diff --git a/runtime/Lib/site_packages/models/__init__.py b/runtime/Lib/site_packages/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..24d86dfd507e25bba3c0b1f054da81902f836f8a --- /dev/null +++ b/runtime/Lib/site_packages/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.error import (AUDIO_IMPORT_ERROR, + TENSORFLOW_IMPORT_WARNING) +from modelscope.utils.import_utils import is_torch_available +from . import audio, cv, multi_modal, nlp +from .base import Head, Model +from .builder import BACKBONES, HEADS, MODELS, build_model + +if is_torch_available(): + from .base.base_torch_model import TorchModel + from .base.base_torch_head import TorchHead diff --git a/runtime/Lib/site_packages/models/audio/__init__.py b/runtime/Lib/site_packages/models/audio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ca0b7562324f8f55fba951f2c609a40005266908 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from . import ans, asr, itn, kws, separation, sv, tts diff --git a/runtime/Lib/site_packages/models/audio/aec/__init__.py b/runtime/Lib/site_packages/models/audio/aec/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/__init__.py b/runtime/Lib/site_packages/models/audio/aec/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/activations.py b/runtime/Lib/site_packages/models/audio/aec/layers/activations.py new file mode 100644 index 0000000000000000000000000000000000000000..f78ad4b54710905194eeb7cd51d09500e0895371 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/layers/activations.py @@ -0,0 +1,62 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch.nn as nn + +from .layer_base import LayerBase + + +class RectifiedLinear(LayerBase): + + def __init__(self, input_dim, output_dim): + super(RectifiedLinear, self).__init__() + self.dim = input_dim + self.relu = nn.ReLU() + + def forward(self, input): + return self.relu(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr + + +class LogSoftmax(LayerBase): + + def __init__(self, input_dim, output_dim): + super(LogSoftmax, self).__init__() + self.dim = input_dim + self.ls = nn.LogSoftmax() + + def forward(self, input): + return self.ls(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr + + +class Sigmoid(LayerBase): + + def __init__(self, input_dim, output_dim): + super(Sigmoid, self).__init__() + self.dim = input_dim + self.sig = nn.Sigmoid() + + def forward(self, input): + return self.sig(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py b/runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..2de8a03ffca8ddd42ec0e71577ab616fcf131995 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/layers/affine_transform.py @@ -0,0 +1,80 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch as th +import torch.nn as nn + +from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number, + to_kaldi_matrix) + + +class AffineTransform(LayerBase): + + def __init__(self, input_dim, output_dim): + super(AffineTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.linear = nn.Linear(input_dim, output_dim) + + def forward(self, input): + return self.linear(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + re_str += ' 1 1 0\n' + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += to_kaldi_matrix(x) + return re_str + + def to_raw_nnet(self, fid): + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + x.tofile(fid) + + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + x.tofile(fid) + + def load_kaldi_nnet(self, instr): + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('AffineTransform format error for ') + instr, lr = output + + output = expect_token_number(instr, '') + if output is None: + raise Exception( + 'AffineTransform format error for ') + instr, lr = output + + output = expect_token_number(instr, '') + if output is None: + raise Exception('AffineTransform format error for ') + instr, lr = output + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('AffineTransform format error for parsing matrix') + instr, mat = output + + print(mat.shape) + self.linear.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('AffineTransform format error for parsing matrix') + instr, mat = output + mat = np.squeeze(mat) + self.linear.bias = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + return instr diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py b/runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..1582b908e53c1471557463f5b6fcaf9eb95d300a --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/layers/deep_fsmn.py @@ -0,0 +1,180 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as F + +from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number, + to_kaldi_matrix) + + +class DeepFsmn(LayerBase): + + def __init__(self, + input_dim, + output_dim, + lorder=None, + rorder=None, + hidden_size=None, + layer_norm=False, + dropout=0): + super(DeepFsmn, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + + if lorder is None: + return + + self.lorder = lorder + self.rorder = rorder + self.hidden_size = hidden_size + self.layer_norm = layer_norm + + self.linear = nn.Linear(input_dim, hidden_size) + self.norm = nn.LayerNorm(hidden_size) + self.drop1 = nn.Dropout(p=dropout) + self.drop2 = nn.Dropout(p=dropout) + self.project = nn.Linear(hidden_size, output_dim, bias=False) + + self.conv1 = nn.Conv2d( + output_dim, + output_dim, [lorder, 1], [1, 1], + groups=output_dim, + bias=False) + self.conv2 = nn.Conv2d( + output_dim, + output_dim, [rorder, 1], [1, 1], + groups=output_dim, + bias=False) + + def forward(self, input): + + f1 = F.relu(self.linear(input)) + + f1 = self.drop1(f1) + if self.layer_norm: + f1 = self.norm(f1) + + p1 = self.project(f1) + + x = th.unsqueeze(p1, 1) + + x_per = x.permute(0, 3, 2, 1) + + y = F.pad(x_per, [0, 0, self.lorder - 1, 0]) + yr = F.pad(x_per, [0, 0, 0, self.rorder]) + yr = yr[:, :, 1:, :] + + out = x_per + self.conv1(y) + self.conv2(yr) + out = self.drop2(out) + + out1 = out.permute(0, 3, 2, 1) + + return input + out1.squeeze() + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n'\ + % (self.output_dim, self.input_dim) + re_str += ' %d %d %d %d 0\n'\ + % (1, self.hidden_size, self.lorder, 1) + lfiters = self.state_dict()['conv1.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + re_str += to_kaldi_matrix(x) + proj_weights = self.state_dict()['project.weight'] + x = proj_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += to_kaldi_matrix(x) + return re_str + + def load_kaldi_nnet(self, instr): + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lr = output + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, hiddensize = output + self.hidden_size = int(hiddensize) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lorder = output + self.lorder = int(lorder) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lstride = output + self.lstride = lstride + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + mat1 = np.fliplr(mat.T).copy() + self.conv1 = nn.Conv2d( + self.output_dim, + self.output_dim, [self.lorder, 1], [1, 1], + groups=self.output_dim, + bias=False) + mat_th = th.from_numpy(mat1).type(th.FloatTensor) + mat_th = mat_th.unsqueeze(1) + mat_th = mat_th.unsqueeze(3) + self.conv1.weight = th.nn.Parameter(mat_th) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + + self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False) + self.linear = nn.Linear(self.input_dim, self.hidden_size) + + self.project.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + self.linear.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + self.linear.bias = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + return instr diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py b/runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py new file mode 100644 index 0000000000000000000000000000000000000000..7c39e5be6ac49b680135f119876178ae736342c6 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/layers/layer_base.py @@ -0,0 +1,52 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import abc +import re + +import numpy as np +import torch.nn as nn + + +def expect_token_number(instr, token): + first_token = re.match(r'^\s*' + token, instr) + if first_token is None: + return None + instr = instr[first_token.end():] + lr = re.match(r'^\s*(-?\d+\.?\d*e?-?\d*?)', instr) + if lr is None: + return None + return instr[lr.end():], lr.groups()[0] + + +def expect_kaldi_matrix(instr): + pos2 = instr.find('[', 0) + pos3 = instr.find(']', pos2) + mat = [] + for stt in instr[pos2 + 1:pos3].split('\n'): + tmp_mat = np.fromstring(stt, dtype=np.float32, sep=' ') + if tmp_mat.size > 0: + mat.append(tmp_mat) + return instr[pos3 + 1:], np.array(mat) + + +def to_kaldi_matrix(np_mat): + """ + function that transform as str numpy mat to standard kaldi str matrix + :param np_mat: numpy mat + :return: str + """ + np.set_printoptions(threshold=np.inf, linewidth=np.nan, suppress=True) + out_str = str(np_mat) + out_str = out_str.replace('[', '') + out_str = out_str.replace(']', '') + return '[ %s ]\n' % out_str + + +class LayerBase(nn.Module, metaclass=abc.ABCMeta): + + def __init__(self): + super(LayerBase, self).__init__() + + @abc.abstractmethod + def to_kaldi_nnet(self): + pass diff --git a/runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py b/runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..a276db051605d9f188565e25628cf35f37818e71 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/layers/uni_deep_fsmn.py @@ -0,0 +1,484 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as F + +from .layer_base import (LayerBase, expect_kaldi_matrix, expect_token_number, + to_kaldi_matrix) + + +class SepConv(nn.Module): + + def __init__(self, + in_channels, + filters, + out_channels, + kernel_size=(5, 2), + dilation=(1, 1)): + """ :param kernel_size (time, frequency) + + """ + super(SepConv, self).__init__() + # depthwise + pointwise + self.dconv = nn.Conv2d( + in_channels, + in_channels * filters, + kernel_size, + dilation=dilation, + groups=in_channels) + self.pconv = nn.Conv2d( + in_channels * filters, out_channels, kernel_size=1) + self.padding = dilation[0] * (kernel_size[0] - 1) + + def forward(self, input): + ''' input: [B, C, T, F] + ''' + x = F.pad(input, [0, 0, self.padding, 0]) + x = self.dconv(x) + x = self.pconv(x) + return x + + +class Conv2d(nn.Module): + + def __init__(self, + input_dim, + output_dim, + lorder=20, + rorder=0, + groups=1, + bias=False, + skip_connect=True): + super(Conv2d, self).__init__() + self.lorder = lorder + self.conv = nn.Conv2d( + input_dim, output_dim, [lorder, 1], groups=groups, bias=bias) + self.rorder = rorder + if self.rorder: + self.conv2 = nn.Conv2d( + input_dim, output_dim, [rorder, 1], groups=groups, bias=bias) + self.skip_connect = skip_connect + + def forward(self, input): + # [B, 1, T, F] + x = th.unsqueeze(input, 1) + # [B, F, T, 1] + x_per = x.permute(0, 3, 2, 1) + y = F.pad(x_per, [0, 0, self.lorder - 1, 0]) + out = self.conv(y) + if self.rorder: + yr = F.pad(x_per, [0, 0, 0, self.rorder]) + yr = yr[:, :, 1:, :] + out += self.conv2(yr) + out = out.permute(0, 3, 2, 1).squeeze(1) + if self.skip_connect: + out = out + input + return out + + +class SelfAttLayer(nn.Module): + + def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None): + super(SelfAttLayer, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + + if lorder is None: + return + + self.lorder = lorder + self.hidden_size = hidden_size + + self.linear = nn.Linear(input_dim, hidden_size) + + self.project = nn.Linear(hidden_size, output_dim, bias=False) + + self.att = nn.Linear(input_dim, lorder, bias=False) + + def forward(self, input): + + f1 = F.relu(self.linear(input)) + + p1 = self.project(f1) + + x = th.unsqueeze(p1, 1) + + x_per = x.permute(0, 3, 2, 1) + + y = F.pad(x_per, [0, 0, self.lorder - 1, 0]) + + # z [B, F, T, lorder] + z = x_per + for i in range(1, self.lorder): + z = th.cat([z, y[:, :, self.lorder - 1 - i:-i, :]], axis=-1) + + # [B, T, lorder] + att = F.softmax(self.att(input), dim=-1) + att = th.unsqueeze(att, 1) + z = th.sum(z * att, axis=-1) + + out1 = z.permute(0, 2, 1) + + return input + out1 + + +class TFFsmn(nn.Module): + + def __init__(self, + input_dim, + output_dim, + lorder=None, + hidden_size=None, + dilation=1, + layer_norm=False, + dropout=0, + skip_connect=True): + super(TFFsmn, self).__init__() + + self.skip_connect = skip_connect + + self.linear = nn.Linear(input_dim, hidden_size) + self.norm = nn.Identity() + if layer_norm: + self.norm = nn.LayerNorm(input_dim) + self.act = nn.ReLU() + self.project = nn.Linear(hidden_size, output_dim, bias=False) + + self.conv1 = nn.Conv2d( + output_dim, + output_dim, [lorder, 1], + dilation=[dilation, 1], + groups=output_dim, + bias=False) + self.padding_left = dilation * (lorder - 1) + dorder = 5 + self.conv2 = nn.Conv2d(1, 1, [dorder, 1], bias=False) + self.padding_freq = dorder - 1 + + def forward(self, input): + return self.compute1(input) + + def compute1(self, input): + ''' linear-dconv-relu(norm)-linear-dconv + ''' + x = self.linear(input) + # [B, 1, F, T] + x = th.unsqueeze(x, 1).permute(0, 1, 3, 2) + z = F.pad(x, [0, 0, self.padding_freq, 0]) + z = self.conv2(z) + x + x = z.permute(0, 3, 2, 1).squeeze(-1) + x = self.act(x) + x = self.norm(x) + x = self.project(x) + x = th.unsqueeze(x, 1).permute(0, 3, 2, 1) + # [B, F, T+lorder-1, 1] + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.conv1(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + + return input + out + + +class CNNFsmn(nn.Module): + ''' use cnn to reduce parameters + ''' + + def __init__(self, + input_dim, + output_dim, + lorder=None, + hidden_size=None, + dilation=1, + layer_norm=False, + dropout=0, + skip_connect=True): + super(CNNFsmn, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + self.skip_connect = skip_connect + + if lorder is None: + return + + self.lorder = lorder + self.hidden_size = hidden_size + + self.linear = nn.Linear(input_dim, hidden_size) + self.act = nn.ReLU() + kernel_size = (3, 8) + stride = (1, 4) + self.conv = nn.Sequential( + nn.ConstantPad2d((stride[1], 0, kernel_size[0] - 1, 0), 0), + nn.Conv2d(1, stride[1], kernel_size=kernel_size, stride=stride)) + + self.dconv = nn.Conv2d( + output_dim, + output_dim, [lorder, 1], + dilation=[dilation, 1], + groups=output_dim, + bias=False) + self.padding_left = dilation * (lorder - 1) + + def forward(self, input): + return self.compute2(input) + + def compute1(self, input): + ''' linear-relu(norm)-conv2d-relu?-dconv + ''' + # [B, T, F] + x = self.linear(input) + x = self.act(x) + x = th.unsqueeze(x, 1) + x = self.conv(x) + # [B, C, T, F] -> [B, 1, T, F] + b, c, t, f = x.shape + x = x.view([b, 1, t, -1]) + x = x.permute(0, 3, 2, 1) + # [B, F, T+lorder-1, 1] + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.dconv(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + return input + out + + def compute2(self, input): + ''' conv2d-relu-linear-relu?-dconv + ''' + x = th.unsqueeze(input, 1) + x = self.conv(x) + x = self.act(x) + # [B, C, T, F] -> [B, T, F] + b, c, t, f = x.shape + x = x.view([b, t, -1]) + x = self.linear(x) + x = th.unsqueeze(x, 1).permute(0, 3, 2, 1) + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.dconv(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + return input + out + + +class UniDeepFsmn(LayerBase): + + def __init__(self, + input_dim, + output_dim, + lorder=None, + hidden_size=None, + dilation=1, + layer_norm=False, + dropout=0, + skip_connect=True): + super(UniDeepFsmn, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + self.skip_connect = skip_connect + + if lorder is None: + return + + self.lorder = lorder + self.hidden_size = hidden_size + + self.linear = nn.Linear(input_dim, hidden_size) + self.norm = nn.Identity() + if layer_norm: + self.norm = nn.LayerNorm(input_dim) + self.act = nn.ReLU() + self.project = nn.Linear(hidden_size, output_dim, bias=False) + + self.conv1 = nn.Conv2d( + output_dim, + output_dim, [lorder, 1], + dilation=[dilation, 1], + groups=output_dim, + bias=False) + self.padding_left = dilation * (lorder - 1) + + def forward(self, input): + return self.compute1(input) + + def compute1(self, input): + ''' linear-relu(norm)-linear-dconv + ''' + # [B, T, F] + x = self.linear(input) + x = self.act(x) + x = self.norm(x) + x = self.project(x) + x = th.unsqueeze(x, 1).permute(0, 3, 2, 1) + # [B, F, T+lorder-1, 1] + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.conv1(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + + return input + out + + def compute2(self, input): + ''' linear-dconv-linear-relu(norm) + ''' + x = self.project(input) + x = th.unsqueeze(x, 1).permute(0, 3, 2, 1) + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.conv1(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + x = self.linear(out) + x = self.act(x) + x = self.norm(x) + + return input + x + + def compute3(self, input): + ''' dconv-linear-relu(norm)-linear + ''' + x = th.unsqueeze(input, 1).permute(0, 3, 2, 1) + y = F.pad(x, [0, 0, self.padding_left, 0]) + out = self.conv1(y) + if self.skip_connect: + out = out + x + out = out.permute(0, 3, 2, 1).squeeze() + x = self.linear(out) + x = self.act(x) + x = self.norm(x) + x = self.project(x) + + return input + x + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' \ + % (self.output_dim, self.input_dim) + re_str += ' %d %d %d %d 0\n' \ + % (1, self.hidden_size, self.lorder, 1) + lfiters = self.state_dict()['conv1.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + re_str += to_kaldi_matrix(x) + proj_weights = self.state_dict()['project.weight'] + x = proj_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += to_kaldi_matrix(x) + return re_str + + def to_raw_nnet(self, fid): + lfiters = self.state_dict()['conv1.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + x.tofile(fid) + + proj_weights = self.state_dict()['project.weight'] + x = proj_weights.squeeze().numpy() + x.tofile(fid) + + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + x.tofile(fid) + + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + x.tofile(fid) + + def load_kaldi_nnet(self, instr): + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lr = output + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, hiddensize = output + self.hidden_size = int(hiddensize) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lorder = output + self.lorder = int(lorder) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + instr, lstride = output + self.lstride = lstride + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error for ') + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + mat1 = np.fliplr(mat.T).copy() + + self.conv1 = nn.Conv2d( + self.output_dim, + self.output_dim, [self.lorder, 1], [1, 1], + groups=self.output_dim, + bias=False) + + mat_th = th.from_numpy(mat1).type(th.FloatTensor) + mat_th = mat_th.unsqueeze(1) + mat_th = mat_th.unsqueeze(3) + self.conv1.weight = th.nn.Parameter(mat_th) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + + self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False) + self.linear = nn.Linear(self.input_dim, self.hidden_size) + + self.project.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + self.linear.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error for parsing matrix') + instr, mat = output + mat = np.squeeze(mat) + self.linear.bias = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + return instr diff --git a/runtime/Lib/site_packages/models/audio/aec/network/__init__.py b/runtime/Lib/site_packages/models/audio/aec/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/aec/network/loss.py b/runtime/Lib/site_packages/models/audio/aec/network/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..1f20072a2f47351bad195c72b92affc50efc54f0 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/network/loss.py @@ -0,0 +1,396 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn.functional as F + +from .modulation_loss import (GaborSTRFConv, MelScale, + ModulationDomainLossModule) + +EPS = 1e-8 + + +def compute_mask(mixed_spec, clean_spec, mask_type='psmiam', clip=1): + ''' + stft: (batch, ..., 2) or complex(batch, ...) + y = x + n + ''' + if torch.is_complex(mixed_spec): + yr, yi = mixed_spec.real, mixed_spec.imag + else: + yr, yi = mixed_spec[..., 0], mixed_spec[..., 1] + if torch.is_complex(clean_spec): + xr, xi = clean_spec.real, clean_spec.imag + else: + xr, xi = clean_spec[..., 0], clean_spec[..., 1] + + if mask_type == 'iam': + ymag = torch.sqrt(yr**2 + yi**2) + xmag = torch.sqrt(xr**2 + xi**2) + iam = xmag / (ymag + EPS) + return torch.clamp(iam, 0, 1) + + elif mask_type == 'psm': + ypow = yr**2 + yi**2 + psm = (xr * yr + xi * yi) / (ypow + EPS) + return torch.clamp(psm, 0, 1) + + elif mask_type == 'psmiam': + ypow = yr**2 + yi**2 + psm = (xr * yr + xi * yi) / (ypow + EPS) + ymag = torch.sqrt(yr**2 + yi**2) + xmag = torch.sqrt(xr**2 + xi**2) + iam = xmag / (ymag + EPS) + psmiam = psm * iam + return torch.clamp(psmiam, 0, 1) + + elif mask_type == 'crm': + ypow = yr**2 + yi**2 + mr = (xr * yr + xi * yi) / (ypow + EPS) + mi = (xi * yr - xr * yi) / (ypow + EPS) + mr = torch.clamp(mr, -clip, clip) + mi = torch.clamp(mi, -clip, clip) + return mr, mi + + +def energy_vad(spec, + thdhigh=320 * 600 * 600 * 2, + thdlow=320 * 300 * 300 * 2, + int16=True): + ''' + energy based vad should be accurate enough + spec: (batch, bins, frames, 2) + returns (batch, frames) + ''' + energy = torch.sum(spec[..., 0]**2 + spec[..., 1]**2, dim=1) + vad = energy > thdhigh + idx = torch.logical_and(vad == 0, energy > thdlow) + vad[idx] = 0.5 + return vad + + +def modulation_loss_init(n_fft): + gabor_strf_parameters = torch.load( + './network/gabor_strf_parameters.pt')['state_dict'] + gabor_modulation_kernels = GaborSTRFConv(supn=30, supk=30, nkern=60) + gabor_modulation_kernels.load_state_dict(gabor_strf_parameters) + + modulation_loss_module = ModulationDomainLossModule( + gabor_modulation_kernels.eval()) + for param in modulation_loss_module.parameters(): + param.requires_grad = False + + stft2mel = MelScale( + n_mels=80, sample_rate=16000, n_stft=n_fft // 2 + 1).cuda() + + return modulation_loss_module, stft2mel + + +def mask_loss_function( + loss_func='psm_loss', + loss_type='mse', # ['mse', 'mae', 'comb'] + mask_type='psmiam', + use_mod_loss=False, + use_wav2vec_loss=False, + n_fft=640, + hop_length=320, + EPS=1e-8, + weight=None): + if weight is not None: + print(f'Use loss weight: {weight}') + winlen = n_fft + window = torch.hamming_window(winlen, periodic=False) + + def stft(x, return_complex=False): + # returns [batch, bins, frames, 2] + return torch.stft( + x, + n_fft, + hop_length, + winlen, + window=window.to(x.device), + center=False, + return_complex=return_complex) + + def istft(x, slen): + return torch.istft( + x, + n_fft, + hop_length, + winlen, + window=window.to(x.device), + center=False, + length=slen) + + def mask_loss(targets, masks, nframes): + ''' [Batch, Time, Frequency] + ''' + with torch.no_grad(): + mask_for_loss = torch.ones_like(targets) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + masks = masks * mask_for_loss + targets = targets * mask_for_loss + + if weight is None: + alpha = 1 + else: # for aec ST + alpha = weight - targets + + if loss_type == 'mse': + loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2)) + elif loss_type == 'mae': + loss = torch.sum(alpha * torch.abs(targets - masks)) + else: # mse(mask), mae(mask) approx 1:2 + loss = 0.5 * torch.sum(alpha * torch.pow(targets - masks, 2) + + 0.1 * alpha * torch.abs(targets - masks)) + loss /= torch.sum(nframes) + return loss + + def spectrum_loss(targets, spec, nframes): + ''' [Batch, Time, Frequency, 2] + ''' + with torch.no_grad(): + mask_for_loss = torch.ones_like(targets[..., 0]) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + xr = spec[..., 0] * mask_for_loss + xi = spec[..., 1] * mask_for_loss + yr = targets[..., 0] * mask_for_loss + yi = targets[..., 1] * mask_for_loss + xmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2) * mask_for_loss + ymag = torch.sqrt(targets[..., 0]**2 + + targets[..., 1]**2) * mask_for_loss + + loss1 = torch.sum(torch.pow(xr - yr, 2) + torch.pow(xi - yi, 2)) + loss2 = torch.sum(torch.pow(xmag - ymag, 2)) + + loss = (loss1 + loss2) / torch.sum(nframes) + return loss + + def sa_loss_dlen(mixed, clean, masks, nframes): + yspec = stft(mixed).permute([0, 2, 1, 3]) / 32768 + xspec = stft(clean).permute([0, 2, 1, 3]) / 32768 + with torch.no_grad(): + mask_for_loss = torch.ones_like(xspec[..., 0]) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + emag = ((yspec[..., 0]**2 + yspec[..., 1]**2)**0.15) * (masks**0.3) + xmag = (xspec[..., 0]**2 + xspec[..., 1]**2)**0.15 + emag = emag * mask_for_loss + xmag = xmag * mask_for_loss + + loss = torch.sum(torch.pow(emag - xmag, 2)) / torch.sum(nframes) + return loss + + def psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask=None): + mixed_spec = stft(mixed) + clean_spec = stft(clean) + targets = compute_mask(mixed_spec, clean_spec, mask_type) + # [B, T, F] + targets = targets.permute(0, 2, 1) + + loss = mask_loss(targets, masks, nframes) + + if subtask is not None: + vadtargets = energy_vad(clean_spec) + with torch.no_grad(): + mask_for_loss = torch.ones_like(targets[:, :, 0]) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:] = 0 + subtask = subtask[:, :, 0] * mask_for_loss + vadtargets = vadtargets * mask_for_loss + + loss_vad = F.binary_cross_entropy(subtask, vadtargets) + return loss + loss_vad + return loss + + def modulation_loss(mixed, clean, masks, nframes, subtask=None): + mixed_spec = stft(mixed, True) + clean_spec = stft(clean, True) + enhanced_mag = torch.abs(mixed_spec) + clean_mag = torch.abs(clean_spec) + with torch.no_grad(): + mask_for_loss = torch.ones_like(clean_mag) + for idx, num in enumerate(nframes): + mask_for_loss[idx, :, num:] = 0 + clean_mag = clean_mag * mask_for_loss + enhanced_mag = enhanced_mag * mask_for_loss * masks.permute([0, 2, 1]) + + # Covert to log-mel representation + # (B,T,#mel_channels) + clean_log_mel = torch.log( + torch.transpose(stft2mel(clean_mag**2), 2, 1) + 1e-8) + enhanced_log_mel = torch.log( + torch.transpose(stft2mel(enhanced_mag**2), 2, 1) + 1e-8) + + alpha = compute_mask(mixed_spec, clean_spec, mask_type) + alpha = alpha.permute(0, 2, 1) + loss = 0.05 * modulation_loss_module(enhanced_log_mel, clean_log_mel, + alpha) + loss2 = psm_vad_loss_dlen(mixed, clean, masks, nframes, subtask) + # print(loss.item(), loss2.item()) #approx 1:4 + loss = loss + loss2 + return loss + + def wav2vec_loss(mixed, clean, masks, nframes, subtask=None): + mixed /= 32768 + clean /= 32768 + mixed_spec = stft(mixed) + with torch.no_grad(): + mask_for_loss = torch.ones_like(masks) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + masks_est = masks * mask_for_loss + + estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3) + est_clean = istft(estimate, clean.shape[1]) + loss = wav2vec_loss_module(est_clean, clean) + return loss + + def sisdr_loss_dlen(mixed, + clean, + masks, + nframes, + subtask=None, + zero_mean=True): + mixed_spec = stft(mixed) + with torch.no_grad(): + mask_for_loss = torch.ones_like(masks) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + masks_est = masks * mask_for_loss + + estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3) + est_clean = istft(estimate, clean.shape[1]) + flen = min(clean.shape[1], est_clean.shape[1]) + clean = clean[:, :flen] + est_clean = est_clean[:, :flen] + + # follow asteroid/losses/sdr.py + if zero_mean: + clean = clean - torch.mean(clean, dim=1, keepdim=True) + est_clean = est_clean - torch.mean(est_clean, dim=1, keepdim=True) + + dot = torch.sum(est_clean * clean, dim=1, keepdim=True) + s_clean_energy = torch.sum(clean**2, dim=1, keepdim=True) + EPS + scaled_clean = dot * clean / s_clean_energy + e_noise = est_clean - scaled_clean + + # [batch] + sisdr = torch.sum( + scaled_clean**2, dim=1) / ( + torch.sum(e_noise**2, dim=1) + EPS) + sisdr = -10 * torch.log10(sisdr + EPS) + loss = sisdr.mean() + return loss + + def sisdr_freq_loss_dlen(mixed, clean, masks, nframes, subtask=None): + mixed_spec = stft(mixed) + clean_spec = stft(clean) + with torch.no_grad(): + mask_for_loss = torch.ones_like(masks) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + masks_est = masks * mask_for_loss + + estimate = mixed_spec * masks_est.permute([0, 2, 1]).unsqueeze(3) + + dot_real = estimate[..., 0] * clean_spec[..., 0] + \ + estimate[..., 1] * clean_spec[..., 1] + dot_imag = estimate[..., 0] * clean_spec[..., 1] - \ + estimate[..., 1] * clean_spec[..., 0] + dot = torch.cat([dot_real.unsqueeze(3), dot_imag.unsqueeze(3)], dim=-1) + s_clean_energy = clean_spec[..., 0] ** 2 + \ + clean_spec[..., 1] ** 2 + EPS + scaled_clean = dot * clean_spec / s_clean_energy.unsqueeze(3) + e_noise = estimate - scaled_clean + + # [batch] + scaled_clean_energy = torch.sum( + scaled_clean[..., 0]**2 + scaled_clean[..., 1]**2, dim=1) + e_noise_energy = torch.sum( + e_noise[..., 0]**2 + e_noise[..., 1]**2, dim=1) + sisdr = torch.sum( + scaled_clean_energy, dim=1) / ( + torch.sum(e_noise_energy, dim=1) + EPS) + sisdr = -10 * torch.log10(sisdr + EPS) + loss = sisdr.mean() + return loss + + def crm_loss_dlen(mixed, clean, masks, nframes, subtask=None): + mixed_spec = stft(mixed).permute([0, 2, 1, 3]) + clean_spec = stft(clean).permute([0, 2, 1, 3]) + mixed_spec = mixed_spec / 32768 + clean_spec = clean_spec / 32768 + tgt_mr, tgt_mi = compute_mask(mixed_spec, clean_spec, mask_type='crm') + + D = int(masks.shape[2] / 2) + with torch.no_grad(): + mask_for_loss = torch.ones_like(clean_spec[..., 0]) + for idx, num in enumerate(nframes): + mask_for_loss[idx, num:, :] = 0 + mr = masks[..., :D] * mask_for_loss + mi = masks[..., D:] * mask_for_loss + tgt_mr = tgt_mr * mask_for_loss + tgt_mi = tgt_mi * mask_for_loss + + if weight is None: + alpha = 1 + else: + alpha = weight - tgt_mr + # signal approximation + yr = mixed_spec[..., 0] + yi = mixed_spec[..., 1] + loss1 = torch.sum(alpha * torch.pow((mr * yr - mi * yi) - clean_spec[..., 0], 2)) \ + + torch.sum(alpha * torch.pow((mr * yi + mi * yr) - clean_spec[..., 1], 2)) + # mask approximation + loss2 = torch.sum(alpha * torch.pow(mr - tgt_mr, 2)) \ + + torch.sum(alpha * torch.pow(mi - tgt_mi, 2)) + loss = 0.5 * (loss1 + loss2) / torch.sum(nframes) + return loss + + def crm_miso_loss_dlen(mixed, clean, masks, nframes): + return crm_loss_dlen(mixed[..., 0], clean[..., 0], masks, nframes) + + def mimo_loss_dlen(mixed, clean, masks, nframes): + chs = mixed.shape[-1] + D = masks.shape[2] // chs + loss = psm_vad_loss_dlen(mixed[..., 0], clean[..., 0], masks[..., :D], + nframes) + for ch in range(1, chs): + loss1 = psm_vad_loss_dlen(mixed[..., ch], clean[..., ch], + masks[..., ch * D:ch * D + D], nframes) + loss = loss + loss1 + return loss / chs + + def spec_loss_dlen(mixed, clean, spec, nframes): + clean_spec = stft(clean).permute([0, 2, 1, 3]) + clean_spec = clean_spec / 32768 + + D = spec.shape[2] // 2 + spec_est = torch.cat([spec[..., :D, None], spec[..., D:, None]], + dim=-1) + loss = spectrum_loss(clean_spec, spec_est, nframes) + return loss + + if loss_func == 'psm_vad_loss_dlen': + return psm_vad_loss_dlen + elif loss_func == 'sisdr_loss_dlen': + return sisdr_loss_dlen + elif loss_func == 'sisdr_freq_loss_dlen': + return sisdr_freq_loss_dlen + elif loss_func == 'crm_loss_dlen': + return crm_loss_dlen + elif loss_func == 'modulation_loss': + return modulation_loss + elif loss_func == 'wav2vec_loss': + return wav2vec_loss + elif loss_func == 'mimo_loss_dlen': + return mimo_loss_dlen + elif loss_func == 'spec_loss_dlen': + return spec_loss_dlen + elif loss_func == 'sa_loss_dlen': + return sa_loss_dlen + else: + print('error loss func') + return None diff --git a/runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py b/runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..3017b5c699fb268d499d5c3e33026a577ba0c24d --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/network/modulation_loss.py @@ -0,0 +1,250 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchaudio.transforms import MelScale + + +class ModulationDomainLossModule(torch.nn.Module): + """Modulation-domain loss function developed in [1] for supervised speech enhancement + + In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram + as the input spectrogram representation. + Specific parameter details are in the paper and in the example below + + Parameters + ---------- + modulation_kernels: nn.Module + Differentiable module that transforms a spectrogram representation to the modulation domain + + modulation_domain = modulation_kernels(input_tf_representation) + Input Spectrogram representation (B, T, F) ---> |(M) modulation_kernels|--->Modulation Domain(B, M, T', F') + + norm: boolean + Normalizes the modulation domain representation to be 0 mean across time + + [1] T. Vuong, Y. Xia, and R. M. Stern, “A modulation-domain lossfor neural-network-based real-time + speech enhancement” + Accepted ICASSP 2021, https://arxiv.org/abs/2102.07330 + + + """ + + def __init__(self, modulation_kernels, norm=True): + super(ModulationDomainLossModule, self).__init__() + + self.modulation_kernels = modulation_kernels + self.mse = nn.MSELoss(reduce=False) + self.norm = norm + + def forward(self, enhanced_spect, clean_spect, weight=None): + """Calculate modulation-domain loss + Args: + enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels). + clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels). + Returns: + Tensor: Modulation-domain loss value. + """ + + clean_mod = self.modulation_kernels(clean_spect) + enhanced_mod = self.modulation_kernels(enhanced_spect) + + if self.norm: + mean_clean_mod = torch.mean(clean_mod, dim=2) + mean_enhanced_mod = torch.mean(enhanced_mod, dim=2) + + clean_mod = clean_mod - mean_clean_mod.unsqueeze(2) + enhanced_mod = enhanced_mod - mean_enhanced_mod.unsqueeze(2) + + if weight is None: + alpha = 1 + else: # TF-mask weight + alpha = 1 + torch.sum(weight, dim=-1, keepdim=True).unsqueeze(1) + mod_mse_loss = self.mse(enhanced_mod, clean_mod) * alpha + mod_mse_loss = torch.mean( + torch.sum(mod_mse_loss, dim=(1, 2, 3)) + / torch.sum(clean_mod**2, dim=(1, 2, 3))) + + return mod_mse_loss + + +class ModulationDomainNCCLossModule(torch.nn.Module): + """Modulation-domain loss function developed in [1] for supervised speech enhancement + + # Speech Intelligibility Prediction Using Spectro-Temporal Modulation Analysis - based off of this + + In our paper, we used the gabor-based STRF kernels as the modulation kernels and used the log-mel spectrogram + as the input spectrogram representation. + Specific parameter details are in the paper and in the example below + + Parameters + ---------- + modulation_kernels: nn.Module + Differentiable module that transforms a spectrogram representation to the modulation domain + + modulation_domain = modulation_kernels(input_tf_representation) + Input Spectrogram representation(B, T, F) --- (M) modulation_kernels---> Modulation Domain(B, M, T', F') + + [1] + + """ + + def __init__(self, modulation_kernels): + super(ModulationDomainNCCLossModule, self).__init__() + + self.modulation_kernels = modulation_kernels + self.mse = nn.MSELoss(reduce=False) + + def forward(self, enhanced_spect, clean_spect): + """Calculate modulation-domain loss + Args: + enhanced_spect (Tensor): spectrogram representation of enhanced signal (B, #frames, #freq_channels). + clean_spect (Tensor): spectrogram representation of clean ground-truth signal (B, #frames, #freq_channels). + Returns: + Tensor: Modulation-domain loss value. + """ + + clean_mod = self.modulation_kernels(clean_spect) + enhanced_mod = self.modulation_kernels(enhanced_spect) + mean_clean_mod = torch.mean(clean_mod, dim=2) + mean_enhanced_mod = torch.mean(enhanced_mod, dim=2) + + normalized_clean = clean_mod - mean_clean_mod.unsqueeze(2) + normalized_enhanced = enhanced_mod - mean_enhanced_mod.unsqueeze(2) + + inner_product = torch.sum( + normalized_clean * normalized_enhanced, dim=2) + normalized_denom = (torch.sum( + normalized_clean * normalized_clean, dim=2))**.5 * (torch.sum( + normalized_enhanced * normalized_enhanced, dim=2))**.5 + + ncc = inner_product / normalized_denom + mod_mse_loss = torch.mean((ncc - 1.0)**2) + + return mod_mse_loss + + +class GaborSTRFConv(nn.Module): + """Gabor-STRF-based cross-correlation kernel.""" + + def __init__(self, + supn, + supk, + nkern, + rates=None, + scales=None, + norm_strf=True, + real_only=False): + """Instantiate a Gabor-based STRF convolution layer. + Parameters + ---------- + supn: int + Time support in number of frames. Also the window length. + supk: int + Frequency support in number of channels. Also the window length. + nkern: int + Number of kernels, each with a learnable rate and scale. + rates: list of float, None + Initial values for temporal modulation. + scales: list of float, None + Initial values for spectral modulation. + norm_strf: Boolean + Normalize STRF kernels to be unit length + real_only: Boolean + If True, nkern REAL gabor-STRF kernels + If False, nkern//2 REAL and nkern//2 IMAGINARY gabor-STRF kernels + """ + super(GaborSTRFConv, self).__init__() + self.numN = supn + self.numK = supk + self.numKern = nkern + self.real_only = real_only + self.norm_strf = norm_strf + + if not real_only: + nkern = nkern // 2 + + if supk % 2 == 0: # force odd number + supk += 1 + self.supk = torch.arange(supk, dtype=torch.float32) + if supn % 2 == 0: # force odd number + supn += 1 + self.supn = torch.arange(supn, dtype=self.supk.dtype) + self.padding = (supn // 2, supk // 2) + # Set up learnable parameters + # for param in (rates, scales): + # assert (not param) or len(param) == nkern + if not rates: + + rates = torch.rand(nkern) * math.pi / 2.0 + + if not scales: + + scales = (torch.rand(nkern) * 2.0 - 1.0) * math.pi / 2.0 + + self.rates_ = nn.Parameter(torch.Tensor(rates)) + self.scales_ = nn.Parameter(torch.Tensor(scales)) + + def strfs(self): + """Make STRFs using the current parameters.""" + + if self.supn.device != self.rates_.device: # for first run + self.supn = self.supn.to(self.rates_.device) + self.supk = self.supk.to(self.rates_.device) + n0, k0 = self.padding + + nwind = .5 - .5 * \ + torch.cos(2 * math.pi * (self.supn + 1) / (len(self.supn) + 1)) + kwind = .5 - .5 * \ + torch.cos(2 * math.pi * (self.supk + 1) / (len(self.supk) + 1)) + + new_wind = torch.matmul((nwind).unsqueeze(-1), (kwind).unsqueeze(0)) + + n_n_0 = self.supn - n0 + k_k_0 = self.supk - k0 + n_mult = torch.matmul( + n_n_0.unsqueeze(1), + torch.ones((1, len(self.supk))).type(torch.FloatTensor).to( + self.rates_.device)) + k_mult = torch.matmul( + torch.ones((len(self.supn), + 1)).type(torch.FloatTensor).to(self.rates_.device), + k_k_0.unsqueeze(0)) + + inside = self.rates_.unsqueeze(1).unsqueeze( + 1) * n_mult + self.scales_.unsqueeze(1).unsqueeze(1) * k_mult + real_strf = torch.cos(inside) * new_wind.unsqueeze(0) + + if self.real_only: + final_strf = real_strf + + else: + imag_strf = torch.sin(inside) * new_wind.unsqueeze(0) + final_strf = torch.cat([real_strf, imag_strf], dim=0) + + if self.norm_strf: + final_strf = final_strf / (torch.sum( + final_strf**2, dim=(1, 2)).unsqueeze(1).unsqueeze(2))**.5 + + return final_strf + + def forward(self, sigspec): + """Forward pass a batch of (real) spectra [Batch x Time x Frequency].""" + if len(sigspec.shape) == 2: # expand batch dimension if single eg + sigspec = sigspec.unsqueeze(0) + strfs = self.strfs().unsqueeze(1).type_as(sigspec) + out = F.conv2d(sigspec.unsqueeze(1), strfs, padding=self.padding) + return out + + def __repr__(self): + """Gabor filter""" + report = """ + +++++ Gabor Filter Kernels [{}], supn[{}], supk[{}] real only [{}] norm strf [{}] +++++ + + """.format(self.numKern, self.numN, self.numK, self.real_only, + self.norm_strf) + + return report diff --git a/runtime/Lib/site_packages/models/audio/aec/network/se_net.py b/runtime/Lib/site_packages/models/audio/aec/network/se_net.py new file mode 100644 index 0000000000000000000000000000000000000000..406396052557b4fa7c5b5c979b5ec5ed6661e135 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/aec/network/se_net.py @@ -0,0 +1,487 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.audio.aec.layers.activations import (RectifiedLinear, + Sigmoid) +from modelscope.models.audio.aec.layers.affine_transform import AffineTransform +from modelscope.models.audio.aec.layers.deep_fsmn import DeepFsmn +from modelscope.models.audio.aec.layers.uni_deep_fsmn import (Conv2d, + UniDeepFsmn) + + +class MaskNet(nn.Module): + + def __init__(self, + indim, + outdim, + layers=9, + hidden_dim=128, + hidden_dim2=None, + lorder=20, + rorder=0, + dilation=1, + layer_norm=False, + dropout=0, + crm=False, + vad=False, + linearout=False): + super(MaskNet, self).__init__() + + self.linear1 = AffineTransform(indim, hidden_dim) + self.relu = RectifiedLinear(hidden_dim, hidden_dim) + if hidden_dim2 is None: + hidden_dim2 = hidden_dim + + if rorder == 0: + repeats = [ + UniDeepFsmn( + hidden_dim, + hidden_dim, + lorder, + hidden_dim2, + dilation=dilation, + layer_norm=layer_norm, + dropout=dropout) for i in range(layers) + ] + else: + repeats = [ + DeepFsmn( + hidden_dim, + hidden_dim, + lorder, + rorder, + hidden_dim2, + layer_norm=layer_norm, + dropout=dropout) for i in range(layers) + ] + self.deepfsmn = nn.Sequential(*repeats) + + self.linear2 = AffineTransform(hidden_dim, outdim) + + self.crm = crm + if self.crm: + self.sig = nn.Tanh() + else: + self.sig = Sigmoid(outdim, outdim) + + self.vad = vad + if self.vad: + self.linear3 = AffineTransform(hidden_dim, 1) + + self.layers = layers + self.linearout = linearout + if self.linearout and self.vad: + print('Warning: not supported nnet') + + def forward(self, feat, ctl=None): + x1 = self.linear1(feat) + x2 = self.relu(x1) + if ctl is not None: + ctl = min(ctl, self.layers - 1) + for i in range(ctl): + x2 = self.deepfsmn[i](x2) + mask = self.sig(self.linear2(x2)) + if self.vad: + vad = torch.sigmoid(self.linear3(x2)) + return mask, vad + else: + return mask + x3 = self.deepfsmn(x2) + if self.linearout: + return self.linear2(x3) + mask = self.sig(self.linear2(x3)) + if self.vad: + vad = torch.sigmoid(self.linear3(x3)) + return mask, vad + else: + return mask + + def to_kaldi_nnet(self): + re_str = '' + re_str += '\n' + re_str += self.linear1.to_kaldi_nnet() + re_str += self.relu.to_kaldi_nnet() + for dfsmn in self.deepfsmn: + re_str += dfsmn.to_kaldi_nnet() + re_str += self.linear2.to_kaldi_nnet() + re_str += self.sig.to_kaldi_nnet() + re_str += '\n' + + return re_str + + def to_raw_nnet(self, fid): + self.linear1.to_raw_nnet(fid) + for dfsmn in self.deepfsmn: + dfsmn.to_raw_nnet(fid) + self.linear2.to_raw_nnet(fid) + + +class StageNet(nn.Module): + + def __init__(self, + indim, + outdim, + layers=9, + layers2=6, + hidden_dim=128, + lorder=20, + rorder=0, + layer_norm=False, + dropout=0, + crm=False, + vad=False, + linearout=False): + super(StageNet, self).__init__() + + self.stage1 = nn.ModuleList() + self.stage2 = nn.ModuleList() + layer = nn.Sequential(nn.Linear(indim, hidden_dim), nn.ReLU()) + self.stage1.append(layer) + for i in range(layers): + layer = UniDeepFsmn( + hidden_dim, + hidden_dim, + lorder, + hidden_dim, + layer_norm=layer_norm, + dropout=dropout) + self.stage1.append(layer) + layer = nn.Sequential(nn.Linear(hidden_dim, 321), nn.Sigmoid()) + self.stage1.append(layer) + # stage2 + layer = nn.Sequential(nn.Linear(321 + indim, hidden_dim), nn.ReLU()) + self.stage2.append(layer) + for i in range(layers2): + layer = UniDeepFsmn( + hidden_dim, + hidden_dim, + lorder, + hidden_dim, + layer_norm=layer_norm, + dropout=dropout) + self.stage2.append(layer) + layer = nn.Sequential( + nn.Linear(hidden_dim, outdim), + nn.Sigmoid() if not crm else nn.Tanh()) + self.stage2.append(layer) + self.crm = crm + self.vad = vad + self.linearout = linearout + self.window = torch.hamming_window(640, periodic=False).cuda() + self.freezed = False + + def freeze(self): + if not self.freezed: + for param in self.stage1.parameters(): + param.requires_grad = False + self.freezed = True + print('freezed stage1') + + def forward(self, feat, mixture, ctl=None): + if ctl == 'off': + x = feat + for i in range(len(self.stage1)): + x = self.stage1[i](x) + return x + else: + self.freeze() + x = feat + for i in range(len(self.stage1)): + x = self.stage1[i](x) + + spec = torch.stft( + mixture / 32768, + 640, + 320, + 640, + self.window, + center=False, + return_complex=True) + spec = torch.view_as_real(spec).permute([0, 2, 1, 3]) + specmag = torch.sqrt(spec[..., 0]**2 + spec[..., 1]**2) + est = x * specmag + y = torch.cat([est, feat], dim=-1) + for i in range(len(self.stage2)): + y = self.stage2[i](y) + return y + + +class Unet(nn.Module): + + def __init__(self, + indim, + outdim, + layers=9, + dims=[256] * 4, + lorder=20, + rorder=0, + dilation=1, + layer_norm=False, + dropout=0, + crm=False, + vad=False, + linearout=False): + super(Unet, self).__init__() + + self.linear1 = AffineTransform(indim, dims[0]) + self.relu = RectifiedLinear(dims[0], dims[0]) + + self.encoder = nn.ModuleList() + self.decoder = nn.ModuleList() + for i in range(len(dims) - 1): + layer = nn.Sequential( + nn.Linear(dims[i], dims[i + 1]), nn.ReLU(), + nn.Linear(dims[i + 1], dims[i + 1], bias=False), + Conv2d( + dims[i + 1], + dims[i + 1], + lorder, + groups=dims[i + 1], + skip_connect=True)) + self.encoder.append(layer) + for i in range(len(dims) - 1, 0, -1): + layer = nn.Sequential( + nn.Linear(dims[i] * 2, dims[i - 1]), nn.ReLU(), + nn.Linear(dims[i - 1], dims[i - 1], bias=False), + Conv2d( + dims[i - 1], + dims[i - 1], + lorder, + groups=dims[i - 1], + skip_connect=True)) + self.decoder.append(layer) + self.tf = nn.ModuleList() + for i in range(layers - 2 * (len(dims) - 1)): + layer = nn.Sequential( + nn.Linear(dims[-1], dims[-1]), nn.ReLU(), + nn.Linear(dims[-1], dims[-1], bias=False), + Conv2d( + dims[-1], + dims[-1], + lorder, + groups=dims[-1], + skip_connect=True)) + self.tf.append(layer) + + self.linear2 = AffineTransform(dims[0], outdim) + self.crm = crm + self.act = nn.Tanh() if self.crm else nn.Sigmoid() + self.vad = False + self.layers = layers + self.linearout = linearout + + def forward(self, x, ctl=None): + x = self.linear1(x) + x = self.relu(x) + + encoder_out = [] + for i in range(len(self.encoder)): + x = self.encoder[i](x) + encoder_out.append(x) + for i in range(len(self.tf)): + x = self.tf[i](x) + for i in range(len(self.decoder)): + x = torch.cat([x, encoder_out[-1 - i]], dim=-1) + x = self.decoder[i](x) + + x = self.linear2(x) + if self.linearout: + return x + return self.act(x) + + +class BranchNet(nn.Module): + + def __init__(self, + indim, + outdim, + layers=9, + hidden_dim=256, + lorder=20, + rorder=0, + dilation=1, + layer_norm=False, + dropout=0, + crm=False, + vad=False, + linearout=False): + super(BranchNet, self).__init__() + + self.linear1 = AffineTransform(indim, hidden_dim) + self.relu = RectifiedLinear(hidden_dim, hidden_dim) + + self.convs = nn.ModuleList() + self.deepfsmn = nn.ModuleList() + self.FREQ = nn.ModuleList() + self.TIME = nn.ModuleList() + self.br1 = nn.ModuleList() + self.br2 = nn.ModuleList() + for i in range(layers): + ''' + layer = nn.Sequential( + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, hidden_dim, bias=False), + Conv2d(hidden_dim, hidden_dim, lorder, + groups=hidden_dim, skip_connect=True) + ) + self.deepfsmn.append(layer) + ''' + layer = nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.ReLU()) + self.FREQ.append(layer) + ''' + layer = nn.GRU(hidden_dim, hidden_dim, + batch_first=True, + bidirectional=False) + self.TIME.append(layer) + + layer = nn.Sequential( + nn.Linear(hidden_dim, hidden_dim//2, bias=False), + Conv2d(hidden_dim//2, hidden_dim//2, lorder, + groups=hidden_dim//2, skip_connect=True) + ) + self.br1.append(layer) + layer = nn.GRU(hidden_dim, hidden_dim//2, + batch_first=True, + bidirectional=False) + self.br2.append(layer) + ''' + + self.linear2 = AffineTransform(hidden_dim, outdim) + self.crm = crm + self.act = nn.Tanh() if self.crm else nn.Sigmoid() + self.vad = False + self.layers = layers + self.linearout = linearout + + def forward(self, x, ctl=None): + return self.forward_branch(x) + + def forward_sepconv(self, x): + x = torch.unsqueeze(x, 1) + for i in range(len(self.convs)): + x = self.convs[i](x) + x = F.relu(x) + B, C, H, W = x.shape + x = x.permute(0, 2, 1, 3) + x = torch.reshape(x, [B, H, C * W]) + x = self.linear1(x) + x = self.relu(x) + for i in range(self.layers): + x = self.deepfsmn[i](x) + x + x = self.linear2(x) + return self.act(x) + + def forward_branch(self, x): + x = self.linear1(x) + x = self.relu(x) + for i in range(self.layers): + z = self.FREQ[i](x) + x = z + x + x = self.linear2(x) + if self.linearout: + return x + return self.act(x) + + +class TACNet(nn.Module): + ''' transform average concatenate for ad hoc dr + ''' + + def __init__(self, + indim, + outdim, + layers=9, + hidden_dim=128, + lorder=20, + rorder=0, + crm=False, + vad=False, + linearout=False): + super(TACNet, self).__init__() + + self.linear1 = AffineTransform(indim, hidden_dim) + self.relu = RectifiedLinear(hidden_dim, hidden_dim) + + if rorder == 0: + repeats = [ + UniDeepFsmn(hidden_dim, hidden_dim, lorder, hidden_dim) + for i in range(layers) + ] + else: + repeats = [ + DeepFsmn(hidden_dim, hidden_dim, lorder, rorder, hidden_dim) + for i in range(layers) + ] + self.deepfsmn = nn.Sequential(*repeats) + + self.ch_transform = nn.ModuleList([]) + self.ch_average = nn.ModuleList([]) + self.ch_concat = nn.ModuleList([]) + for i in range(layers): + self.ch_transform.append( + nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU())) + self.ch_average.append( + nn.Sequential(nn.Linear(hidden_dim, hidden_dim), nn.PReLU())) + self.ch_concat.append( + nn.Sequential( + nn.Linear(hidden_dim * 2, hidden_dim), nn.PReLU())) + + self.linear2 = AffineTransform(hidden_dim, outdim) + + self.crm = crm + if self.crm: + self.sig = nn.Tanh() + else: + self.sig = Sigmoid(outdim, outdim) + + self.vad = vad + if self.vad: + self.linear3 = AffineTransform(hidden_dim, 1) + + self.layers = layers + self.linearout = linearout + if self.linearout and self.vad: + print('Warning: not supported nnet') + + def forward(self, feat, ctl=None): + B, T, F = feat.shape + # assume 4ch + ch = 4 + zlist = [] + for c in range(ch): + z = self.linear1(feat[..., c * (F // 4):(c + 1) * (F // 4)]) + z = self.relu(z) + zlist.append(z) + for i in range(self.layers): + # forward + for c in range(ch): + zlist[c] = self.deepfsmn[i](zlist[c]) + + # transform + olist = [] + for c in range(ch): + z = self.ch_transform[i](zlist[c]) + olist.append(z) + # average + avg = 0 + for c in range(ch): + avg = avg + olist[c] + avg = avg / ch + avg = self.ch_average[i](avg) + # concate + for c in range(ch): + tac = torch.cat([olist[c], avg], dim=-1) + tac = self.ch_concat[i](tac) + zlist[c] = zlist[c] + tac + + for c in range(ch): + zlist[c] = self.sig(self.linear2(zlist[c])) + mask = torch.cat(zlist, dim=-1) + return mask + + def to_kaldi_nnet(self): + pass diff --git a/runtime/Lib/site_packages/models/audio/ans/__init__.py b/runtime/Lib/site_packages/models/audio/ans/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b88a787a117d3dd53e462538904f7008358cfd2e --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .frcrn import FRCRNDecorator + +else: + _import_structure = { + 'frcrn': ['FRCRNDecorator'], + 'dnoise_net': ['DenoiseNet'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/ans/complex_nn.py b/runtime/Lib/site_packages/models/audio/ans/complex_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..98bfd8b53c998f0a97a9fe776b04879680cf53b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/complex_nn.py @@ -0,0 +1,206 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# +# The implementation of class ComplexConv2d, ComplexConvTranspose2d and +# ComplexBatchNorm2d here is modified based on Jongho Choi(sweetcocoa@snu.ac.kr +# / Seoul National Univ., ESTsoft ) and publicly available at +# https://github.com/sweetcocoa/DeepComplexUNetPyTorch + +import torch +import torch.nn as nn + +from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn + + +class ComplexUniDeepFsmn(nn.Module): + + def __init__(self, nIn, nHidden=128, nOut=128): + super(ComplexUniDeepFsmn, self).__init__() + + self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden) + self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden) + self.fsmn_re_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden) + self.fsmn_im_L2 = UniDeepFsmn(nHidden, nOut, 20, nHidden) + + def forward(self, x): + r""" + + Args: + x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2] + + Returns: + [batch, feature, sequence, 2], eg: [6, 99, 1024, 2] + """ + # + b, c, h, T, d = x.size() + x = torch.reshape(x, (b, c * h, T, d)) + # x: [b,h,T,2], [6, 256, 106, 2] + x = torch.transpose(x, 1, 2) + # x: [b,T,h,2], [6, 106, 256, 2] + + real_L1 = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1]) + imaginary_L1 = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0]) + # GRU output: [99, 6, 128] + real = self.fsmn_re_L2(real_L1) - self.fsmn_im_L2(imaginary_L1) + imaginary = self.fsmn_re_L2(imaginary_L1) + self.fsmn_im_L2(real_L1) + # output: [b,T,h,2], [99, 6, 1024, 2] + output = torch.stack((real, imaginary), dim=-1) + + # output: [b,h,T,2], [6, 99, 1024, 2] + output = torch.transpose(output, 1, 2) + output = torch.reshape(output, (b, c, h, T, d)) + + return output + + +class ComplexUniDeepFsmn_L1(nn.Module): + + def __init__(self, nIn, nHidden=128, nOut=128): + super(ComplexUniDeepFsmn_L1, self).__init__() + self.fsmn_re_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden) + self.fsmn_im_L1 = UniDeepFsmn(nIn, nHidden, 20, nHidden) + + def forward(self, x): + r""" + + Args: + x: torch with shape [batch, channel, feature, sequence, 2], eg: [6, 256, 1, 106, 2] + """ + b, c, h, T, d = x.size() + # x : [b,T,h,c,2] + x = torch.transpose(x, 1, 3) + x = torch.reshape(x, (b * T, h, c, d)) + + real = self.fsmn_re_L1(x[..., 0]) - self.fsmn_im_L1(x[..., 1]) + imaginary = self.fsmn_re_L1(x[..., 1]) + self.fsmn_im_L1(x[..., 0]) + # output: [b*T,h,c,2], [6*106, h, 256, 2] + output = torch.stack((real, imaginary), dim=-1) + + output = torch.reshape(output, (b, T, h, c, d)) + output = torch.transpose(output, 1, 3) + return output + + +class ComplexConv2d(nn.Module): + # https://github.com/litcoderr/ComplexCNN/blob/master/complexcnn/modules.py + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + **kwargs): + super().__init__() + + # Model components + self.conv_re = nn.Conv2d( + in_channel, + out_channel, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + **kwargs) + self.conv_im = nn.Conv2d( + in_channel, + out_channel, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + **kwargs) + + def forward(self, x): + r""" + + Args: + x: torch with shape: [batch,channel,axis1,axis2,2] + """ + real = self.conv_re(x[..., 0]) - self.conv_im(x[..., 1]) + imaginary = self.conv_re(x[..., 1]) + self.conv_im(x[..., 0]) + output = torch.stack((real, imaginary), dim=-1) + return output + + +class ComplexConvTranspose2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + bias=True, + **kwargs): + super().__init__() + + # Model components + self.tconv_re = nn.ConvTranspose2d( + in_channel, + out_channel, + kernel_size=kernel_size, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + bias=bias, + dilation=dilation, + **kwargs) + self.tconv_im = nn.ConvTranspose2d( + in_channel, + out_channel, + kernel_size=kernel_size, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + bias=bias, + dilation=dilation, + **kwargs) + + def forward(self, x): # shpae of x : [batch,channel,axis1,axis2,2] + real = self.tconv_re(x[..., 0]) - self.tconv_im(x[..., 1]) + imaginary = self.tconv_re(x[..., 1]) + self.tconv_im(x[..., 0]) + output = torch.stack((real, imaginary), dim=-1) + return output + + +class ComplexBatchNorm2d(nn.Module): + + def __init__(self, + num_features, + eps=1e-5, + momentum=0.1, + affine=True, + track_running_stats=True, + **kwargs): + super().__init__() + self.bn_re = nn.BatchNorm2d( + num_features=num_features, + momentum=momentum, + affine=affine, + eps=eps, + track_running_stats=track_running_stats, + **kwargs) + self.bn_im = nn.BatchNorm2d( + num_features=num_features, + momentum=momentum, + affine=affine, + eps=eps, + track_running_stats=track_running_stats, + **kwargs) + + def forward(self, x): + real = self.bn_re(x[..., 0]) + imag = self.bn_im(x[..., 1]) + output = torch.stack((real, imag), dim=-1) + return output diff --git a/runtime/Lib/site_packages/models/audio/ans/conv_stft.py b/runtime/Lib/site_packages/models/audio/ans/conv_stft.py new file mode 100644 index 0000000000000000000000000000000000000000..3d37f1aae4ca6136b77d0eef6ee91c309d0ce181 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/conv_stft.py @@ -0,0 +1,113 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from scipy.signal import get_window + + +def init_kernels(win_len, win_inc, fft_len, win_type=None, invers=False): + if win_type == 'None' or win_type is None: + window = np.ones(win_len) + else: + window = get_window(win_type, win_len, fftbins=True)**0.5 + + N = fft_len + fourier_basis = np.fft.rfft(np.eye(N))[:win_len] + real_kernel = np.real(fourier_basis) + imag_kernel = np.imag(fourier_basis) + kernel = np.concatenate([real_kernel, imag_kernel], 1).T + + if invers: + kernel = np.linalg.pinv(kernel).T + + kernel = kernel * window + kernel = kernel[:, None, :] + return torch.from_numpy(kernel.astype(np.float32)), torch.from_numpy( + window[None, :, None].astype(np.float32)) + + +class ConvSTFT(nn.Module): + + def __init__(self, + win_len, + win_inc, + fft_len=None, + win_type='hamming', + feature_type='real', + fix=True): + super(ConvSTFT, self).__init__() + + if fft_len is None: + self.fft_len = int(2**np.ceil(np.log2(win_len))) + else: + self.fft_len = fft_len + + kernel, _ = init_kernels(win_len, win_inc, self.fft_len, win_type) + self.weight = nn.Parameter(kernel, requires_grad=(not fix)) + self.feature_type = feature_type + self.stride = win_inc + self.win_len = win_len + self.dim = self.fft_len + + def forward(self, inputs): + if inputs.dim() == 2: + inputs = torch.unsqueeze(inputs, 1) + + outputs = F.conv1d(inputs, self.weight, stride=self.stride) + + if self.feature_type == 'complex': + return outputs + else: + dim = self.dim // 2 + 1 + real = outputs[:, :dim, :] + imag = outputs[:, dim:, :] + mags = torch.sqrt(real**2 + imag**2) + phase = torch.atan2(imag, real) + return mags, phase + + +class ConviSTFT(nn.Module): + + def __init__(self, + win_len, + win_inc, + fft_len=None, + win_type='hamming', + feature_type='real', + fix=True): + super(ConviSTFT, self).__init__() + if fft_len is None: + self.fft_len = int(2**np.ceil(np.log2(win_len))) + else: + self.fft_len = fft_len + kernel, window = init_kernels( + win_len, win_inc, self.fft_len, win_type, invers=True) + self.weight = nn.Parameter(kernel, requires_grad=(not fix)) + self.feature_type = feature_type + self.win_type = win_type + self.win_len = win_len + self.win_inc = win_inc + self.stride = win_inc + self.dim = self.fft_len + self.register_buffer('window', window) + self.register_buffer('enframe', torch.eye(win_len)[:, None, :]) + + def forward(self, inputs, phase=None): + """ + Args: + inputs : [B, N+2, T] (complex spec) or [B, N//2+1, T] (mags) + phase: [B, N//2+1, T] (if not none) + """ + + if phase is not None: + real = inputs * torch.cos(phase) + imag = inputs * torch.sin(phase) + inputs = torch.cat([real, imag], 1) + outputs = F.conv_transpose1d(inputs, self.weight, stride=self.stride) + + # this is from torch-stft: https://github.com/pseeth/torch-stft + t = self.window.repeat(1, 1, inputs.size(-1))**2 + coff = F.conv_transpose1d(t, self.enframe, stride=self.stride) + outputs = outputs / (coff + 1e-8) + return outputs diff --git a/runtime/Lib/site_packages/models/audio/ans/denoise_net.py b/runtime/Lib/site_packages/models/audio/ans/denoise_net.py new file mode 100644 index 0000000000000000000000000000000000000000..9d20074b47f3b4b5c5a866ca3f6372134a1cb606 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/denoise_net.py @@ -0,0 +1,73 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Related papers: +# Shengkui Zhao, Trung Hieu Nguyen, Bin Ma, “Monaural Speech Enhancement with Complex Convolutional +# Block Attention Module and Joint Time Frequency Losses”, ICASSP 2021. +# Shiliang Zhang, Ming Lei, Zhijie Yan, Lirong Dai, “Deep-FSMN for Large Vocabulary Continuous Speech +# Recognition “, arXiv:1803.05030, 2018. + +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.ans.layers.activations import (RectifiedLinear, + Sigmoid) +from modelscope.models.audio.ans.layers.affine_transform import AffineTransform +from modelscope.models.audio.ans.layers.uni_deep_fsmn import UniDeepFsmn +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.acoustic_noise_suppression, module_name=Models.speech_dfsmn_ans) +class DfsmnAns(TorchModel): + """Denoise model with DFSMN. + + Args: + model_dir (str): the model path. + fsmn_depth (int): the depth of deepfsmn + lorder (int): + """ + + def __init__(self, + model_dir: str, + fsmn_depth=9, + lorder=20, + *args, + **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.lorder = lorder + self.linear1 = AffineTransform(120, 256) + self.relu = RectifiedLinear(256, 256) + repeats = [ + UniDeepFsmn(256, 256, lorder, 256) for i in range(fsmn_depth) + ] + self.deepfsmn = nn.Sequential(*repeats) + self.linear2 = AffineTransform(256, 961) + self.sig = Sigmoid(961, 961) + + def forward(self, input): + """ + Args: + input: fbank feature [batch_size,number_of_frame,feature_dimension] + + Returns: + mask value [batch_size, number_of_frame, FFT_size/2+1] + """ + x1 = self.linear1(input) + x2 = self.relu(x1) + x3 = self.deepfsmn(x2) + x4 = self.linear2(x3) + x5 = self.sig(x4) + return x5 + + def to_kaldi_nnet(self): + re_str = '' + re_str += '\n' + re_str += self.linear1.to_kaldi_nnet() + re_str += self.relu.to_kaldi_nnet() + for dfsmn in self.deepfsmn: + re_str += dfsmn.to_kaldi_nnet() + re_str += self.linear2.to_kaldi_nnet() + re_str += self.sig.to_kaldi_nnet() + re_str += '\n' + + return re_str diff --git a/runtime/Lib/site_packages/models/audio/ans/frcrn.py b/runtime/Lib/site_packages/models/audio/ans/frcrn.py new file mode 100644 index 0000000000000000000000000000000000000000..0a83dfae0e3a8a5a3b063eaaf6f77fe6858b033c --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/frcrn.py @@ -0,0 +1,280 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Dict + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from .conv_stft import ConviSTFT, ConvSTFT +from .unet import UNet + + +@MODELS.register_module( + Tasks.acoustic_noise_suppression, + module_name=Models.speech_frcrn_ans_cirm_16k) +class FRCRNDecorator(TorchModel): + r""" A decorator of FRCRN for integrating into modelscope framework """ + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the frcrn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.model = FRCRN(*args, **kwargs) + model_bin_file = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + if os.path.exists(model_bin_file): + checkpoint = torch.load( + model_bin_file, map_location=torch.device('cpu')) + if isinstance(checkpoint, dict) and 'state_dict' in checkpoint: + # the new trained model by user is based on FRCRNDecorator + self.load_state_dict(checkpoint['state_dict']) + else: + # The released model on Modelscope is based on FRCRN + self.model.load_state_dict(checkpoint, strict=False) + + def forward(self, inputs: Dict[str, Tensor]) -> Dict[str, Tensor]: + result_list = self.model.forward(inputs['noisy']) + output = { + 'spec_l1': result_list[0], + 'wav_l1': result_list[1], + 'mask_l1': result_list[2], + 'spec_l2': result_list[3], + 'wav_l2': result_list[4], + 'mask_l2': result_list[5] + } + if 'clean' in inputs: + mix_result = self.model.loss( + inputs['noisy'], inputs['clean'], result_list, mode='Mix') + output.update(mix_result) + sisnr_result = self.model.loss( + inputs['noisy'], inputs['clean'], result_list, mode='SiSNR') + output.update(sisnr_result) + # logger hooker will use items under 'log_vars' + output['log_vars'] = {k: mix_result[k].item() for k in mix_result} + output['log_vars'].update( + {k: sisnr_result[k].item() + for k in sisnr_result}) + return output + + +class FRCRN(nn.Module): + r""" Frequency Recurrent CRN """ + + def __init__(self, + complex, + model_complexity, + model_depth, + log_amp, + padding_mode, + win_len=400, + win_inc=100, + fft_len=512, + win_type='hann', + **kwargs): + r""" + Args: + complex: Whether to use complex networks. + model_complexity: define the model complexity with the number of layers + model_depth: Only two options are available : 10, 20 + log_amp: Whether to use log amplitude to estimate signals + padding_mode: Encoder's convolution filter. 'zeros', 'reflect' + win_len: length of window used for defining one frame of sample points + win_inc: length of window shifting (equivalent to hop_size) + fft_len: number of Short Time Fourier Transform (STFT) points + win_type: windowing type used in STFT, eg. 'hanning', 'hamming' + """ + super().__init__() + self.feat_dim = fft_len // 2 + 1 + + self.win_len = win_len + self.win_inc = win_inc + self.fft_len = fft_len + self.win_type = win_type + + fix = True + self.stft = ConvSTFT( + self.win_len, + self.win_inc, + self.fft_len, + self.win_type, + feature_type='complex', + fix=fix) + self.istft = ConviSTFT( + self.win_len, + self.win_inc, + self.fft_len, + self.win_type, + feature_type='complex', + fix=fix) + self.unet = UNet( + 1, + complex=complex, + model_complexity=model_complexity, + model_depth=model_depth, + padding_mode=padding_mode) + self.unet2 = UNet( + 1, + complex=complex, + model_complexity=model_complexity, + model_depth=model_depth, + padding_mode=padding_mode) + + def forward(self, inputs): + out_list = [] + # [B, D*2, T] + cmp_spec = self.stft(inputs) + # [B, 1, D*2, T] + cmp_spec = torch.unsqueeze(cmp_spec, 1) + + # to [B, 2, D, T] real_part/imag_part + cmp_spec = torch.cat([ + cmp_spec[:, :, :self.feat_dim, :], + cmp_spec[:, :, self.feat_dim:, :], + ], 1) + + # [B, 2, D, T] + cmp_spec = torch.unsqueeze(cmp_spec, 4) + # [B, 1, D, T, 2] + cmp_spec = torch.transpose(cmp_spec, 1, 4) + unet1_out = self.unet(cmp_spec) + cmp_mask1 = torch.tanh(unet1_out) + unet2_out = self.unet2(unet1_out) + cmp_mask2 = torch.tanh(unet2_out) + est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask1) + out_list.append(est_spec) + out_list.append(est_wav) + out_list.append(est_mask) + cmp_mask2 = cmp_mask2 + cmp_mask1 + est_spec, est_wav, est_mask = self.apply_mask(cmp_spec, cmp_mask2) + out_list.append(est_spec) + out_list.append(est_wav) + out_list.append(est_mask) + return out_list + + def apply_mask(self, cmp_spec, cmp_mask): + est_spec = torch.cat([ + cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 0] + - cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 1], + cmp_spec[:, :, :, :, 0] * cmp_mask[:, :, :, :, 1] + + cmp_spec[:, :, :, :, 1] * cmp_mask[:, :, :, :, 0] + ], 1) + est_spec = torch.cat([est_spec[:, 0, :, :], est_spec[:, 1, :, :]], 1) + cmp_mask = torch.squeeze(cmp_mask, 1) + cmp_mask = torch.cat([cmp_mask[:, :, :, 0], cmp_mask[:, :, :, 1]], 1) + + est_wav = self.istft(est_spec) + est_wav = torch.squeeze(est_wav, 1) + return est_spec, est_wav, cmp_mask + + def get_params(self, weight_decay=0.0): + # add L2 penalty + weights, biases = [], [] + for name, param in self.named_parameters(): + if 'bias' in name: + biases += [param] + else: + weights += [param] + params = [{ + 'params': weights, + 'weight_decay': weight_decay, + }, { + 'params': biases, + 'weight_decay': 0.0, + }] + return params + + def loss(self, noisy, labels, out_list, mode='Mix'): + if mode == 'SiSNR': + count = 0 + while count < len(out_list): + est_spec = out_list[count] + count = count + 1 + est_wav = out_list[count] + count = count + 1 + est_mask = out_list[count] + count = count + 1 + if count != 3: + loss = self.loss_1layer(noisy, est_spec, est_wav, labels, + est_mask, mode) + return dict(sisnr=loss) + + elif mode == 'Mix': + count = 0 + while count < len(out_list): + est_spec = out_list[count] + count = count + 1 + est_wav = out_list[count] + count = count + 1 + est_mask = out_list[count] + count = count + 1 + if count != 3: + amp_loss, phase_loss, SiSNR_loss = self.loss_1layer( + noisy, est_spec, est_wav, labels, est_mask, mode) + loss = amp_loss + phase_loss + SiSNR_loss + return dict(loss=loss, amp_loss=amp_loss, phase_loss=phase_loss) + + def loss_1layer(self, noisy, est, est_wav, labels, cmp_mask, mode='Mix'): + r""" Compute the loss by mode + mode == 'Mix' + est: [B, F*2, T] + labels: [B, F*2,T] + mode == 'SiSNR' + est: [B, T] + labels: [B, T] + """ + if mode == 'SiSNR': + if labels.dim() == 3: + labels = torch.squeeze(labels, 1) + if est_wav.dim() == 3: + est_wav = torch.squeeze(est_wav, 1) + return -si_snr(est_wav, labels) + elif mode == 'Mix': + + if labels.dim() == 3: + labels = torch.squeeze(labels, 1) + if est_wav.dim() == 3: + est_wav = torch.squeeze(est_wav, 1) + SiSNR_loss = -si_snr(est_wav, labels) + + b, d, t = est.size() + S = self.stft(labels) + Sr = S[:, :self.feat_dim, :] + Si = S[:, self.feat_dim:, :] + Y = self.stft(noisy) + Yr = Y[:, :self.feat_dim, :] + Yi = Y[:, self.feat_dim:, :] + Y_pow = Yr**2 + Yi**2 + gth_mask = torch.cat([(Sr * Yr + Si * Yi) / (Y_pow + 1e-8), + (Si * Yr - Sr * Yi) / (Y_pow + 1e-8)], 1) + gth_mask[gth_mask > 2] = 1 + gth_mask[gth_mask < -2] = -1 + amp_loss = F.mse_loss(gth_mask[:, :self.feat_dim, :], + cmp_mask[:, :self.feat_dim, :]) * d + phase_loss = F.mse_loss(gth_mask[:, self.feat_dim:, :], + cmp_mask[:, self.feat_dim:, :]) * d + return amp_loss, phase_loss, SiSNR_loss + + +def l2_norm(s1, s2): + norm = torch.sum(s1 * s2, -1, keepdim=True) + return norm + + +def si_snr(s1, s2, eps=1e-8): + s1_s2_norm = l2_norm(s1, s2) + s2_s2_norm = l2_norm(s2, s2) + s_target = s1_s2_norm / (s2_s2_norm + eps) * s2 + e_noise = s1 - s_target + target_norm = l2_norm(s_target, s_target) + noise_norm = l2_norm(e_noise, e_noise) + snr = 10 * torch.log10((target_norm) / (noise_norm + eps) + eps) + return torch.mean(snr) diff --git a/runtime/Lib/site_packages/models/audio/ans/layers/__init__.py b/runtime/Lib/site_packages/models/audio/ans/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/ans/layers/activations.py b/runtime/Lib/site_packages/models/audio/ans/layers/activations.py new file mode 100644 index 0000000000000000000000000000000000000000..406de73694a614634423f37ead10baecd19451ae --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/layers/activations.py @@ -0,0 +1,62 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch.nn as nn + +from modelscope.models.audio.ans.layers.layer_base import LayerBase + + +class RectifiedLinear(LayerBase): + + def __init__(self, input_dim, output_dim): + super(RectifiedLinear, self).__init__() + self.dim = input_dim + self.relu = nn.ReLU() + + def forward(self, input): + return self.relu(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr + + +class LogSoftmax(LayerBase): + + def __init__(self, input_dim, output_dim): + super(LogSoftmax, self).__init__() + self.dim = input_dim + self.ls = nn.LogSoftmax() + + def forward(self, input): + return self.ls(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr + + +class Sigmoid(LayerBase): + + def __init__(self, input_dim, output_dim): + super(Sigmoid, self).__init__() + self.dim = input_dim + self.sig = nn.Sigmoid() + + def forward(self, input): + return self.sig(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + return re_str + + def load_kaldi_nnet(self, instr): + return instr diff --git a/runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py b/runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..d3cad18184eb53bec0bd9bae2a4d18fbea95037d --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/layers/affine_transform.py @@ -0,0 +1,86 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch as th +import torch.nn as nn + +from modelscope.models.audio.ans.layers.layer_base import (LayerBase, + to_kaldi_matrix) +from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix, + expect_token_number) + + +class AffineTransform(LayerBase): + + def __init__(self, input_dim, output_dim): + super(AffineTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + + self.linear = nn.Linear(input_dim, output_dim) + + def forward(self, input): + return self.linear(input) + + def to_kaldi_nnet(self): + re_str = '' + + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + + re_str += ' 1 1 0\n' + + linear_weights = self.state_dict()['linear.weight'] + + x = linear_weights.squeeze().numpy() + + re_str += to_kaldi_matrix(x) + + linear_bias = self.state_dict()['linear.bias'] + + x = linear_bias.squeeze().numpy() + + re_str += to_kaldi_matrix(x) + + return re_str + + def load_kaldi_nnet(self, instr): + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('AffineTransform format error') + + instr, lr = output + + output = expect_token_number(instr, '') + if output is None: + raise Exception('AffineTransform format error') + + instr, lr = output + + output = expect_token_number(instr, '') + if output is None: + raise Exception('AffineTransform format error') + + instr, lr = output + + output = expect_kaldi_matrix(instr) + + if output is None: + raise Exception('AffineTransform format error') + + instr, mat = output + + self.linear.weight = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('AffineTransform format error') + + instr, mat = output + self.linear.bias = th.nn.Parameter( + th.from_numpy(mat).type(th.FloatTensor)) + return instr diff --git a/runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py b/runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py new file mode 100644 index 0000000000000000000000000000000000000000..ca713d2fb67e8b19692af701114087543fddfd12 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/layers/layer_base.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import abc + +import numpy as np +import six +import torch.nn as nn + + +def to_kaldi_matrix(np_mat): + """ function that transform as str numpy mat to standard kaldi str matrix + + Args: + np_mat: numpy mat + """ + np.set_printoptions(threshold=np.inf, linewidth=np.nan) + out_str = str(np_mat) + out_str = out_str.replace('[', '') + out_str = out_str.replace(']', '') + return '[ %s ]\n' % out_str + + +@six.add_metaclass(abc.ABCMeta) +class LayerBase(nn.Module): + + def __init__(self): + super(LayerBase, self).__init__() + + @abc.abstractmethod + def to_kaldi_nnet(self): + pass diff --git a/runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py b/runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..772e6048e88929576617b3db7a071b88608c86e1 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/layers/uni_deep_fsmn.py @@ -0,0 +1,156 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.audio.ans.layers.layer_base import (LayerBase, + to_kaldi_matrix) +from modelscope.utils.audio.audio_utils import (expect_kaldi_matrix, + expect_token_number) + + +class UniDeepFsmn(LayerBase): + + def __init__(self, input_dim, output_dim, lorder=1, hidden_size=None): + super(UniDeepFsmn, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + self.lorder = lorder + self.hidden_size = hidden_size + + self.linear = nn.Linear(input_dim, hidden_size) + self.project = nn.Linear(hidden_size, output_dim, bias=False) + self.conv1 = nn.Conv2d( + output_dim, + output_dim, (lorder, 1), (1, 1), + groups=output_dim, + bias=False) + + def forward(self, input): + """ + + Args: + input: torch with shape: batch (b) x sequence(T) x feature (h) + + Returns: + batch (b) x channel (c) x sequence(T) x feature (h) + """ + f1 = F.relu(self.linear(input)) + p1 = self.project(f1) + x = torch.unsqueeze(p1, 1) + # x: batch (b) x channel (c) x sequence(T) x feature (h) + x_per = x.permute(0, 3, 2, 1) + # x_per: batch (b) x feature (h) x sequence(T) x channel (c) + y = F.pad(x_per, [0, 0, self.lorder - 1, 0]) + + out = x_per + self.conv1(y) + out1 = out.permute(0, 3, 2, 1) + # out1: batch (b) x channel (c) x sequence(T) x feature (h) + return input + out1.squeeze() + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n'\ + % (self.output_dim, self.input_dim) + re_str += ' %d %d %d %d 0\n'\ + % (1, self.hidden_size, self.lorder, 1) + + lfiters = self.state_dict()['conv1.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + re_str += to_kaldi_matrix(x) + proj_weights = self.state_dict()['project.weight'] + x = proj_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += to_kaldi_matrix(x) + return re_str + + def load_kaldi_nnet(self, instr): + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, lr = output + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, hiddensize = output + self.hidden_size = int(hiddensize) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, lorder = output + self.lorder = int(lorder) + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, lstride = output + self.lstride = lstride + + output = expect_token_number( + instr, + '', + ) + if output is None: + raise Exception('UniDeepFsmn format error') + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('Fsmn format error') + instr, mat = output + mat1 = np.fliplr(mat.T).copy() + self.conv1 = nn.Conv2d( + self.output_dim, + self.output_dim, (self.lorder, 1), (1, 1), + groups=self.output_dim, + bias=False) + mat_th = torch.from_numpy(mat1).type(torch.FloatTensor) + mat_th = mat_th.unsqueeze(1) + mat_th = mat_th.unsqueeze(3) + self.conv1.weight = torch.nn.Parameter(mat_th) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, mat = output + + self.project = nn.Linear(self.hidden_size, self.output_dim, bias=False) + self.linear = nn.Linear(self.input_dim, self.hidden_size) + self.project.weight = torch.nn.Parameter( + torch.from_numpy(mat).type(torch.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, mat = output + self.linear.weight = torch.nn.Parameter( + torch.from_numpy(mat).type(torch.FloatTensor)) + + output = expect_kaldi_matrix(instr) + if output is None: + raise Exception('UniDeepFsmn format error') + instr, mat = output + self.linear.bias = torch.nn.Parameter( + torch.from_numpy(mat).type(torch.FloatTensor)) + return instr diff --git a/runtime/Lib/site_packages/models/audio/ans/se_module_complex.py b/runtime/Lib/site_packages/models/audio/ans/se_module_complex.py new file mode 100644 index 0000000000000000000000000000000000000000..b58eb6bafdd4b40631f9917340920481455dd36e --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/se_module_complex.py @@ -0,0 +1,27 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +from torch import nn + + +class SELayer(nn.Module): + + def __init__(self, channel, reduction=16): + super(SELayer, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc_r = nn.Sequential( + nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), nn.Sigmoid()) + self.fc_i = nn.Sequential( + nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), nn.Sigmoid()) + + def forward(self, x): + b, c, _, _, _ = x.size() + x_r = self.avg_pool(x[:, :, :, :, 0]).view(b, c) + x_i = self.avg_pool(x[:, :, :, :, 1]).view(b, c) + y_r = self.fc_r(x_r).view(b, c, 1, 1, 1) - self.fc_i(x_i).view( + b, c, 1, 1, 1) + y_i = self.fc_r(x_i).view(b, c, 1, 1, 1) + self.fc_i(x_r).view( + b, c, 1, 1, 1) + y = torch.cat([y_r, y_i], 4) + return x * y diff --git a/runtime/Lib/site_packages/models/audio/ans/unet.py b/runtime/Lib/site_packages/models/audio/ans/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..7b4df1e9f31e967aedfa549e48fb0968a5bbe8b2 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/ans/unet.py @@ -0,0 +1,276 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# +# The implementation here is modified based on +# Jongho Choi(sweetcocoa@snu.ac.kr / Seoul National Univ., ESTsoft ) +# and publicly available at +# https://github.com/sweetcocoa/DeepComplexUNetPyTorch + +import torch +import torch.nn as nn + +from . import complex_nn +from .se_module_complex import SELayer + + +class Encoder(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + padding=None, + complex=False, + padding_mode='zeros'): + super().__init__() + if padding is None: + padding = [(i - 1) // 2 for i in kernel_size] # 'SAME' padding + + if complex: + conv = complex_nn.ComplexConv2d + bn = complex_nn.ComplexBatchNorm2d + else: + conv = nn.Conv2d + bn = nn.BatchNorm2d + + self.conv = conv( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + padding_mode=padding_mode) + self.bn = bn(out_channels) + self.relu = nn.LeakyReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class Decoder(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + padding=(0, 0), + complex=False): + super().__init__() + if complex: + tconv = complex_nn.ComplexConvTranspose2d + bn = complex_nn.ComplexBatchNorm2d + else: + tconv = nn.ConvTranspose2d + bn = nn.BatchNorm2d + + self.transconv = tconv( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding) + self.bn = bn(out_channels) + self.relu = nn.LeakyReLU(inplace=True) + + def forward(self, x): + x = self.transconv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class UNet(nn.Module): + + def __init__(self, + input_channels=1, + complex=False, + model_complexity=45, + model_depth=20, + padding_mode='zeros'): + super().__init__() + + if complex: + model_complexity = int(model_complexity // 1.414) + + self.set_size( + model_complexity=model_complexity, + input_channels=input_channels, + model_depth=model_depth) + self.encoders = [] + self.model_length = model_depth // 2 + self.fsmn = complex_nn.ComplexUniDeepFsmn(128, 128, 128) + self.se_layers_enc = [] + self.fsmn_enc = [] + for i in range(self.model_length): + fsmn_enc = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128) + self.add_module('fsmn_enc{}'.format(i), fsmn_enc) + self.fsmn_enc.append(fsmn_enc) + module = Encoder( + self.enc_channels[i], + self.enc_channels[i + 1], + kernel_size=self.enc_kernel_sizes[i], + stride=self.enc_strides[i], + padding=self.enc_paddings[i], + complex=complex, + padding_mode=padding_mode) + self.add_module('encoder{}'.format(i), module) + self.encoders.append(module) + se_layer_enc = SELayer(self.enc_channels[i + 1], 8) + self.add_module('se_layer_enc{}'.format(i), se_layer_enc) + self.se_layers_enc.append(se_layer_enc) + self.decoders = [] + self.fsmn_dec = [] + self.se_layers_dec = [] + for i in range(self.model_length): + fsmn_dec = complex_nn.ComplexUniDeepFsmn_L1(128, 128, 128) + self.add_module('fsmn_dec{}'.format(i), fsmn_dec) + self.fsmn_dec.append(fsmn_dec) + module = Decoder( + self.dec_channels[i] * 2, + self.dec_channels[i + 1], + kernel_size=self.dec_kernel_sizes[i], + stride=self.dec_strides[i], + padding=self.dec_paddings[i], + complex=complex) + self.add_module('decoder{}'.format(i), module) + self.decoders.append(module) + if i < self.model_length - 1: + se_layer_dec = SELayer(self.dec_channels[i + 1], 8) + self.add_module('se_layer_dec{}'.format(i), se_layer_dec) + self.se_layers_dec.append(se_layer_dec) + if complex: + conv = complex_nn.ComplexConv2d + else: + conv = nn.Conv2d + + linear = conv(self.dec_channels[-1], 1, 1) + + self.add_module('linear', linear) + self.complex = complex + self.padding_mode = padding_mode + + self.decoders = nn.ModuleList(self.decoders) + self.encoders = nn.ModuleList(self.encoders) + self.se_layers_enc = nn.ModuleList(self.se_layers_enc) + self.se_layers_dec = nn.ModuleList(self.se_layers_dec) + self.fsmn_enc = nn.ModuleList(self.fsmn_enc) + self.fsmn_dec = nn.ModuleList(self.fsmn_dec) + + def forward(self, inputs): + x = inputs + # go down + xs = [] + xs_se = [] + xs_se.append(x) + for i, encoder in enumerate(self.encoders): + xs.append(x) + if i > 0: + x = self.fsmn_enc[i](x) + x = encoder(x) + xs_se.append(self.se_layers_enc[i](x)) + # xs : x0=input x1 ... x9 + x = self.fsmn(x) + + p = x + for i, decoder in enumerate(self.decoders): + p = decoder(p) + if i < self.model_length - 1: + p = self.fsmn_dec[i](p) + if i == self.model_length - 1: + break + if i < self.model_length - 2: + p = self.se_layers_dec[i](p) + p = torch.cat([p, xs_se[self.model_length - 1 - i]], dim=1) + + # cmp_spec: [12, 1, 513, 64, 2] + cmp_spec = self.linear(p) + return cmp_spec + + def set_size(self, model_complexity, model_depth=20, input_channels=1): + + if model_depth == 14: + self.enc_channels = [ + input_channels, 128, 128, 128, 128, 128, 128, 128 + ] + self.enc_kernel_sizes = [(5, 2), (5, 2), (5, 2), (5, 2), (5, 2), + (5, 2), (2, 2)] + self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1), + (2, 1)] + self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1), + (0, 1), (0, 1)] + self.dec_channels = [64, 128, 128, 128, 128, 128, 128, 1] + self.dec_kernel_sizes = [(2, 2), (5, 2), (5, 2), (5, 2), (6, 2), + (5, 2), (5, 2)] + self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1), (2, 1), + (2, 1)] + self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1), + (0, 1), (0, 1)] + + elif model_depth == 10: + self.enc_channels = [ + input_channels, + 16, + 32, + 64, + 128, + 256, + ] + self.enc_kernel_sizes = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3)] + self.enc_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)] + self.enc_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)] + self.dec_channels = [128, 128, 64, 32, 16, 1] + self.dec_kernel_sizes = [(3, 3), (3, 3), (3, 3), (4, 3), (3, 3)] + self.dec_strides = [(2, 1), (2, 1), (2, 1), (2, 1), (2, 1)] + self.dec_paddings = [(0, 1), (0, 1), (0, 1), (0, 1), (0, 1)] + + elif model_depth == 20: + self.enc_channels = [ + input_channels, model_complexity, model_complexity, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, 128 + ] + + self.enc_kernel_sizes = [(7, 1), (1, 7), (6, 4), (7, 5), (5, 3), + (5, 3), (5, 3), (5, 3), (5, 3), (5, 3)] + + self.enc_strides = [(1, 1), (1, 1), (2, 2), (2, 1), (2, 2), (2, 1), + (2, 2), (2, 1), (2, 2), (2, 1)] + + self.enc_paddings = [ + (3, 0), + (0, 3), + None, # (0, 2), + None, + None, # (3,1), + None, # (3,1), + None, # (1,2), + None, + None, + None + ] + + self.dec_channels = [ + 0, model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2, model_complexity * 2, + model_complexity * 2 + ] + + self.dec_kernel_sizes = [(4, 3), (4, 2), (4, 3), (4, 2), (4, 3), + (4, 2), (6, 3), (7, 4), (1, 7), (7, 1)] + + self.dec_strides = [(2, 1), (2, 2), (2, 1), (2, 2), (2, 1), (2, 2), + (2, 1), (2, 2), (1, 1), (1, 1)] + + self.dec_paddings = [(1, 1), (1, 0), (1, 1), (1, 0), (1, 1), + (1, 0), (2, 1), (2, 1), (0, 3), (3, 0)] + else: + raise ValueError('Unknown model depth : {}'.format(model_depth)) diff --git a/runtime/Lib/site_packages/models/audio/asr/__init__.py b/runtime/Lib/site_packages/models/audio/asr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d6ca2c4ee0d065de44e2b5b1873fd6b8f78b3bb0 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/asr/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .generic_automatic_speech_recognition import GenericAutomaticSpeechRecognition + +else: + _import_structure = { + 'generic_automatic_speech_recognition': + ['GenericAutomaticSpeechRecognition'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py b/runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..5e02076ee1bfd1f4902130e365c38421555bd628 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/asr/generic_automatic_speech_recognition.py @@ -0,0 +1,51 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Frameworks, Tasks + +__all__ = ['GenericAutomaticSpeechRecognition'] + + +@MODELS.register_module( + Tasks.auto_speech_recognition, module_name=Models.generic_asr) +@MODELS.register_module( + Tasks.voice_activity_detection, module_name=Models.generic_asr) +@MODELS.register_module( + Tasks.speech_separation, module_name=Models.generic_asr) +@MODELS.register_module( + Tasks.language_score_prediction, module_name=Models.generic_asr) +@MODELS.register_module(Tasks.speech_timestamp, module_name=Models.generic_asr) +class GenericAutomaticSpeechRecognition(Model): + + def __init__(self, model_dir: str, am_model_name: str, + model_config: Dict[str, Any], *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + am_model_name (str): the am model name from configuration.json + model_config (Dict[str, Any]): the detail config about model from configuration.json + """ + super().__init__(model_dir, am_model_name, model_config, *args, + **kwargs) + self.model_cfg = { + # the recognition model dir path + 'model_workspace': model_dir, + # the am model name + 'am_model': am_model_name, + # the am model file path + 'am_model_path': os.path.join(model_dir, am_model_name), + # the recognition model config dict + 'model_config': model_config + } + + def forward(self) -> Dict[str, Any]: + """preload model and return the info of the model + """ + + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py b/runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..feb822d4c8b7470dc85f136aec7e15ac00926760 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/asr/wenet_automatic_speech_recognition.py @@ -0,0 +1,38 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +import json +import wenetruntime as wenet + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks + +__all__ = ['WeNetAutomaticSpeechRecognition'] + + +@MODELS.register_module( + Tasks.auto_speech_recognition, module_name=Models.wenet_asr) +class WeNetAutomaticSpeechRecognition(Model): + + def __init__(self, model_dir: str, am_model_name: str, + model_config: Dict[str, Any], *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, am_model_name, model_config, *args, + **kwargs) + self.decoder = wenet.Decoder(model_dir, lang='chs') + + def forward(self, inputs: Dict[str, Any]) -> str: + if inputs['audio_format'] == 'wav': + rst = self.decoder.decode_wav(inputs['audio']) + else: + rst = self.decoder.decode(inputs['audio']) + text = json.loads(rst)['nbest'][0]['sentence'] + return {'text': text} diff --git a/runtime/Lib/site_packages/models/audio/itn/__init__.py b/runtime/Lib/site_packages/models/audio/itn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..be299f4a4ad28210c0e1d592a900196c80ec9e87 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/itn/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .generic_inverse_text_processing import GenericInverseTextProcessing + +else: + _import_structure = { + 'generic_inverse_text_processing': ['GenericInverseTextProcessing'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py b/runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..b71c776986f2e8c72b21bf65c74d8b3513a379f6 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/itn/generic_inverse_text_processing.py @@ -0,0 +1,44 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Frameworks, Tasks + + +@MODELS.register_module( + Tasks.inverse_text_processing, module_name=Models.generic_itn) +class GenericInverseTextProcessing(Model): + + def __init__(self, model_dir: str, itn_model_name: str, + model_config: Dict[str, Any], *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + itn_model_name (str): the itn model name from configuration.json + model_config (Dict[str, Any]): the detail config about model from configuration.json + """ + super().__init__(model_dir, itn_model_name, model_config, *args, + **kwargs) + self.model_cfg = { + # the recognition model dir path + 'model_workspace': model_dir, + # the itn model name + 'itn_model': itn_model_name, + # the am model file path + 'itn_model_path': os.path.join(model_dir, itn_model_name), + # the recognition model config dict + 'model_config': model_config + } + + def forward(self) -> Dict[str, Any]: + """ + just return the model config + + """ + + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/audio/kws/__init__.py b/runtime/Lib/site_packages/models/audio/kws/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ee39be366b47c7729de7369dd0b25d08a5cc9e64 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .generic_key_word_spotting import GenericKeyWordSpotting + from .farfield.model import FSMNSeleNetV2Decorator + from .nearfield.model import FSMNDecorator + +else: + _import_structure = { + 'generic_key_word_spotting': ['GenericKeyWordSpotting'], + 'farfield.model': ['FSMNSeleNetV2Decorator'], + 'nearfield.model': ['FSMNDecorator'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py b/runtime/Lib/site_packages/models/audio/kws/farfield/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..e06d791118c9f2b0e5802a9d97e5057e0b5f20b7 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn.py @@ -0,0 +1,497 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .model_def import (HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32, + printNeonMatrix, printNeonVector) + +DEBUG = False + + +def to_kaldi_matrix(np_mat): + """ function that transform as str numpy mat to standard kaldi str matrix + + Args: + np_mat: numpy mat + + Returns: str + """ + np.set_printoptions(threshold=np.inf, linewidth=np.nan) + out_str = str(np_mat) + out_str = out_str.replace('[', '') + out_str = out_str.replace(']', '') + return '[ %s ]\n' % out_str + + +def print_tensor(torch_tensor): + """ print torch tensor for debug + + Args: + torch_tensor: a tensor + """ + re_str = '' + x = torch_tensor.detach().squeeze().numpy() + re_str += to_kaldi_matrix(x) + re_str += '\n' + print(re_str) + + +class LinearTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(LinearTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.linear = nn.Linear(input_dim, output_dim, bias=False) + + self.debug = False + self.dataout = None + + def forward(self, input): + output = self.linear(input) + + if self.debug: + self.dataout = output + + return output + + def print_model(self): + printNeonMatrix(self.linear.weight) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + re_str += ' 1\n' + + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + re_str += '\n' + + return re_str + + +class AffineTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(AffineTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + + self.linear = nn.Linear(input_dim, output_dim) + + self.debug = False + self.dataout = None + + def forward(self, input): + output = self.linear(input) + + if self.debug: + self.dataout = output + + return output + + def print_model(self): + printNeonMatrix(self.linear.weight) + printNeonVector(self.linear.bias) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + re_str += ' 1 1 0\n' + + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += to_kaldi_matrix(x) + + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += to_kaldi_matrix(x) + re_str += '\n' + + return re_str + + +class Fsmn(nn.Module): + """ + FSMN implementation. + """ + + def __init__(self, + input_dim, + output_dim, + lorder=None, + rorder=None, + lstride=None, + rstride=None): + super(Fsmn, self).__init__() + + self.dim = input_dim + + if lorder is None: + return + + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + + self.conv_left = nn.Conv2d( + self.dim, + self.dim, (lorder, 1), + dilation=(lstride, 1), + groups=self.dim, + bias=False) + + if rorder > 0: + self.conv_right = nn.Conv2d( + self.dim, + self.dim, (rorder, 1), + dilation=(rstride, 1), + groups=self.dim, + bias=False) + else: + self.conv_right = None + + self.debug = False + self.dataout = None + + def forward(self, input): + x = torch.unsqueeze(input, 1) + x_per = x.permute(0, 3, 2, 1) + + y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0]) + + if self.conv_right is not None: + y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride]) + y_right = y_right[:, :, self.rstride:, :] + out = x_per + self.conv_left(y_left) + self.conv_right(y_right) + else: + out = x_per + self.conv_left(y_left) + + out1 = out.permute(0, 3, 2, 1) + output = out1.squeeze(1) + + if self.debug: + self.dataout = output + + return output + + def print_model(self): + tmpw = self.conv_left.weight + tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0]) + for j in range(tmpw.shape[0]): + tmpwm[:, j] = tmpw[j, 0, :, 0] + + printNeonMatrix(tmpwm) + + if self.conv_right is not None: + tmpw = self.conv_right.weight + tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0]) + for j in range(tmpw.shape[0]): + tmpwm[:, j] = tmpw[j, 0, :, 0] + + printNeonMatrix(tmpwm) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + re_str += ' %d %d %d %d %d 0\n' % ( + 1, self.lorder, self.rorder, self.lstride, self.rstride) + + lfiters = self.state_dict()['conv_left.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + re_str += to_kaldi_matrix(x) + + if self.conv_right is not None: + rfiters = self.state_dict()['conv_right.weight'] + x = (rfiters.squeeze().numpy().T) + re_str += to_kaldi_matrix(x) + re_str += '\n' + + return re_str + + +class RectifiedLinear(nn.Module): + + def __init__(self, input_dim, output_dim): + super(RectifiedLinear, self).__init__() + self.dim = input_dim + self.relu = nn.ReLU() + + def forward(self, input): + return self.relu(input) + + def to_kaldi_nnet(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + re_str += '\n' + return re_str + + +class FSMNNet(nn.Module): + """ + FSMN net for keyword spotting + """ + + def __init__(self, + input_dim=200, + linear_dim=128, + proj_dim=128, + lorder=10, + rorder=1, + num_syn=5, + fsmn_layers=4): + """ + Args: + input_dim: input dimension + linear_dim: fsmn input dimension + proj_dim: fsmn projection dimension + lorder: fsmn left order + rorder: fsmn right order + num_syn: output dimension + fsmn_layers: no. of sequential fsmn layers + """ + super(FSMNNet, self).__init__() + + self.input_dim = input_dim + self.linear_dim = linear_dim + self.proj_dim = proj_dim + self.lorder = lorder + self.rorder = rorder + self.num_syn = num_syn + self.fsmn_layers = fsmn_layers + + self.linear1 = AffineTransform(input_dim, linear_dim) + self.relu = RectifiedLinear(linear_dim, linear_dim) + + self.fsmn = self._build_repeats(linear_dim, proj_dim, lorder, rorder, + fsmn_layers) + + self.linear2 = AffineTransform(linear_dim, num_syn) + + @staticmethod + def _build_repeats(linear_dim=136, + proj_dim=68, + lorder=3, + rorder=2, + fsmn_layers=5): + repeats = [ + nn.Sequential( + LinearTransform(linear_dim, proj_dim), + Fsmn(proj_dim, proj_dim, lorder, rorder, 1, 1), + AffineTransform(proj_dim, linear_dim), + RectifiedLinear(linear_dim, linear_dim)) + for i in range(fsmn_layers) + ] + + return nn.Sequential(*repeats) + + def forward(self, input): + x1 = self.linear1(input) + x2 = self.relu(x1) + x3 = self.fsmn(x2) + x4 = self.linear2(x3) + return x4 + + def print_model(self): + self.linear1.print_model() + + for layer in self.fsmn: + layer[0].print_model() + layer[1].print_model() + layer[2].print_model() + + self.linear2.print_model() + + def print_header(self): + # + # write total header + # + header = [0.0] * HEADER_BLOCK_SIZE * 4 + # numins + header[0] = 0.0 + # numouts + header[1] = 0.0 + # dimins + header[2] = self.input_dim + # dimouts + header[3] = self.num_syn + # numlayers + header[4] = 3 + + # + # write each layer's header + # + hidx = 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = self.input_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = self.linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_RELU.value) + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_SEQUENTIAL_FSMN.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = self.proj_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = self.lorder + header[HEADER_BLOCK_SIZE * hidx + 5] = self.rorder + header[HEADER_BLOCK_SIZE * hidx + 6] = self.fsmn_layers + header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0 + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = self.linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = self.num_syn + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_SOFTMAX.value) + + for h in header: + print(f32ToI32(h)) + + def to_kaldi_nnet(self): + re_str = '' + re_str += '\n' + re_str += self.linear1.to_kaldi_nnet() + re_str += self.relu.to_kaldi_nnet() + + for fsmn in self.fsmn: + re_str += fsmn[0].to_kaldi_nnet() + re_str += fsmn[1].to_kaldi_nnet() + re_str += fsmn[2].to_kaldi_nnet() + re_str += fsmn[3].to_kaldi_nnet() + + re_str += self.linear2.to_kaldi_nnet() + re_str += ' %d %d\n' % (self.num_syn, self.num_syn) + re_str += '\n' + re_str += '\n' + + return re_str + + +class DFSMN(nn.Module): + """ + One deep fsmn layer + """ + + def __init__(self, + dimproj=64, + dimlinear=128, + lorder=20, + rorder=1, + lstride=1, + rstride=1): + """ + Args: + dimproj: projection dimension, input and output dimension of memory blocks + dimlinear: dimension of mapping layer + lorder: left order + rorder: right order + lstride: left stride + rstride: right stride + """ + super(DFSMN, self).__init__() + + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + + self.expand = AffineTransform(dimproj, dimlinear) + self.shrink = LinearTransform(dimlinear, dimproj) + + self.conv_left = nn.Conv2d( + dimproj, + dimproj, (lorder, 1), + dilation=(lstride, 1), + groups=dimproj, + bias=False) + + if rorder > 0: + self.conv_right = nn.Conv2d( + dimproj, + dimproj, (rorder, 1), + dilation=(rstride, 1), + groups=dimproj, + bias=False) + else: + self.conv_right = None + + def forward(self, input): + f1 = F.relu(self.expand(input)) + p1 = self.shrink(f1) + + x = torch.unsqueeze(p1, 1) + x_per = x.permute(0, 3, 2, 1) + + y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0]) + + if self.conv_right is not None: + y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride]) + y_right = y_right[:, :, self.rstride:, :] + out = x_per + self.conv_left(y_left) + self.conv_right(y_right) + else: + out = x_per + self.conv_left(y_left) + + out1 = out.permute(0, 3, 2, 1) + output = input + out1.squeeze(1) + + return output + + def print_model(self): + self.expand.print_model() + self.shrink.print_model() + + tmpw = self.conv_left.weight + tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0]) + for j in range(tmpw.shape[0]): + tmpwm[:, j] = tmpw[j, 0, :, 0] + + printNeonMatrix(tmpwm) + + if self.conv_right is not None: + tmpw = self.conv_right.weight + tmpwm = torch.zeros(tmpw.shape[2], tmpw.shape[0]) + for j in range(tmpw.shape[0]): + tmpwm[:, j] = tmpw[j, 0, :, 0] + + printNeonMatrix(tmpwm) + + +def build_dfsmn_repeats(linear_dim=128, + proj_dim=64, + lorder=20, + rorder=1, + fsmn_layers=6): + """ + build stacked dfsmn layers + Args: + linear_dim: + proj_dim: + lorder: + rorder: + fsmn_layers: + + Returns: + + """ + repeats = [ + nn.Sequential(DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1)) + for i in range(fsmn_layers) + ] + + return nn.Sequential(*repeats) diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..a258e00493f90150cf5a5f1ca50afcb57bb84093 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v2.py @@ -0,0 +1,238 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear +from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32 + + +class FSMNUnit(nn.Module): + """ A multi-channel fsmn unit + + """ + + def __init__(self, dimlinear=128, dimproj=64, lorder=20, rorder=1): + """ + Args: + dimlinear: input / output dimension + dimproj: fsmn input / output dimension + lorder: left order + rorder: right order + """ + super(FSMNUnit, self).__init__() + + self.shrink = LinearTransform(dimlinear, dimproj) + self.fsmn = Fsmn(dimproj, dimproj, lorder, rorder, 1, 1) + self.expand = AffineTransform(dimproj, dimlinear) + + self.debug = False + self.dataout = None + + ''' + batch, time, channel, feature + ''' + + def forward(self, input): + if torch.cuda.is_available(): + out = torch.zeros(input.shape).cuda() + else: + out = torch.zeros(input.shape) + + for n in range(input.shape[2]): + out1 = self.shrink(input[:, :, n, :]) + out2 = self.fsmn(out1) + out[:, :, n, :] = F.relu(self.expand(out2)) + + if self.debug: + self.dataout = out + + return out + + def print_model(self): + self.shrink.print_model() + self.fsmn.print_model() + self.expand.print_model() + + def to_kaldi_nnet(self): + re_str = self.shrink.to_kaldi_nnet() + re_str += self.fsmn.to_kaldi_nnet() + re_str += self.expand.to_kaldi_nnet() + + relu = RectifiedLinear(self.expand.linear.out_features, + self.expand.linear.out_features) + re_str += relu.to_kaldi_nnet() + + return re_str + + +class FSMNSeleNetV2(nn.Module): + """ FSMN model with channel selection. + """ + + def __init__(self, + input_dim=120, + linear_dim=128, + proj_dim=64, + lorder=20, + rorder=1, + num_syn=5, + fsmn_layers=5, + sele_layer=0): + """ + Args: + input_dim: input dimension + linear_dim: fsmn input dimension + proj_dim: fsmn projection dimension + lorder: fsmn left order + rorder: fsmn right order + num_syn: output dimension + fsmn_layers: no. of fsmn units + sele_layer: channel selection layer index + """ + super(FSMNSeleNetV2, self).__init__() + + self.sele_layer = sele_layer + + self.featmap = AffineTransform(input_dim, linear_dim) + + self.mem = [] + for i in range(fsmn_layers): + unit = FSMNUnit(linear_dim, proj_dim, lorder, rorder) + self.mem.append(unit) + self.add_module('mem_{:d}'.format(i), unit) + + self.decision = AffineTransform(linear_dim, num_syn) + + def forward(self, input): + # multi-channel feature mapping + if torch.cuda.is_available(): + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.featmap.linear.out_features).cuda() + else: + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.featmap.linear.out_features) + + for n in range(input.shape[2]): + x[:, :, n, :] = F.relu(self.featmap(input[:, :, n, :])) + + for i, unit in enumerate(self.mem): + y = unit(x) + + # perform channel selection + if i == self.sele_layer: + pool = nn.MaxPool2d((y.shape[2], 1), stride=(y.shape[2], 1)) + y = pool(y) + + x = y + + # remove channel dimension + y = torch.squeeze(y, -2) + z = self.decision(y) + + return z + + def print_model(self): + self.featmap.print_model() + + for unit in self.mem: + unit.print_model() + + self.decision.print_model() + + def print_header(self): + ''' + get FSMN params + ''' + input_dim = self.featmap.linear.in_features + linear_dim = self.featmap.linear.out_features + proj_dim = self.mem[0].shrink.linear.out_features + lorder = self.mem[0].fsmn.conv_left.kernel_size[0] + rorder = 0 + if self.mem[0].fsmn.conv_right is not None: + rorder = self.mem[0].fsmn.conv_right.kernel_size[0] + + num_syn = self.decision.linear.out_features + fsmn_layers = len(self.mem) + + # no. of output channels, 0.0 means the same as numins + # numouts = 0.0 + numouts = 1.0 + + # + # write total header + # + header = [0.0] * HEADER_BLOCK_SIZE * 4 + # numins + header[0] = 0.0 + # numouts + header[1] = numouts + # dimins + header[2] = input_dim + # dimouts + header[3] = num_syn + # numlayers + header[4] = 3 + + # + # write each layer's header + # + hidx = 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_RELU.value) + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_SEQUENTIAL_FSMN.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = proj_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = lorder + header[HEADER_BLOCK_SIZE * hidx + 5] = rorder + header[HEADER_BLOCK_SIZE * hidx + 6] = fsmn_layers + if numouts == 1.0: + header[HEADER_BLOCK_SIZE * hidx + 7] = float(self.sele_layer) + else: + header[HEADER_BLOCK_SIZE * hidx + 7] = -1.0 + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = numouts + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_SOFTMAX.value) + + for h in header: + print(f32ToI32(h)) + + def to_kaldi_nnet(self): + re_str = '\n' + + re_str = self.featmap.to_kaldi_nnet() + + relu = RectifiedLinear(self.featmap.linear.out_features, + self.featmap.linear.out_features) + re_str += relu.to_kaldi_nnet() + + for unit in self.mem: + re_str += unit.to_kaldi_nnet() + + re_str += self.decision.to_kaldi_nnet() + + re_str += ' %d %d\n' % (self.decision.linear.out_features, + self.decision.linear.out_features) + re_str += '\n' + re_str += '\n' + + return re_str diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..d57354d0c563659050a1a16b9365e134f70a002b --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/farfield/fsmn_sele_v3.py @@ -0,0 +1,233 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .fsmn import AffineTransform, Fsmn, LinearTransform, RectifiedLinear +from .model_def import HEADER_BLOCK_SIZE, ActivationType, LayerType, f32ToI32 + + +class DFSMNUnit(nn.Module): + """ one multi-channel deep fsmn unit + Args: + dimin: input dimension + dimexpand: feature expansion dimension + dimout: output dimension + lorder: left ofder + rorder: right order + """ + + def __init__(self, + dimin=64, + dimexpand=128, + dimout=64, + lorder=10, + rorder=1): + super(DFSMNUnit, self).__init__() + + self.expand = AffineTransform(dimin, dimexpand) + self.shrink = LinearTransform(dimexpand, dimout) + self.fsmn = Fsmn(dimout, dimout, lorder, rorder, 1, 1) + + self.debug = False + self.dataout = None + + def forward(self, input): + """ + Args: + input: [batch, time, feature] + """ + out1 = F.relu(self.expand(input)) + out2 = self.shrink(out1) + out3 = self.fsmn(out2) + + # add skip connection for matched data + if input.shape[-1] == out3.shape[-1]: + out3 = input + out3 + if self.debug: + self.dataout = out3 + return out3 + + def print_model(self): + self.expand.printModel() + self.shrink.printModel() + self.fsmn.printModel() + + def to_kaldi_nnet(self): + re_str = self.expand.toKaldiNNet() + relu = RectifiedLinear(self.expand.linear.out_features, + self.expand.linear.out_features) + re_str += relu.toKaldiNNet() + re_str = self.shrink.toKaldiNNet() + re_str += self.fsmn.toKaldiNNet() + return re_str + + +class FSMNSeleNetV3(nn.Module): + """ Deep FSMN model with channel selection performs multi-channel kws. + Zhang, Shiliang, et al. "Deep-FSMN for large vocabulary continuous speech + recognition." 2018 IEEE International Conference on Acoustics, Speech and + Signal Processing (ICASSP). IEEE, 2018. + + Args: + input_dim: input dimension + linear_dim: fsmn input dimension + proj_dim: fsmn projection dimension + lorder: fsmn left order + rorder: fsmn right order + num_syn: output dimension + fsmn_layers: no. of fsmn units + """ + + def __init__(self, + input_dim=120, + linear_dim=128, + proj_dim=64, + lorder=10, + rorder=1, + num_syn=5, + fsmn_layers=5): + super(FSMNSeleNetV3, self).__init__() + + self.mem = [] + # the first unit, mapping input dim to proj dim + unit = DFSMNUnit(input_dim, linear_dim, proj_dim, lorder, rorder) + self.mem.append(unit) + self.add_module('mem_{:d}'.format(0), unit) + + # deep fsmn layers with skip connection + for i in range(1, fsmn_layers): + unit = DFSMNUnit(proj_dim, linear_dim, proj_dim, lorder, rorder) + self.mem.append(unit) + self.add_module('mem_{:d}'.format(i), unit) + + self.expand2 = AffineTransform(proj_dim, linear_dim) + self.decision = AffineTransform(linear_dim, num_syn) + + def forward(self, input): + # multi-channel temp space, [batch, time, channel, feature] + if torch.cuda.is_available(): + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.expand2.linear.out_features).cuda() + else: + x = torch.zeros(input.shape[0], input.shape[1], input.shape[2], + self.expand2.linear.out_features) + + for n in range(input.shape[2]): + chin = input[:, :, n, :] + + for unit in self.mem: + chout = unit(chin) + chin = chout + + x[:, :, n, :] = F.relu(self.expand2(chout)) + + # perform max pooling + pool = nn.MaxPool2d((x.shape[2], 1), stride=(x.shape[2], 1)) + y = pool(x) + + # remove channel dimension + y = torch.squeeze(y, -2) + z = self.decision(y) + + return z + + def print_model(self): + for unit in self.mem: + unit.print_model() + + self.expand2.printModel() + self.decision.printModel() + + def print_header(self): + """ get DFSMN params + """ + input_dim = self.mem[0].expand.linear.in_features + linear_dim = self.mem[0].expand.linear.out_features + proj_dim = self.mem[0].shrink.linear.out_features + lorder = self.mem[0].fsmn.conv_left.kernel_size[0] + rorder = 0 + if self.mem[0].fsmn.conv_right is not None: + rorder = self.mem[0].fsmn.conv_right.kernel_size[0] + + num_syn = self.decision.linear.out_features + fsmn_layers = len(self.mem) + + # no. of output channels, 0.0 means the same as numins + numouts = 1.0 + + # + # write total header + # + header = [0.0] * HEADER_BLOCK_SIZE * 5 + # numins + header[0] = 0.0 + # numouts + header[1] = numouts + # dimins + header[2] = input_dim + # dimouts + header[3] = num_syn + # numlayers + header[4] = 4 + + # + # write each layer's header + # + hidx = 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DFSMN.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = input_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = proj_dim + header[HEADER_BLOCK_SIZE * hidx + 5] = lorder + header[HEADER_BLOCK_SIZE * hidx + 6] = rorder + header[HEADER_BLOCK_SIZE * hidx + 7] = fsmn_layers + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = proj_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_RELU.value) + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_MAX_POOLING.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = 0.0 + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + hidx += 1 + + header[HEADER_BLOCK_SIZE * hidx + 0] = float( + LayerType.LAYER_DENSE.value) + header[HEADER_BLOCK_SIZE * hidx + 1] = numouts + header[HEADER_BLOCK_SIZE * hidx + 2] = linear_dim + header[HEADER_BLOCK_SIZE * hidx + 3] = num_syn + header[HEADER_BLOCK_SIZE * hidx + 4] = 1.0 + header[HEADER_BLOCK_SIZE * hidx + 5] = float( + ActivationType.ACTIVATION_SOFTMAX.value) + + for h in header: + print(f32ToI32(h)) + + def to_kaldi_nnet(self): + re_str = '\n' + for unit in self.mem: + re_str += unit.to_kaldi_nnet() + re_str = self.expand2.toKaldiNNet() + relu = RectifiedLinear(self.expand2.linear.out_features, + self.expand2.linear.out_features) + re_str += relu.toKaldiNNet() + re_str += self.decision.toKaldiNNet() + re_str += ' %d %d\n' % (self.decision.linear.out_features, + self.decision.linear.out_features) + re_str += '\n' + re_str += '\n' + + return re_str diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/model.py b/runtime/Lib/site_packages/models/audio/kws/farfield/model.py new file mode 100644 index 0000000000000000000000000000000000000000..670ac97ca6bb79cedb51f7f3b3fd75b5aa379e27 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/farfield/model.py @@ -0,0 +1,98 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import tempfile +from typing import Dict, Optional + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.utils.audio.audio_utils import update_conf +from modelscope.utils.constant import Tasks +from .fsmn_sele_v2 import FSMNSeleNetV2 +from .fsmn_sele_v3 import FSMNSeleNetV3 + + +@MODELS.register_module( + Tasks.keyword_spotting, module_name=Models.speech_dfsmn_kws_char_farfield) +class FSMNSeleNetV2Decorator(TorchModel): + r""" A decorator of FSMNSeleNetV2 for integrating into modelscope framework """ + + MODEL_CLASS = FSMNSeleNetV2 + MODEL_TXT = 'model.txt' + SC_CONFIG = 'sound_connect.conf' + + def __init__(self, + model_dir: str, + training: Optional[bool] = False, + *args, + **kwargs): + """initialize the dfsmn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + if training: + self.model = self.MODEL_CLASS(*args, **kwargs) + else: + sc_config_file = os.path.join(model_dir, self.SC_CONFIG) + model_txt_file = os.path.join(model_dir, self.MODEL_TXT) + self.tmp_dir = tempfile.TemporaryDirectory() + new_config_file = os.path.join(self.tmp_dir.name, self.SC_CONFIG) + + self._sc = None + if os.path.exists(model_txt_file): + conf_dict = dict(kws_model=model_txt_file) + update_conf(sc_config_file, new_config_file, conf_dict) + import py_sound_connect + self._sc = py_sound_connect.SoundConnect(new_config_file) + self.size_in = self._sc.bytesPerBlockIn() + self.size_out = self._sc.bytesPerBlockOut() + else: + raise Exception( + f'Invalid model directory! Failed to load model file:' + f' {model_txt_file}.') + + def __del__(self): + if hasattr(self, 'tmp_dir'): + self.tmp_dir.cleanup() + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + return self.model.forward(input) + + def forward_decode(self, data: bytes): + result = {'pcm': self._sc.process(data, self.size_out)} + state = self._sc.kwsState() + if state == 2: + result['kws'] = { + 'keyword': + self._sc.kwsKeyword(self._sc.kwsSpottedKeywordIndex()), + 'offset': self._sc.kwsKeywordOffset(), + 'channel': self._sc.kwsBestChannel(), + 'length': self._sc.kwsKeywordLength(), + 'confidence': self._sc.kwsConfidence() + } + return result + + +@MODELS.register_module( + Tasks.keyword_spotting, + module_name=Models.speech_dfsmn_kws_char_farfield_iot) +class FSMNSeleNetV3Decorator(FSMNSeleNetV2Decorator): + r""" A decorator of FSMNSeleNetV3 for integrating into modelscope framework """ + + MODEL_CLASS = FSMNSeleNetV3 + + def __init__(self, + model_dir: str, + training: Optional[bool] = False, + *args, + **kwargs): + """initialize the dfsmn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, training, *args, **kwargs) diff --git a/runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py b/runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py new file mode 100644 index 0000000000000000000000000000000000000000..be9cca2c56b0533df4e3fd867f47260e5b42deb7 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/farfield/model_def.py @@ -0,0 +1,123 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import struct +from enum import Enum + +HEADER_BLOCK_SIZE = 10 + + +class LayerType(Enum): + LAYER_DENSE = 1 + LAYER_GRU = 2 + LAYER_ATTENTION = 3 + LAYER_FSMN = 4 + LAYER_SEQUENTIAL_FSMN = 5 + LAYER_FSMN_SELE = 6 + LAYER_GRU_ATTENTION = 7 + LAYER_DFSMN = 8 + + +class ActivationType(Enum): + ACTIVATION_NONE = 0 + ACTIVATION_RELU = 1 + ACTIVATION_TANH = 2 + ACTIVATION_SIGMOID = 3 + ACTIVATION_SOFTMAX = 4 + ACTIVATION_LOGSOFTMAX = 5 + + +def f32ToI32(f): + """ + print layer + """ + bs = struct.pack('f', f) + + ba = bytearray() + ba.append(bs[0]) + ba.append(bs[1]) + ba.append(bs[2]) + ba.append(bs[3]) + + return struct.unpack('i', ba)[0] + + +def printNeonMatrix(w): + """ + print matrix with neon padding + """ + numrows, numcols = w.shape + numnecols = math.ceil(numcols / 4) + + for i in range(numrows): + for j in range(numcols): + print(f32ToI32(w[i, j])) + + for j in range(numnecols * 4 - numcols): + print(0) + + +def printNeonVector(b): + """ + print vector with neon padding + """ + size = b.shape[0] + nesize = math.ceil(size / 4) + + for i in range(size): + print(f32ToI32(b[i])) + + for i in range(nesize * 4 - size): + print(0) + + +def printDense(layer): + """ + save dense layer + """ + statedict = layer.state_dict() + printNeonMatrix(statedict['weight']) + printNeonVector(statedict['bias']) + + +def printGRU(layer): + """ + save gru layer + """ + statedict = layer.state_dict() + weight = [statedict['weight_ih_l0'], statedict['weight_hh_l0']] + bias = [statedict['bias_ih_l0'], statedict['bias_hh_l0']] + numins, numouts = weight[0].shape + numins = numins // 3 + + # output input weights + w_rx = weight[0][:numins, :] + w_zx = weight[0][numins:numins * 2, :] + w_x = weight[0][numins * 2:, :] + printNeonMatrix(w_zx) + printNeonMatrix(w_rx) + printNeonMatrix(w_x) + + # output recurrent weights + w_rh = weight[1][:numins, :] + w_zh = weight[1][numins:numins * 2, :] + w_h = weight[1][numins * 2:, :] + printNeonMatrix(w_zh) + printNeonMatrix(w_rh) + printNeonMatrix(w_h) + + # output input bias + b_rx = bias[0][:numins] + b_zx = bias[0][numins:numins * 2] + b_x = bias[0][numins * 2:] + printNeonVector(b_zx) + printNeonVector(b_rx) + printNeonVector(b_x) + + # output recurrent bias + b_rh = bias[1][:numins] + b_zh = bias[1][numins:numins * 2] + b_h = bias[1][numins * 2:] + printNeonVector(b_zh) + printNeonVector(b_rh) + printNeonVector(b_h) diff --git a/runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py b/runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py new file mode 100644 index 0000000000000000000000000000000000000000..2f70327d6ea622c44de61171bc80d20c453409eb --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/generic_key_word_spotting.py @@ -0,0 +1,32 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks + +__all__ = ['GenericKeyWordSpotting'] + + +@MODELS.register_module(Tasks.keyword_spotting, module_name=Models.kws_kwsbp) +class GenericKeyWordSpotting(Model): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.model_cfg = { + 'model_workspace': model_dir, + 'config_path': os.path.join(model_dir, 'config.yaml') + } + + def forward(self) -> Dict[str, Any]: + """return the info of the model + """ + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py b/runtime/Lib/site_packages/models/audio/kws/nearfield/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py b/runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py new file mode 100644 index 0000000000000000000000000000000000000000..bad065f70b6518c4244140d9aa889c22083a2731 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/nearfield/cmvn.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Binbin Zhang +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import numpy as np +import torch + + +class GlobalCMVN(torch.nn.Module): + + def __init__(self, + mean: torch.Tensor, + istd: torch.Tensor, + norm_var: bool = True): + """ + Args: + mean (torch.Tensor): mean stats + istd (torch.Tensor): inverse std, std which is 1.0 / std + """ + super().__init__() + assert mean.shape == istd.shape + self.norm_var = norm_var + # The buffer can be accessed from this module using self.mean + self.register_buffer('mean', mean) + self.register_buffer('istd', istd) + + def forward(self, x: torch.Tensor): + """ + Args: + x (torch.Tensor): (batch, max_len, feat_dim) + + Returns: + (torch.Tensor): normalized feature + """ + x = x - self.mean + if self.norm_var: + x = x * self.istd + return x + + +def load_kaldi_cmvn(cmvn_file): + """ Load the kaldi format cmvn stats file and no need to calculate + + Args: + cmvn_file: cmvn stats file in kaldi format + + Returns: + a numpy array of [means, vars] + """ + + means = None + variance = None + with open(cmvn_file) as f: + all_lines = f.readlines() + for idx, line in enumerate(all_lines): + if line.find('AddShift') != -1: + segs = line.strip().split(' ') + assert len(segs) == 3 + next_line = all_lines[idx + 1] + means_str = re.findall(r'[\[](.*?)[\]]', next_line)[0] + means_list = means_str.strip().split(' ') + means = [0 - float(s) for s in means_list] + assert len(means) == int(segs[1]) + elif line.find('Rescale') != -1: + segs = line.strip().split(' ') + assert len(segs) == 3 + next_line = all_lines[idx + 1] + vars_str = re.findall(r'[\[](.*?)[\]]', next_line)[0] + vars_list = vars_str.strip().split(' ') + variance = [float(s) for s in vars_list] + assert len(variance) == int(segs[1]) + elif line.find('Splice') != -1: + segs = line.strip().split(' ') + assert len(segs) == 3 + next_line = all_lines[idx + 1] + splice_str = re.findall(r'[\[](.*?)[\]]', next_line)[0] + splice_list = splice_str.strip().split(' ') + assert len(splice_list) * int(segs[2]) == int(segs[1]) + copy_times = len(splice_list) + else: + continue + + cmvn = np.array([means, variance]) + cmvn = np.tile(cmvn, (1, copy_times)) + + return cmvn diff --git a/runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py b/runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..094dacd22e0efc1752970bb6b5e436293f47a5fd --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/nearfield/fsmn.py @@ -0,0 +1,521 @@ +''' +FSMN implementation. + +Copyright: 2022-03-09 yueyue.nyy +''' + +from typing import Tuple + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def toKaldiMatrix(np_mat): + np.set_printoptions(threshold=np.inf, linewidth=np.nan) + out_str = str(np_mat) + out_str = out_str.replace('[', '') + out_str = out_str.replace(']', '') + return '[ %s ]\n' % out_str + + +def printTensor(torch_tensor): + re_str = '' + x = torch_tensor.detach().squeeze().numpy() + re_str += toKaldiMatrix(x) + # re_str += '\n' + print(re_str) + + +class LinearTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(LinearTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.linear = nn.Linear(input_dim, output_dim, bias=False) + self.quant = torch.quantization.QuantStub() + self.dequant = torch.quantization.DeQuantStub() + + def forward(self, input): + output = self.quant(input) + output = self.linear(output) + output = self.dequant(output) + + return output + + def to_kaldi_net(self): + re_str = '' + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + re_str += ' 1\n' + + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += toKaldiMatrix(x) + # re_str += '\n' + + return re_str + + def to_pytorch_net(self, fread): + linear_line = fread.readline() + linear_split = linear_line.strip().split() + assert len(linear_split) == 3 + assert linear_split[0] == '' + self.output_dim = int(linear_split[1]) + self.input_dim = int(linear_split[2]) + + learn_rate_line = fread.readline() + assert learn_rate_line.find('LearnRateCoef') != -1 + + self.linear.reset_parameters() + + # linear_weights = self.state_dict()['linear.weight'] + # print(linear_weights.shape) + new_weights = torch.zeros((self.output_dim, self.input_dim), + dtype=torch.float32) + for i in range(self.output_dim): + line = fread.readline() + splits = line.strip().strip('[]').strip().split() + assert len(splits) == self.input_dim + cols = torch.tensor([float(item) for item in splits], + dtype=torch.float32) + new_weights[i, :] = cols + + self.linear.weight.data = new_weights + + +class AffineTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(AffineTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + + self.linear = nn.Linear(input_dim, output_dim) + self.quant = torch.quantization.QuantStub() + self.dequant = torch.quantization.DeQuantStub() + + def forward(self, input): + output = self.quant(input) + output = self.linear(output) + output = self.dequant(output) + + return output + + def to_kaldi_net(self): + re_str = '' + re_str += ' %d %d\n' % (self.output_dim, + self.input_dim) + re_str += ' 1 1 0\n' + + linear_weights = self.state_dict()['linear.weight'] + x = linear_weights.squeeze().numpy() + re_str += toKaldiMatrix(x) + + linear_bias = self.state_dict()['linear.bias'] + x = linear_bias.squeeze().numpy() + re_str += toKaldiMatrix(x) + # re_str += '\n' + + return re_str + + def to_pytorch_net(self, fread): + affine_line = fread.readline() + affine_split = affine_line.strip().split() + assert len(affine_split) == 3 + assert affine_split[0] == '' + self.output_dim = int(affine_split[1]) + self.input_dim = int(affine_split[2]) + print('AffineTransform output/input dim: %d %d' % + (self.output_dim, self.input_dim)) + + learn_rate_line = fread.readline() + assert learn_rate_line.find('LearnRateCoef') != -1 + + # linear_weights = self.state_dict()['linear.weight'] + # print(linear_weights.shape) + self.linear.reset_parameters() + + new_weights = torch.zeros((self.output_dim, self.input_dim), + dtype=torch.float32) + for i in range(self.output_dim): + line = fread.readline() + splits = line.strip().strip('[]').strip().split() + assert len(splits) == self.input_dim + cols = torch.tensor([float(item) for item in splits], + dtype=torch.float32) + new_weights[i, :] = cols + + self.linear.weight.data = new_weights + + # linear_bias = self.state_dict()['linear.bias'] + # print(linear_bias.shape) + bias_line = fread.readline() + splits = bias_line.strip().strip('[]').strip().split() + assert len(splits) == self.output_dim + new_bias = torch.tensor([float(item) for item in splits], + dtype=torch.float32) + + self.linear.bias.data = new_bias + + +class FSMNBlock(nn.Module): + + def __init__( + self, + input_dim: int, + output_dim: int, + lorder=None, + rorder=None, + lstride=1, + rstride=1, + ): + super(FSMNBlock, self).__init__() + + self.dim = input_dim + + if lorder is None: + return + + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + + self.conv_left = nn.Conv2d( + self.dim, + self.dim, [lorder, 1], + dilation=[lstride, 1], + groups=self.dim, + bias=False) + + if rorder > 0: + self.conv_right = nn.Conv2d( + self.dim, + self.dim, [rorder, 1], + dilation=[rstride, 1], + groups=self.dim, + bias=False) + else: + self.conv_right = None + + self.quant = torch.quantization.QuantStub() + self.dequant = torch.quantization.DeQuantStub() + + def forward(self, input): + x = torch.unsqueeze(input, 1) + x_per = x.permute(0, 3, 2, 1) + + y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0]) + y_left = self.quant(y_left) + y_left = self.conv_left(y_left) + y_left = self.dequant(y_left) + out = x_per + y_left + + if self.conv_right is not None: + y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride]) + y_right = y_right[:, :, self.rstride:, :] + y_right = self.quant(y_right) + y_right = self.conv_right(y_right) + y_right = self.dequant(y_right) + out += y_right + + out_per = out.permute(0, 3, 2, 1) + output = out_per.squeeze(1) + + return output + + def to_kaldi_net(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + re_str += ' %d %d %d %d %d 0\n' % ( + 1, self.lorder, self.rorder, self.lstride, self.rstride) + + # print(self.conv_left.weight,self.conv_right.weight) + lfiters = self.state_dict()['conv_left.weight'] + x = np.flipud(lfiters.squeeze().numpy().T) + re_str += toKaldiMatrix(x) + + if self.conv_right is not None: + rfiters = self.state_dict()['conv_right.weight'] + x = (rfiters.squeeze().numpy().T) + re_str += toKaldiMatrix(x) + # re_str += '\n' + + return re_str + + def to_pytorch_net(self, fread): + fsmn_line = fread.readline() + fsmn_split = fsmn_line.strip().split() + assert len(fsmn_split) == 3 + assert fsmn_split[0] == '' + self.dim = int(fsmn_split[1]) + + params_line = fread.readline() + params_split = params_line.strip().strip('[]').strip().split() + assert len(params_split) == 12 + assert params_split[0] == '' + assert params_split[2] == '' + self.lorder = int(params_split[3]) + assert params_split[4] == '' + self.rorder = int(params_split[5]) + assert params_split[6] == '' + self.lstride = int(params_split[7]) + assert params_split[8] == '' + self.rstride = int(params_split[9]) + assert params_split[10] == '' + + # lfilters = self.state_dict()['conv_left.weight'] + # print(lfilters.shape) + print('read conv_left weight') + new_lfilters = torch.zeros((self.lorder, 1, self.dim, 1), + dtype=torch.float32) + for i in range(self.lorder): + print('read conv_left weight -- %d' % i) + line = fread.readline() + splits = line.strip().strip('[]').strip().split() + assert len(splits) == self.dim + cols = torch.tensor([float(item) for item in splits], + dtype=torch.float32) + new_lfilters[self.lorder - 1 - i, 0, :, 0] = cols + + new_lfilters = torch.transpose(new_lfilters, 0, 2) + # print(new_lfilters.shape) + + self.conv_left.reset_parameters() + self.conv_left.weight.data = new_lfilters + # print(self.conv_left.weight.shape) + + if self.rorder > 0: + # rfilters = self.state_dict()['conv_right.weight'] + # print(rfilters.shape) + print('read conv_right weight') + new_rfilters = torch.zeros((self.rorder, 1, self.dim, 1), + dtype=torch.float32) + line = fread.readline() + for i in range(self.rorder): + print('read conv_right weight -- %d' % i) + line = fread.readline() + splits = line.strip().strip('[]').strip().split() + assert len(splits) == self.dim + cols = torch.tensor([float(item) for item in splits], + dtype=torch.float32) + new_rfilters[i, 0, :, 0] = cols + + new_rfilters = torch.transpose(new_rfilters, 0, 2) + # print(new_rfilters.shape) + self.conv_right.reset_parameters() + self.conv_right.weight.data = new_rfilters + # print(self.conv_right.weight.shape) + + +class RectifiedLinear(nn.Module): + + def __init__(self, input_dim, output_dim): + super(RectifiedLinear, self).__init__() + self.dim = input_dim + self.relu = nn.ReLU() + self.dropout = nn.Dropout(0.1) + + def forward(self, input): + out = self.relu(input) + # out = self.dropout(out) + return out + + def to_kaldi_net(self): + re_str = '' + re_str += ' %d %d\n' % (self.dim, self.dim) + # re_str += '\n' + return re_str + + # re_str = '' + # re_str += ' %d %d\n' % (self.dim, self.dim) + # re_str += ' 0 0\n' + # re_str += toKaldiMatrix(np.ones((self.dim), dtype = 'int32')) + # re_str += toKaldiMatrix(np.zeros((self.dim), dtype = 'int32')) + # re_str += '\n' + # return re_str + + def to_pytorch_net(self, fread): + line = fread.readline() + splits = line.strip().split() + assert len(splits) == 3 + assert splits[0] == '' + assert int(splits[1]) == int(splits[2]) + assert int(splits[1]) == self.dim + self.dim = int(splits[1]) + + +def _build_repeats( + fsmn_layers: int, + linear_dim: int, + proj_dim: int, + lorder: int, + rorder: int, + lstride=1, + rstride=1, +): + repeats = [ + nn.Sequential( + LinearTransform(linear_dim, proj_dim), + FSMNBlock(proj_dim, proj_dim, lorder, rorder, 1, 1), + AffineTransform(proj_dim, linear_dim), + RectifiedLinear(linear_dim, linear_dim)) + for i in range(fsmn_layers) + ] + + return nn.Sequential(*repeats) + + +class FSMN(nn.Module): + + def __init__( + self, + input_dim: int, + input_affine_dim: int, + fsmn_layers: int, + linear_dim: int, + proj_dim: int, + lorder: int, + rorder: int, + lstride: int, + rstride: int, + output_affine_dim: int, + output_dim: int, + ): + """ + Args: + input_dim: input dimension + input_affine_dim: input affine layer dimension + fsmn_layers: no. of fsmn units + linear_dim: fsmn input dimension + proj_dim: fsmn projection dimension + lorder: fsmn left order + rorder: fsmn right order + lstride: fsmn left stride + rstride: fsmn right stride + output_affine_dim: output affine layer dimension + output_dim: output dimension + """ + super(FSMN, self).__init__() + + self.input_dim = input_dim + self.input_affine_dim = input_affine_dim + self.fsmn_layers = fsmn_layers + self.linear_dim = linear_dim + self.proj_dim = proj_dim + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + self.output_affine_dim = output_affine_dim + self.output_dim = output_dim + + self.in_linear1 = AffineTransform(input_dim, input_affine_dim) + self.in_linear2 = AffineTransform(input_affine_dim, linear_dim) + self.relu = RectifiedLinear(linear_dim, linear_dim) + + self.fsmn = _build_repeats(fsmn_layers, linear_dim, proj_dim, lorder, + rorder, lstride, rstride) + + self.out_linear1 = AffineTransform(linear_dim, output_affine_dim) + self.out_linear2 = AffineTransform(output_affine_dim, output_dim) + # self.softmax = nn.Softmax(dim = -1) + + def fuse_modules(self): + pass + + def forward( + self, + input: torch.Tensor, + in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float) + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + input (torch.Tensor): Input tensor (B, T, D) + in_cache(torch.Tensor): (B, D, C), C is the accumulated cache size + """ + + # print("FSMN forward!!!!") + # print(input.shape) + # print(input) + # print(self.in_linear1.input_dim) + # print(self.in_linear1.output_dim) + + x1 = self.in_linear1(input) + x2 = self.in_linear2(x1) + x3 = self.relu(x2) + x4 = self.fsmn(x3) + x5 = self.out_linear1(x4) + x6 = self.out_linear2(x5) + # x7 = self.softmax(x6) + + # return x7, None + return x6, in_cache + + def to_kaldi_net(self): + re_str = '' + re_str += '\n' + re_str += self.in_linear1.to_kaldi_net() + re_str += self.in_linear2.to_kaldi_net() + re_str += self.relu.to_kaldi_net() + + for fsmn in self.fsmn: + re_str += fsmn[0].to_kaldi_net() + re_str += fsmn[1].to_kaldi_net() + re_str += fsmn[2].to_kaldi_net() + re_str += fsmn[3].to_kaldi_net() + + re_str += self.out_linear1.to_kaldi_net() + re_str += self.out_linear2.to_kaldi_net() + re_str += ' %d %d\n' % (self.output_dim, self.output_dim) + # re_str += '\n' + re_str += '\n' + + return re_str + + def to_pytorch_net(self, kaldi_file): + with open(kaldi_file, 'r', encoding='utf8') as fread: + fread = open(kaldi_file, 'r') + nnet_start_line = fread.readline() + assert nnet_start_line.strip() == '' + + self.in_linear1.to_pytorch_net(fread) + self.in_linear2.to_pytorch_net(fread) + self.relu.to_pytorch_net(fread) + + for fsmn in self.fsmn: + fsmn[0].to_pytorch_net(fread) + fsmn[1].to_pytorch_net(fread) + fsmn[2].to_pytorch_net(fread) + fsmn[3].to_pytorch_net(fread) + + self.out_linear1.to_pytorch_net(fread) + self.out_linear2.to_pytorch_net(fread) + + softmax_line = fread.readline() + softmax_split = softmax_line.strip().split() + assert softmax_split[0].strip() == '' + assert int(softmax_split[1]) == self.output_dim + assert int(softmax_split[2]) == self.output_dim + # '\n' + + nnet_end_line = fread.readline() + assert nnet_end_line.strip() == '' + fread.close() + + +if __name__ == '__main__': + fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599) + print(fsmn) + + num_params = sum(p.numel() for p in fsmn.parameters()) + print('the number of model params: {}'.format(num_params)) + x = torch.zeros(128, 200, 400) # batch-size * time * dim + y, _ = fsmn(x) # batch-size * time * dim + print('input shape: {}'.format(x.shape)) + print('output shape: {}'.format(y.shape)) + + print(fsmn.to_kaldi_net()) diff --git a/runtime/Lib/site_packages/models/audio/kws/nearfield/model.py b/runtime/Lib/site_packages/models/audio/kws/nearfield/model.py new file mode 100644 index 0000000000000000000000000000000000000000..023d5011a07f784b73c7ca70b56c1d5ff08de2d6 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/kws/nearfield/model.py @@ -0,0 +1,178 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import sys +import tempfile +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.utils.audio.audio_utils import update_conf +from modelscope.utils.constant import Tasks +from .cmvn import GlobalCMVN, load_kaldi_cmvn +from .fsmn import FSMN + + +@MODELS.register_module( + Tasks.keyword_spotting, + module_name=Models.speech_kws_fsmn_char_ctc_nearfield) +class FSMNDecorator(TorchModel): + r""" A decorator of FSMN for integrating into modelscope framework """ + + def __init__(self, + model_dir: str, + cmvn_file: str = None, + backbone: dict = None, + input_dim: int = 400, + output_dim: int = 2599, + training: Optional[bool] = False, + *args, + **kwargs): + """initialize the fsmn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + cmvn_file (str): cmvn file + backbone (dict): params related to backbone + input_dim (int): input dimension of network + output_dim (int): output dimension of network + training (bool): training or inference mode + """ + super().__init__(model_dir, *args, **kwargs) + + self.model = None + self.model_cfg = None + + if training: + self.model = self.init_model(cmvn_file, backbone, input_dim, + output_dim) + else: + self.model_cfg = { + 'model_workspace': model_dir, + 'config_path': os.path.join(model_dir, 'config.yaml') + } + + def __del__(self): + if hasattr(self, 'tmp_dir'): + self.tmp_dir.cleanup() + + def forward(self, input) -> Dict[str, Tensor]: + """ + Args: + input (torch.Tensor): Input tensor (B, T, D) + """ + if self.model is not None and input is not None: + return self.model.forward(input) + else: + return self.model_cfg + + def init_model(self, cmvn_file, backbone, input_dim, output_dim): + if cmvn_file is not None: + mean, istd = load_kaldi_cmvn(cmvn_file) + global_cmvn = GlobalCMVN( + torch.from_numpy(mean).float(), + torch.from_numpy(istd).float(), + ) + else: + global_cmvn = None + + hidden_dim = 128 + preprocessing = None + + input_affine_dim = backbone['input_affine_dim'] + num_layers = backbone['num_layers'] + linear_dim = backbone['linear_dim'] + proj_dim = backbone['proj_dim'] + left_order = backbone['left_order'] + right_order = backbone['right_order'] + left_stride = backbone['left_stride'] + right_stride = backbone['right_stride'] + output_affine_dim = backbone['output_affine_dim'] + backbone = FSMN(input_dim, input_affine_dim, num_layers, linear_dim, + proj_dim, left_order, right_order, left_stride, + right_stride, output_affine_dim, output_dim) + + classifier = None + activation = None + + kws_model = KWSModel(input_dim, output_dim, hidden_dim, global_cmvn, + preprocessing, backbone, classifier, activation) + return kws_model + + +class KWSModel(nn.Module): + """Our model consists of four parts: + 1. global_cmvn: Optional, (idim, idim) + 2. preprocessing: feature dimension projection, (idim, hdim) + 3. backbone: backbone or feature extractor of the whole network, (hdim, hdim) + 4. classifier: output layer or classifier of KWS model, (hdim, odim) + 5. activation: + nn.Sigmoid for wakeup word + nn.Identity for speech command dataset + """ + + def __init__( + self, + idim: int, + odim: int, + hdim: int, + global_cmvn: Optional[nn.Module], + preprocessing: Optional[nn.Module], + backbone: nn.Module, + classifier: nn.Module, + activation: nn.Module, + ): + """ + Args: + idim (int): input dimension of network + odim (int): output dimension of network + hdim (int): hidden dimension of network + global_cmvn (nn.Module): cmvn for input feature, (idim, idim) + preprocessing (nn.Module): feature dimension projection, (idim, hdim) + backbone (nn.Module): backbone or feature extractor of the whole network, (hdim, hdim) + classifier (nn.Module): output layer or classifier of KWS model, (hdim, odim) + activation (nn.Module): nn.Identity for training, nn.Sigmoid for inference + """ + super().__init__() + self.idim = idim + self.odim = odim + self.hdim = hdim + self.global_cmvn = global_cmvn + self.preprocessing = preprocessing + self.backbone = backbone + self.classifier = classifier + self.activation = activation + + def to_kaldi_net(self): + return self.backbone.to_kaldi_net() + + def to_pytorch_net(self, kaldi_file): + return self.backbone.to_pytorch_net(kaldi_file) + + def forward( + self, + x: torch.Tensor, + in_cache: torch.Tensor = torch.zeros(0, 0, 0, dtype=torch.float) + ) -> Tuple[torch.Tensor, torch.Tensor]: + if self.global_cmvn is not None: + x = self.global_cmvn(x) + if self.preprocessing is not None: + x = self.preprocessing(x) + + x, out_cache = self.backbone(x, in_cache) + + if self.classifier is not None: + x = self.classifier(x) + if self.activation is not None: + x = self.activation(x) + return x, out_cache + + def fuse_modules(self): + if self.preprocessing is not None: + self.preprocessing.fuse_modules() + self.backbone.fuse_modules() diff --git a/runtime/Lib/site_packages/models/audio/punc/__init__.py b/runtime/Lib/site_packages/models/audio/punc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py b/runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py new file mode 100644 index 0000000000000000000000000000000000000000..dabb60905001f1dc5f99acfaa8344d69cc000cbe --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/punc/generic_punctuation.py @@ -0,0 +1,43 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Frameworks, Tasks + + +@MODELS.register_module(Tasks.punctuation, module_name=Models.generic_punc) +class PunctuationProcessing(Model): + + def __init__(self, model_dir: str, punc_model_name: str, + punc_model_config: Dict[str, Any], *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + punc_model_name (str): the itn model name from configuration.json + punc_model_config (Dict[str, Any]): the detail config about model from configuration.json + """ + super().__init__(model_dir, punc_model_name, punc_model_config, *args, + **kwargs) + self.model_cfg = { + # the recognition model dir path + 'model_workspace': model_dir, + # the itn model name + 'punc_model': punc_model_name, + # the am model file path + 'punc_model_path': os.path.join(model_dir, punc_model_name), + # the recognition model config dict + 'model_config': punc_model_config + } + + def forward(self) -> Dict[str, Any]: + """ + just return the model config + + """ + + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/audio/separation/__init__.py b/runtime/Lib/site_packages/models/audio/separation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f957b9096b32699e3c6c2a661ef32a76c399fc1a --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mossformer import MossFormer + from .m2.mossformer import MossFormer2 + +else: + _import_structure = { + 'mossformer': ['MossFormer'], + 'm2.mossformer': ['MossFormer2'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/separation/layer_norm.py b/runtime/Lib/site_packages/models/audio/separation/layer_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..a4145cbcf95a02f97c6349b90660b0fbb9ffc510 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/layer_norm.py @@ -0,0 +1,68 @@ +# Copyright 2018 Northwestern Polytechnical University (author: Ke Wang) +# made publicly available under the MIT License +# at https://github.com/wangkenpu/Conv-TasNet-PyTorch/blob/64188ffa48971218fdd68b66906970f215d7eca2/model/layer_norm.py + +from __future__ import absolute_import, division, print_function + +import torch +import torch.nn as nn + + +class CLayerNorm(nn.LayerNorm): + """Channel-wise layer normalization.""" + + def __init__(self, *args, **kwargs): + super(CLayerNorm, self).__init__(*args, **kwargs) + + def forward(self, sample): + """Forward function. + + Args: + sample: [batch_size, channels, length] + """ + if sample.dim() != 3: + raise RuntimeError('{} only accept 3-D tensor as input'.format( + self.__name__)) + # [N, C, T] -> [N, T, C] + sample = torch.transpose(sample, 1, 2) + # LayerNorm + sample = super().forward(sample) + # [N, T, C] -> [N, C, T] + sample = torch.transpose(sample, 1, 2) + return sample + + +class GLayerNorm(nn.Module): + """Global Layer Normalization for TasNet.""" + + def __init__(self, channels, eps=1e-5): + super(GLayerNorm, self).__init__() + self.eps = eps + self.norm_dim = channels + self.gamma = nn.Parameter(torch.Tensor(channels)) + self.beta = nn.Parameter(torch.Tensor(channels)) + self.reset_parameters() + + def reset_parameters(self): + nn.init.ones_(self.gamma) + nn.init.zeros_(self.beta) + + def forward(self, sample): + """Forward function. + + Args: + sample: [batch_size, channels, length] + """ + if sample.dim() != 3: + raise RuntimeError('{} only accept 3-D tensor as input'.format( + self.__name__)) + # [N, C, T] -> [N, T, C] + sample = torch.transpose(sample, 1, 2) + # Mean and variance [N, 1, 1] + mean = torch.mean(sample, (1, 2), keepdim=True) + var = torch.mean((sample - mean)**2, (1, 2), keepdim=True) + sample = (sample + - mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta + # [N, T, C] -> [N, C, T] + sample = torch.transpose(sample, 1, 2) + return sample diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/__init__.py b/runtime/Lib/site_packages/models/audio/separation/m2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py b/runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..f6238e1001faaf789238377fe8bb684277abdabe --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/m2/conv_module.py @@ -0,0 +1,278 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.init as init +from torch import Tensor + +EPS = 1e-8 + + +class GlobalLayerNorm(nn.Module): + """Calculate Global Layer Normalization. + + Args: + dim : (int or list or torch.Size) + Input shape from an expected input of size. + eps : float + A value added to the denominator for numerical stability. + elementwise_affine : bool + A boolean value that when set to True, + this module has learnable per-element affine parameters + initialized to ones (for weights) and zeros (for biases). + + Example: + ------- + >>> x = torch.randn(5, 10, 20) + >>> GLN = GlobalLayerNorm(10, 3) + >>> x_norm = GLN(x) + """ + + def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True): + super(GlobalLayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + if shape == 3: + self.weight = nn.Parameter(torch.ones(self.dim, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1)) + if shape == 4: + self.weight = nn.Parameter(torch.ones(self.dim, 1, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + + def forward(self, x): + """Returns the normalized tensor. + + Args: + x : torch.Tensor + Tensor of size [N, C, K, S] or [N, C, L]. + """ + # x = N x C x K x S or N x C x L + # N x 1 x 1 + # cln: mean,var N x 1 x K x S + # gln: mean,var N x 1 x 1 + if x.dim() == 3: + mean = torch.mean(x, (1, 2), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2), keepdim=True) + if self.elementwise_affine: + # yapf: disable + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) + # yapf: enable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + + if x.dim() == 4: + mean = torch.mean(x, (1, 2, 3), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2, 3), keepdim=True) + if self.elementwise_affine: + # yapf: disable + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) + # yapf: enable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + return x + + +class CumulativeLayerNorm(nn.LayerNorm): + """Calculate Cumulative Layer Normalization. + + Args: + dim : int + Dimension that you want to normalize. + elementwise_affine : True + Learnable per-element affine parameters. + + Example: + ------- + >>> x = torch.randn(5, 10, 20) + >>> CLN = CumulativeLayerNorm(10) + >>> x_norm = CLN(x) + """ + + def __init__(self, dim, elementwise_affine=True): + super(CumulativeLayerNorm, self).__init__( + dim, elementwise_affine=elementwise_affine, eps=1e-8) + + def forward(self, x): + """Returns the normalized tensor. + + Args: + x : torch.Tensor + Tensor size [N, C, K, S] or [N, C, L] + """ + # x: N x C x K x S or N x C x L + # N x K x S x C + if x.dim() == 4: + x = x.permute(0, 2, 3, 1).contiguous() + # N x K x S x C == only channel norm + x = super().forward(x) + # N x C x K x S + x = x.permute(0, 3, 1, 2).contiguous() + if x.dim() == 3: + x = torch.transpose(x, 1, 2) + # N x L x C == only channel norm + x = super().forward(x) + # N x C x L + x = torch.transpose(x, 1, 2) + return x + + +class Transpose(nn.Module): + """ Wrapper class of torch.transpose() for Sequential module. """ + + def __init__(self, shape: tuple): + super(Transpose, self).__init__() + self.shape = shape + + def forward(self, x: Tensor) -> Tensor: + return x.transpose(*self.shape) + + +class DepthwiseConv1d(nn.Module): + """When groups == in_channels and out_channels == K * in_channels, where K is a positive integer, + this operation is termed in literature as depthwise convolution. + + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + bias (bool, optional): If True, adds a learnable bias to the output. Default: True + Inputs: inputs + - **inputs** (batch, in_channels, time): Tensor containing input vector + Returns: outputs + - **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + bias: bool = False, + ) -> None: + super(DepthwiseConv1d, self).__init__() + assert out_channels % in_channels == 0, 'out_channels should be constant multiple of in_channels' + self.conv = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + groups=in_channels, + stride=stride, + padding=padding, + bias=bias, + ) + + def forward(self, inputs: Tensor) -> Tensor: + return self.conv(inputs) + + +class ConvModule(nn.Module): + """ + Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU). + This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution + to aid training deep models. + + Args: + in_channels (int): Number of channels in the input + kernel_size (int or tuple, optional): Size of the convolving kernel Default: 17 + dropout_p (float, optional): probability of dropout + Inputs: inputs + inputs (batch, time, dim): Tensor contains input sequences + Outputs: outputs + outputs (batch, time, dim): Tensor produces by conformer convolution module. + """ + + def __init__( + self, + in_channels: int, + kernel_size: int = 17, + expansion_factor: int = 2, + dropout_p: float = 0.1, + ) -> None: + super(ConvModule, self).__init__() + assert ( + kernel_size - 1 + ) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding" + assert expansion_factor == 2, 'Currently, Only Supports expansion_factor 2' + + self.sequential = nn.Sequential( + Transpose(shape=(1, 2)), + DepthwiseConv1d( + in_channels, + in_channels, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2), + ) + + def forward(self, inputs: Tensor) -> Tensor: + return inputs + self.sequential(inputs).transpose(1, 2) + + +class DilatedDenseNet(nn.Module): + + def __init__(self, depth=4, lorder=20, in_channels=64): + super(DilatedDenseNet, self).__init__() + self.depth = depth + self.in_channels = in_channels + self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.) + self.twidth = lorder * 2 - 1 + self.kernel_size = (self.twidth, 1) + for i in range(self.depth): + dil = 2**i + pad_length = lorder + (dil - 1) * (lorder - 1) - 1 + setattr(self, 'pad{}'.format(i + 1), + nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.)) + setattr( + self, 'conv{}'.format(i + 1), + nn.Conv2d( + self.in_channels * (i + 1), + self.in_channels, + kernel_size=self.kernel_size, + dilation=(dil, 1), + groups=self.in_channels, + bias=False)) + setattr(self, 'norm{}'.format(i + 1), + nn.InstanceNorm2d(in_channels, affine=True)) + setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels)) + + def forward(self, x): + x = torch.unsqueeze(x, 1) + x_per = x.permute(0, 3, 2, 1) + skip = x_per + for i in range(self.depth): + out = getattr(self, 'pad{}'.format(i + 1))(skip) + out = getattr(self, 'conv{}'.format(i + 1))(out) + out = getattr(self, 'norm{}'.format(i + 1))(out) + out = getattr(self, 'prelu{}'.format(i + 1))(out) + skip = torch.cat([out, skip], dim=1) + out1 = out.permute(0, 3, 2, 1) + return out1.squeeze(1) + + +class FFConvMDilated(nn.Module): + + def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1): + super().__init__() + self.mdl = nn.Sequential( + norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(), + DilatedDenseNet(depth=2, lorder=17, in_channels=dim_out), + nn.Dropout(dropout)) + + def forward( + self, + x, + ): + output = self.mdl(x) + return output diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py b/runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py new file mode 100644 index 0000000000000000000000000000000000000000..97400888fc6bb04d5d43c73d95d0ce801b238925 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/m2/fsmn.py @@ -0,0 +1,144 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch as th +import torch.nn as nn +import torch.nn.functional as F + + +class UniDeepFsmn(nn.Module): + + def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None): + super(UniDeepFsmn, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + if lorder is None: + return + self.lorder = lorder + self.hidden_size = hidden_size + self.linear = nn.Linear(input_dim, hidden_size) + self.project = nn.Linear(hidden_size, output_dim, bias=False) + self.conv1 = nn.Conv2d( + output_dim, + output_dim, [lorder + lorder - 1, 1], [1, 1], + groups=output_dim, + bias=False) + + def forward(self, input): + f1 = F.relu(self.linear(input)) + p1 = self.project(f1) + x = th.unsqueeze(p1, 1) + x_per = x.permute(0, 3, 2, 1) + y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1]) + out = x_per + self.conv1(y) + out1 = out.permute(0, 3, 2, 1) + return input + out1.squeeze() + + +class UniDeepFsmnDual(nn.Module): + + def __init__(self, input_dim, output_dim, lorder=None, hidden_size=None): + super(UniDeepFsmnDual, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + if lorder is None: + return + self.lorder = lorder + self.hidden_size = hidden_size + self.linear = nn.Linear(input_dim, hidden_size) + self.project = nn.Linear(hidden_size, output_dim, bias=False) + self.conv1 = nn.Conv2d( + output_dim, + output_dim, [lorder + lorder - 1, 1], [1, 1], + groups=output_dim, + bias=False) + self.conv2 = nn.Conv2d( + output_dim, + output_dim, [lorder + lorder - 1, 1], [1, 1], + groups=output_dim // 4, + bias=False) + + def forward(self, input): + + f1 = F.relu(self.linear(input)) + p1 = self.project(f1) + x = th.unsqueeze(p1, 1) + x_per = x.permute(0, 3, 2, 1) + y = F.pad(x_per, [0, 0, self.lorder - 1, self.lorder - 1]) + conv1_out = x_per + self.conv1(y) + z = F.pad(conv1_out, [0, 0, self.lorder - 1, self.lorder - 1]) + out = conv1_out + self.conv2(z) + out1 = out.permute(0, 3, 2, 1) + return input + out1.squeeze() + + +class DilatedDenseNet(nn.Module): + + def __init__(self, depth=4, lorder=20, in_channels=64): + super(DilatedDenseNet, self).__init__() + self.depth = depth + self.in_channels = in_channels + self.pad = nn.ConstantPad2d((1, 1, 1, 0), value=0.) + self.twidth = lorder * 2 - 1 + self.kernel_size = (self.twidth, 1) + for i in range(self.depth): + dil = 2**i + pad_length = lorder + (dil - 1) * (lorder - 1) - 1 + setattr(self, 'pad{}'.format(i + 1), + nn.ConstantPad2d((0, 0, pad_length, pad_length), value=0.)) + setattr( + self, 'conv{}'.format(i + 1), + nn.Conv2d( + self.in_channels * (i + 1), + self.in_channels, + kernel_size=self.kernel_size, + dilation=(dil, 1), + groups=self.in_channels, + bias=False)) + setattr(self, 'norm{}'.format(i + 1), + nn.InstanceNorm2d(in_channels, affine=True)) + setattr(self, 'prelu{}'.format(i + 1), nn.PReLU(self.in_channels)) + + def forward(self, x): + skip = x + for i in range(self.depth): + out = getattr(self, 'pad{}'.format(i + 1))(skip) + out = getattr(self, 'conv{}'.format(i + 1))(out) + out = getattr(self, 'norm{}'.format(i + 1))(out) + out = getattr(self, 'prelu{}'.format(i + 1))(out) + skip = th.cat([out, skip], dim=1) + return out + + +class UniDeepFsmnDilated(nn.Module): + + def __init__(self, + input_dim, + output_dim, + lorder=None, + hidden_size=None, + depth=2): + super(UniDeepFsmnDilated, self).__init__() + + self.input_dim = input_dim + self.output_dim = output_dim + self.depth = depth + if lorder is None: + return + self.lorder = lorder + self.hidden_size = hidden_size + self.linear = nn.Linear(input_dim, hidden_size) + self.project = nn.Linear(hidden_size, output_dim, bias=False) + self.conv = DilatedDenseNet( + depth=self.depth, lorder=lorder, in_channels=output_dim) + + def forward(self, input): + f1 = F.relu(self.linear(input)) + p1 = self.project(f1) + x = th.unsqueeze(p1, 1) + x_per = x.permute(0, 3, 2, 1) + out = self.conv(x_per) + out1 = out.permute(0, 3, 2, 1) + + return input + out1.squeeze() diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py b/runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..811702f7b16a6c326f179bfa8fac5a24104f1d9d --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/m2/layer_norm.py @@ -0,0 +1,125 @@ +# Copyright 2018 Northwestern Polytechnical University (author: Ke Wang) + +from __future__ import absolute_import, division, print_function + +import torch +import torch.nn as nn + + +class CLayerNorm(nn.LayerNorm): + """Channel-wise layer normalization.""" + + def __init__(self, *args, **kwargs): + super(CLayerNorm, self).__init__(*args, **kwargs) + + def forward(self, sample): + """Forward function. + + Args: + sample: [batch_size, channels, length] + """ + if sample.dim() != 3: + raise RuntimeError('{} only accept 3-D tensor as input'.format( + self.__name__)) + # [N, C, T] -> [N, T, C] + sample = torch.transpose(sample, 1, 2) + # LayerNorm + sample = super().forward(sample) + # [N, T, C] -> [N, C, T] + sample = torch.transpose(sample, 1, 2) + return sample + + +class ILayerNorm(nn.InstanceNorm1d): + """Channel-wise layer normalization.""" + + def __init__(self, *args, **kwargs): + super(ILayerNorm, self).__init__(*args, **kwargs) + + def forward(self, sample): + """Forward function. + + Args: + sample: [batch_size, channels, length] + """ + if sample.dim() != 3: + raise RuntimeError('{} only accept 3-D tensor as input'.format( + self.__name__)) + # [N, C, T] -> [N, T, C] + sample = torch.transpose(sample, 1, 2) + # LayerNorm + sample = super().forward(sample) + # [N, T, C] -> [N, C, T] + sample = torch.transpose(sample, 1, 2) + return sample + + +class GLayerNorm(nn.Module): + """Global Layer Normalization for TasNet.""" + + def __init__(self, channels, eps=1e-5): + super(GLayerNorm, self).__init__() + self.eps = eps + self.norm_dim = channels + self.gamma = nn.Parameter(torch.Tensor(channels)) + self.beta = nn.Parameter(torch.Tensor(channels)) + # self.register_parameter('weight', self.gamma) + # self.register_parameter('bias', self.beta) + self.reset_parameters() + + def reset_parameters(self): + nn.init.ones_(self.gamma) + nn.init.zeros_(self.beta) + + def forward(self, sample): + """Forward function. + + Args: + sample: [batch_size, channels, length] + """ + if sample.dim() != 3: + raise RuntimeError('{} only accept 3-D tensor as input'.format( + self.__name__)) + # [N, C, T] -> [N, T, C] + sample = torch.transpose(sample, 1, 2) + # Mean and variance [N, 1, 1] + mean = torch.mean(sample, (1, 2), keepdim=True) + var = torch.mean((sample - mean)**2, (1, 2), keepdim=True) + sample = (sample + - mean) / torch.sqrt(var + self.eps) * self.gamma + self.beta + # [N, T, C] -> [N, C, T] + sample = torch.transpose(sample, 1, 2) + return sample + + +class _LayerNorm(nn.Module): + """Layer Normalization base class.""" + + def __init__(self, channel_size): + super(_LayerNorm, self).__init__() + self.channel_size = channel_size + self.gamma = nn.Parameter(torch.ones(channel_size), requires_grad=True) + self.beta = nn.Parameter(torch.zeros(channel_size), requires_grad=True) + + def apply_gain_and_bias(self, normed_x): + """ Assumes input of size `[batch, chanel, *]`. """ + return (self.gamma * normed_x.transpose(1, -1) + self.beta).transpose( + 1, -1) + + +class GlobLayerNorm(_LayerNorm): + """Global Layer Normalization (globLN).""" + + def forward(self, x): + """ Applies forward pass. + Works for any input size > 2D. + + Args: + x (:class:`torch.Tensor`): Shape `[batch, chan, *]` + Returns: + :class:`torch.Tensor`: gLN_x `[batch, chan, *]` + """ + dims = list(range(1, len(x.shape))) + mean = x.mean(dim=dims, keepdim=True) + var = torch.pow(x - mean, 2).mean(dim=dims, keepdim=True) + return self.apply_gain_and_bias((x - mean) / (var + 1e-8).sqrt()) diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/mossformer.py b/runtime/Lib/site_packages/models/audio/separation/m2/mossformer.py new file mode 100644 index 0000000000000000000000000000000000000000..8249d451b7b52438db389fd147c1f2caf0903b29 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/m2/mossformer.py @@ -0,0 +1,599 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Some code here is modified based on speechbrain and can be found on github +# https://github.com/speechbrain/speechbrain/blob/develop/speechbrain/lobes/models/dual_path.py +"""Library to support dual-path speech separation. + +Authors + * Cem Subakan 2020 + * Mirco Ravanelli 2020 + * Samuele Cornell 2020 + * Mirko Bronzi 2020 + * Jianyuan Zhong 2020 +""" + +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import ModelFile, Tasks +from .mossformer_block import MossformerBlockGFSMN, ScaledSinuEmbedding + +EPS = 1e-8 + + +class GlobalLayerNorm(nn.Module): + """Calculate Global Layer Normalization. + + Args: + dim : (int or list or torch.Size) + Input shape from an expected input of size. + eps : float + A value added to the denominator for numerical stability. + elementwise_affine : bool + A boolean value that when set to True, + this module has learnable per-element affine parameters + initialized to ones (for weights) and zeros (for biases). + + Example: + >>> x = torch.randn(5, 10, 20) + >>> GLN = GlobalLayerNorm(10, 3) + >>> x_norm = GLN(x) + """ + + def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True): + super(GlobalLayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + if shape == 3: + self.weight = nn.Parameter(torch.ones(self.dim, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1)) + if shape == 4: + self.weight = nn.Parameter(torch.ones(self.dim, 1, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + + def forward(self, x): + """Returns the normalized tensor. + + Args: + x : torch.Tensor + Tensor of size [N, C, K, S] or [N, C, L]. + """ + # x = N x C x K x S or N x C x L + # N x 1 x 1 + # cln: mean,var N x 1 x K x S + # gln: mean,var N x 1 x 1 + if x.dim() == 3: + mean = torch.mean(x, (1, 2), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2), keepdim=True) + if self.elementwise_affine: + # yapf: disable + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) + # yapf: enable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + + if x.dim() == 4: + mean = torch.mean(x, (1, 2, 3), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2, 3), keepdim=True) + if self.elementwise_affine: + # yapf: disable + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) + # yapf: enable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + return x + + +class CumulativeLayerNorm(nn.LayerNorm): + """Calculate Cumulative Layer Normalization. + + Args: + dim : int + Dimension that you want to normalize. + elementwise_affine : True + Learnable per-element affine parameters. + + Example + ------- + >>> x = torch.randn(5, 10, 20) + >>> CLN = CumulativeLayerNorm(10) + >>> x_norm = CLN(x) + """ + + def __init__(self, dim, elementwise_affine=True): + super(CumulativeLayerNorm, self).__init__( + dim, elementwise_affine=elementwise_affine, eps=1e-8) + + def forward(self, x): + """Returns the normalized tensor. + + Arguments + --------- + x : torch.Tensor + Tensor size [N, C, K, S] or [N, C, L] + """ + # x: N x C x K x S or N x C x L + # N x K x S x C + if x.dim() == 4: + x = x.permute(0, 2, 3, 1).contiguous() + # N x K x S x C == only channel norm + x = super().forward(x) + # N x C x K x S + x = x.permute(0, 3, 1, 2).contiguous() + if x.dim() == 3: + x = torch.transpose(x, 1, 2) + # N x L x C == only channel norm + x = super().forward(x) + # N x C x L + x = torch.transpose(x, 1, 2) + return x + + +def select_norm(norm, dim, shape): + """Just a wrapper to select the normalization type. + """ + + if norm == 'gln': + return GlobalLayerNorm(dim, shape, elementwise_affine=True) + if norm == 'cln': + return CumulativeLayerNorm(dim, elementwise_affine=True) + if norm == 'ln': + return nn.GroupNorm(1, dim, eps=1e-8) + else: + return nn.BatchNorm1d(dim) + + +class Encoder(nn.Module): + """Convolutional Encoder Layer. + + Args: + kernel_size : int + Length of filters. + in_channels : int + Number of input channels. + out_channels : int + Number of output channels. + + Example: + >>> x = torch.randn(2, 1000) + >>> encoder = Encoder(kernel_size=4, out_channels=64) + >>> h = encoder(x) + >>> h.shape + torch.Size([2, 64, 499]) + """ + + def __init__(self, kernel_size=2, out_channels=64, in_channels=1): + super(Encoder, self).__init__() + self.conv1d = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=kernel_size // 2, + groups=1, + bias=False, + ) + self.in_channels = in_channels + + def forward(self, x): + """Return the encoded output. + + Args: + x : torch.Tensor + Input tensor with dimensionality [B, L]. + + Returns: + x : torch.Tensor + Encoded tensor with dimensionality [B, N, T_out]. + where B = Batchsize + L = Number of timepoints + N = Number of filters + T_out = Number of timepoints at the output of the encoder + """ + # B x L -> B x 1 x L + if self.in_channels == 1: + x = torch.unsqueeze(x, dim=1) + # B x 1 x L -> B x N x T_out + x = self.conv1d(x) + x = F.relu(x) + + return x + + +class Decoder(nn.ConvTranspose1d): + """A decoder layer that consists of ConvTranspose1d. + + Args: + kernel_size : int + Length of filters. + in_channels : int + Number of input channels. + out_channels : int + Number of output channels. + + + Example: + --------- + >>> x = torch.randn(2, 100, 1000) + >>> decoder = Decoder(kernel_size=4, in_channels=100, out_channels=1) + >>> h = decoder(x) + >>> h.shape + torch.Size([2, 1003]) + """ + + def __init__(self, *args, **kwargs): + super(Decoder, self).__init__(*args, **kwargs) + + def forward(self, x): + """Return the decoded output. + + Args: + x : torch.Tensor + Input tensor with dimensionality [B, N, L]. + where, B = Batchsize, + N = number of filters + L = time points + """ + + if x.dim() not in [2, 3]: + raise RuntimeError('{} accept 3/4D tensor as input'.format( + self.__name__)) + x = super().forward(x if x.dim() == 3 else torch.unsqueeze(x, 1)) + + if torch.squeeze(x).dim() == 1: + x = torch.squeeze(x, dim=1) + else: + x = torch.squeeze(x) + return x + + +class MossFormerM(nn.Module): + """This class implements the transformer encoder. + + Args: + num_blocks : int + Number of mossformer blocks to include. + d_model : int + The dimension of the input embedding. + attn_dropout : float + Dropout for the self-attention (Optional). + group_size: int + the chunk size + query_key_dim: int + the attention vector dimension + expansion_factor: int + the expansion factor for the linear projection in conv module + causal: bool + true for causal / false for non causal + + Example: + ------- + >>> import torch + >>> x = torch.rand((8, 60, 512)) + >>> net = TransformerEncoder_MossFormerM(num_blocks=8, d_model=512) + >>> output, _ = net(x) + >>> output.shape + torch.Size([8, 60, 512]) + """ + + def __init__(self, + num_blocks, + d_model=None, + causal=False, + group_size=256, + query_key_dim=128, + expansion_factor=4., + attn_dropout=0.1): + super().__init__() + + self.mossformerM = MossformerBlockGFSMN( + dim=d_model, + depth=num_blocks, + group_size=group_size, + query_key_dim=query_key_dim, + expansion_factor=expansion_factor, + causal=causal, + attn_dropout=attn_dropout) + self.norm = nn.LayerNorm(d_model, eps=1e-6) + + def forward(self, src): + """ + Args: + src : torch.Tensor + Tensor shape [B, L, N], + where, B = Batchsize, + L = time points + N = number of filters + The sequence to the encoder layer (required). + """ + output = self.mossformerM(src) + output = self.norm(output) + + return output + + +class ComputationBlock(nn.Module): + """Computation block for dual-path processing. + + Args: + num_blocks : int + Number of mossformer blocks to include. + out_channels : int + Dimensionality of inter/intra model. + norm : str + Normalization type. + skip_around_intra : bool + Skip connection around the intra layer. + + Example: + --------- + >>> comp_block = ComputationBlock(64) + >>> x = torch.randn(10, 64, 100) + >>> x = comp_block(x) + >>> x.shape + torch.Size([10, 64, 100]) + """ + + def __init__( + self, + num_blocks, + out_channels, + norm='ln', + skip_around_intra=True, + ): + super(ComputationBlock, self).__init__() + + # MossFormer+: MossFormer with recurrence + self.intra_mdl = MossFormerM( + num_blocks=num_blocks, d_model=out_channels) + self.skip_around_intra = skip_around_intra + + # Norm + self.norm = norm + if norm is not None: + self.intra_norm = select_norm(norm, out_channels, 3) + + def forward(self, x): + """Returns the output tensor. + + Args: + x : torch.Tensor + Input tensor of dimension [B, N, S]. + + Returns: + out: torch.Tensor + Output tensor of dimension [B, N, S]. + where, B = Batchsize, + N = number of filters + S = sequence time index + """ + B, N, S = x.shape + # intra RNN + # [B, S, N] + intra = x.permute(0, 2, 1).contiguous() + + intra = self.intra_mdl(intra) + + # [B, N, S] + intra = intra.permute(0, 2, 1).contiguous() + if self.norm is not None: + intra = self.intra_norm(intra) + + # [B, N, S] + if self.skip_around_intra: + intra = intra + x + + out = intra + return out + + +class MossFormerMaskNet(nn.Module): + """The dual path model which is the basis for dualpathrnn, sepformer, dptnet. + + Args: + in_channels : int + Number of channels at the output of the encoder. + out_channels : int + Number of channels that would be inputted to the intra and inter blocks. + norm : str + Normalization type. + num_spks : int + Number of sources (speakers). + skip_around_intra : bool + Skip connection around intra. + use_global_pos_enc : bool + Global positional encodings. + max_length : int + Maximum sequence length. + + Example: + --------- + >>> mossformer_block = MossFormerM(1, 64, 8) + >>> mossformer_masknet = MossFormerMaskNet(64, 64, intra_block, num_spks=2) + >>> x = torch.randn(10, 64, 2000) + >>> x = mossformer_masknet(x) + >>> x.shape + torch.Size([2, 10, 64, 2000]) + """ + + def __init__( + self, + in_channels, + out_channels, + num_blocks=24, + norm='ln', + num_spks=2, + skip_around_intra=True, + use_global_pos_enc=True, + max_length=20000, + ): + super(MossFormerMaskNet, self).__init__() + self.num_spks = num_spks + self.num_blocks = num_blocks + self.norm = select_norm(norm, in_channels, 3) + self.conv1d_encoder = nn.Conv1d( + in_channels, out_channels, 1, bias=False) + self.use_global_pos_enc = use_global_pos_enc + + if self.use_global_pos_enc: + self.pos_enc = ScaledSinuEmbedding(out_channels) + + self.mdl = ComputationBlock( + num_blocks, + out_channels, + norm, + skip_around_intra=skip_around_intra, + ) + + self.conv1d_out = nn.Conv1d( + out_channels, out_channels * num_spks, kernel_size=1) + self.conv1_decoder = nn.Conv1d( + out_channels, in_channels, 1, bias=False) + self.prelu = nn.PReLU() + self.activation = nn.ReLU() + # gated output layer + self.output = nn.Sequential( + nn.Conv1d(out_channels, out_channels, 1), nn.Tanh()) + self.output_gate = nn.Sequential( + nn.Conv1d(out_channels, out_channels, 1), nn.Sigmoid()) + + def forward(self, x): + """Returns the output tensor. + + Args: + x : torch.Tensor + Input tensor of dimension [B, N, S]. + + Returns: + out : torch.Tensor + Output tensor of dimension [spks, B, N, S] + where, spks = Number of speakers + B = Batchsize, + N = number of filters + S = the number of time frames + """ + # before each line we indicate the shape after executing the line + # [B, N, L] + x = self.norm(x) + + # [B, N, L] + x = self.conv1d_encoder(x) + if self.use_global_pos_enc: + base = x + x = x.transpose(1, -1) + emb = self.pos_enc(x) + emb = emb.transpose(0, -1) + x = base + emb + + # [B, N, S] + x = self.mdl(x) + x = self.prelu(x) + + # [B, N*spks, S] + x = self.conv1d_out(x) + B, _, S = x.shape + + # [B*spks, N, S] + x = x.view(B * self.num_spks, -1, S) + + # [B*spks, N, S] + x = self.output(x) * self.output_gate(x) + + # [B*spks, N, S] + x = self.conv1_decoder(x) + + # [B, spks, N, S] + _, N, L = x.shape + x = x.view(B, self.num_spks, N, L) + x = self.activation(x) + + # [spks, B, N, S] + x = x.transpose(0, 1) + + return x + + +@MODELS.register_module( + Tasks.speech_separation, + module_name=Models.speech_mossformer2_separation_temporal_8k) +class MossFormer2(TorchModel): + """Library to support MossFormer speech separation. + + Args: + model_dir (str): the model path. + """ + + def __init__(self, + model_dir: str, + in_channels=512, + out_channels=512, + num_blocks=24, + kernel_size=16, + norm='ln', + num_spks=2, + skip_around_intra=True, + use_global_pos_enc=True, + max_length=20000, + *args, + **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.num_spks = num_spks + self.enc = Encoder( + kernel_size=kernel_size, out_channels=in_channels, in_channels=1) + self.mask_net = MossFormerMaskNet( + in_channels=in_channels, + out_channels=out_channels, + num_blocks=num_blocks, + norm=norm, + num_spks=num_spks, + skip_around_intra=skip_around_intra, + use_global_pos_enc=use_global_pos_enc, + max_length=max_length, + ) + self.dec = Decoder( + in_channels=out_channels, + out_channels=1, + kernel_size=kernel_size, + stride=kernel_size // 2, + bias=False) + + def forward(self, input): + x = self.enc(input) + mask = self.mask_net(x) + x = torch.stack([x] * self.num_spks) + sep_x = x * mask + + # Decoding + est_source = torch.cat( + [self.dec(sep_x[i]).unsqueeze(-1) for i in range(self.num_spks)], + dim=-1, + ) + T_origin = input.size(1) + T_est = est_source.size(1) + if T_origin > T_est: + est_source = F.pad(est_source, (0, 0, 0, T_origin - T_est)) + else: + est_source = est_source[:, :T_origin, :] + return est_source + + def load_check_point(self, load_path=None, device=None): + if not load_path: + load_path = self.model_dir + if not device: + device = torch.device('cpu') + self.load_state_dict( + torch.load( + os.path.join(load_path, ModelFile.TORCH_MODEL_FILE), + map_location=device), + strict=False) diff --git a/runtime/Lib/site_packages/models/audio/separation/m2/mossformer_block.py b/runtime/Lib/site_packages/models/audio/separation/m2/mossformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..6aee53eedf6acfe1edafd929be6737d9b7e481da --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/m2/mossformer_block.py @@ -0,0 +1,548 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn.functional as F +from einops import rearrange +from rotary_embedding_torch import RotaryEmbedding +from torch import einsum, nn + +from .conv_module import ConvModule, FFConvMDilated +from .fsmn import UniDeepFsmn, UniDeepFsmnDilated +from .layer_norm import CLayerNorm + +# functions + + +def identity(t, *args, **kwargs): + return t + + +def append_dims(x, num_dims): + if num_dims <= 0: + return x + return x.view(*x.shape, *((1, ) * num_dims)) + + +def exists(val): + return val is not None + + +def default(val, d): + return val if exists(val) else d + + +def padding_to_multiple_of(n, mult): + remainder = n % mult + if remainder == 0: + return 0 + return mult - remainder + + +# scalenorm + + +class ScaleNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.scale = dim**-0.5 + self.eps = eps + self.g = nn.Parameter(torch.ones(1)) + + def forward(self, x): + norm = torch.norm(x, dim=-1, keepdim=True) * self.scale + return x / norm.clamp(min=self.eps) * self.g + + +# absolute positional encodings + + +class ScaledSinuEmbedding(nn.Module): + + def __init__(self, dim): + super().__init__() + self.scale = nn.Parameter(torch.ones(1, )) + inv_freq = 1. / (10000**(torch.arange(0, dim, 2).float() / dim)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, x): + n, device = x.shape[1], x.device + t = torch.arange(n, device=device).type_as(self.inv_freq) + sinu = einsum('i , j -> i j', t, self.inv_freq) + emb = torch.cat((sinu.sin(), sinu.cos()), dim=-1) + return emb * self.scale + + +class OffsetScale(nn.Module): + + def __init__(self, dim, heads=1): + super().__init__() + self.gamma = nn.Parameter(torch.ones(heads, dim)) + self.beta = nn.Parameter(torch.zeros(heads, dim)) + nn.init.normal_(self.gamma, std=0.02) + + def forward(self, x): + out = einsum('... d, h d -> ... h d', x, self.gamma) + self.beta + return out.unbind(dim=-2) + + +class FFConvM(nn.Module): + + def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1): + super().__init__() + self.mdl = nn.Sequential( + norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(), + ConvModule(dim_out), nn.Dropout(dropout)) + + def forward( + self, + x, + ): + output = self.mdl(x) + return output + + +class GroupLinear(nn.Module): + + def __init__(self, dim_in, dim_out, K=4): + super().__init__() + hidden = dim_in // 2 + self.group_conv = nn.Conv1d( + dim_in, hidden, groups=dim_in // K, kernel_size=1) + self.norm = nn.LayerNorm(hidden) + self.linear = nn.Linear(hidden, dim_out) + + def forward( + self, + x, + ): + x1 = x.transpose(2, 1) + conv_out = self.group_conv(x1) + x2 = self.norm(conv_out.transpose(2, 1)) + x3 = self.linear(x2) + return x3 + + +class FFM(nn.Module): + + def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1): + super().__init__() + self.mdl = nn.Sequential( + norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(), + nn.Dropout(dropout)) + + def forward( + self, + x, + ): + output = self.mdl(x) + return output + + +# FLASH +class FLASH_ShareA_FFConvM(nn.Module): + + def __init__(self, + *, + dim, + group_size=256, + query_key_dim=128, + expansion_factor=1., + causal=False, + dropout=0.1, + rotary_pos_emb=None, + norm_klass=nn.LayerNorm, + shift_tokens=True): + super().__init__() + hidden_dim = int(dim * expansion_factor) + self.group_size = group_size + self.causal = causal + self.shift_tokens = shift_tokens + + # positional embeddings + self.rotary_pos_emb = rotary_pos_emb + # norm + self.dropout = nn.Dropout(dropout) + # projections + self.to_hidden = FFConvM( + dim_in=dim, + dim_out=hidden_dim, + norm_klass=norm_klass, + dropout=dropout, + ) + self.to_qk = FFConvM( + dim_in=dim, + dim_out=query_key_dim, + norm_klass=norm_klass, + dropout=dropout, + ) + + self.qk_offset_scale = OffsetScale(query_key_dim, heads=4) + + self.to_out = FFConvM( + dim_in=dim * 2, + dim_out=dim, + norm_klass=norm_klass, + dropout=dropout, + ) + + self.gateActivate = nn.Sigmoid() + + def forward(self, x, *, mask=None): + """ + b - batch + n - sequence length (within groups) + g - group dimension + d - feature dimension (keys) + e - feature dimension (values) + i - sequence dimension (source) + j - sequence dimension (target) + """ + # prenorm + normed_x = x + + if self.shift_tokens: + x_shift, x_pass = normed_x.chunk(2, dim=-1) + x_shift = F.pad(x_shift, (0, 0, 1, -1), value=0.) + normed_x = torch.cat((x_shift, x_pass), dim=-1) + + # initial projections + v, u = self.to_hidden(normed_x).chunk(2, dim=-1) + qk = self.to_qk(normed_x) + + # offset and scale + quad_q, lin_q, quad_k, lin_k = self.qk_offset_scale(qk) + att_v, att_u = self.cal_attention(x, quad_q, lin_q, quad_k, lin_k, v, + u) + out = (att_u * v) * self.gateActivate(att_v * u) + + x = x + self.to_out(out) + return x + + def cal_attention(self, x, quad_q, lin_q, quad_k, lin_k, v, u, mask=None): + b, n, device, g = x.shape[0], x.shape[-2], x.device, self.group_size + + if exists(mask): + lin_mask = rearrange(mask, '... -> ... 1') + lin_k = lin_k.masked_fill(~lin_mask, 0.) + + # rotate queries and keys + if exists(self.rotary_pos_emb): + quad_q, lin_q, quad_k, lin_k = map( + self.rotary_pos_emb.rotate_queries_or_keys, + (quad_q, lin_q, quad_k, lin_k)) + + # padding for groups + padding = padding_to_multiple_of(n, g) + + if padding > 0: + quad_q, quad_k, lin_q, lin_k, v, u = map( + lambda t: F.pad(t, (0, 0, 0, padding), value=0.), + (quad_q, quad_k, lin_q, lin_k, v, u)) + + mask = default(mask, + torch.ones((b, n), device=device, dtype=torch.bool)) + mask = F.pad(mask, (0, padding), value=False) + + # group along sequence + quad_q, quad_k, lin_q, lin_k, v, u = map( + lambda t: rearrange(t, 'b (g n) d -> b g n d', n=self.group_size), + (quad_q, quad_k, lin_q, lin_k, v, u)) + + if exists(mask): + mask = rearrange(mask, 'b (g j) -> b g 1 j', j=g) + + # calculate quadratic attention output + sim = einsum('... i d, ... j d -> ... i j', quad_q, quad_k) / g + + attn = F.relu(sim)**2 + attn = self.dropout(attn) + + if exists(mask): + attn = attn.masked_fill(~mask, 0.) + + if self.causal: + causal_mask = torch.ones((g, g), dtype=torch.bool, + device=device).triu(1) + attn = attn.masked_fill(causal_mask, 0.) + + quad_out_v = einsum('... i j, ... j d -> ... i d', attn, v) + quad_out_u = einsum('... i j, ... j d -> ... i d', attn, u) + + # calculate linear attention output + if self.causal: + lin_kv = einsum('b g n d, b g n e -> b g d e', lin_k, v) / g + # exclusive cumulative sum along group dimension + lin_kv = lin_kv.cumsum(dim=1) + lin_kv = F.pad(lin_kv, (0, 0, 0, 0, 1, -1), value=0.) + lin_out_v = einsum('b g d e, b g n d -> b g n e', lin_kv, lin_q) + + lin_ku = einsum('b g n d, b g n e -> b g d e', lin_k, u) / g + # exclusive cumulative sum along group dimension + lin_ku = lin_ku.cumsum(dim=1) + lin_ku = F.pad(lin_ku, (0, 0, 0, 0, 1, -1), value=0.) + lin_out_u = einsum('b g d e, b g n d -> b g n e', lin_ku, lin_q) + else: + lin_kv = einsum('b g n d, b g n e -> b d e', lin_k, v) / n + lin_out_v = einsum('b g n d, b d e -> b g n e', lin_q, lin_kv) + + lin_ku = einsum('b g n d, b g n e -> b d e', lin_k, u) / n + lin_out_u = einsum('b g n d, b d e -> b g n e', lin_q, lin_ku) + + # fold back groups into full sequence, and excise out padding + return map(lambda t: rearrange(t, 'b g n d -> b (g n) d')[:, :n], + (quad_out_v + lin_out_v, quad_out_u + lin_out_u)) + + +class GatedFSMNDilated(nn.Module): + + def __init__(self, in_channels, out_channels, lorder, hidden_size): + super().__init__() + self.to_u = FFConvM( + dim_in=in_channels, + dim_out=hidden_size, + norm_klass=nn.LayerNorm, + dropout=0.1, + ) + self.to_v = FFConvM( + dim_in=in_channels, + dim_out=hidden_size, + norm_klass=nn.LayerNorm, + dropout=0.1, + ) + self.fsmn = UniDeepFsmnDilated(in_channels, out_channels, lorder, + hidden_size) + + def forward( + self, + x, + ): + input = x + x_u = self.to_u(x) + x_v = self.to_v(x) + x_u = self.fsmn(x_u) + x = x_v * x_u + input + return x + + +class GatedFSMNDilatedDual(nn.Module): + + def __init__(self, in_channels, out_channels, lorder, hidden_size): + super().__init__() + self.to_u = FFConvMDilated( + dim_in=in_channels, + dim_out=hidden_size, + norm_klass=nn.LayerNorm, + dropout=0.1, + ) + self.to_v = FFConvMDilated( + dim_in=in_channels, + dim_out=hidden_size, + norm_klass=nn.LayerNorm, + dropout=0.1, + ) + self.fsmn = UniDeepFsmnDilated(in_channels, out_channels, lorder, + hidden_size) + + def forward( + self, + x, + ): + input = x + x_u = self.to_u(x) + x_v = self.to_v(x) + x_u = self.fsmn(x_u) + x = x_v * x_u + input + return x + + +class GatedFSMNBlockDilatedDual(nn.Module): + """1-D convolutional block.""" + + def __init__( + self, + dim, + inner_channels=256, + ): + super(GatedFSMNBlockDilatedDual, self).__init__() + + self.conv1 = nn.Sequential( + nn.Conv1d(dim, inner_channels, kernel_size=1), + nn.PReLU(), + ) + self.norm1 = CLayerNorm(inner_channels) + self.gated_fsmn = GatedFSMNDilatedDual( + inner_channels, + inner_channels, + lorder=20, + hidden_size=inner_channels) + self.norm2 = CLayerNorm(inner_channels) + self.conv2 = nn.Conv1d(inner_channels, dim, kernel_size=1) + + def forward(self, input): + conv1 = self.conv1(input.transpose(2, 1)) + norm1 = self.norm1(conv1) + seq_out = self.gated_fsmn(norm1.transpose(2, 1)) + norm2 = self.norm2(seq_out.transpose(2, 1)) + conv2 = self.conv2(norm2) + return conv2.transpose(2, 1) + input + + +class GatedFSMNBlockDilated(nn.Module): + """1-D convolutional block.""" + + def __init__( + self, + dim, + inner_channels=256, + group_size=256, + norm_type='scalenorm', + ): + super(GatedFSMNBlockDilated, self).__init__() + + self.group_size = group_size + + self.conv1 = nn.Sequential( + nn.Conv1d(dim, inner_channels, kernel_size=1), + nn.PReLU(), + ) + self.norm1 = CLayerNorm(inner_channels) + # block dilated with gating + self.gated_fsmn = GatedFSMNDilated( + inner_channels, + inner_channels, + lorder=20, + hidden_size=inner_channels) + self.norm2 = CLayerNorm(inner_channels) + self.conv2 = nn.Conv1d(inner_channels, dim, kernel_size=1) + + def forward(self, input): + conv1 = self.conv1(input.transpose(2, 1)) + norm1 = self.norm1(conv1) + seq_out = self.gated_fsmn(norm1.transpose(2, 1)) + norm2 = self.norm2(seq_out.transpose(2, 1)) + conv2 = self.conv2(norm2) + return conv2.transpose(2, 1) + input + + +class MossformerBlockGFSMN(nn.Module): + + def __init__(self, + *, + dim, + depth, + group_size=256, + query_key_dim=128, + expansion_factor=4., + causal=False, + attn_dropout=0.1, + norm_type='scalenorm', + shift_tokens=True): + super().__init__() + assert norm_type in ( + 'scalenorm', + 'layernorm'), 'norm_type must be one of scalenorm or layernorm' + + if norm_type == 'scalenorm': + norm_klass = ScaleNorm + elif norm_type == 'layernorm': + norm_klass = nn.LayerNorm + + self.group_size = group_size + + rotary_pos_emb = RotaryEmbedding(dim=min(32, query_key_dim)) + # max rotary embedding dimensions of 32, partial Rotary embeddings, from Wang et al - GPT-J + self.fsmn = nn.ModuleList( + [GatedFSMNBlockDilated(dim) for _ in range(depth)]) + self.layers = nn.ModuleList([ + FLASH_ShareA_FFConvM( + dim=dim, + group_size=group_size, + query_key_dim=query_key_dim, + expansion_factor=expansion_factor, + causal=causal, + dropout=attn_dropout, + rotary_pos_emb=rotary_pos_emb, + norm_klass=norm_klass, + shift_tokens=shift_tokens) for _ in range(depth) + ]) + + def _build_repeats(self, + in_channels, + out_channels, + lorder, + hidden_size, + repeats=1): + repeats = [ + UniDeepFsmn(in_channels, out_channels, lorder, hidden_size) + for i in range(repeats) + ] + return nn.Sequential(*repeats) + + def forward(self, x, *, mask=None): + ii = 0 + for flash in self.layers: + x = flash(x, mask=mask) + x = self.fsmn[ii](x) + ii = ii + 1 + return x + + +class MossformerBlock(nn.Module): + + def __init__(self, + *, + dim, + depth, + group_size=256, + query_key_dim=128, + expansion_factor=4., + causal=False, + attn_dropout=0.1, + norm_type='scalenorm', + shift_tokens=True): + super().__init__() + assert norm_type in ( + 'scalenorm', + 'layernorm'), 'norm_type must be one of scalenorm or layernorm' + + if norm_type == 'scalenorm': + norm_klass = ScaleNorm + elif norm_type == 'layernorm': + norm_klass = nn.LayerNorm + + self.group_size = group_size + + rotary_pos_emb = RotaryEmbedding(dim=min(32, query_key_dim)) + # max rotary embedding dimensions of 32, partial Rotary embeddings, from Wang et al - GPT-J + self.layers = nn.ModuleList([ + FLASH_ShareA_FFConvM( + dim=dim, + group_size=group_size, + query_key_dim=query_key_dim, + expansion_factor=expansion_factor, + causal=causal, + dropout=attn_dropout, + rotary_pos_emb=rotary_pos_emb, + norm_klass=norm_klass, + shift_tokens=shift_tokens) for _ in range(depth) + ]) + + def _build_repeats(self, + in_channels, + out_channels, + lorder, + hidden_size, + repeats=1): + repeats = [ + UniDeepFsmn(in_channels, out_channels, lorder, hidden_size) + for i in range(repeats) + ] + return nn.Sequential(*repeats) + + def forward(self, x, *, mask=None): + ii = 0 + for flash in self.layers: + x = flash(x, mask=mask) + ii = ii + 1 + return x diff --git a/runtime/Lib/site_packages/models/audio/separation/mossformer.py b/runtime/Lib/site_packages/models/audio/separation/mossformer.py new file mode 100644 index 0000000000000000000000000000000000000000..5264beef987dd7ee8dcefdc4b11ee1bc452384e5 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/mossformer.py @@ -0,0 +1,475 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy +import os +from typing import Any, Dict + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.separation.mossformer_block import ( + MossFormerModule, ScaledSinuEmbedding) +from modelscope.models.audio.separation.mossformer_conv_module import ( + CumulativeLayerNorm, GlobalLayerNorm) +from modelscope.models.base import Tensor +from modelscope.utils.constant import Tasks + +EPS = 1e-8 + + +@MODELS.register_module( + Tasks.speech_separation, + module_name=Models.speech_mossformer_separation_temporal_8k) +class MossFormer(TorchModel): + """Library to support MossFormer speech separation. + + Args: + model_dir (str): the model path. + """ + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.encoder = Encoder( + kernel_size=kwargs['kernel_size'], + out_channels=kwargs['out_channels']) + self.decoder = Decoder( + in_channels=kwargs['in_channels'], + out_channels=1, + kernel_size=kwargs['kernel_size'], + stride=kwargs['stride'], + bias=kwargs['bias']) + self.mask_net = MossFormerMaskNet( + kwargs['in_channels'], + kwargs['out_channels'], + MossFormerM(kwargs['num_blocks'], kwargs['d_model'], + kwargs['attn_dropout'], kwargs['group_size'], + kwargs['query_key_dim'], kwargs['expansion_factor'], + kwargs['causal']), + norm=kwargs['norm'], + num_spks=kwargs['num_spks']) + self.num_spks = kwargs['num_spks'] + + def forward(self, inputs: Tensor) -> Dict[str, Any]: + # Separation + mix_w = self.encoder(inputs) + est_mask = self.mask_net(mix_w) + mix_w = torch.stack([mix_w] * self.num_spks) + sep_h = mix_w * est_mask + # Decoding + est_source = torch.cat( + [ + self.decoder(sep_h[i]).unsqueeze(-1) + for i in range(self.num_spks) + ], + dim=-1, + ) + # T changed after conv1d in encoder, fix it here + t_origin = inputs.size(1) + t_est = est_source.size(1) + if t_origin > t_est: + est_source = F.pad(est_source, (0, 0, 0, t_origin - t_est)) + else: + est_source = est_source[:, :t_origin, :] + return est_source + + def load_check_point(self, load_path=None, device=None): + if not load_path: + load_path = self.model_dir + if not device: + device = torch.device('cpu') + self.encoder.load_state_dict( + torch.load( + os.path.join(load_path, 'encoder.bin'), map_location=device), + strict=True) + self.decoder.load_state_dict( + torch.load( + os.path.join(load_path, 'decoder.bin'), map_location=device), + strict=True) + self.mask_net.load_state_dict( + torch.load( + os.path.join(load_path, 'masknet.bin'), map_location=device), + strict=True) + + def as_dict(self): + return dict( + encoder=self.encoder, decoder=self.decoder, masknet=self.mask_net) + + +def select_norm(norm, dim, shape): + """Just a wrapper to select the normalization type. + """ + + if norm == 'gln': + return GlobalLayerNorm(dim, shape, elementwise_affine=True) + if norm == 'cln': + return CumulativeLayerNorm(dim, elementwise_affine=True) + if norm == 'ln': + return nn.GroupNorm(1, dim, eps=1e-8) + else: + return nn.BatchNorm1d(dim) + + +class Encoder(nn.Module): + """Convolutional Encoder Layer. + + Args: + kernel_size: Length of filters. + in_channels: Number of input channels. + out_channels: Number of output channels. + + Examples: + + >>> x = torch.randn(2, 1000) + >>> encoder = Encoder(kernel_size=4, out_channels=64) + >>> h = encoder(x) + >>> h.shape # torch.Size([2, 64, 499]) + """ + + def __init__(self, + kernel_size: int = 2, + out_channels: int = 64, + in_channels: int = 1): + super(Encoder, self).__init__() + self.conv1d = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=kernel_size // 2, + groups=1, + bias=False, + ) + self.in_channels = in_channels + + def forward(self, x: torch.Tensor): + """Return the encoded output. + + Args: + x: Input tensor with dimensionality [B, L]. + + Returns: + Encoded tensor with dimensionality [B, N, T_out]. + where B = Batchsize + L = Number of timepoints + N = Number of filters + T_out = Number of timepoints at the output of the encoder + """ + # B x L -> B x 1 x L + if self.in_channels == 1: + x = torch.unsqueeze(x, dim=1) + # B x 1 x L -> B x N x T_out + x = self.conv1d(x) + x = F.relu(x) + + return x + + +class Decoder(nn.ConvTranspose1d): + """A decoder layer that consists of ConvTranspose1d. + + Args: + kernel_size: Length of filters. + in_channels: Number of input channels. + out_channels: Number of output channels. + + Example + --------- + >>> x = torch.randn(2, 100, 1000) + >>> decoder = Decoder(kernel_size=4, in_channels=100, out_channels=1) + >>> h = decoder(x) + >>> h.shape + torch.Size([2, 1003]) + """ + + def __init__(self, *args, **kwargs): + super(Decoder, self).__init__(*args, **kwargs) + + def forward(self, x): + """Return the decoded output. + + Args: + x: Input tensor with dimensionality [B, N, L]. + where, B = Batchsize, + N = number of filters + L = time points + """ + + if x.dim() not in [2, 3]: + raise RuntimeError('{} accept 3/4D tensor as input'.format( + self.__name__)) + x = super().forward(x if x.dim() == 3 else torch.unsqueeze(x, 1)) + + if torch.squeeze(x).dim() == 1: + x = torch.squeeze(x, dim=1) + else: + x = torch.squeeze(x) + return x + + +class IdentityBlock: + """This block is used when we want to have identity transformation within the Dual_path block. + + Example + ------- + >>> x = torch.randn(10, 100) + >>> IB = IdentityBlock() + >>> xhat = IB(x) + """ + + def _init__(self, **kwargs): + pass + + def __call__(self, x): + return x + + +class MossFormerM(nn.Module): + """This class implements the transformer encoder. + + Args: + num_blocks : int + Number of mossformer blocks to include. + d_model : int + The dimension of the input embedding. + attn_dropout : float + Dropout for the self-attention (Optional). + group_size: int + the chunk size + query_key_dim: int + the attention vector dimension + expansion_factor: int + the expansion factor for the linear projection in conv module + causal: bool + true for causal / false for non causal + + Example + ------- + >>> import torch + >>> x = torch.rand((8, 60, 512)) #B, S, N + >>> net = MossFormerM(num_blocks=8, d_model=512) + >>> output, _ = net(x) + >>> output.shape + torch.Size([8, 60, 512]) + """ + + def __init__(self, + num_blocks, + d_model=None, + attn_dropout=0.1, + group_size=256, + query_key_dim=128, + expansion_factor=4., + causal=False): + super().__init__() + + self.mossformerM = MossFormerModule( + dim=d_model, + depth=num_blocks, + group_size=group_size, + query_key_dim=query_key_dim, + expansion_factor=expansion_factor, + causal=causal, + attn_dropout=attn_dropout) + import speechbrain as sb + self.norm = sb.nnet.normalization.LayerNorm(d_model, eps=1e-6) + + def forward(self, src: torch.Tensor): + """ + Args: + src: Tensor shape [B, S, N], + where, B = Batchsize, + S = time points + N = number of filters + The sequence to the encoder layer (required). + """ + output = self.mossformerM(src) + output = self.norm(output) + + return output + + +class ComputeAttention(nn.Module): + """Computation block for dual-path processing. + + Args: + att_mdl : torch.nn.module + Model to process within the chunks. + out_channels : int + Dimensionality of attention model. + norm : str + Normalization type. + skip_connection : bool + Skip connection around the attention module. + + Example + --------- + >>> att_block = MossFormerM(num_blocks=8, d_model=512) + >>> comp_att = ComputeAttention(att_block, 512) + >>> x = torch.randn(10, 64, 512) + >>> x = comp_att(x) + >>> x.shape + torch.Size([10, 64, 512]) + """ + + def __init__( + self, + att_mdl, + out_channels, + norm='ln', + skip_connection=True, + ): + super(ComputeAttention, self).__init__() + + self.att_mdl = att_mdl + self.skip_connection = skip_connection + + # Norm + self.norm = norm + if norm is not None: + self.att_norm = select_norm(norm, out_channels, 3) + + def forward(self, x: torch.Tensor): + """Returns the output tensor. + + Args: + x: Input tensor of dimension [B, S, N]. + + Returns: + out: Output tensor of dimension [B, S, N]. + where, B = Batchsize, + N = number of filters + S = time points + """ + # [B, S, N] + att_out = x.permute(0, 2, 1).contiguous() + + att_out = self.att_mdl(att_out) + + # [B, N, S] + att_out = att_out.permute(0, 2, 1).contiguous() + if self.norm is not None: + att_out = self.att_norm(att_out) + + # [B, N, S] + if self.skip_connection: + att_out = att_out + x + + out = att_out + return out + + +class MossFormerMaskNet(nn.Module): + """The dual path model which is the basis for dualpathrnn, sepformer, dptnet. + + Args: + in_channels : int + Number of channels at the output of the encoder. + out_channels : int + Number of channels that would be inputted to the intra and inter blocks. + att_model : torch.nn.module + Attention model to process the input sequence. + norm : str + Normalization type. + num_spks : int + Number of sources (speakers). + skip_connection : bool + Skip connection around attention module. + use_global_pos_enc : bool + Global positional encodings. + + Example + --------- + >>> mossformer_block = MossFormerM(num_blocks=8, d_model=512) + >>> mossformer_masknet = MossFormerMaskNet(64, 64, att_model, num_spks=2) + >>> x = torch.randn(10, 64, 2000) + >>> x = mossformer_masknet(x) + >>> x.shape + torch.Size([2, 10, 64, 2000]) + """ + + def __init__( + self, + in_channels, + out_channels, + att_model, + norm='ln', + num_spks=2, + skip_connection=True, + use_global_pos_enc=True, + ): + super(MossFormerMaskNet, self).__init__() + self.num_spks = num_spks + self.norm = select_norm(norm, in_channels, 3) + self.conv1d_encoder = nn.Conv1d( + in_channels, out_channels, 1, bias=False) + self.use_global_pos_enc = use_global_pos_enc + + if self.use_global_pos_enc: + self.pos_enc = ScaledSinuEmbedding(out_channels) + + self.mdl = copy.deepcopy( + ComputeAttention( + att_model, + out_channels, + norm, + skip_connection=skip_connection, + )) + + self.conv1d_out = nn.Conv1d( + out_channels, out_channels * num_spks, kernel_size=1) + self.conv1_decoder = nn.Conv1d( + out_channels, in_channels, 1, bias=False) + self.prelu = nn.PReLU() + self.activation = nn.ReLU() + # gated output layer + self.output = nn.Sequential( + nn.Conv1d(out_channels, out_channels, 1), nn.Tanh()) + self.output_gate = nn.Sequential( + nn.Conv1d(out_channels, out_channels, 1), nn.Sigmoid()) + + def forward(self, x: torch.Tensor): + """Returns the output tensor. + + Args: + x: Input tensor of dimension [B, N, S]. + + Returns: + out: Output tensor of dimension [spks, B, N, S] + where, spks = Number of speakers + B = Batchsize, + N = number of filters + S = the number of time frames + """ + + # before each line we indicate the shape after executing the line + # [B, N, L] + x = self.norm(x) + # [B, N, L] + x = self.conv1d_encoder(x) + if self.use_global_pos_enc: + base = x + x = x.transpose(1, -1) + emb = self.pos_enc(x) + emb = emb.transpose(0, -1) + x = base + emb + # [B, N, S] + x = self.mdl(x) + x = self.prelu(x) + # [B, N*spks, S] + x = self.conv1d_out(x) + b, _, s = x.shape + # [B*spks, N, S] + x = x.view(b * self.num_spks, -1, s) + # [B*spks, N, S] + x = self.output(x) * self.output_gate(x) + # [B*spks, N, S] + x = self.conv1_decoder(x) + # [B, spks, N, S] + _, n, L = x.shape + x = x.view(b, self.num_spks, n, L) + x = self.activation(x) + # [spks, B, N, S] + x = x.transpose(0, 1) + return x diff --git a/runtime/Lib/site_packages/models/audio/separation/mossformer_block.py b/runtime/Lib/site_packages/models/audio/separation/mossformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..1db8d010048591d8d5e1d6ad924dcf7043e81ade --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/mossformer_block.py @@ -0,0 +1,265 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn.functional as F +from torch import einsum, nn + +from modelscope.models.audio.separation.mossformer_conv_module import \ + MossFormerConvModule + + +def exists(val): + return val is not None + + +def default(val, d): + return val if exists(val) else d + + +def padding_to_multiple_of(n, mult): + remainder = n % mult + if remainder == 0: + return 0 + return mult - remainder + + +class ScaleNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.scale = dim**-0.5 + self.eps = eps + self.g = nn.Parameter(torch.ones(1)) + + def forward(self, x): + norm = torch.norm(x, dim=-1, keepdim=True) * self.scale + return x / norm.clamp(min=self.eps) * self.g + + +class ScaledSinuEmbedding(nn.Module): + + def __init__(self, dim): + super().__init__() + self.scale = nn.Parameter(torch.ones(1, )) + inv_freq = 1. / (10000**(torch.arange(0, dim, 2).float() / dim)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, x): + n, device = x.shape[1], x.device + t = torch.arange(n, device=device).type_as(self.inv_freq) + sinu = einsum('i , j -> i j', t, self.inv_freq) + emb = torch.cat((sinu.sin(), sinu.cos()), dim=-1) + return emb * self.scale + + +class OffsetScale(nn.Module): + + def __init__(self, dim, heads=1): + super().__init__() + self.gamma = nn.Parameter(torch.ones(heads, dim)) + self.beta = nn.Parameter(torch.zeros(heads, dim)) + nn.init.normal_(self.gamma, std=0.02) + + def forward(self, x): + out = einsum('... d, h d -> ... h d', x, self.gamma) + self.beta + return out.unbind(dim=-2) + + +class FFConvM(nn.Module): + + def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1): + super().__init__() + self.mdl = nn.Sequential( + norm_klass(dim_in), nn.Linear(dim_in, dim_out), nn.SiLU(), + MossFormerConvModule(dim_out), nn.Dropout(dropout)) + + def forward(self, x): + output = self.mdl(x) + return output + + +class MossFormerBlock(nn.Module): + + def __init__(self, + dim, + group_size=256, + query_key_dim=128, + expansion_factor=1., + causal=False, + dropout=0.1, + rotary_pos_emb=None, + norm_klass=nn.LayerNorm, + shift_tokens=True): + super().__init__() + hidden_dim = int(dim * expansion_factor) + self.group_size = group_size + self.causal = causal + self.shift_tokens = shift_tokens + # positional embeddings + self.rotary_pos_emb = rotary_pos_emb + # norm + self.dropout = nn.Dropout(dropout) + # projections + self.to_hidden = FFConvM( + dim_in=dim, + dim_out=hidden_dim, + norm_klass=norm_klass, + dropout=dropout, + ) + self.to_qk = FFConvM( + dim_in=dim, + dim_out=query_key_dim, + norm_klass=norm_klass, + dropout=dropout, + ) + self.qk_offset_scale = OffsetScale(query_key_dim, heads=4) + self.to_out = FFConvM( + dim_in=dim * 2, + dim_out=dim, + norm_klass=norm_klass, + dropout=dropout, + ) + self.gateActivate = nn.Sigmoid() + + def forward(self, x): + # prenorm + normed_x = x + # do token shift - a great, costless trick from an independent AI researcher in Shenzhen + if self.shift_tokens: + x_shift, x_pass = normed_x.chunk(2, dim=-1) + x_shift = F.pad(x_shift, (0, 0, 1, -1), value=0.) + normed_x = torch.cat((x_shift, x_pass), dim=-1) + + # initial projections + v, u = self.to_hidden(normed_x).chunk(2, dim=-1) + qk = self.to_qk(normed_x) + # offset and scale + quad_q, lin_q, quad_k, lin_k = self.qk_offset_scale(qk) + att_v, att_u = self.cal_attention(x, quad_q, lin_q, quad_k, lin_k, v, + u) + + # projection out and residual + out = (att_u * v) * self.gateActivate(att_v * u) + x = x + self.to_out(out) + return x + + def cal_attention(self, x, quad_q, lin_q, quad_k, lin_k, v, u, mask=None): + b, n, device, g = x.shape[0], x.shape[-2], x.device, self.group_size + + from einops import rearrange + if exists(mask): + lin_mask = rearrange(mask, '... -> ... 1') + lin_k = lin_k.masked_fill(~lin_mask, 0.) + + # rotate queries and keys + if exists(self.rotary_pos_emb): + quad_q, lin_q, quad_k, lin_k = map( + self.rotary_pos_emb.rotate_queries_or_keys, + (quad_q, lin_q, quad_k, lin_k)) + + # padding for groups + padding = padding_to_multiple_of(n, g) + if padding > 0: + quad_q, quad_k, lin_q, lin_k, v, u = map( + lambda t: F.pad(t, (0, 0, 0, padding), value=0.), + (quad_q, quad_k, lin_q, lin_k, v, u)) + mask = default(mask, + torch.ones((b, n), device=device, dtype=torch.bool)) + mask = F.pad(mask, (0, padding), value=False) + + # group along sequence + quad_q, quad_k, lin_q, lin_k, v, u = map( + lambda t: rearrange(t, 'b (g n) d -> b g n d', n=self.group_size), + (quad_q, quad_k, lin_q, lin_k, v, u)) + + if exists(mask): + mask = rearrange(mask, 'b (g j) -> b g 1 j', j=g) + + # calculate quadratic attention output + sim = einsum('... i d, ... j d -> ... i j', quad_q, quad_k) / g + attn = F.relu(sim)**2 + attn = self.dropout(attn) + if exists(mask): + attn = attn.masked_fill(~mask, 0.) + if self.causal: + causal_mask = torch.ones((g, g), dtype=torch.bool, + device=device).triu(1) + attn = attn.masked_fill(causal_mask, 0.) + + quad_out_v = einsum('... i j, ... j d -> ... i d', attn, v) + quad_out_u = einsum('... i j, ... j d -> ... i d', attn, u) + + # calculate linear attention output + if self.causal: + lin_kv = einsum('b g n d, b g n e -> b g d e', lin_k, v) / g + # exclusive cumulative sum along group dimension + lin_kv = lin_kv.cumsum(dim=1) + lin_kv = F.pad(lin_kv, (0, 0, 0, 0, 1, -1), value=0.) + lin_out_v = einsum('b g d e, b g n d -> b g n e', lin_kv, lin_q) + + lin_ku = einsum('b g n d, b g n e -> b g d e', lin_k, u) / g + # exclusive cumulative sum along group dimension + lin_ku = lin_ku.cumsum(dim=1) + lin_ku = F.pad(lin_ku, (0, 0, 0, 0, 1, -1), value=0.) + lin_out_u = einsum('b g d e, b g n d -> b g n e', lin_ku, lin_q) + else: + lin_kv = einsum('b g n d, b g n e -> b d e', lin_k, v) / n + lin_out_v = einsum('b g n d, b d e -> b g n e', lin_q, lin_kv) + + lin_ku = einsum('b g n d, b g n e -> b d e', lin_k, u) / n + lin_out_u = einsum('b g n d, b d e -> b g n e', lin_q, lin_ku) + + # fold back groups into full sequence, and excise out padding + quad_attn_out_v, lin_attn_out_v = map( + lambda t: rearrange(t, 'b g n d -> b (g n) d')[:, :n], + (quad_out_v, lin_out_v)) + quad_attn_out_u, lin_attn_out_u = map( + lambda t: rearrange(t, 'b g n d -> b (g n) d')[:, :n], + (quad_out_u, lin_out_u)) + + # gate + return quad_attn_out_v + lin_attn_out_v, quad_attn_out_u + lin_attn_out_u + + +class MossFormerModule(nn.Module): + + def __init__(self, + dim, + depth, + group_size=256, + query_key_dim=128, + expansion_factor=4., + causal=False, + attn_dropout=0.1, + norm_type='scalenorm', + shift_tokens=True): + super().__init__() + assert norm_type in ( + 'scalenorm', + 'layernorm'), 'norm_type must be one of scalenorm or layernorm' + + if norm_type == 'scalenorm': + norm_klass = ScaleNorm + elif norm_type == 'layernorm': + norm_klass = nn.LayerNorm + + from rotary_embedding_torch import RotaryEmbedding + rotary_pos_emb = RotaryEmbedding(dim=min(32, query_key_dim)) + # max rotary embedding dimensions of 32, partial Rotary embeddings, from Wang et al - GPT-J + self.layers = nn.ModuleList([ + MossFormerBlock( + dim=dim, + group_size=group_size, + query_key_dim=query_key_dim, + expansion_factor=expansion_factor, + causal=causal, + dropout=attn_dropout, + rotary_pos_emb=rotary_pos_emb, + norm_klass=norm_klass, + shift_tokens=shift_tokens) for _ in range(depth) + ]) + + def forward(self, x): + for mossformer_layer in self.layers: + x = mossformer_layer(x) + return x diff --git a/runtime/Lib/site_packages/models/audio/separation/mossformer_conv_module.py b/runtime/Lib/site_packages/models/audio/separation/mossformer_conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..283269b3eb376e25c1f50fc6928ff6117b7a5ebf --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/separation/mossformer_conv_module.py @@ -0,0 +1,272 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.init as init +from torch import Tensor + +EPS = 1e-8 + + +class GlobalLayerNorm(nn.Module): + """Calculate Global Layer Normalization. + + Args: + dim : (int or list or torch.Size) + Input shape from an expected input of size. + eps : float + A value added to the denominator for numerical stability. + elementwise_affine : bool + A boolean value that when set to True, + this module has learnable per-element affine parameters + initialized to ones (for weights) and zeros (for biases). + + Example + ------- + >>> x = torch.randn(5, 10, 20) + >>> GLN = GlobalLayerNorm(10, 3) + >>> x_norm = GLN(x) + """ + + def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True): + super(GlobalLayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.elementwise_affine = elementwise_affine + + if self.elementwise_affine: + if shape == 3: + self.weight = nn.Parameter(torch.ones(self.dim, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1)) + if shape == 4: + self.weight = nn.Parameter(torch.ones(self.dim, 1, 1)) + self.bias = nn.Parameter(torch.zeros(self.dim, 1, 1)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + + def forward(self, x): + """Returns the normalized tensor. + + Args: + x: Tensor of size [N, C, K, S] or [N, C, L]. + """ + # N x 1 x 1 + # cln: mean,var N x 1 x K x S + # gln: mean,var N x 1 x 1 + if x.dim() == 3: + mean = torch.mean(x, (1, 2), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2), keepdim=True) + if self.elementwise_affine: + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) # yapf: disable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + + if x.dim() == 4: + mean = torch.mean(x, (1, 2, 3), keepdim=True) + var = torch.mean((x - mean)**2, (1, 2, 3), keepdim=True) + if self.elementwise_affine: + x = (self.weight * (x - mean) / torch.sqrt(var + self.eps) + + self.bias) # yapf: disable + else: + x = (x - mean) / torch.sqrt(var + self.eps) + return x + + +class CumulativeLayerNorm(nn.LayerNorm): + """Calculate Cumulative Layer Normalization. + + Args: + dim: Dimension that you want to normalize. + elementwise_affine: Learnable per-element affine parameters. + + Example + ------- + >>> x = torch.randn(5, 10, 20) + >>> CLN = CumulativeLayerNorm(10) + >>> x_norm = CLN(x) + """ + + def __init__(self, dim, elementwise_affine=True): + super(CumulativeLayerNorm, self).__init__( + dim, elementwise_affine=elementwise_affine, eps=1e-8) + + def forward(self, x): + """Returns the normalized tensor. + + Args: + x: Tensor size [N, C, K, S] or [N, C, L] + """ + # N x K x S x C + if x.dim() == 4: + x = x.permute(0, 2, 3, 1).contiguous() + # N x K x S x C == only channel norm + x = super().forward(x) + # N x C x K x S + x = x.permute(0, 3, 1, 2).contiguous() + if x.dim() == 3: + x = torch.transpose(x, 1, 2) + # N x L x C == only channel norm + x = super().forward(x) + # N x C x L + x = torch.transpose(x, 1, 2) + return x + + +def select_norm(norm, dim, shape): + """Just a wrapper to select the normalization type. + """ + + if norm == 'gln': + return GlobalLayerNorm(dim, shape, elementwise_affine=True) + if norm == 'cln': + return CumulativeLayerNorm(dim, elementwise_affine=True) + if norm == 'ln': + return nn.GroupNorm(1, dim, eps=1e-8) + else: + return nn.BatchNorm1d(dim) + + +class Swish(nn.Module): + """ + Swish is a smooth, non-monotonic function that consistently matches or outperforms ReLU on deep networks applied + to a variety of challenging domains such as Image classification and Machine translation. + """ + + def __init__(self): + super(Swish, self).__init__() + + def forward(self, inputs: Tensor) -> Tensor: + return inputs * inputs.sigmoid() + + +class GLU(nn.Module): + """ + The gating mechanism is called Gated Linear Units (GLU), which was first introduced for natural language processing + in the paper “Language Modeling with Gated Convolutional Networks” + """ + + def __init__(self, dim: int) -> None: + super(GLU, self).__init__() + self.dim = dim + + def forward(self, inputs: Tensor) -> Tensor: + outputs, gate = inputs.chunk(2, dim=self.dim) + return outputs * gate.sigmoid() + + +class Transpose(nn.Module): + """ Wrapper class of torch.transpose() for Sequential module. """ + + def __init__(self, shape: tuple): + super(Transpose, self).__init__() + self.shape = shape + + def forward(self, x: Tensor) -> Tensor: + return x.transpose(*self.shape) + + +class Linear(nn.Module): + """ + Wrapper class of torch.nn.Linear + Weight initialize by xavier initialization and bias initialize to zeros. + """ + + def __init__(self, + in_features: int, + out_features: int, + bias: bool = True) -> None: + super(Linear, self).__init__() + self.linear = nn.Linear(in_features, out_features, bias=bias) + init.xavier_uniform_(self.linear.weight) + if bias: + init.zeros_(self.linear.bias) + + def forward(self, x: Tensor) -> Tensor: + return self.linear(x) + + +class DepthwiseConv1d(nn.Module): + """ + When groups == in_channels and out_channels == K * in_channels, where K is a positive integer, + this operation is termed in literature as depthwise convolution. + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + bias (bool, optional): If True, adds a learnable bias to the output. Default: True + Inputs: inputs + - **inputs** (batch, in_channels, time): Tensor containing input vector + Returns: outputs + - **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + bias: bool = False, + ) -> None: + super(DepthwiseConv1d, self).__init__() + assert out_channels % in_channels == 0, 'out_channels should be constant multiple of in_channels' + self.conv = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + groups=in_channels, + stride=stride, + padding=padding, + bias=bias, + ) + + def forward(self, inputs: Tensor) -> Tensor: + return self.conv(inputs) + + +class MossFormerConvModule(nn.Module): + """Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU). + + This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution + to aid training deep models. + + Args: + in_channels (int): Number of channels in the input + kernel_size (int or tuple, optional): Size of the convolving kernel Default: 17 + dropout_p (float, optional): probability of dropout + """ + + def __init__(self, + in_channels: int, + kernel_size: int = 17, + expansion_factor: int = 2) -> None: + super(MossFormerConvModule, self).__init__() + assert ( + kernel_size - 1 + ) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding" + assert expansion_factor == 2, 'Currently, Only Supports expansion_factor 2' + + self.sequential = nn.Sequential( + Transpose(shape=(1, 2)), + DepthwiseConv1d( + in_channels, + in_channels, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2), + ) + + def forward(self, inputs: Tensor) -> Tensor: + """ + Args: + inputs (batch, time, dim): Tensor contains input sequences + + Returns: + outputs (batch, time, dim): Tensor produces by conformer convolution module. + """ + return inputs + self.sequential(inputs).transpose(1, 2) diff --git a/runtime/Lib/site_packages/models/audio/sv/DTDNN.py b/runtime/Lib/site_packages/models/audio/sv/DTDNN.py new file mode 100644 index 0000000000000000000000000000000000000000..4fc7fedc889650ae919d0f3964adfa8016567fbe --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/DTDNN.py @@ -0,0 +1,204 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN_layers import (BasicResBlock, + CAMDenseTDNNBlock, + DenseLayer, StatsPool, + TDNNLayer, TransitLayer, + get_nonlinear) +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class FCM(nn.Module): + + def __init__(self, + block=BasicResBlock, + num_blocks=[2, 2], + m_channels=32, + feat_dim=80): + super(FCM, self).__init__() + self.in_planes = m_channels + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=2) + self.layer2 = self._make_layer( + block, m_channels, num_blocks[0], stride=2) + + self.conv2 = nn.Conv2d( + m_channels, + m_channels, + kernel_size=3, + stride=(2, 1), + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(m_channels) + self.out_channels = m_channels * (feat_dim // 8) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.unsqueeze(1) + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = F.relu(self.bn2(self.conv2(out))) + + shape = out.shape + out = out.reshape(shape[0], shape[1] * shape[2], shape[3]) + return out + + +class CAMPPlus(nn.Module): + + def __init__(self, + feat_dim=80, + embedding_size=512, + growth_rate=32, + bn_size=4, + init_channels=128, + config_str='batchnorm-relu', + memory_efficient=True, + output_level='segment'): + super(CAMPPlus, self).__init__() + + self.head = FCM(feat_dim=feat_dim) + channels = self.head.out_channels + self.output_level = output_level + + self.xvector = nn.Sequential( + OrderedDict([ + ('tdnn', + TDNNLayer( + channels, + init_channels, + 5, + stride=2, + dilation=1, + padding=-1, + config_str=config_str)), + ])) + channels = init_channels + for i, (num_layers, kernel_size, dilation) in enumerate( + zip((12, 24, 16), (3, 3, 3), (1, 2, 2))): + block = CAMDenseTDNNBlock( + num_layers=num_layers, + in_channels=channels, + out_channels=growth_rate, + bn_channels=bn_size * growth_rate, + kernel_size=kernel_size, + dilation=dilation, + config_str=config_str, + memory_efficient=memory_efficient) + self.xvector.add_module('block%d' % (i + 1), block) + channels = channels + num_layers * growth_rate + self.xvector.add_module( + 'transit%d' % (i + 1), + TransitLayer( + channels, channels // 2, bias=False, + config_str=config_str)) + channels //= 2 + + self.xvector.add_module('out_nonlinear', + get_nonlinear(config_str, channels)) + + if self.output_level == 'segment': + self.xvector.add_module('stats', StatsPool()) + self.xvector.add_module( + 'dense', + DenseLayer( + channels * 2, embedding_size, config_str='batchnorm_')) + else: + assert self.output_level == 'frame', '`output_level` should be set to \'segment\' or \'frame\'. ' + + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Linear)): + nn.init.kaiming_normal_(m.weight.data) + if m.bias is not None: + nn.init.zeros_(m.bias) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + x = self.head(x) + x = self.xvector(x) + if self.output_level == 'frame': + x = x.transpose(1, 2) + return x + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.campplus_sv) +class SpeakerVerificationCAMPPlus(TorchModel): + r"""A fast and efficient speaker embedding model, using a 2-dimensional convolution residual network as the head + and a densely connected time delay neural network as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + + self.feature_dim = self.model_config['fbank_dim'] + self.emb_size = self.model_config['emb_size'] + self.device = create_device(self.other_config['device']) + + self.embedding_model = CAMPPlus(self.feature_dim, self.emb_size) + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + return embedding.detach().cpu() + + def __extract_feature(self, audio): + features = [] + for au in audio: + feature = Kaldi.fbank( + au.unsqueeze(0), num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + features.append(feature.unsqueeze(0)) + features = torch.cat(features) + return features + + def __load_check_point(self, pretrained_model_name): + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=torch.device('cpu')), + strict=True) diff --git a/runtime/Lib/site_packages/models/audio/sv/DTDNN_layers.py b/runtime/Lib/site_packages/models/audio/sv/DTDNN_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..21e34621c933a8fe3a1458d4ca5f866f84932b0c --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/DTDNN_layers.py @@ -0,0 +1,266 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Some implementations are adapted from https://github.com/yuyq96/D-TDNN +""" + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from torch import nn + + +def get_nonlinear(config_str, channels): + nonlinear = nn.Sequential() + for name in config_str.split('-'): + if name == 'relu': + nonlinear.add_module('relu', nn.ReLU(inplace=True)) + elif name == 'prelu': + nonlinear.add_module('prelu', nn.PReLU(channels)) + elif name == 'batchnorm': + nonlinear.add_module('batchnorm', nn.BatchNorm1d(channels)) + elif name == 'batchnorm_': + nonlinear.add_module('batchnorm', + nn.BatchNorm1d(channels, affine=False)) + else: + raise ValueError('Unexpected module ({}).'.format(name)) + return nonlinear + + +def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2): + mean = x.mean(dim=dim) + std = x.std(dim=dim, unbiased=unbiased) + stats = torch.cat([mean, std], dim=-1) + if keepdim: + stats = stats.unsqueeze(dim=dim) + return stats + + +class StatsPool(nn.Module): + + def forward(self, x): + return statistics_pooling(x) + + +class TDNNLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + bias=False, + config_str='batchnorm-relu'): + super(TDNNLayer, self).__init__() + if padding < 0: + assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format( + kernel_size) + padding = (kernel_size - 1) // 2 * dilation + self.linear = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + x = self.linear(x) + x = self.nonlinear(x) + return x + + +class CAMLayer(nn.Module): + + def __init__(self, + bn_channels, + out_channels, + kernel_size, + stride, + padding, + dilation, + bias, + reduction=2): + super(CAMLayer, self).__init__() + self.linear_local = nn.Conv1d( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + self.linear1 = nn.Conv1d(bn_channels, bn_channels // reduction, 1) + self.relu = nn.ReLU(inplace=True) + self.linear2 = nn.Conv1d(bn_channels // reduction, out_channels, 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + y = self.linear_local(x) + context = x.mean(-1, keepdim=True) + self.seg_pooling(x) + context = self.relu(self.linear1(context)) + m = self.sigmoid(self.linear2(context)) + return y * m + + def seg_pooling(self, x, seg_len=100, stype='avg'): + if stype == 'avg': + seg = F.avg_pool1d( + x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + elif stype == 'max': + seg = F.max_pool1d( + x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + else: + raise ValueError('Wrong segment pooling type.') + shape = seg.shape + seg = seg.unsqueeze(-1).expand(*shape, + seg_len).reshape(*shape[:-1], -1) + seg = seg[..., :x.shape[-1]] + return seg + + +class CAMDenseTDNNLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str='batchnorm-relu', + memory_efficient=False): + super(CAMDenseTDNNLayer, self).__init__() + assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format( + kernel_size) + padding = (kernel_size - 1) // 2 * dilation + self.memory_efficient = memory_efficient + self.nonlinear1 = get_nonlinear(config_str, in_channels) + self.linear1 = nn.Conv1d(in_channels, bn_channels, 1, bias=False) + self.nonlinear2 = get_nonlinear(config_str, bn_channels) + self.cam_layer = CAMLayer( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + + def bn_function(self, x): + return self.linear1(self.nonlinear1(x)) + + def forward(self, x): + if self.training and self.memory_efficient: + x = cp.checkpoint(self.bn_function, x) + else: + x = self.bn_function(x) + x = self.cam_layer(self.nonlinear2(x)) + return x + + +class CAMDenseTDNNBlock(nn.ModuleList): + + def __init__(self, + num_layers, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str='batchnorm-relu', + memory_efficient=False): + super(CAMDenseTDNNBlock, self).__init__() + for i in range(num_layers): + layer = CAMDenseTDNNLayer( + in_channels=in_channels + i * out_channels, + out_channels=out_channels, + bn_channels=bn_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + config_str=config_str, + memory_efficient=memory_efficient) + self.add_module('tdnnd%d' % (i + 1), layer) + + def forward(self, x): + for layer in self: + x = torch.cat([x, layer(x)], dim=1) + return x + + +class TransitLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bias=True, + config_str='batchnorm-relu'): + super(TransitLayer, self).__init__() + self.nonlinear = get_nonlinear(config_str, in_channels) + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + + def forward(self, x): + x = self.nonlinear(x) + x = self.linear(x) + return x + + +class DenseLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bias=False, + config_str='batchnorm-relu'): + super(DenseLayer, self).__init__() + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + if len(x.shape) == 2: + x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1) + else: + x = self.linear(x) + x = self.nonlinear(x) + return x + + +class BasicResBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicResBlock, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, + planes, + kernel_size=3, + stride=(stride, 1), + padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d( + planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=(stride, 1), + bias=False), nn.BatchNorm2d(self.expansion * planes)) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out diff --git a/runtime/Lib/site_packages/models/audio/sv/ERes2Net.py b/runtime/Lib/site_packages/models/audio/sv/ERes2Net.py new file mode 100644 index 0000000000000000000000000000000000000000..3c07390b4bc8c14a208b57951b0174ca48b40785 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/ERes2Net.py @@ -0,0 +1,356 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Res2Net implementation is adapted from https://github.com/wenet-e2e/wespeaker. + ERes2Net incorporates both local and global feature fusion techniques to improve the performance. The local feature + fusion (LFF) fuses the features within one single residual block to extract the local signal. + The global feature fusion (GFF) takes acoustic features of different scales as input to aggregate global signal. +""" +import math +import os +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +import modelscope.models.audio.sv.pooling_layers as pooling_layers +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.fusion import AFF +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class ReLU(nn.Hardtanh): + + def __init__(self, inplace=False): + super(ReLU, self).__init__(0, 20, inplace) + + def __repr__(self): + inplace_str = 'inplace' if self.inplace else '' + return self.__class__.__name__ + ' (' \ + + inplace_str + ')' + + +def conv1x1(in_planes, out_planes, stride=1): + '1x1 convolution without padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=stride, + padding=0, + bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + '3x3 convolution with padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlockRes2Net(nn.Module): + expansion = 2 + + def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): + super(BasicBlockRes2Net, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + + convs = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class BasicBlockRes2Net_diff_AFF(nn.Module): + expansion = 2 + + def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): + super(BasicBlockRes2Net_diff_AFF, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + + convs = [] + fuse_models = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + for j in range(self.nums - 1): + fuse_models.append(AFF(channels=width)) + + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.fuse_models = nn.ModuleList(fuse_models) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = self.fuse_models[i - 1](sp, spx[i]) + + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class ERes2Net(nn.Module): + + def __init__(self, + block=BasicBlockRes2Net, + block_fuse=BasicBlockRes2Net_diff_AFF, + num_blocks=[3, 4, 6, 3], + m_channels=32, + feat_dim=80, + embed_dim=192, + pooling_func='TSTP', + two_emb_layer=False): + super(ERes2Net, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embed_dim = embed_dim + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer( + block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer( + block_fuse, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer( + block_fuse, m_channels * 8, num_blocks[3], stride=2) + + # downsampling + self.layer1_downsample = nn.Conv2d( + m_channels * 2, + m_channels * 4, + kernel_size=3, + stride=2, + padding=1, + bias=False) + self.layer2_downsample = nn.Conv2d( + m_channels * 4, + m_channels * 8, + kernel_size=3, + padding=1, + stride=2, + bias=False) + self.layer3_downsample = nn.Conv2d( + m_channels * 8, + m_channels * 16, + kernel_size=3, + padding=1, + stride=2, + bias=False) + + # bottom-up fusion + self.fuse_mode12 = AFF(channels=m_channels * 4) + self.fuse_mode123 = AFF(channels=m_channels * 8) + self.fuse_mode1234 = AFF(channels=m_channels * 16) + + self.n_stats = 1 if pooling_func == 'TAP' or pooling_func == 'TSDP' else 2 + self.pool = getattr(pooling_layers, pooling_func)( + in_dim=self.stats_dim * block.expansion) + self.seg_1 = nn.Linear(self.stats_dim * block.expansion * self.n_stats, + embed_dim) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embed_dim, affine=False) + self.seg_2 = nn.Linear(embed_dim, embed_dim) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) + + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out1 = self.layer1(out) + + # bottom-up fusion + out2 = self.layer2(out1) + out1_downsample = self.layer1_downsample(out1) + fuse_out12 = self.fuse_mode12(out2, out1_downsample) + + out3 = self.layer3(out2) + fuse_out12_downsample = self.layer2_downsample(fuse_out12) + fuse_out123 = self.fuse_mode123(out3, fuse_out12_downsample) + + out4 = self.layer4(out3) + fuse_out123_downsample = self.layer3_downsample(fuse_out123) + fuse_out1234 = self.fuse_mode1234(out4, fuse_out123_downsample) + stats = self.pool(fuse_out1234) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.eres2net_sv) +class SpeakerVerificationERes2Net(TorchModel): + r"""Enhanced Res2Net architecture with local and global feature fusion. ERes2Net is mainly composed + of LFF and GFF. The LFF extracts localization-preserved speaker features and strengthen the local information + interaction. GFF fuses multi-scale feature maps in bottom-up pathway to obtain global information. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.embed_dim = self.model_config['embed_dim'] + self.m_channels = self.model_config['channels'] + self.other_config = kwargs + self.feature_dim = 80 + self.device = create_device(self.other_config['device']) + + self.embedding_model = ERes2Net( + embed_dim=self.embed_dim, m_channels=self.m_channels) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + + return embedding.detach().cpu() + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/runtime/Lib/site_packages/models/audio/sv/ERes2Net_aug.py b/runtime/Lib/site_packages/models/audio/sv/ERes2Net_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..5540ff3eff7089f4bd20036fee9ae2d80a11440e --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/ERes2Net_aug.py @@ -0,0 +1,348 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Res2Net implementation is adapted from https://github.com/wenet-e2e/wespeaker. + ERes2Net_aug incorporates both local and global feature fusion techniques + to improve the performance. The training code is located on the following + GitHub repository: https://github.com/alibaba-damo-academy/3D-Speaker. +""" +import math +import os +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +import modelscope.models.audio.sv.pooling_layers as pooling_layers +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.fusion import AFF +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class ReLU(nn.Hardtanh): + + def __init__(self, inplace=False): + super(ReLU, self).__init__(0, 20, inplace) + + def __repr__(self): + inplace_str = 'inplace' if self.inplace else '' + return self.__class__.__name__ + ' (' \ + + inplace_str + ')' + + +def conv1x1(in_planes, out_planes, stride=1): + '1x1 convolution without padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=stride, + padding=0, + bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + '3x3 convolution with padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlockERes2Net(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3): + super(BasicBlockERes2Net, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + + convs = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class BasicBlockERes2Net_diff_AFF(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3): + super(BasicBlockERes2Net_diff_AFF, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = conv1x1(in_planes, width * scale, stride) + self.bn1 = nn.BatchNorm2d(width * scale) + + self.nums = scale + + convs = [] + fuse_models = [] + bns = [] + for i in range(self.nums): + convs.append(conv3x3(width, width)) + bns.append(nn.BatchNorm2d(width)) + # Add different fuse_model parameters + for j in range(self.nums - 1): + fuse_models.append(AFF(channels=width)) + + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.fuse_models = nn.ModuleList(fuse_models) + self.relu = ReLU(inplace=True) + + self.conv3 = conv1x1(width * scale, planes * self.expansion) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = self.fuse_models[i - 1](sp, spx[i]) + + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class ERes2Net_aug(nn.Module): + + def __init__(self, + block=BasicBlockERes2Net, + block_fuse=BasicBlockERes2Net_diff_AFF, + num_blocks=[3, 4, 6, 3], + m_channels=64, + feat_dim=80, + embedding_size=192, + pooling_func='TSTP', + two_emb_layer=False): + super(ERes2Net_aug, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embedding_size = embedding_size + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer( + block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer( + block_fuse, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer( + block_fuse, m_channels * 8, num_blocks[3], stride=2) + + self.layer1_downsample = nn.Conv2d( + m_channels * 4, + m_channels * 8, + kernel_size=3, + padding=1, + stride=2, + bias=False) + self.layer2_downsample = nn.Conv2d( + m_channels * 8, + m_channels * 16, + kernel_size=3, + padding=1, + stride=2, + bias=False) + self.layer3_downsample = nn.Conv2d( + m_channels * 16, + m_channels * 32, + kernel_size=3, + padding=1, + stride=2, + bias=False) + + self.fuse_mode12 = AFF(channels=m_channels * 8) + self.fuse_mode123 = AFF(channels=m_channels * 16) + self.fuse_mode1234 = AFF(channels=m_channels * 32) + + self.n_stats = 1 if pooling_func == 'TAP' or pooling_func == 'TSDP' else 2 + self.pool = getattr(pooling_layers, pooling_func)( + in_dim=self.stats_dim * block.expansion) + self.seg_1 = nn.Linear(self.stats_dim * block.expansion * self.n_stats, + embedding_size) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) + self.seg_2 = nn.Linear(embedding_size, embedding_size) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out1 = self.layer1(out) + out2 = self.layer2(out1) + out1_downsample = self.layer1_downsample(out1) + fuse_out12 = self.fuse_mode12(out2, out1_downsample) + out3 = self.layer3(out2) + fuse_out12_downsample = self.layer2_downsample(fuse_out12) + fuse_out123 = self.fuse_mode123(out3, fuse_out12_downsample) + out4 = self.layer4(out3) + fuse_out123_downsample = self.layer3_downsample(fuse_out123) + fuse_out1234 = self.fuse_mode1234(out4, fuse_out123_downsample) + stats = self.pool(fuse_out1234) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.eres2net_aug_sv) +class SpeakerVerificationERes2Net(TorchModel): + r"""Enhanced Res2Net_aug architecture with local and global feature fusion. + ERes2Net_aug is an upgraded version of ERes2Net that uses a larger number of + parameters to achieve better recognition performance. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + self.feature_dim = 80 + self.device = create_device(self.other_config['device']) + self.embedding_model = ERes2Net_aug() + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + + return embedding.detach().cpu() + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/runtime/Lib/site_packages/models/audio/sv/TDNN.py b/runtime/Lib/site_packages/models/audio/sv/TDNN.py new file mode 100644 index 0000000000000000000000000000000000000000..9cc35c1f80a44b2638d0f8b684c6c5b0c872cb0b --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/TDNN.py @@ -0,0 +1,303 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Conv1d_O(nn.Module): + + def __init__( + self, + out_channels, + kernel_size, + input_shape=None, + in_channels=None, + stride=1, + dilation=1, + padding='same', + groups=1, + bias=True, + padding_mode='reflect', + skip_transpose=False, + ): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.padding = padding + self.padding_mode = padding_mode + self.unsqueeze = False + self.skip_transpose = skip_transpose + + if input_shape is None and in_channels is None: + raise ValueError('Must provide one of input_shape or in_channels') + + if in_channels is None: + in_channels = self._check_input_shape(input_shape) + + self.conv = nn.Conv1d( + in_channels, + out_channels, + self.kernel_size, + stride=self.stride, + dilation=self.dilation, + padding=0, + groups=groups, + bias=bias, + ) + + def forward(self, x): + """Returns the output of the convolution. + + Arguments + --------- + x : torch.Tensor (batch, time, channel) + input to convolve. 2d or 4d tensors are expected. + """ + + if not self.skip_transpose: + x = x.transpose(1, -1) + + if self.unsqueeze: + x = x.unsqueeze(1) + + if self.padding == 'same': + x = self._manage_padding(x, self.kernel_size, self.dilation, + self.stride) + + elif self.padding == 'causal': + num_pad = (self.kernel_size - 1) * self.dilation + x = F.pad(x, (num_pad, 0)) + + elif self.padding == 'valid': + pass + + else: + raise ValueError( + "Padding must be 'same', 'valid' or 'causal'. Got " + + self.padding) + + wx = self.conv(x) + + if self.unsqueeze: + wx = wx.squeeze(1) + + if not self.skip_transpose: + wx = wx.transpose(1, -1) + + return wx + + def _manage_padding( + self, + x, + kernel_size: int, + dilation: int, + stride: int, + ): + # Detecting input shape + L_in = x.shape[-1] + + # Time padding + padding = get_padding_elem(L_in, stride, kernel_size, dilation) + + # Applying padding + x = F.pad(x, padding, mode=self.padding_mode) + + return x + + def _check_input_shape(self, shape): + """Checks the input shape and returns the number of input channels. + """ + + if len(shape) == 2: + self.unsqueeze = True + in_channels = 1 + elif self.skip_transpose: + in_channels = shape[1] + elif len(shape) == 3: + in_channels = shape[2] + else: + raise ValueError('conv1d expects 2d, 3d inputs. Got ' + + str(len(shape))) + + # Kernel size must be odd + if self.kernel_size % 2 == 0: + raise ValueError( + 'The field kernel size must be an odd number. Got %s.' % + (self.kernel_size)) + return in_channels + + +# Skip transpose as much as possible for efficiency +class Conv1d(Conv1d_O): + + def __init__(self, *args, **kwargs): + super().__init__(skip_transpose=True, *args, **kwargs) + + +def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int): + """This function computes the number of elements to add for zero-padding. + + Arguments + --------- + L_in : int + stride: int + kernel_size : int + dilation : int + """ + if stride > 1: + n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1) + L_out = stride * (n_steps - 1) + kernel_size * dilation + padding = [kernel_size // 2, kernel_size // 2] + + else: + L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1 + + padding = [(L_in - L_out) // 2, (L_in - L_out) // 2] + return padding + + +class BatchNorm1d_O(nn.Module): + + def __init__( + self, + input_shape=None, + input_size=None, + eps=1e-05, + momentum=0.1, + affine=True, + track_running_stats=True, + combine_batch_time=False, + skip_transpose=False, + ): + super().__init__() + self.combine_batch_time = combine_batch_time + self.skip_transpose = skip_transpose + + if input_size is None and skip_transpose: + input_size = input_shape[1] + elif input_size is None: + input_size = input_shape[-1] + + self.norm = nn.BatchNorm1d( + input_size, + eps=eps, + momentum=momentum, + affine=affine, + track_running_stats=track_running_stats, + ) + + def forward(self, x): + """Returns the normalized input tensor. + + Arguments + --------- + x : torch.Tensor (batch, time, [channels]) + input to normalize. 2d or 3d tensors are expected in input + 4d tensors can be used when combine_dims=True. + """ + shape_or = x.shape + if self.combine_batch_time: + if x.ndim == 3: + x = x.reshape(shape_or[0] * shape_or[1], shape_or[2]) + else: + x = x.reshape(shape_or[0] * shape_or[1], shape_or[3], + shape_or[2]) + + elif not self.skip_transpose: + x = x.transpose(-1, 1) + + x_n = self.norm(x) + + if self.combine_batch_time: + x_n = x_n.reshape(shape_or) + elif not self.skip_transpose: + x_n = x_n.transpose(1, -1) + + return x_n + + +class BatchNorm1d(BatchNorm1d_O): + + def __init__(self, *args, **kwargs): + super().__init__(skip_transpose=True, *args, **kwargs) + + +class Xvector(torch.nn.Module): + """This model extracts X-vectors for speaker recognition and diarization. + + Arguments + --------- + device : str + Device used e.g. "cpu" or "cuda". + activation : torch class + A class for constructing the activation layers. + tdnn_blocks : int + Number of time-delay neural (TDNN) layers. + tdnn_channels : list of ints + Output channels for TDNN layer. + tdnn_kernel_sizes : list of ints + List of kernel sizes for each TDNN layer. + tdnn_dilations : list of ints + List of dilations for kernels in each TDNN layer. + lin_neurons : int + Number of neurons in linear layers. + + Example + ------- + >>> compute_xvect = Xvector('cpu') + >>> input_feats = torch.rand([5, 10, 40]) + >>> outputs = compute_xvect(input_feats) + >>> outputs.shape + torch.Size([5, 1, 512]) + """ + + def __init__( + self, + device='cpu', + activation=torch.nn.LeakyReLU, + tdnn_blocks=5, + tdnn_channels=[512, 512, 512, 512, 1500], + tdnn_kernel_sizes=[5, 3, 3, 1, 1], + tdnn_dilations=[1, 2, 3, 1, 1], + lin_neurons=512, + in_channels=80, + ): + + super().__init__() + self.blocks = nn.ModuleList() + + # TDNN layers + for block_index in range(tdnn_blocks): + out_channels = tdnn_channels[block_index] + self.blocks.extend([ + Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=tdnn_kernel_sizes[block_index], + dilation=tdnn_dilations[block_index], + ), + activation(), + BatchNorm1d(input_size=out_channels), + ]) + in_channels = tdnn_channels[block_index] + + def forward(self, x, lens=None): + """Returns the x-vectors. + + Arguments + --------- + x : torch.Tensor + """ + + x = x.transpose(1, 2) + + for layer in self.blocks: + try: + x = layer(x, lengths=lens) + except TypeError: + x = layer(x) + x = x.transpose(1, 2) + return x diff --git a/runtime/Lib/site_packages/models/audio/sv/__init__.py b/runtime/Lib/site_packages/models/audio/sv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cac0191aea8208eb42688b8b57b1d611a913aa70 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .ecapa_tdnn import SpeakerVerificationECAPATDNN + +else: + _import_structure = {'ecapa_tdnn': ['SpeakerVerificationECAPATDNN']} + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/sv/cluster_backend.py b/runtime/Lib/site_packages/models/audio/sv/cluster_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..fb027876e3cd272d27225274c0fc0a416833ba07 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/cluster_backend.py @@ -0,0 +1,197 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict, Union + +import hdbscan +import numpy as np +import scipy +import sklearn +import umap +from sklearn.cluster._kmeans import k_means + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks + + +class SpectralCluster: + r"""A spectral clustering mehtod using unnormalized Laplacian of affinity matrix. + This implementation is adapted from https://github.com/speechbrain/speechbrain. + """ + + def __init__(self, min_num_spks=1, max_num_spks=15, pval=0.022): + self.min_num_spks = min_num_spks + self.max_num_spks = max_num_spks + self.pval = pval + + def __call__(self, X, oracle_num=None): + # Similarity matrix computation + sim_mat = self.get_sim_mat(X) + + # Refining similarity matrix with pval + prunned_sim_mat = self.p_pruning(sim_mat) + + # Symmetrization + sym_prund_sim_mat = 0.5 * (prunned_sim_mat + prunned_sim_mat.T) + + # Laplacian calculation + laplacian = self.get_laplacian(sym_prund_sim_mat) + + # Get Spectral Embeddings + emb, num_of_spk = self.get_spec_embs(laplacian, oracle_num) + + # Perform clustering + labels = self.cluster_embs(emb, num_of_spk) + + return labels + + def get_sim_mat(self, X): + # Cosine similarities + M = sklearn.metrics.pairwise.cosine_similarity(X, X) + return M + + def p_pruning(self, A): + if A.shape[0] * self.pval < 6: + pval = 6. / A.shape[0] + else: + pval = self.pval + + n_elems = int((1 - pval) * A.shape[0]) + + # For each row in a affinity matrix + for i in range(A.shape[0]): + low_indexes = np.argsort(A[i, :]) + low_indexes = low_indexes[0:n_elems] + + # Replace smaller similarity values by 0s + A[i, low_indexes] = 0 + return A + + def get_laplacian(self, M): + M[np.diag_indices(M.shape[0])] = 0 + D = np.sum(np.abs(M), axis=1) + D = np.diag(D) + L = D - M + return L + + def get_spec_embs(self, L, k_oracle=None): + lambdas, eig_vecs = scipy.linalg.eigh(L) + + if k_oracle is not None: + num_of_spk = k_oracle + else: + lambda_gap_list = self.getEigenGaps( + lambdas[self.min_num_spks - 1:self.max_num_spks + 1]) + num_of_spk = np.argmax(lambda_gap_list) + self.min_num_spks + + emb = eig_vecs[:, :num_of_spk] + return emb, num_of_spk + + def cluster_embs(self, emb, k): + _, labels, _ = k_means(emb, k) + return labels + + def getEigenGaps(self, eig_vals): + eig_vals_gap_list = [] + for i in range(len(eig_vals) - 1): + gap = float(eig_vals[i + 1]) - float(eig_vals[i]) + eig_vals_gap_list.append(gap) + return eig_vals_gap_list + + +class UmapHdbscan: + r""" + Reference: + - Siqi Zheng, Hongbin Suo. Reformulating Speaker Diarization as Community Detection With + Emphasis On Topological Structure. ICASSP2022 + """ + + def __init__(self, + n_neighbors=20, + n_components=60, + min_samples=10, + min_cluster_size=10, + metric='cosine'): + self.n_neighbors = n_neighbors + self.n_components = n_components + self.min_samples = min_samples + self.min_cluster_size = min_cluster_size + self.metric = metric + + def __call__(self, X): + umap_X = umap.UMAP( + n_neighbors=self.n_neighbors, + min_dist=0.0, + n_components=min(self.n_components, X.shape[0] - 2), + metric=self.metric, + ).fit_transform(X) + labels = hdbscan.HDBSCAN( + min_samples=self.min_samples, + min_cluster_size=self.min_cluster_size, + allow_single_cluster=True).fit_predict(umap_X) + return labels + + +@MODELS.register_module( + Tasks.speaker_diarization, module_name=Models.cluster_backend) +class ClusterBackend(TorchModel): + r"""Perfom clustering for input embeddings and output the labels. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + + self.spectral_cluster = SpectralCluster() + self.umap_hdbscan_cluster = UmapHdbscan() + + def forward(self, X, **params): + # clustering and return the labels + k = params['oracle_num'] if 'oracle_num' in params else None + assert len( + X.shape + ) == 2, 'modelscope error: the shape of input should be [N, C]' + if X.shape[0] < 20: + return np.zeros(X.shape[0], dtype='int') + if X.shape[0] < 2048 or k is not None: + labels = self.spectral_cluster(X, k) + else: + labels = self.umap_hdbscan_cluster(X) + + if k is None and 'merge_thr' in self.model_config: + labels = self.merge_by_cos(labels, X, + self.model_config['merge_thr']) + + return labels + + def merge_by_cos(self, labels, embs, cos_thr): + # merge the similar speakers by cosine similarity + assert cos_thr > 0 and cos_thr <= 1 + while True: + spk_num = labels.max() + 1 + if spk_num == 1: + break + spk_center = [] + for i in range(spk_num): + spk_emb = embs[labels == i].mean(0) + spk_center.append(spk_emb) + assert len(spk_center) > 0 + spk_center = np.stack(spk_center, axis=0) + norm_spk_center = spk_center / np.linalg.norm( + spk_center, axis=1, keepdims=True) + affinity = np.matmul(norm_spk_center, norm_spk_center.T) + affinity = np.triu(affinity, 1) + spks = np.unravel_index(np.argmax(affinity), affinity.shape) + if affinity[spks] < cos_thr: + break + for i in range(len(labels)): + if labels[i] == spks[1]: + labels[i] = spks[0] + elif labels[i] > spks[1]: + labels[i] -= 1 + return labels diff --git a/runtime/Lib/site_packages/models/audio/sv/ecapa_tdnn.py b/runtime/Lib/site_packages/models/audio/sv/ecapa_tdnn.py new file mode 100644 index 0000000000000000000000000000000000000000..a068efa2750a43fa211bd89f742d18848a1825eb --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/ecapa_tdnn.py @@ -0,0 +1,515 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" This ECAPA-TDNN implementation is adapted from https://github.com/speechbrain/speechbrain. +""" +import math +import os +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +def length_to_mask(length, max_len=None, dtype=None, device=None): + assert len(length.shape) == 1 + + if max_len is None: + max_len = length.max().long().item() + mask = torch.arange( + max_len, device=length.device, dtype=length.dtype).expand( + len(length), max_len) < length.unsqueeze(1) + + if dtype is None: + dtype = length.dtype + + if device is None: + device = length.device + + mask = torch.as_tensor(mask, dtype=dtype, device=device) + return mask + + +def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int): + if stride > 1: + n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1) + L_out = stride * (n_steps - 1) + kernel_size * dilation + padding = [kernel_size // 2, kernel_size // 2] + + else: + L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1 + + padding = [(L_in - L_out) // 2, (L_in - L_out) // 2] + return padding + + +class Conv1d(nn.Module): + + def __init__( + self, + out_channels, + kernel_size, + in_channels, + stride=1, + dilation=1, + padding='same', + groups=1, + bias=True, + padding_mode='reflect', + ): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.padding = padding + self.padding_mode = padding_mode + + self.conv = nn.Conv1d( + in_channels, + out_channels, + self.kernel_size, + stride=self.stride, + dilation=self.dilation, + padding=0, + groups=groups, + bias=bias, + ) + + def forward(self, x): + if self.padding == 'same': + x = self._manage_padding(x, self.kernel_size, self.dilation, + self.stride) + + elif self.padding == 'causal': + num_pad = (self.kernel_size - 1) * self.dilation + x = F.pad(x, (num_pad, 0)) + + elif self.padding == 'valid': + pass + + else: + raise ValueError( + "Padding must be 'same', 'valid' or 'causal'. Got " + + self.padding) + + wx = self.conv(x) + + return wx + + def _manage_padding( + self, + x, + kernel_size: int, + dilation: int, + stride: int, + ): + L_in = x.shape[-1] + padding = get_padding_elem(L_in, stride, kernel_size, dilation) + x = F.pad(x, padding, mode=self.padding_mode) + + return x + + +class BatchNorm1d(nn.Module): + + def __init__( + self, + input_size, + eps=1e-05, + momentum=0.1, + ): + super().__init__() + self.norm = nn.BatchNorm1d( + input_size, + eps=eps, + momentum=momentum, + ) + + def forward(self, x): + return self.norm(x) + + +class TDNNBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + dilation, + activation=nn.ReLU, + groups=1, + ): + super(TDNNBlock, self).__init__() + self.conv = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + dilation=dilation, + groups=groups, + ) + self.activation = activation() + self.norm = BatchNorm1d(input_size=out_channels) + + def forward(self, x): + return self.norm(self.activation(self.conv(x))) + + +class Res2NetBlock(torch.nn.Module): + + def __init__(self, + in_channels, + out_channels, + scale=8, + kernel_size=3, + dilation=1): + super(Res2NetBlock, self).__init__() + assert in_channels % scale == 0 + assert out_channels % scale == 0 + + in_channel = in_channels // scale + hidden_channel = out_channels // scale + + self.blocks = nn.ModuleList([ + TDNNBlock( + in_channel, + hidden_channel, + kernel_size=kernel_size, + dilation=dilation, + ) for i in range(scale - 1) + ]) + self.scale = scale + + def forward(self, x): + y = [] + for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)): + if i == 0: + y_i = x_i + elif i == 1: + y_i = self.blocks[i - 1](x_i) + else: + y_i = self.blocks[i - 1](x_i + y_i) + y.append(y_i) + y = torch.cat(y, dim=1) + return y + + +class SEBlock(nn.Module): + + def __init__(self, in_channels, se_channels, out_channels): + super(SEBlock, self).__init__() + + self.conv1 = Conv1d( + in_channels=in_channels, out_channels=se_channels, kernel_size=1) + self.relu = torch.nn.ReLU(inplace=True) + self.conv2 = Conv1d( + in_channels=se_channels, out_channels=out_channels, kernel_size=1) + self.sigmoid = torch.nn.Sigmoid() + + def forward(self, x, lengths=None): + L = x.shape[-1] + if lengths is not None: + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + total = mask.sum(dim=2, keepdim=True) + s = (x * mask).sum(dim=2, keepdim=True) / total + else: + s = x.mean(dim=2, keepdim=True) + + s = self.relu(self.conv1(s)) + s = self.sigmoid(self.conv2(s)) + + return s * x + + +class AttentiveStatisticsPooling(nn.Module): + + def __init__(self, channels, attention_channels=128, global_context=True): + super().__init__() + + self.eps = 1e-12 + self.global_context = global_context + if global_context: + self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) + else: + self.tdnn = TDNNBlock(channels, attention_channels, 1, 1) + self.tanh = nn.Tanh() + self.conv = Conv1d( + in_channels=attention_channels, + out_channels=channels, + kernel_size=1) + + def forward(self, x, lengths=None): + L = x.shape[-1] + + def _compute_statistics(x, m, dim=2, eps=self.eps): + mean = (m * x).sum(dim) + std = torch.sqrt( + (m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)) + return mean, std + + if lengths is None: + lengths = torch.ones(x.shape[0], device=x.device) + + # Make binary mask of shape [N, 1, L] + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + + # Expand the temporal context of the pooling layer by allowing the + # self-attention to look at global properties of the utterance. + if self.global_context: + # torch.std is unstable for backward computation + # https://github.com/pytorch/pytorch/issues/4320 + total = mask.sum(dim=2, keepdim=True).float() + mean, std = _compute_statistics(x, mask / total) + mean = mean.unsqueeze(2).repeat(1, 1, L) + std = std.unsqueeze(2).repeat(1, 1, L) + attn = torch.cat([x, mean, std], dim=1) + else: + attn = x + + # Apply layers + attn = self.conv(self.tanh(self.tdnn(attn))) + + # Filter out zero-paddings + attn = attn.masked_fill(mask == 0, float('-inf')) + + attn = F.softmax(attn, dim=2) + mean, std = _compute_statistics(x, attn) + # Append mean and std of the batch + pooled_stats = torch.cat((mean, std), dim=1) + pooled_stats = pooled_stats.unsqueeze(2) + + return pooled_stats + + +class SERes2NetBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + res2net_scale=8, + se_channels=128, + kernel_size=1, + dilation=1, + activation=torch.nn.ReLU, + groups=1, + ): + super().__init__() + self.out_channels = out_channels + self.tdnn1 = TDNNBlock( + in_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.res2net_block = Res2NetBlock(out_channels, out_channels, + res2net_scale, kernel_size, dilation) + self.tdnn2 = TDNNBlock( + out_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.se_block = SEBlock(out_channels, se_channels, out_channels) + + self.shortcut = None + if in_channels != out_channels: + self.shortcut = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + residual = x + if self.shortcut: + residual = self.shortcut(x) + + x = self.tdnn1(x) + x = self.res2net_block(x) + x = self.tdnn2(x) + x = self.se_block(x, lengths) + + return x + residual + + +class ECAPA_TDNN(nn.Module): + """An implementation of the speaker embedding model in a paper. + "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in + TDNN Based Speaker Verification" (https://arxiv.org/abs/2005.07143). + """ + + def __init__( + self, + input_size, + device='cpu', + lin_neurons=192, + activation=torch.nn.ReLU, + channels=[512, 512, 512, 512, 1536], + kernel_sizes=[5, 3, 3, 3, 1], + dilations=[1, 2, 3, 4, 1], + attention_channels=128, + res2net_scale=8, + se_channels=128, + global_context=True, + groups=[1, 1, 1, 1, 1], + ): + + super().__init__() + assert len(channels) == len(kernel_sizes) + assert len(channels) == len(dilations) + self.channels = channels + self.blocks = nn.ModuleList() + + # The initial TDNN layer + self.blocks.append( + TDNNBlock( + input_size, + channels[0], + kernel_sizes[0], + dilations[0], + activation, + groups[0], + )) + + # SE-Res2Net layers + for i in range(1, len(channels) - 1): + self.blocks.append( + SERes2NetBlock( + channels[i - 1], + channels[i], + res2net_scale=res2net_scale, + se_channels=se_channels, + kernel_size=kernel_sizes[i], + dilation=dilations[i], + activation=activation, + groups=groups[i], + )) + + # Multi-layer feature aggregation + self.mfa = TDNNBlock( + channels[-1], + channels[-1], + kernel_sizes[-1], + dilations[-1], + activation, + groups=groups[-1], + ) + + # Attentive Statistical Pooling + self.asp = AttentiveStatisticsPooling( + channels[-1], + attention_channels=attention_channels, + global_context=global_context, + ) + self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2) + + # Final linear transformation + self.fc = Conv1d( + in_channels=channels[-1] * 2, + out_channels=lin_neurons, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + """Returns the embedding vector. + + Arguments + --------- + x : torch.Tensor + Tensor of shape (batch, time, channel). + """ + x = x.transpose(1, 2) + + xl = [] + for layer in self.blocks: + try: + x = layer(x, lengths=lengths) + except TypeError: + x = layer(x) + xl.append(x) + + # Multi-layer feature aggregation + x = torch.cat(xl[1:], dim=1) + x = self.mfa(x) + + # Attentive Statistical Pooling + x = self.asp(x, lengths=lengths) + x = self.asp_bn(x) + + # Final linear transformation + x = self.fc(x) + + x = x.transpose(1, 2).squeeze(1) + return x + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.ecapa_tdnn_sv) +class SpeakerVerificationECAPATDNN(TorchModel): + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + if self.model_config['channel'] != 1024: + raise ValueError( + 'modelscope error: Currently only 1024-channel ecapa tdnn is supported.' + ) + + self.feature_dim = 80 + channels_config = [1024, 1024, 1024, 1024, 3072] + self.device = create_device(self.other_config['device']) + print(self.device) + + self.embedding_model = ECAPA_TDNN( + self.feature_dim, channels=channels_config) + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + + return embedding.detach().cpu() + + def __extract_feature(self, audio): + features = [] + for au in audio: + feature = Kaldi.fbank( + au.unsqueeze(0), num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + features.append(feature.unsqueeze(0)) + features = torch.cat(features) + return features + + def __load_check_point(self, pretrained_model_name): + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=torch.device('cpu')), + strict=True) diff --git a/runtime/Lib/site_packages/models/audio/sv/fusion.py b/runtime/Lib/site_packages/models/audio/sv/fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..615529bdbf40f2aed3049f423e9ce42b238dbb8b --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/fusion.py @@ -0,0 +1,32 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + + +class AFF(nn.Module): + + def __init__(self, channels=64, r=4): + super(AFF, self).__init__() + inter_channels = int(channels // r) + + self.local_att = nn.Sequential( + nn.Conv2d( + channels * 2, + inter_channels, + kernel_size=1, + stride=1, + padding=0), + nn.BatchNorm2d(inter_channels), + nn.SiLU(inplace=True), + nn.Conv2d( + inter_channels, channels, kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(channels), + ) + + def forward(self, x, ds_y): + xa = torch.cat((x, ds_y), dim=1) + x_att = self.local_att(xa) + x_att = 1.0 + torch.tanh(x_att) + xo = torch.mul(x, x_att) + torch.mul(ds_y, 2.0 - x_att) + + return xo diff --git a/runtime/Lib/site_packages/models/audio/sv/generic_speaker_verification.py b/runtime/Lib/site_packages/models/audio/sv/generic_speaker_verification.py new file mode 100644 index 0000000000000000000000000000000000000000..788ccf7c7e067943895c3ac23e785ff7493e490a --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/generic_speaker_verification.py @@ -0,0 +1,45 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Frameworks, Tasks + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.generic_sv) +@MODELS.register_module( + Tasks.speaker_diarization, module_name=Models.generic_sv) +class SpeakerVerification(Model): + + def __init__(self, model_dir: str, model_name: str, + model_config: Dict[str, Any], *args, **kwargs): + """initialize the info of model. + + Args: + model_dir (str): the model path. + model_name (str): the itn model name from configuration.json + model_config (Dict[str, Any]): the detail config about model from configuration.json + """ + super().__init__(model_dir, model_name, model_config, *args, **kwargs) + self.model_cfg = { + # the recognition model dir path + 'model_workspace': model_dir, + # the itn model name + 'model_name': model_name, + # the am model file path + 'model_path': os.path.join(model_dir, model_name), + # the recognition model config dict + 'model_config': model_config + } + + def forward(self) -> Dict[str, Any]: + """ + just return the model config + + """ + + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_eres2net.py b/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_eres2net.py new file mode 100644 index 0000000000000000000000000000000000000000..0876cd2e5b21357443f820248509996183be8e61 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_eres2net.py @@ -0,0 +1,120 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +import numpy as np +import torch +import torch.nn as nn +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN import CAMPPlus +from modelscope.models.audio.sv.DTDNN_layers import DenseLayer +from modelscope.models.audio.sv.ERes2Net import ERes2Net +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class LinearClassifier(nn.Module): + + def __init__( + self, + input_dim, + num_blocks=0, + inter_dim=512, + out_neurons=1000, + ): + + super().__init__() + self.blocks = nn.ModuleList() + + self.nonlinear = nn.ReLU(inplace=True) + for _ in range(num_blocks): + self.blocks.append(DenseLayer(input_dim, inter_dim, bias=True)) + input_dim = inter_dim + + self.linear = nn.Linear(input_dim, out_neurons, bias=True) + + def forward(self, x): + # x: [B, dim] + x = self.nonlinear(x) + for layer in self.blocks: + x = layer(x) + x = self.linear(x) + return x + + +@MODELS.register_module( + Tasks.speech_language_recognition, module_name=Models.eres2net_lre) +class LanguageRecognitionERes2Net(TorchModel): + r"""A speech language recognition model using the ERes2Net architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.embed_dim = self.model_config['embed_dim'] + self.m_channels = self.model_config['channels'] + self.feature_dim = self.model_config['fbank_dim'] + self.sample_rate = self.model_config['sample_rate'] + self.device = create_device(kwargs['device']) + + self.encoder = ERes2Net( + embed_dim=self.embed_dim, m_channels=self.m_channels) + self.backend = LinearClassifier( + input_dim=self.embed_dim, + out_neurons=len(self.model_config['languages'])) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self._load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.to(self.device) + self.backend.to(self.device) + self.encoder.eval() + self.backend.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len(audio.shape) == 2, \ + 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self._extract_feature(audio) + embs = self.encoder(feature.to(self.device)) + output = self.backend(embs) + output = output.detach().cpu().argmax(-1) + return output + + def _extract_feature(self, audio): + features = [] + for au in audio: + feature = Kaldi.fbank( + au.unsqueeze(0), + num_mel_bins=self.feature_dim, + sample_frequency=self.sample_rate) + feature = feature - feature.mean(dim=0, keepdim=True) + features.append(feature.unsqueeze(0)) + features = torch.cat(features) + return features + + def _load_check_point(self, pretrained_encoder, pretrained_backend): + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=torch.device('cpu'))) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=torch.device('cpu'))) diff --git a/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_model.py b/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_model.py new file mode 100644 index 0000000000000000000000000000000000000000..3ab531282887dabb621d2fd38c1fe7f96a075e11 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/lanuage_recognition_model.py @@ -0,0 +1,117 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict + +import numpy as np +import torch +import torch.nn as nn +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN import CAMPPlus +from modelscope.models.audio.sv.DTDNN_layers import DenseLayer +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class LinearClassifier(nn.Module): + + def __init__( + self, + input_dim, + num_blocks=0, + inter_dim=512, + out_neurons=1000, + ): + + super().__init__() + self.blocks = nn.ModuleList() + + self.nonlinear = nn.ReLU(inplace=True) + for _ in range(num_blocks): + self.blocks.append(DenseLayer(input_dim, inter_dim, bias=True)) + input_dim = inter_dim + + self.linear = nn.Linear(input_dim, out_neurons, bias=True) + + def forward(self, x): + # x: [B, dim] + x = self.nonlinear(x) + for layer in self.blocks: + x = layer(x) + x = self.linear(x) + return x + + +@MODELS.register_module( + Tasks.speech_language_recognition, module_name=Models.campplus_lre) +class LanguageRecognitionCAMPPlus(TorchModel): + r"""A speech language recognition model using the CAM++ architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.emb_size = self.model_config['emb_size'] + self.feature_dim = self.model_config['fbank_dim'] + self.sample_rate = self.model_config['sample_rate'] + self.device = create_device(kwargs['device']) + + self.encoder = CAMPPlus(self.feature_dim, self.emb_size) + self.backend = LinearClassifier( + input_dim=self.emb_size, + out_neurons=len(self.model_config['languages'])) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self._load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.to(self.device) + self.backend.to(self.device) + self.encoder.eval() + self.backend.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len(audio.shape) == 2, \ + 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self._extract_feature(audio) + embs = self.encoder(feature.to(self.device)) + output = self.backend(embs) + output = output.detach().cpu().argmax(-1) + return output + + def _extract_feature(self, audio): + features = [] + for au in audio: + feature = Kaldi.fbank( + au.unsqueeze(0), + num_mel_bins=self.feature_dim, + sample_frequency=self.sample_rate) + feature = feature - feature.mean(dim=0, keepdim=True) + features.append(feature.unsqueeze(0)) + features = torch.cat(features) + return features + + def _load_check_point(self, pretrained_encoder, pretrained_backend): + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=torch.device('cpu'))) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=torch.device('cpu'))) diff --git a/runtime/Lib/site_packages/models/audio/sv/pooling_layers.py b/runtime/Lib/site_packages/models/audio/sv/pooling_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0fdc44ca98a4660087b38ffe82108ab73634fd49 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/pooling_layers.py @@ -0,0 +1,107 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" This implementation is adapted from https://github.com/wenet-e2e/wespeaker. +""" +import torch +import torch.nn as nn + + +class TAP(nn.Module): + """ + Temporal average pooling, only first-order mean is considered + """ + + def __init__(self, **kwargs): + super(TAP, self).__init__() + + def forward(self, x): + pooling_mean = x.mean(dim=-1) + # To be compatable with 2D input + pooling_mean = pooling_mean.flatten(start_dim=1) + return pooling_mean + + +class TSDP(nn.Module): + """ + Temporal standard deviation pooling, only second-order std is considered + """ + + def __init__(self, **kwargs): + super(TSDP, self).__init__() + + def forward(self, x): + # The last dimension is the temporal axis + pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) + pooling_std = pooling_std.flatten(start_dim=1) + return pooling_std + + +class TSTP(nn.Module): + """ + Temporal statistics pooling, concatenate mean and std, which is used in + x-vector + Comment: simple concatenation can not make full use of both statistics + """ + + def __init__(self, **kwargs): + super(TSTP, self).__init__() + + def forward(self, x): + # The last dimension is the temporal axis + pooling_mean = x.mean(dim=-1) + pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8) + pooling_mean = pooling_mean.flatten(start_dim=1) + pooling_std = pooling_std.flatten(start_dim=1) + + stats = torch.cat((pooling_mean, pooling_std), 1) + return stats + + +class ASTP(nn.Module): + """ Attentive statistics pooling: Channel- and context-dependent + statistics pooling, first used in ECAPA_TDNN. + """ + + def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False): + super(ASTP, self).__init__() + self.global_context_att = global_context_att + + # Use Conv1d with stride == 1 rather than Linear, then we don't + # need to transpose inputs. + if global_context_att: + self.linear1 = nn.Conv1d( + in_dim * 3, bottleneck_dim, + kernel_size=1) # equals W and b in the paper + else: + self.linear1 = nn.Conv1d( + in_dim, bottleneck_dim, + kernel_size=1) # equals W and b in the paper + self.linear2 = nn.Conv1d( + bottleneck_dim, in_dim, + kernel_size=1) # equals V and k in the paper + + def forward(self, x): + """ + x: a 3-dimensional tensor in tdnn-based architecture (B,F,T) + or a 4-dimensional tensor in resnet architecture (B,C,F,T) + 0-dim: batch-dimension, last-dim: time-dimension (frame-dimension) + """ + if len(x.shape) == 4: + x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3]) + assert len(x.shape) == 3 + + if self.global_context_att: + context_mean = torch.mean(x, dim=-1, keepdim=True).expand_as(x) + context_std = torch.sqrt( + torch.var(x, dim=-1, keepdim=True) + 1e-10).expand_as(x) + x_in = torch.cat((x, context_mean, context_std), dim=1) + else: + x_in = x + + # DON'T use ReLU here! ReLU may be hard to converge. + alpha = torch.tanh( + self.linear1(x_in)) # alpha = F.relu(self.linear1(x_in)) + alpha = torch.softmax(self.linear2(alpha), dim=2) + mean = torch.sum(alpha * x, dim=2) + var = torch.sum(alpha * (x**2), dim=2) - mean**2 + std = torch.sqrt(var.clamp(min=1e-10)) + return torch.cat([mean, std], dim=1) diff --git a/runtime/Lib/site_packages/models/audio/sv/rdino.py b/runtime/Lib/site_packages/models/audio/sv/rdino.py new file mode 100644 index 0000000000000000000000000000000000000000..0d51ee7a8cb50d57da3f29ddc9a73bcc7f955e9b --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/rdino.py @@ -0,0 +1,573 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" This ECAPA-TDNN implementation is adapted from https://github.com/speechbrain/speechbrain. + RDINOHead implementation is adapted from DINO framework. +""" +import math +import os +from typing import Any, Dict, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks + + +def length_to_mask(length, max_len=None, dtype=None, device=None): + assert len(length.shape) == 1 + + if max_len is None: + max_len = length.max().long().item() + mask = torch.arange( + max_len, device=length.device, dtype=length.dtype).expand( + len(length), max_len) < length.unsqueeze(1) + + if dtype is None: + dtype = length.dtype + + if device is None: + device = length.device + + mask = torch.as_tensor(mask, dtype=dtype, device=device) + return mask + + +def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int): + if stride > 1: + n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1) + L_out = stride * (n_steps - 1) + kernel_size * dilation + padding = [kernel_size // 2, kernel_size // 2] + + else: + L_out = (L_in - dilation * (kernel_size - 1) - 1) // stride + 1 + + padding = [(L_in - L_out) // 2, (L_in - L_out) // 2] + return padding + + +class Conv1d(nn.Module): + + def __init__( + self, + out_channels, + kernel_size, + in_channels, + stride=1, + dilation=1, + padding='same', + groups=1, + bias=True, + padding_mode='reflect', + ): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.padding = padding + self.padding_mode = padding_mode + + self.conv = nn.Conv1d( + in_channels, + out_channels, + self.kernel_size, + stride=self.stride, + dilation=self.dilation, + padding=0, + groups=groups, + bias=bias, + ) + + def forward(self, x): + if self.padding == 'same': + x = self._manage_padding(x, self.kernel_size, self.dilation, + self.stride) + + elif self.padding == 'causal': + num_pad = (self.kernel_size - 1) * self.dilation + x = F.pad(x, (num_pad, 0)) + + elif self.padding == 'valid': + pass + + else: + raise ValueError( + "Padding must be 'same', 'valid' or 'causal'. Got " + + self.padding) + + wx = self.conv(x) + + return wx + + def _manage_padding( + self, + x, + kernel_size: int, + dilation: int, + stride: int, + ): + L_in = x.shape[-1] + padding = get_padding_elem(L_in, stride, kernel_size, dilation) + x = F.pad(x, padding, mode=self.padding_mode) + + return x + + +class BatchNorm1d(nn.Module): + + def __init__( + self, + input_size, + eps=1e-05, + momentum=0.1, + ): + super().__init__() + self.norm = nn.BatchNorm1d( + input_size, + eps=eps, + momentum=momentum, + ) + + def forward(self, x): + return self.norm(x) + + +class TDNNBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + dilation, + activation=nn.ReLU, + groups=1, + ): + super(TDNNBlock, self).__init__() + self.conv = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + dilation=dilation, + groups=groups, + ) + self.activation = activation() + self.norm = BatchNorm1d(input_size=out_channels) + + def forward(self, x): + return self.norm(self.activation(self.conv(x))) + + +class Res2NetBlock(torch.nn.Module): + + def __init__(self, + in_channels, + out_channels, + scale=8, + kernel_size=3, + dilation=1): + super(Res2NetBlock, self).__init__() + assert in_channels % scale == 0 + assert out_channels % scale == 0 + + in_channel = in_channels // scale + hidden_channel = out_channels // scale + + self.blocks = nn.ModuleList([ + TDNNBlock( + in_channel, + hidden_channel, + kernel_size=kernel_size, + dilation=dilation, + ) for i in range(scale - 1) + ]) + self.scale = scale + + def forward(self, x): + y = [] + for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)): + if i == 0: + y_i = x_i + elif i == 1: + y_i = self.blocks[i - 1](x_i) + else: + y_i = self.blocks[i - 1](x_i + y_i) + y.append(y_i) + y = torch.cat(y, dim=1) + return y + + +class SEBlock(nn.Module): + + def __init__(self, in_channels, se_channels, out_channels): + super(SEBlock, self).__init__() + + self.conv1 = Conv1d( + in_channels=in_channels, out_channels=se_channels, kernel_size=1) + self.relu = torch.nn.ReLU(inplace=True) + self.conv2 = Conv1d( + in_channels=se_channels, out_channels=out_channels, kernel_size=1) + self.sigmoid = torch.nn.Sigmoid() + + def forward(self, x, lengths=None): + L = x.shape[-1] + if lengths is not None: + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + total = mask.sum(dim=2, keepdim=True) + s = (x * mask).sum(dim=2, keepdim=True) / total + else: + s = x.mean(dim=2, keepdim=True) + + s = self.relu(self.conv1(s)) + s = self.sigmoid(self.conv2(s)) + + return s * x + + +class AttentiveStatisticsPooling(nn.Module): + + def __init__(self, channels, attention_channels=128, global_context=True): + super().__init__() + + self.eps = 1e-12 + self.global_context = global_context + if global_context: + self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) + else: + self.tdnn = TDNNBlock(channels, attention_channels, 1, 1) + self.tanh = nn.Tanh() + self.conv = Conv1d( + in_channels=attention_channels, + out_channels=channels, + kernel_size=1) + + def forward(self, x, lengths=None): + L = x.shape[-1] + + def _compute_statistics(x, m, dim=2, eps=self.eps): + mean = (m * x).sum(dim) + std = torch.sqrt( + (m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps)) + return mean, std + + if lengths is None: + lengths = torch.ones(x.shape[0], device=x.device) + + # Make binary mask of shape [N, 1, L] + mask = length_to_mask(lengths * L, max_len=L, device=x.device) + mask = mask.unsqueeze(1) + + # Expand the temporal context of the pooling layer by allowing the + # self-attention to look at global properties of the utterance. + if self.global_context: + # torch.std is unstable for backward computation + # https://github.com/pytorch/pytorch/issues/4320 + total = mask.sum(dim=2, keepdim=True).float() + mean, std = _compute_statistics(x, mask / total) + mean = mean.unsqueeze(2).repeat(1, 1, L) + std = std.unsqueeze(2).repeat(1, 1, L) + attn = torch.cat([x, mean, std], dim=1) + else: + attn = x + + # Apply layers + attn = self.conv(self.tanh(self.tdnn(attn))) + + # Filter out zero-paddings + attn = attn.masked_fill(mask == 0, float('-inf')) + + attn = F.softmax(attn, dim=2) + mean, std = _compute_statistics(x, attn) + # Append mean and std of the batch + pooled_stats = torch.cat((mean, std), dim=1) + pooled_stats = pooled_stats.unsqueeze(2) + + return pooled_stats + + +class SERes2NetBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + res2net_scale=8, + se_channels=128, + kernel_size=1, + dilation=1, + activation=torch.nn.ReLU, + groups=1, + ): + super().__init__() + self.out_channels = out_channels + self.tdnn1 = TDNNBlock( + in_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.res2net_block = Res2NetBlock(out_channels, out_channels, + res2net_scale, kernel_size, dilation) + self.tdnn2 = TDNNBlock( + out_channels, + out_channels, + kernel_size=1, + dilation=1, + activation=activation, + groups=groups, + ) + self.se_block = SEBlock(out_channels, se_channels, out_channels) + + self.shortcut = None + if in_channels != out_channels: + self.shortcut = Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + residual = x + if self.shortcut: + residual = self.shortcut(x) + + x = self.tdnn1(x) + x = self.res2net_block(x) + x = self.tdnn2(x) + x = self.se_block(x, lengths) + + return x + residual + + +class ECAPA_TDNN(nn.Module): + """An implementation of the speaker embedding model in a paper. + "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in + TDNN Based Speaker Verification" (https://arxiv.org/abs/2005.07143). + """ + + def __init__( + self, + input_size, + device='cpu', + lin_neurons=512, + activation=torch.nn.ReLU, + channels=[512, 512, 512, 512, 1536], + kernel_sizes=[5, 3, 3, 3, 1], + dilations=[1, 2, 3, 4, 1], + attention_channels=128, + res2net_scale=8, + se_channels=128, + global_context=True, + groups=[1, 1, 1, 1, 1], + ): + + super().__init__() + assert len(channels) == len(kernel_sizes) + assert len(channels) == len(dilations) + self.channels = channels + self.blocks = nn.ModuleList() + + # The initial TDNN layer + self.blocks.append( + TDNNBlock( + input_size, + channels[0], + kernel_sizes[0], + dilations[0], + activation, + groups[0], + )) + + # SE-Res2Net layers + for i in range(1, len(channels) - 1): + self.blocks.append( + SERes2NetBlock( + channels[i - 1], + channels[i], + res2net_scale=res2net_scale, + se_channels=se_channels, + kernel_size=kernel_sizes[i], + dilation=dilations[i], + activation=activation, + groups=groups[i], + )) + + # Multi-layer feature aggregation + self.mfa = TDNNBlock( + channels[-1], + channels[-1], + kernel_sizes[-1], + dilations[-1], + activation, + groups=groups[-1], + ) + + # Attentive Statistical Pooling + self.asp = AttentiveStatisticsPooling( + channels[-1], + attention_channels=attention_channels, + global_context=global_context, + ) + self.asp_bn = BatchNorm1d(input_size=channels[-1] * 2) + + # Final linear transformation + self.fc = Conv1d( + in_channels=channels[-1] * 2, + out_channels=lin_neurons, + kernel_size=1, + ) + + def forward(self, x, lengths=None): + """Returns the embedding vector. + + Arguments + --------- + x : torch.Tensor + Tensor of shape (batch, time, channel). + """ + x = x.transpose(1, 2) + + xl = [] + for layer in self.blocks: + try: + x = layer(x, lengths=lengths) + except TypeError: + x = layer(x) + xl.append(x) + + # Multi-layer feature aggregation + x = torch.cat(xl[1:], dim=1) + x = self.mfa(x) + + # Attentive Statistical Pooling + x = self.asp(x, lengths=lengths) + x = self.asp_bn(x) + + # Final linear transformation + x = self.fc(x) + + x = x.transpose(1, 2).squeeze(1) + return x + + +class RDINOHead(nn.Module): + + def __init__(self, + in_dim, + out_dim, + use_bn=False, + norm_last_layer=True, + nlayers=3, + hidden_dim=2048, + bottleneck_dim=256, + add_dim=8192): + super().__init__() + nlayers = max(nlayers, 1) + if nlayers == 1: + self.mlp = nn.Linear(in_dim, bottleneck_dim) + else: + layers = [nn.Linear(in_dim, hidden_dim)] + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + for _ in range(nlayers - 2): + layers.append(nn.Linear(hidden_dim, hidden_dim)) + if use_bn: + layers.append(nn.BatchNorm1d(hidden_dim)) + layers.append(nn.GELU()) + + layers.append(nn.Linear(hidden_dim, add_dim)) + self.mlp = nn.Sequential(*layers) + self.add_layer = nn.Linear(add_dim, bottleneck_dim) + self.apply(self._init_weights) + self.last_layer = nn.utils.weight_norm( + nn.Linear(bottleneck_dim, out_dim, bias=False)) + self.last_layer.weight_g.data.fill_(1) + if norm_last_layer: + self.last_layer.weight_g.requires_grad = False + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + torch.nn.init.trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward(self, x): + vicr_out = self.mlp(x) + x = self.add_layer(vicr_out) + x = nn.functional.normalize(x, dim=-1, p=2) + x = self.last_layer(x) + return vicr_out, x + + +class Combine(nn.Module): + + def __init__(self, backbone, head): + super(Combine, self).__init__() + self.backbone = backbone + self.head = head + + def forward(self, x): + x = self.backbone(x) + output = self.head(x) + return output + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.rdino_tdnn_sv) +class SpeakerVerification_RDINO(TorchModel): + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + if self.model_config['channel'] != 1024: + raise ValueError( + 'modelscope error: Currently only 1024-channel ecapa tdnn is supported.' + ) + + self.feature_dim = 80 + channels_config = [1024, 1024, 1024, 1024, 3072] + + self.embedding_model = ECAPA_TDNN( + self.feature_dim, channels=channels_config) + self.embedding_model = Combine(self.embedding_model, + RDINOHead(512, 65536, True)) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.eval() + + def forward(self, audio): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model.backbone(feature) + + return embedding + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + state_dict = torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device) + state_dict_tea = { + k.replace('module.', ''): v + for k, v in state_dict['teacher'].items() + } + self.embedding_model.load_state_dict(state_dict_tea, strict=True) diff --git a/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator.py b/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator.py new file mode 100644 index 0000000000000000000000000000000000000000..4926196e6ce2ac1e3d7690300f50a1b3dad0b6da --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator.py @@ -0,0 +1,327 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN import CAMPPlus +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class MultiHeadSelfAttention(nn.Module): + + def __init__(self, n_units, h=8, dropout=0.1): + super(MultiHeadSelfAttention, self).__init__() + self.linearQ = nn.Linear(n_units, n_units) + self.linearK = nn.Linear(n_units, n_units) + self.linearV = nn.Linear(n_units, n_units) + self.linearO = nn.Linear(n_units, n_units) + self.d_k = n_units // h + self.h = h + self.dropout = nn.Dropout(p=dropout) + self.att = None + + def forward(self, x, batch_size): + # x: (BT, F) + q = self.linearQ(x).reshape(batch_size, -1, self.h, self.d_k) + k = self.linearK(x).reshape(batch_size, -1, self.h, self.d_k) + v = self.linearV(x).reshape(batch_size, -1, self.h, self.d_k) + scores = torch.matmul(q.transpose(1, 2), k.permute( + 0, 2, 3, 1)) / np.sqrt(self.d_k) + # scores: (B, h, T, T) + self.att = F.softmax(scores, dim=3) + p_att = self.dropout(self.att) + # v : (B, T, h, d_k) + # p_att : (B, h, T, T) + x = torch.matmul(p_att, v.transpose(1, 2)) + # x : (B, h, T, d_k) + x = x.transpose(1, 2).reshape(-1, self.h * self.d_k) + return self.linearO(x) + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, n_units, d_units, dropout): + super(PositionwiseFeedForward, self).__init__() + self.linear1 = nn.Linear(n_units, d_units) + self.linear2 = nn.Linear(d_units, n_units) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x): + return self.linear2(self.dropout(F.relu(self.linear1(x)))) + + +class PosEncoding(nn.Module): + + def __init__(self, max_seq_len, d_word_vec): + super(PosEncoding, self).__init__() + pos_enc = np.array([[ + pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) + for j in range(d_word_vec) + ] for pos in range(max_seq_len)]) + pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2]) + pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2]) + pad_row = np.zeros([1, d_word_vec]) + pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32) + + self.pos_enc = torch.nn.Embedding(max_seq_len + 1, d_word_vec) + self.pos_enc.weight = torch.nn.Parameter( + torch.from_numpy(pos_enc), requires_grad=False) + + def forward(self, input_len): + max_len = torch.max(input_len) + input_pos = torch.LongTensor([ + list(range(1, len + 1)) + [0] * (max_len - len) + for len in input_len + ]) + + input_pos = input_pos.to(list(self.pos_enc.parameters())[0].device) + return self.pos_enc(input_pos) + + +class TransformerEncoder(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: [B, num_anchors, T, n_in] + bs, num, tframe, dim = x.size() + x = x.reshape(bs * num, tframe, -1) # [B*num_anchors, T, dim] + # x: (B, T, F) ... batch, time, (mel)freq + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, F, T) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + output = output.reshape(bs, num, tframe, + -1) # [B, num_anchors, T, dim] + return output + + +class TransformerEncoder_out(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder_out, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: (B, T, F) + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, T, F) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + return output + + +class OutLayer(nn.Module): + + def __init__(self, n_units=256, num_anchors=2): + super(OutLayer, self).__init__() + self.combine = TransformerEncoder_out(num_anchors * n_units, n_units) + self.out_linear = nn.Linear(n_units // num_anchors, 1) + + def forward(self, input): + # input: [B, num_anchors, T, dim] + bs, num, tframe, dim = input.size() + output = input.permute(0, 2, 1, + 3).reshape(bs, tframe, + -1) # [Bs, t, num_anchors*dim] + output = self.combine(output) # [Bs, t, n_units] + output = output.reshape( + bs, tframe, num, -1) # [Bs, t, num_anchors, n_units//num_anchors] + output = self.out_linear(output).squeeze(-1) # [Bs, t, num_anchors] + + return output + + +class TransformerDetector(nn.Module): + + def __init__(self, + frame_dim=512, + anchor_dim=192, + hidden_dim=256, + max_seq_len=1000): + super(TransformerDetector, self).__init__() + self.detection = TransformerEncoder( + idim=frame_dim + anchor_dim, n_units=hidden_dim) + self.output = OutLayer(n_units=hidden_dim) + self.pos_enc = PosEncoding(max_seq_len, hidden_dim) + + def forward(self, feats, anchors): + # feats: [1, t, fdim] + num_frames = feats.shape[1] + num_anchors = anchors.shape[1] + bs = feats.shape[0] + feats = feats.unsqueeze(1).repeat( + 1, num_anchors, 1, 1) # shape: [Bs, num_anchors, t, fdim] + anchors = anchors.unsqueeze(2).repeat( + 1, 1, num_frames, 1) # shape: [Bs, num_anchors, t, xdim] + sd_in = torch.cat((feats, anchors), + dim=-1) # shape: [Bs, num_anchors, t, fdim+xdim] + sd_out = self.detection(sd_in) # shape: [Bs, num_anchors, t, sd_dim] + + # pos + pos_emb = self.pos_enc(torch.tensor([num_frames] * (bs * num_anchors))) + pos_emb = pos_emb.reshape(bs, num_anchors, num_frames, -1) + sd_out += pos_emb + + # output + output = self.output(sd_out) # shape: [Bs, t, num_anchors] + + return output + + +@MODELS.register_module(Tasks.speaker_diarization, module_name=Models.scl_sd) +class SpeakerChangeLocatorTransformer(TorchModel): + r"""A speaekr change locator using the transformer architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.feature_dim = self.model_config['fbank_dim'] + frame_size = self.model_config['frame_size'] + anchor_size = self.model_config['anchor_size'] + self.device = create_device(kwargs['device']) + + self.encoder = CAMPPlus(self.feature_dim, output_level='frame') + self.backend = TransformerDetector( + frame_dim=frame_size, anchor_dim=anchor_size) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self.__load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.to(self.device) + self.backend.to(self.device) + self.encoder.eval() + self.backend.eval() + + def forward(self, audio, anchors): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if isinstance(anchors, np.ndarray): + anchors = torch.from_numpy(anchors) + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + assert len( + anchors.shape + ) == 3 and anchors.shape[0] == 1 and anchors.shape[ + 1] == 2, 'modelscope error: the shape of input anchors to model needs to be [1, 2, D]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + frame_state = self.encoder(feature.to(self.device)) + output = self.backend(frame_state, anchors.to(self.device)) + output = output.squeeze(0).detach().cpu().sigmoid() + + time_scale_factor = int(np.ceil(feature.shape[1] / output.shape[0])) + output = output.unsqueeze(1).expand(-1, time_scale_factor, + -1).reshape(-1, output.shape[-1]) + return output + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point( + self, + pretrained_encoder, + pretrained_backend, + ): + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=torch.device('cpu'))) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=torch.device('cpu'))) diff --git a/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator_xvector.py b/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator_xvector.py new file mode 100644 index 0000000000000000000000000000000000000000..72e7b3d01431332aec273611b31bac1da4bbc05b --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/speaker_change_locator_xvector.py @@ -0,0 +1,329 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.TDNN import Xvector +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class MultiHeadSelfAttention(nn.Module): + + def __init__(self, n_units, h=8, dropout=0.1): + super(MultiHeadSelfAttention, self).__init__() + self.linearQ = nn.Linear(n_units, n_units) + self.linearK = nn.Linear(n_units, n_units) + self.linearV = nn.Linear(n_units, n_units) + self.linearO = nn.Linear(n_units, n_units) + self.d_k = n_units // h + self.h = h + self.dropout = nn.Dropout(p=dropout) + self.att = None + + def forward(self, x, batch_size): + # x: (BT, F) + q = self.linearQ(x).reshape(batch_size, -1, self.h, self.d_k) + k = self.linearK(x).reshape(batch_size, -1, self.h, self.d_k) + v = self.linearV(x).reshape(batch_size, -1, self.h, self.d_k) + scores = torch.matmul(q.transpose(1, 2), k.permute( + 0, 2, 3, 1)) / np.sqrt(self.d_k) + # scores: (B, h, T, T) + self.att = F.softmax(scores, dim=3) + p_att = self.dropout(self.att) + # v : (B, T, h, d_k) + # p_att : (B, h, T, T) + x = torch.matmul(p_att, v.transpose(1, 2)) + # x : (B, h, T, d_k) + x = x.transpose(1, 2).reshape(-1, self.h * self.d_k) + return self.linearO(x) + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, n_units, d_units, dropout): + super(PositionwiseFeedForward, self).__init__() + self.linear1 = nn.Linear(n_units, d_units) + self.linear2 = nn.Linear(d_units, n_units) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x): + return self.linear2(self.dropout(F.relu(self.linear1(x)))) + + +class PosEncoding(nn.Module): + + def __init__(self, max_seq_len, d_word_vec): + super(PosEncoding, self).__init__() + pos_enc = np.array([[ + pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) + for j in range(d_word_vec) + ] for pos in range(max_seq_len)]) + pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2]) + pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2]) + pad_row = np.zeros([1, d_word_vec]) + pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32) + + self.pos_enc = torch.nn.Embedding(max_seq_len + 1, d_word_vec) + self.pos_enc.weight = torch.nn.Parameter( + torch.from_numpy(pos_enc), requires_grad=False) + + def forward(self, input_len): + max_len = torch.max(input_len) + input_pos = torch.LongTensor([ + list(range(1, len + 1)) + [0] * (max_len - len) + for len in input_len + ]) + + input_pos = input_pos.to(list(self.pos_enc.parameters())[0].device) + return self.pos_enc(input_pos) + + +class TransformerEncoder(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: [B, num_anchors, T, n_in] + bs, num, tframe, dim = x.size() + x = x.reshape(bs * num, tframe, -1) # [B*num_anchors, T, dim] + # x: (B, T, F) ... batch, time, (mel)freq + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, F, T) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + output = output.reshape(bs, num, tframe, + -1) # [B, num_anchors, T, dim] + return output + + +class TransformerEncoder_out(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder_out, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: (B, T, F) + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, T, F) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + return output + + +class OutLayer(nn.Module): + + def __init__(self, n_units=256, num_anchors=2): + super(OutLayer, self).__init__() + self.rnn_combine = TransformerEncoder_out(num_anchors * n_units, + n_units) + self.out_linear = nn.Linear(n_units // num_anchors, 1) + + def forward(self, input): + # input: [B, num_anchors, T, dim] + bs, num, tframe, dim = input.size() + output = input.permute(0, 2, 1, + 3).reshape(bs, tframe, + -1) # [Bs, t, num_anchors*dim] + output = self.rnn_combine(output) # [Bs, t, n_units] + output = output.reshape( + bs, tframe, num, -1) # [Bs, t, num_anchors, n_units//num_anchors] + output = self.out_linear(output).squeeze(-1) # [Bs, t, num_anchors] + + return output + + +class TransformerDetector(nn.Module): + + def __init__(self, + frame_dim=512, + anchor_dim=192, + hidden_dim=256, + max_seq_len=500): + super(TransformerDetector, self).__init__() + self.detection = TransformerEncoder( + idim=frame_dim + anchor_dim, n_units=hidden_dim) + self.output = OutLayer(n_units=hidden_dim) + self.pos_enc = PosEncoding(max_seq_len, hidden_dim) + + def forward(self, feats, anchors): + # feats: [1, t, fdim] + num_frames = feats.shape[1] + num_anchors = anchors.shape[1] + bs = feats.shape[0] + feats = feats.unsqueeze(1).repeat( + 1, num_anchors, 1, 1) # shape: [Bs, num_anchors, t, fdim] + anchors = anchors.unsqueeze(2).repeat( + 1, 1, num_frames, 1) # shape: [Bs, num_anchors, t, xdim] + sd_in = torch.cat((feats, anchors), + dim=-1) # shape: [Bs, num_anchors, t, fdim+xdim] + sd_out = self.detection(sd_in) # shape: [Bs, num_anchors, t, sd_dim] + + # pos + pos_emb = self.pos_enc(torch.tensor([num_frames] * (bs * num_anchors))) + pos_emb = pos_emb.reshape(bs, num_anchors, num_frames, -1) + sd_out += pos_emb + + # output + output = self.output(sd_out) # shape: [Bs, t, num_anchors] + + return output + + +@MODELS.register_module( + Tasks.speaker_diarization, module_name=Models.scl_sd_xvector) +class SpeakerChangeLocatorTransformer(TorchModel): + r"""A speaekr change locator using the transformer architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.feature_dim = self.model_config['fbank_dim'] + frame_size = self.model_config['frame_size'] + anchor_size = self.model_config['anchor_size'] + self.device = create_device(kwargs['device']) + + self.encoder = Xvector(in_channels=self.feature_dim) + self.backend = TransformerDetector( + frame_dim=frame_size, anchor_dim=anchor_size) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self.__load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.to(self.device) + self.backend.to(self.device) + self.encoder.eval() + self.backend.eval() + + def forward(self, audio, anchors): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if isinstance(anchors, np.ndarray): + anchors = torch.from_numpy(anchors) + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + assert len( + anchors.shape + ) == 3 and anchors.shape[0] == 1 and anchors.shape[ + 1] == 2, 'modelscope error: the shape of input anchors to model needs to be [1, 2, D]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + frame_state = self.encoder(feature.to(self.device)) + output = self.backend(frame_state, anchors.to(self.device)) + output = output.squeeze(0).detach().cpu().sigmoid() + + time_scale_factor = int(np.ceil(feature.shape[1] / output.shape[0])) + output = output.unsqueeze(1).expand(-1, time_scale_factor, + -1).reshape(-1, output.shape[-1]) + return output + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point( + self, + pretrained_encoder, + pretrained_backend, + ): + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=torch.device('cpu'))) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=torch.device('cpu'))) diff --git a/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_dialogue_detection.py b/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_dialogue_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..e1aaafb58d9acd8e2d1a67d3e605931308b5c48e --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_dialogue_detection.py @@ -0,0 +1,98 @@ +# copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Heads, Models, TaskModels +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS, MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.outputs import (AttentionTextClassificationModelOutput, + ModelOutputBase, OutputKeys) +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import parse_label_mapping + +logger = logging.get_logger() + + +@HEADS.register_module( + Tasks.speaker_diarization_dialogue_detection, + module_name=Heads.text_classification) +class TextClassificationHead(TorchHead): + + def __init__(self, + hidden_size=768, + classifier_dropout=0.1, + num_labels=None, + **kwargs): + super().__init__( + hidden_size=hidden_size, + classifier_dropout=classifier_dropout, + num_labels=num_labels, + ) + assert num_labels is not None + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(hidden_size, num_labels) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + pooler_output = inputs.pooler_output + pooler_output = self.dropout(pooler_output) + logits = self.classifier(pooler_output) + loss = None + if labels is not None: + loss = self.compute_loss(logits, labels) + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def compute_loss(self, logits: torch.Tensor, labels) -> torch.Tensor: + return F.cross_entropy(logits, labels) + + +@MODELS.register_module( + Tasks.speaker_diarization_dialogue_detection, + module_name=TaskModels.text_classification) +class ModelForTextClassification(EncoderModel): + task = Tasks.text_classification + + head_type = Heads.text_classification + + def __init__(self, model_dir: str, *args, **kwargs): + # get the num_labels from label_mapping.json + self.id2label = {} + + # get the num_labels + num_labels = kwargs.get('num_labels') + if num_labels is None: + label2id = parse_label_mapping(model_dir) + if label2id is not None and len(label2id) > 0: + num_labels = len(label2id) + self.id2label = {id: label for label, id in label2id.items()} + kwargs['num_labels'] = num_labels + super().__init__(model_dir, *args, **kwargs) + + def parse_head_cfg(self): + head_cfg = super().parse_head_cfg() + if hasattr(head_cfg, 'classifier_dropout'): + head_cfg['classifier_dropout'] = ( + head_cfg.classifier_dropout if head_cfg['classifier_dropout'] + is not None else head_cfg.hidden_dropout_prob) + else: + head_cfg['classifier_dropout'] = head_cfg.hidden_dropout_prob + head_cfg['num_labels'] = self.config.num_labels + return head_cfg + + +@MODELS.register_module( + Tasks.speaker_diarization_dialogue_detection, module_name=Models.bert) +class BertForSequenceClassification(ModelForTextClassification): + base_model_type = 'bert' diff --git a/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_semantic_speaker_turn_detection.py b/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_semantic_speaker_turn_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..b7b14b08b960b183ca9ecf1b4bb92dcd87e74a48 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/sv/speaker_diarization_semantic_speaker_turn_detection.py @@ -0,0 +1,137 @@ +# copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Heads, Models, TaskModels +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS, MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.outputs import (AttentionTokenClassificationModelOutput, + ModelOutputBase, OutputKeys) +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import parse_label_mapping + +logger = logging.get_logger() + + +@HEADS.register_module( + Tasks.speaker_diarization_semantic_speaker_turn_detection, + module_name=Heads.token_classification) +class TokenClassificationHead(TorchHead): + + def __init__(self, + hidden_size=768, + classifier_dropout=0.1, + num_labels=None, + **kwargs): + super().__init__( + num_labels=num_labels, + classifier_dropout=classifier_dropout, + hidden_size=hidden_size, + ) + assert num_labels is not None + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(hidden_size, num_labels) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + sequence_output = inputs.last_hidden_state + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + loss = None + if labels is not None: + loss = self.compute_loss(logits, attention_mask, labels) + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions) + + def compute_loss(self, logits: torch.Tensor, attention_mask, + labels) -> torch.Tensor: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + return loss + + +@MODELS.register_module( + Tasks.speaker_diarization_semantic_speaker_turn_detection, + module_name=TaskModels.token_classification) +class ModelForTokenClassification(EncoderModel): + task = Tasks.token_classification + head_type = Heads.token_classification + base_model_prefix = 'bert' + override_base_model_prefix = True + + def __init__(self, model_dir: str, *args, **kwargs): + self.id2label = {} + + num_labels = kwargs.get('num_labels') + if num_labels is None: + label2id = parse_label_mapping(model_dir) + if label2id is not None and len(label2id) > 0: + num_labels = len(label2id) + self.id2label = {id: label for label, id in label2id.items()} + kwargs['num_labels'] = num_labels + super(ModelForTokenClassification, + self).__init__(model_dir, *args, **kwargs) + + def parse_head_cfg(self): + head_cfg = super().parse_head_cfg() + if hasattr(head_cfg, 'classifier_dropout'): + head_cfg['classifier_dropout'] = ( + head_cfg.classifier_dropout if head_cfg['classifier_dropout'] + is not None else head_cfg.hidden_dropout_prob) + else: + head_cfg['classifier_dropout'] = head_cfg.hidden_dropout_prob + head_cfg['num_labels'] = self.config.num_labels + return head_cfg + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + offset_mapping=None, + label_mask=None, + **kwargs): + kwargs['offset_mapping'] = offset_mapping + kwargs['label_mask'] = label_mask + outputs = super().forward(input_ids, attention_mask, token_type_ids, + position_ids, head_mask, inputs_embeds, + labels, output_attentions, + output_hidden_states, **kwargs) + + outputs.offset_mapping = offset_mapping + outputs.label_mask = label_mask + + return outputs + + +@MODELS.register_module( + Tasks.speaker_diarization_semantic_speaker_turn_detection, + module_name=Models.bert) +class BertForTokenClassification(ModelForTokenClassification): + base_model_type = 'bert' diff --git a/runtime/Lib/site_packages/models/audio/tts/__init__.py b/runtime/Lib/site_packages/models/audio/tts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8af35c5a34e9fc58b2abdabf02174cc6ebb8e70a --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/tts/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .sambert_hifi import SambertHifigan + +else: + _import_structure = {'sambert_hifi': ['SambertHifigan']} + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/audio/tts/sambert_hifi.py b/runtime/Lib/site_packages/models/audio/tts/sambert_hifi.py new file mode 100644 index 0000000000000000000000000000000000000000..6df9ec97aaa6e1f5f140f1f0569d167f126213e5 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/tts/sambert_hifi.py @@ -0,0 +1,275 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import datetime +import os +import shutil +import wave +import zipfile + +import json +import matplotlib.pyplot as plt +import numpy as np +import yaml + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.utils.audio.audio_utils import (TtsCustomParams, TtsTrainType, + ndarray_pcm_to_wav) +from modelscope.utils.audio.tts_exceptions import ( + TtsFrontendInitializeFailedException, + TtsFrontendLanguageTypeInvalidException, TtsModelConfigurationException, + TtsVoiceNotExistsException) +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .voice import Voice + +__all__ = ['SambertHifigan'] + +logger = get_logger() + + +@MODELS.register_module( + Tasks.text_to_speech, module_name=Models.sambert_hifigan) +class SambertHifigan(Model): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.sample_rate = kwargs.get('sample_rate', 16000) + self.is_train = False + if 'is_train' in kwargs: + is_train = kwargs['is_train'] + if isinstance(is_train, bool): + self.is_train = is_train + # check legacy modelcard + self.ignore_mask = False + if 'am' in kwargs: + if 'linguistic_unit' in kwargs['am']: + self.ignore_mask = not kwargs['am']['linguistic_unit'].get( + 'has_mask', True) + self.voices, self.voice_cfg, self.lang_type = self.load_voice( + model_dir, kwargs.get('custom_ckpt', {})) + if len(self.voices) == 0 or len(self.voice_cfg.get('voices', [])) == 0: + raise TtsVoiceNotExistsException('modelscope error: voices empty') + if self.voice_cfg['voices']: + self.default_voice_name = self.voice_cfg['voices'][0] + else: + raise TtsVoiceNotExistsException( + 'modelscope error: voices is empty in voices.json') + # initialize frontend + import ttsfrd + frontend = ttsfrd.TtsFrontendEngine() + zip_file = os.path.join(model_dir, 'resource.zip') + self.res_path = os.path.join(model_dir, 'resource') + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(model_dir) + if not frontend.initialize(self.res_path): + raise TtsFrontendInitializeFailedException( + 'modelscope error: resource invalid: {}'.format(self.res_path)) + if not frontend.set_lang_type(self.lang_type): + raise TtsFrontendLanguageTypeInvalidException( + 'modelscope error: language type invalid: {}'.format( + self.lang_type)) + self.frontend = frontend + + def build_voice_from_custom(self, model_dir, custom_ckpt): + necessary_files = (TtsCustomParams.VOICE_NAME, TtsCustomParams.AM_CKPT, + TtsCustomParams.VOC_CKPT, TtsCustomParams.AM_CONFIG, + TtsCustomParams.VOC_CONFIG) + voices = {} + voices_cfg = {} + lang_type = 'PinYin' + for k in necessary_files: + if k not in custom_ckpt: + raise TtsModelNotExistsException( + f'custom ckpt must have: {necessary_files}') + voice_name = custom_ckpt[TtsCustomParams.VOICE_NAME] + voice = Voice( + voice_name=voice_name, + voice_path=model_dir, + custom_ckpt=custom_ckpt, + ignore_mask=self.ignore_mask, + is_train=self.is_train) + voices[voice_name] = voice + voices_cfg['voices'] = [voice_name] + lang_type = voice.lang_type + return voices, voices_cfg, lang_type + + def load_voice(self, model_dir, custom_ckpt): + voices = {} + voices_path = os.path.join(model_dir, 'voices') + voices_json_path = os.path.join(voices_path, 'voices.json') + lang_type = 'PinYin' + if len(custom_ckpt) != 0: + return self.build_voice_from_custom(model_dir, custom_ckpt) + if not os.path.exists(voices_path) or not os.path.exists( + voices_json_path): + return voices, {}, lang_type + with open(voices_json_path, 'r', encoding='utf-8') as f: + voice_cfg = json.load(f) + if 'voices' not in voice_cfg: + return voices, {}, lang_type + for name in voice_cfg['voices']: + voice_path = os.path.join(voices_path, name) + if not os.path.exists(voice_path): + continue + voices[name] = Voice( + name, + voice_path, + ignore_mask=self.ignore_mask, + is_train=self.is_train) + lang_type = voices[name].lang_type + return voices, voice_cfg, lang_type + + def save_voices(self): + voices_json_path = os.path.join(self.model_dir, 'voices', + 'voices.json') + if os.path.exists(voices_json_path): + os.remove(voices_json_path) + save_voices = {} + save_voices['voices'] = [] + for k in self.voices.keys(): + save_voices['voices'].append(k) + with open(voices_json_path, 'w', encoding='utf-8') as f: + json.dump(save_voices, f) + + def get_voices(self): + return self.voices, self.voice_cfg + + def create_empty_voice(self, voice_name, audio_config, am_config_path, + voc_config_path): + voice_name_path = os.path.join(self.model_dir, 'voices', voice_name) + if os.path.exists(voice_name_path): + shutil.rmtree(voice_name_path) + os.makedirs(voice_name_path, exist_ok=True) + if audio_config and os.path.exists(audio_config) and os.path.isfile( + audio_config): + shutil.copy(audio_config, voice_name_path) + voice_am_path = os.path.join(voice_name_path, 'am') + voice_voc_path = os.path.join(voice_name_path, 'voc') + if am_config_path and os.path.exists( + am_config_path) and os.path.isfile(am_config): + am_config_name = os.path.join(voice_am_path, 'config.yaml') + shutil.copy(am_config_path, am_config_name) + if voc_config_path and os.path.exists( + voc_config_path) and os.path.isfile(voc_config): + voc_config_name = os.path.join(voice_am_path, 'config.yaml') + shutil.copy(voc_config_path, voc_config_name) + am_ckpt_path = os.path.join(voice_am_path, 'ckpt') + voc_ckpt_path = os.path.join(voice_voc_path, 'ckpt') + os.makedirs(am_ckpt_path, exist_ok=True) + os.makedirs(voc_ckpt_path, exist_ok=True) + self.voices[voice_name] = Voice( + voice_name=voice_name, + voice_path=voice_name_path, + allow_empty=True) + + def get_voice_audio_config_path(self, voice): + if voice not in self.voices: + return '' + return self.voices[voice].audio_config + + def get_voice_se_model_path(self, voice): + if voice not in self.voices: + return '' + if self.voices[voice].se_enable: + return self.voices[voice].se_model_path + else: + return '' + + def get_voice_lang_path(self, voice): + if voice not in self.voices: + return '' + return self.voices[voice].lang_dir + + def synthesis_one_sentences(self, voice_name, text): + if voice_name not in self.voices: + raise TtsVoiceNotExistsException( + f'modelscope error: Voice {voice_name} not exists') + return self.voices[voice_name].forward(text) + + def train(self, + voice, + dirs, + train_type, + configs_path_dict=None, + ignore_pretrain=False, + create_if_not_exists=False, + hparam=None): + plt.set_loglevel('info') + work_dir = dirs['work_dir'] + am_dir = dirs['am_tmp_dir'] + voc_dir = dirs['voc_tmp_dir'] + data_dir = dirs['data_dir'] + target_voice = None + if voice not in self.voices: + if not create_if_not_exists: + raise TtsVoiceNotExistsException( + f'modelscope error: Voice {voice_name} not exists') + am_config_path = configs_path_dict.get('am_config', + 'am_config.yaml') + voc_config_path = configs_path_dict.get('voc_config', + 'voc_config.yaml') + if TtsTrainType.TRAIN_TYPE_SAMBERT in train_type and not am_config: + raise TtsTrainingCfgNotExistsException( + 'training new voice am with empty am_config') + if TtsTrainType.TRAIN_TYPE_VOC in train_type and not voc_config: + raise TtsTrainingCfgNotExistsException( + 'training new voice voc with empty voc_config') + else: + target_voice = self.voices[voice] + am_config_path = target_voice.am_config_path + voc_config_path = target_voice.voc_config_path + if configs_path_dict: + if 'am_config' in configs_path_dict: + am_override = configs_path_dict['am_config'] + if os.path.exists(am_override): + am_config_path = am_override + if 'voc_config' in configs_path_dict: + voc_override = configs_path_dict['voc_config'] + if os.path.exists(voc_override): + voc_config_path = voc_override + + logger.info('Start training....') + if TtsTrainType.TRAIN_TYPE_SAMBERT in train_type: + logger.info('Start SAMBERT training...') + totaltime = datetime.datetime.now() + hparams = train_type[TtsTrainType.TRAIN_TYPE_SAMBERT] + target_voice.train_sambert(work_dir, am_dir, data_dir, + am_config_path, ignore_pretrain, + hparams) + totaltime = datetime.datetime.now() - totaltime + logger.info('SAMBERT training spent: {:.2f} hours\n'.format( + totaltime.total_seconds() / 3600.0)) + else: + logger.info('skip SAMBERT training...') + + if TtsTrainType.TRAIN_TYPE_VOC in train_type: + logger.info('Start HIFIGAN training...') + totaltime = datetime.datetime.now() + hparams = train_type[TtsTrainType.TRAIN_TYPE_VOC] + target_voice.train_hifigan(work_dir, voc_dir, data_dir, + voc_config_path, ignore_pretrain, + hparams) + totaltime = datetime.datetime.now() - totaltime + logger.info('HIFIGAN training spent: {:.2f} hours\n'.format( + totaltime.total_seconds() / 3600.0)) + else: + logger.info('skip HIFIGAN training...') + + def forward(self, text: str, voice_name: str = None): + voice = self.default_voice_name + if voice_name is not None: + voice = voice_name + result = self.frontend.gen_tacotron_symbols(text) + texts = [s for s in result.splitlines() if s != ''] + audio_total = np.empty((0), dtype='int16') + for line in texts: + line = line.strip().split('\t') + audio = self.synthesis_one_sentences(voice, line[1]) + audio = 32768.0 * audio + audio_total = np.append(audio_total, audio.astype('int16'), axis=0) + return ndarray_pcm_to_wav(self.sample_rate, audio_total) diff --git a/runtime/Lib/site_packages/models/audio/tts/voice.py b/runtime/Lib/site_packages/models/audio/tts/voice.py new file mode 100644 index 0000000000000000000000000000000000000000..ed9edf4332871db555890f92659a2aa195d8e4a2 --- /dev/null +++ b/runtime/Lib/site_packages/models/audio/tts/voice.py @@ -0,0 +1,653 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import pickle as pkl +import time +from collections import OrderedDict +from threading import Lock + +import json +import numpy as np +import torch +import yaml +from kantts.datasets.dataset import get_am_datasets, get_voc_datasets +from kantts.models import model_builder +from kantts.train.loss import criterion_builder +from kantts.train.trainer import GAN_Trainer, Sambert_Trainer, distributed_init +from kantts.utils.ling_unit.ling_unit import KanTtsLinguisticUnit +from torch.utils.data import DataLoader + +from modelscope.utils.audio.audio_utils import TtsCustomParams +from modelscope.utils.audio.tts_exceptions import ( + TtsModelConfigurationException, TtsModelNotExistsException) +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def denorm_f0(mel, + f0_threshold=30, + uv_threshold=0.6, + norm_type='mean_std', + f0_feature=None): + if norm_type == 'mean_std': + f0_mvn = f0_feature + + f0 = mel[:, -2] + uv = mel[:, -1] + + uv[uv < uv_threshold] = 0.0 + uv[uv >= uv_threshold] = 1.0 + + f0 = f0 * f0_mvn[1:, :] + f0_mvn[0:1, :] + f0[f0 < f0_threshold] = f0_threshold + + mel[:, -2] = f0 + mel[:, -1] = uv + else: # global + f0_global_max_min = f0_feature + + f0 = mel[:, -2] + uv = mel[:, -1] + + uv[uv < uv_threshold] = 0.0 + uv[uv >= uv_threshold] = 1.0 + + f0 = f0 * (f0_global_max_min[0] + - f0_global_max_min[1]) + f0_global_max_min[1] + f0[f0 < f0_threshold] = f0_threshold + + mel[:, -2] = f0 + mel[:, -1] = uv + + return mel + + +def binarize(mel, threshold=0.6): + # vuv binarize + res_mel = mel.clone() + index = torch.where(mel[:, -1] < threshold)[0] + res_mel[:, -1] = 1.0 + res_mel[:, -1][index] = 0.0 + return res_mel + + +class Voice: + + def __init__(self, + voice_name, + voice_path=None, + custom_ckpt={}, + ignore_mask=True, + is_train=False): + self.voice_name = voice_name + self.voice_path = voice_path + self.ignore_mask = ignore_mask + self.is_train = is_train + if not torch.cuda.is_available(): + self.device = torch.device('cpu') + self.distributed = False + else: + torch.backends.cudnn.benchmark = True + self.distributed, self.device, self.local_rank, self.world_size = distributed_init( + ) + + if len(custom_ckpt) != 0: + self.am_config_path = custom_ckpt[TtsCustomParams.AM_CONFIG] + self.voc_config_path = custom_ckpt[TtsCustomParams.VOC_CONFIG] + if not os.path.isabs(self.am_config_path): + self.am_config_path = os.path.join(voice_path, + self.am_config_path) + if not os.path.isabs(self.voc_config_path): + self.voc_config_path = os.path.join(voice_path, + self.voc_config_path) + am_ckpt = custom_ckpt[TtsCustomParams.AM_CKPT] + voc_ckpt = custom_ckpt[TtsCustomParams.VOC_CKPT] + if not os.path.isabs(am_ckpt): + am_ckpt = os.path.join(voice_path, am_ckpt) + if not os.path.isabs(voc_ckpt): + voc_ckpt = os.path.join(voice_path, voc_ckpt) + self.am_ckpts = self.scan_ckpt(am_ckpt) + self.voc_ckpts = self.scan_ckpt(voc_ckpt) + self.se_path = custom_ckpt.get(TtsCustomParams.SE_FILE, 'se.npy') + if not os.path.isabs(self.se_path): + self.se_path = os.path.join(voice_path, self.se_path) + self.se_model_path = custom_ckpt.get(TtsCustomParams.SE_MODEL, + 'se.onnx') + if not os.path.isabs(self.se_model_path): + self.se_model_path = os.path.join(voice_path, + self.se_model_path) + self.audio_config = custom_ckpt.get(TtsCustomParams.AUIDO_CONFIG, + 'audio_config.yaml') + if not os.path.isabs(self.audio_config): + self.audio_config = os.path.join(voice_path, self.audio_config) + self.mvn_path = custom_ckpt.get(TtsCustomParams.MVN_FILE, + 'mvn.npy') + if not os.path.isabs(self.mvn_path): + self.mvn_path = os.path.join(voice_path, self.mvn_path) + else: + self.audio_config = os.path.join(voice_path, 'audio_config.yaml') + self.am_config_path = os.path.join(voice_path, 'am', 'config.yaml') + self.voc_config_path = os.path.join(voice_path, 'voc', + 'config.yaml') + + self.se_path = os.path.join(voice_path, 'am', 'se.npy') + self.am_ckpts = self.scan_ckpt( + os.path.join(voice_path, 'am', 'ckpt')) + self.voc_ckpts = self.scan_ckpt( + os.path.join(voice_path, 'voc', 'ckpt')) + self.mvn_path = os.path.join(voice_path, 'am', 'mvn.npy') + self.se_model_path = os.path.join(voice_path, 'se', 'ckpt', + 'se.onnx') + + logger.info( + f'am_config={self.am_config_path} voc_config={self.voc_config_path}' + ) + logger.info(f'audio_config={self.audio_config}') + logger.info(f'am_ckpts={self.am_ckpts}') + logger.info(f'voc_ckpts={self.voc_ckpts}') + logger.info( + f'se_path={self.se_path} se_model_path={self.se_model_path}') + logger.info(f'mvn_path={self.mvn_path}') + + if not os.path.exists(self.am_config_path): + raise TtsModelConfigurationException( + 'modelscope error: am configuration not found') + if not os.path.exists(self.voc_config_path): + raise TtsModelConfigurationException( + 'modelscope error: voc configuration not found') + if len(self.am_ckpts) == 0: + raise TtsModelNotExistsException( + 'modelscope error: am model file not found') + if len(self.voc_ckpts) == 0: + raise TtsModelNotExistsException( + 'modelscope error: voc model file not found') + with open(self.am_config_path, 'r') as f: + self.am_config = yaml.load(f, Loader=yaml.Loader) + with open(self.voc_config_path, 'r') as f: + self.voc_config = yaml.load(f, Loader=yaml.Loader) + if 'linguistic_unit' not in self.am_config: + raise TtsModelConfigurationException( + 'no linguistic_unit in am config') + self.lang_type = self.am_config['linguistic_unit'].get( + 'language', 'PinYin') + self.model_loaded = False + self.lock = Lock() + self.ling_unit = KanTtsLinguisticUnit(self.am_config) + self.ling_unit_size = self.ling_unit.get_unit_size() + if self.ignore_mask: + target_set = set(('sy', 'tone', 'syllable_flag', 'word_segment', + 'emotion', 'speaker')) + for k, v in self.ling_unit_size.items(): + if k in target_set: + self.ling_unit_size[k] = v - 1 + + self.am_config['Model']['KanTtsSAMBERT']['params'].update( + self.ling_unit_size) + + self.se_enable = self.am_config['Model']['KanTtsSAMBERT'][ + 'params'].get('SE', False) + if self.se_enable and not self.is_train: + if not os.path.exists(self.se_path): + raise TtsModelConfigurationException( + f'se enabled but se_file:{self.se_path} not exists') + self.se = np.load(self.se_path) + + self.nsf_enable = self.am_config['Model']['KanTtsSAMBERT'][ + 'params'].get('NSF', False) + if self.nsf_enable and not self.is_train: + self.nsf_norm_type = self.am_config['Model']['KanTtsSAMBERT'][ + 'params'].get('nsf_norm_type', 'mean_std') + if self.nsf_norm_type == 'mean_std': + if not os.path.exists(self.mvn_path): + raise TtsModelNotExistsException( + f'f0_mvn_file: {self.mvn_path} not exists') + self.f0_feature = np.load(self.mvn_path) + else: # global + nsf_f0_global_minimum = self.am_config['Model'][ + 'KanTtsSAMBERT']['params'].get('nsf_f0_global_minimum', + 30.0) + nsf_f0_global_maximum = self.am_config['Model'][ + 'KanTtsSAMBERT']['params'].get('nsf_f0_global_maximum', + 730.0) + self.f0_feature = [ + nsf_f0_global_maximum, nsf_f0_global_minimum + ] + + def scan_ckpt(self, ckpt_path): + select_target = ckpt_path + input_not_dir = False + if not os.path.isdir(ckpt_path): + input_not_dir = True + ckpt_path = os.path.dirname(ckpt_path) + filelist = os.listdir(ckpt_path) + if len(filelist) == 0: + return {} + ckpts = {} + for filename in filelist: + # checkpoint_X.pth + if len(filename) - 15 <= 0: + continue + if filename[-4:] == '.pth' and filename[0:10] == 'checkpoint': + filename_prefix = filename.split('.')[0] + idx = int(filename_prefix.split('_')[-1]) + path = os.path.join(ckpt_path, filename) + if input_not_dir and path != select_target: + continue + ckpts[idx] = path + od = OrderedDict(sorted(ckpts.items())) + return od + + def load_am(self): + self.am_model, _, _ = model_builder(self.am_config, self.device) + self.am = self.am_model['KanTtsSAMBERT'] + state_dict = torch.load( + self.am_ckpts[next(reversed(self.am_ckpts))], + map_location=self.device) + self.am.load_state_dict(state_dict['model'], strict=False) + self.am.eval() + + def load_vocoder(self): + from kantts.models.hifigan.hifigan import Generator + self.voc_model = Generator( + **self.voc_config['Model']['Generator']['params']) + states = torch.load( + self.voc_ckpts[next(reversed(self.voc_ckpts))], + map_location=self.device) + self.voc_model.load_state_dict(states['model']['generator']) + if self.voc_config['Model']['Generator']['params']['out_channels'] > 1: + from kantts.models.pqmf import PQMF + self.voc_model = PQMF() + self.voc_model.remove_weight_norm() + self.voc_model.eval().to(self.device) + + def am_forward(self, symbol_seq): + with self.lock: + with torch.no_grad(): + inputs_feat_lst = self.ling_unit.encode_symbol_sequence( + symbol_seq) + inputs_feat_index = 0 + if self.ling_unit.using_byte(): + inputs_byte_index = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device)) + inputs_ling = torch.stack([inputs_byte_index], + dim=-1).unsqueeze(0) + else: + inputs_sy = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device)) + inputs_feat_index = inputs_feat_index + 1 + inputs_tone = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device)) + inputs_feat_index = inputs_feat_index + 1 + inputs_syllable = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device)) + inputs_feat_index = inputs_feat_index + 1 + inputs_ws = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device)) + inputs_ling = torch.stack( + [inputs_sy, inputs_tone, inputs_syllable, inputs_ws], + dim=-1).unsqueeze(0) + inputs_feat_index = inputs_feat_index + 1 + inputs_emo = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device).unsqueeze(0)) + inputs_feat_index = inputs_feat_index + 1 + if self.se_enable: + inputs_spk = ( + torch.from_numpy( + self.se.repeat( + len(inputs_feat_lst[inputs_feat_index]), + axis=0)).float().to( + self.device).unsqueeze(0)[:, :-1, :]) + else: + inputs_spk = ( + torch.from_numpy( + inputs_feat_lst[inputs_feat_index]).long().to( + self.device).unsqueeze(0)[:, :-1]) + inputs_len = (torch.zeros(1).to(self.device).long() + + inputs_emo.size(1) - 1) # minus 1 for "~" + res = self.am(inputs_ling[:, :-1, :], inputs_emo[:, :-1], + inputs_spk, inputs_len) + postnet_outputs = res['postnet_outputs'] + LR_length_rounded = res['LR_length_rounded'] + valid_length = int(LR_length_rounded[0].item()) + mel_post = postnet_outputs[0, :valid_length, :].cpu() + if self.nsf_enable: + mel_post = denorm_f0( + mel_post, + norm_type=self.nsf_norm_type, + f0_feature=self.f0_feature) + return mel_post + + def vocoder_forward(self, melspec): + with torch.no_grad(): + x = melspec.to(self.device) + if self.voc_model.nsf_enable: + x = binarize(x) + x = x.transpose(1, 0).unsqueeze(0) + y = self.voc_model(x) + if hasattr(self.voc_model, 'pqmf'): + y = self.voc_model.synthesis(y) + y = y.view(-1).cpu().numpy() + return y + + def train_sambert(self, + work_dir, + stage_dir, + data_dir, + config_path, + ignore_pretrain=False, + hparams=dict()): + logger.info('TRAIN SAMBERT....') + if len(self.am_ckpts) == 0: + raise TtsTrainingInvalidModelException( + 'resume pretrain but model is empty') + + from_steps = hparams.get('resume_from_steps', -1) + if from_steps < 0: + from_latest = hparams.get('resume_from_latest', True) + else: + from_latest = hparams.get('resume_from_latest', False) + train_steps = hparams.get('train_steps', 0) + + with open(self.audio_config, 'r') as f: + config = yaml.load(f, Loader=yaml.Loader) + with open(config_path, 'r') as f: + config.update(yaml.load(f, Loader=yaml.Loader)) + config.update(hparams) + + resume_from = None + if from_latest: + from_steps = next(reversed(self.am_ckpts)) + resume_from = self.am_ckpts[from_steps] + if not os.path.exists(resume_from): + raise TtsTrainingInvalidModelException( + f'latest model:{resume_from} not exists') + else: + if from_steps not in self.am_ckpts: + raise TtsTrainingInvalidModelException( + f'no such model from steps:{from_steps}') + else: + resume_from = self.am_ckpts[from_steps] + + if train_steps > 0: + train_max_steps = train_steps + from_steps + config['train_max_steps'] = train_max_steps + + logger.info(f'TRAINING steps: {train_max_steps}') + config['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime()) + from modelscope import __version__ + config['modelscope_version'] = __version__ + + with open(os.path.join(stage_dir, 'config.yaml'), 'w') as f: + yaml.dump(config, f, Dumper=yaml.Dumper, default_flow_style=None) + + for key, value in config.items(): + logger.info(f'{key} = {value}') + + if self.distributed: + config['rank'] = torch.distributed.get_rank() + config['distributed'] = True + + if self.se_enable: + valid_enable = False + valid_split_ratio = 0.00 + else: + valid_enable = True + valid_split_ratio = 0.02 + + fp_enable = config['Model']['KanTtsSAMBERT']['params'].get('FP', False) + meta_file = [ + os.path.join( + d, + 'raw_metafile.txt' if not fp_enable else 'fprm_metafile.txt') + for d in data_dir + ] + + train_dataset, valid_dataset = get_am_datasets( + meta_file, + data_dir, + config, + config['allow_cache'], + split_ratio=1.0 - valid_split_ratio) + + logger.info(f'The number of training files = {len(train_dataset)}.') + logger.info(f'The number of validation files = {len(valid_dataset)}.') + + sampler = {'train': None, 'valid': None} + + if self.distributed: + # setup sampler for distributed training + from torch.utils.data.distributed import DistributedSampler + + sampler['train'] = DistributedSampler( + dataset=train_dataset, + num_replicas=self.world_size, + shuffle=True, + ) + sampler['valid'] = DistributedSampler( + dataset=valid_dataset, + num_replicas=self.world_size, + shuffle=False, + ) if valid_enable else None + + train_dataloader = DataLoader( + train_dataset, + shuffle=False if self.distributed else True, + collate_fn=train_dataset.collate_fn, + batch_size=config['batch_size'], + num_workers=config['num_workers'], + sampler=sampler['train'], + pin_memory=config['pin_memory'], + ) + + valid_dataloader = DataLoader( + valid_dataset, + shuffle=False if self.distributed else True, + collate_fn=valid_dataset.collate_fn, + batch_size=config['batch_size'], + num_workers=config['num_workers'], + sampler=sampler['valid'], + pin_memory=config['pin_memory'], + ) if valid_enable else None + + ling_unit_size = train_dataset.ling_unit.get_unit_size() + + config['Model']['KanTtsSAMBERT']['params'].update(ling_unit_size) + model, optimizer, scheduler = model_builder(config, self.device, + self.local_rank, + self.distributed) + + criterion = criterion_builder(config, self.device) + + trainer = Sambert_Trainer( + config=config, + model=model, + optimizer=optimizer, + scheduler=scheduler, + criterion=criterion, + device=self.device, + sampler=sampler, + train_loader=train_dataloader, + valid_loader=valid_dataloader, + max_steps=train_max_steps, + save_dir=stage_dir, + save_interval=config['save_interval_steps'], + valid_interval=config['eval_interval_steps'], + log_interval=config['log_interval'], + grad_clip=config['grad_norm'], + ) + + if resume_from is not None: + trainer.load_checkpoint(resume_from, True, True) + logger.info(f'Successfully resumed from {resume_from}.') + + try: + trainer.train() + except (Exception, KeyboardInterrupt) as e: + logger.error(e, exc_info=True) + trainer.save_checkpoint( + os.path.join( + os.path.join(stage_dir, 'ckpt'), + f'checkpoint-{trainer.steps}.pth')) + logger.info( + f'Successfully saved checkpoint @ {trainer.steps}steps.') + + def train_hifigan(self, + work_dir, + stage_dir, + data_dir, + config_path, + ignore_pretrain=False, + hparams=dict()): + logger.info('TRAIN HIFIGAN....') + if len(self.voc_ckpts) == 0: + raise TtsTrainingInvalidModelException( + 'resume pretrain but model is empty') + + from_steps = hparams.get('resume_from_steps', -1) + if from_steps < 0: + from_latest = hparams.get('resume_from_latest', True) + else: + from_latest = hparams.get('resume_from_latest', False) + train_steps = hparams.get('train_steps', 0) + + with open(self.audio_config, 'r') as f: + config = yaml.load(f, Loader=yaml.Loader) + + with open(config_path, 'r') as f: + config.update(yaml.load(f, Loader=yaml.Loader)) + config.update(hparams) + + resume_from = None + if from_latest: + from_steps = next(reversed(self.voc_ckpts)) + resume_from = self.voc_ckpts[from_steps] + if not os.path.exists(resume_from): + raise TtsTrainingInvalidModelException( + f'latest model:{resume_from} not exists') + else: + if from_steps not in self.voc_ckpts: + raise TtsTrainingInvalidModelException( + f'no such model from steps:{from_steps}') + else: + resume_from = self.voc_ckpts[from_steps] + + if train_steps > 0: + train_max_steps = train_steps + config['train_max_steps'] = train_max_steps + + logger.info(f'TRAINING steps: {train_max_steps}') + logger.info(f'resume from: {resume_from}') + config['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', + time.localtime()) + from modelscope import __version__ + config['modelscope_version'] = __version__ + + with open(os.path.join(stage_dir, 'config.yaml'), 'w') as f: + yaml.dump(config, f, Dumper=yaml.Dumper, default_flow_style=None) + + for key, value in config.items(): + logger.info(f'{key} = {value}') + + train_dataset, valid_dataset = get_voc_datasets(config, data_dir) + + logger.info(f'The number of training files = {len(train_dataset)}.') + logger.info(f'The number of validation files = {len(valid_dataset)}.') + + sampler = {'train': None, 'valid': None} + if self.distributed: + # setup sampler for distributed training + from torch.utils.data.distributed import DistributedSampler + + sampler['train'] = DistributedSampler( + dataset=train_dataset, + num_replicas=self.world_size, + shuffle=True, + ) + sampler['valid'] = DistributedSampler( + dataset=valid_dataset, + num_replicas=self.world_size, + shuffle=False, + ) + + train_dataloader = DataLoader( + train_dataset, + shuffle=False if self.distributed else True, + collate_fn=train_dataset.collate_fn, + batch_size=config['batch_size'], + num_workers=config['num_workers'], + sampler=sampler['train'], + pin_memory=config['pin_memory'], + ) + + valid_dataloader = DataLoader( + valid_dataset, + shuffle=False if self.distributed else True, + collate_fn=valid_dataset.collate_fn, + batch_size=config['batch_size'], + num_workers=config['num_workers'], + sampler=sampler['valid'], + pin_memory=config['pin_memory'], + ) + + model, optimizer, scheduler = model_builder(config, self.device, + self.local_rank, + self.distributed) + + criterion = criterion_builder(config, self.device) + trainer = GAN_Trainer( + config=config, + model=model, + optimizer=optimizer, + scheduler=scheduler, + criterion=criterion, + device=self.device, + sampler=sampler, + train_loader=train_dataloader, + valid_loader=valid_dataloader, + max_steps=train_max_steps, + save_dir=stage_dir, + save_interval=config['save_interval_steps'], + valid_interval=config['eval_interval_steps'], + log_interval=config['log_interval_steps'], + ) + + if resume_from is not None: + trainer.load_checkpoint(resume_from) + logger.info(f'Successfully resumed from {resume_from}.') + + try: + trainer.train() + except (Exception, KeyboardInterrupt) as e: + logger.error(e, exc_info=True) + trainer.save_checkpoint( + os.path.join( + os.path.join(stage_dir, 'ckpt'), + f'checkpoint-{trainer.steps}.pth')) + logger.info( + f'Successfully saved checkpoint @ {trainer.steps}steps.') + + def forward(self, symbol_seq): + with self.lock: + if not self.model_loaded: + self.load_am() + self.load_vocoder() + self.model_loaded = True + return self.vocoder_forward(self.am_forward(symbol_seq)) diff --git a/runtime/Lib/site_packages/models/base/__init__.py b/runtime/Lib/site_packages/models/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b727354446ee8d4eee5de7aff61ec7cff969c1b --- /dev/null +++ b/runtime/Lib/site_packages/models/base/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from modelscope.utils.import_utils import is_torch_available +from .base_head import * # noqa F403 +from .base_model import * # noqa F403 + +if is_torch_available(): + from .base_torch_model import TorchModel + from .base_torch_head import TorchHead diff --git a/runtime/Lib/site_packages/models/base/base_head.py b/runtime/Lib/site_packages/models/base/base_head.py new file mode 100644 index 0000000000000000000000000000000000000000..a3cf1b071f05695559b52ce2beda36a48ada4e0d --- /dev/null +++ b/runtime/Lib/site_packages/models/base/base_head.py @@ -0,0 +1,40 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from abc import ABC, abstractmethod +from typing import Any, Dict, Union + +from modelscope.models.base.base_model import Model +from modelscope.utils.config import ConfigDict +from modelscope.utils.logger import get_logger + +logger = get_logger() + +Tensor = Union['torch.Tensor', 'tf.Tensor'] +Input = Union[Dict[str, Tensor], Model] + + +class Head(ABC): + """The head base class is for the tasks head method definition + + """ + + def __init__(self, **kwargs): + self.config = ConfigDict(kwargs) + + @abstractmethod + def forward(self, *args, **kwargs) -> Dict[str, Any]: + """ + This method will use the output from backbone model to do any + downstream tasks. Receive The output from backbone model. + + Returns (Dict[str, Any]): The output from downstream task. + """ + pass + + @abstractmethod + def compute_loss(self, *args, **kwargs) -> Dict[str, Any]: + """ + compute loss for head during the finetuning. + + Returns (Dict[str, Any]): The loss dict + """ + pass diff --git a/runtime/Lib/site_packages/models/base/base_model.py b/runtime/Lib/site_packages/models/base/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..9beb156bd01076880457b544f9372b8489376c0b --- /dev/null +++ b/runtime/Lib/site_packages/models/base/base_model.py @@ -0,0 +1,216 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import os.path as osp +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Union + +from modelscope.hub.snapshot_download import snapshot_download +from modelscope.metainfo import Tasks +from modelscope.models.builder import build_backbone, build_model +from modelscope.utils.automodel_utils import (can_load_by_ms, + try_to_load_hf_model) +from modelscope.utils.config import Config, ConfigDict +from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile +from modelscope.utils.device import verify_device +from modelscope.utils.logger import get_logger +from modelscope.utils.plugins import (register_modelhub_repo, + register_plugins_repo) + +logger = get_logger() + +Tensor = Union['torch.Tensor', 'tf.Tensor'] + + +class Model(ABC): + """Base model interface. + """ + + def __init__(self, model_dir, *args, **kwargs): + self.model_dir = model_dir + device_name = kwargs.get('device', 'gpu') + verify_device(device_name) + self._device_name = device_name + + def __call__(self, *args, **kwargs) -> Dict[str, Any]: + return self.postprocess(self.forward(*args, **kwargs)) + + @abstractmethod + def forward(self, *args, **kwargs) -> Dict[str, Any]: + """ + Run the forward pass for a model. + + Returns: + Dict[str, Any]: output from the model forward pass + """ + pass + + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: + """ Model specific postprocess and convert model output to + standard model outputs. + + Args: + inputs: input data + + Return: + dict of results: a dict containing outputs of model, each + output should have the standard output name. + """ + return inputs + + @classmethod + def _instantiate(cls, **kwargs): + """ Define the instantiation method of a model,default method is by + calling the constructor. Note that in the case of no loading model + process in constructor of a task model, a load_model method is + added, and thus this method is overloaded + """ + return cls(**kwargs) + + @classmethod + def from_pretrained(cls, + model_name_or_path: str, + revision: Optional[str] = DEFAULT_MODEL_REVISION, + cfg_dict: Config = None, + device: str = None, + **kwargs): + """Instantiate a model from local directory or remote model repo. Note + that when loading from remote, the model revision can be specified. + + Args: + model_name_or_path(str): A model dir or a model id to be loaded + revision(str, `optional`): The revision used when the model_name_or_path is + a model id of the remote hub. default `master`. + cfg_dict(Config, `optional`): An optional model config. If provided, it will replace + the config read out of the `model_name_or_path` + device(str, `optional`): The device to load the model. + **kwargs: + task(str, `optional`): The `Tasks` enumeration value to replace the task value + read out of config in the `model_name_or_path`. This is useful when the model to be loaded is not + equal to the model saved. + For example, load a `backbone` into a `text-classification` model. + Other kwargs will be directly fed into the `model` key, to replace the default configs. + use_hf(bool, `optional`): + If set to True, it will initialize the model using AutoModel or AutoModelFor* from hf. + If set to False, the model is loaded using the modelscope mode. + If set to None, the loading mode will be automatically selected. + ignore_file_pattern(List[str], `optional`): + This parameter is passed to snapshot_download + device_map(str | Dict[str, str], `optional`): + This parameter is passed to AutoModel or AutoModelFor* + torch_dtype(torch.dtype, `optional`): + This parameter is passed to AutoModel or AutoModelFor* + config(PretrainedConfig, `optional`): + This parameter is passed to AutoModel or AutoModelFor* + Returns: + A model instance. + + Examples: + >>> from modelscope.models import Model + >>> Model.from_pretrained('damo/nlp_structbert_backbone_base_std', task='text-classification') + """ + prefetched = kwargs.get('model_prefetched') + if prefetched is not None: + kwargs.pop('model_prefetched') + invoked_by = kwargs.get(Invoke.KEY) + if invoked_by is not None: + kwargs.pop(Invoke.KEY) + else: + invoked_by = Invoke.PRETRAINED + + ignore_file_pattern = kwargs.pop('ignore_file_pattern', None) + if osp.exists(model_name_or_path): + local_model_dir = model_name_or_path + else: + if prefetched is True: + raise RuntimeError( + 'Expecting model is pre-fetched locally, but is not found.' + ) + + invoked_by = '%s/%s' % (Invoke.KEY, invoked_by) + local_model_dir = snapshot_download( + model_name_or_path, + revision, + user_agent=invoked_by, + ignore_file_pattern=ignore_file_pattern) + logger.info(f'initialize model from {local_model_dir}') + + configuration_path = osp.join(local_model_dir, ModelFile.CONFIGURATION) + cfg = None + if cfg_dict is not None: + cfg = cfg_dict + elif os.path.exists(configuration_path): + cfg = Config.from_file(configuration_path) + task_name = getattr(cfg, 'task', None) + if 'task' in kwargs: + task_name = kwargs.pop('task') + model_cfg = getattr(cfg, 'model', ConfigDict()) + if hasattr(model_cfg, 'model_type') and not hasattr(model_cfg, 'type'): + model_cfg.type = model_cfg.model_type + model_type = getattr(model_cfg, 'type', None) + if isinstance(device, str) and device.startswith('gpu'): + device = 'cuda' + device[3:] + use_hf = kwargs.pop('use_hf', None) + if use_hf is None and can_load_by_ms(local_model_dir, task_name, + model_type): + use_hf = False + model = None + if use_hf in {True, None}: + model = try_to_load_hf_model(local_model_dir, task_name, use_hf, + **kwargs) + if model is not None: + device_map = kwargs.pop('device_map', None) + if device_map is None and device is not None: + model = model.to(device) + return model + # use ms + if cfg is None: + raise FileNotFoundError( + f'`{ModelFile.CONFIGURATION}` file not found.') + model_cfg.model_dir = local_model_dir + + # install and import remote repos before build + register_plugins_repo(cfg.safe_get('plugins')) + register_modelhub_repo(local_model_dir, cfg.get('allow_remote', False)) + + for k, v in kwargs.items(): + model_cfg[k] = v + if device is not None: + model_cfg.device = device + if task_name is Tasks.backbone: + model_cfg.init_backbone = True + model = build_backbone(model_cfg) + else: + model = build_model(model_cfg, task_name=task_name) + + # dynamically add pipeline info to model for pipeline inference + if hasattr(cfg, 'pipeline'): + model.pipeline = cfg.pipeline + + if not hasattr(model, 'cfg'): + model.cfg = cfg + + model_cfg.pop('model_dir', None) + model.name = model_name_or_path + model.model_dir = local_model_dir + return model + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + config: Optional[dict] = None, + **kwargs): + """save the pretrained model, its configuration and other related files to a directory, + so that it can be re-loaded + + Args: + target_folder (Union[str, os.PathLike]): + Directory to which to save. Will be created if it doesn't exist. + + save_checkpoint_names (Union[str, List[str]]): + The checkpoint names to be saved in the target_folder + + config (Optional[dict], optional): + The config for the configuration.json, might not be identical with model.config + """ + raise NotImplementedError( + 'save_pretrained method need to be implemented by the subclass.') diff --git a/runtime/Lib/site_packages/models/base/base_torch_head.py b/runtime/Lib/site_packages/models/base/base_torch_head.py new file mode 100644 index 0000000000000000000000000000000000000000..fb69be4dd092c60407f8aa6506c48311d2bf700a --- /dev/null +++ b/runtime/Lib/site_packages/models/base/base_torch_head.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import torch + +from modelscope.models.base.base_head import Head +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class TorchHead(Head, torch.nn.Module): + """ Base head interface for pytorch + + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + torch.nn.Module.__init__(self) + + def forward(self, *args, **kwargs) -> Dict[str, Any]: + raise NotImplementedError + + def compute_loss(self, *args, **kwargs) -> Dict[str, Any]: + raise NotImplementedError diff --git a/runtime/Lib/site_packages/models/base/base_torch_model.py b/runtime/Lib/site_packages/models/base/base_torch_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2caeb41b3bb9b957de9aad2e1e9ed784491554b2 --- /dev/null +++ b/runtime/Lib/site_packages/models/base/base_torch_model.py @@ -0,0 +1,147 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from copy import deepcopy +from functools import partial +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from packaging import version +from torch import nn +from torch.nn.parallel import DataParallel, DistributedDataParallel + +from modelscope.utils.checkpoint import (save_checkpoint, save_configuration, + save_pretrained) +from modelscope.utils.file_utils import func_receive_dict_inputs +from modelscope.utils.logger import get_logger +from .base_model import Model + +logger = get_logger() + + +class TorchModel(Model, torch.nn.Module): + """ Base model interface for pytorch + + """ + + def __init__(self, model_dir=None, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + torch.nn.Module.__init__(self) + + def __call__(self, *args, **kwargs) -> Dict[str, Any]: + # Adapting a model with only one dict arg, and the arg name must be input or inputs + if func_receive_dict_inputs(self.forward): + return self.postprocess(self.forward(args[0], **kwargs)) + else: + return self.postprocess(self.forward(*args, **kwargs)) + + def _load_pretrained(self, + net, + load_path, + strict=True, + param_key='params'): + if isinstance(net, (DataParallel, DistributedDataParallel)): + net = net.module + load_net = torch.load( + load_path, map_location=lambda storage, loc: storage) + if param_key is not None: + if param_key not in load_net and 'params' in load_net: + param_key = 'params' + logger.info( + f'Loading: {param_key} does not exist, use params.') + if param_key in load_net: + load_net = load_net[param_key] + logger.info( + f'Loading {net.__class__.__name__} model from {load_path}, with param key: [{param_key}].' + ) + # remove unnecessary 'module.' + for k, v in deepcopy(load_net).items(): + if k.startswith('module.'): + load_net[k[7:]] = v + load_net.pop(k) + net.load_state_dict(load_net, strict=strict) + logger.info('load model done.') + return net + + def forward(self, *args, **kwargs) -> Dict[str, Any]: + raise NotImplementedError + + def post_init(self): + """ + A method executed at the end of each model initialization, to execute code that needs the model's + modules properly initialized (such as weight initialization). + """ + self.init_weights() + + def init_weights(self): + # Initialize weights + self.apply(self._init_weights) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=0.02) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=0.02) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = partial( + save_checkpoint, with_meta=False), + config: Optional[dict] = None, + save_config_function: Callable = save_configuration, + **kwargs): + """save the pretrained model, its configuration and other related files to a directory, + so that it can be re-loaded + + Args: + target_folder (Union[str, os.PathLike]): + Directory to which to save. Will be created if it doesn't exist. + + save_checkpoint_names (Union[str, List[str]]): + The checkpoint names to be saved in the target_folder + + save_function (Callable, optional): + The function to use to save the state dictionary. + + config (Optional[dict], optional): + The config for the configuration.json, might not be identical with model.config + + save_config_function (Callble, optional): + The function to use to save the configuration. + + """ + if config is None and hasattr(self, 'cfg'): + config = self.cfg + + save_pretrained(self, target_folder, save_checkpoint_names, + save_function, **kwargs) + + if config is not None: + save_config_function(target_folder, config) + + def compile(self, **kwargs): + """Compile torch model with torch>=2.0 + + Args: + kwargs: + backend: The backend param of torch.compile + mode: The mode param of torch.compile + """ + if version.parse(torch.__version__) >= version.parse('2.0.0.dev'): + return torch.compile(self, **kwargs) + else: + logger.warning( + f'Torch compiling needs torch version >= 2.0.0, your torch version is : {torch.__version__},' + f' returns original model') + return self diff --git a/runtime/Lib/site_packages/models/builder.py b/runtime/Lib/site_packages/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..b57fba53ec4dd549e0faba0cc91e64b48706dc00 --- /dev/null +++ b/runtime/Lib/site_packages/models/builder.py @@ -0,0 +1,97 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from modelscope.metainfo import Models +from modelscope.utils.config import ConfigDict +from modelscope.utils.constant import Tasks +from modelscope.utils.import_utils import INDEX_KEY, LazyImportModule +from modelscope.utils.logger import get_logger +from modelscope.utils.registry import Registry, build_from_cfg +from modelscope.utils.task_utils import get_task_by_subtask_name + +logger = get_logger() + +MODELS = Registry('models') +BACKBONES = MODELS +HEADS = Registry('heads') + +modules = LazyImportModule.AST_INDEX[INDEX_KEY] +for module_index in list(modules.keys()): + if module_index[1] == Tasks.backbone and module_index[0] == 'BACKBONES': + modules[(MODELS.name.upper(), module_index[1], + module_index[2])] = modules[module_index] + + +def build_model(cfg: ConfigDict, + task_name: str = None, + default_args: dict = None): + """ build model given model config dict + + Args: + cfg (:obj:`ConfigDict`): config dict for model object. + task_name (str, optional): task name, refer to + :obj:`Tasks` for more details + default_args (dict, optional): Default initialization arguments. + """ + try: + model = build_from_cfg( + cfg, MODELS, group_key=task_name, default_args=default_args) + except KeyError as e: + # Handle subtask with a backbone model that hasn't been registered + # All the subtask with a parent task should have a task model, otherwise it is not a + # valid subtask + parent_task, task_model_type = get_task_by_subtask_name(task_name) + if task_model_type is None: + raise KeyError(e) + cfg['type'] = task_model_type + model = build_from_cfg( + cfg, MODELS, group_key=parent_task, default_args=default_args) + return model + + +def build_backbone(cfg: ConfigDict, default_args: dict = None): + """ build backbone given backbone config dict + + Args: + cfg (:obj:`ConfigDict`): config dict for backbone object. + default_args (dict, optional): Default initialization arguments. + """ + if not cfg.get('init_backbone', False): + model_dir = cfg.pop('model_dir', None) + else: + model_dir = cfg.get('model_dir', None) + + try: + model = build_from_cfg( + cfg, + BACKBONES, + group_key=Tasks.backbone, + default_args=default_args) + except KeyError: + # Handle backbone that is not in the register group by using transformers AutoModel. + # AutoModel are mostly using in NLP and part of Multi-Modal, while the number of backbone in CV、Audio and MM + # is limited, thus could be added and registered in Modelscope directly + logger.warning( + f'The backbone {cfg.type} is not registered in modelscope, try to import the backbone from hf transformers.' + ) + cfg['type'] = Models.transformers + cfg['model_dir'] = model_dir + model = build_from_cfg( + cfg, + BACKBONES, + group_key=Tasks.backbone, + default_args=default_args) + return model + + +def build_head(cfg: ConfigDict, + task_name: str = None, + default_args: dict = None): + """ build head given config dict + + Args: + cfg (:obj:`ConfigDict`): config dict for head object. + task_name (str, optional): task name, refer to + :obj:`Tasks` for more details + default_args (dict, optional): Default initialization arguments. + """ + return build_from_cfg( + cfg, HEADS, group_key=task_name, default_args=default_args) diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/__init__.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a40a349401d0812604e647e5c6f53e068a6f91b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mmdet_model import AbnormalDetectionModel + +else: + _import_structure = {'mmdet_model': ['AbnormalDetectionModel']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_model.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_model.py new file mode 100644 index 0000000000000000000000000000000000000000..d341b13a9f4ee2121dc582ca0d4ee5b9b198effb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_model.py @@ -0,0 +1,103 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .mmdet_ms import MaskScoringNRoIHead, SingleRoINExtractor + + +@MODELS.register_module( + Tasks.image_object_detection, module_name=Models.mask_scoring) +class AbnormalDetectionModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + + from mmcv.runner import load_checkpoint + from mmdet.datasets import replace_ImageToTensor + from mmdet.datasets.pipelines import Compose + from mmdet.models import build_detector + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + config_path = osp.join(model_dir, 'mmcv_config.py') + config = Config.from_file(config_path) + config.model.pretrained = None + self.model = build_detector( + config.model, test_cfg=config.get('test_cfg')) + + checkpoint = load_checkpoint( + self.model, model_path, map_location='cpu') + self.class_names = checkpoint['meta']['CLASSES'] + config.test_pipeline[0].type = 'LoadImageFromWebcam' + self.transform_input = Compose( + replace_ImageToTensor(config.test_pipeline)) + self.model.cfg = config + self.model.eval() + self.score_thr = config.score_thr + + def inference(self, data): + """data is dict,contain img and img_metas,follow with mmdet. + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + + with torch.no_grad(): + results = self.model( + return_loss=False, + rescale=True, + img=data['img'], + img_metas=data['img_metas']) + return results + + def preprocess(self, image): + """image is numpy return is dict contain img and img_metas,follow with mmdet.""" + + from mmcv.parallel import collate, scatter + data = dict(img=image) + data = self.transform_input(data) + data = collate([data], samples_per_gpu=1) + data['img_metas'] = [ + img_metas.data[0] for img_metas in data['img_metas'] + ] + data['img'] = [img.data[0] for img in data['img']] + + if next(self.model.parameters()).is_cuda: + data = scatter(data, [next(self.model.parameters()).device])[0] + + return data + + def postprocess(self, inputs): + + if isinstance(inputs[0], tuple): + bbox_result, _ = inputs[0] + else: + bbox_result, _ = inputs[0], None + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + + bbox_result = np.vstack(bbox_result) + scores = bbox_result[:, -1] + inds = scores > self.score_thr + if np.sum(np.array(inds).astype('int')) == 0: + return None, None, None + bboxes = bbox_result[inds, :] + labels = labels[inds] + scores = np.around(bboxes[:, 4], 6) + bboxes = (bboxes[:, 0:4]).astype(int) + labels = [self.class_names[i_label] for i_label in labels] + return bboxes, scores, labels diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/__init__.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d51550b7502835ed1b46d09247bb252aefbce03a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .roi_head import MaskScoringNRoIHead, SingleRoINExtractor diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/__init__.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b6b0975b02d45b1cbf0ea8fba0240ba43fa6ef1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .mask_scoring_roi_head import MaskScoringNRoIHead +from .roi_extractors import SingleRoINExtractor + +__all__ = ['MaskScoringNRoIHead', 'SingleRoINExtractor'] diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/mask_scoring_roi_head.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/mask_scoring_roi_head.py new file mode 100644 index 0000000000000000000000000000000000000000..7f89dca162efdfdeede28529dfa9dc991a99fc95 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/mask_scoring_roi_head.py @@ -0,0 +1,138 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on mmdetection +# Originally Apache 2.0 License and publicly available at https://github.com/open-mmlab/mmdetection +import torch +from mmdet.core import bbox2roi +from mmdet.models.builder import HEADS, build_head +from mmdet.models.roi_heads.standard_roi_head import StandardRoIHead + + +@HEADS.register_module() +class MaskScoringNRoIHead(StandardRoIHead): + """Mask Scoring RoIHead for Mask Scoring RCNN. + + https://arxiv.org/abs/1903.00241 + """ + + def __init__(self, + mask_iou_head=None, + reg_roi_scale_factor=None, + **kwargs): + # assert mask_iou_head is not None + super(MaskScoringNRoIHead, self).__init__(**kwargs) + if mask_iou_head is not None: + self.mask_iou_head = build_head(mask_iou_head) + self.reg_roi_scale_factor = reg_roi_scale_factor + + def _bbox_forward(self, x, rois): + """Box head forward function used in both training and testing time.""" + bbox_cls_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], rois) + bbox_reg_feats = self.bbox_roi_extractor( + x[:self.bbox_roi_extractor.num_inputs], + rois, + roi_scale_factor=self.reg_roi_scale_factor) + if self.with_shared_head: + bbox_cls_feats = self.shared_head(bbox_cls_feats) + bbox_reg_feats = self.shared_head(bbox_reg_feats) + cls_score, bbox_pred = self.bbox_head(bbox_cls_feats, bbox_reg_feats) + + bbox_results = dict( + cls_score=cls_score, + bbox_pred=bbox_pred, + bbox_feats=bbox_cls_feats) + return bbox_results + + def _mask_forward_train(self, x, sampling_results, bbox_feats, gt_masks, + img_metas): + """Run forward function and calculate loss for Mask head in + training.""" + pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) + mask_results = super(MaskScoringNRoIHead, + self)._mask_forward_train(x, sampling_results, + bbox_feats, gt_masks, + img_metas) + if mask_results['loss_mask'] is None: + return mask_results + + # mask iou head forward and loss + pos_mask_pred = mask_results['mask_pred'][ + range(mask_results['mask_pred'].size(0)), pos_labels] + mask_iou_pred = self.mask_iou_head(mask_results['mask_feats'], + pos_mask_pred) + pos_mask_iou_pred = mask_iou_pred[range(mask_iou_pred.size(0)), + pos_labels] + + mask_iou_targets = self.mask_iou_head.get_targets( + sampling_results, gt_masks, pos_mask_pred, + mask_results['mask_targets'], self.train_cfg) + loss_mask_iou = self.mask_iou_head.loss(pos_mask_iou_pred, + mask_iou_targets) + mask_results['loss_mask'].update(loss_mask_iou) + return mask_results + + def simple_test_mask(self, + x, + img_metas, + det_bboxes, + det_labels, + rescale=False): + """Obtain mask prediction without augmentation.""" + # image shapes of images in the batch + ori_shapes = tuple(meta['ori_shape'] for meta in img_metas) + scale_factors = tuple(meta['scale_factor'] for meta in img_metas) + + num_imgs = len(det_bboxes) + if all(det_bbox.shape[0] == 0 for det_bbox in det_bboxes): + num_classes = self.mask_head.num_classes + segm_results = [[[] for _ in range(num_classes)] + for _ in range(num_imgs)] + mask_scores = [[[] for _ in range(num_classes)] + for _ in range(num_imgs)] + else: + # if det_bboxes is rescaled to the original image size, we need to + # rescale it back to the testing scale to obtain RoIs. + if rescale and not isinstance(scale_factors[0], float): + scale_factors = [ + torch.from_numpy(scale_factor).to(det_bboxes[0].device) + for scale_factor in scale_factors + ] + _bboxes = [ + det_bboxes[i][:, :4] + * scale_factors[i] if rescale else det_bboxes[i] + for i in range(num_imgs) + ] + mask_rois = bbox2roi(_bboxes) + mask_results = self._mask_forward(x, mask_rois) + concat_det_labels = torch.cat(det_labels) + # get mask scores with mask iou head + mask_feats = mask_results['mask_feats'] + mask_pred = mask_results['mask_pred'] + mask_iou_pred = self.mask_iou_head( + mask_feats, mask_pred[range(concat_det_labels.size(0)), + concat_det_labels]) + # split batch mask prediction back to each image + num_bboxes_per_img = tuple(len(_bbox) for _bbox in _bboxes) + mask_preds = mask_pred.split(num_bboxes_per_img, 0) + mask_iou_preds = mask_iou_pred.split(num_bboxes_per_img, 0) + + # apply mask post-processing to each image individually + segm_results = [] + mask_scores = [] + for i in range(num_imgs): + if det_bboxes[i].shape[0] == 0: + segm_results.append( + [[] for _ in range(self.mask_head.num_classes)]) + mask_scores.append( + [[] for _ in range(self.mask_head.num_classes)]) + else: + segm_result = self.mask_head.get_seg_masks( + mask_preds[i], _bboxes[i], det_labels[i], + self.test_cfg, ori_shapes[i], scale_factors[i], + rescale) + # get mask scores with mask iou head + mask_score = self.mask_iou_head.get_mask_scores( + mask_iou_preds[i], det_bboxes[i], det_labels[i]) + segm_results.append(segm_result) + mask_scores.append(mask_score) + return list(zip(segm_results, mask_scores)) diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/__init__.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7dac4ac6d0d49b802cc48d9dfceb333b1c45b02b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .single_level_roi_extractor import SingleRoINExtractor + +__all__ = ['SingleRoINExtractor'] diff --git a/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/single_level_roi_extractor.py b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/single_level_roi_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..e044c603cc329027db6bdc5816aa98c4106a2666 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/abnormal_object_detection/mmdet_ms/roi_head/roi_extractors/single_level_roi_extractor.py @@ -0,0 +1,153 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on mmdetection +# Originally Apache 2.0 License and publicly available at https://github.com/open-mmlab/mmdetection +import torch +from mmcv.runner import force_fp32 +from mmdet.models.builder import ROI_EXTRACTORS +from mmdet.models.roi_heads.roi_extractors.base_roi_extractor import \ + BaseRoIExtractor + + +@ROI_EXTRACTORS.register_module() +class SingleRoINExtractor(BaseRoIExtractor): + """Extract RoI features from a single level feature map. + + If there are multiple input feature levels, each RoI is mapped to a level + according to its scale. The mapping rule is proposed in + `FPN `_. + + Args: + roi_layer (dict): Specify RoI layer type and arguments. + out_channels (int): Output channels of RoI layers. + featmap_strides (List[int]): Strides of input feature maps. + finest_scale (int): Scale threshold of mapping to level 0. Default: 56. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + + def __init__(self, + roi_layer, + out_channels, + featmap_strides, + finest_scale=56, + init_cfg=None, + gc_context=False, + offset_feature=False): + super(SingleRoINExtractor, self).__init__(roi_layer, out_channels, + featmap_strides, init_cfg) + self.finest_scale = finest_scale + self.gc_context = gc_context + self.offset_feature = offset_feature + self.pool = torch.nn.AdaptiveAvgPool2d(7) + + def map_roi_levels(self, rois, num_levels): + """Map rois to corresponding feature levels by scales. + + - scale < finest_scale * 2: level 0 + - finest_scale * 2 <= scale < finest_scale * 4: level 1 + - finest_scale * 4 <= scale < finest_scale * 8: level 2 + - scale >= finest_scale * 8: level 3 + + Args: + rois (Tensor): Input RoIs, shape (k, 5). + num_levels (int): Total level number. + + Returns: + Tensor: Level index (0-based) of each RoI, shape (k, ) + """ + a = rois[:, 3] - rois[:, 1] + b = rois[:, 4] - rois[:, 2] + scale = torch.sqrt(a * b) + target_lvls = torch.floor(torch.log2(scale / self.finest_scale + 1e-6)) + target_lvls = target_lvls.clamp(min=0, max=num_levels - 1).long() + return target_lvls + + @force_fp32(apply_to=('feats', ), out_fp16=True) + def forward(self, feats, rois, roi_scale_factor=None): + """Forward function.""" + out_size = self.roi_layers[0].output_size + num_levels = len(feats) + expand_dims = (-1, self.out_channels * out_size[0] * out_size[1]) + if torch.onnx.is_in_onnx_export(): + # Work around to export mask-rcnn to onnx + roi_feats = rois[:, :1].clone().detach() + roi_feats = roi_feats.expand(*expand_dims) + roi_feats = roi_feats.reshape(-1, self.out_channels, *out_size) + roi_feats = roi_feats * 0 + else: + roi_feats = feats[0].new_zeros( + rois.size(0), self.out_channels, *out_size) + # TODO: remove this when parrots supports + if torch.__version__ == 'parrots': + roi_feats.requires_grad = True + + if num_levels == 1: + if len(rois) == 0: + return roi_feats + return self.roi_layers[0](feats[0], rois) + + if self.gc_context: + context = [] + for feat in feats: + context.append(self.pool(feat)) + + batch_size = feats[0].shape[0] + target_lvls = self.map_roi_levels(rois, num_levels) + + if roi_scale_factor is not None: + rois = self.roi_rescale(rois, roi_scale_factor) + + for i in range(num_levels): + mask = target_lvls == i + if torch.onnx.is_in_onnx_export(): + # To keep all roi_align nodes exported to onnx + # and skip nonzero op + mask = mask.float().unsqueeze(-1) + # select target level rois and reset the rest rois to zero. + rois_i = rois.clone().detach() + rois_i *= mask + mask_exp = mask.expand(*expand_dims).reshape(roi_feats.shape) + roi_feats_t = self.roi_layers[i](feats[i], rois_i) + roi_feats_t *= mask_exp + roi_feats += roi_feats_t + continue + inds = mask.nonzero(as_tuple=False).squeeze(1) + if inds.numel() > 0: + rois_ = rois[inds] + # todo offset + rois_offset = rois[inds] + offset = torch.zeros(rois_.size(0), 5) + _, _, x_max, y_max = rois_[:, 1].min().item(), rois_[:, 2].min( + ).item(), rois_[:, 3].max().item(), rois_[:, 4].max().item() + offset[:, 1:3] = -100 * torch.ones(rois_.size(0), 1) + offset[:, 3:5] = 100 * torch.ones(rois_.size(0), 1) + rois_offset += offset.cuda() + rois_offset_thsxy = torch.clamp(rois_offset[:, 1:3], min=0.) + rois_offset_ths_xmax = torch.clamp( + rois_offset[:, 3], max=x_max) + rois_offset_ths_ymax = torch.clamp( + rois_offset[:, 4], max=y_max) + rois_offset[:, 1:3] = rois_offset_thsxy + rois_offset[:, + 3], rois_offset[:, + 4] = rois_offset_ths_xmax, rois_offset_ths_ymax + roi_feats_t = self.roi_layers[i](feats[i], rois_) + roi_feats_t_offset = self.roi_layers[i](feats[i], rois_offset) + if self.gc_context: + for j in range(batch_size): + roi_feats_t[rois_[:, 0] == j] += context[i][j] + elif self.offset_feature: + roi_feats_t += roi_feats_t_offset + + roi_feats[inds] = roi_feats_t + else: + # Sometimes some pyramid levels will not be used for RoI + # feature extraction and this will cause an incomplete + # computation graph in one GPU, which is different from those + # in other GPUs and will cause a hanging error. + # Therefore, we add it to ensure each feature pyramid is + # included in the computation graph to avoid runtime bugs. + roi_feats += sum( + x.view(-1)[0] + for x in self.parameters()) * 0. + feats[i].sum() * 0. + return roi_feats diff --git a/runtime/Lib/site_packages/models/cv/action_detection/__init__.py b/runtime/Lib/site_packages/models/cv/action_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fedbe19cc108dd85f484bb5149ae2b469bfbdbf9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_detection/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .action_detection_onnx import ActionDetONNX + +else: + _import_structure = {'action_detection_onnx': ['ActionDetONNX']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/action_detection/action_detection_onnx.py b/runtime/Lib/site_packages/models/cv/action_detection/action_detection_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..ea93d0ddd8d46ab0d59a283e7d2aeb3349d501a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_detection/action_detection_onnx.py @@ -0,0 +1,192 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import os.path as osp +import shutil +import subprocess +import uuid +from tempfile import TemporaryDirectory +from urllib.parse import urlparse + +import cv2 +import numpy as np +import onnxruntime as rt + +from modelscope.hub.file_download import http_get_file +from modelscope.models import Model +from modelscope.utils.constant import Devices +from modelscope.utils.device import verify_device + + +class ActionDetONNX(Model): + + def __init__(self, model_dir, config, *args, **kwargs): + super().__init__(self, model_dir, *args, **kwargs) + model_file = osp.join(config['model_file']) + device_type, device_id = verify_device(self._device_name) + options = rt.SessionOptions() + op_num_threads = config.get('op_num_threads', 1) + options.intra_op_num_threads = op_num_threads + options.inter_op_num_threads = op_num_threads + if device_type == Devices.gpu: + sess = rt.InferenceSession( + model_file, + providers=['CUDAExecutionProvider'], + sess_options=options, + provider_options=[{ + 'device_id': device_id + }]) + else: + sess = rt.InferenceSession( + model_file, + providers=['CPUExecutionProvider'], + sess_options=options) + self.input_name = sess.get_inputs()[0].name + self.sess = sess + self.num_stride = len(config['fpn_strides']) + self.score_thresh = np.asarray( + config['pre_nms_thresh'], dtype='float32').reshape((1, -1)) + self.size_divisibility = config['size_divisibility'] + self.nms_threshold = config['nms_thresh'] + self.tmp_dir = config['tmp_dir'] + self.temporal_stride = config['step'] + self.input_data_type = config['input_type'] + self.action_names = config['action_names'] + self.video_length_limit = config['video_length_limit'] + + def resize_box(self, det, height, width, scale_h, scale_w): + bboxs = det[0] + bboxs[:, [0, 2]] *= scale_w + bboxs[:, [1, 3]] *= scale_h + bboxs[:, [0, 2]] = bboxs[:, [0, 2]].clip(0, width - 1) + bboxs[:, [1, 3]] = bboxs[:, [1, 3]].clip(0, height - 1) + result = { + 'boxes': bboxs.round().astype('int32').tolist(), + 'scores': det[1].tolist(), + 'labels': [self.action_names[i] for i in det[2].tolist()] + } + return result + + def parse_frames(self, frame_names): + imgs = [cv2.imread(name)[:, :, ::-1] for name in frame_names] + imgs = np.stack(imgs).astype(self.input_data_type).transpose( + (3, 0, 1, 2)) # c,t,h,w + imgs = imgs[None] + return imgs + + def forward_img(self, imgs, h, w): + pred = self.sess.run(None, { + self.input_name: imgs, + 'height': np.asarray(h), + 'width': np.asarray(w) + }) + dets = self.post_nms( + pred, + score_threshold=self.score_thresh, + nms_threshold=self.nms_threshold) + return dets + + def forward_video(self, video_name, scale): + min_size, max_size = self._get_sizes(scale) + url_parsed = urlparse(video_name) + frame_rate = 2 + with TemporaryDirectory() as temporary_cache_dir: + if url_parsed.scheme in ('file', '') and osp.exists( + url_parsed.path): + local_video_name = video_name + else: + random_str = str(uuid.uuid1()) + http_get_file( + url=video_name, + local_dir=temporary_cache_dir, + file_name=random_str, + headers={}, + cookies=None) + local_video_name = osp.join(temporary_cache_dir, random_str) + cmd = f'ffmpeg -y -loglevel quiet -ss 0 -t {self.video_length_limit}' + \ + f' -i {local_video_name} -r {frame_rate} -f' + \ + f' image2 {temporary_cache_dir}/%06d_out.jpg' + cmd = cmd.split(' ') + subprocess.call(cmd) + + frame_names = [ + osp.join(temporary_cache_dir, name) + for name in sorted(os.listdir(temporary_cache_dir)) + if name.endswith('_out.jpg') + ] + frame_names = [ + frame_names[i:i + frame_rate * 2] + for i in range(0, + len(frame_names) - frame_rate * 2 + + 1, frame_rate * self.temporal_stride) + ] + timestamp = list( + range(1, + len(frame_names) * self.temporal_stride, + self.temporal_stride)) + batch_imgs = [self.parse_frames(names) for names in frame_names] + N, _, T, H, W = batch_imgs[0].shape + scale_min = min_size / min(H, W) + h, w = min(int(scale_min * H), + max_size), min(int(scale_min * W), max_size) + h = round(h / self.size_divisibility) * self.size_divisibility + w = round(w / self.size_divisibility) * self.size_divisibility + scale_h, scale_w = H / h, W / w + + results = [] + for imgs in batch_imgs: + det = self.forward_img(imgs, h, w) + det = self.resize_box(det[0], H, W, scale_h, scale_w) + results.append(det) + results = [{ + 'timestamp': t, + 'actions': res + } for t, res in zip(timestamp, results)] + return results + + def forward(self, video_name): + return self.forward_video(video_name, scale=1) + + def post_nms(self, pred, score_threshold, nms_threshold=0.3): + pred_bboxes, pred_scores = pred + N = len(pred_bboxes) + dets = [] + for i in range(N): + bboxes, scores = pred_bboxes[i], pred_scores[i] + candidate_inds = scores > score_threshold + scores = scores[candidate_inds] + candidate_nonzeros = candidate_inds.nonzero() + bboxes = bboxes[candidate_nonzeros[0]] + labels = candidate_nonzeros[1] + keep = self._nms(bboxes, scores, labels, nms_threshold) + bbox = bboxes[keep] + score = scores[keep] + label = labels[keep] + dets.append((bbox, score, label)) + return dets + + def _nms(self, boxes, scores, idxs, nms_threshold): + if len(boxes) == 0: + return [] + max_coordinate = boxes.max() + offsets = idxs * (max_coordinate + 1) + boxes_for_nms = boxes + offsets[:, None].astype('float32') + boxes_for_nms[:, 2] = boxes_for_nms[:, 2] - boxes_for_nms[:, 0] + boxes_for_nms[:, 3] = boxes_for_nms[:, 3] - boxes_for_nms[:, 1] + keep = cv2.dnn.NMSBoxes( + boxes_for_nms.tolist(), + scores.tolist(), + score_threshold=0, + nms_threshold=nms_threshold) + if len(keep.shape) == 2: + keep = np.squeeze(keep, 1) + return keep + + def _get_sizes(self, scale): + if scale == 1: + min_size, max_size = 512, 896 + elif scale == 2: + min_size, max_size = 768, 1280 + else: + min_size, max_size = 1024, 1792 + return min_size, max_size diff --git a/runtime/Lib/site_packages/models/cv/action_detection/modules/__init__.py b/runtime/Lib/site_packages/models/cv/action_detection/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/action_detection/modules/action_detection_pytorch.py b/runtime/Lib/site_packages/models/cv/action_detection/modules/action_detection_pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..a8600ae0a710a45138a2efa99f5410cc47515aae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_detection/modules/action_detection_pytorch.py @@ -0,0 +1,232 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import logging +from typing import Dict, List + +import torch +import torch.nn as nn +from detectron2.layers import ShapeSpec +from detectron2.modeling import postprocessing +from detectron2.modeling.backbone.fpn import FPN, LastLevelP6P7 +from detectron2.modeling.box_regression import _dense_box_regression_loss +from detectron2.modeling.meta_arch.fcos import FCOS, FCOSHead +from detectron2.structures import (Boxes, ImageList, Instances, + pairwise_point_box_distance) +from fvcore.nn import sigmoid_focal_loss_jit +from torch.nn import functional as F + +from modelscope.models.base import TorchModel +from .resnet import Bottleneck3D, ResNet3D + +logger = logging.getLogger('detectron2.modelscope.' + __name__) + + +class ActionDetector(FCOS, TorchModel): + + def __init__(self, **kargs): + super().__init__(**kargs) + + @torch.no_grad() + def load_init_backbone(self, path): + from fvcore.common import checkpoint + state = torch.load(path, map_location=torch.device('cpu')) + model_state = state.pop('model') + prefix = 'backbone.bottom_up.' + keys = sorted(model_state.keys()) + for k in keys: + if not k.startswith(prefix): + model_state.pop(k) + checkpoint._strip_prefix_if_present(model_state, prefix) + t = self.backbone.bottom_up.load_state_dict(model_state, strict=False) + logger.info(str(t)) + logger.info(f'Load pretrained backbone weights from {path}') + + def preprocess_image(self, batched_inputs): + """ + Normalize, pad and batch the input images. + """ + images = [x['frames'].to(self.device) for x in batched_inputs] + images = [x.float() / 255.0 for x in images] + images = ImageList.from_tensors(images, + self.backbone.size_divisibility) + return images + + @torch.no_grad() + def match_anchors(self, anchors: List[Boxes], + gt_instances: List[Instances]): + """ + Match anchors with ground truth boxes. + + Args: + anchors: #level boxes, from the highest resolution to lower resolution + gt_instances: ground truth instances per image + + Returns: + List[Tensor]: + #image tensors, each is a vector of matched gt + indices (or -1 for unmatched anchors) for all anchors. + """ + num_anchors_per_level = [len(x) for x in anchors] + anchors = Boxes.cat(anchors) # Rx4 + anchor_centers = anchors.get_centers() # Rx2 + anchor_sizes = anchors.tensor[:, 2] - anchors.tensor[:, 0] # R + + lower_bound = anchor_sizes * 4 + lower_bound[:num_anchors_per_level[0]] = 0 + upper_bound = anchor_sizes * 8 + upper_bound[-num_anchors_per_level[-1]:] = float('inf') + + matched_indices = [] + for gt_per_image in gt_instances: + if len(gt_per_image) == 0: + matched_indices.append( + torch.full((len(anchors), ), + -1, + dtype=torch.int64, + device=anchors.tensor.device)) + continue + gt_centers = gt_per_image.gt_boxes.get_centers() # Nx2 + # FCOS with center sampling: anchor point must be close enough to gt center. + center_dist = (anchor_centers[:, None, :] + - gt_centers[None, :, :]).abs_().max(dim=2).values + pairwise_match = center_dist < self.center_sampling_radius * anchor_sizes[:, + None] + pairwise_dist = pairwise_point_box_distance( + anchor_centers, gt_per_image.gt_boxes) + + # The original FCOS anchor matching rule: anchor point must be inside gt + pairwise_match &= pairwise_dist.min(dim=2).values > 0 + + # Multilevel anchor matching in FCOS: each anchor is only responsible + # for certain scale range. + pairwise_dist = pairwise_dist.max(dim=2).values + pairwise_match &= (pairwise_dist > lower_bound[:, None]) & ( + pairwise_dist < upper_bound[:, None]) + + # Match the GT box with minimum area, if there are multiple GT matches + gt_areas = gt_per_image.gt_boxes.area() # N + pairwise_match = pairwise_match.to( + torch.float32) * (1e8 - gt_areas[None, :]) + min_values, matched_idx = pairwise_match.max( + dim=1) # R, per-anchor match + matched_idx[ + min_values < 1e-5] = -1 # Unmatched anchors are assigned -1 + + matched_indices.append(matched_idx) + return matched_indices + + def losses(self, anchors, pred_logits, gt_labels, pred_anchor_deltas, + gt_boxes, pred_centerness): + """ + This method is almost identical to :meth:`RetinaNet.losses`, with an extra + "loss_centerness" in the returned dict. + """ + gt_labels = torch.stack(gt_labels) # (N, R) + + pos_mask = (gt_labels >= 0) & (gt_labels != self.num_classes) + num_pos_anchors = pos_mask.sum().item() + normalizer = self._ema_update('loss_normalizer', + max(num_pos_anchors, 1), 300) + + # classification and regression loss + gt_labels_target = F.one_hot( + gt_labels, num_classes=self.num_classes + + 1)[:, :, :-1] # no loss for the last (background) class + loss_cls = sigmoid_focal_loss_jit( + torch.cat(pred_logits, dim=1), + gt_labels_target.to(pred_logits[0].dtype), + alpha=self.focal_loss_alpha, + gamma=self.focal_loss_gamma, + reduction='sum', + ) + + loss_box_reg = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + [x.tensor for x in gt_boxes], + pos_mask, + box_reg_loss_type='giou', + ) + + ctrness_targets = self.compute_ctrness_targets(anchors, + gt_boxes) # NxR + pred_centerness = torch.cat( + pred_centerness, dim=1).squeeze(dim=2) # NxR + ctrness_loss = F.binary_cross_entropy_with_logits( + pred_centerness[pos_mask], + ctrness_targets[pos_mask], + reduction='sum') + return { + 'loss_fcos_cls': loss_cls / normalizer, + 'loss_fcos_loc': loss_box_reg / normalizer, + 'loss_fcos_ctr': ctrness_loss / normalizer, + } + + @torch.no_grad() + def label_anchors(self, anchors, gt_instances): + """ + Same interface as :meth:`RetinaNet.label_anchors`, but implemented with FCOS + anchor matching rule. + + Unlike RetinaNet, there are no ignored anchors. + """ + matched_indices = self.match_anchors(anchors, gt_instances) + + matched_labels, matched_boxes = [], [] + for gt_index, gt_per_image in zip(matched_indices, gt_instances): + if len(gt_per_image) > 0: + label = gt_per_image.gt_classes[gt_index.clip(min=0)] + matched_gt_boxes = gt_per_image.gt_boxes[gt_index.clip(min=0)] + else: + label = gt_per_image.gt_classes.new_zeros((len(gt_index), )) + matched_gt_boxes = Boxes( + gt_per_image.gt_boxes.tensor.new_zeros((len(gt_index), 4))) + label[gt_index < 0] = self.num_classes # background + matched_labels.append(label) + matched_boxes.append(matched_gt_boxes) + return matched_labels, matched_boxes + + def compute_ctrness_targets(self, anchors, gt_boxes): # NxR + anchors = Boxes.cat(anchors).tensor # Rx4 + reg_targets = [ + self.box2box_transform.get_deltas(anchors, m.tensor) + for m in gt_boxes + ] + reg_targets = torch.stack(reg_targets, dim=0) # NxRx4 + if len(reg_targets) == 0: + # return reg_targets.new_zeros(len(reg_targets)) + return reg_targets.new_zeros(reg_targets.size()[:-1]) + left_right = reg_targets[:, :, [0, 2]] + top_bottom = reg_targets[:, :, [1, 3]] + ctrness = (left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * ( + top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0]) + return torch.sqrt(ctrness) + + +def build_action_detection_model(num_classes, device='cpu'): + backbone = ResNet3D( + Bottleneck3D, [3, 4, 6, 3], + ops=['c2d', 'p3d'] * 8, + t_stride=[1, 1, 1, 1, 1], + num_classes=None) + in_features = ['res3', 'res4', 'res5'] + out_channels = 512 + top_block = LastLevelP6P7(out_channels, out_channels, in_feature='p5') + fpnbackbone = FPN( + bottom_up=backbone, + in_features=in_features, + out_channels=out_channels, + top_block=top_block, + ) + head = FCOSHead( + input_shape=[ShapeSpec(channels=out_channels)] * 5, + conv_dims=[out_channels] * 2, + num_classes=num_classes) + model = ActionDetector( + backbone=fpnbackbone, + head=head, + num_classes=num_classes, + pixel_mean=[0, 0, 0], + pixel_std=[0, 0, 0]) + return model diff --git a/runtime/Lib/site_packages/models/cv/action_detection/modules/resnet.py b/runtime/Lib/site_packages/models/cv/action_detection/modules/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..7f5529a48ab9698a0f3c214410e3c3933f5f9591 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_detection/modules/resnet.py @@ -0,0 +1,382 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +from detectron2.modeling import Backbone + + +def conv1x3x3(in_planes, out_planes, stride=(1, 1, 1)): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=(1, 3, 3), + stride=stride, + padding=(0, 1, 1), + bias=False) + + +def conv3x3x3(in_planes, out_planes, stride=(1, 1, 1)): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=(3, 3, 3), + stride=stride, + padding=(1, 1, 1), + bias=False) + + +def conv1x1x1(in_planes, out_planes, stride=(1, 1, 1)): + return nn.Conv3d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +def conv3x1x1(in_planes, out_planes, stride=(1, 1, 1)): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=(3, 1, 1), + stride=stride, + padding=(1, 0, 0), + bias=False) + + +class BasicBlock3D(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + op='c2d', + downsample=None, + base_width=64, + norm_layer=None): + super(BasicBlock3D, self).__init__() + dilation = 1 + groups = 1 + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + stride = [stride] * 3 if isinstance(stride, int) else stride + self.t_stride = stride[0] + self.stride = stride + stride = [1] + list(stride[1:]) + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + # self.conv1 = conv3x3(inplanes, planes, stride) + self.conv1 = conv1x3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv1d = conv3x1x1(planes, planes) + # self.conv2 = conv3x3(planes, planes) + self.conv2 = conv1x3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + + def forward(self, x): + identity = x + + out = self.conv1(x) + if self.t_stride > 1: + out = torch.max_pool3d( + out, [self.t_stride, 1, 1], stride=[self.t_stride, 1, 1]) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv1d(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + @property + def out_channels(self): + return self.conv2.out_channels + + +class Bottleneck3D(nn.Module): + expansion = 2 + + def __init__(self, + inplanes, + planes, + stride=1, + op='c2d', + downsample=None, + base_width=64, + norm_layer=None): + super(Bottleneck3D, self).__init__() + self.op = op + if norm_layer is None: + norm_layer = nn.BatchNorm3d + width = int(planes * (base_width / 64.)) + stride = [stride] * 3 if isinstance(stride, int) else stride + self.conv1 = conv3x1x1(inplanes, width) if op == 'p3d' else conv1x1x1( + inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3x3(width, width, + stride) if op == 'c3d' else conv1x3x3( + width, width, stride) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + if self.op == 'tsm': + out = self.tsm(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + @property + def out_channels(self): + return self.conv3.out_channels + + +class ResNet3D(Backbone): + + def __init__(self, + block, + layers, + ops, + t_stride, + num_classes=1000, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + reduce_dim=0, + norm_layer=None): + self.reduce_dim = reduce_dim + self.num_classes = num_classes + super(ResNet3D, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + self._norm_layer = norm_layer + self._out_feature_strides = {'res3': 8, 'res4': 16, 'res5': 32} + self._out_features = ['res3', 'res4', 'res5'] + self._out_feature_channels = {} + self.outputs = {} + self.inplanes = 64 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv3d( + 3, + self.inplanes, (1, 7, 7), + stride=(t_stride[0], 2, 2), + padding=(0, 3, 3), + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool3d( + kernel_size=(1, 3, 3), + stride=(t_stride[1], 2, 2), + padding=(0, 1, 1)) + self.layer1 = self._make_layer( + block, 64, layers[0], stride=(1, 1, 1), ops=ops[:layers[0]]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=(t_stride[2], 2, 2), + ops=ops[layers[0]:][:layers[1]]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=(t_stride[3], 2, 2), + ops=ops[sum(layers[:2], 0):][:layers[2]]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=(t_stride[4], 2, 2), + ops=ops[sum(layers[:3], 0):][:layers[3]]) + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.sptial_atten = nn.Conv2d(2, 1, kernel_size=7, padding=3) + self.drop = nn.Dropout(0.5) + if reduce_dim > 0: + self.rd_conv = nn.Conv2d( + 512 * block.expansion, reduce_dim, kernel_size=1) + self.clc = nn.Conv2d(reduce_dim, num_classes, kernel_size=1) + else: + self.clc = nn.Conv2d( + 512 * block.expansion, num_classes, kernel_size=1) + + self._out_feature_channels['res3'] = self.layer2[-1].out_channels + self._out_feature_channels['res4'] = self.layer3[-1].out_channels + self._out_feature_channels['res5'] = self.layer4[-1].out_channels + + for m in self.modules(): + if isinstance(m, nn.Conv3d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm3d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck3D): + nn.init.constant_(m.bn3.weight, 0) + + def _make_layer(self, block, planes, blocks, ops, stride=(1, 1, 1)): + norm_layer = self._norm_layer + downsample = None + if stride != (1, 1, 1) or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, ops[0], downsample, + self.base_width, norm_layer)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + op=ops[i], + base_width=self.base_width, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def features(self, x): + x = self.norm_x(x) + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + self.outputs['res3'] = x.mean(dim=2) + x = self.layer3(x) + self.outputs['res4'] = x.mean(dim=2) + x = self.layer4(x) # N,C,T,H,W + self.outputs['res5'] = x.mean(dim=2) + if self.num_classes is not None: + x = torch.mean(x, dim=2) # 解决时间维度, N,C,H,W + # spatial attention + ftr = torch.cat( + (x.max(dim=1, keepdim=True)[0], x.mean(dim=1, keepdim=True)), + dim=1) + score = self.sptial_atten(ftr) # N,1,H,W + x = x * torch.sigmoid(score) # N,C,H,W + self.score = score + + x = self.avgpool(x) # ,N,C,1,1 + if self.reduce_dim > 0: + x = self.rd_conv(x) + self.outputs['ftr'] = x.mean(dim=(2, 3)) + return x + + def logits(self, x): + x = self.features(x) + x = self.clc(x) + return x + + def forward(self, x): + ftr = self.features(x) + if self.num_classes is not None: + x = self.drop(ftr) + x = self.clc(x) + x = torch.mean(x, (2, 3)) + return x + + return self.outputs + + @torch.no_grad() + def norm_x(self, x): + m = x.new_tensor([0.485, 0.456, 0.406]).reshape([1, 3, 1, 1, 1]) + s = x.new_tensor([0.229, 0.224, 0.225]).reshape([1, 3, 1, 1, 1]) + x -= m + x /= s + return x + + +def resnet101_3d(ops, t_stride, num_class, reduce_dim=0): + net = ResNet3D( + Bottleneck3D, [3, 4, 23, 3], + ops=ops, + t_stride=t_stride, + num_classes=num_class, + reduce_dim=reduce_dim) + return net + + +def resnet50_3d(ops, t_stride, num_class, reduce_dim=0): + net = ResNet3D( + Bottleneck3D, [3, 4, 6, 3], + ops=ops, + t_stride=t_stride, + num_classes=num_class, + reduce_dim=reduce_dim) + return net + + +def resnet34_3d(ops, t_stride, num_class, reduce_dim=0): + net = ResNet3D( + BasicBlock3D, [3, 4, 6, 3], + ops=ops, + t_stride=t_stride, + num_classes=num_class, + reduce_dim=reduce_dim) + return net + + +def resnet18_3d(ops, t_stride, num_class, reduce_dim=0): + net = ResNet3D( + BasicBlock3D, [2, 2, 2, 2], + ops=ops, + t_stride=t_stride, + num_classes=num_class, + reduce_dim=reduce_dim) + return net diff --git a/runtime/Lib/site_packages/models/cv/action_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/action_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5e9dc31064e064ef946bdc40465c02c37366453a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_recognition/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .models import BaseVideoModel + from .tada_convnext import TadaConvNeXt + from .temporal_patch_shift_transformer import PatchShiftTransformer + +else: + _import_structure = { + 'models': ['BaseVideoModel'], + 'tada_convnext': ['TadaConvNeXt'], + 'temporal_patch_shift_transformer': ['PatchShiftTransformer'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/action_recognition/models.py b/runtime/Lib/site_packages/models/cv/action_recognition/models.py new file mode 100644 index 0000000000000000000000000000000000000000..f16805fbd70f938ecb85aa22e07035d8cdeea535 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_recognition/models.py @@ -0,0 +1,134 @@ +# The implementation is also open-sourced by the authors, +# and available at https://github.com/alibaba-mmai-research/TAdaConv +# Copyright 2021-2022 The Alibaba FVI Team Authors. All rights reserved. +import torch.nn as nn + +from .s3dg import Inception3D +from .tada_convnext import TadaConvNeXt + + +class BaseVideoModel(nn.Module): + """ + Standard video model. + The model is divided into the backbone and the head, where the backbone + extracts features and the head performs classification. + + The backbones can be defined in model/base/backbone.py or anywhere else + as long as the backbone is registered by the BACKBONE_REGISTRY. + The heads can be defined in model/module_zoo/heads/ or anywhere else + as long as the head is registered by the HEAD_REGISTRY. + + The registries automatically finds the registered modules and construct + the base video model. + """ + + def __init__(self, cfg): + """ + Args: + cfg (Config): global config object. + """ + super(BaseVideoModel, self).__init__() + # the backbone is created according to meta-architectures + # defined in models/base/backbone.py + if cfg.MODEL.NAME == 'ConvNeXt_tiny': + self.backbone = TadaConvNeXt(cfg) + elif cfg.MODEL.NAME == 'S3DG': + self.backbone = Inception3D(cfg) + else: + error_str = 'backbone {} is not supported, ConvNeXt_tiny or S3DG is supported'.format( + cfg.MODEL.NAME) + raise NotImplementedError(error_str) + + # the head is created according to the heads + # defined in models/module_zoo/heads + if cfg.VIDEO.HEAD.NAME == 'BaseHead': + self.head = BaseHead(cfg) + elif cfg.VIDEO.HEAD.NAME == 'AvgHead': + self.head = AvgHead(cfg) + else: + error_str = 'head {} is not supported, BaseHead or AvgHead is supported'.format( + cfg.VIDEO.HEAD.NAME) + raise NotImplementedError(error_str) + + def forward(self, x): + x = self.backbone(x) + x = self.head(x) + return x + + +class BaseHead(nn.Module): + """ + Constructs base head. + """ + + def __init__( + self, + cfg, + ): + """ + Args: + cfg (Config): global config object. + """ + super(BaseHead, self).__init__() + self.cfg = cfg + dim = cfg.VIDEO.BACKBONE.NUM_OUT_FEATURES + num_classes = cfg.VIDEO.HEAD.NUM_CLASSES + dropout_rate = cfg.VIDEO.HEAD.DROPOUT_RATE + activation_func = cfg.VIDEO.HEAD.ACTIVATION + self._construct_head(dim, num_classes, dropout_rate, activation_func) + + def _construct_head(self, dim, num_classes, dropout_rate, activation_func): + self.global_avg_pool = nn.AdaptiveAvgPool3d(1) + + if dropout_rate > 0.0: + self.dropout = nn.Dropout(dropout_rate) + + self.out = nn.Linear(dim, num_classes, bias=True) + + if activation_func == 'softmax': + self.activation = nn.Softmax(dim=-1) + elif activation_func == 'sigmoid': + self.activation = nn.Sigmoid() + else: + raise NotImplementedError('{} is not supported as an activation' + 'function.'.format(activation_func)) + + def forward(self, x): + if len(x.shape) == 5: + x = self.global_avg_pool(x) + # (N, C, T, H, W) -> (N, T, H, W, C). + x = x.permute((0, 2, 3, 4, 1)) + if hasattr(self, 'dropout'): + out = self.dropout(x) + else: + out = x + out = self.out(out) + out = self.activation(out) + out = out.view(out.shape[0], -1) + return out, x.view(x.shape[0], -1) + + +class AvgHead(nn.Module): + """ + Constructs base head. + """ + + def __init__( + self, + cfg, + ): + """ + Args: + cfg (Config): global config object. + """ + super(AvgHead, self).__init__() + self.cfg = cfg + self.global_avg_pool = nn.AdaptiveAvgPool3d(1) + + def forward(self, x): + if len(x.shape) == 5: + x = self.global_avg_pool(x) + # (N, C, T, H, W) -> (N, T, H, W, C). + x = x.permute((0, 2, 3, 4, 1)) + out = x.view(x.shape[0], -1) + return out, x.view(x.shape[0], -1) diff --git a/runtime/Lib/site_packages/models/cv/action_recognition/s3dg.py b/runtime/Lib/site_packages/models/cv/action_recognition/s3dg.py new file mode 100644 index 0000000000000000000000000000000000000000..46e7689271f8e574a0ff178fbdd4ca567d6a1bf3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_recognition/s3dg.py @@ -0,0 +1,304 @@ +# The implementation is adopted from https://github.com/TengdaHan/CoCLR, +# made pubicly available under the Apache License, Version 2.0 at https://github.com/TengdaHan/CoCLR +# Copyright 2021-2022 The Alibaba FVI Team Authors. All rights reserved. +import torch +import torch.nn as nn + + +class InceptionBaseConv3D(nn.Module): + """ + Constructs basic inception 3D conv. + Modified from https://github.com/TengdaHan/CoCLR/blob/main/backbone/s3dg.py. + """ + + def __init__(self, + cfg, + in_planes, + out_planes, + kernel_size, + stride, + padding=0): + super(InceptionBaseConv3D, self).__init__() + self.conv = nn.Conv3d( + in_planes, + out_planes, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=False) + self.bn = nn.BatchNorm3d(out_planes) + self.relu = nn.ReLU(inplace=True) + + # init + self.conv.weight.data.normal_( + mean=0, std=0.01) # original s3d is truncated normal within 2 std + self.bn.weight.data.fill_(1) + self.bn.bias.data.zero_() + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class InceptionBlock3D(nn.Module): + """ + Element constructing the S3D/S3DG. + See models/base/backbone.py L99-186. + + Modifed from https://github.com/TengdaHan/CoCLR/blob/main/backbone/s3dg.py. + """ + + def __init__(self, cfg, in_planes, out_planes): + super(InceptionBlock3D, self).__init__() + + _gating = cfg.VIDEO.BACKBONE.BRANCH.GATING + + assert len(out_planes) == 6 + assert isinstance(out_planes, list) + + [ + num_out_0_0a, num_out_1_0a, num_out_1_0b, num_out_2_0a, + num_out_2_0b, num_out_3_0b + ] = out_planes + + self.branch0 = nn.Sequential( + InceptionBaseConv3D( + cfg, in_planes, num_out_0_0a, kernel_size=1, stride=1), ) + self.branch1 = nn.Sequential( + InceptionBaseConv3D( + cfg, in_planes, num_out_1_0a, kernel_size=1, stride=1), + STConv3d( + cfg, + num_out_1_0a, + num_out_1_0b, + kernel_size=3, + stride=1, + padding=1), + ) + self.branch2 = nn.Sequential( + InceptionBaseConv3D( + cfg, in_planes, num_out_2_0a, kernel_size=1, stride=1), + STConv3d( + cfg, + num_out_2_0a, + num_out_2_0b, + kernel_size=3, + stride=1, + padding=1), + ) + self.branch3 = nn.Sequential( + nn.MaxPool3d(kernel_size=(3, 3, 3), stride=1, padding=1), + InceptionBaseConv3D( + cfg, in_planes, num_out_3_0b, kernel_size=1, stride=1), + ) + + self.out_channels = sum( + [num_out_0_0a, num_out_1_0b, num_out_2_0b, num_out_3_0b]) + + self.gating = _gating + if _gating: + self.gating_b0 = SelfGating(num_out_0_0a) + self.gating_b1 = SelfGating(num_out_1_0b) + self.gating_b2 = SelfGating(num_out_2_0b) + self.gating_b3 = SelfGating(num_out_3_0b) + + def forward(self, x): + x0 = self.branch0(x) + x1 = self.branch1(x) + x2 = self.branch2(x) + x3 = self.branch3(x) + if self.gating: + x0 = self.gating_b0(x0) + x1 = self.gating_b1(x1) + x2 = self.gating_b2(x2) + x3 = self.gating_b3(x3) + + out = torch.cat((x0, x1, x2, x3), 1) + + return out + + +class SelfGating(nn.Module): + + def __init__(self, input_dim): + super(SelfGating, self).__init__() + self.fc = nn.Linear(input_dim, input_dim) + + def forward(self, input_tensor): + """Feature gating as used in S3D-G""" + spatiotemporal_average = torch.mean(input_tensor, dim=[2, 3, 4]) + weights = self.fc(spatiotemporal_average) + weights = torch.sigmoid(weights) + return weights[:, :, None, None, None] * input_tensor + + +class STConv3d(nn.Module): + """ + Element constructing the S3D/S3DG. + See models/base/backbone.py L99-186. + + Modifed from https://github.com/TengdaHan/CoCLR/blob/main/backbone/s3dg.py. + """ + + def __init__(self, + cfg, + in_planes, + out_planes, + kernel_size, + stride, + padding=0): + super(STConv3d, self).__init__() + if isinstance(stride, tuple): + t_stride = stride[0] + stride = stride[-1] + else: # int + t_stride = stride + + self.bn_mmt = cfg.BN.MOMENTUM + self.bn_eps = float(cfg.BN.EPS) + self._construct_branch(cfg, in_planes, out_planes, kernel_size, stride, + t_stride, padding) + + def _construct_branch(self, + cfg, + in_planes, + out_planes, + kernel_size, + stride, + t_stride, + padding=0): + self.conv1 = nn.Conv3d( + in_planes, + out_planes, + kernel_size=(1, kernel_size, kernel_size), + stride=(1, stride, stride), + padding=(0, padding, padding), + bias=False) + self.conv2 = nn.Conv3d( + out_planes, + out_planes, + kernel_size=(kernel_size, 1, 1), + stride=(t_stride, 1, 1), + padding=(padding, 0, 0), + bias=False) + + self.bn1 = nn.BatchNorm3d( + out_planes, eps=self.bn_eps, momentum=self.bn_mmt) + self.bn2 = nn.BatchNorm3d( + out_planes, eps=self.bn_eps, momentum=self.bn_mmt) + self.relu = nn.ReLU(inplace=True) + + # init + self.conv1.weight.data.normal_( + mean=0, std=0.01) # original s3d is truncated normal within 2 std + self.conv2.weight.data.normal_( + mean=0, std=0.01) # original s3d is truncated normal within 2 std + self.bn1.weight.data.fill_(1) + self.bn1.bias.data.zero_() + self.bn2.weight.data.fill_(1) + self.bn2.bias.data.zero_() + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + return x + + +class Inception3D(nn.Module): + """ + Backbone architecture for I3D/S3DG. + Modifed from https://github.com/TengdaHan/CoCLR/blob/main/backbone/s3dg.py. + """ + + def __init__(self, cfg): + """ + Args: + cfg (Config): global config object. + """ + super(Inception3D, self).__init__() + _input_channel = cfg.DATA.NUM_INPUT_CHANNELS + self._construct_backbone(cfg, _input_channel) + + def _construct_backbone(self, cfg, input_channel): + # ------------------- Block 1 ------------------- + self.Conv_1a = STConv3d( + cfg, input_channel, 64, kernel_size=7, stride=2, padding=3) + + self.block1 = nn.Sequential(self.Conv_1a) # (64, 32, 112, 112) + + # ------------------- Block 2 ------------------- + self.MaxPool_2a = nn.MaxPool3d( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) + self.Conv_2b = InceptionBaseConv3D( + cfg, 64, 64, kernel_size=1, stride=1) + self.Conv_2c = STConv3d( + cfg, 64, 192, kernel_size=3, stride=1, padding=1) + + self.block2 = nn.Sequential( + self.MaxPool_2a, # (64, 32, 56, 56) + self.Conv_2b, # (64, 32, 56, 56) + self.Conv_2c) # (192, 32, 56, 56) + + # ------------------- Block 3 ------------------- + self.MaxPool_3a = nn.MaxPool3d( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) + self.Mixed_3b = InceptionBlock3D( + cfg, in_planes=192, out_planes=[64, 96, 128, 16, 32, 32]) + self.Mixed_3c = InceptionBlock3D( + cfg, in_planes=256, out_planes=[128, 128, 192, 32, 96, 64]) + + self.block3 = nn.Sequential( + self.MaxPool_3a, # (192, 32, 28, 28) + self.Mixed_3b, # (256, 32, 28, 28) + self.Mixed_3c) # (480, 32, 28, 28) + + # ------------------- Block 4 ------------------- + self.MaxPool_4a = nn.MaxPool3d( + kernel_size=(3, 3, 3), stride=(2, 2, 2), padding=(1, 1, 1)) + self.Mixed_4b = InceptionBlock3D( + cfg, in_planes=480, out_planes=[192, 96, 208, 16, 48, 64]) + self.Mixed_4c = InceptionBlock3D( + cfg, in_planes=512, out_planes=[160, 112, 224, 24, 64, 64]) + self.Mixed_4d = InceptionBlock3D( + cfg, in_planes=512, out_planes=[128, 128, 256, 24, 64, 64]) + self.Mixed_4e = InceptionBlock3D( + cfg, in_planes=512, out_planes=[112, 144, 288, 32, 64, 64]) + self.Mixed_4f = InceptionBlock3D( + cfg, in_planes=528, out_planes=[256, 160, 320, 32, 128, 128]) + + self.block4 = nn.Sequential( + self.MaxPool_4a, # (480, 16, 14, 14) + self.Mixed_4b, # (512, 16, 14, 14) + self.Mixed_4c, # (512, 16, 14, 14) + self.Mixed_4d, # (512, 16, 14, 14) + self.Mixed_4e, # (528, 16, 14, 14) + self.Mixed_4f) # (832, 16, 14, 14) + + # ------------------- Block 5 ------------------- + self.MaxPool_5a = nn.MaxPool3d( + kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=(0, 0, 0)) + self.Mixed_5b = InceptionBlock3D( + cfg, in_planes=832, out_planes=[256, 160, 320, 32, 128, 128]) + self.Mixed_5c = InceptionBlock3D( + cfg, in_planes=832, out_planes=[384, 192, 384, 48, 128, 128]) + + self.block5 = nn.Sequential( + self.MaxPool_5a, # (832, 8, 7, 7) + self.Mixed_5b, # (832, 8, 7, 7) + self.Mixed_5c) # (1024, 8, 7, 7) + + def forward(self, x): + if isinstance(x, dict): + x = x['video'] + x = self.block1(x) + x = self.block2(x) + x = self.block3(x) + x = self.block4(x) + x = self.block5(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/action_recognition/tada_convnext.py b/runtime/Lib/site_packages/models/cv/action_recognition/tada_convnext.py new file mode 100644 index 0000000000000000000000000000000000000000..b1de7af8f78a19a8525832c2be8464f435dfb772 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_recognition/tada_convnext.py @@ -0,0 +1,476 @@ +# The implementation is adopted from https://github.com/facebookresearch/ConvNeXt, +# made pubicly available under the MIT License at https://github.com/facebookresearch/ConvNeXt +# Copyright 2021-2022 The Alibaba FVI Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair, _triple + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """ + From https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py. + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """ + From https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py. + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +class TadaConvNeXt(nn.Module): + r""" ConvNeXt + A PyTorch impl of : `A ConvNet for the 2020s` - + https://arxiv.org/pdf/2201.03545.pdf + + Args: + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3] + dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768] + drop_path_rate (float): Stochastic depth rate. Default: 0. + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1. + """ + + def __init__( + self, cfg + # in_chans=3, num_classes=1000, + # depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], drop_path_rate=0., + # layer_scale_init_value=1e-6, head_init_scale=1., + ): + super().__init__() + in_chans = cfg.VIDEO.BACKBONE.NUM_INPUT_CHANNELS + dims = cfg.VIDEO.BACKBONE.NUM_FILTERS + drop_path_rate = cfg.VIDEO.BACKBONE.DROP_PATH + depths = cfg.VIDEO.BACKBONE.DEPTH + layer_scale_init_value = cfg.VIDEO.BACKBONE.LARGE_SCALE_INIT_VALUE + stem_t_kernel_size = cfg.VIDEO.BACKBONE.STEM.T_KERNEL_SIZE if hasattr( + cfg.VIDEO.BACKBONE.STEM, 'T_KERNEL_SIZE') else 2 + t_stride = cfg.VIDEO.BACKBONE.STEM.T_STRIDE if hasattr( + cfg.VIDEO.BACKBONE.STEM, 'T_STRIDE') else 2 + + self.downsample_layers = nn.ModuleList( + ) # stem and 3 intermediate downsampling conv layers + stem = nn.Sequential( + nn.Conv3d( + in_chans, + dims[0], + kernel_size=(stem_t_kernel_size, 4, 4), + stride=(t_stride, 4, 4), + padding=((stem_t_kernel_size - 1) // 2, 0, 0)), + LayerNorm(dims[0], eps=1e-6, data_format='channels_first')) + self.downsample_layers.append(stem) + for i in range(3): + downsample_layer = nn.Sequential( + LayerNorm(dims[i], eps=1e-6, data_format='channels_first'), + nn.Conv3d( + dims[i], + dims[i + 1], + kernel_size=(1, 2, 2), + stride=(1, 2, 2)), + ) + self.downsample_layers.append(downsample_layer) + + self.stages = nn.ModuleList( + ) # 4 feature resolution stages, each consisting of multiple residual blocks + dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + cur = 0 + for i in range(4): + stage = nn.Sequential(*[ + TAdaConvNeXtBlock( + cfg, + dim=dims[i], + drop_path=dp_rates[cur + j], + layer_scale_init_value=layer_scale_init_value) + for j in range(depths[i]) + ]) + self.stages.append(stage) + cur += depths[i] + + self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer + + def forward_features(self, x): + for i in range(4): + x = self.downsample_layers[i](x) + x = self.stages[i](x) + return self.norm(x.mean( + [-3, -2, -1])) # global average pooling, (N, C, H, W) -> (N, C) + + def forward(self, x): + if isinstance(x, dict): + x = x['video'] + x = self.forward_features(x) + return x + + def get_num_layers(self): + return 12, 0 + + +class ConvNeXtBlock(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__(self, cfg, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + self.dwconv = nn.Conv3d( + dim, dim, kernel_size=(1, 7, 7), padding=(0, 3, 3), + groups=dim) # depthwise conv + self.norm = LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, + 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = x.permute(0, 2, 3, 4, 1) # (N, C, T, H, W) -> (N, T, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 4, 1, 2, 3) # (N, T, H, W, C) -> (N, C, T, H, W) + + x = input + self.drop_path(x) + return x + + +class LayerNorm(nn.Module): + r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with + shape (batch_size, height, width, channels) while channels_first corresponds to inputs + with shape (batch_size, channels, height, width). + """ + + def __init__(self, + normalized_shape, + eps=1e-6, + data_format='channels_last'): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ['channels_last', 'channels_first']: + raise NotImplementedError + self.normalized_shape = (normalized_shape, ) + + def forward(self, x): + if self.data_format == 'channels_last': + return F.layer_norm(x, self.normalized_shape, self.weight, + self.bias, self.eps) + elif self.data_format == 'channels_first': + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None, None] * x + self.bias[:, None, None, + None] + return x + + +class TAdaConvNeXtBlock(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_fi rst) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__(self, cfg, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + layer_scale_init_value = float(layer_scale_init_value) + self.dwconv = TAdaConv2d( + dim, + dim, + kernel_size=(1, 7, 7), + padding=(0, 3, 3), + groups=dim, + cal_dim='cout') + route_func_type = cfg.VIDEO.BACKBONE.BRANCH.ROUTE_FUNC_TYPE + if route_func_type == 'normal': + self.dwconv_rf = RouteFuncMLP( + c_in=dim, + ratio=cfg.VIDEO.BACKBONE.BRANCH.ROUTE_FUNC_R, + kernels=cfg.VIDEO.BACKBONE.BRANCH.ROUTE_FUNC_K, + with_bias_cal=self.dwconv.bias is not None) + elif route_func_type == 'normal_lngelu': + self.dwconv_rf = RouteFuncMLPLnGelu( + c_in=dim, + ratio=cfg.VIDEO.BACKBONE.BRANCH.ROUTE_FUNC_R, + kernels=cfg.VIDEO.BACKBONE.BRANCH.ROUTE_FUNC_K, + with_bias_cal=self.dwconv.bias is not None) + else: + raise ValueError( + 'Unknown route_func_type: {}'.format(route_func_type)) + self.norm = LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, + 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x, self.dwconv_rf(x)) + x = x.permute(0, 2, 3, 4, 1) # (N, C, T, H, W) -> (N, T, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 4, 1, 2, 3) # (N, T, H, W, C) -> (N, C, T, H, W) + + x = input + self.drop_path(x) + return x + + +class RouteFuncMLPLnGelu(nn.Module): + """ + The routing function for generating the calibration weights. + """ + + def __init__(self, + c_in, + ratio, + kernels, + with_bias_cal=False, + bn_eps=1e-5, + bn_mmt=0.1): + """ + Args: + c_in (int): number of input channels. + ratio (int): reduction ratio for the routing function. + kernels (list): temporal kernel size of the stacked 1D convolutions + """ + super(RouteFuncMLPLnGelu, self).__init__() + self.c_in = c_in + self.with_bias_cal = with_bias_cal + self.avgpool = nn.AdaptiveAvgPool3d((None, 1, 1)) + self.globalpool = nn.AdaptiveAvgPool3d(1) + self.g = nn.Conv3d( + in_channels=c_in, + out_channels=c_in, + kernel_size=1, + padding=0, + ) + self.a = nn.Conv3d( + in_channels=c_in, + out_channels=int(c_in // ratio), + kernel_size=[kernels[0], 1, 1], + padding=[kernels[0] // 2, 0, 0], + ) + # self.bn = nn.BatchNorm3d(int(c_in//ratio), eps=bn_eps, momentum=bn_mmt) + self.ln = LayerNorm( + int(c_in // ratio), eps=1e-6, data_format='channels_first') + self.gelu = nn.GELU() + # self.relu = nn.ReLU(inplace=True) + self.b = nn.Conv3d( + in_channels=int(c_in // ratio), + out_channels=c_in, + kernel_size=[kernels[1], 1, 1], + padding=[kernels[1] // 2, 0, 0], + bias=False) + self.b.skip_init = True + self.b.weight.data.zero_() # to make sure the initial values + # for the output is 1. + if with_bias_cal: + self.b_bias = nn.Conv3d( + in_channels=int(c_in // ratio), + out_channels=c_in, + kernel_size=[kernels[1], 1, 1], + padding=[kernels[1] // 2, 0, 0], + bias=False) + self.b_bias.skip_init = True + self.b_bias.weight.data.zero_() # to make sure the initial values + # for the output is 1. + + def forward(self, x): + g = self.globalpool(x) + x = self.avgpool(x) + x = self.a(x + self.g(g)) + # x = self.bn(x) + # x = self.relu(x) + x = self.ln(x) + x = self.gelu(x) + if self.with_bias_cal: + return [self.b(x) + 1, self.b_bias(x) + 1] + else: + return self.b(x) + 1 + + +class TAdaConv2d(nn.Module): + """ + Performs temporally adaptive 2D convolution. + Currently, only application on 5D tensors is supported, which makes TAdaConv2d + essentially a 3D convolution with temporal kernel size of 1. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + cal_dim='cin'): + super(TAdaConv2d, self).__init__() + """ + Args: + in_channels (int): number of input channels. + out_channels (int): number of output channels. + kernel_size (list): kernel size of TAdaConv2d. + stride (list): stride for the convolution in TAdaConv2d. + padding (list): padding for the convolution in TAdaConv2d. + dilation (list): dilation of the convolution in TAdaConv2d. + groups (int): number of groups for TAdaConv2d. + bias (bool): whether to use bias in TAdaConv2d. + calibration_mode (str): calibrated dimension in TAdaConv2d. + Supported input "cin", "cout". + """ + + kernel_size = _triple(kernel_size) + stride = _triple(stride) + padding = _triple(padding) + dilation = _triple(dilation) + + assert kernel_size[0] == 1 + assert stride[0] == 1 + assert padding[0] == 0 + assert dilation[0] == 1 + assert cal_dim in ['cin', 'cout'] + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.cal_dim = cal_dim + + # base weights (W_b) + self.weight = nn.Parameter( + torch.Tensor(1, 1, out_channels, in_channels // groups, + kernel_size[1], kernel_size[2])) + if bias: + self.bias = nn.Parameter(torch.Tensor(1, 1, out_channels)) + else: + self.register_parameter('bias', None) + + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + nn.init.uniform_(self.bias, -bound, bound) + + def forward(self, x, alpha): + """ + Args: + x (tensor): feature to perform convolution on. + alpha (tensor): calibration weight for the base weights. + W_t = alpha_t * W_b + """ + if isinstance(alpha, list): + w_alpha, b_alpha = alpha[0], alpha[1] + else: + w_alpha = alpha + b_alpha = None + _, _, c_out, c_in, kh, kw = self.weight.size() + b, c_in, t, h, w = x.size() + x = x.permute(0, 2, 1, 3, 4).reshape(1, -1, h, w) + + if self.cal_dim == 'cin': + # w_alpha: B, C, T, H(1), W(1) -> B, T, C, H(1), W(1) -> B, T, 1, C, H(1), W(1) + # corresponding to calibrating the input channel + weight = (w_alpha.permute(0, 2, 1, 3, 4).unsqueeze(2) + * self.weight).reshape(-1, c_in // self.groups, kh, kw) + elif self.cal_dim == 'cout': + # w_alpha: B, C, T, H(1), W(1) -> B, T, C, H(1), W(1) -> B, T, C, 1, H(1), W(1) + # corresponding to calibrating the input channel + weight = (w_alpha.permute(0, 2, 1, 3, 4).unsqueeze(3) + * self.weight).reshape(-1, c_in // self.groups, kh, kw) + + bias = None + if self.bias is not None: + if b_alpha is not None: + # b_alpha: B, C, T, H(1), W(1) -> B, T, C, H(1), W(1) -> B, T, C + bias = (b_alpha.permute(0, 2, 1, 3, 4).squeeze() + * self.bias).reshape(-1) + else: + bias = self.bias.repeat(b, t, 1).reshape(-1) + output = F.conv2d( + x, + weight=weight, + bias=bias, + stride=self.stride[1:], + padding=self.padding[1:], + dilation=self.dilation[1:], + groups=self.groups * b * t) + + output = output.view(b, t, c_out, output.size(-2), + output.size(-1)).permute(0, 2, 1, 3, 4) + + return output + + def __repr__(self): + return f'TAdaConv2d({self.in_channels}, {self.out_channels}, kernel_size={self.kernel_size}, ' +\ + f"stride={self.stride}, padding={self.padding}, bias={self.bias is not None}, cal_dim=\"{self.cal_dim}\")" diff --git a/runtime/Lib/site_packages/models/cv/action_recognition/temporal_patch_shift_transformer.py b/runtime/Lib/site_packages/models/cv/action_recognition/temporal_patch_shift_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..6abe3025718dd951e1d851ec10077e1b702ce76a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/action_recognition/temporal_patch_shift_transformer.py @@ -0,0 +1,1198 @@ +# Part of the implementation is borrowed and modified from Video Swin Transformer, +# publicly available at https://github.com/SwinTransformer/Video-Swin-Transformer + +from abc import ABCMeta, abstractmethod +from functools import lru_cache, reduce +from operator import mul + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import torchvision.transforms as T +from einops import rearrange +from timm.models.layers import DropPath, Mlp, trunc_normal_ + +from modelscope.models import TorchModel + + +def normal_init(module, mean=0., std=1., bias=0.): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.normal_(module.weight, mean, std) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def window_partition(x, window_size): + """ window_partition function. + Args: + x: (B, D, H, W, C) + window_size (tuple[int]): window size + + Returns: + windows: (B*num_windows, window_size*window_size, C) + """ + B, D, H, W, C = x.shape + x = x.view(B, D // window_size[0], window_size[0], H // window_size[1], + window_size[1], W // window_size[2], window_size[2], C) + windows = x.permute(0, 1, 3, 5, 2, 4, 6, + 7).contiguous().view(-1, reduce(mul, window_size), C) + return windows + + +def window_reverse(windows, window_size, B, D, H, W): + """ window_reverse function. + Args: + windows: (B*num_windows, window_size, window_size, C) + window_size (tuple[int]): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, D, H, W, C) + """ + x = windows.view(B, D // window_size[0], H // window_size[1], + W // window_size[2], window_size[0], window_size[1], + window_size[2], -1) + x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous().view(B, D, H, W, -1) + return x + + +def get_window_size(x_size, window_size, shift_size=None): + use_window_size = list(window_size) + if shift_size is not None: + use_shift_size = list(shift_size) + for i in range(len(x_size)): + if x_size[i] <= window_size[i]: + use_window_size[i] = x_size[i] + if shift_size is not None: + use_shift_size[i] = 0 + + if shift_size is None: + return tuple(use_window_size) + else: + return tuple(use_window_size), tuple(use_shift_size) + + +class WindowAttention3D(nn.Module): + """ This is PyTorch impl of TPS + + Window based multi-head self attention (W-MSA) module with relative position bias. + The coordinates of patches and patches are shifted together using Pattern C. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The temporal length, height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + shift (bool, optional): If True, conduct shift operation + shift_type (str, optional): shift operation type, either using 'psm' or 'tsm' + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + shift=False, + shift_type='psm'): + + super().__init__() + self.dim = dim + window_size = (16, 7, 7) + self.window_size = window_size # Wd, Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + self.shift = shift + self.shift_type = shift_type + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros( + np.prod([2 * ws - 1 for ws in window_size]), + num_heads)) # 2*Wd-1 * 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_d = torch.arange(self.window_size[0]) + coords_h = torch.arange(self.window_size[1]) + coords_w = torch.arange(self.window_size[2]) + coords = torch.stack( + torch.meshgrid(coords_d, coords_h, coords_w, + indexing='ij')) # 3, Wd, Wh, Ww + # Do the same rotation to coords + coords_old = coords.clone() + + # pattern patternC - 9 + coords[:, :, 0::3, 0::3] = torch.roll( + coords[:, :, 0::3, 0::3], shifts=-4, dims=1) + coords[:, :, 0::3, 1::3] = torch.roll( + coords[:, :, 0::3, 1::3], shifts=1, dims=1) + coords[:, :, 0::3, 2::3] = torch.roll( + coords[:, :, 0::3, 2::3], shifts=2, dims=1) + coords[:, :, 1::3, 2::3] = torch.roll( + coords[:, :, 1::3, 2::3], shifts=3, dims=1) + coords[:, :, 1::3, 0::3] = torch.roll( + coords[:, :, 1::3, 0::3], shifts=-1, dims=1) + coords[:, :, 2::3, 0::3] = torch.roll( + coords[:, :, 2::3, 0::3], shifts=-2, dims=1) + coords[:, :, 2::3, 1::3] = torch.roll( + coords[:, :, 2::3, 1::3], shifts=-3, dims=1) + coords[:, :, 2::3, 2::3] = torch.roll( + coords[:, :, 2::3, 2::3], shifts=4, dims=1) + + coords_flatten = torch.flatten(coords, 1) # 3, Wd*Wh*Ww + coords_old_flatten = torch.flatten(coords_old, 1) + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 3, Wd*Wh*Ww, Wd*Wh*Ww + relative_coords_old = coords_old_flatten[:, :, + None] - coords_old_flatten[:, + None, :] # 3, Wd*Wh*Ww, Wd*Wh*Ww + + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wd*Wh*Ww, Wd*Wh*Ww, 3 + relative_coords_old = relative_coords_old.permute( + 1, 2, 0).contiguous() # Wd*Wh*Ww, Wd*Wh*Ww, 3 + + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 2] += self.window_size[2] - 1 + + relative_coords_old[:, :, 0] += self.window_size[ + 0] - 1 # shift to start from 0 + relative_coords_old[:, :, 1] += self.window_size[1] - 1 + relative_coords_old[:, :, 2] += self.window_size[2] - 1 + + relative_coords[:, :, 0] *= (2 * self.window_size[1] + - 1) * (2 * self.window_size[2] - 1) + relative_coords[:, :, 1] *= (2 * self.window_size[2] - 1) + + relative_coords_old[:, :, 0] *= (2 * self.window_size[1] + - 1) * (2 * self.window_size[2] - 1) + relative_coords_old[:, :, 1] *= (2 * self.window_size[2] - 1) + + relative_position_index = relative_coords.sum(-1) # Wd*Wh*Ww, Wd*Wh*Ww + + relative_position_index_old = relative_coords_old.sum(-1) + relative_position_index = relative_position_index.view( + window_size[0], window_size[1] * window_size[2], window_size[0], + window_size[1] * window_size[2]).permute(0, 2, 1, 3).reshape( + window_size[0] * window_size[0], + window_size[1] * window_size[2], + window_size[1] * window_size[2])[::window_size[0], :, :] + + relative_position_index_old = relative_position_index_old.view( + window_size[0], window_size[1] * window_size[2], window_size[0], + window_size[1] * window_size[2]).permute(0, 2, 1, 3).reshape( + window_size[0] * window_size[0], + window_size[1] * window_size[2], + window_size[1] * window_size[2])[::window_size[0], :, :] + + self.register_buffer('relative_position_index', + relative_position_index) + self.register_buffer('relative_position_index_old', + relative_position_index_old) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + if self.shift and self.shift_type == 'psm': + self.shift_op = PatchShift(False, 1) + self.shift_op_back = PatchShift(True, 1) + elif self.shift and self.shift_type == 'tsm': + self.shift_op = TemporalShift(8) + + def forward(self, x, mask=None, batch_size=8, frame_len=8): + """ Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, N, N) or None + """ + B_, N, C = x.shape + if self.shift: + x = x.view(B_, N, self.num_heads, + C // self.num_heads).permute(0, 2, 1, 3) + + x = self.shift_op(x, batch_size, frame_len) + x = x.permute(0, 2, 1, 3).reshape(B_, N, C) + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # B_, nH, N, C + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + if self.shift and self.shift_type == 'psm': + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index[:].reshape(-1), :].reshape( + frame_len, N, N, -1) # 8frames ,Wd*Wh*Ww,Wd*Wh*Ww,nH + else: + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index_old[:].reshape(-1), :].reshape( + frame_len, N, N, -1) # 8frames ,Wd*Wh*Ww,Wd*Wh*Ww,nH + + relative_position_bias = relative_position_bias.permute( + 0, 3, 1, 2).contiguous() # Frames, nH, Wd*Wh*Ww, Wd*Wh*Ww + + attn = attn.view( + batch_size, frame_len, -1, self.num_heads, N, N).permute( + 0, + 2, 1, 3, 4, 5) + relative_position_bias.unsqueeze(0).unsqueeze( + 1) # B_, nH, N, N + attn = attn.permute(0, 2, 1, 3, 4, 5).view(-1, self.num_heads, N, N) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + # Shift back for psm + if self.shift and self.shift_type == 'psm': + x = self.shift_op_back(attn @ v, batch_size, + frame_len).transpose(1, + 2).reshape(B_, N, C) + else: + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class PatchShift(nn.Module): + """ This is PyTorch impl of TPS + + The patches are shifted using Pattern C. + + It supports both of shifted and shift back. + + Args: + inv (bool): whether using inverse shifted (shift back) + ratio (float): ratio of channels to be shifted, patch shift using 1.0 + """ + + def __init__(self, inv=False, ratio=1): + super(PatchShift, self).__init__() + self.inv = inv + self.ratio = ratio + # if inv: + # print('=> Using inverse PatchShift, ratio {}, tps'.format(ratio)) + # else: + # print('=> Using bayershift, ratio {}, tps'.format(ratio)) + + def forward(self, x, batch_size, frame_len): + x = self.shift( + x, + inv=self.inv, + ratio=self.ratio, + batch_size=batch_size, + frame_len=frame_len) + return x + + @staticmethod + def shift(x, inv=False, ratio=0.5, batch_size=8, frame_len=8): + B, num_heads, N, c = x.size() + fold = int(num_heads * ratio) + feat = x + feat = feat.view(batch_size, frame_len, -1, num_heads, 7, 7, c) + out = feat.clone() + multiplier = 1 + stride = 1 + if inv: + multiplier = -1 + + # Pattern C + out[:, :, :, :fold, 0::3, 0::3, :] = torch.roll( + feat[:, :, :, :fold, 0::3, 0::3, :], + shifts=-4 * multiplier * stride, + dims=1) + out[:, :, :, :fold, 0::3, 1::3, :] = torch.roll( + feat[:, :, :, :fold, 0::3, 1::3, :], + shifts=multiplier * stride, + dims=1) + out[:, :, :, :fold, 1::3, 0::3, :] = torch.roll( + feat[:, :, :, :fold, 1::3, 0::3, :], + shifts=-multiplier * stride, + dims=1) + out[:, :, :, :fold, 0::3, 2::3, :] = torch.roll( + feat[:, :, :, :fold, 0::3, 2::3, :], + shifts=2 * multiplier * stride, + dims=1) + out[:, :, :, :fold, 2::3, 0::3, :] = torch.roll( + feat[:, :, :, :fold, 2::3, 0::3, :], + shifts=-2 * multiplier * stride, + dims=1) + out[:, :, :, :fold, 1::3, 2::3, :] = torch.roll( + feat[:, :, :, :fold, 1::3, 2::3, :], + shifts=3 * multiplier * stride, + dims=1) + out[:, :, :, :fold, 2::3, 1::3, :] = torch.roll( + feat[:, :, :, :fold, 2::3, 1::3, :], + shifts=-3 * multiplier * stride, + dims=1) + out[:, :, :, :fold, 2::3, 2::3, :] = torch.roll( + feat[:, :, :, :fold, 2::3, 2::3, :], + shifts=4 * multiplier * stride, + dims=1) + + out = out.view(B, num_heads, N, c) + return out + + +class TemporalShift(nn.Module): + """ This is PyTorch impl of TPS + + The temporal channel shift. + + The code is adopted from TSM: Temporal Shift Module for Efficient Video Understanding. ICCV19 + + https://github.com/mit-han-lab/temporal-shift-module/blob/master/ops/temporal_shift.py + + Args: + n_div (int): propotion of channel to be shifted. + """ + + def __init__(self, n_div=8): + super(TemporalShift, self).__init__() + self.fold_div = n_div + + def forward(self, x, batch_size, frame_len): + x = self.shift( + x, + fold_div=self.fold_div, + batch_size=batch_size, + frame_len=frame_len) + return x + + @staticmethod + def shift(x, fold_div=8, batch_size=8, frame_len=8): + B, num_heads, N, c = x.size() + fold = c // fold_div + feat = x + feat = feat.view(batch_size, frame_len, -1, num_heads, N, c) + out = feat.clone() + + out[:, 1:, :, :, :, :fold] = feat[:, :-1, :, :, :, :fold] # shift left + out[:, :-1, :, :, :, + fold:2 * fold] = feat[:, 1:, :, :, :, fold:2 * fold] # shift right + + out = out.view(B, num_heads, N, c) + + return out + + +class SwinTransformerBlock3D(nn.Module): + """ Swin Transformer Block from Video Swin Transformer. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (tuple[int]): Window size. + shift_size (tuple[int]): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + window_size=(2, 7, 7), + shift_size=(0, 0, 0), + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + use_checkpoint=False, + shift=False, + shift_type='psm'): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.use_checkpoint = use_checkpoint + self.shift = shift + self.shift_type = shift_type + + assert 0 <= self.shift_size[0] < self.window_size[ + 0], 'shift_size must in 0-window_size' + assert 0 <= self.shift_size[1] < self.window_size[ + 1], 'shift_size must in 0-window_size' + assert 0 <= self.shift_size[2] < self.window_size[ + 2], 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention3D( + dim, + window_size=self.window_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + shift=self.shift, + shift_type=self.shift_type) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + def forward_part1(self, x, mask_matrix): + B, D, H, W, C = x.shape + window_size, shift_size = get_window_size((D, H, W), self.window_size, + self.shift_size) + + x = self.norm1(x) + # pad feature maps to multiples of window size + pad_l = pad_t = pad_d0 = 0 + pad_d1 = (window_size[0] - D % window_size[0]) % window_size[0] + pad_b = (window_size[1] - H % window_size[1]) % window_size[1] + pad_r = (window_size[2] - W % window_size[2]) % window_size[2] + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b, pad_d0, pad_d1)) + _, Dp, Hp, Wp, _ = x.shape + # cyclic shift + if any(i > 0 for i in shift_size): + shifted_x = torch.roll( + x, + shifts=(-shift_size[0], -shift_size[1], -shift_size[2]), + dims=(1, 2, 3)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + # partition windows + x_windows = window_partition(shifted_x, + window_size) # B*nW, Wd*Wh*Ww, C + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask, batch_size=B, + frame_len=D) # B*nW, Wd*Wh*Ww, C + # merge windows + attn_windows = attn_windows.view(-1, *(window_size + (C, ))) + shifted_x = window_reverse(attn_windows, window_size, B, Dp, Hp, + Wp) # B D' H' W' C + # reverse cyclic shift + if any(i > 0 for i in shift_size): + x = torch.roll( + shifted_x, + shifts=(shift_size[0], shift_size[1], shift_size[2]), + dims=(1, 2, 3)) + else: + x = shifted_x + + if pad_d1 > 0 or pad_r > 0 or pad_b > 0: + x = x[:, :D, :H, :W, :].contiguous() + return x + + def forward_part2(self, x): + return self.drop_path(self.mlp(self.norm2(x))) + + def forward(self, x, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, D, H, W, C). + mask_matrix: Attention mask for cyclic shift. + """ + + shortcut = x + if self.use_checkpoint: + x = checkpoint.checkpoint(self.forward_part1, x, mask_matrix) + else: + x = self.forward_part1(x, mask_matrix) + x = shortcut + self.drop_path(x) + + if self.use_checkpoint: + x = x + checkpoint.checkpoint(self.forward_part2, x) + else: + x = x + self.forward_part2(x) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer from Video Swin Transformer. + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ Forward function. + + Args: + x: Input feature, tensor size (B, D, H, W, C). + """ + B, D, H, W, C = x.shape + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, :, 0::2, 0::2, :] # B D H/2 W/2 C + x1 = x[:, :, 1::2, 0::2, :] # B D H/2 W/2 C + x2 = x[:, :, 0::2, 1::2, :] # B D H/2 W/2 C + x3 = x[:, :, 1::2, 1::2, :] # B D H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B D H/2 W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +@lru_cache() +def compute_mask(D, H, W, window_size, shift_size, device): + img_mask = torch.zeros((1, D, H, W, 1), device=device) # 1 Dp Hp Wp 1 + cnt = 0 + for d in slice(-window_size[0]), slice(-window_size[0], + -shift_size[0]), slice( + -shift_size[0], None): + for h in slice(-window_size[1]), slice(-window_size[1], + -shift_size[1]), slice( + -shift_size[1], None): + for w in slice(-window_size[2]), slice(-window_size[2], + -shift_size[2]), slice( + -shift_size[2], None): + img_mask[:, d, h, w, :] = cnt + cnt += 1 + mask_windows = window_partition(img_mask, + window_size) # nW, ws[0]*ws[1]*ws[2], 1 + mask_windows = mask_windows.squeeze(-1) # nW, ws[0]*ws[1]*ws[2] + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + return attn_mask + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage from Video Swin Transformer. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (tuple[int]): Local window size. Default: (1,7,7). + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=(1, 7, 7), + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + shift_type='psm'): + super().__init__() + self.window_size = window_size + self.shift_size = tuple(i // 2 for i in window_size) + self.depth = depth + self.use_checkpoint = use_checkpoint + self.shift_type = shift_type + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock3D( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=(0, 0, 0) if (i % 2 == 0) else self.shift_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + use_checkpoint=use_checkpoint, + shift=True, + shift_type='tsm' if (i % 2 == 0 and self.shift_type == 'psm') + or self.shift_type == 'tsm' else 'psm', + ) for i in range(depth) + ]) + + self.downsample = downsample + if self.downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + + def forward(self, x): + """ Forward function. + + Args: + x: Input feature, tensor size (B, C, D, H, W). + """ + # calculate attention mask for SW-MSA + B, C, D, H, W = x.shape + window_size, shift_size = get_window_size((D, H, W), self.window_size, + self.shift_size) + x = rearrange(x, 'b c d h w -> b d h w c') + Dp = int(np.ceil(D / window_size[0])) * window_size[0] + Hp = int(np.ceil(H / window_size[1])) * window_size[1] + Wp = int(np.ceil(W / window_size[2])) * window_size[2] + attn_mask = compute_mask(Dp, Hp, Wp, window_size, shift_size, x.device) + for blk in self.blocks: + x = blk(x, attn_mask) + x = x.view(B, D, H, W, -1) + + if self.downsample is not None: + x = self.downsample(x) + x = rearrange(x, 'b d h w c -> b c d h w') + return x + + +class PatchEmbed3D(nn.Module): + """ Video to Patch Embedding from Video Swin Transformer. + + Args: + patch_size (int): Patch token size. Default: (2,4,4). + in_chans (int): Number of input video channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=(2, 4, 4), + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv3d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, D, H, W = x.size() + if W % self.patch_size[2] != 0: + x = F.pad(x, (0, self.patch_size[2] - W % self.patch_size[2])) + if H % self.patch_size[1] != 0: + x = F.pad(x, + (0, 0, 0, self.patch_size[1] - H % self.patch_size[1])) + if D % self.patch_size[0] != 0: + x = F.pad( + x, + (0, 0, 0, 0, 0, self.patch_size[0] - D % self.patch_size[0])) + + x = self.proj(x) # B C D Wh Ww + if self.norm is not None: + D, Wh, Ww = x.size(2), x.size(3), x.size(4) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, D, Wh, Ww) + + return x + + +class SwinTransformer2D_TPS(nn.Module): + """ + Code is adopted from Video Swin Transformer. + + Args: + patch_size (int | tuple(int)): Patch size. Default: (4,4,4). + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer: Normalization layer. Default: nn.LayerNorm. + patch_norm (bool): If True, add normalization after patch embedding. Default: False. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + """ + + def __init__(self, + pretrained=None, + pretrained2d=True, + patch_size=(4, 4, 4), + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=(2, 7, 7), + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + patch_norm=False, + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrained = pretrained + self.pretrained2d = pretrained2d + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.patch_norm = patch_norm + self.frozen_stages = frozen_stages + self.window_size = window_size + self.patch_size = patch_size + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed3D( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging + if i_layer < self.num_layers - 1 else None, + use_checkpoint=use_checkpoint, + shift_type='psm') + self.layers.append(layer) + + self.num_features = int(embed_dim * 2**(self.num_layers - 1)) + + # add a norm layer for each output + self.norm = norm_layer(self.num_features) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1: + self.pos_drop.eval() + for i in range(0, self.frozen_stages): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def inflate_weights(self): + """Inflate the swin2d parameters to swin3d. + + The differences between swin3d and swin2d mainly lie in an extra + axis. To utilize the pretrained parameters in 2d model, + the weight of swin2d models should be inflated to fit in the shapes of + the 3d counterpart. + + Args: + logger (logging.Logger): The logger used to print + debugging information. + """ + checkpoint = torch.load(self.pretrained, map_location='cpu') + state_dict = checkpoint['model'] + + # delete relative_position_index since we always re-init it + relative_position_index_keys = [ + k for k in state_dict.keys() if 'relative_position_index' in k + ] + for k in relative_position_index_keys: + del state_dict[k] + + # delete attn_mask since we always re-init it + attn_mask_keys = [k for k in state_dict.keys() if 'attn_mask' in k] + for k in attn_mask_keys: + del state_dict[k] + + state_dict['patch_embed.proj.weight'] = state_dict[ + 'patch_embed.proj.weight'].unsqueeze(2).repeat( + 1, 1, self.patch_size[0], 1, 1) / self.patch_size[0] + + # bicubic interpolate relative_position_bias_table if not match + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for k in relative_position_bias_table_keys: + relative_position_bias_table_pretrained = state_dict[k] + relative_position_bias_table_current = self.state_dict()[k] + L1, nH1 = relative_position_bias_table_pretrained.size() + L2, nH2 = relative_position_bias_table_current.size() + L2 = (2 * self.window_size[1] - 1) * (2 * self.window_size[2] - 1) + # wd = self.window_size[0] + # to make it match + wd = 16 + if nH1 != nH2: + print(f'Error in loading {k}, passing') + else: + if L1 != L2: + S1 = int(L1**0.5) + relative_position_bias_table_pretrained_resized = torch.nn.functional.interpolate( + relative_position_bias_table_pretrained.permute( + 1, 0).view(1, nH1, S1, S1), + size=(2 * self.window_size[1] - 1, + 2 * self.window_size[2] - 1), + mode='bicubic') + relative_position_bias_table_pretrained = relative_position_bias_table_pretrained_resized.view( + nH2, L2).permute(1, 0) + state_dict[k] = relative_position_bias_table_pretrained.repeat( + 2 * wd - 1, 1) + + msg = self.load_state_dict(state_dict, strict=False) + print(msg) + print(f"=> loaded successfully '{self.pretrained}'") + del checkpoint + torch.cuda.empty_cache() + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if pretrained: + self.pretrained = pretrained + if isinstance(self.pretrained, str): + self.apply(_init_weights) + print(f'load model from: {self.pretrained}') + + if self.pretrained2d: + # Inflate 2D model into 3D model. + # self.inflate_weights(logger) + self.inflate_weights() + else: + # Directly load 3D model. + torch.load_checkpoint(self, self.pretrained, strict=False) + elif self.pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + x = self.pos_drop(x) + + for layer in self.layers: + x = layer(x.contiguous()) + + x = rearrange(x, 'n c d h w -> n d h w c') + x = self.norm(x) + x = rearrange(x, 'n d h w c -> n c d h w') + + return x + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer2D_TPS, self).train(mode) + self._freeze_stages() + + +def top_k_accuracy(scores, labels, topk=(1, )): + """Calculate top k accuracy score from mmaction. + + Args: + scores (list[np.ndarray]): Prediction scores for each class. + labels (list[int]): Ground truth labels. + topk (tuple[int]): K value for top_k_accuracy. Default: (1, ). + + Returns: + list[float]: Top k accuracy score for each k. + """ + res = [] + labels = np.array(labels)[:, np.newaxis] + for k in topk: + max_k_preds = np.argsort(scores, axis=1)[:, -k:][:, ::-1] + match_array = np.logical_or.reduce(max_k_preds == labels, axis=1) + topk_acc_score = match_array.sum() / match_array.shape[0] + res.append(topk_acc_score) + + return res + + +class BaseHead(nn.Module, metaclass=ABCMeta): + """Base class for head from mmaction. + + All Head should subclass it. + All subclass should overwrite: + - Methods:``init_weights``, initializing weights in some modules. + - Methods:``forward``, supporting to forward both for training and testing. + + Args: + num_classes (int): Number of classes to be classified. + in_channels (int): Number of channels in input feature. + loss_cls (dict): Config for building loss. + Default: dict(type='CrossEntropyLoss', loss_weight=1.0). + multi_class (bool): Determines whether it is a multi-class + recognition task. Default: False. + label_smooth_eps (float): Epsilon used in label smooth. + Reference: arxiv.org/abs/1906.02629. Default: 0. + """ + + def __init__(self, + num_classes, + in_channels, + loss_cls=dict(type='CrossEntropyLoss', loss_weight=1.0), + multi_class=False, + label_smooth_eps=0.0): + super().__init__() + self.num_classes = num_classes + self.in_channels = in_channels + self.loss_cls = torch.nn.CrossEntropyLoss() + self.multi_class = multi_class + self.label_smooth_eps = label_smooth_eps + + @abstractmethod + def init_weights(self): + """Initiate the parameters either from existing checkpoint or from + scratch.""" + + @abstractmethod + def forward(self, x): + """Defines the computation performed at every call.""" + + def loss(self, cls_score, labels, **kwargs): + """Calculate the loss given output ``cls_score``, target ``labels``. + + Args: + cls_score (torch.Tensor): The output of the model. + labels (torch.Tensor): The target output of the model. + + Returns: + dict: A dict containing field 'loss_cls'(mandatory) + and 'top1_acc', 'top5_acc'(optional). + """ + losses = dict() + if labels.shape == torch.Size([]): + labels = labels.unsqueeze(0) + elif labels.dim() == 1 and labels.size()[0] == self.num_classes \ + and cls_score.size()[0] == 1: + # Fix a bug when training with soft labels and batch size is 1. + # When using soft labels, `labels` and `cls_score` share the same + # shape. + labels = labels.unsqueeze(0) + + if not self.multi_class and cls_score.size() != labels.size(): + top_k_acc = top_k_accuracy(cls_score.detach().cpu().numpy(), + labels.detach().cpu().numpy(), (1, 5)) + losses['top1_acc'] = torch.tensor( + top_k_acc[0], device=cls_score.device) + losses['top5_acc'] = torch.tensor( + top_k_acc[1], device=cls_score.device) + + elif self.multi_class and self.label_smooth_eps != 0: + labels = ((1 - self.label_smooth_eps) * labels + + self.label_smooth_eps / self.num_classes) + + loss_cls = self.loss_cls(cls_score, labels, **kwargs) + # loss_cls may be dictionary or single tensor + if isinstance(loss_cls, dict): + losses.update(loss_cls) + else: + losses['loss_cls'] = loss_cls + + return losses + + +class I3DHead(BaseHead): + """Classification head for I3D from mmaction. + + Args: + num_classes (int): Number of classes to be classified. + in_channels (int): Number of channels in input feature. + loss_cls (dict): Config for building loss. + Default: dict(type='CrossEntropyLoss') + spatial_type (str): Pooling type in spatial dimension. Default: 'avg'. + dropout_ratio (float): Probability of dropout layer. Default: 0.5. + init_std (float): Std value for Initiation. Default: 0.01. + kwargs (dict, optional): Any keyword argument to be used to initialize + the head. + """ + + def __init__(self, + num_classes, + in_channels, + loss_cls=dict(type='CrossEntropyLoss'), + spatial_type='avg', + dropout_ratio=0.5, + init_std=0.01, + **kwargs): + super().__init__(num_classes, in_channels, loss_cls, **kwargs) + + self.spatial_type = spatial_type + self.dropout_ratio = dropout_ratio + self.init_std = init_std + if self.dropout_ratio != 0: + self.dropout = nn.Dropout(p=self.dropout_ratio) + else: + self.dropout = None + self.fc_cls = nn.Linear(self.in_channels, self.num_classes) + + if self.spatial_type == 'avg': + # use `nn.AdaptiveAvgPool3d` to adaptively match the in_channels. + self.avg_pool = nn.AdaptiveAvgPool3d((1, 1, 1)) + else: + self.avg_pool = None + + def init_weights(self): + """Initiate the parameters from scratch.""" + normal_init(self.fc_cls, std=self.init_std) + + def forward(self, x): + """Defines the computation performed at every call. + + Args: + x (torch.Tensor): The input data. + + Returns: + torch.Tensor: The classification scores for input samples. + """ + # [N, in_channels, 4, 7, 7] + if self.avg_pool is not None: + x = self.avg_pool(x) + # [N, in_channels, 1, 1, 1] + if self.dropout is not None: + x = self.dropout(x) + # [N, in_channels, 1, 1, 1] + x = x.view(x.shape[0], -1) + # [N, in_channels] + cls_score = self.fc_cls(x) + # [N, num_classes] + return cls_score + + +class PatchShiftTransformer(TorchModel): + """ This is PyTorch impl of PST: + Spatiotemporal Self-attention Modeling with Temporal Patch Shift for Action Recognition, ECCV22. + """ + + def __init__(self, + model_dir=None, + num_classes=400, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + embed_dim=96, + in_channels=768, + pretrained=None): + super().__init__(model_dir) + self.backbone = SwinTransformer2D_TPS( + pretrained=pretrained, + pretrained2d=True, + patch_size=(2, 4, 4), + in_chans=3, + embed_dim=embed_dim, + depths=depths, + num_heads=num_heads, + window_size=(1, 7, 7), + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + patch_norm=True, + frozen_stages=-1, + use_checkpoint=False) + self.cls_head = I3DHead( + num_classes=num_classes, in_channels=in_channels) + + def forward(self, x): + feature = self.backbone(x) + output = self.cls_head(feature) + return output diff --git a/runtime/Lib/site_packages/models/cv/animal_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/animal_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..00a37a3f405cd5672be716ea06a61f1777f7640f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/animal_recognition/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .resnet import ResNet, Bottleneck + from .splat import SplAtConv2d + +else: + _import_structure = { + 'resnet': ['ResNet', 'Bottleneck'], + 'splat': ['SplAtConv2d'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/animal_recognition/resnet.py b/runtime/Lib/site_packages/models/cv/animal_recognition/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..d7c03c299c542838124bc24310c07f9f7c1ca049 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/animal_recognition/resnet.py @@ -0,0 +1,430 @@ +# The implementation is adopted from Split-Attention Network, A New ResNet Variant, +# made pubicly available under the Apache License 2.0 License +# at https://github.com/zhanghang1989/ResNeSt/blob/master/resnest/torch/models/resnet.py +import math + +import torch +import torch.nn as nn + +from .splat import SplAtConv2d + +__all__ = ['ResNet', 'Bottleneck'] + + +class DropBlock2D(object): + + def __init__(self, *args, **kwargs): + raise NotImplementedError + + +class GlobalAvgPool2d(nn.Module): + + def __init__(self): + """Global average pooling over the input's spatial dimensions""" + super(GlobalAvgPool2d, self).__init__() + + def forward(self, inputs): + return nn.functional.adaptive_avg_pool2d(inputs, + 1).view(inputs.size(0), -1) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + radix=1, + cardinality=1, + bottleneck_width=64, + avd=False, + avd_first=False, + dilation=1, + is_first=False, + rectified_conv=False, + rectify_avg=False, + norm_layer=None, + dropblock_prob=0.0, + last_gamma=False): + super(Bottleneck, self).__init__() + group_width = int(planes * (bottleneck_width / 64.)) * cardinality + self.conv1 = nn.Conv2d( + inplanes, group_width, kernel_size=1, bias=False) + self.bn1 = norm_layer(group_width) + self.dropblock_prob = dropblock_prob + self.radix = radix + self.avd = avd and (stride > 1 or is_first) + self.avd_first = avd_first + + if self.avd: + self.avd_layer = nn.AvgPool2d(3, stride, padding=1) + stride = 1 + + if dropblock_prob > 0.0: + self.dropblock1 = DropBlock2D(dropblock_prob, 3) + if radix == 1: + self.dropblock2 = DropBlock2D(dropblock_prob, 3) + self.dropblock3 = DropBlock2D(dropblock_prob, 3) + + if radix >= 1: + self.conv2 = SplAtConv2d( + group_width, + group_width, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=cardinality, + bias=False, + radix=radix, + rectify=rectified_conv, + rectify_avg=rectify_avg, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + elif rectified_conv: + self.conv2 = nn.Conv2d( + group_width, + group_width, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=cardinality, + bias=False) + self.bn2 = norm_layer(group_width) + else: + self.conv2 = nn.Conv2d( + group_width, + group_width, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=cardinality, + bias=False) + self.bn2 = norm_layer(group_width) + + self.conv3 = nn.Conv2d( + group_width, planes * 4, kernel_size=1, bias=False) + self.bn3 = norm_layer(planes * 4) + + if last_gamma: + from torch.nn.init import zeros_ + zeros_(self.bn3.weight) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.dilation = dilation + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + if self.dropblock_prob > 0.0: + out = self.dropblock1(out) + out = self.relu(out) + + if self.avd and self.avd_first: + out = self.avd_layer(out) + + out = self.conv2(out) + if self.radix == 0: + out = self.bn2(out) + if self.dropblock_prob > 0.0: + out = self.dropblock2(out) + out = self.relu(out) + + if self.avd and not self.avd_first: + out = self.avd_layer(out) + + out = self.conv3(out) + out = self.bn3(out) + if self.dropblock_prob > 0.0: + out = self.dropblock3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, + block, + layers, + radix=1, + groups=1, + bottleneck_width=64, + num_classes=1000, + dilated=False, + dilation=1, + deep_stem=False, + stem_width=64, + avg_down=False, + rectified_conv=False, + rectify_avg=False, + avd=False, + avd_first=False, + final_drop=0.0, + dropblock_prob=0, + last_gamma=False, + norm_layer=nn.BatchNorm2d): + self.cardinality = groups + self.bottleneck_width = bottleneck_width + # ResNet-D params + self.inplanes = stem_width * 2 if deep_stem else 64 + self.avg_down = avg_down + self.last_gamma = last_gamma + # ResNeSt params + self.radix = radix + self.avd = avd + self.avd_first = avd_first + + super(ResNet, self).__init__() + self.rectified_conv = rectified_conv + self.rectify_avg = rectify_avg + if rectified_conv: + conv_layer = nn.Conv2d + else: + conv_layer = nn.Conv2d + conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {} + if deep_stem: + self.conv1 = nn.Sequential( + conv_layer( + 3, + stem_width, + kernel_size=3, + stride=2, + padding=1, + bias=False, + **conv_kwargs), + norm_layer(stem_width), + nn.ReLU(inplace=True), + conv_layer( + stem_width, + stem_width, + kernel_size=3, + stride=1, + padding=1, + bias=False, + **conv_kwargs), + norm_layer(stem_width), + nn.ReLU(inplace=True), + conv_layer( + stem_width, + stem_width * 2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + **conv_kwargs), + ) + else: + self.conv1 = conv_layer( + 3, + 64, + kernel_size=7, + stride=2, + padding=3, + bias=False, + **conv_kwargs) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer( + block, 64, layers[0], norm_layer=norm_layer, is_first=False) + self.layer2 = self._make_layer( + block, 128, layers[1], stride=2, norm_layer=norm_layer) + if dilated or dilation == 4: + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=1, + dilation=2, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=1, + dilation=4, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + elif dilation == 2: + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilation=1, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=1, + dilation=2, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + else: + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob) + self.avgpool = GlobalAvgPool2d() + self.drop = nn.Dropout(final_drop) if final_drop > 0.0 else None + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, norm_layer): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, + block, + planes, + blocks, + stride=1, + dilation=1, + norm_layer=None, + dropblock_prob=0.0, + is_first=True): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + down_layers = [] + if self.avg_down: + if dilation == 1: + down_layers.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + else: + down_layers.append( + nn.AvgPool2d( + kernel_size=1, + stride=1, + ceil_mode=True, + count_include_pad=False)) + down_layers.append( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=1, + bias=False)) + else: + down_layers.append( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False)) + down_layers.append(norm_layer(planes * block.expansion)) + downsample = nn.Sequential(*down_layers) + + layers = [] + if dilation == 1 or dilation == 2: + layers.append( + block( + self.inplanes, + planes, + stride, + downsample=downsample, + radix=self.radix, + cardinality=self.cardinality, + bottleneck_width=self.bottleneck_width, + avd=self.avd, + avd_first=self.avd_first, + dilation=1, + is_first=is_first, + rectified_conv=self.rectified_conv, + rectify_avg=self.rectify_avg, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob, + last_gamma=self.last_gamma)) + elif dilation == 4: + layers.append( + block( + self.inplanes, + planes, + stride, + downsample=downsample, + radix=self.radix, + cardinality=self.cardinality, + bottleneck_width=self.bottleneck_width, + avd=self.avd, + avd_first=self.avd_first, + dilation=2, + is_first=is_first, + rectified_conv=self.rectified_conv, + rectify_avg=self.rectify_avg, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob, + last_gamma=self.last_gamma)) + else: + raise RuntimeError('=> unknown dilation size: {}'.format(dilation)) + + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + radix=self.radix, + cardinality=self.cardinality, + bottleneck_width=self.bottleneck_width, + avd=self.avd, + avd_first=self.avd_first, + dilation=dilation, + rectified_conv=self.rectified_conv, + rectify_avg=self.rectify_avg, + norm_layer=norm_layer, + dropblock_prob=dropblock_prob, + last_gamma=self.last_gamma)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = torch.flatten(x, 1) + if self.drop: + x = self.drop(x) + x = self.fc(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/animal_recognition/splat.py b/runtime/Lib/site_packages/models/cv/animal_recognition/splat.py new file mode 100644 index 0000000000000000000000000000000000000000..a10d0abe190f18a26e6281cd85c7e97aace290de --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/animal_recognition/splat.py @@ -0,0 +1,126 @@ +# The implementation is adopted from Split-Attention Network, A New ResNet Variant, +# made pubicly available under the Apache License 2.0 License +# at https://github.com/zhanghang1989/ResNeSt/blob/master/resnest/torch/models/splat.py +"""Split-Attention""" + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import BatchNorm2d, Conv2d, Linear, Module, ReLU +from torch.nn.modules.utils import _pair + +__all__ = ['SplAtConv2d'] + + +class SplAtConv2d(Module): + """Split-Attention Conv2d + """ + + def __init__(self, + in_channels, + channels, + kernel_size, + stride=(1, 1), + padding=(0, 0), + dilation=(1, 1), + groups=1, + bias=True, + radix=2, + reduction_factor=4, + rectify=False, + rectify_avg=False, + norm_layer=None, + dropblock_prob=0.0, + **kwargs): + super(SplAtConv2d, self).__init__() + padding = _pair(padding) + self.rectify = rectify and (padding[0] > 0 or padding[1] > 0) + self.rectify_avg = rectify_avg + inter_channels = max(in_channels * radix // reduction_factor, 32) + self.radix = radix + self.cardinality = groups + self.channels = channels + self.dropblock_prob = dropblock_prob + if self.rectify: + self.conv = Conv2d( + in_channels, + channels * radix, + kernel_size, + stride, + padding, + dilation, + groups=groups * radix, + bias=bias, + **kwargs) + else: + self.conv = Conv2d( + in_channels, + channels * radix, + kernel_size, + stride, + padding, + dilation, + groups=groups * radix, + bias=bias, + **kwargs) + self.use_bn = norm_layer is not None + if self.use_bn: + self.bn0 = norm_layer(channels * radix) + self.relu = ReLU(inplace=True) + self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality) + if self.use_bn: + self.bn1 = norm_layer(inter_channels) + self.fc2 = Conv2d( + inter_channels, channels * radix, 1, groups=self.cardinality) + if dropblock_prob > 0.0: + self.dropblock = DropBlock2D(dropblock_prob, 3) + self.rsoftmax = rSoftMax(radix, groups) + + def forward(self, x): + x = self.conv(x) + if self.use_bn: + x = self.bn0(x) + if self.dropblock_prob > 0.0: + x = self.dropblock(x) + x = self.relu(x) + + batch, rchannel = x.shape[:2] + if self.radix > 1: + splited = torch.split(x, rchannel // self.radix, dim=1) + gap = sum(splited) + else: + gap = x + gap = F.adaptive_avg_pool2d(gap, 1) + gap = self.fc1(gap) + + if self.use_bn: + gap = self.bn1(gap) + gap = self.relu(gap) + + atten = self.fc2(gap) + atten = self.rsoftmax(atten).view(batch, -1, 1, 1) + + if self.radix > 1: + attens = torch.split(atten, rchannel // self.radix, dim=1) + out = sum([att * split for (att, split) in zip(attens, splited)]) + else: + out = atten * x + return out.contiguous() + + +class rSoftMax(nn.Module): + + def __init__(self, radix, cardinality): + super().__init__() + self.radix = radix + self.cardinality = cardinality + + def forward(self, x): + batch = x.size(0) + if self.radix > 1: + x = x.view(batch, self.cardinality, self.radix, -1).transpose(1, 2) + x = F.softmax(x, dim=1) + x = x.reshape(batch, -1) + else: + x = torch.sigmoid(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/bad_image_detecting/__init__.py b/runtime/Lib/site_packages/models/cv/bad_image_detecting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8fb20550e6fd04617645ae951486a3fda28342cf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/bad_image_detecting/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .bad_image_detecting import BadImageDetecting + +else: + _import_structure = {'bad_image_detecting': ['BadImageDetecting']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/bad_image_detecting/bad_image_detecting.py b/runtime/Lib/site_packages/models/cv/bad_image_detecting/bad_image_detecting.py new file mode 100644 index 0000000000000000000000000000000000000000..f173f479e332e4c8e0abe99f2a69f0e34990cda2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/bad_image_detecting/bad_image_detecting.py @@ -0,0 +1,81 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import numpy as np +import torch.cuda +from torchvision import models + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['BadImageDetecting'] + + +@MODELS.register_module( + Tasks.bad_image_detecting, module_name=Models.bad_image_detecting) +class BadImageDetecting(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image_quality_assessment_mos model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.model = models.mobilenet_v2( + pretrained=False, width_mult=0.35, num_classes=3) + self.model = self._load_pretrained(self.model, model_path) + self.model.eval() + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + losses = dict() + return losses + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + + ret = self.model(input) + + return {'output': ret} + + def _evaluate_postprocess(self, input: Tensor) -> Dict[str, list]: + torch.cuda.empty_cache() + with torch.no_grad(): + preds = self.model(input) + _, pred_ = torch.max(preds, dim=1) + del input + torch.cuda.empty_cache() + return {OutputKeys.LABEL: pred_} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif OutputKeys.LABEL in inputs: + infeat = inputs['input'] + return self._evaluate_postprocess(infeat) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/body_2d_keypoints/__init__.py b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ddc00cb3cbb095af3567e207e62ea88fd1931cd7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .hrnet_v2 import PoseHighResolutionNetV2 + +else: + _import_structure = { + 'hrnet_v2': ['PoseHighResolutionNetV2'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_basic_modules.py b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_basic_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..3b960688caaeadc9c227b9eaac1d041ecc064139 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_basic_modules.py @@ -0,0 +1,397 @@ +# The implementation is based on HRNET, available at https://github.com/HRNet/HigherHRNet-Human-Pose-Estimation. + +import torch +import torch.nn as nn + +BN_MOMENTUM = 0.1 + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d( + planes * self.expansion, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class HighResolutionModule(nn.Module): + + def __init__(self, + num_branches, + blocks, + num_blocks, + num_inchannels, + num_channels, + fuse_method, + multi_scale_output=True): + super(HighResolutionModule, self).__init__() + self._check_branches(num_branches, blocks, num_blocks, num_inchannels, + num_channels) + + self.num_inchannels = num_inchannels + self.fuse_method = fuse_method + self.num_branches = num_branches + + self.multi_scale_output = multi_scale_output + + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(True) + + def _check_branches(self, num_branches, blocks, num_blocks, num_inchannels, + num_channels): + if num_branches != len(num_blocks): + error_msg = 'NUM_BRANCHES({}) <> NUM_BLOCKS({})'.format( + num_branches, len(num_blocks)) + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = 'NUM_BRANCHES({}) <> NUM_CHANNELS({})'.format( + num_branches, len(num_channels)) + raise ValueError(error_msg) + + if num_branches != len(num_inchannels): + error_msg = 'NUM_BRANCHES({}) <> NUM_INCHANNELS({})'.format( + num_branches, len(num_inchannels)) + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + downsample = None + if stride != 1 or \ + self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.num_inchannels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d( + num_channels[branch_index] * block.expansion, + momentum=BN_MOMENTUM), + ) + layers = [] + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index], stride, downsample)) + self.num_inchannels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index])) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + if self.num_branches == 1: + return None + + num_branches = self.num_branches + num_inchannels = self.num_inchannels + fuse_layers = [] + for i in range(num_branches if self.multi_scale_output else 1): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_inchannels[i], + 1, + 1, + 0, + bias=False), nn.BatchNorm2d(num_inchannels[i]), + nn.Upsample( + scale_factor=2**(j - i), mode='nearest'))) + elif j == i: + fuse_layer.append(None) + else: + conv3x3s = [] + for k in range(i - j): + if k == i - j - 1: + num_outchannels_conv3x3 = num_inchannels[i] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + nn.BatchNorm2d(num_outchannels_conv3x3))) + else: + num_outchannels_conv3x3 = num_inchannels[j] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + nn.BatchNorm2d(num_outchannels_conv3x3), + nn.ReLU(True))) + fuse_layer.append(nn.Sequential(*conv3x3s)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def get_num_inchannels(self): + return self.num_inchannels + + def forward(self, x): + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + + for i in range(len(self.fuse_layers)): + y = x[0] if i == 0 else self.fuse_layers[i][0](x[0]) + for j in range(1, self.num_branches): + if i == j: + y = y + x[j] + else: + y = y + self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + + return x_fuse + + +def conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1): + result = nn.Sequential() + result.add_module( + 'conv', + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False)) + result.add_module('bn', nn.BatchNorm2d(num_features=out_channels)) + return result + + +def upsample(scale, oup): + return nn.Sequential( + nn.Upsample(scale_factor=scale, mode='bilinear'), + nn.Conv2d( + in_channels=oup, + out_channels=oup, + kernel_size=3, + stride=1, + padding=1, + groups=1, + bias=False), nn.BatchNorm2d(oup), nn.PReLU()) + + +class SE_Block(nn.Module): + + def __init__(self, c, r=16): + super().__init__() + self.squeeze = nn.AdaptiveAvgPool2d(1) + self.excitation = nn.Sequential( + nn.Linear(c, c // r, bias=False), nn.ReLU(inplace=True), + nn.Linear(c // r, c, bias=False), nn.Sigmoid()) + + def forward(self, x): + bs, c, _, _ = x.shape + y = self.squeeze(x).view(bs, c) + y = self.excitation(y).view(bs, c, 1, 1) + return x * y.expand_as(x) + + +class BasicBlockSE(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, r=64): + super(BasicBlockSE, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + self.se = SE_Block(planes, r) + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.se(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class BottleneckSE(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, r=64): + super(BottleneckSE, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d( + planes * self.expansion, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + self.se = SE_Block(planes * self.expansion, r) + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out = self.se(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +blocks_dict = { + 'BASIC': BasicBlock, + 'BOTTLENECK': Bottleneck, + 'BASICSE': BasicBlockSE, + 'BOTTLENECKSE': BottleneckSE, +} diff --git a/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_v2.py b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..19e426b2131d9c7bfcd165b1de5502daf8687fda --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/hrnet_v2.py @@ -0,0 +1,223 @@ +# The implementation is based on HRNET, available at https://github.com/HRNet/HigherHRNet-Human-Pose-Estimation. + +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.body_2d_keypoints.hrnet_basic_modules import ( + BN_MOMENTUM, BasicBlock, Bottleneck, HighResolutionModule, blocks_dict) +from modelscope.models.cv.body_2d_keypoints.w48 import cfg_128x128_15 +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.body_2d_keypoints, module_name=Models.body_2d_keypoints) +class PoseHighResolutionNetV2(TorchModel): + + def __init__(self, cfg=None, **kwargs): + if cfg is None: + cfg = cfg_128x128_15 + self.inplanes = 64 + extra = cfg['MODEL']['EXTRA'] + super(PoseHighResolutionNetV2, self).__init__(**kwargs) + + # stem net + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + 64, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(Bottleneck, 64, 4) + + self.stage2_cfg = cfg['MODEL']['EXTRA']['STAGE2'] + num_channels = self.stage2_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage2_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition1 = self._make_transition_layer([256], num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + self.stage3_cfg = cfg['MODEL']['EXTRA']['STAGE3'] + num_channels = self.stage3_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage3_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + self.stage4_cfg = cfg['MODEL']['EXTRA']['STAGE4'] + num_channels = self.stage4_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage4_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels, multi_scale_output=True) + """final four layers""" + last_inp_channels = int(np.sum(pre_stage_channels)) + self.final_layer = nn.Sequential( + nn.Conv2d( + in_channels=last_inp_channels, + out_channels=last_inp_channels, + kernel_size=1, + stride=1, + padding=0), + nn.BatchNorm2d(last_inp_channels, momentum=BN_MOMENTUM), + nn.ReLU(inplace=False), + nn.Conv2d( + in_channels=last_inp_channels, + out_channels=cfg['MODEL']['NUM_JOINTS'], + kernel_size=extra['FINAL_CONV_KERNEL'], + stride=1, + padding=1 if extra['FINAL_CONV_KERNEL'] == 3 else 0)) + + self.pretrained_layers = cfg['MODEL']['EXTRA']['PRETRAINED_LAYERS'] + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + nn.Conv2d( + num_channels_pre_layer[i], + num_channels_cur_layer[i], + 3, + 1, + 1, + bias=False), + nn.BatchNorm2d(num_channels_cur_layer[i]), + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv3x3s = [] + for j in range(i + 1 - num_branches_pre): + inchannels = num_channels_pre_layer[-1] + outchannels = num_channels_cur_layer[ + i] if j == i - num_branches_pre else inchannels + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + inchannels, outchannels, 3, 2, 1, bias=False), + nn.BatchNorm2d(outchannels), + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv3x3s)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_stage(self, + layer_config, + num_inchannels, + multi_scale_output=True): + num_modules = layer_config['NUM_MODULES'] + num_branches = layer_config['NUM_BRANCHES'] + num_blocks = layer_config['NUM_BLOCKS'] + num_channels = layer_config['NUM_CHANNELS'] + block = blocks_dict[layer_config['BLOCK']] + fuse_method = layer_config['FUSE_METHOD'] + + modules = [] + for i in range(num_modules): + if not multi_scale_output and i == num_modules - 1: + reset_multi_scale_output = False + else: + reset_multi_scale_output = True + + modules.append( + HighResolutionModule(num_branches, block, num_blocks, + num_inchannels, num_channels, fuse_method, + reset_multi_scale_output)) + num_inchannels = modules[-1].get_num_inchannels() + + return nn.Sequential(*modules), num_inchannels + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['NUM_BRANCHES']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['NUM_BRANCHES']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['NUM_BRANCHES']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + + y_list = self.stage4(x_list) + + y0_h, y0_w = y_list[0].size(2), y_list[0].size(3) + y1 = F.upsample(y_list[1], size=(y0_h, y0_w), mode='bilinear') + y2 = F.upsample(y_list[2], size=(y0_h, y0_w), mode='bilinear') + y3 = F.upsample(y_list[3], size=(y0_h, y0_w), mode='bilinear') + + y = torch.cat([y_list[0], y1, y2, y3], 1) + output = self.final_layer(y) + + return output diff --git a/runtime/Lib/site_packages/models/cv/body_2d_keypoints/w48.py b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/w48.py new file mode 100644 index 0000000000000000000000000000000000000000..e0317991af751e80a6c3b8f2f61ceaf0e3e974b2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_2d_keypoints/w48.py @@ -0,0 +1,53 @@ +# The implementation is based on HRNET, available at https://github.com/HRNet/HigherHRNet-Human-Pose-Estimation. + +cfg_128x128_15 = { + 'DATASET': { + 'TYPE': 'DAMO', + 'PARENT_IDS': [0, 0, 1, 2, 3, 1, 5, 6, 14, 8, 9, 14, 11, 12, 1], + 'LEFT_IDS': [2, 3, 4, 8, 9, 10], + 'RIGHT_IDS': [5, 6, 7, 11, 12, 13], + 'SPINE_IDS': [0, 1, 14] + }, + 'MODEL': { + 'INIT_WEIGHTS': True, + 'NAME': 'pose_hrnet', + 'NUM_JOINTS': 15, + 'PRETRAINED': '', + 'TARGET_TYPE': 'gaussian', + 'IMAGE_SIZE': [128, 128], + 'HEATMAP_SIZE': [32, 32], + 'SIGMA': 2.0, + 'EXTRA': { + 'PRETRAINED_LAYERS': [ + 'conv1', 'bn1', 'conv2', 'bn2', 'layer1', 'transition1', + 'stage2', 'transition2', 'stage3', 'transition3', 'stage4' + ], + 'FINAL_CONV_KERNEL': + 1, + 'STAGE2': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 2, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4], + 'NUM_CHANNELS': [48, 96], + 'FUSE_METHOD': 'SUM' + }, + 'STAGE3': { + 'NUM_MODULES': 4, + 'NUM_BRANCHES': 3, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4, 4], + 'NUM_CHANNELS': [48, 96, 192], + 'FUSE_METHOD': 'SUM' + }, + 'STAGE4': { + 'NUM_MODULES': 3, + 'NUM_BRANCHES': 4, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4, 4, 4], + 'NUM_CHANNELS': [48, 96, 192, 384], + 'FUSE_METHOD': 'SUM' + }, + } + } +} diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/__init__.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2672ba9a8448970f6ba4cabd0cf1972cadfd14a1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .cannonical_pose import BodyKeypointsDetection3D + from .hdformer import HDFormerDetector +else: + _import_structure = { + 'cannonical_pose': ['BodyKeypointsDetection3D'], + 'hdformer': ['HDFormerDetector'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/__init__.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..928d0a501f72c97aec6b0ff389ae30a4e82cfb65 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/__init__.py @@ -0,0 +1 @@ +from .body_3d_pose import BodyKeypointsDetection3D diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/body_3d_pose.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/body_3d_pose.py new file mode 100644 index 0000000000000000000000000000000000000000..e9c0839509ab932db550a24b8e1c0408fef13951 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/body_3d_pose.py @@ -0,0 +1,248 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import logging +import os.path as osp +from typing import Any, Dict, List, Union + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.body_3d_keypoints.cannonical_pose.canonical_pose_modules import ( + TemporalModel, TransCan3Dkeys) +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['BodyKeypointsDetection3D'] + + +class KeypointsTypes(object): + POSES_CAMERA = 'poses_camera' + POSES_TRAJ = 'poses_traj' + + +@MODELS.register_module( + Tasks.body_3d_keypoints, module_name=Models.body_3d_keypoints) +class BodyKeypointsDetection3D(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + + super().__init__(model_dir, *args, **kwargs) + + self.model_dir = model_dir + model_path = osp.join(self.model_dir, ModelFile.TORCH_MODEL_FILE) + cfg_path = osp.join(self.model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(cfg_path) + self._create_model() + + if not osp.exists(model_path): + raise IOError(f'{model_path} is not exists.') + + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.pretrained_state_dict = torch.load( + model_path, map_location=self._device) + + self.load_pretrained() + self.to_device(self._device) + self.eval() + + def _create_model(self): + self.model_pos = TemporalModel( + self.cfg.model.MODEL.IN_NUM_JOINTS, + self.cfg.model.MODEL.IN_2D_FEATURE, + self.cfg.model.MODEL.OUT_NUM_JOINTS, + filter_widths=self.cfg.model.MODEL.FILTER_WIDTHS, + causal=self.cfg.model.MODEL.CAUSAL, + dropout=self.cfg.model.MODEL.DROPOUT, + channels=self.cfg.model.MODEL.CHANNELS, + dense=self.cfg.model.MODEL.DENSE) + + receptive_field = self.model_pos.receptive_field() + self.pad = (receptive_field - 1) // 2 + if self.cfg.model.MODEL.CAUSAL: + self.causal_shift = self.pad + else: + self.causal_shift = 0 + + self.model_traj = TransCan3Dkeys( + in_channels=self.cfg.model.MODEL.IN_NUM_JOINTS + * self.cfg.model.MODEL.IN_2D_FEATURE, + num_features=1024, + out_channels=self.cfg.model.MODEL.OUT_3D_FEATURE, + num_blocks=4, + time_window=receptive_field) + + def eval(self): + self.model_pos.eval() + self.model_traj.eval() + + def train(self): + self.model_pos.train() + self.model_traj.train() + + def to_device(self, device): + self.model_pos = self.model_pos.to(device) + self.model_traj = self.model_traj.to(device) + + def load_pretrained(self): + if 'model_pos' in self.pretrained_state_dict: + self.model_pos.load_state_dict( + self.pretrained_state_dict['model_pos'], strict=False) + else: + logging.error( + 'Not load model pos from pretrained_state_dict, not in pretrained_state_dict' + ) + + if 'model_traj' in self.pretrained_state_dict: + self.model_traj.load_state_dict( + self.pretrained_state_dict['model_traj'], strict=False) + else: + logging.error( + 'Not load model traj from pretrained_state_dict, not in pretrained_state_dict' + ) + logging.info('Load pretrained model done.') + + def preprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + """Proprocess of 2D input joints. + + Args: + input (Dict[str, Any]): [NUM_FRAME, NUM_JOINTS, 2], input 2d human body keypoints. + + Returns: + Dict[str, Any]: canonical 2d points and root relative joints. + """ + if 'cuda' == input.device.type: + input = input.data.cpu().numpy() + elif 'cpu' == input.device.type: + input = input.data.numpy() + pose2d = input + + pose2d_canonical = self.canonicalize_2Ds( + pose2d, self.cfg.model.INPUT.FOCAL_LENGTH, + self.cfg.model.INPUT.CENTER) + pose2d_normalized = self.normalize_screen_coordinates( + pose2d, self.cfg.model.INPUT.RES_W, self.cfg.model.INPUT.RES_H) + pose2d_rr = pose2d_normalized + pose2d_rr[:, 1:] -= pose2d_rr[:, :1] + + # expand [NUM_FRAME, NUM_JOINTS, 2] to [1, NUM_FRAME, NUM_JOINTS, 2] + pose2d_rr = np.expand_dims( + np.pad( + pose2d_rr, + ((self.pad + self.causal_shift, self.pad - self.causal_shift), + (0, 0), (0, 0)), 'edge'), + axis=0) + pose2d_canonical = np.expand_dims( + np.pad( + pose2d_canonical, + ((self.pad + self.causal_shift, self.pad - self.causal_shift), + (0, 0), (0, 0)), 'edge'), + axis=0) + pose2d_rr = torch.from_numpy(pose2d_rr.astype(np.float32)) + pose2d_canonical = torch.from_numpy( + pose2d_canonical.astype(np.float32)) + + inputs_2d = pose2d_rr.clone() + if torch.cuda.is_available(): + inputs_2d = inputs_2d.cuda(non_blocking=True) + + # Positional model + if self.cfg.model.MODEL.USE_2D_OFFSETS: + inputs_2d[:, :, 0] = 0 + else: + inputs_2d[:, :, 1:] += inputs_2d[:, :, :1] + + return { + 'inputs_2d': inputs_2d, + 'pose2d_rr': pose2d_rr, + 'pose2d_canonical': pose2d_canonical + } + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """3D human pose estimation. + + Args: + input (Dict): + inputs_2d: [1, NUM_FRAME, NUM_JOINTS, 2] + pose2d_rr: [1, NUM_FRAME, NUM_JOINTS, 2] + pose2d_canonical: [1, NUM_FRAME, NUM_JOINTS, 2] + NUM_FRAME = max(receptive_filed + video_frame_number, video_frame_number) + + Returns: + Dict[str, Any]: + "camera_pose": Tensor, [1, NUM_FRAME, OUT_NUM_JOINTS, OUT_3D_FEATURE_DIM], + 3D human pose keypoints in camera frame. + "camera_traj": Tensor, [1, NUM_FRAME, 1, 3], + root keypoints coordinates in camera frame. + """ + inputs_2d = input['inputs_2d'] + pose2d_rr = input['pose2d_rr'] + pose2d_canonical = input['pose2d_canonical'] + with torch.no_grad(): + # predict 3D pose keypoints + predicted_3d_pos = self.model_pos(inputs_2d) + + # predict global trajectory + b1, w1, n1, d1 = inputs_2d.shape + + input_pose2d_abs = self.get_abs_2d_pts(w1, pose2d_rr, + pose2d_canonical) + b1, w1, n1, d1 = input_pose2d_abs.size() + b2, w2, n2, d2 = predicted_3d_pos.size() + + if torch.cuda.is_available(): + input_pose2d_abs = input_pose2d_abs.cuda(non_blocking=True) + + predicted_3d_traj = self.model_traj( + input_pose2d_abs.view(b1, w1, n1 * d1), + predicted_3d_pos.view(b2 * w2, n2 * d2)).view(b2, w2, -1, 3) + + predict_dict = { + KeypointsTypes.POSES_CAMERA: predicted_3d_pos, + KeypointsTypes.POSES_TRAJ: predicted_3d_traj + } + + return predict_dict + + def get_abs_2d_pts(self, input_video_frame_num, pose2d_rr, + pose2d_canonical): + pad = self.pad + w = input_video_frame_num - pad * 2 + + lst_pose2d_rr = [] + lst_pose2d_cannoical = [] + for i in range(pad, w + pad): + lst_pose2d_rr.append(pose2d_rr[:, i - pad:i + pad + 1]) + lst_pose2d_cannoical.append(pose2d_canonical[:, + i - pad:i + pad + 1]) + + input_pose2d_rr = torch.cat(lst_pose2d_cannoical, axis=0) + input_pose2d_cannoical = torch.cat(lst_pose2d_cannoical, axis=0) + + if self.cfg.model.MODEL.USE_CANONICAL_COORDS: + input_pose2d_abs = input_pose2d_cannoical.clone() + else: + input_pose2d_abs = input_pose2d_rr.clone() + input_pose2d_abs[:, :, 1:] += input_pose2d_abs[:, :, :1] + + return input_pose2d_abs + + def canonicalize_2Ds(self, pos2d, f, c): + cs = np.array([c[0], c[1]]).reshape(1, 1, 2) + fs = np.array([f[0], f[1]]).reshape(1, 1, 2) + canoical_2Ds = (pos2d - cs) / fs + return canoical_2Ds + + def normalize_screen_coordinates(self, X, w, h): + assert X.shape[-1] == 2 + + # Normalize so that [0, w] is mapped to [-1, 1], while preserving the aspect ratio + return X / w * 2 - [1, h / w] diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/canonical_pose_modules.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/canonical_pose_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..b7f0c4a35418a28579db5824e6a5467f82c01d89 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/cannonical_pose/canonical_pose_modules.py @@ -0,0 +1,233 @@ +# The implementation is based on VideoPose3D, available at https://github.com/facebookresearch/VideoPose3D +import torch +import torch.nn as nn + + +class TemporalModelBase(nn.Module): + """ + Do not instantiate this class. + """ + + def __init__(self, num_joints_in, in_features, num_joints_out, + filter_widths, causal, dropout, channels): + super().__init__() + + # Validate input + for fw in filter_widths: + assert fw % 2 != 0, 'Only odd filter widths are supported' + + self.num_joints_in = num_joints_in + self.in_features = in_features + self.num_joints_out = num_joints_out + self.filter_widths = filter_widths + + self.drop = nn.Dropout(dropout) + self.relu = nn.ReLU(inplace=True) + + self.pad = [filter_widths[0] // 2] + self.expand_bn = nn.BatchNorm1d(channels, momentum=0.1) + self.shrink = nn.Conv1d(channels, num_joints_out * 3, 1) + + def set_bn_momentum(self, momentum): + self.expand_bn.momentum = momentum + for bn in self.layers_bn: + bn.momentum = momentum + + def receptive_field(self): + """ + Return the total receptive field of this model as # of frames. + """ + frames = 0 + for f in self.pad: + frames += f + return 1 + 2 * frames + + def total_causal_shift(self): + """ + Return the asymmetric offset for sequence padding. + The returned value is typically 0 if causal convolutions are disabled, + otherwise it is half the receptive field. + """ + frames = self.causal_shift[0] + next_dilation = self.filter_widths[0] + for i in range(1, len(self.filter_widths)): + frames += self.causal_shift[i] * next_dilation + next_dilation *= self.filter_widths[i] + return frames + + def forward(self, x): + assert len(x.shape) == 4 + assert x.shape[-2] == self.num_joints_in + assert x.shape[-1] == self.in_features + + sz = x.shape[:3] + x = x.view(x.shape[0], x.shape[1], -1) + x = x.permute(0, 2, 1) + + x = self._forward_blocks(x) + + x = x.permute(0, 2, 1) + x = x.view(sz[0], -1, self.num_joints_out, 3) + + return x + + +class TemporalModel(TemporalModelBase): + """ + Reference 3D pose estimation model with temporal convolutions. + This implementation can be used for all use-cases. + """ + + def __init__(self, + num_joints_in, + in_features, + num_joints_out, + filter_widths, + causal=False, + dropout=0.25, + channels=1024, + dense=False): + """ + Initialize this model. + + Arguments: + num_joints_in -- number of input joints (e.g. 17 for Human3.6M) + in_features -- number of input features for each joint (typically 2 for 2D input) + num_joints_out -- number of output joints (can be different than input) + filter_widths -- list of convolution widths, which also determines the # of blocks and receptive field + causal -- use causal convolutions instead of symmetric convolutions (for real-time applications) + dropout -- dropout probability + channels -- number of convolution channels + dense -- use regular dense convolutions instead of dilated convolutions (ablation experiment) + """ + super().__init__(num_joints_in, in_features, num_joints_out, + filter_widths, causal, dropout, channels) + + self.expand_conv = nn.Conv1d( + num_joints_in * in_features, + channels, + filter_widths[0], + bias=False) + + layers_conv = [] + layers_bn = [] + + self.causal_shift = [(filter_widths[0]) // 2 if causal else 0] + next_dilation = filter_widths[0] + for i in range(1, len(filter_widths)): + self.pad.append((filter_widths[i] - 1) * next_dilation // 2) + self.causal_shift.append((filter_widths[i] // 2 + * next_dilation) if causal else 0) + + layers_conv.append( + nn.Conv1d( + channels, + channels, + filter_widths[i] if not dense else (2 * self.pad[-1] + 1), + dilation=next_dilation if not dense else 1, + bias=False)) + layers_bn.append(nn.BatchNorm1d(channels, momentum=0.1)) + layers_conv.append( + nn.Conv1d(channels, channels, 1, dilation=1, bias=False)) + layers_bn.append(nn.BatchNorm1d(channels, momentum=0.1)) + + next_dilation *= filter_widths[i] + + self.layers_conv = nn.ModuleList(layers_conv) + self.layers_bn = nn.ModuleList(layers_bn) + + def _forward_blocks(self, x): + x = self.drop(self.relu(self.expand_bn(self.expand_conv(x)))) + for i in range(len(self.pad) - 1): + pad = self.pad[i + 1] + shift = self.causal_shift[i + 1] + res = x[:, :, pad + shift:x.shape[2] - pad + shift] + x = self.drop( + self.relu(self.layers_bn[2 * i](self.layers_conv[2 * i](x)))) + x = res + self.drop( + self.relu(self.layers_bn[2 * i + 1]( + self.layers_conv[2 * i + 1](x)))) + + x = self.shrink(x) + return x + + +# regression of the trajectory +class TransCan3Dkeys(nn.Module): + + def __init__(self, + in_channels=74, + num_features=256, + out_channels=44, + time_window=10, + num_blocks=2): + super().__init__() + self.in_channels = in_channels + self.num_features = num_features + self.out_channels = out_channels + self.num_blocks = num_blocks + self.time_window = time_window + + self.expand_bn = nn.BatchNorm1d(self.num_features, momentum=0.1) + self.conv1 = nn.Sequential( + nn.ReplicationPad1d(1), + nn.Conv1d( + self.in_channels, self.num_features, kernel_size=3, + bias=False), self.expand_bn, nn.ReLU(inplace=True), + nn.Dropout(p=0.25)) + self._make_blocks() + self.pad = nn.ReplicationPad1d(4) + self.relu = nn.ReLU(inplace=True) + self.drop = nn.Dropout(p=0.25) + self.reduce = nn.Conv1d( + self.num_features, self.num_features, kernel_size=self.time_window) + self.embedding_3d_1 = nn.Linear(in_channels // 2 * 3, 500) + self.embedding_3d_2 = nn.Linear(500, 500) + self.LReLU1 = nn.LeakyReLU() + self.LReLU2 = nn.LeakyReLU() + self.LReLU3 = nn.LeakyReLU() + self.out1 = nn.Linear(self.num_features + 500, self.num_features) + self.out2 = nn.Linear(self.num_features, self.out_channels) + + def _make_blocks(self): + layers_conv = [] + layers_bn = [] + for i in range(self.num_blocks): + layers_conv.append( + nn.Conv1d( + self.num_features, + self.num_features, + kernel_size=5, + bias=False, + dilation=2)) + layers_bn.append(nn.BatchNorm1d(self.num_features)) + self.layers_conv = nn.ModuleList(layers_conv) + self.layers_bn = nn.ModuleList(layers_bn) + + def set_bn_momentum(self, momentum): + self.expand_bn.momentum = momentum + for bn in self.layers_bn: + bn.momentum = momentum + + def forward(self, p2ds, p3d): + """ + Args: + x - (B x T x J x C) + """ + B, T, C = p2ds.shape + x = p2ds.permute((0, 2, 1)) + x = self.conv1(x) + for i in range(self.num_blocks): + pre = x + x = self.pad(x) + x = self.layers_conv[i](x) + x = self.layers_bn[i](x) + x = self.drop(self.relu(x)) + x = pre + x + x_2d = self.relu(self.reduce(x)) + x_2d = x_2d.view(B, -1) + x_3d = self.LReLU1(self.embedding_3d_1(p3d)) + x = torch.cat((x_2d, x_3d), 1) + x = self.LReLU3(self.out1(x)) + x = self.out2(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/__init__.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9e40d1238f11518540108ac93235368d4766053 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .hdformer_detector import HDFormerDetector diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/backbone.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..7850eda86ca0f66d9057ccdbec13c7c7d7c06319 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/backbone.py @@ -0,0 +1,306 @@ +# -------------------------------------------------------- +# The implementation is also open-sourced by the authors as Hanyuan Chen, and is available publicly on +# https://github.com/hyer/HDFormer +# -------------------------------------------------------- +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.body_3d_keypoints.hdformer.block import \ + HightOrderAttentionBlock +from modelscope.models.cv.body_3d_keypoints.hdformer.directed_graph import ( + DiGraph, Graph) +from modelscope.models.cv.body_3d_keypoints.hdformer.skeleton import \ + get_skeleton + + +class HDFormerNet(nn.Module): + + def __init__(self, cfg): + super(HDFormerNet, self).__init__() + in_channels = cfg.in_channels + dropout = cfg.dropout + self.cfg = cfg + self.PLANES = [16, 32, 64, 128, 256] + + # load graph + skeleton = get_skeleton() + self.di_graph = DiGraph(skeleton=skeleton) + self.graph = Graph( + skeleton=skeleton, strategy='agcn', max_hop=1, dilation=1) + self.A = torch.tensor( + self.graph.A, + dtype=torch.float32, + requires_grad=True, + device='cuda') + + # build networks + spatial_kernel_size = self.A.size(0) + temporal_kernel_size = 9 + kernel_size = (temporal_kernel_size, spatial_kernel_size) + + if not cfg.data_bn: + self.data_bn = None + else: + n_joints = self.cfg.IN_NUM_JOINTS \ + if hasattr(self.cfg, 'IN_NUM_JOINTS') \ + else self.cfg.n_joints + self.data_bn = nn.BatchNorm1d(in_channels * n_joints) if hasattr(cfg, 'PJN') and cfg.PJN \ + else nn.BatchNorm2d(in_channels) + + self.downsample = nn.ModuleList( + ( + HightOrderAttentionBlock( + in_channels, + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + residual=False, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=0), + HightOrderAttentionBlock( + self.PLANES[0], + self.PLANES[1], + kernel_size, + A=self.A, + di_graph=self.di_graph, + stride=2, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=0), + HightOrderAttentionBlock( + self.PLANES[1], + self.PLANES[1], + kernel_size, + A=self.A, + di_graph=self.di_graph, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=0), + HightOrderAttentionBlock( + self.PLANES[1], + self.PLANES[2], + kernel_size, + A=self.A, + di_graph=self.di_graph, + stride=2, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=0), + HightOrderAttentionBlock( + self.PLANES[2], + self.PLANES[2], + kernel_size, + A=self.A, + di_graph=self.di_graph, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=0), + HightOrderAttentionBlock( + self.PLANES[2], + self.PLANES[3], + kernel_size, + A=self.A, + di_graph=self.di_graph, + stride=2, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=dropout), + HightOrderAttentionBlock( + self.PLANES[3], + self.PLANES[3], + kernel_size, + A=self.A, + di_graph=self.di_graph, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=dropout), + HightOrderAttentionBlock( + self.PLANES[3], + self.PLANES[4], + kernel_size, + A=self.A, + di_graph=self.di_graph, + stride=2, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=dropout), + HightOrderAttentionBlock( + self.PLANES[4], + self.PLANES[4], + kernel_size, + A=self.A, + di_graph=self.di_graph, + adj_len=self.A.size(1), + attention=cfg.attention_down if hasattr( + cfg, 'attention_down') else False, + dropout=dropout), + )) + + self.upsample = nn.ModuleList(( + HightOrderAttentionBlock( + self.PLANES[4], + self.PLANES[3], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_up + if hasattr(cfg, 'attention_up') else False, + adj_len=self.A.size(1), + dropout=dropout), + HightOrderAttentionBlock( + self.PLANES[3], + self.PLANES[2], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_up + if hasattr(cfg, 'attention_up') else False, + adj_len=self.A.size(1), + dropout=dropout), + HightOrderAttentionBlock( + self.PLANES[2], + self.PLANES[1], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_up + if hasattr(cfg, 'attention_up') else False, + adj_len=self.A.size(1), + dropout=0), + HightOrderAttentionBlock( + self.PLANES[1], + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_up + if hasattr(cfg, 'attention_up') else False, + adj_len=self.A.size(1), + dropout=0), + )) + + self.merge = nn.ModuleList(( + HightOrderAttentionBlock( + self.PLANES[4], + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_merge if hasattr( + cfg, 'attention_merge') else False, + adj_len=self.A.size(1), + dropout=dropout, + max_hop=self.cfg.max_hop), + HightOrderAttentionBlock( + self.PLANES[3], + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_merge if hasattr( + cfg, 'attention_merge') else False, + adj_len=self.A.size(1), + dropout=dropout, + max_hop=self.cfg.max_hop), + HightOrderAttentionBlock( + self.PLANES[2], + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_merge if hasattr( + cfg, 'attention_merge') else False, + adj_len=self.A.size(1), + dropout=0, + max_hop=self.cfg.max_hop), + HightOrderAttentionBlock( + self.PLANES[1], + self.PLANES[0], + kernel_size, + A=self.A, + di_graph=self.di_graph, + attention=cfg.attention_merge if hasattr( + cfg, 'attention_merge') else False, + adj_len=self.A.size(1), + dropout=0, + max_hop=self.cfg.max_hop), + )) + + def get_edge_fea(self, x_v): + x_e = (x_v[..., [c for p, c in self.di_graph.directed_edges_hop1]] + - x_v[..., [p for p, c in self.di_graph.directed_edges_hop1]] + ).contiguous() + N, C, T, V = x_v.shape + edeg_append = torch.zeros((N, C, T, 1), device=x_e.device) + x_e = torch.cat((x_e, edeg_append), dim=-1) + return x_e + + def forward(self, x_v: torch.Tensor): + """ + x: shape [B,C,T,V_v] + """ + B, C, T, V = x_v.shape + # data normalization + if self.data_bn is not None: + if hasattr(self.cfg, 'PJN') and self.cfg.PJN: + x_v = self.data_bn(x_v.permute(0, 1, 3, 2).contiguous().view(B, -1, T)).view(B, C, V, T) \ + .contiguous().permute(0, 1, 3, 2) + else: + x_v = self.data_bn(x_v) + + x_e = self.get_edge_fea(x_v) + + # forward + feature = [] + for idx, hoa_block in enumerate(self.downsample): + x_v, x_e = hoa_block(x_v, x_e) + if idx == 0 or idx == 2 or idx == 4 or idx == 6: + feature.append((x_v, x_e)) + + feature.append((x_v, x_e)) + feature = feature[::-1] + + x_v, x_e = feature[0] + identity_feature = feature[1:] + + ushape_feature = [] + ushape_feature.append((x_v, x_e)) + for idx, (hoa_block, id) in \ + enumerate(zip(self.upsample, identity_feature)): + x_v, x_e = hoa_block(x_v, x_e) + if hasattr(self.cfg, 'deterministic') and self.cfg.deterministic: + x_v = F.interpolate(x_v, scale_factor=(2, 1), mode='nearest') + else: + x_v = F.interpolate( + x_v, + scale_factor=(2, 1), + mode='bilinear', + align_corners=False) + x_v += id[0] + ushape_feature.append((x_v, x_e)) + + ushape_feature = ushape_feature[:-1] + for idx, (hoa_block, u) in \ + enumerate(zip(self.merge, ushape_feature)): + x_v2, x_e2 = hoa_block(*u) + if hasattr(self.cfg, 'deterministic') and self.cfg.deterministic: + x_v += F.interpolate( + x_v2, scale_factor=(2**(4 - idx), 1), mode='nearest') + else: + x_v += F.interpolate( + x_v2, + scale_factor=(2**(4 - idx), 1), + mode='bilinear', + align_corners=False) + return x_v, x_e diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/block.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/block.py new file mode 100644 index 0000000000000000000000000000000000000000..5886658e7c479b7a645c0529f39f3e32f92b0c6b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/block.py @@ -0,0 +1,380 @@ +# Part of the implementation is borrowed and modified from 2s-AGCN, publicly available at +# https://github.com/lshiwjx/2s-AGCN +import math + +import torch +import torch.nn as nn +from einops import rearrange + + +def import_class(name): + components = name.split('.') + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + return mod + + +def conv_branch_init(conv, branches): + weight = conv.weight + n = weight.size(0) + k1 = weight.size(1) + k2 = weight.size(2) + nn.init.normal_(weight, 0, math.sqrt(2. / (n * k1 * k2 * branches))) + if conv.bias is not None: + nn.init.constant_(conv.bias, 0) + + +def conv_init(conv): + if conv.weight is not None: + nn.init.kaiming_normal_(conv.weight, mode='fan_out') + if conv.bias is not None: + nn.init.constant_(conv.bias, 0) + + +def bn_init(bn, scale): + nn.init.constant_(bn.weight, scale) + nn.init.constant_(bn.bias, 0) + + +def zero(x): + """return zero.""" + return 0 + + +def iden(x): + """return input itself.""" + return x + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0., + changedim=False, + currentdim=0, + depth=0): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + comb=False, + vis=False): + """Attention is all you need + + Args: + dim (_type_): _description_ + num_heads (int, optional): _description_. Defaults to 8. + qkv_bias (bool, optional): _description_. Defaults to False. + qk_scale (_type_, optional): _description_. Defaults to None. + attn_drop (_type_, optional): _description_. Defaults to 0.. + proj_drop (_type_, optional): _description_. Defaults to 0.. + comb (bool, optional): Defaults to False. + True: q transpose * k. + False: q * k transpose. + vis (bool, optional): _description_. Defaults to False. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim**-0.5 + + self.to_q = nn.Linear(dim, dim, bias=qkv_bias) + self.to_k = nn.Linear(dim, dim, bias=qkv_bias) + self.to_v = nn.Linear(dim, dim, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + + self.proj_drop = nn.Dropout(proj_drop) + self.comb = comb + self.vis = vis + + def forward(self, fv, fe): + B, N, C = fv.shape + B, E, C = fe.shape + q = self.to_q(fv).reshape(B, N, self.num_heads, + C // self.num_heads).permute(0, 2, 1, 3) + k = self.to_k(fe).reshape(B, E, self.num_heads, + C // self.num_heads).permute(0, 2, 1, 3) + v = self.to_v(fe).reshape(B, E, self.num_heads, + C // self.num_heads).permute(0, 2, 1, 3) + # Now fv shape (B, H, N, C//heads) + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + if self.comb: + fv = (attn @ v.transpose(-2, -1)).transpose(-2, -1) + fv = rearrange(fv, 'B H N C -> B N (H C)') + elif self.comb is False: + fv = (attn @ v).transpose(1, 2).reshape(B, N, C) + fv = self.proj(fv) + fv = self.proj_drop(fv) + return fv + + +class FirstOrderAttention(nn.Module): + """First Order Attention block for spatial relationship. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + A, + t_kernel_size=1, + t_stride=1, + t_padding=0, + t_dilation=1, + adj_len=17, + bias=True): + super().__init__() + + self.kernel_size = kernel_size + self.A = A + self.PA = nn.Parameter(torch.FloatTensor(3, adj_len, adj_len)) + torch.nn.init.constant_(self.PA, 1e-6) + + self.num_subset = 3 + inter_channels = out_channels // 4 + self.inter_c = inter_channels + self.conv_a = nn.ModuleList() + self.conv_b = nn.ModuleList() + self.conv_d = nn.ModuleList() + self.linears = nn.ModuleList() + for i in range(self.num_subset): + self.conv_a.append(nn.Conv2d(in_channels, inter_channels, 1)) + self.conv_b.append(nn.Conv2d(in_channels, inter_channels, 1)) + self.conv_d.append(nn.Conv2d(in_channels, out_channels, 1)) + self.linears.append(nn.Linear(in_channels, in_channels)) + + if in_channels != out_channels: + self.down = nn.Sequential( + nn.Conv2d(in_channels, out_channels, 1), + nn.BatchNorm2d(out_channels)) + else: + self.down = lambda x: x + + self.bn = nn.BatchNorm2d(out_channels) + self.soft = nn.Softmax(-2) + self.relu = nn.ReLU() + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + conv_init(m) + elif isinstance(m, nn.BatchNorm2d): + bn_init(m, 1) + bn_init(self.bn, 1e-6) + for i in range(self.num_subset): + conv_branch_init(self.conv_d[i], self.num_subset) + + def forward(self, x): + assert self.A.shape[0] == self.kernel_size[1] + + N, C, T, V = x.size() + A = self.A + self.PA + + y = None + for i in range(self.num_subset): + x_in = rearrange(x, 'N C T V -> N T V C') + x_in = self.linears[i](x_in) + A0 = rearrange(x_in, 'N T V C -> N (C T) V') + + A1 = self.conv_a[i](x).permute(0, 3, 1, 2).contiguous().view( + N, V, self.inter_c * T) + A2 = self.conv_b[i](x).view(N, self.inter_c * T, V) + A1 = self.soft(torch.matmul(A1, A2) / A1.size(-1)) + A1 = A1 + A[i] + z = self.conv_d[i](torch.matmul(A0, A1).view(N, C, T, V)) + y = z + y if y is not None else z + y = self.bn(y) + y += self.down(x) + + return self.relu(y) + + +class HightOrderAttentionBlock(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + A, + di_graph, + attention=False, + stride=1, + adj_len=17, + dropout=0, + residual=True, + norm_layer=nn.BatchNorm2d, + edge_importance=False, + graph=None, + conditional=False, + experts=4, + bias=True, + share_tcn=False, + max_hop=2): + super().__init__() + + t_kernel_size = kernel_size[0] + assert t_kernel_size % 2 == 1 + padding = ((t_kernel_size - 1) // 2, 0) + self.max_hop = max_hop + self.attention = attention + self.di_graph = di_graph + + self.foa_block = FirstOrderAttention( + in_channels, + out_channels, + kernel_size, + A, + bias=bias, + adj_len=adj_len) + + self.tcn_v = nn.Sequential( + norm_layer(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d( + out_channels, + out_channels, (t_kernel_size, 1), (stride, 1), + padding, + bias=bias), + norm_layer(out_channels), + nn.Dropout(dropout, inplace=True), + ) + + if not residual: + self.residual_v = zero + elif (in_channels == out_channels) and (stride == 1): + self.residual_v = iden + else: + self.residual_v = nn.Sequential( + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=(stride, 1), + bias=bias), + norm_layer(out_channels), + ) + + self.relu = nn.ReLU(inplace=True) + + if self.attention: + self.cross_attn = Attention( + dim=out_channels, + num_heads=8, + qkv_bias=True, + qk_scale=None, + attn_drop=dropout, + proj_drop=dropout) + self.norm_v = nn.LayerNorm(out_channels) + self.mlp = Mlp( + in_features=out_channels, + out_features=out_channels, + hidden_features=out_channels * 2, + act_layer=nn.GELU, + drop=dropout) + self.norm_mlp = nn.LayerNorm(out_channels) + + # linear to change fep channels + self.linears = nn.ModuleList() + for hop_i in range(self.max_hop - 1): + hop_linear = nn.ModuleList() + for i in range( + len( + eval(f'self.di_graph.directed_edges_hop{hop_i+2}')) + ): + hop_linear.append(nn.Linear(hop_i + 2, 1)) + self.linears.append(hop_linear) + + def forward(self, fv, fe): + # `fv` (node features) has shape (B, C, T, V_node) + # `fe` (edge features) has shape (B, C, T, V_edge) + N, C, T, V = fv.size() + + res_v = self.residual_v(fv) + + fvp = self.foa_block(fv) + fep_out = ( + fvp[..., [c for p, c in self.di_graph.directed_edges_hop1]] + - fvp[..., [p for p, c in self.di_graph.directed_edges_hop1]] + ).contiguous() + + if self.attention: + fep_concat = None + for hop_i in range(self.max_hop): + if 0 == hop_i: + fep_hop_i = (fvp[..., [ + c for p, c in eval( + f'self.di_graph.directed_edges_hop{hop_i+1}') + ]] - fvp[..., [ + p for p, c in eval( + f'self.di_graph.directed_edges_hop{hop_i+1}') + ]]).contiguous() + fep_hop_i = rearrange(fep_hop_i, 'N C T E -> (N T) E C') + else: + joints_parts = eval( + f'self.di_graph.directed_edges_hop{hop_i+1}') + fep_hop_i = None + for part_idx, part in enumerate(joints_parts): + fep_part = None + for j in range(len(part) - 1): + fep = (fvp[..., part[j + 1]] + - fvp[..., part[j]]).contiguous().unsqueeze( + dim=-1) + if fep_part is None: + fep_part = fep + else: + fep_part = torch.cat((fep_part, fep), dim=-1) + fep_part = self.linears[hop_i - 1][part_idx](fep_part) + if fep_hop_i is None: + fep_hop_i = fep_part + else: + fep_hop_i = torch.cat((fep_hop_i, fep_part), + dim=-1) + + fep_hop_i = rearrange(fep_hop_i, 'N C T E -> (N T) E C') + + if fep_concat is None: + fep_concat = fep_hop_i + else: + fep_concat = torch.cat((fep_concat, fep_hop_i), + dim=-2) # dim=-2 represent edge dim + fvp = rearrange(fvp, 'N C T V -> (N T) V C') + fvp = self.norm_v(self.cross_attn(fvp, fep_concat)) + iden(fvp) + fvp = self.mlp(self.norm_mlp(fvp)) + iden( + fvp) # make output joint number = adj_len + fvp = rearrange(fvp, '(N T) V C -> N C T V', N=N) + + fvp = self.tcn_v(fvp) + res_v + + return self.relu(fvp), fep_out diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/directed_graph.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/directed_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..9fe61d7f7e71672278b904984d4ec4e6e42f59b3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/directed_graph.py @@ -0,0 +1,209 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import sys +from typing import List, Tuple + +import numpy as np + +sys.path.insert(0, './') + + +def edge2mat(link, num_node): + """According to the directed edge link, the adjacency matrix is constructed. + link: [V, 2], each row is a tuple(start node, end node). + """ + A = np.zeros((num_node, num_node)) + for i, j in link: + A[j, i] = 1 + return A + + +def normalize_incidence_matrix(im: np.ndarray) -> np.ndarray: + Dl = im.sum(-1) + num_node = im.shape[0] + Dn = np.zeros((num_node, num_node)) + for i in range(num_node): + if Dl[i] > 0: + Dn[i, i] = Dl[i]**(-1) + res = Dn @ im + return res + + +def build_digraph_incidence_matrix(num_nodes: int, + edges: List[Tuple]) -> np.ndarray: + source_graph = np.zeros((num_nodes, len(edges)), dtype='float32') + target_graph = np.zeros((num_nodes, len(edges)), dtype='float32') + for edge_id, (source_node, target_node) in enumerate(edges): + source_graph[source_node, edge_id] = 1. + target_graph[target_node, edge_id] = 1. + source_graph = normalize_incidence_matrix(source_graph) + target_graph = normalize_incidence_matrix(target_graph) + return source_graph, target_graph + + +class DiGraph(): + + def __init__(self, skeleton): + super().__init__() + self.num_nodes = len(skeleton.parents()) + self.directed_edges_hop1 = [ + (parent, child) for child, parent in enumerate(skeleton.parents()) + if parent >= 0 + ] + self.directed_edges_hop2 = [(0, 1, 2), (0, 4, 5), (0, 7, 8), (1, 2, 3), + (4, 5, 6), (7, 8, 9), + (7, 8, 11), (7, 8, 14), (8, 9, 10), + (8, 11, 12), (8, 14, 15), (11, 12, 13), + (14, 15, 16)] # (parent, child) + self.directed_edges_hop3 = [(0, 1, 2, 3), (0, 4, 5, 6), (0, 7, 8, 9), + (7, 8, 9, 10), (7, 8, 11, 12), + (7, 8, 14, 15), (8, 11, 12, 13), + (8, 14, 15, 16)] + self.directed_edges_hop4 = [(0, 7, 8, 9, 10), (0, 7, 8, 11, 12), + (0, 7, 8, 14, 15), (7, 8, 11, 12, 13), + (7, 8, 14, 15, 16)] + + self.num_edges = len(self.directed_edges_hop1) + self.edge_left = [0, 1, 2, 10, 11, 12] + self.edge_right = [3, 4, 5, 13, 14, 15] + self.edge_middle = [6, 7, 8, 9] + self.center = 0 # for h36m data skeleton + # Incidence matrices + self.source_M, self.target_M = \ + build_digraph_incidence_matrix(self.num_nodes, self.directed_edges_hop1) + + +class Graph(): + """ The Graph to model the skeletons extracted by the openpose + Args: + strategy (string): must be one of the follow candidates + - uniform: Uniform Labeling + - distance: Distance Partitioning + - spatial: Spatial Configuration + - agcn: AGCN Configuration + For more information, please refer to the section 'Partition Strategies' + in our paper (https://arxiv.org/abs/1801.07455). + layout (string): must be one of the follow candidates + - openpose: Is consists of 18 joints. For more information, please + refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose#output + - ntu-rgb+d: Is consists of 25 joints. For more information, please + refer to https://github.com/shahroudy/NTURGB-D + max_hop (int): the maximal distance between two connected nodes + dilation (int): controls the spacing between the kernel points + """ + + def __init__(self, + skeleton=None, + strategy='uniform', + max_hop=1, + dilation=1): + self.max_hop = max_hop + self.dilation = dilation + + assert strategy in ['uniform', 'distance', 'spatial', 'agcn'] + self.get_edge(skeleton) + self.hop_dis = get_hop_distance( + self.num_node, self.edge, max_hop=max_hop) + self.get_adjacency(strategy) + + def __str__(self): + return self.A + + def get_edge(self, skeleton): + # edge is a list of [child, parent] paris + self.num_node = len(skeleton.parents()) + self_link = [(i, i) for i in range(self.num_node)] + neighbor_link = [(child, parent) + for child, parent in enumerate(skeleton.parents())] + self.self_link = self_link + self.neighbor_link = neighbor_link + self.edge = self_link + neighbor_link + self.center = 0 # for h36m data skeleton, root node idx + + def get_adjacency(self, strategy): + valid_hop = range(0, self.max_hop + 1, self.dilation) + adjacency = np.zeros((self.num_node, self.num_node)) + for hop in valid_hop: + adjacency[self.hop_dis == hop] = 1 + normalize_adjacency = normalize_digraph(adjacency) + + if strategy == 'uniform': + A = np.zeros((1, self.num_node, self.num_node)) + A[0] = normalize_adjacency + self.A = A + elif strategy == 'distance': + A = np.zeros((len(valid_hop), self.num_node, self.num_node)) + for i, hop in enumerate(valid_hop): + A[i][self.hop_dis == hop] = \ + normalize_adjacency[self.hop_dis == hop] + self.A = A + elif strategy == 'spatial': + A = [] + for hop in valid_hop: + a_root = np.zeros((self.num_node, self.num_node)) + a_close = np.zeros((self.num_node, self.num_node)) + a_further = np.zeros((self.num_node, self.num_node)) + for i in range(self.num_node): + for j in range(self.num_node): + if self.hop_dis[j, i] == hop: + if self.hop_dis[j, self.center] == self.hop_dis[ + i, self.center]: + a_root[j, i] = normalize_adjacency[j, i] + elif self.hop_dis[j, self.center] > self.hop_dis[ + i, self.center]: + a_close[j, i] = normalize_adjacency[j, i] + else: + a_further[j, i] = normalize_adjacency[j, i] + if hop == 0: + A.append(a_root) + else: + A.append(a_root + a_close) + A.append(a_further) + A = np.stack(A) + self.A = A + elif strategy == 'agcn': + A = [] + link_mat = edge2mat(self.self_link, self.num_node) + In = normalize_digraph(edge2mat(self.neighbor_link, self.num_node)) + outward = [(j, i) for (i, j) in self.neighbor_link] + Out = normalize_digraph(edge2mat(outward, self.num_node)) + A = np.stack((link_mat, In, Out)) + self.A = A + else: + raise ValueError('Do Not Exist This Strategy') + + +def get_hop_distance(num_node, edge, max_hop=1): + A = np.zeros((num_node, num_node)) + for i, j in edge: + A[j, i] = 1 + A[i, j] = 1 + + # compute hop steps + hop_dis = np.zeros((num_node, num_node)) + np.inf + transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)] + arrive_mat = (np.stack(transfer_mat) > 0) + for d in range(max_hop, -1, -1): + hop_dis[arrive_mat[d]] = d + return hop_dis + + +def normalize_digraph(A): + Dl = np.sum(A, 0) + num_node = A.shape[0] + Dn = np.zeros((num_node, num_node)) + for i in range(num_node): + if Dl[i] > 0: + Dn[i, i] = Dl[i]**(-1) + AD = np.dot(A, Dn) + return AD + + +def normalize_undigraph(A): + Dl = np.sum(A, 0) + num_node = A.shape[0] + Dn = np.zeros((num_node, num_node)) + for i in range(num_node): + if Dl[i] > 0: + Dn[i, i] = Dl[i]**(-0.5) + DAD = np.dot(np.dot(Dn, A), Dn) + return DAD diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer.py new file mode 100644 index 0000000000000000000000000000000000000000..2873657edd25da485fc74449308fa87eb8339523 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer.py @@ -0,0 +1,64 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from modelscope.models.cv.body_3d_keypoints.hdformer.backbone import \ + HDFormerNet + + +class HDFormer(nn.Module): + + def __init__(self, cfg): + super(HDFormer, self).__init__() + self.regress_with_edge = hasattr( + cfg, 'regress_with_edge') and cfg.regress_with_edge + self.backbone = HDFormerNet(cfg) + num_v, num_e = self.backbone.di_graph.source_M.shape + self.regressor_type = cfg.regressor_type if hasattr( + cfg, 'regressor_type') else 'conv' + if self.regressor_type == 'conv': + self.joint_regressor = nn.Conv2d( + self.backbone.PLANES[0], + 3 * (num_v - 1), + kernel_size=(3, num_v + num_e) if self.regress_with_edge else + (3, num_v), + padding=(1, 0), + bias=True) + elif self.regressor_type == 'fc': + self.joint_regressor = nn.Conv1d( + self.backbone.PLANES[0] * (num_v + num_e) + if self.regress_with_edge else self.backbone.PLANES[0] * num_v, + 3 * (num_v - 1), + kernel_size=3, + padding=1, + bias=True) + else: + raise NotImplementedError + + def forward(self, x_v: torch.Tensor, mean_3d: torch.Tensor, + std_3d: torch.Tensor): + """ + x: shape [B,C,T,V_v] + """ + fv, fe = self.backbone(x_v) + B, C, T, V = fv.shape + + if self.regressor_type == 'conv': + pre_joints = self.joint_regressor(torch.cat([ + fv, fe + ], dim=-1)) if self.regress_with_edge else self.joint_regressor(fv) + elif self.regressor_type == 'fc': + x = (torch.cat([fv, fe], dim=-1) if self.regress_with_edge else fv) \ + .permute(0, 1, 3, 2).contiguous().view(B, -1, T) + pre_joints = self.joint_regressor(x) + else: + raise NotImplementedError + pre_joints = pre_joints.view(B, 3, V - 1, + T).permute(0, 1, 3, + 2).contiguous() # [B,3,T,V-1] + root_node = torch.zeros((B, 3, T, 1), + dtype=pre_joints.dtype, + device=pre_joints.device) + pre_joints = torch.cat((root_node, pre_joints), dim=-1) + pre_joints = pre_joints * std_3d + mean_3d + return pre_joints diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer_detector.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..73c9b4be35713ca38cd958a4ed13d65061c8ad95 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/hdformer_detector.py @@ -0,0 +1,196 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Any, Dict, List, Union + +import numpy as np +import torch +import torch.backends.cudnn as cudnn + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.body_3d_keypoints.hdformer.hdformer import HDFormer +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + + +class KeypointsTypes(object): + POSES_CAMERA = 'poses_camera' + + +logger = get_logger() + + +@MODELS.register_module( + Tasks.body_3d_keypoints, module_name=Models.body_3d_keypoints_hdformer) +class HDFormerDetector(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + + cudnn.benchmark = True + self.model_path = osp.join(self.model_dir, ModelFile.TORCH_MODEL_FILE) + self.mean_std_2d = np.load( + osp.join(self.model_dir, 'mean_std_2d.npy'), allow_pickle=True) + self.mean_std_3d = np.load( + osp.join(self.model_dir, 'mean_std_3d.npy'), allow_pickle=True) + self.left_right_symmetry_2d = np.array( + [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13]) + cfg_path = osp.join(self.model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(cfg_path) + if torch.cuda.is_available(): + self.device = torch.device('cuda') + else: + self.device = torch.device('cpu') + self.net = HDFormer(self.cfg.model.MODEL) + + self.load_model() + self.net = self.net.to(self.device) + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, + map_location=torch.device('cuda') + if torch.cuda.is_available() else torch.device('cpu')) + self.net.load_state_dict(pretrained_dict['state_dict'], strict=False) + self.net.eval() + + def preprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + """Proprocess of 2D input joints. + + Args: + input (Dict[str, Any]): [NUM_FRAME, NUM_JOINTS, 2], input 2d human body keypoints. + + Returns: + Dict[str, Any]: canonical 2d points and root relative joints. + """ + if 'cuda' == input.device.type: + input = input.data.cpu().numpy() + elif 'cpu' == input.device.type: + input = input.data.numpy() + pose2d = input + num_frames, num_joints, in_channels = pose2d.shape + logger.info(f'2d pose frame number: {num_frames}') + + # [NUM_FRAME, NUM_JOINTS, 2] + c = np.array(self.cfg.model.INPUT.center) + f = np.array(self.cfg.model.INPUT.focal_length) + self.window_size = self.cfg.model.INPUT.window_size + receptive_field = self.cfg.model.INPUT.n_frames + + # split the 2D pose sequences into fixed length frames + inputs_2d = [] + inputs_2d_flip = [] + n = 0 + indices = [] + while n + receptive_field <= num_frames: + indices.append((n, n + receptive_field)) + n += self.window_size + self.valid_length = n - self.window_size + receptive_field + + if 0 == len(indices): + logger.warn( + f'Fail to construct test sequences, total_frames = {num_frames}, \ + while receptive_filed ={receptive_field}') + + self.mean_2d = self.mean_std_2d[0] + self.std_2d = self.mean_std_2d[1] + for (start, end) in indices: + data_2d = pose2d[start:end] + data_2d = (data_2d - 0.5 - c) / f + data_2d_flip = data_2d.copy() + data_2d_flip[:, :, 0] *= -1 + data_2d_flip = data_2d_flip[:, self.left_right_symmetry_2d, :] + data_2d_flip = (data_2d_flip - self.mean_2d) / self.std_2d + + data_2d = (data_2d - self.mean_2d) / self.std_2d + data_2d = torch.from_numpy(data_2d.transpose( + (2, 0, 1))).float() # [C,T,V] + + data_2d_flip = torch.from_numpy(data_2d_flip.transpose( + (2, 0, 1))).float() # [C,T,V] + + inputs_2d.append(data_2d) + inputs_2d_flip.append(data_2d_flip) + + self.mean_3d = self.mean_std_3d[0] + self.std_3d = self.mean_std_3d[1] + mean_3d = torch.from_numpy(self.mean_3d).float().unsqueeze(-1) + mean_3d = mean_3d.permute(1, 2, 0) # [3, 1, 17] + std_3d = torch.from_numpy(self.std_3d).float().unsqueeze(-1) + std_3d = std_3d.permute(1, 2, 0) + + return { + 'inputs_2d': inputs_2d, + 'inputs_2d_flip': inputs_2d_flip, + 'mean_3d': mean_3d, + 'std_3d': std_3d + } + + def avg_flip(self, pre, pre_flip): + left_right_symmetry = [ + 0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 14, 15, 16, 11, 12, 13 + ] + pre_flip[:, 0, :, :] *= -1 + pre_flip = pre_flip[:, :, :, left_right_symmetry] + pred_avg = (pre + pre_flip) / 2. + return pred_avg + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """3D human pose estimation. + + Args: + input (Dict): + inputs_2d: [1, NUM_FRAME, NUM_JOINTS, 2] + + Returns: + Dict[str, Any]: + "camera_pose": Tensor, [1, NUM_FRAME, OUT_NUM_JOINTS, OUT_3D_FEATURE_DIM], + 3D human pose keypoints in camera frame. + "success": 3D pose estimation success or failed. + """ + inputs_2d = input['inputs_2d'] + inputs_2d_flip = input['inputs_2d_flip'] + mean_3d = input['mean_3d'] + std_3d = input['std_3d'] + preds_3d = None + vertex_pre = None + + if [] == inputs_2d: + predict_dict = {'success': False, KeypointsTypes.POSES_CAMERA: []} + return predict_dict + + with torch.no_grad(): + for i, pose_2d in enumerate(inputs_2d): + pose_2d = pose_2d.unsqueeze(0).cuda(non_blocking=True) \ + if torch.cuda.is_available() else pose_2d.unsqueeze(0) + pose_2d_flip = inputs_2d_flip[i] + pose_2d_flip = pose_2d_flip.unsqueeze(0).cuda(non_blocking=True) \ + if torch.cuda.is_available() else pose_2d_flip.unsqueeze(0) + mean_3d = mean_3d.unsqueeze(0).cuda(non_blocking=True) \ + if torch.cuda.is_available() else mean_3d.unsqueeze(0) + std_3d = std_3d.unsqueeze(0).cuda(non_blocking=True) \ + if torch.cuda.is_available() else std_3d.unsqueeze(0) + + vertex_pre = self.net(pose_2d, mean_3d, std_3d) + vertex_pre_flip = self.net(pose_2d_flip, mean_3d, std_3d) + vertex_pre = self.avg_flip(vertex_pre, vertex_pre_flip) + + # concat the prediction results for each window_size + predict_3d = vertex_pre.permute( + 0, 2, 3, 1).contiguous()[0][:self.window_size] + if preds_3d is None: + preds_3d = predict_3d + else: + preds_3d = torch.concat((preds_3d, predict_3d), dim=0) + remain_pose_results = vertex_pre.permute( + 0, 2, 3, 1).contiguous()[0][self.window_size:] + preds_3d = torch.concat((preds_3d, remain_pose_results), dim=0) + + preds_3d = preds_3d.unsqueeze(0) # add batch dim + preds_3d = preds_3d / self.cfg.model.INPUT.res_w # Normalize to [-1, 1] + predict_dict = {'success': True, KeypointsTypes.POSES_CAMERA: preds_3d} + + return predict_dict diff --git a/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/skeleton.py b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/skeleton.py new file mode 100644 index 0000000000000000000000000000000000000000..1abf90d742b544cc57f4c487d6d2f0ac85abe8b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/body_3d_keypoints/hdformer/skeleton.py @@ -0,0 +1,103 @@ +# Copyright (c) 2018-present, Facebook, Inc. All rights reserved. +import numpy as np + + +class Skeleton: + + def __init__(self, parents, joints_left, joints_right): + assert len(joints_left) == len(joints_right) + + self._parents = np.array(parents) + self._joints_left = joints_left + self._joints_right = joints_right + self._compute_metadata() + + def num_joints(self): + return len(self._parents) + + def parents(self): + return self._parents + + def has_children(self): + return self._has_children + + def children(self): + return self._children + + def remove_joints(self, joints_to_remove): + """ + Remove the joints specified in 'joints_to_remove'. + """ + valid_joints = [] + for joint in range(len(self._parents)): + if joint not in joints_to_remove: + valid_joints.append(joint) + + for i in range(len(self._parents)): + while self._parents[i] in joints_to_remove: + self._parents[i] = self._parents[self._parents[i]] + + index_offsets = np.zeros(len(self._parents), dtype=int) + new_parents = [] + for i, parent in enumerate(self._parents): + if i not in joints_to_remove: + new_parents.append(parent - index_offsets[parent]) + else: + index_offsets[i:] += 1 + self._parents = np.array(new_parents) + + if self._joints_left is not None: + new_joints_left = [] + for joint in self._joints_left: + if joint in valid_joints: + new_joints_left.append(joint - index_offsets[joint]) + self._joints_left = new_joints_left + if self._joints_right is not None: + new_joints_right = [] + for joint in self._joints_right: + if joint in valid_joints: + new_joints_right.append(joint - index_offsets[joint]) + self._joints_right = new_joints_right + + self._compute_metadata() + + return valid_joints + + def joints_left(self): + return self._joints_left + + def joints_right(self): + return self._joints_right + + def _compute_metadata(self): + self._has_children = np.zeros(len(self._parents)).astype(bool) + for i, parent in enumerate(self._parents): + if parent != -1: + self._has_children[parent] = True + + self._children = [] + for i, parent in enumerate(self._parents): + self._children.append([]) + for i, parent in enumerate(self._parents): + if parent != -1: + self._children[parent].append(i) + + +def get_skeleton(): + skeleton = Skeleton( + parents=[ + -1, 0, 1, 2, 3, 4, 0, 6, 7, 8, 9, 0, 11, 12, 13, 14, 12, 16, 17, + 18, 19, 20, 19, 22, 12, 24, 25, 26, 27, 28, 27, 30 + ], + joints_left=[6, 7, 8, 9, 10, 16, 17, 18, 19, 20, 21, 22, 23], + joints_right=[1, 2, 3, 4, 5, 24, 25, 26, 27, 28, 29, 30, 31]) + # Bring the skeleton to 17 joints instead of the original 32 + skeleton.remove_joints( + [4, 5, 9, 10, 11, 16, 20, 21, 22, 23, 24, 28, 29, 30, 31]) + # Rewire shoulders to the correct parents + skeleton._parents[11] = 8 + skeleton._parents[14] = 8 + # Fix children error + skeleton._children[7] = [8] + skeleton._children[8] = [9, 11, 14] + return skeleton diff --git a/runtime/Lib/site_packages/models/cv/cartoon/__init__.py b/runtime/Lib/site_packages/models/cv/cartoon/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..447335c6cee7bcb009509e4f06753b106de1f0b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/__init__.py @@ -0,0 +1,38 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .facelib.facer import FaceAna + from .mtcnn_pytorch.src.align_trans import (get_reference_facial_points, + warp_and_crop_face) + from .utils import (get_f5p, padTo16x, resize_size, all_file, + tf_data_loader, write_batch_image) + from .network import disc_sn + from .loss import simple_superpixel + from .model_tf import CartoonModel + +else: + _import_structure = { + 'facelib.facer': ['FaceAna'], + 'mtcnn_pytorch.src.align_trans': + ['get_reference_facial_points', 'warp_and_crop_face'], + 'utils': [ + 'get_f5p', 'padTo16x', 'resize_size', 'all_file', 'tf_data_loader', + 'write_batch_image' + ], + 'network': ['disc_sn'], + 'loss': ['simple_superpixel'], + 'model_tf': ['CartoonModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/LK/__init__.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/LK/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/LK/lk.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/LK/lk.py new file mode 100644 index 0000000000000000000000000000000000000000..6fd95ad66ae57d6c88fed8c5ef9613a2306b110e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/facelib/LK/lk.py @@ -0,0 +1,99 @@ +# The implementation is adopted from https://github.com/610265158/Peppa_Pig_Face_Engine + +import numpy as np + +from modelscope.models.cv.cartoon.facelib.config import config as cfg + + +class GroupTrack(): + + def __init__(self): + self.old_frame = None + self.previous_landmarks_set = None + self.with_landmark = True + self.thres = cfg.TRACE.pixel_thres + self.alpha = cfg.TRACE.smooth_landmark + self.iou_thres = cfg.TRACE.iou_thres + + def calculate(self, img, current_landmarks_set): + if self.previous_landmarks_set is None: + self.previous_landmarks_set = current_landmarks_set + result = current_landmarks_set + else: + previous_lm_num = self.previous_landmarks_set.shape[0] + if previous_lm_num == 0: + self.previous_landmarks_set = current_landmarks_set + result = current_landmarks_set + return result + else: + result = [] + for i in range(current_landmarks_set.shape[0]): + not_in_flag = True + for j in range(previous_lm_num): + if self.iou(current_landmarks_set[i], + self.previous_landmarks_set[j] + ) > self.iou_thres: + result.append( + self.smooth(current_landmarks_set[i], + self.previous_landmarks_set[j])) + not_in_flag = False + break + if not_in_flag: + result.append(current_landmarks_set[i]) + + result = np.array(result) + self.previous_landmarks_set = result + + return result + + def iou(self, p_set0, p_set1): + rec1 = [ + np.min(p_set0[:, 0]), + np.min(p_set0[:, 1]), + np.max(p_set0[:, 0]), + np.max(p_set0[:, 1]) + ] + rec2 = [ + np.min(p_set1[:, 0]), + np.min(p_set1[:, 1]), + np.max(p_set1[:, 0]), + np.max(p_set1[:, 1]) + ] + + # computing area of each rectangles + S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1]) + S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1]) + + # computing the sum_area + sum_area = S_rec1 + S_rec2 + + # find the each edge of intersect rectangle + x1 = max(rec1[0], rec2[0]) + y1 = max(rec1[1], rec2[1]) + x2 = min(rec1[2], rec2[2]) + y2 = min(rec1[3], rec2[3]) + + # judge if there is an intersect + intersect = max(0, x2 - x1) * max(0, y2 - y1) + + iou = intersect / (sum_area - intersect) + return iou + + def smooth(self, now_landmarks, previous_landmarks): + result = [] + for i in range(now_landmarks.shape[0]): + x = now_landmarks[i][0] - previous_landmarks[i][0] + y = now_landmarks[i][1] - previous_landmarks[i][1] + dis = np.sqrt(np.square(x) + np.square(y)) + if dis < self.thres: + result.append(previous_landmarks[i]) + else: + result.append( + self.do_moving_average(now_landmarks[i], + previous_landmarks[i])) + + return np.array(result) + + def do_moving_average(self, p_now, p_previous): + p = self.alpha * p_now + (1 - self.alpha) * p_previous + return p diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/__init__.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/config.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/config.py new file mode 100644 index 0000000000000000000000000000000000000000..92b39db046a40c6aeafbe18d739b41a020832a9a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/facelib/config.py @@ -0,0 +1,25 @@ +# The implementation is adopted from https://github.com/610265158/Peppa_Pig_Face_Engine + +import os + +import numpy as np +from easydict import EasyDict as edict + +config = edict() +os.environ['CUDA_VISIBLE_DEVICES'] = '0' + +config.DETECT = edict() +config.DETECT.topk = 10 +config.DETECT.thres = 0.8 +config.DETECT.input_shape = (512, 512, 3) +config.KEYPOINTS = edict() +config.KEYPOINTS.p_num = 68 +config.KEYPOINTS.base_extend_range = [0.2, 0.3] +config.KEYPOINTS.input_shape = (160, 160, 3) +config.TRACE = edict() +config.TRACE.pixel_thres = 1 +config.TRACE.smooth_box = 0.3 +config.TRACE.smooth_landmark = 0.95 +config.TRACE.iou_thres = 0.5 +config.DATA = edict() +config.DATA.pixel_means = np.array([123., 116., 103.]) # RGB diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_detector.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..fa36d662b99467402f192ca59aed5529b54d3551 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_detector.py @@ -0,0 +1,118 @@ +# The implementation is adopted from https://github.com/610265158/Peppa_Pig_Face_Engine + +import time + +import cv2 +import numpy as np +import tensorflow as tf + +from .config import config as cfg + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class FaceDetector: + + def __init__(self, dir): + + self.model_path = dir + '/detector.pb' + self.thres = cfg.DETECT.thres + self.input_shape = cfg.DETECT.input_shape + + self._graph = tf.Graph() + + with self._graph.as_default(): + self._graph, self._sess = self.init_model(self.model_path) + + self.input_image = tf.get_default_graph().get_tensor_by_name( + 'tower_0/images:0') + self.training = tf.get_default_graph().get_tensor_by_name( + 'training_flag:0') + self.output_ops = [ + tf.get_default_graph().get_tensor_by_name('tower_0/boxes:0'), + tf.get_default_graph().get_tensor_by_name('tower_0/scores:0'), + tf.get_default_graph().get_tensor_by_name( + 'tower_0/num_detections:0'), + ] + + def __call__(self, image): + + image, scale_x, scale_y = self.preprocess( + image, + target_width=self.input_shape[1], + target_height=self.input_shape[0]) + + image = np.expand_dims(image, 0) + + boxes, scores, num_boxes = self._sess.run( + self.output_ops, + feed_dict={ + self.input_image: image, + self.training: False + }) + + num_boxes = num_boxes[0] + boxes = boxes[0][:num_boxes] + + scores = scores[0][:num_boxes] + + to_keep = scores > self.thres + boxes = boxes[to_keep] + scores = scores[to_keep] + + y1 = self.input_shape[0] / scale_y + x1 = self.input_shape[1] / scale_x + y2 = self.input_shape[0] / scale_y + x2 = self.input_shape[1] / scale_x + scaler = np.array([y1, x1, y2, x2], dtype='float32') + boxes = boxes * scaler + + scores = np.expand_dims(scores, 0).reshape([-1, 1]) + + for i in range(boxes.shape[0]): + boxes[i] = np.array( + [boxes[i][1], boxes[i][0], boxes[i][3], boxes[i][2]]) + return np.concatenate([boxes, scores], axis=1) + + def preprocess(self, image, target_height, target_width, label=None): + + h, w, c = image.shape + + bimage = np.zeros( + shape=[target_height, target_width, c], + dtype=image.dtype) + np.array( + cfg.DATA.pixel_means, dtype=image.dtype) + long_side = max(h, w) + + scale_x = scale_y = target_height / long_side + + image = cv2.resize(image, None, fx=scale_x, fy=scale_y) + + h_, w_, _ = image.shape + bimage[:h_, :w_, :] = image + + return bimage, scale_x, scale_y + + def init_model(self, *args): + pb_path = args[0] + + def init_pb(model_path): + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.2 + compute_graph = tf.Graph() + compute_graph.as_default() + sess = tf.Session(config=config) + with tf.gfile.GFile(model_path, 'rb') as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + tf.import_graph_def(graph_def, name='') + + return (compute_graph, sess) + + model = init_pb(pb_path) + + graph = model[0] + sess = model[1] + + return graph, sess diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_landmark.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_landmark.py new file mode 100644 index 0000000000000000000000000000000000000000..3c53f3a651b47638307180166ad40459cecd09d6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/facelib/face_landmark.py @@ -0,0 +1,156 @@ +# The implementation is adopted from https://github.com/610265158/Peppa_Pig_Face_Engine + +import cv2 +import numpy as np +import tensorflow as tf + +from .config import config as cfg + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class FaceLandmark: + + def __init__(self, dir): + self.model_path = dir + '/keypoints.pb' + self.min_face = 60 + self.keypoint_num = cfg.KEYPOINTS.p_num * 2 + + self._graph = tf.Graph() + + with self._graph.as_default(): + + self._graph, self._sess = self.init_model(self.model_path) + self.img_input = tf.get_default_graph().get_tensor_by_name( + 'tower_0/images:0') + self.embeddings = tf.get_default_graph().get_tensor_by_name( + 'tower_0/prediction:0') + self.training = tf.get_default_graph().get_tensor_by_name( + 'training_flag:0') + + self.landmark = self.embeddings[:, :self.keypoint_num] + self.headpose = self.embeddings[:, -7:-4] * 90. + self.state = tf.nn.sigmoid(self.embeddings[:, -4:]) + + def __call__(self, img, bboxes): + landmark_result = [] + state_result = [] + for i, bbox in enumerate(bboxes): + landmark, state = self._one_shot_run(img, bbox, i) + if landmark is not None: + landmark_result.append(landmark) + state_result.append(state) + return np.array(landmark_result), np.array(state_result) + + def simple_run(self, cropped_img): + with self._graph.as_default(): + + cropped_img = np.expand_dims(cropped_img, axis=0) + landmark, p, states = self._sess.run( + [self.landmark, self.headpose, self.state], + feed_dict={ + self.img_input: cropped_img, + self.training: False + }) + + return landmark, states + + def _one_shot_run(self, image, bbox, i): + + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + if (bbox_width <= self.min_face and bbox_height <= self.min_face): + return None, None + add = int(max(bbox_width, bbox_height)) + bimg = cv2.copyMakeBorder( + image, + add, + add, + add, + add, + borderType=cv2.BORDER_CONSTANT, + value=cfg.DATA.pixel_means) + bbox += add + + one_edge = (1 + 2 * cfg.KEYPOINTS.base_extend_range[0]) * bbox_width + center = [(bbox[0] + bbox[2]) // 2, (bbox[1] + bbox[3]) // 2] + + bbox[0] = center[0] - one_edge // 2 + bbox[1] = center[1] - one_edge // 2 + bbox[2] = center[0] + one_edge // 2 + bbox[3] = center[1] + one_edge // 2 + + bbox = bbox.astype(int) + crop_image = bimg[bbox[1]:bbox[3], bbox[0]:bbox[2], :] + h, w, _ = crop_image.shape + crop_image = cv2.resize( + crop_image, + (cfg.KEYPOINTS.input_shape[1], cfg.KEYPOINTS.input_shape[0])) + crop_image = crop_image.astype(np.float32) + + keypoints, state = self.simple_run(crop_image) + + res = keypoints[0][:self.keypoint_num].reshape((-1, 2)) + res[:, 0] = res[:, 0] * w / cfg.KEYPOINTS.input_shape[1] + res[:, 1] = res[:, 1] * h / cfg.KEYPOINTS.input_shape[0] + + landmark = [] + for _index in range(res.shape[0]): + x_y = res[_index] + landmark.append([ + int(x_y[0] * cfg.KEYPOINTS.input_shape[0] + bbox[0] - add), + int(x_y[1] * cfg.KEYPOINTS.input_shape[1] + bbox[1] - add) + ]) + + landmark = np.array(landmark, np.float32) + + return landmark, state + + def init_model(self, *args): + + if len(args) == 1: + use_pb = True + pb_path = args[0] + else: + use_pb = False + meta_path = args[0] + restore_model_path = args[1] + + def ini_ckpt(): + graph = tf.Graph() + graph.as_default() + configProto = tf.ConfigProto() + configProto.gpu_options.allow_growth = True + sess = tf.Session(config=configProto) + # load_model(model_path, sess) + saver = tf.train.import_meta_graph(meta_path) + saver.restore(sess, restore_model_path) + + print('Model restred!') + return (graph, sess) + + def init_pb(model_path): + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.2 + compute_graph = tf.Graph() + compute_graph.as_default() + sess = tf.Session(config=config) + with tf.gfile.GFile(model_path, 'rb') as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + tf.import_graph_def(graph_def, name='') + + # saver = tf.train.Saver(tf.global_variables()) + # saver.save(sess, save_path='./tmp.ckpt') + return (compute_graph, sess) + + if use_pb: + model = init_pb(pb_path) + else: + model = ini_ckpt() + + graph = model[0] + sess = model[1] + + return graph, sess diff --git a/runtime/Lib/site_packages/models/cv/cartoon/facelib/facer.py b/runtime/Lib/site_packages/models/cv/cartoon/facelib/facer.py new file mode 100644 index 0000000000000000000000000000000000000000..7e312cf3b0bdb8429d36f3df9fb662c16d0ff6b0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/facelib/facer.py @@ -0,0 +1,151 @@ +# The implementation is adopted from https://github.com/610265158/Peppa_Pig_Face_Engine + +import time + +import cv2 +import numpy as np + +from .config import config as cfg +from .face_detector import FaceDetector +from .face_landmark import FaceLandmark +from .LK.lk import GroupTrack + + +class FaceAna(): + ''' + by default the top3 facea sorted by area will be calculated for time reason + ''' + + def __init__(self, model_dir): + self.face_detector = FaceDetector(model_dir) + self.face_landmark = FaceLandmark(model_dir) + self.trace = GroupTrack() + + self.track_box = None + self.previous_image = None + self.previous_box = None + + self.diff_thres = 5 + self.top_k = cfg.DETECT.topk + self.iou_thres = cfg.TRACE.iou_thres + self.alpha = cfg.TRACE.smooth_box + + def run(self, image): + + boxes = self.face_detector(image) + + if boxes.shape[0] > self.top_k: + boxes = self.sort(boxes) + + boxes_return = np.array(boxes) + landmarks, states = self.face_landmark(image, boxes) + + if 1: + track = [] + for i in range(landmarks.shape[0]): + track.append([ + np.min(landmarks[i][:, 0]), + np.min(landmarks[i][:, 1]), + np.max(landmarks[i][:, 0]), + np.max(landmarks[i][:, 1]) + ]) + tmp_box = np.array(track) + + self.track_box = self.judge_boxs(boxes_return, tmp_box) + + self.track_box, landmarks = self.sort_res(self.track_box, landmarks) + return self.track_box, landmarks, states + + def sort_res(self, bboxes, points): + area = [] + for bbox in bboxes: + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + area.append(bbox_height * bbox_width) + + area = np.array(area) + picked = area.argsort()[::-1] + sorted_bboxes = [bboxes[x] for x in picked] + sorted_points = [points[x] for x in picked] + return np.array(sorted_bboxes), np.array(sorted_points) + + def diff_frames(self, previous_frame, image): + if previous_frame is None: + return True + else: + _diff = cv2.absdiff(previous_frame, image) + diff = np.sum( + _diff) / previous_frame.shape[0] / previous_frame.shape[1] / 3. + return diff > self.diff_thres + + def sort(self, bboxes): + if self.top_k > 100: + return bboxes + area = [] + for bbox in bboxes: + + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + area.append(bbox_height * bbox_width) + + area = np.array(area) + + picked = area.argsort()[-self.top_k:][::-1] + sorted_bboxes = [bboxes[x] for x in picked] + return np.array(sorted_bboxes) + + def judge_boxs(self, previous_bboxs, now_bboxs): + + def iou(rec1, rec2): + + # computing area of each rectangles + S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1]) + S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1]) + + # computing the sum_area + sum_area = S_rec1 + S_rec2 + + # find the each edge of intersect rectangle + x1 = max(rec1[0], rec2[0]) + y1 = max(rec1[1], rec2[1]) + x2 = min(rec1[2], rec2[2]) + y2 = min(rec1[3], rec2[3]) + + # judge if there is an intersect + intersect = max(0, x2 - x1) * max(0, y2 - y1) + + return intersect / (sum_area - intersect) + + if previous_bboxs is None: + return now_bboxs + + result = [] + + for i in range(now_bboxs.shape[0]): + contain = False + for j in range(previous_bboxs.shape[0]): + if iou(now_bboxs[i], previous_bboxs[j]) > self.iou_thres: + result.append(self.smooth(now_bboxs[i], previous_bboxs[j])) + contain = True + break + if not contain: + result.append(now_bboxs[i]) + + return np.array(result) + + def smooth(self, now_box, previous_box): + + return self.do_moving_average(now_box[:4], previous_box[:4]) + + def do_moving_average(self, p_now, p_previous): + p = self.alpha * p_now + (1 - self.alpha) * p_previous + return p + + def reset(self): + ''' + reset the previous info used foe tracking, + :return: + ''' + self.track_box = None + self.previous_image = None + self.previous_box = None diff --git a/runtime/Lib/site_packages/models/cv/cartoon/loss.py b/runtime/Lib/site_packages/models/cv/cartoon/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..9010d29aa8c2099365e161267544115eef478a0a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/loss.py @@ -0,0 +1,274 @@ +''' +CVPR 2020 submission, Paper ID 6791 +Source code for 'Learning to Cartoonize Using White-Box Cartoon Representations' +''' + +import os.path as osp + +import numpy as np +import scipy.stats as st +import tensorflow as tf +from joblib import Parallel, delayed +from skimage import color, segmentation + +from .network import disc_sn + +VGG_MEAN = [103.939, 116.779, 123.68] + + +class Vgg19: + + def __init__(self, vgg19_npy_path=None): + + self.data_dict = np.load( + vgg19_npy_path, encoding='latin1', allow_pickle=True).item() + + print('Finished loading vgg19.npy') + + def build_conv4_4(self, rgb, include_fc=False): + + rgb_scaled = (rgb + 1) * 127.5 + + blue, green, red = tf.split( + axis=3, num_or_size_splits=3, value=rgb_scaled) + bgr = tf.concat( + axis=3, + values=[ + blue - VGG_MEAN[0], green - VGG_MEAN[1], red - VGG_MEAN[2] + ]) + + self.conv1_1 = self.conv_layer(bgr, 'conv1_1') + self.relu1_1 = tf.nn.relu(self.conv1_1) + self.conv1_2 = self.conv_layer(self.relu1_1, 'conv1_2') + self.relu1_2 = tf.nn.relu(self.conv1_2) + self.pool1 = self.max_pool(self.relu1_2, 'pool1') + + self.conv2_1 = self.conv_layer(self.pool1, 'conv2_1') + self.relu2_1 = tf.nn.relu(self.conv2_1) + self.conv2_2 = self.conv_layer(self.relu2_1, 'conv2_2') + self.relu2_2 = tf.nn.relu(self.conv2_2) + self.pool2 = self.max_pool(self.relu2_2, 'pool2') + + self.conv3_1 = self.conv_layer(self.pool2, 'conv3_1') + self.relu3_1 = tf.nn.relu(self.conv3_1) + self.conv3_2 = self.conv_layer(self.relu3_1, 'conv3_2') + self.relu3_2 = tf.nn.relu(self.conv3_2) + self.conv3_3 = self.conv_layer(self.relu3_2, 'conv3_3') + self.relu3_3 = tf.nn.relu(self.conv3_3) + self.conv3_4 = self.conv_layer(self.relu3_3, 'conv3_4') + self.relu3_4 = tf.nn.relu(self.conv3_4) + self.pool3 = self.max_pool(self.relu3_4, 'pool3') + + self.conv4_1 = self.conv_layer(self.pool3, 'conv4_1') + self.relu4_1 = tf.nn.relu(self.conv4_1) + self.conv4_2 = self.conv_layer(self.relu4_1, 'conv4_2') + self.relu4_2 = tf.nn.relu(self.conv4_2) + self.conv4_3 = self.conv_layer(self.relu4_2, 'conv4_3') + self.relu4_3 = tf.nn.relu(self.conv4_3) + self.conv4_4 = self.conv_layer(self.relu4_3, 'conv4_4') + self.relu4_4 = tf.nn.relu(self.conv4_4) + self.pool4 = self.max_pool(self.relu4_4, 'pool4') + + return self.conv4_4 + + def max_pool(self, bottom, name): + return tf.nn.max_pool( + bottom, + ksize=[1, 2, 2, 1], + strides=[1, 2, 2, 1], + padding='SAME', + name=name) + + def conv_layer(self, bottom, name): + with tf.variable_scope(name): + filt = self.get_conv_filter(name) + conv = tf.nn.conv2d(bottom, filt, [1, 1, 1, 1], padding='SAME') + conv_biases = self.get_bias(name) + bias = tf.nn.bias_add(conv, conv_biases) + return bias + + def fc_layer(self, bottom, name): + with tf.variable_scope(name): + shape = bottom.get_shape().as_list() + dim = 1 + for d in shape[1:]: + dim *= d + x = tf.reshape(bottom, [-1, dim]) + weights = self.get_fc_weight(name) + biases = self.get_bias(name) + fc = tf.nn.bias_add(tf.matmul(x, weights), biases) + + return fc + + def get_conv_filter(self, name): + return tf.constant(self.data_dict[name][0], name='filter') + + def get_bias(self, name): + return tf.constant(self.data_dict[name][1], name='biases') + + def get_fc_weight(self, name): + return tf.constant(self.data_dict[name][0], name='weights') + + +def content_loss(model_dir, input_photo, transfer_res, input_superpixel): + vgg_model = Vgg19(osp.join(model_dir, 'vgg19.npy')) + vgg_photo = vgg_model.build_conv4_4(input_photo) + vgg_output = vgg_model.build_conv4_4(transfer_res) + vgg_superpixel = vgg_model.build_conv4_4(input_superpixel) + h, w, c = vgg_photo.get_shape().as_list()[1:] + abs_photo = tf.losses.absolute_difference(vgg_photo, vgg_output) + photo_loss = tf.reduce_mean(abs_photo) / (h * w * c) + abs_superpixel = tf.losses.absolute_difference(vgg_superpixel, vgg_output) + superpixel_loss = tf.reduce_mean(abs_superpixel) / (h * w * c) + loss = photo_loss + superpixel_loss + + return loss + + +def style_loss(input_cartoon, output_cartoon): + blur_fake = guided_filter(output_cartoon, output_cartoon, r=5, eps=2e-1) + blur_cartoon = guided_filter(input_cartoon, input_cartoon, r=5, eps=2e-1) + + gray_fake, gray_cartoon = color_shift(output_cartoon, input_cartoon) + + d_loss_gray, g_loss_gray = lsgan_loss( + disc_sn, + gray_cartoon, + gray_fake, + scale=1, + patch=True, + name='disc_gray') + d_loss_blur, g_loss_blur = lsgan_loss( + disc_sn, + blur_cartoon, + blur_fake, + scale=1, + patch=True, + name='disc_blur') + sty_g_loss = (g_loss_blur) + g_loss_gray + sty_d_loss = d_loss_blur + d_loss_gray + + return sty_g_loss, sty_d_loss + + +def gan_loss(discriminator, + real, + fake, + scale=1, + channel=32, + patch=False, + name='discriminator'): + + real_logit = discriminator( + real, scale, channel, name=name, patch=patch, reuse=False) + fake_logit = discriminator( + fake, scale, channel, name=name, patch=patch, reuse=True) + + real_logit = tf.nn.sigmoid(real_logit) + fake_logit = tf.nn.sigmoid(fake_logit) + + g_loss_blur = -tf.reduce_mean(tf.log(fake_logit)) + d_loss_blur = -tf.reduce_mean(tf.log(real_logit) + tf.log(1. - fake_logit)) + + return d_loss_blur, g_loss_blur + + +def lsgan_loss(discriminator, + real, + fake, + scale=1, + channel=32, + patch=False, + name='discriminator'): + + real_logit = discriminator( + real, scale, channel, name=name, patch=patch, reuse=False) + fake_logit = discriminator( + fake, scale, channel, name=name, patch=patch, reuse=True) + + g_loss = tf.reduce_mean((fake_logit - 1)**2) + d_loss = 0.5 * ( + tf.reduce_mean((real_logit - 1)**2) + tf.reduce_mean(fake_logit**2)) + + return d_loss, g_loss + + +def total_variation_loss(image, k_size=1): + h, w = image.get_shape().as_list()[1:3] + tv_h = tf.reduce_mean( + (image[:, k_size:, :, :] - image[:, :h - k_size, :, :])**2) + tv_w = tf.reduce_mean( + (image[:, :, k_size:, :] - image[:, :, :w - k_size, :])**2) + tv_loss = (tv_h + tv_w) / (3 * h * w) + return tv_loss + + +def guided_filter(x, y, r, eps=1e-2): + x_shape = tf.shape(x) + N = tf_box_filter( + tf.ones((1, x_shape[1], x_shape[2], 1), dtype=x.dtype), r) + + mean_x = tf_box_filter(x, r) / N + mean_y = tf_box_filter(y, r) / N + cov_xy = tf_box_filter(x * y, r) / N - mean_x * mean_y + var_x = tf_box_filter(x * x, r) / N - mean_x * mean_x + + A = cov_xy / (var_x + eps) + b = mean_y - A * mean_x + + mean_A = tf_box_filter(A, r) / N + mean_b = tf_box_filter(b, r) / N + + output = mean_A * x + mean_b + + return output + + +def color_shift(image1, image2, mode='uniform'): + b1, g1, r1 = tf.split(image1, num_or_size_splits=3, axis=3) + b2, g2, r2 = tf.split(image2, num_or_size_splits=3, axis=3) + if mode == 'normal': + b_weight = tf.random.normal(shape=[1], mean=0.114, stddev=0.1) + g_weight = np.random.normal(shape=[1], mean=0.587, stddev=0.1) + r_weight = np.random.normal(shape=[1], mean=0.299, stddev=0.1) + elif mode == 'uniform': + b_weight = tf.random.uniform(shape=[1], minval=0.014, maxval=0.214) + g_weight = tf.random.uniform(shape=[1], minval=0.487, maxval=0.687) + r_weight = tf.random.uniform(shape=[1], minval=0.199, maxval=0.399) + output1 = (b_weight * b1 + g_weight * g1 + r_weight * r1) / ( + b_weight + g_weight + r_weight) + output2 = (b_weight * b2 + g_weight * g2 + r_weight * r2) / ( + b_weight + g_weight + r_weight) + return output1, output2 + + +def simple_superpixel(batch_image, seg_num=200): + + def process_slic(image): + seg_label = segmentation.slic( + image, + n_segments=seg_num, + sigma=1, + compactness=10, + convert2lab=True, + start_label=1) + image = color.label2rgb(seg_label, image, kind='avg', bg_label=0) + return image + + num_job = np.shape(batch_image)[0] + batch_out = Parallel(n_jobs=num_job)( + delayed(process_slic)(image) for image in batch_image) + return np.array(batch_out) + + +def tf_box_filter(x, r): + ch = x.get_shape().as_list()[-1] + weight = 1 / ((2 * r + 1)**2) + box_kernel = weight * np.ones((2 * r + 1, 2 * r + 1, ch, 1)) + box_kernel = np.array(box_kernel).astype(np.float32) + output = tf.nn.depthwise_conv2d(x, box_kernel, [1, 1, 1, 1], 'SAME') + return output + + +if __name__ == '__main__': + pass diff --git a/runtime/Lib/site_packages/models/cv/cartoon/model_tf.py b/runtime/Lib/site_packages/models/cv/cartoon/model_tf.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7cf4ed1352ea242105dcca7cf767d7a43a5b57 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/model_tf.py @@ -0,0 +1,64 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Dict + +import tensorflow as tf + +from modelscope.models.base import Model, Tensor +from .loss import content_loss, guided_filter, style_loss, total_variation_loss +from .network import unet_generator + + +class CartoonModel(Model): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + + def __call__( + self, + input_photo: Dict[str, Tensor], + input_cartoon: Dict[str, Tensor] = None, + input_superpixel: Dict[str, Tensor] = None) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input_photo: the preprocessed input photo image + input_cartoon: the preprocessed input cartoon image + input_superpixel: the computed input superpixel image + + Returns: + output_dict: output dict of target ids + """ + if input_cartoon is None: + output = unet_generator(input_photo) + output_cartoon = guided_filter(input_photo, output, r=1) + return {'output_cartoon': output_cartoon} + else: + output = unet_generator(input_photo) + output_cartoon = guided_filter(input_photo, output, r=1) + + con_loss = content_loss(self.model_dir, input_photo, + output_cartoon, input_superpixel) + sty_g_loss, sty_d_loss = style_loss(input_cartoon, output_cartoon) + tv_loss = total_variation_loss(output_cartoon) + + g_loss = 1e-1 * sty_g_loss + 2e2 * con_loss + 1e4 * tv_loss + d_loss = sty_d_loss + + return { + 'output_cartoon': output_cartoon, + 'g_loss': g_loss, + 'd_loss': d_loss, + } + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """ + Run the forward pass for a model. + + Args: + input (Dict[str, Tensor]): the dict of the model inputs for the forward method + + Returns: + Dict[str, Tensor]: output from the model forward pass + """ diff --git a/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/__init__.py b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/__init__.py b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py new file mode 100644 index 0000000000000000000000000000000000000000..eb5420424e295fa4f1c2e8e4d0c32670f045a8c4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py @@ -0,0 +1,185 @@ +# The implementation is adopted from https://github.com/TreB1eN/InsightFace_Pytorch/tree/master/mtcnn_pytorch + +import cv2 +import numpy as np + +from .matlab_cp2tform import get_similarity_transform_for_cv2 + +# reference facial points, a list of coordinates (x,y) +dx = 1 +dy = 1 +REFERENCE_FACIAL_POINTS = [ + [30.29459953 + dx, 51.69630051 + dy], # left eye + [65.53179932 + dx, 51.50139999 + dy], # right eye + [48.02519989 + dx, 71.73660278 + dy], # nose + [33.54930115 + dx, 92.3655014 + dy], # left mouth + [62.72990036 + dx, 92.20410156 + dy] # right mouth +] + +DEFAULT_CROP_SIZE = (96, 112) + +global FACIAL_POINTS + + +class FaceWarpException(Exception): + + def __str__(self): + return 'In File {}:{}'.format(__file__, super.__str__(self)) + + +def get_reference_facial_points(output_size=None, + inner_padding_factor=0.0, + outer_padding=(0, 0), + default_square=False): + + tmp_5pts = np.array(REFERENCE_FACIAL_POINTS) + tmp_crop_size = np.array(DEFAULT_CROP_SIZE) + + # 0) make the inner region a square + if default_square: + size_diff = max(tmp_crop_size) - tmp_crop_size + tmp_5pts += size_diff / 2 + tmp_crop_size += size_diff + + h_crop = tmp_crop_size[0] + w_crop = tmp_crop_size[1] + if (output_size): + if (output_size[0] == h_crop and output_size[1] == w_crop): + return tmp_5pts + + if (inner_padding_factor == 0 and outer_padding == (0, 0)): + if output_size is None: + return tmp_5pts + else: + raise FaceWarpException( + 'No paddings to do, output_size must be None or {}'.format( + tmp_crop_size)) + + # check output size + if not (0 <= inner_padding_factor <= 1.0): + raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)') + + factor = inner_padding_factor > 0 or outer_padding[0] > 0 + factor = factor or outer_padding[1] > 0 + if (factor and output_size is None): + output_size = tmp_crop_size * \ + (1 + inner_padding_factor * 2).astype(np.int32) + output_size += np.array(outer_padding) + + cond1 = outer_padding[0] < output_size[0] + cond2 = outer_padding[1] < output_size[1] + if not (cond1 and cond2): + raise FaceWarpException('Not (outer_padding[0] < output_size[0]' + 'and outer_padding[1] < output_size[1])') + + # 1) pad the inner region according inner_padding_factor + if inner_padding_factor > 0: + size_diff = tmp_crop_size * inner_padding_factor * 2 + tmp_5pts += size_diff / 2 + tmp_crop_size += np.round(size_diff).astype(np.int32) + + # 2) resize the padded inner region + size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2 + + if size_bf_outer_pad[0] * tmp_crop_size[1] != size_bf_outer_pad[ + 1] * tmp_crop_size[0]: + raise FaceWarpException( + 'Must have (output_size - outer_padding)' + '= some_scale * (crop_size * (1.0 + inner_padding_factor)') + + scale_factor = size_bf_outer_pad[0].astype(np.float32) / tmp_crop_size[0] + tmp_5pts = tmp_5pts * scale_factor + + # 3) add outer_padding to make output_size + reference_5point = tmp_5pts + np.array(outer_padding) + + return reference_5point + + +def get_affine_transform_matrix(src_pts, dst_pts): + + tfm = np.float32([[1, 0, 0], [0, 1, 0]]) + n_pts = src_pts.shape[0] + ones = np.ones((n_pts, 1), src_pts.dtype) + src_pts_ = np.hstack([src_pts, ones]) + dst_pts_ = np.hstack([dst_pts, ones]) + + A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_) + + if rank == 3: + tfm = np.float32([[A[0, 0], A[1, 0], A[2, 0]], + [A[0, 1], A[1, 1], A[2, 1]]]) + elif rank == 2: + tfm = np.float32([[A[0, 0], A[1, 0], 0], [A[0, 1], A[1, 1], 0]]) + + return tfm + + +def warp_and_crop_face(src_img, + facial_pts, + ratio=0.84, + reference_pts=None, + crop_size=(96, 112), + align_type='similarity' + '', + return_trans_inv=False): + + if reference_pts is None: + if crop_size[0] == 96 and crop_size[1] == 112: + reference_pts = REFERENCE_FACIAL_POINTS + else: + default_square = False + inner_padding_factor = 0 + outer_padding = (0, 0) + output_size = crop_size + + reference_pts = get_reference_facial_points( + output_size, inner_padding_factor, outer_padding, + default_square) + + ref_pts = np.float32(reference_pts) + + factor = ratio + ref_pts = (ref_pts - 112 / 2) * factor + 112 / 2 + ref_pts *= crop_size[0] / 112. + + ref_pts_shp = ref_pts.shape + if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: + raise FaceWarpException( + 'reference_pts.shape must be (K,2) or (2,K) and K>2') + + if ref_pts_shp[0] == 2: + ref_pts = ref_pts.T + + src_pts = np.float32(facial_pts) + src_pts_shp = src_pts.shape + if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: + raise FaceWarpException( + 'facial_pts.shape must be (K,2) or (2,K) and K>2') + + if src_pts_shp[0] == 2: + src_pts = src_pts.T + + if src_pts.shape != ref_pts.shape: + raise FaceWarpException( + 'facial_pts and reference_pts must have the same shape') + + if align_type == 'cv2_affine': + tfm = cv2.getAffineTransform(src_pts, ref_pts) + tfm_inv = cv2.getAffineTransform(ref_pts, src_pts) + + elif align_type == 'affine': + tfm = get_affine_transform_matrix(src_pts, ref_pts) + tfm_inv = get_affine_transform_matrix(ref_pts, src_pts) + else: + tfm, tfm_inv = get_similarity_transform_for_cv2(src_pts, ref_pts) + + face_img = cv2.warpAffine( + src_img, + tfm, (crop_size[0], crop_size[1]), + borderValue=(255, 255, 255)) + + if return_trans_inv: + return face_img, tfm_inv + else: + return face_img diff --git a/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py new file mode 100644 index 0000000000000000000000000000000000000000..ea9fbacf121d3f47edad127e95ecd92b0780fc18 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py @@ -0,0 +1,335 @@ +# The implementation is adopted from https://github.com/TreB1eN/InsightFace_Pytorch/tree/master/mtcnn_pytorch + +import numpy as np +from numpy.linalg import inv, lstsq +from numpy.linalg import matrix_rank as rank +from numpy.linalg import norm + + +class MatlabCp2tormException(Exception): + + def __str__(self): + return 'In File {}:{}'.format(__file__, super.__str__(self)) + + +def tformfwd(trans, uv): + """ + Function: + ---------- + apply affine transform 'trans' to uv + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix + @uv: Kx2 np.array + each row is a pair of coordinates (x, y) + + Returns: + ---------- + @xy: Kx2 np.array + each row is a pair of transformed coordinates (x, y) + """ + uv = np.hstack((uv, np.ones((uv.shape[0], 1)))) + xy = np.dot(uv, trans) + xy = xy[:, 0:-1] + return xy + + +def tforminv(trans, uv): + """ + Function: + ---------- + apply the inverse of affine transform 'trans' to uv + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix + @uv: Kx2 np.array + each row is a pair of coordinates (x, y) + + Returns: + ---------- + @xy: Kx2 np.array + each row is a pair of inverse-transformed coordinates (x, y) + """ + Tinv = inv(trans) + xy = tformfwd(Tinv, uv) + return xy + + +def findNonreflectiveSimilarity(uv, xy, options=None): + + options = {'K': 2} + + K = options['K'] + M = xy.shape[0] + x = xy[:, 0].reshape((-1, 1)) # use reshape to keep a column vector + y = xy[:, 1].reshape((-1, 1)) # use reshape to keep a column vector + # print('--->x, y:\n', x, y + + tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1)))) + tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1)))) + X = np.vstack((tmp1, tmp2)) + # print('--->X.shape: ', X.shape + # print('X:\n', X + + u = uv[:, 0].reshape((-1, 1)) # use reshape to keep a column vector + v = uv[:, 1].reshape((-1, 1)) # use reshape to keep a column vector + U = np.vstack((u, v)) + # print('--->U.shape: ', U.shape + # print('U:\n', U + + # We know that X * r = U + if rank(X) >= 2 * K: + r, _, _, _ = lstsq(X, U) + r = np.squeeze(r) + else: + raise Exception('cp2tform:twoUniquePointsReq') + + # print('--->r:\n', r + + sc = r[0] + ss = r[1] + tx = r[2] + ty = r[3] + + Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]]) + + # print('--->Tinv:\n', Tinv + + T = inv(Tinv) + # print('--->T:\n', T + + T[:, 2] = np.array([0, 0, 1]) + + return T, Tinv + + +def findSimilarity(uv, xy, options=None): + + options = {'K': 2} + + # uv = np.array(uv) + # xy = np.array(xy) + + # Solve for trans1 + trans1, trans1_inv = findNonreflectiveSimilarity(uv, xy, options) + + # Solve for trans2 + + # manually reflect the xy data across the Y-axis + xyR = xy + xyR[:, 0] = -1 * xyR[:, 0] + + trans2r, trans2r_inv = findNonreflectiveSimilarity(uv, xyR, options) + + # manually reflect the tform to undo the reflection done on xyR + TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]]) + + trans2 = np.dot(trans2r, TreflectY) + + # Figure out if trans1 or trans2 is better + xy1 = tformfwd(trans1, uv) + norm1 = norm(xy1 - xy) + + xy2 = tformfwd(trans2, uv) + norm2 = norm(xy2 - xy) + + if norm1 <= norm2: + return trans1, trans1_inv + else: + trans2_inv = inv(trans2) + return trans2, trans2_inv + + +def get_similarity_transform(src_pts, dst_pts, reflective=True): + """ + Function: + ---------- + Find Similarity Transform Matrix 'trans': + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y, 1] = [u, v, 1] * trans + + Parameters: + ---------- + @src_pts: Kx2 np.array + source points, each row is a pair of coordinates (x, y) + @dst_pts: Kx2 np.array + destination points, each row is a pair of transformed + coordinates (x, y) + @reflective: True or False + if True: + use reflective similarity transform + else: + use non-reflective similarity transform + + Returns: + ---------- + @trans: 3x3 np.array + transform matrix from uv to xy + trans_inv: 3x3 np.array + inverse of trans, transform matrix from xy to uv + """ + + if reflective: + trans, trans_inv = findSimilarity(src_pts, dst_pts) + else: + trans, trans_inv = findNonreflectiveSimilarity(src_pts, dst_pts) + + return trans, trans_inv + + +def cvt_tform_mat_for_cv2(trans): + """ + Function: + ---------- + Convert Transform Matrix 'trans' into 'cv2_trans' which could be + directly used by cv2.warpAffine(): + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y].T = cv_trans * [u, v, 1].T + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix from uv to xy + + Returns: + ---------- + @cv2_trans: 2x3 np.array + transform matrix from src_pts to dst_pts, could be directly used + for cv2.warpAffine() + """ + cv2_trans = trans[:, 0:2].T + + return cv2_trans + + +def get_similarity_transform_for_cv2(src_pts, dst_pts, reflective=True): + """ + Function: + ---------- + Find Similarity Transform Matrix 'cv2_trans' which could be + directly used by cv2.warpAffine(): + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y].T = cv_trans * [u, v, 1].T + + Parameters: + ---------- + @src_pts: Kx2 np.array + source points, each row is a pair of coordinates (x, y) + @dst_pts: Kx2 np.array + destination points, each row is a pair of transformed + coordinates (x, y) + reflective: True or False + if True: + use reflective similarity transform + else: + use non-reflective similarity transform + + Returns: + ---------- + @cv2_trans: 2x3 np.array + transform matrix from src_pts to dst_pts, could be directly used + for cv2.warpAffine() + """ + trans, trans_inv = get_similarity_transform(src_pts, dst_pts, reflective) + cv2_trans = cvt_tform_mat_for_cv2(trans) + cv2_trans_inv = cvt_tform_mat_for_cv2(trans_inv) + + return cv2_trans, cv2_trans_inv + + +if __name__ == '__main__': + """ + u = [0, 6, -2] + v = [0, 3, 5] + x = [-1, 0, 4] + y = [-1, -10, 4] + + # In Matlab, run: + # + # uv = [u'; v']; + # xy = [x'; y']; + # tform_sim=cp2tform(uv,xy,'similarity'); + # + # trans = tform_sim.tdata.T + # ans = + # -0.0764 -1.6190 0 + # 1.6190 -0.0764 0 + # -3.2156 0.0290 1.0000 + # trans_inv = tform_sim.tdata.Tinv + # ans = + # + # -0.0291 0.6163 0 + # -0.6163 -0.0291 0 + # -0.0756 1.9826 1.0000 + # xy_m=tformfwd(tform_sim, u,v) + # + # xy_m = + # + # -3.2156 0.0290 + # 1.1833 -9.9143 + # 5.0323 2.8853 + # uv_m=tforminv(tform_sim, x,y) + # + # uv_m = + # + # 0.5698 1.3953 + # 6.0872 2.2733 + # -2.6570 4.3314 + """ + u = [0, 6, -2] + v = [0, 3, 5] + x = [-1, 0, 4] + y = [-1, -10, 4] + + uv = np.array((u, v)).T + xy = np.array((x, y)).T + + print('\n--->uv:') + print(uv) + print('\n--->xy:') + print(xy) + + trans, trans_inv = get_similarity_transform(uv, xy) + + print('\n--->trans matrix:') + print(trans) + + print('\n--->trans_inv matrix:') + print(trans_inv) + + print('\n---> apply transform to uv') + print('\nxy_m = uv_augmented * trans') + uv_aug = np.hstack((uv, np.ones((uv.shape[0], 1)))) + xy_m = np.dot(uv_aug, trans) + print(xy_m) + + print('\nxy_m = tformfwd(trans, uv)') + xy_m = tformfwd(trans, uv) + print(xy_m) + + print('\n---> apply inverse transform to xy') + print('\nuv_m = xy_augmented * trans_inv') + xy_aug = np.hstack((xy, np.ones((xy.shape[0], 1)))) + uv_m = np.dot(xy_aug, trans_inv) + print(uv_m) + + print('\nuv_m = tformfwd(trans_inv, xy)') + uv_m = tformfwd(trans_inv, xy) + print(uv_m) + + uv_m = tforminv(trans, xy) + print('\nuv_m = tforminv(trans, xy)') + print(uv_m) diff --git a/runtime/Lib/site_packages/models/cv/cartoon/network.py b/runtime/Lib/site_packages/models/cv/cartoon/network.py new file mode 100644 index 0000000000000000000000000000000000000000..6a00a50662d6996ba77aa1472235d306acdc3be7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/network.py @@ -0,0 +1,146 @@ +import tensorflow as tf +import tensorflow.contrib.slim as slim + + +def resblock(inputs, out_channel=32, name='resblock'): + with tf.variable_scope(name): + x = slim.convolution2d( + inputs, out_channel, [3, 3], activation_fn=None, scope='conv1') + + x = tf.nn.leaky_relu(x) + x = slim.convolution2d( + x, out_channel, [3, 3], activation_fn=None, scope='conv2') + + return x + inputs + + +def spectral_norm(w, iteration=1): + w_shape = w.shape.as_list() + w = tf.reshape(w, [-1, w_shape[-1]]) + + u = tf.get_variable( + 'u', [1, w_shape[-1]], + initializer=tf.random_normal_initializer(), + trainable=False) + + u_hat = u + v_hat = None + for i in range(iteration): + """ + power iteration + Usually iteration = 1 will be enough + """ + v_ = tf.matmul(u_hat, tf.transpose(w)) + v_hat = tf.nn.l2_normalize(v_) + + u_ = tf.matmul(v_hat, w) + u_hat = tf.nn.l2_normalize(u_) + + u_hat = tf.stop_gradient(u_hat) + v_hat = tf.stop_gradient(v_hat) + + sigma = tf.matmul(tf.matmul(v_hat, w), tf.transpose(u_hat)) + + with tf.control_dependencies([u.assign(u_hat)]): + w_norm = w / sigma + w_norm = tf.reshape(w_norm, w_shape) + + return w_norm + + +def conv_spectral_norm(x, channel, k_size, stride=1, name='conv_snorm'): + with tf.variable_scope(name): + w = tf.get_variable( + 'kernel', shape=[k_size[0], k_size[1], + x.get_shape()[-1], channel]) + b = tf.get_variable( + 'bias', [channel], initializer=tf.constant_initializer(0.0)) + + x = tf.nn.conv2d( + input=x, + filter=spectral_norm(w), + strides=[1, stride, stride, 1], + padding='SAME') + b + + return x + + +def unet_generator(inputs, + channel=32, + num_blocks=4, + name='generator', + reuse=False): + with tf.variable_scope(name, reuse=reuse): + + x0 = slim.convolution2d(inputs, channel, [7, 7], activation_fn=None) + x0 = tf.nn.leaky_relu(x0) + + x1 = slim.convolution2d( + x0, channel, [3, 3], stride=2, activation_fn=None) + x1 = tf.nn.leaky_relu(x1) + x1 = slim.convolution2d(x1, channel * 2, [3, 3], activation_fn=None) + x1 = tf.nn.leaky_relu(x1) + + x2 = slim.convolution2d( + x1, channel * 2, [3, 3], stride=2, activation_fn=None) + x2 = tf.nn.leaky_relu(x2) + x2 = slim.convolution2d(x2, channel * 4, [3, 3], activation_fn=None) + x2 = tf.nn.leaky_relu(x2) + + for idx in range(num_blocks): + x2 = resblock( + x2, out_channel=channel * 4, name='block_{}'.format(idx)) + + x2 = slim.convolution2d(x2, channel * 2, [3, 3], activation_fn=None) + x2 = tf.nn.leaky_relu(x2) + + h1, w1 = tf.shape(x2)[1], tf.shape(x2)[2] + x3 = tf.image.resize_bilinear(x2, (h1 * 2, w1 * 2)) + x3 = slim.convolution2d( + x3 + x1, channel * 2, [3, 3], activation_fn=None) + x3 = tf.nn.leaky_relu(x3) + x3 = slim.convolution2d(x3, channel, [3, 3], activation_fn=None) + x3 = tf.nn.leaky_relu(x3) + + h2, w2 = tf.shape(x3)[1], tf.shape(x3)[2] + x4 = tf.image.resize_bilinear(x3, (h2 * 2, w2 * 2)) + x4 = slim.convolution2d(x4 + x0, channel, [3, 3], activation_fn=None) + x4 = tf.nn.leaky_relu(x4) + x4 = slim.convolution2d(x4, 3, [7, 7], activation_fn=None) + + # x4 = tf.clip_by_value(x4, -1, 1) + return x4 + + +def disc_sn(x, + scale=1, + channel=32, + patch=True, + name='discriminator', + reuse=False): + with tf.variable_scope(name, reuse=reuse): + + for idx in range(3): + x = conv_spectral_norm( + x, + channel * 2**idx, [3, 3], + stride=2, + name='conv{}_1'.format(idx)) + x = tf.nn.leaky_relu(x) + + x = conv_spectral_norm( + x, channel * 2**idx, [3, 3], name='conv{}_2'.format(idx)) + x = tf.nn.leaky_relu(x) + + if patch is True: + x = conv_spectral_norm(x, 1, [1, 1], name='conv_out') + + else: + x = tf.reduce_mean(x, axis=[1, 2]) + x = slim.fully_connected(x, 1, activation_fn=None) + + return x + + +if __name__ == '__main__': + pass diff --git a/runtime/Lib/site_packages/models/cv/cartoon/utils.py b/runtime/Lib/site_packages/models/cv/cartoon/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f93e332487adc39033231cfe508899f524a7f1f0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cartoon/utils.py @@ -0,0 +1,171 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import random + +import cv2 +import numpy as np +import tensorflow as tf + + +def resize_size(image, size=720): + h, w, c = np.shape(image) + if min(h, w) > size: + if h > w: + h, w = int(size * h / w), size + else: + h, w = size, int(size * w / h) + image = cv2.resize(image, (w, h), interpolation=cv2.INTER_AREA) + return image + + +def padTo16x(image): + h, w, c = np.shape(image) + if h % 16 == 0 and w % 16 == 0: + return image, h, w + nh, nw = (h // 16 + 1) * 16, (w // 16 + 1) * 16 + img_new = np.ones((nh, nw, 3), np.uint8) * 255 + img_new[:h, :w, :] = image + + return img_new, h, w + + +def get_f5p(landmarks, np_img): + eye_left = find_pupil(landmarks[36:41], np_img) + eye_right = find_pupil(landmarks[42:47], np_img) + if eye_left is None or eye_right is None: + print('cannot find 5 points with find_puil, used mean instead.!') + eye_left = landmarks[36:41].mean(axis=0) + eye_right = landmarks[42:47].mean(axis=0) + nose = landmarks[30] + mouth_left = landmarks[48] + mouth_right = landmarks[54] + f5p = [[eye_left[0], eye_left[1]], [eye_right[0], eye_right[1]], + [nose[0], nose[1]], [mouth_left[0], mouth_left[1]], + [mouth_right[0], mouth_right[1]]] + return f5p + + +def find_pupil(landmarks, np_img): + h, w, _ = np_img.shape + xmax = int(landmarks[:, 0].max()) + xmin = int(landmarks[:, 0].min()) + ymax = int(landmarks[:, 1].max()) + ymin = int(landmarks[:, 1].min()) + + if ymin >= ymax or xmin >= xmax or ymin < 0 or xmin < 0 or ymax > h or xmax > w: + return None + eye_img_bgr = np_img[ymin:ymax, xmin:xmax, :] + eye_img = cv2.cvtColor(eye_img_bgr, cv2.COLOR_BGR2GRAY) + eye_img = cv2.equalizeHist(eye_img) + n_marks = landmarks - np.array([xmin, ymin]).reshape([1, 2]) + eye_mask = cv2.fillConvexPoly( + np.zeros_like(eye_img), n_marks.astype(np.int32), 1) + ret, thresh = cv2.threshold(eye_img, 100, 255, + cv2.THRESH_BINARY | cv2.THRESH_OTSU) + thresh = (1 - thresh / 255.) * eye_mask + cnt = 0 + xm = [] + ym = [] + for i in range(thresh.shape[0]): + for j in range(thresh.shape[1]): + if thresh[i, j] > 0.5: + xm.append(j) + ym.append(i) + cnt += 1 + if cnt != 0: + xm.sort() + ym.sort() + xm = xm[cnt // 2] + ym = ym[cnt // 2] + else: + xm = thresh.shape[1] / 2 + ym = thresh.shape[0] / 2 + + return xm + xmin, ym + ymin + + +def next_batch(filename_list, batch_size, fineSize=256): + idx = np.arange(0, len(filename_list)) + np.random.shuffle(idx) + idx = idx[:batch_size] + batch_data = [] + for i in range(batch_size): + image = cv2.imread(filename_list[idx[i]]) + h, w, c = image.shape + rw = random.randint(0, w - fineSize) + rh = random.randint(0, h - fineSize) + image = image[rh:rh + fineSize, rw:rw + fineSize, :] + image = image.astype(np.float32) / 127.5 - 1 + batch_data.append(image) + + return np.asarray(batch_data) + + +def read_image(image_path, IMAGE_SIZE=256): + image = tf.io.read_file(image_path) + image = tf.image.decode_image(image, channels=3) + image = tf.image.convert_image_dtype(image, tf.float32) + image.set_shape([None, None, 3]) + image = tf.image.resize(images=image, size=[IMAGE_SIZE, IMAGE_SIZE]) + image = image[..., ::-1] + # image = image / 127.5 - 1 + image = (image - 0.5) * 2 + + return image + + +def load_data(photo_list): + photo = read_image(photo_list) + return photo + + +def tf_data_loader(image_list, batch_size): + dataset = tf.data.Dataset.from_tensor_slices((image_list)) + dataset = dataset.shuffle(len(image_list)) + dataset = dataset.map(load_data, num_parallel_calls=4) + dataset = dataset.batch(batch_size) + dataset = dataset.prefetch(1) + return dataset + + +def write_batch_image(image, save_dir, name, n): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + fused_dir = os.path.join(save_dir, name) + fused_image = [0] * n + for i in range(n): + fused_image[i] = [] + for j in range(n): + k = i * n + j + image[k] = (image[k] + 1) * 127.5 + image[k] = np.clip(image[k], 0, 255) + fused_image[i].append(image[k]) + fused_image[i] = np.hstack(fused_image[i]) + fused_image = np.vstack(fused_image) + cv2.imwrite(fused_dir, fused_image.astype(np.uint8)) + + +def grid_batch_image(image, n): + fused_image = [0] * n + for i in range(n): + fused_image[i] = [] + for j in range(n): + k = i * n + j + image[k] = (image[k] + 1) * 127.5 + image[k] = np.clip(image[k], 0, 255) + fused_image[i].append(image[k]) + fused_image[i] = np.hstack(fused_image[i]) + fused_image = np.vstack(fused_image) + return fused_image + + +def all_file(file_dir): + L = [] + for root, dirs, files in os.walk(file_dir): + for file in files: + extend = os.path.splitext(file)[1] + if extend == '.png' or extend == '.jpg' or extend == '.jpeg' or extend == '.JPG': + L.append(os.path.join(root, file)) + return L diff --git a/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/__init__.py b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5bc67b633e4b688e250aa801fb9e00070ebd9732 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .c3d import C3D + from .resnet2p1d import resnet26_2p1d + from .resnet3d import resnet26_3d + +else: + _import_structure = { + 'c3d': ['C3D'], + 'resnet2p1d': ['resnet26_2p1d'], + 'resnet3d': ['resnet26_3d'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/c3d.py b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/c3d.py new file mode 100644 index 0000000000000000000000000000000000000000..53dd05a1a6e3b0510edd728408be0cbccfba8e71 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/c3d.py @@ -0,0 +1,129 @@ +# Copyright 2022 Davide Abati. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +# The implementation here is modified based on c3d-pytorch, +# originally MIT License, Copyright (c) 2022 Davide Abati, +# and publicly available at https://github.com/DavideA/c3d-pytorch +""" C3D Model Architecture.""" + +import torch +import torch.nn as nn + + +def conv3x3x3(in_planes, out_planes, stride=1): + return nn.Conv3d( + in_planes, out_planes, kernel_size=3, stride=stride, padding=1) + + +class C3D(nn.Module): + + def __init__(self, + num_classes=1000, + dropout=0.5, + inplanes=3, + norm_layer=None, + last_pool=True): + super(C3D, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if not last_pool and num_classes is not None: + raise ValueError('num_classes should be None when last_pool=False') + + self.conv1 = conv3x3x3(inplanes, 64) + self.bn1 = norm_layer(64) + self.relu1 = nn.ReLU(inplace=True) + self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2)) + + self.conv2 = conv3x3x3(64, 128) + self.bn2 = norm_layer(128) + self.relu2 = nn.ReLU(inplace=True) + self.pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) + + self.conv3a = conv3x3x3(128, 256) + self.bn3a = norm_layer(256) + self.relu3a = nn.ReLU(inplace=True) + + self.conv3b = conv3x3x3(256, 256) + self.bn3b = norm_layer(256) + self.relu3b = nn.ReLU(inplace=True) + self.pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) + + self.conv4a = conv3x3x3(256, 512) + self.bn4a = norm_layer(512) + self.relu4a = nn.ReLU(inplace=True) + + self.conv4b = conv3x3x3(512, 512) + self.bn4b = norm_layer(512) + self.relu4b = nn.ReLU(inplace=True) + self.pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) + + self.conv5a = conv3x3x3(512, 512) + self.bn5a = norm_layer(512) + self.relu5a = nn.ReLU(inplace=True) + + self.conv5b = conv3x3x3(512, 512) + self.bn5b = norm_layer(512) + self.relu5b = nn.ReLU(inplace=True) + self.pool5 = nn.AdaptiveAvgPool3d((1, 1, 1)) if last_pool else None + + if num_classes is None: + self.dropout = None + self.fc = None + else: + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(512, num_classes) + self.out_planes = 512 + + for m in self.modules(): + if isinstance(m, nn.Conv3d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm3d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu1(x) + x = self.pool1(x) + + x = self.conv2(x) + x = self.bn2(x) + x = self.relu2(x) + x = self.pool2(x) + + x = self.conv3a(x) + x = self.bn3a(x) + x = self.relu3a(x) + + x = self.conv3b(x) + x = self.bn3b(x) + x = self.relu3b(x) + x = self.pool3(x) + + x = self.conv4a(x) + x = self.bn4a(x) + x = self.relu4a(x) + + x = self.conv4b(x) + x = self.bn4b(x) + x = self.relu4b(x) + x = self.pool4(x) + + x = self.conv5a(x) + x = self.bn5a(x) + x = self.relu5a(x) + + x = self.conv5b(x) + x = self.bn5b(x) + x = self.relu5b(x) + + if self.pool5: + x = self.pool5(x) + x = torch.flatten(x, 1) + if self.dropout and self.fc: + x = self.dropout(x) + x = self.fc(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet2p1d.py b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet2p1d.py new file mode 100644 index 0000000000000000000000000000000000000000..b49069d12e459273535479cd44ef3968fd981565 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet2p1d.py @@ -0,0 +1,347 @@ +# Copyright (c) 2022 Kensho Hara. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +# The implementation here is modified based on 3D-ResNets-PyTorch, +# originally MIT License, Copyright (c) 2022 Kensho Hara, +# and publicly available at https://github.com/kenshohara/3D-ResNets-PyTorch/blob/master/models/resnet2p1d.py +""" ResNet2plus1d Model Architecture.""" + +import torch +import torch.nn as nn + + +def conv1x3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=(1, 3, 3), + stride=(1, stride, stride), + padding=(0, dilation, dilation), + groups=groups, + bias=False, + dilation=(1, dilation, dilation)) + + +def conv3x1x1(in_planes, out_planes, stride=1, groups=1, dilation=1): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=(3, 1, 1), + stride=(stride, 1, 1), + padding=(dilation, 0, 0), + groups=groups, + bias=False, + dilation=(dilation, 1, 1)) + + +def conv1x1x1(in_planes, out_planes, stride=1): + return nn.Conv3d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + + midplanes1 = (inplanes * planes * 3 * 3 * 3) // ( + inplanes * 3 * 3 + planes * 3) + self.conv1_s = conv1x3x3(inplanes, midplanes1, stride) + self.bn1_s = norm_layer(midplanes1) + self.conv1_t = conv3x1x1(midplanes1, planes, stride) + self.bn1_t = norm_layer(planes) + + midplanes2 = (planes * planes * 3 * 3 * 3) // ( + planes * 3 * 3 + planes * 3) + self.conv2_s = conv1x3x3(planes, midplanes2) + self.bn2_s = norm_layer(midplanes2) + self.conv2_t = conv3x1x1(midplanes2, planes) + self.bn2_t = norm_layer(planes) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1_s(x) + out = self.bn1_s(out) + out = self.relu(out) + out = self.conv1_t(out) + out = self.bn1_t(out) + out = self.relu(out) + + out = self.conv2_s(out) + out = self.bn2_s(out) + out = self.relu(out) + out = self.conv2_t(out) + out = self.bn2_t(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + width = int(planes * (base_width / 64.)) * groups + + self.conv1 = conv1x1x1(inplanes, width) + self.bn1 = norm_layer(width) + + midplanes = (width * width * 3 * 3 * 3) // (width * 3 * 3 + width * 3) + self.conv2_s = conv1x3x3(width, midplanes, stride, groups, dilation) + self.bn2_s = norm_layer(midplanes) + self.conv2_t = conv3x1x1(midplanes, width, stride, groups, dilation) + self.bn2_t = norm_layer(width) + + self.conv3 = conv1x1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2_s(out) + out = self.bn2_s(out) + out = self.relu(out) + out = self.conv2_t(out) + out = self.bn2_t(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet2p1d(nn.Module): + + def __init__(self, + block, + layers, + num_classes=None, + zero_init_residual=True, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + dropout=0.5, + inplanes=3, + first_stride=2, + norm_layer=None, + last_pool=True): + super(ResNet2p1d, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if not last_pool and num_classes is not None: + raise ValueError('num_classes should be None when last_pool=False') + self._norm_layer = norm_layer + self.first_stride = first_stride + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + midplanes = (3 * self.inplanes * 3 * 7 * 7) // (3 * 7 * 7 + + self.inplanes * 3) + self.conv1_s = nn.Conv3d( + inplanes, + midplanes, + kernel_size=(1, 7, 7), + stride=(1, first_stride, first_stride), + padding=(0, 3, 3), + bias=False) + self.bn1_s = norm_layer(midplanes) + self.conv1_t = nn.Conv3d( + midplanes, + self.inplanes, + kernel_size=(3, 1, 1), + stride=(1, 1, 1), + padding=(1, 0, 0), + bias=False) + self.bn1_t = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool3d( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1)) if last_pool else None + if num_classes is None: + self.dropout = None + self.fc = None + else: + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(512 * block.expansion, num_classes) + self.out_planes = 512 * block.expansion + + for m in self.modules(): + if isinstance(m, nn.Conv3d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm3d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2_t.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion)) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1_s(x) + x = self.bn1_s(x) + x = self.relu(x) + x = self.conv1_t(x) + x = self.bn1_t(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + if self.avgpool: + x = self.avgpool(x) + x = torch.flatten(x, 1) + if self.dropout and self.fc: + x = self.dropout(x) + x = self.fc(x) + + return x + + +def resnet10_2p1d(**kwargs): + return ResNet2p1d(BasicBlock, [1, 1, 1, 1], **kwargs) + + +def resnet18_2p1d(**kwargs): + return ResNet2p1d(BasicBlock, [2, 2, 2, 2], **kwargs) + + +def resnet26_2p1d(**kwargs): + return ResNet2p1d(Bottleneck, [2, 2, 2, 2], **kwargs) + + +def resnet34_2p1d(**kwargs): + return ResNet2p1d(BasicBlock, [3, 4, 6, 3], **kwargs) + + +def resnet50_2p1d(**kwargs): + return ResNet2p1d(Bottleneck, [3, 4, 6, 3], **kwargs) + + +def resnet101_2p1d(**kwargs): + return ResNet2p1d(Bottleneck, [3, 4, 23, 3], **kwargs) + + +def resnet152_2p1d(**kwargs): + return ResNet2p1d(Bottleneck, [3, 8, 36, 3], **kwargs) + + +def resnet200_2p1d(**kwargs): + return ResNet2p1d(Bottleneck, [3, 24, 36, 3], **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet3d.py b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet3d.py new file mode 100644 index 0000000000000000000000000000000000000000..dddba06f9d1dae146b30c0b55ab3aa6ab5a6f48e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/cmdssl_video_embedding/resnet3d.py @@ -0,0 +1,292 @@ +# Copyright (c) 2022 Kensho Hara. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +# The implementation here is modified based on 3D-ResNets-PyTorch, +# originally MIT License, Copyright (c) 2022 Kensho Hara, +# and publicly available at https://github.com/kenshohara/3D-ResNets-PyTorch/blob/master/models/resnet.py +""" ResNet3D Model Architecture.""" + +import torch +import torch.nn as nn + + +def conv3x3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + return nn.Conv3d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1x1(in_planes, out_planes, stride=1): + return nn.Conv3d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + self.conv1 = conv3x3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + width = int(planes * (base_width / 64.)) * groups + self.conv1 = conv1x1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet3d(nn.Module): + + def __init__(self, + block, + layers, + num_classes=1000, + zero_init_residual=True, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + dropout=0.5, + inplanes=3, + first_stride=2, + norm_layer=None, + last_pool=True): + super(ResNet3d, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm3d + if not last_pool and num_classes is not None: + raise ValueError('num_classes should be None when last_pool=False') + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv3d( + inplanes, + self.inplanes, + kernel_size=(3, 7, 7), + stride=(1, first_stride, first_stride), + padding=(1, 3, 3), + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool3d( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1)) if last_pool else None + if num_classes is None: + self.dropout = None + self.fc = None + else: + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(512 * block.expansion, num_classes) + self.out_planes = 512 * block.expansion + + for m in self.modules(): + if isinstance(m, nn.Conv3d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm3d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion)) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + if self.avgpool: + x = self.avgpool(x) + x = torch.flatten(x, 1) + if self.dropout and self.fc: + x = self.dropout(x) + x = self.fc(x) + + return x + + +def resnet10_3d(**kwargs): + return ResNet3d(BasicBlock, [1, 1, 1, 1], **kwargs) + + +def resnet18_3d(**kwargs): + return ResNet3d(BasicBlock, [2, 2, 2, 2], **kwargs) + + +def resnet26_3d(**kwargs): + return ResNet3d(Bottleneck, [2, 2, 2, 2], **kwargs) + + +def resnet34_3d(**kwargs): + return ResNet3d(BasicBlock, [3, 4, 6, 3], **kwargs) + + +def resnet50_3d(**kwargs): + return ResNet3d(Bottleneck, [3, 4, 6, 3], **kwargs) + + +def resnet101_3d(**kwargs): + return ResNet3d(Bottleneck, [3, 4, 23, 3], **kwargs) + + +def resnet152_3d(**kwargs): + return ResNet3d(Bottleneck, [3, 8, 36, 3], **kwargs) + + +def resnet200_3d(**kwargs): + return ResNet3d(Bottleneck, [3, 24, 36, 3], **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7cb371bd7d2a60b8a0bf7f05bd927f6955908de1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .controlnet import ControlNet + +else: + _import_structure = {'controlnet': ['ControlNet']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/annotator.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/annotator.py new file mode 100644 index 0000000000000000000000000000000000000000..12076080eac04e6a158ef8f1720e8dca81985289 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/annotator.py @@ -0,0 +1,379 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import os + +import cv2 +import mmcv +import numpy as np +import torch +from einops import rearrange +from mmseg.apis import inference_segmentor, init_segmentor +from mmseg.core.evaluation import get_palette + +from .midas.api import MiDaSInference +from .mlsd.mbv2_mlsd_large import MobileV2_MLSD_Large +from .mlsd.utils import pred_lines +from .openpose import util +from .openpose.body import Body +from .openpose.hand import Hand + +os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' + + +class OpenposeDetector: + + def __init__(self, annotator_ckpts_path, device='cuda'): + body_modelpath = os.path.join(annotator_ckpts_path, + 'body_pose_model.pth') + hand_modelpath = os.path.join(annotator_ckpts_path, + 'hand_pose_model.pth') + + self.body_estimation = Body(body_modelpath, device) + self.hand_estimation = Hand(hand_modelpath, device) + + def __call__(self, oriImg, hand=False): + oriImg = oriImg[:, :, ::-1].copy() + with torch.no_grad(): + candidate, subset = self.body_estimation(oriImg) + canvas = np.zeros_like(oriImg) + canvas = util.draw_bodypose(canvas, candidate, subset) + if hand: + hands_list = util.handDetect(candidate, subset, oriImg) + all_hand_peaks = [] + for x, y, w, is_left in hands_list: + peaks = self.hand_estimation(oriImg[y:y + w, x:x + w, :]) + peaks[:, 0] = np.where(peaks[:, 0] == 0, peaks[:, 0], + peaks[:, 0] + x) + peaks[:, 1] = np.where(peaks[:, 1] == 0, peaks[:, 1], + peaks[:, 1] + y) + all_hand_peaks.append(peaks) + canvas = util.draw_handpose(canvas, all_hand_peaks) + return canvas, dict( + candidate=candidate.tolist(), subset=subset.tolist()) + + +class MLSDdetector: + + def __init__(self, annotator_ckpts_path, device='cuda'): + model_path = os.path.join(annotator_ckpts_path, + 'mlsd_large_512_fp32.pth') + model = MobileV2_MLSD_Large() + model.load_state_dict(torch.load(model_path), strict=True) + self.model = model.to(device).eval() + + def __call__(self, input_image, thr_v, thr_d): + assert input_image.ndim == 3 + img = input_image + img_output = np.zeros_like(img) + try: + with torch.no_grad(): + lines = pred_lines(img, self.model, + [img.shape[0], img.shape[1]], thr_v, thr_d) + for line in lines: + x_start, y_start, x_end, y_end = [int(val) for val in line] + cv2.line(img_output, (x_start, y_start), (x_end, y_end), + [255, 255, 255], 1) + except Exception: + pass + return img_output[:, :, 0] + + +class MidasDetector: + + def __init__(self, model_root_path, device='cuda'): + self.model = MiDaSInference( + model_type='dpt_hybrid', + model_root_path=model_root_path).to(device) + + def __call__(self, input_image, a=np.pi * 2.0, bg_th=0.1): + assert input_image.ndim == 3 + image_depth = input_image + with torch.no_grad(): + image_depth = torch.from_numpy(image_depth).float().cuda() + image_depth = image_depth / 127.5 - 1.0 + image_depth = rearrange(image_depth, 'h w c -> 1 c h w') + depth = self.model(image_depth)[0] + + depth_pt = depth.clone() + depth_pt -= torch.min(depth_pt) + depth_pt /= torch.max(depth_pt) + depth_pt = depth_pt.cpu().numpy() + depth_image = (depth_pt * 255.0).clip(0, 255).astype(np.uint8) + + depth_np = depth.cpu().numpy() + x = cv2.Sobel(depth_np, cv2.CV_32F, 1, 0, ksize=3) + y = cv2.Sobel(depth_np, cv2.CV_32F, 0, 1, ksize=3) + z = np.ones_like(x) * a + x[depth_pt < bg_th] = 0 + y[depth_pt < bg_th] = 0 + normal = np.stack([x, y, z], axis=2) + normal /= np.sum(normal**2.0, axis=2, keepdims=True)**0.5 + normal_image = (normal * 127.5 + 127.5).clip(0, + 255).astype(np.uint8) + + return depth_image, normal_image + + +class HEDNetwork(torch.nn.Module): + + def __init__(self, model_path): + super().__init__() + + self.netVggOne = torch.nn.Sequential( + torch.nn.Conv2d( + in_channels=3, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=64, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False)) + + self.netVggTwo = torch.nn.Sequential( + torch.nn.MaxPool2d(kernel_size=2, stride=2), + torch.nn.Conv2d( + in_channels=64, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=128, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False)) + + self.netVggThr = torch.nn.Sequential( + torch.nn.MaxPool2d(kernel_size=2, stride=2), + torch.nn.Conv2d( + in_channels=128, + out_channels=256, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=256, + out_channels=256, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=256, + out_channels=256, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False)) + + self.netVggFou = torch.nn.Sequential( + torch.nn.MaxPool2d(kernel_size=2, stride=2), + torch.nn.Conv2d( + in_channels=256, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=512, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=512, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False)) + + self.netVggFiv = torch.nn.Sequential( + torch.nn.MaxPool2d(kernel_size=2, stride=2), + torch.nn.Conv2d( + in_channels=512, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=512, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False), + torch.nn.Conv2d( + in_channels=512, + out_channels=512, + kernel_size=3, + stride=1, + padding=1), torch.nn.ReLU(inplace=False)) + + self.netScoreOne = torch.nn.Conv2d( + in_channels=64, out_channels=1, kernel_size=1, stride=1, padding=0) + self.netScoreTwo = torch.nn.Conv2d( + in_channels=128, + out_channels=1, + kernel_size=1, + stride=1, + padding=0) + self.netScoreThr = torch.nn.Conv2d( + in_channels=256, + out_channels=1, + kernel_size=1, + stride=1, + padding=0) + self.netScoreFou = torch.nn.Conv2d( + in_channels=512, + out_channels=1, + kernel_size=1, + stride=1, + padding=0) + self.netScoreFiv = torch.nn.Conv2d( + in_channels=512, + out_channels=1, + kernel_size=1, + stride=1, + padding=0) + + self.netCombine = torch.nn.Sequential( + torch.nn.Conv2d( + in_channels=5, + out_channels=1, + kernel_size=1, + stride=1, + padding=0), torch.nn.Sigmoid()) + + self.load_state_dict({ + strKey.replace('module', 'net'): tenWeight + for strKey, tenWeight in torch.load(model_path).items() + }) + + def forward(self, tenInput): + tenInput = tenInput * 255.0 + tenInput = tenInput - torch.tensor( + data=[104.00698793, 116.66876762, 122.67891434], + dtype=tenInput.dtype, + device=tenInput.device).view(1, 3, 1, 1) + + tenVggOne = self.netVggOne(tenInput) + tenVggTwo = self.netVggTwo(tenVggOne) + tenVggThr = self.netVggThr(tenVggTwo) + tenVggFou = self.netVggFou(tenVggThr) + tenVggFiv = self.netVggFiv(tenVggFou) + + tenScoreOne = self.netScoreOne(tenVggOne) + tenScoreTwo = self.netScoreTwo(tenVggTwo) + tenScoreThr = self.netScoreThr(tenVggThr) + tenScoreFou = self.netScoreFou(tenVggFou) + tenScoreFiv = self.netScoreFiv(tenVggFiv) + + tenScoreOne = torch.nn.functional.interpolate( + input=tenScoreOne, + size=(tenInput.shape[2], tenInput.shape[3]), + mode='bilinear', + align_corners=False) + tenScoreTwo = torch.nn.functional.interpolate( + input=tenScoreTwo, + size=(tenInput.shape[2], tenInput.shape[3]), + mode='bilinear', + align_corners=False) + tenScoreThr = torch.nn.functional.interpolate( + input=tenScoreThr, + size=(tenInput.shape[2], tenInput.shape[3]), + mode='bilinear', + align_corners=False) + tenScoreFou = torch.nn.functional.interpolate( + input=tenScoreFou, + size=(tenInput.shape[2], tenInput.shape[3]), + mode='bilinear', + align_corners=False) + tenScoreFiv = torch.nn.functional.interpolate( + input=tenScoreFiv, + size=(tenInput.shape[2], tenInput.shape[3]), + mode='bilinear', + align_corners=False) + + return self.netCombine( + torch.cat([ + tenScoreOne, tenScoreTwo, tenScoreThr, tenScoreFou, tenScoreFiv + ], 1)) + + +class CannyDetector: + + def __call__(self, img, low_threshold, high_threshold): + return cv2.Canny(img, low_threshold, high_threshold) + + +class HEDdetector: + + def __init__(self, annotator_ckpts_path, device='cuda'): + modelpath = os.path.join(annotator_ckpts_path, 'network-bsds500.pth') + self.netNetwork = HEDNetwork(modelpath).to(device).eval() + + def __call__(self, input_image): + assert input_image.ndim == 3 + input_image = input_image[:, :, ::-1].copy() + with torch.no_grad(): + image_hed = torch.from_numpy(input_image).float().cuda() + image_hed = image_hed / 255.0 + image_hed = rearrange(image_hed, 'h w c -> 1 c h w') + edge = self.netNetwork(image_hed)[0] + edge = (edge.cpu().numpy() * 255.0).clip(0, 255).astype(np.uint8) + return edge[0] + + +def nms(x, t, s): + x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s) + + f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8) + f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8) + f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8) + + y = np.zeros_like(x) + + for f in [f1, f2, f3, f4]: + np.putmask(y, cv2.dilate(x, kernel=f) == x, x) + + z = np.zeros_like(y, dtype=np.uint8) + z[y > t] = 255 + return z + + +def show_result_pyplot(model, + img, + result, + palette=None, + fig_size=(15, 10), + opacity=0.5, + title='', + block=True): + if hasattr(model, 'module'): + model = model.module + img = model.show_result( + img, result, palette=palette, show=False, opacity=opacity) + return mmcv.bgr2rgb(img) + + +class SegformerDetector: + + def __init__(self, annotator_ckpts_path, device='cuda'): + modelpath = os.path.join( + annotator_ckpts_path, + 'segformer_mit-b4_512x512_160k_ade20k_20220620_112216-4fa4f58f.pth' + ) + config_file = os.path.join( + annotator_ckpts_path.replace('ckpt/annotator/', ''), + 'config/config.py') + self.model = init_segmentor(config_file, modelpath).to(device) + + def __call__(self, img): + result = inference_segmentor(self.model, img) + res_img = show_result_pyplot( + self.model, img, result, get_palette('ade'), opacity=1) + + return res_img diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/api.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/api.py new file mode 100644 index 0000000000000000000000000000000000000000..cc8ca77332bf81ea81dcf29072ad27be24afb24b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/api.py @@ -0,0 +1,165 @@ +# based on https://github.com/isl-org/MiDaS + +import os + +import cv2 +import torch +import torch.nn as nn +from torchvision.transforms import Compose + +from .midas.dpt_depth import DPTDepthModel +from .midas.midas_net import MidasNet +from .midas.midas_net_custom import MidasNet_small +from .midas.transforms import NormalizeImage, PrepareForNet, Resize + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +def load_midas_transform(model_type): + # https://github.com/isl-org/MiDaS/blob/master/run.py + # load transform only + if model_type == 'dpt_large': # DPT-Large + net_w, net_h = 384, 384 + resize_mode = 'minimal' + normalization = NormalizeImage( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == 'dpt_hybrid': # DPT-Hybrid + net_w, net_h = 384, 384 + resize_mode = 'minimal' + normalization = NormalizeImage( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == 'midas_v21': + net_w, net_h = 384, 384 + resize_mode = 'upper_bound' + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + elif model_type == 'midas_v21_small': + net_w, net_h = 256, 256 + resize_mode = 'upper_bound' + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + else: + assert False, f"model_type '{model_type}' not implemented, use: --model_type large" + + transform = Compose([ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ]) + + return transform + + +def load_model(model_type, model_root_path): + # https://github.com/isl-org/MiDaS/blob/master/run.py + # load network + ISL_PATHS = { + 'dpt_large': os.path.join(model_root_path, + 'dpt_large-midas-2f21e586.pt'), + 'dpt_hybrid': os.path.join(model_root_path, + 'dpt_hybrid-midas-501f0c75.pt'), + 'midas_v21': '', + 'midas_v21_small': '', + } + model_path = ISL_PATHS[model_type] + if model_type == 'dpt_large': # DPT-Large + model = DPTDepthModel( + path=model_path, + backbone='vitl16_384', + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = 'minimal' + normalization = NormalizeImage( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == 'dpt_hybrid': # DPT-Hybrid + model = DPTDepthModel( + path=model_path, + backbone='vitb_rn50_384', + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = 'minimal' + normalization = NormalizeImage( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == 'midas_v21': + model = MidasNet(model_path, non_negative=True) + net_w, net_h = 384, 384 + resize_mode = 'upper_bound' + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + elif model_type == 'midas_v21_small': + model = MidasNet_small( + model_path, + features=64, + backbone='efficientnet_lite3', + exportable=True, + non_negative=True, + blocks={'expand': True}) + net_w, net_h = 256, 256 + resize_mode = 'upper_bound' + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + else: + print( + f"model_type '{model_type}' not implemented, use: --model_type large" + ) + assert False + + transform = Compose([ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ]) + + return model.eval(), transform + + +class MiDaSInference(nn.Module): + MODEL_TYPES_TORCH_HUB = ['DPT_Large', 'DPT_Hybrid', 'MiDaS_small'] + MODEL_TYPES_ISL = [ + 'dpt_large', + 'dpt_hybrid', + 'midas_v21', + 'midas_v21_small', + ] + + def __init__(self, model_type, model_root_path): + super().__init__() + assert (model_type in self.MODEL_TYPES_ISL) + model, _ = load_model(model_type, model_root_path) + self.model = model + self.model.train = disabled_train + + def forward(self, x): + with torch.no_grad(): + prediction = self.model(x) + return prediction diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/base_model.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5c76c352cc4938492d46b4aa536e9e45275a5e1d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/base_model.py @@ -0,0 +1,19 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import torch + + +class BaseModel(torch.nn.Module): + + def load(self, path): + """Load model from file. + + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device('cpu')) + + if 'optimizer' in parameters: + parameters = parameters['model'] + + self.load_state_dict(parameters) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/blocks.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..6a3fc2f0d4aca1faabbbe986d5a740b3c8653173 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/blocks.py @@ -0,0 +1,394 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import torch +import torch.nn as nn + +from .vit import (_make_pretrained_vitb16_384, _make_pretrained_vitb_rn50_384, + _make_pretrained_vitl16_384, forward_vit) + + +def _make_encoder( + backbone, + features, + use_pretrained, + groups=1, + expand=False, + exportable=True, + hooks=None, + use_vit_only=False, + use_readout='ignore', +): + if backbone == 'vitl16_384': + pretrained = _make_pretrained_vitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, + expand=expand) # ViT-L/16 - 85.0% Top1 (backbone) + elif backbone == 'vitb_rn50_384': + pretrained = _make_pretrained_vitb_rn50_384( + use_pretrained, + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) + scratch = _make_scratch( + [256, 512, 768, 768], features, groups=groups, + expand=expand) # ViT-H/16 - 85.0% Top1 (backbone) + elif backbone == 'vitb16_384': + pretrained = _make_pretrained_vitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, + expand=expand) # ViT-B/16 - 84.6% Top1 (backbone) + elif backbone == 'resnext101_wsl': + pretrained = _make_pretrained_resnext101_wsl(use_pretrained) + scratch = _make_scratch([256, 512, 1024, 2048], + features, + groups=groups, + expand=expand) # efficientnet_lite3 + elif backbone == 'efficientnet_lite3': + pretrained = _make_pretrained_efficientnet_lite3( + use_pretrained, exportable=exportable) + scratch = _make_scratch([32, 48, 136, 384], + features, + groups=groups, + expand=expand) # efficientnet_lite3 + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand is True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups) + + return scratch + + +def _make_pretrained_efficientnet_lite3(use_pretrained, exportable=False): + efficientnet = torch.hub.load( + 'rwightman/gen-efficientnet-pytorch', + 'tf_efficientnet_lite3', + pretrained=use_pretrained, + exportable=exportable) + return _make_efficientnet_backbone(efficientnet) + + +def _make_efficientnet_backbone(effnet): + pretrained = nn.Module() + + pretrained.layer1 = nn.Sequential(effnet.conv_stem, effnet.bn1, + effnet.act1, *effnet.blocks[0:2]) + pretrained.layer2 = nn.Sequential(*effnet.blocks[2:3]) + pretrained.layer3 = nn.Sequential(*effnet.blocks[3:5]) + pretrained.layer4 = nn.Sequential(*effnet.blocks[5:9]) + + return pretrained + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu, + resnet.maxpool, resnet.layer1) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + + +def _make_pretrained_resnext101_wsl(use_pretrained): + resnet = torch.hub.load('facebookresearch/WSL-Images', + 'resnext101_32x8d_wsl') + return _make_resnet_backbone(resnet) + + +class Interpolate(nn.Module): + """Interpolation module. + """ + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode='bilinear', align_corners=True) + + return output + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=True, + groups=self.groups) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=True, + groups=self.groups) + + if self.bn is True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn is True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn is True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block. + """ + + def __init__(self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand is True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners) + + output = self.out_conv(output) + + return output diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/dpt_depth.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/dpt_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..f3b795978c9eeefac70e4b67b7ce536ebe4dde0c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/dpt_depth.py @@ -0,0 +1,107 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .base_model import BaseModel +from .blocks import (FeatureFusionBlock, FeatureFusionBlock_custom, + Interpolate, _make_encoder, forward_vit) + + +def _make_fusion_block(features, use_bn): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + ) + + +class DPT(BaseModel): + + def __init__( + self, + head, + features=256, + backbone='vitb_rn50_384', + readout='project', + channels_last=False, + use_bn=False, + ): + + super(DPT, self).__init__() + + self.channels_last = channels_last + + hooks = { + 'vitb_rn50_384': [0, 1, 8, 11], + 'vitb16_384': [2, 5, 8, 11], + 'vitl16_384': [5, 11, 17, 23], + } + + # Instantiate backbone and reassemble blocks + self.pretrained, self.scratch = _make_encoder( + backbone, + features, + False, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + hooks=hooks[backbone], + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.scratch.output_conv = head + + def forward(self, x): + if self.channels_last is True: + x.contiguous(memory_format=torch.channels_last) + + layer_1, layer_2, layer_3, layer_4 = forward_vit(self.pretrained, x) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return out + + +class DPTDepthModel(DPT): + + def __init__(self, path=None, non_negative=True, **kwargs): + features = kwargs['features'] if 'features' in kwargs else 256 + + head = nn.Sequential( + nn.Conv2d( + features, features // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode='bilinear', align_corners=True), + nn.Conv2d(features // 2, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + def forward(self, x): + return super().forward(x).squeeze(dim=1) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net.py new file mode 100644 index 0000000000000000000000000000000000000000..374ea568bdf6554ba9ef5e1555f0189fe5c5d77a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net.py @@ -0,0 +1,79 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, Interpolate, _make_encoder + + +class MidasNet(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=256, non_negative=True): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print('Loading weights: ', path) + + super(MidasNet, self).__init__() + + use_pretrained = False if path is None else True + + self.pretrained, self.scratch = _make_encoder( + backbone='resnext101_wsl', + features=features, + use_pretrained=use_pretrained) + + self.scratch.refinenet4 = FeatureFusionBlock(features) + self.scratch.refinenet3 = FeatureFusionBlock(features) + self.scratch.refinenet2 = FeatureFusionBlock(features) + self.scratch.refinenet1 = FeatureFusionBlock(features) + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, 128, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode='bilinear'), + nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + ) + + if path: + self.load(path) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net_custom.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net_custom.py new file mode 100644 index 0000000000000000000000000000000000000000..56ff7a3625fe1f94a9d8d595e08cd759e3e59c4a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/midas_net_custom.py @@ -0,0 +1,169 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import (FeatureFusionBlock, FeatureFusionBlock_custom, + Interpolate, _make_encoder) + + +class MidasNet_small(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, + path=None, + features=64, + backbone='efficientnet_lite3', + non_negative=True, + exportable=True, + channels_last=False, + align_corners=True, + blocks={'expand': True}): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print('Loading weights: ', path) + + super(MidasNet_small, self).__init__() + + use_pretrained = False if path else True + + self.channels_last = channels_last + self.blocks = blocks + self.backbone = backbone + + self.groups = 1 + + features1 = features + features2 = features + features3 = features + features4 = features + self.expand = False + if 'expand' in self.blocks and self.blocks['expand'] is True: + self.expand = True + features1 = features + features2 = features * 2 + features3 = features * 4 + features4 = features * 8 + + self.pretrained, self.scratch = _make_encoder( + self.backbone, + features, + use_pretrained, + groups=self.groups, + expand=self.expand, + exportable=exportable) + + self.scratch.activation = nn.ReLU(False) + + self.scratch.refinenet4 = FeatureFusionBlock_custom( + features4, + self.scratch.activation, + deconv=False, + bn=False, + expand=self.expand, + align_corners=align_corners) + self.scratch.refinenet3 = FeatureFusionBlock_custom( + features3, + self.scratch.activation, + deconv=False, + bn=False, + expand=self.expand, + align_corners=align_corners) + self.scratch.refinenet2 = FeatureFusionBlock_custom( + features2, + self.scratch.activation, + deconv=False, + bn=False, + expand=self.expand, + align_corners=align_corners) + self.scratch.refinenet1 = FeatureFusionBlock_custom( + features1, + self.scratch.activation, + deconv=False, + bn=False, + align_corners=align_corners) + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d( + features, + features // 2, + kernel_size=3, + stride=1, + padding=1, + groups=self.groups), + Interpolate(scale_factor=2, mode='bilinear'), + nn.Conv2d(features // 2, 32, kernel_size=3, stride=1, padding=1), + self.scratch.activation, + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + if path: + self.load(path) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + if self.channels_last is True: + print('self.channels_last = ', self.channels_last) + x.contiguous(memory_format=torch.channels_last) + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) + + +def fuse_model(m): + prev_previous_type = nn.Identity() + prev_previous_name = '' + previous_type = nn.Identity() + previous_name = '' + for name, module in m.named_modules(): + if prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d and type( + module) == nn.ReLU: + # print("FUSED ", prev_previous_name, previous_name, name) + torch.quantization.fuse_modules( + m, [prev_previous_name, previous_name, name], inplace=True) + elif prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d: + # print("FUSED ", prev_previous_name, previous_name) + torch.quantization.fuse_modules( + m, [prev_previous_name, previous_name], inplace=True) + # elif previous_type == nn.Conv2d and type(module) == nn.ReLU: + # print("FUSED ", previous_name, name) + # torch.quantization.fuse_modules(m, [previous_name, name], inplace=True) + + prev_previous_type = previous_type + prev_previous_name = previous_name + previous_type = type(module) + previous_name = name diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/transforms.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..078cc2ec8927a9d2bf9cdf913a11b2a003a844e4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/transforms.py @@ -0,0 +1,237 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import math + +import cv2 +import numpy as np + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample['disparity'].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample['image'] = cv2.resize( + sample['image'], + tuple(shape[::-1]), + interpolation=image_interpolation_method) + + sample['disparity'] = cv2.resize( + sample['disparity'], + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST) + sample['mask'] = cv2.resize( + sample['mask'].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample['mask'] = sample['mask'].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method='lower_bound', + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. + (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == 'lower_bound': + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == 'upper_bound': + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == 'minimal': + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f'resize_method {self.__resize_method} not implemented') + + if self.__resize_method == 'lower_bound': + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width) + elif self.__resize_method == 'upper_bound': + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width) + elif self.__resize_method == 'minimal': + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError( + f'resize_method {self.__resize_method} not implemented') + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size(sample['image'].shape[1], + sample['image'].shape[0]) + + # resize sample + sample['image'] = cv2.resize( + sample['image'], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if 'disparity' in sample: + sample['disparity'] = cv2.resize( + sample['disparity'], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if 'depth' in sample: + sample['depth'] = cv2.resize( + sample['depth'], (width, height), + interpolation=cv2.INTER_NEAREST) + + sample['mask'] = cv2.resize( + sample['mask'].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample['mask'] = sample['mask'].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample['image'] = (sample['image'] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample['image'], (2, 0, 1)) + sample['image'] = np.ascontiguousarray(image).astype(np.float32) + + if 'mask' in sample: + sample['mask'] = sample['mask'].astype(np.float32) + sample['mask'] = np.ascontiguousarray(sample['mask']) + + if 'disparity' in sample: + disparity = sample['disparity'].astype(np.float32) + sample['disparity'] = np.ascontiguousarray(disparity) + + if 'depth' in sample: + depth = sample['depth'].astype(np.float32) + sample['depth'] = np.ascontiguousarray(depth) + + return sample diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/vit.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c851b3f84e07ae9cca4585230204cf2a733d47 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/midas/vit.py @@ -0,0 +1,516 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import math +import types + +import timm +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Slice(nn.Module): + + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index:] + + +class AddReadout(nn.Module): + + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index:] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential( + nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index:]) + features = torch.cat((x[:, self.start_index:], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +def forward_vit(pretrained, x): + b, c, h, w = x.shape + + _ = pretrained.model.forward_flex(x) + + layer_1 = pretrained.activations['1'] + layer_2 = pretrained.activations['2'] + layer_3 = pretrained.activations['3'] + layer_4 = pretrained.activations['4'] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size([ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ]), + )) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3:len(pretrained.act_postprocess1)]( + layer_1) + layer_2 = pretrained.act_postprocess2[3:len(pretrained.act_postprocess2)]( + layer_2) + layer_3 = pretrained.act_postprocess3[3:len(pretrained.act_postprocess3)]( + layer_3) + layer_4 = pretrained.act_postprocess4[3:len(pretrained.act_postprocess4)]( + layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, :self.start_index], + posemb[0, self.start_index:], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, + -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate( + posemb_grid, size=(gs_h, gs_w), mode='bilinear') + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed(self.pos_embed, h // self.patch_size[1], + w // self.patch_size[0]) + + B = x.shape[0] + + if hasattr(self.patch_embed, 'backbone'): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[ + -1] # last feature if backbone outputs list/tuple of features + + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, 'dist_token', None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +activations = {} + + +def get_activation(name): + + def hook(model, input, output): + activations[name] = output + + return hook + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == 'ignore': + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == 'add': + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == 'project': + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout='ignore', + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook( + get_activation('1')) + pretrained.model.blocks[hooks[1]].register_forward_hook( + get_activation('2')) + pretrained.model.blocks[hooks[2]].register_forward_hook( + get_activation('3')) + pretrained.model.blocks[hooks[3]].register_forward_hook( + get_activation('4')) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, + start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, + pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model) + + return pretrained + + +def _make_pretrained_vitl16_384(pretrained, use_readout='ignore', hooks=None): + model = timm.create_model('vit_large_patch16_384', pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks is None else hooks + return _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_vitb16_384(pretrained, use_readout='ignore', hooks=None): + model = timm.create_model('vit_base_patch16_384', pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks is None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout) + + +def _make_pretrained_deitb16_384(pretrained, use_readout='ignore', hooks=None): + model = timm.create_model( + 'vit_deit_base_patch16_384', pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks is None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout) + + +def _make_pretrained_deitb16_distil_384(pretrained, + use_readout='ignore', + hooks=None): + model = timm.create_model( + 'vit_deit_base_distilled_patch16_384', pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks is None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + start_index=2, + ) + + +def _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=[0, 1, 8, 11], + vit_features=768, + use_vit_only=False, + use_readout='ignore', + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + + if use_vit_only is True: + pretrained.model.blocks[hooks[0]].register_forward_hook( + get_activation('1')) + pretrained.model.blocks[hooks[1]].register_forward_hook( + get_activation('2')) + else: + pretrained.model.patch_embed.backbone.stages[0].register_forward_hook( + get_activation('1')) + pretrained.model.patch_embed.backbone.stages[1].register_forward_hook( + get_activation('2')) + + pretrained.model.blocks[hooks[2]].register_forward_hook( + get_activation('3')) + pretrained.model.blocks[hooks[3]].register_forward_hook( + get_activation('4')) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, + start_index) + + if use_vit_only is True: + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + else: + pretrained.act_postprocess1 = nn.Sequential(nn.Identity(), + nn.Identity(), + nn.Identity()) + pretrained.act_postprocess2 = nn.Sequential(nn.Identity(), + nn.Identity(), + nn.Identity()) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, + pretrained.model) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model) + + return pretrained + + +def _make_pretrained_vitb_rn50_384(pretrained, + use_readout='ignore', + hooks=None, + use_vit_only=False): + model = timm.create_model('vit_base_resnet50_384', pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks is None else hooks + return _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/utils.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a3edf820acf18c9334dbe29107bc7211c9788c69 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/midas/utils.py @@ -0,0 +1,194 @@ +"""Utils for monoDepth.""" +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import re +import sys + +import cv2 +import numpy as np +import torch + + +def read_pfm(path): + """Read pfm file. + + Args: + path (str): path to file + + Returns: + tuple: (data, scale) + """ + with open(path, 'rb') as file: + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode('ascii') == 'PF': + color = True + elif header.decode('ascii') == 'Pf': + color = False + else: + raise Exception('Not a PFM file: ' + path) + + dim_match = re.match(r'^(\d+)\s(\d+)\s$', + file.readline().decode('ascii')) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception('Malformed PFM header.') + + scale = float(file.readline().decode('ascii').rstrip()) + if scale < 0: + # little-endian + endian = '<' + scale = -scale + else: + # big-endian + endian = '>' + + data = np.fromfile(file, endian + 'f') + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + + return data, scale + + +def write_pfm(path, image, scale=1): + """Write pfm file. + + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, 'wb') as file: + color = None + + if image.dtype.name != 'float32': + raise Exception('Image dtype must be float32.') + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif (len(image.shape) == 2 + or len(image.shape) == 3 and image.shape[2] == 1): # greyscale + color = False + else: + raise Exception( + 'Image must have H x W x 3, H x W x 1 or H x W dimensions.') + + file.write('PF\n' if color else 'Pf\n'.encode()) + file.write('%d %d\n'.encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == '<' or endian == '=' and sys.byteorder == 'little': + scale = -scale + + file.write('%f\n'.encode() % scale) + + image.tofile(file) + + +def read_image(path): + """Read image and output RGB image (0-1). + + Args: + path (str): path to file + + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + + +def resize_image(img): + """Resize image and make it fit for network. + + Args: + img (array): image + + Returns: + tensor: data ready for network + """ + height_orig = img.shape[0] + width_orig = img.shape[1] + + if width_orig > height_orig: + scale = width_orig / 384 + else: + scale = height_orig / 384 + + height = (np.ceil(height_orig / scale / 32) * 32).astype(int) + width = (np.ceil(width_orig / scale / 32) * 32).astype(int) + + img_resized = cv2.resize( + img, (width, height), interpolation=cv2.INTER_AREA) + + img_resized = ( + torch.from_numpy(np.transpose(img_resized, + (2, 0, 1))).contiguous().float()) + img_resized = img_resized.unsqueeze(0) + + return img_resized + + +def resize_depth(depth, width, height): + """Resize depth map and bring to CPU (numpy). + + Args: + depth (tensor): depth + width (int): image width + height (int): image height + + Returns: + array: processed depth + """ + depth = torch.squeeze(depth[0, :, :, :]).to('cpu') + + depth_resized = cv2.resize( + depth.numpy(), (width, height), interpolation=cv2.INTER_CUBIC) + + return depth_resized + + +def write_depth(path, depth, bits=1): + """Write depth map to pfm and png file. + + Args: + path (str): filepath without extension + depth (array): depth + """ + write_pfm(path + '.pfm', depth.astype(np.float32)) + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8 * bits)) - 1 + + if depth_max - depth_min > np.finfo('float').eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = np.zeros(depth.shape, dtype=depth.type) + + if bits == 1: + cv2.imwrite(path + '.png', out.astype('uint8')) + elif bits == 2: + cv2.imwrite(path + '.png', out.astype('uint16')) + + return diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/mbv2_mlsd_large.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/mbv2_mlsd_large.py new file mode 100644 index 0000000000000000000000000000000000000000..0b5ba451e69b44374e567c9054a064df8e3b8b90 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/mbv2_mlsd_large.py @@ -0,0 +1,305 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import os +import sys + +import torch +import torch.nn as nn +import torch.utils.model_zoo as model_zoo +from torch.nn import functional as F + + +class BlockTypeA(nn.Module): + + def __init__(self, in_c1, in_c2, out_c1, out_c2, upscale=True): + super(BlockTypeA, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c2, out_c2, kernel_size=1), nn.BatchNorm2d(out_c2), + nn.ReLU(inplace=True)) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c1, out_c1, kernel_size=1), nn.BatchNorm2d(out_c1), + nn.ReLU(inplace=True)) + self.upscale = upscale + + def forward(self, a, b): + b = self.conv1(b) + a = self.conv2(a) + if self.upscale: + b = F.interpolate( + b, scale_factor=2.0, mode='bilinear', align_corners=True) + return torch.cat((a, b), dim=1) + + +class BlockTypeB(nn.Module): + + def __init__(self, in_c, out_c): + super(BlockTypeB, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), nn.ReLU()) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, out_c, kernel_size=3, padding=1), + nn.BatchNorm2d(out_c), nn.ReLU()) + + def forward(self, x): + x = self.conv1(x) + x + x = self.conv2(x) + return x + + +class BlockTypeC(nn.Module): + + def __init__(self, in_c, out_c): + super(BlockTypeC, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=5, dilation=5), + nn.BatchNorm2d(in_c), nn.ReLU()) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), nn.ReLU()) + self.conv3 = nn.Conv2d(in_c, out_c, kernel_size=1) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + return x + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + + def __init__(self, + in_planes, + out_planes, + kernel_size=3, + stride=1, + groups=1): + self.channel_pad = out_planes - in_planes + self.stride = stride + # padding = (kernel_size - 1) // 2 + + # TFLite uses slightly different padding than PyTorch + if stride == 2: + padding = 0 + else: + padding = (kernel_size - 1) // 2 + + super(ConvBNReLU, self).__init__( + nn.Conv2d( + in_planes, + out_planes, + kernel_size, + stride, + padding, + groups=groups, + bias=False), nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True)) + self.max_pool = nn.MaxPool2d(kernel_size=stride, stride=stride) + + def forward(self, x): + # TFLite uses different padding + if self.stride == 2: + x = F.pad(x, (0, 1, 0, 1), 'constant', 0) + # print(x.shape) + + for module in self: + if not isinstance(module, nn.MaxPool2d): + x = module(x) + return x + + +class InvertedResidual(nn.Module): + + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU( + hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + + def __init__(self, pretrained=True): + """ + MobileNet V2 main class + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + """ + super(MobileNetV2, self).__init__() + + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + width_mult = 1.0 + round_nearest = 8 + + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + # [6, 160, 3, 2], + # [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len( + inverted_residual_setting[0]) != 4: + raise ValueError('inverted_residual_setting should be non-empty ' + 'or a 4-element list, got {}'.format( + inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, + round_nearest) + self.last_channel = _make_divisible( + last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(4, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append( + block( + input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + + self.features = nn.Sequential(*features) + self.fpn_selected = [1, 3, 6, 10, 13] + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + if pretrained: + self._load_pretrained_model() + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + fpn_features = [] + for i, f in enumerate(self.features): + if i > self.fpn_selected[-1]: + break + x = f(x) + if i in self.fpn_selected: + fpn_features.append(x) + + c1, c2, c3, c4, c5 = fpn_features + return c1, c2, c3, c4, c5 + + def forward(self, x): + return self._forward_impl(x) + + def _load_pretrained_model(self): + pretrain_dict = model_zoo.load_url( + 'https://download.pytorch.org/models/mobilenet_v2-b0353104.pth') + model_dict = {} + state_dict = self.state_dict() + for k, v in pretrain_dict.items(): + if k in state_dict: + model_dict[k] = v + state_dict.update(model_dict) + self.load_state_dict(state_dict) + + +class MobileV2_MLSD_Large(nn.Module): + + def __init__(self): + super(MobileV2_MLSD_Large, self).__init__() + + self.backbone = MobileNetV2(pretrained=False) + # A, B + self.block15 = BlockTypeA( + in_c1=64, in_c2=96, out_c1=64, out_c2=64, upscale=False) + self.block16 = BlockTypeB(128, 64) + + # A, B + self.block17 = BlockTypeA(in_c1=32, in_c2=64, out_c1=64, out_c2=64) + self.block18 = BlockTypeB(128, 64) + + # A, B + self.block19 = BlockTypeA(in_c1=24, in_c2=64, out_c1=64, out_c2=64) + self.block20 = BlockTypeB(128, 64) + + # A, B, C + self.block21 = BlockTypeA(in_c1=16, in_c2=64, out_c1=64, out_c2=64) + self.block22 = BlockTypeB(128, 64) + + self.block23 = BlockTypeC(64, 16) + + def forward(self, x): + c1, c2, c3, c4, c5 = self.backbone(x) + + x = self.block15(c4, c5) + x = self.block16(x) + + x = self.block17(c3, x) + x = self.block18(x) + + x = self.block19(c2, x) + x = self.block20(x) + + x = self.block21(c1, x) + x = self.block22(x) + x = self.block23(x) + x = x[:, 7:, :, :] + + return x diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/utils.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d348d1542944194d26297a63704968e3f43fbc60 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/mlsd/utils.py @@ -0,0 +1,643 @@ +''' +modified by lihaoweicv +pytorch version +M-LSD +Copyright 2021-present NAVER Corp. +Apache License v2.0 +''' +import os + +import cv2 +import numpy as np +import torch +from torch.nn import functional as F + + +def deccode_output_score_and_ptss(tpMap, topk_n=200, ksize=5): + ''' + tpMap: + center: tpMap[1, 0, :, :] + displacement: tpMap[1, 1:5, :, :] + ''' + b, c, h, w = tpMap.shape + assert b == 1, 'only support bsize==1' + displacement = tpMap[:, 1:5, :, :][0] + center = tpMap[:, 0, :, :] + heat = torch.sigmoid(center) + hmax = F.max_pool2d( + heat, (ksize, ksize), stride=1, padding=(ksize - 1) // 2) + keep = (hmax == heat).float() + heat = heat * keep + heat = heat.reshape(-1, ) + + scores, indices = torch.topk(heat, topk_n, dim=-1, largest=True) + yy = torch.floor_divide(indices, w).unsqueeze(-1) + xx = torch.fmod(indices, w).unsqueeze(-1) + ptss = torch.cat((yy, xx), dim=-1) + + ptss = ptss.detach().cpu().numpy() + scores = scores.detach().cpu().numpy() + displacement = displacement.detach().cpu().numpy() + displacement = displacement.transpose((1, 2, 0)) + return ptss, scores, displacement + + +def pred_lines(image, + model, + input_shape=[512, 512], + score_thr=0.10, + dist_thr=20.0): + h, w, _ = image.shape + h_ratio, w_ratio = [h / input_shape[0], w / input_shape[1]] + temp_img_02 = cv2.resize( + image, (input_shape[1], input_shape[0]), interpolation=cv2.INTER_AREA) + resized_image = np.concatenate( + [temp_img_02, + np.ones([input_shape[0], input_shape[1], 1])], axis=-1) + + resized_image = resized_image.transpose((2, 0, 1)) + batch_image = np.expand_dims(resized_image, axis=0).astype('float32') + batch_image = (batch_image / 127.5) - 1.0 + + batch_image = torch.from_numpy(batch_image).float().cuda() + outputs = model(batch_image) + pts, pts_score, vmap = deccode_output_score_and_ptss(outputs, 200, 3) + start = vmap[:, :, :2] + end = vmap[:, :, 2:] + dist_map = np.sqrt(np.sum((start - end)**2, axis=-1)) + + segments_list = [] + for center, score in zip(pts, pts_score): + y, x = center + distance = dist_map[y, x] + if score > score_thr and distance > dist_thr: + disp_x_start, disp_y_start, disp_x_end, disp_y_end = vmap[y, x, :] + x_start = x + disp_x_start + y_start = y + disp_y_start + x_end = x + disp_x_end + y_end = y + disp_y_end + segments_list.append([x_start, y_start, x_end, y_end]) + + lines = 2 * np.array(segments_list) # 256 > 512 + lines[:, 0] = lines[:, 0] * w_ratio + lines[:, 1] = lines[:, 1] * h_ratio + lines[:, 2] = lines[:, 2] * w_ratio + lines[:, 3] = lines[:, 3] * h_ratio + + return lines + + +params_glob = { + 'score': 0.06, + 'outside_ratio': 0.28, + 'inside_ratio': 0.45, + 'w_overlap': 0.0, + 'w_degree': 1.95, + 'w_length': 0.0, + 'w_area': 1.86, + 'w_center': 0.14 +} + + +def pred_squares(image, model, input_shape=[512, 512], params=params_glob): + ''' + shape = [height, width] + ''' + h, w, _ = image.shape + original_shape = [h, w] + temp_img01 = cv2.resize( + image, (input_shape[0], input_shape[1]), interpolation=cv2.INTER_AREA) + resized_image = np.concatenate( + [temp_img01, np.ones([input_shape[0], input_shape[1], 1])], axis=-1) + resized_image = resized_image.transpose((2, 0, 1)) + batch_image = np.expand_dims(resized_image, axis=0).astype('float32') + batch_image = (batch_image / 127.5) - 1.0 + + batch_image = torch.from_numpy(batch_image).float().cuda() + outputs = model(batch_image) + + pts, pts_score, vmap = deccode_output_score_and_ptss(outputs, 200, 3) + start = vmap[:, :, :2] # (x, y) + end = vmap[:, :, 2:] # (x, y) + dist_map = np.sqrt(np.sum((start - end)**2, axis=-1)) + + junc_list = [] + segments_list = [] + for junc, score in zip(pts, pts_score): + y, x = junc + distance = dist_map[y, x] + if score > params['score'] and distance > 20.0: + junc_list.append([x, y]) + disp_x_start, disp_y_start, disp_x_end, disp_y_end = vmap[y, x, :] + d_arrow = 1.0 + x_start = x + d_arrow * disp_x_start + y_start = y + d_arrow * disp_y_start + x_end = x + d_arrow * disp_x_end + y_end = y + d_arrow * disp_y_end + segments_list.append([x_start, y_start, x_end, y_end]) + + segments = np.array(segments_list) + + # post processing for squares + # 1. get unique lines + point = np.array([[0, 0]]) + point = point[0] + start = segments[:, :2] + end = segments[:, 2:] + diff = start - end + a = diff[:, 1] + b = -diff[:, 0] + c = a * start[:, 0] + b * start[:, 1] + + d = np.abs(a * point[0] + b * point[1] - c) / np.sqrt(a**2 + b**2 + 1e-10) + theta = np.arctan2(diff[:, 0], diff[:, 1]) * 180 / np.pi + theta[theta < 0.0] += 180 + hough = np.concatenate([d[:, None], theta[:, None]], axis=-1) + + d_quant = 1 + theta_quant = 2 + hough[:, 0] //= d_quant + hough[:, 1] //= theta_quant + _, indices, counts = np.unique( + hough, axis=0, return_index=True, return_counts=True) + + acc_map = np.zeros([512 // d_quant + 1, 360 // theta_quant + 1], + dtype='float32') + idx_map = np.zeros([512 // d_quant + 1, 360 // theta_quant + 1], + dtype='int32') - 1 + yx_indices = hough[indices, :].astype('int32') + acc_map[yx_indices[:, 0], yx_indices[:, 1]] = counts + idx_map[yx_indices[:, 0], yx_indices[:, 1]] = indices + + acc_map_np = acc_map + # acc_map = acc_map[None, :, :, None] + # + # ### fast suppression using tensorflow op + # acc_map = tf.constant(acc_map, dtype=tf.float32) + # max_acc_map = tf.keras.layers.MaxPool2D(pool_size=(5, 5), strides=1, padding='same')(acc_map) + # acc_map = acc_map * tf.cast(tf.math.equal(acc_map, max_acc_map), tf.float32) + # flatten_acc_map = tf.reshape(acc_map, [1, -1]) + # topk_values, topk_indices = tf.math.top_k(flatten_acc_map, k=len(pts)) + # _, h, w, _ = acc_map.shape + # y = tf.expand_dims(topk_indices // w, axis=-1) + # x = tf.expand_dims(topk_indices % w, axis=-1) + # yx = tf.concat([y, x], axis=-1) + + # fast suppression using pytorch op + acc_map = torch.from_numpy(acc_map_np).unsqueeze(0).unsqueeze(0) + _, _, h, w = acc_map.shape + max_acc_map = F.max_pool2d(acc_map, kernel_size=5, stride=1, padding=2) + acc_map = acc_map * ((acc_map == max_acc_map).float()) + flatten_acc_map = acc_map.reshape([ + -1, + ]) + + scores, indices = torch.topk( + flatten_acc_map, len(pts), dim=-1, largest=True) + yy = torch.div(indices, w, rounding_mode='floor').unsqueeze(-1) + xx = torch.fmod(indices, w).unsqueeze(-1) + yx = torch.cat((yy, xx), dim=-1) + + yx = yx.detach().cpu().numpy() + + topk_values = scores.detach().cpu().numpy() + indices = idx_map[yx[:, 0], yx[:, 1]] + basis = 5 // 2 + + merged_segments = [] + for yx_pt, max_indice, value in zip(yx, indices, topk_values): + y, x = yx_pt + if max_indice == -1 or value == 0: + continue + segment_list = [] + for y_offset in range(-basis, basis + 1): + for x_offset in range(-basis, basis + 1): + indice = idx_map[y + y_offset, x + x_offset] + cnt = int(acc_map_np[y + y_offset, x + x_offset]) + if indice != -1: + segment_list.append(segments[indice]) + if cnt > 1: + check_cnt = 1 + current_hough = hough[indice] + for new_indice, new_hough in enumerate(hough): + if (current_hough + == new_hough).all() and indice != new_indice: + segment_list.append(segments[new_indice]) + check_cnt += 1 + if check_cnt == cnt: + break + group_segments = np.array(segment_list).reshape([-1, 2]) + sorted_group_segments = np.sort(group_segments, axis=0) + x_min, y_min = sorted_group_segments[0, :] + x_max, y_max = sorted_group_segments[-1, :] + + deg = theta[max_indice] + if deg >= 90: + merged_segments.append([x_min, y_max, x_max, y_min]) + else: + merged_segments.append([x_min, y_min, x_max, y_max]) + + # 2. get intersections + new_segments = np.array(merged_segments) # (x1, y1, x2, y2) + start = new_segments[:, :2] # (x1, y1) + end = new_segments[:, 2:] # (x2, y2) + new_centers = (start + end) / 2.0 + diff = start - end + dist_segments = np.sqrt(np.sum(diff**2, axis=-1)) + + # ax + by = c + a = diff[:, 1] + b = -diff[:, 0] + c = a * start[:, 0] + b * start[:, 1] + pre_det = a[:, None] * b[None, :] + det = pre_det - np.transpose(pre_det) + + pre_inter_y = a[:, None] * c[None, :] + inter_y = (pre_inter_y - np.transpose(pre_inter_y)) / (det + 1e-10) + pre_inter_x = c[:, None] * b[None, :] + inter_x = (pre_inter_x - np.transpose(pre_inter_x)) / (det + 1e-10) + inter_pts = np.concatenate([inter_x[:, :, None], inter_y[:, :, None]], + axis=-1).astype('int32') + + # 3. get corner information + # 3.1 get distance + ''' + dist_segments: + | dist(0), dist(1), dist(2), ...| + dist_inter_to_segment1: + | dist(inter,0), dist(inter,0), dist(inter,0), ... | + | dist(inter,1), dist(inter,1), dist(inter,1), ... | + ... + dist_inter_to_semgnet2: + | dist(inter,0), dist(inter,1), dist(inter,2), ... | + | dist(inter,0), dist(inter,1), dist(inter,2), ... | + ... + ''' + + dist_inter_to_segment1_start = np.sqrt( + np.sum(((inter_pts - start[:, None, :])**2), axis=-1, + keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment1_end = np.sqrt( + np.sum(((inter_pts - end[:, None, :])**2), axis=-1, + keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment2_start = np.sqrt( + np.sum(((inter_pts - start[None, :, :])**2), axis=-1, + keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment2_end = np.sqrt( + np.sum(((inter_pts - end[None, :, :])**2), axis=-1, + keepdims=True)) # [n_batch, n_batch, 1] + + # sort ascending + dist_inter_to_segment1 = np.sort( + np.concatenate( + [dist_inter_to_segment1_start, dist_inter_to_segment1_end], + axis=-1), + axis=-1) # [n_batch, n_batch, 2] + dist_inter_to_segment2 = np.sort( + np.concatenate( + [dist_inter_to_segment2_start, dist_inter_to_segment2_end], + axis=-1), + axis=-1) # [n_batch, n_batch, 2] + + # 3.2 get degree + inter_to_start = new_centers[:, None, :] - inter_pts + deg_inter_to_start = np.arctan2(inter_to_start[:, :, 1], + inter_to_start[:, :, 0]) * 180 / np.pi + deg_inter_to_start[deg_inter_to_start < 0.0] += 360 + inter_to_end = new_centers[None, :, :] - inter_pts + deg_inter_to_end = np.arctan2(inter_to_end[:, :, 1], + inter_to_end[:, :, 0]) * 180 / np.pi + deg_inter_to_end[deg_inter_to_end < 0.0] += 360 + ''' + B -- G + | | + C -- R + B : blue / G: green / C: cyan / R: red + + 0 -- 1 + | | + 3 -- 2 + ''' + # rename variables + deg1_map, deg2_map = deg_inter_to_start, deg_inter_to_end + # sort deg ascending + deg_sort = np.sort( + np.concatenate([deg1_map[:, :, None], deg2_map[:, :, None]], axis=-1), + axis=-1) + + deg_diff_map = np.abs(deg1_map - deg2_map) + # we only consider the smallest degree of intersect + deg_diff_map[deg_diff_map > 180] = 360 - deg_diff_map[deg_diff_map > 180] + + # define available degree range + deg_range = [60, 120] + + corner_dict = {corner_info: [] for corner_info in range(4)} + inter_points = [] + for i in range(inter_pts.shape[0]): + for j in range(i + 1, inter_pts.shape[1]): + # i, j > line index, always i < j + x, y = inter_pts[i, j, :] + deg1, deg2 = deg_sort[i, j, :] + deg_diff = deg_diff_map[i, j] + + check_degree = deg_diff > deg_range[0] and deg_diff < deg_range[1] + + outside_ratio = params['outside_ratio'] # over ratio >>> drop it! + inside_ratio = params['inside_ratio'] # over ratio >>> drop it! + temp01 = dist_inter_to_segment1[i, j, 1] >= dist_segments[i] + temp02 = dist_inter_to_segment1[ + i, j, 0] <= dist_segments[i] * outside_ratio + temp03 = dist_inter_to_segment1[i, j, 1] <= dist_segments[i] + temp04 = dist_inter_to_segment1[ + i, j, 0] <= dist_segments[i] * inside_ratio + temp_cond01 = ((temp01 and temp02) or (temp03 and temp04)) + temp05 = dist_inter_to_segment2[i, j, 1] >= dist_segments[j] + temp06 = dist_inter_to_segment2[ + i, j, 0] <= dist_segments[j] * outside_ratio + temp07 = dist_inter_to_segment2[i, j, 1] <= dist_segments[j] + temp08 = dist_inter_to_segment2[ + i, j, 0] <= dist_segments[j] * inside_ratio + temp_cond02 = ((temp05 and temp06) or (temp07 and temp08)) + check_distance = temp_cond01 and temp_cond02 + + if check_degree and check_distance: + corner_info = None + + if (deg1 >= 0 and deg1 <= 45 and deg2 >= 45 and deg2 <= 120) or \ + (deg2 >= 315 and deg1 >= 45 and deg1 <= 120): + corner_info = 0 + elif (deg1 >= 45 and deg1 <= 125 and deg2 >= 125 + and deg2 <= 225): + corner_info = 1 + elif (deg1 >= 125 and deg1 <= 225 and deg2 >= 225 + and deg2 <= 315): + corner_info = 2 + elif (deg1 >= 0 and deg1 <= 45 and deg2 >= 225 and deg2 <= 315) or \ + (deg2 >= 315 and deg1 >= 225 and deg1 <= 315): + corner_info = 3 + else: + corner_info = 4 # we don't use it + continue + + corner_dict[corner_info].append([x, y, i, j]) + inter_points.append([x, y]) + + square_list = [] + connect_list = [] + segments_list = [] + for corner0 in corner_dict[0]: + for corner1 in corner_dict[1]: + connect01 = False + for corner0_line in corner0[2:]: + if corner0_line in corner1[2:]: + connect01 = True + break + if connect01: + for corner2 in corner_dict[2]: + connect12 = False + for corner1_line in corner1[2:]: + if corner1_line in corner2[2:]: + connect12 = True + break + if connect12: + for corner3 in corner_dict[3]: + connect23 = False + for corner2_line in corner2[2:]: + if corner2_line in corner3[2:]: + connect23 = True + break + if connect23: + for corner3_line in corner3[2:]: + if corner3_line in corner0[2:]: + # SQUARE!!! + ''' + 0 -- 1 + | | + 3 -- 2 + square_list: + order: 0 > 1 > 2 > 3 + | x0, y0, x1, y1, x2, y2, x3, y3 | + | x0, y0, x1, y1, x2, y2, x3, y3 | + ... + connect_list: + order: 01 > 12 > 23 > 30 + | line_idx01, line_idx12, line_idx23, line_idx30 | + | line_idx01, line_idx12, line_idx23, line_idx30 | + ... + segments_list: + order: 0 > 1 > 2 > 3 + | line_idx0_i, line_idx0_j, line_idx1_i, line_idx1_j, + line_idx2_i, line_idx2_j, line_idx3_i, line_idx3_j | + | line_idx0_i, line_idx0_j, line_idx1_i, line_idx1_j, + line_idx2_i, line_idx2_j, line_idx3_i, line_idx3_j | + ... + ''' + square_list.append(corner0[:2] + + corner1[:2] + + corner2[:2] + + corner3[:2]) + connect_list.append([ + corner0_line, corner1_line, + corner2_line, corner3_line + ]) + segments_list.append(corner0[2:] + + corner1[2:] + + corner2[2:] + + corner3[2:]) + + def check_outside_inside(segments_info, connect_idx): + # return 'outside or inside', min distance, cover_param, peri_param + if connect_idx == segments_info[0]: + check_dist_mat = dist_inter_to_segment1 + else: + check_dist_mat = dist_inter_to_segment2 + + i, j = segments_info + min_dist, max_dist = check_dist_mat[i, j, :] + connect_dist = dist_segments[connect_idx] + if max_dist > connect_dist: + return 'outside', min_dist, 0, 1 + else: + return 'inside', min_dist, -1, -1 + + # top_square = None + + try: + map_size = input_shape[0] / 2 + squares = np.array(square_list).reshape([-1, 4, 2]) + score_array = [] + connect_array = np.array(connect_list) + segments_array = np.array(segments_list).reshape([-1, 4, 2]) + + # get degree of corners: + squares_rollup = np.roll(squares, 1, axis=1) + squares_rolldown = np.roll(squares, -1, axis=1) + vec1 = squares_rollup - squares + normalized_vec1 = vec1 / ( + np.linalg.norm(vec1, axis=-1, keepdims=True) + 1e-10) + vec2 = squares_rolldown - squares + normalized_vec2 = vec2 / ( + np.linalg.norm(vec2, axis=-1, keepdims=True) + 1e-10) + inner_products = np.sum( + normalized_vec1 * normalized_vec2, axis=-1) # [n_squares, 4] + squares_degree = np.arccos( + inner_products) * 180 / np.pi # [n_squares, 4] + + # get square score + overlap_scores = [] + degree_scores = [] + length_scores = [] + + for connects, segments, square, degree in zip(connect_array, + segments_array, squares, + squares_degree): + ''' + 0 -- 1 + | | + 3 -- 2 + + # segments: [4, 2] + # connects: [4] + ''' + + # OVERLAP SCORES + cover = 0 + perimeter = 0 + # check 0 > 1 > 2 > 3 + square_length = [] + + for start_idx in range(4): + end_idx = (start_idx + 1) % 4 + + connect_idx = connects[start_idx] # segment idx of segment01 + start_segments = segments[start_idx] + end_segments = segments[end_idx] + + # start_point = square[start_idx] + # end_point = square[end_idx] + + # check whether outside or inside + start_position, start_min, start_cover_param, start_peri_param = check_outside_inside( + start_segments, connect_idx) + end_position, end_min, end_cover_param, end_peri_param = check_outside_inside( + end_segments, connect_idx) + + cover += dist_segments[ + connect_idx] + start_cover_param * start_min + end_cover_param * end_min + perimeter += dist_segments[ + connect_idx] + start_peri_param * start_min + end_peri_param * end_min + + square_length.append(dist_segments[connect_idx] + + start_peri_param * start_min + + end_peri_param * end_min) + + overlap_scores.append(cover / perimeter) + # OVERLAP SCORES + # DEGREE SCORES + ''' + deg0 vs deg2 + deg1 vs deg3 + ''' + deg0, deg1, deg2, deg3 = degree + deg_ratio1 = deg0 / deg2 + if deg_ratio1 > 1.0: + deg_ratio1 = 1 / deg_ratio1 + deg_ratio2 = deg1 / deg3 + if deg_ratio2 > 1.0: + deg_ratio2 = 1 / deg_ratio2 + degree_scores.append((deg_ratio1 + deg_ratio2) / 2) + # DEGREE SCORES + # LENGTH SCORES + ''' + len0 vs len2 + len1 vs len3 + ''' + len0, len1, len2, len3 = square_length + len_ratio1 = len0 / len2 if len2 > len0 else len2 / len0 + len_ratio2 = len1 / len3 if len3 > len1 else len3 / len1 + length_scores.append((len_ratio1 + len_ratio2) / 2) + + # LENGTH SCORES + + overlap_scores = np.array(overlap_scores) + overlap_scores /= np.max(overlap_scores) + + degree_scores = np.array(degree_scores) + # degree_scores /= np.max(degree_scores) + + length_scores = np.array(length_scores) + + # AREA SCORES + area_scores = np.reshape(squares, [-1, 4, 2]) + area_x = area_scores[:, :, 0] + area_y = area_scores[:, :, 1] + correction = area_x[:, -1] * area_y[:, 0] - area_y[:, -1] * area_x[:, + 0] + area_scores = np.sum( + area_x[:, :-1] * area_y[:, 1:], axis=-1) - np.sum( + area_y[:, :-1] * area_x[:, 1:], axis=-1) + area_scores = 0.5 * np.abs(area_scores + correction) + area_scores /= (map_size * map_size) # np.max(area_scores) + # AREA SCORES + + # CENTER SCORES + centers = np.array([[256 // 2, 256 // 2]], dtype='float32') # [1, 2] + # squares: [n, 4, 2] + square_centers = np.mean(squares, axis=1) # [n, 2] + center2center = np.sqrt(np.sum((centers - square_centers)**2)) + center_scores = center2center / (map_size / np.sqrt(2.0)) + ''' + score_w = [overlap, degree, area, center, length] + ''' + # score_w = [0.0, 1.0, 10.0, 0.5, 1.0] + temp_value001 = params['w_overlap'] * overlap_scores + temp_value002 = params['w_degree'] * degree_scores + temp_value003 = params['w_area'] * area_scores + temp_value004 = params['w_center'] * center_scores + temp_value005 = params['w_length'] * length_scores + score_array = temp_value001 + temp_value002 + temp_value003 - temp_value004 + temp_value005 + + # best_square = [] + + sorted_idx = np.argsort(score_array)[::-1] + score_array = score_array[sorted_idx] + squares = squares[sorted_idx] + + except Exception: + pass + '''return list + merged_lines, squares, scores + ''' + + try: + new_segments[:, 0] = new_segments[:, 0] * 2 / input_shape[ + 1] * original_shape[1] + new_segments[:, 1] = new_segments[:, 1] * 2 / input_shape[ + 0] * original_shape[0] + new_segments[:, 2] = new_segments[:, 2] * 2 / input_shape[ + 1] * original_shape[1] + new_segments[:, 3] = new_segments[:, 3] * 2 / input_shape[ + 0] * original_shape[0] + except Exception: + new_segments = [] + + try: + squares[:, :, + 0] = squares[:, :, 0] * 2 / input_shape[1] * original_shape[1] + squares[:, :, + 1] = squares[:, :, 1] * 2 / input_shape[0] * original_shape[0] + except Exception: + squares = [] + score_array = [] + + try: + inter_points = np.array(inter_points) + inter_points[:, 0] = inter_points[:, 0] * 2 / input_shape[ + 1] * original_shape[1] + inter_points[:, 1] = inter_points[:, 1] * 2 / input_shape[ + 0] * original_shape[0] + except Exception: + inter_points = [] + + return new_segments, squares, score_array, inter_points diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/__init__.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/body.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/body.py new file mode 100644 index 0000000000000000000000000000000000000000..11e33c2fe0ccba594db181dafd306b5122e57363 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/body.py @@ -0,0 +1,285 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import math +import time + +import cv2 +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import torch +from scipy.ndimage.filters import gaussian_filter +from torchvision import transforms + +from . import util +from .model import bodypose_model + + +class Body(object): + + def __init__(self, model_path, device): + self.model = bodypose_model() + self.model = self.model.to(device) + model_dict = util.transfer(self.model, torch.load(model_path)) + self.model.load_state_dict(model_dict) + self.model.eval() + + def __call__(self, oriImg): + # scale_search = [0.5, 1.0, 1.5, 2.0] + scale_search = [0.5] + boxsize = 368 + stride = 8 + padValue = 128 + thre1 = 0.1 + thre2 = 0.05 + multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search] + heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) + paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) + + for m in range(len(multiplier)): + scale = multiplier[m] + imageToTest = cv2.resize( + oriImg, (0, 0), + fx=scale, + fy=scale, + interpolation=cv2.INTER_CUBIC) + imageToTest_padded, pad = util.padRightDownCorner( + imageToTest, stride, padValue) + im = np.transpose( + np.float32(imageToTest_padded[:, :, :, np.newaxis]), + (3, 2, 0, 1)) / 256 - 0.5 + im = np.ascontiguousarray(im) + + data = torch.from_numpy(im).float() + if torch.cuda.is_available(): + data = data.cuda() + # data = data.permute([2, 0, 1]).unsqueeze(0).float() + with torch.no_grad(): + Mconv7_stage6_L1, Mconv7_stage6_L2 = self.model(data) + Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy() + Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy() + + # extract outputs, resize, and remove padding + # heatmap = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[1]].data), (1, 2, 0)) + # output 1 is heatmaps + heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), + (1, 2, 0)) # output 1 is heatmaps + heatmap = cv2.resize( + heatmap, (0, 0), + fx=stride, + fy=stride, + interpolation=cv2.INTER_CUBIC) + heatmap = heatmap[:imageToTest_padded.shape[0] + - pad[2], :imageToTest_padded.shape[1] + - pad[3], :] + heatmap = cv2.resize( + heatmap, (oriImg.shape[1], oriImg.shape[0]), + interpolation=cv2.INTER_CUBIC) + + # paf = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[0]].data), (1, 2, 0)) # output 0 is PAFs + paf = np.transpose(np.squeeze(Mconv7_stage6_L1), + (1, 2, 0)) # output 0 is PAFs + paf = cv2.resize( + paf, (0, 0), + fx=stride, + fy=stride, + interpolation=cv2.INTER_CUBIC) + paf = paf[:imageToTest_padded.shape[0] + - pad[2], :imageToTest_padded.shape[1] - pad[3], :] + paf = cv2.resize( + paf, (oriImg.shape[1], oriImg.shape[0]), + interpolation=cv2.INTER_CUBIC) + + heatmap_avg += heatmap_avg + heatmap / len(multiplier) + paf_avg += +paf / len(multiplier) + + all_peaks = [] + peak_counter = 0 + + for part in range(18): + map_ori = heatmap_avg[:, :, part] + one_heatmap = gaussian_filter(map_ori, sigma=3) + + map_left = np.zeros(one_heatmap.shape) + map_left[1:, :] = one_heatmap[:-1, :] + map_right = np.zeros(one_heatmap.shape) + map_right[:-1, :] = one_heatmap[1:, :] + map_up = np.zeros(one_heatmap.shape) + map_up[:, 1:] = one_heatmap[:, :-1] + map_down = np.zeros(one_heatmap.shape) + map_down[:, :-1] = one_heatmap[:, 1:] + + peaks_binary = np.logical_and.reduce( + (one_heatmap >= map_left, one_heatmap >= map_right, + one_heatmap >= map_up, one_heatmap >= map_down, + one_heatmap > thre1)) + peaks = list( + zip(np.nonzero(peaks_binary)[1], + np.nonzero(peaks_binary)[0])) # note reverse + peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks] + peak_id = range(peak_counter, peak_counter + len(peaks)) + peaks_with_score_and_id = [ + peaks_with_score[i] + (peak_id[i], ) + for i in range(len(peak_id)) + ] + + all_peaks.append(peaks_with_score_and_id) + peak_counter += len(peaks) + + # find connection in the specified sequence, center 29 is in the position 15 + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], + [9, 10], [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], + [1, 15], [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]] + # the middle joints heatmap correpondence + mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], + [19, 20], [21, 22], [23, 24], [25, 26], [27, 28], [29, 30], + [47, 48], [49, 50], [53, 54], [51, 52], [55, 56], [37, 38], + [45, 46]] + + connection_all = [] + special_k = [] + mid_num = 10 + + for k in range(len(mapIdx)): + score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] + candA = all_peaks[limbSeq[k][0] - 1] + candB = all_peaks[limbSeq[k][1] - 1] + nA = len(candA) + nB = len(candB) + indexA, indexB = limbSeq[k] + if (nA != 0 and nB != 0): + connection_candidate = [] + for i in range(nA): + for j in range(nB): + vec = np.subtract(candB[j][:2], candA[i][:2]) + norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) + norm = max(0.001, norm) + vec = np.divide(vec, norm) + + startend = list( + zip( + np.linspace( + candA[i][0], candB[j][0], num=mid_num), + np.linspace( + candA[i][1], candB[j][1], num=mid_num))) + + vec_x = np.array([ + score_mid[int(round(startend[Ivalue2][1])), + int(round(startend[Ivalue2][0])), 0] + for Ivalue2 in range(len(startend)) + ]) + vec_y = np.array([ + score_mid[int(round(startend[Ivalue1][1])), + int(round(startend[Ivalue1][0])), 1] + for Ivalue1 in range(len(startend)) + ]) + + score_midpts = np.multiply( + vec_x, vec[0]) + np.multiply(vec_y, vec[1]) + score_with_dist_prior = sum(score_midpts) / len( + score_midpts) + min( + 0.5 * oriImg.shape[0] / norm - 1, 0) + criterion1 = len(np.nonzero( + score_midpts > thre2)[0]) > 0.8 * len(score_midpts) + criterion2 = score_with_dist_prior > 0 + if criterion1 and criterion2: + connection_candidate.append([ + i, j, score_with_dist_prior, + score_with_dist_prior + candA[i][2] + + candB[j][2] + ]) + + connection_candidate = sorted( + connection_candidate, key=lambda x: x[2], reverse=True) + connection = np.zeros((0, 5)) + for c in range(len(connection_candidate)): + i, j, s = connection_candidate[c][0:3] + if (i not in connection[:, 3] + and j not in connection[:, 4]): + connection = np.vstack( + [connection, [candA[i][3], candB[j][3], s, i, j]]) + if (len(connection) >= min(nA, nB)): + break + + connection_all.append(connection) + else: + special_k.append(k) + connection_all.append([]) + + # last number in each row is the total parts number of that person + # the second last number in each row is the score of the overall configuration + subset = -1 * np.ones((0, 20)) + candidate = np.array( + [item for sublist in all_peaks for item in sublist]) + + for k in range(len(mapIdx)): + if k not in special_k: + partAs = connection_all[k][:, 0] + partBs = connection_all[k][:, 1] + indexA, indexB = np.array(limbSeq[k]) - 1 + + for i in range(len(connection_all[k])): # = 1:size(temp,1) + found = 0 + subset_idx = [-1, -1] + for j in range(len(subset)): # 1:size(subset,1): + if subset[j][indexA] == partAs[i] or subset[j][ + indexB] == partBs[i]: + subset_idx[found] = j + found += 1 + + if found == 1: + j = subset_idx[0] + if subset[j][indexB] != partBs[i]: + subset[j][indexB] = partBs[i] + subset[j][-1] += 1 + subset[j][-2] += candidate[ + partBs[i].astype(int), + 2] + connection_all[k][i][2] + elif found == 2: # if found 2 and disjoint, merge them + j1, j2 = subset_idx + temp_value01 = (subset[j1] >= 0).astype(int) + temp_value02 = (subset[j2] >= 0).astype(int) + membership = (temp_value01 + temp_value02)[:-2] + if len(np.nonzero(membership == 2)[0]) == 0: # merge + subset[j1][:-2] += (subset[j2][:-2] + 1) + subset[j1][-2:] += subset[j2][-2:] + subset[j1][-2] += connection_all[k][i][2] + subset = np.delete(subset, j2, 0) + else: # as like found == 1 + subset[j1][indexB] = partBs[i] + subset[j1][-1] += 1 + subset[j1][-2] += candidate[ + partBs[i].astype(int), + 2] + connection_all[k][i][2] + + # if find no partA in the subset, create a new subset + elif not found and k < 17: + row = -1 * np.ones(20) + row[indexA] = partAs[i] + row[indexB] = partBs[i] + row[-1] = 2 + row[-2] = sum( + candidate[connection_all[k][i, :2].astype(int), + 2]) + connection_all[k][i][2] + subset = np.vstack([subset, row]) + # delete some rows of subset which has few parts occur + deleteIdx = [] + for i in range(len(subset)): + if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: + deleteIdx.append(i) + subset = np.delete(subset, deleteIdx, axis=0) + + # subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts + # candidate: x, y, score, id + return candidate, subset + + +if __name__ == '__main__': + body_estimation = Body('../model/body_pose_model.pth') + + test_image = '../images/ski.jpg' + oriImg = cv2.imread(test_image) # B,G,R order + candidate, subset = body_estimation(oriImg) + canvas = util.draw_bodypose(oriImg, candidate, subset) + plt.imshow(canvas[:, :, [2, 1, 0]]) + plt.show() diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/hand.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/hand.py new file mode 100644 index 0000000000000000000000000000000000000000..e1c11e7b172c1fc79876a6815d663abf70abb55e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/hand.py @@ -0,0 +1,110 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import math +import time + +import cv2 +import json +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import torch +from scipy.ndimage.filters import gaussian_filter +from skimage.measure import label + +from . import util +from .model import handpose_model + + +class Hand(object): + + def __init__(self, model_path, device): + self.model = handpose_model() + self.model = self.model.to(device) + model_dict = util.transfer(self.model, torch.load(model_path)) + self.model.load_state_dict(model_dict) + self.model.eval() + + def __call__(self, oriImg): + scale_search = [0.5, 1.0, 1.5, 2.0] + # scale_search = [0.5] + boxsize = 368 + stride = 8 + padValue = 128 + thre = 0.05 + multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search] + heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 22)) + # paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) + + for m in range(len(multiplier)): + scale = multiplier[m] + imageToTest = cv2.resize( + oriImg, (0, 0), + fx=scale, + fy=scale, + interpolation=cv2.INTER_CUBIC) + imageToTest_padded, pad = util.padRightDownCorner( + imageToTest, stride, padValue) + im = np.transpose( + np.float32(imageToTest_padded[:, :, :, np.newaxis]), + (3, 2, 0, 1)) / 256 - 0.5 + im = np.ascontiguousarray(im) + + data = torch.from_numpy(im).float() + if torch.cuda.is_available(): + data = data.cuda() + # data = data.permute([2, 0, 1]).unsqueeze(0).float() + with torch.no_grad(): + output = self.model(data).cpu().numpy() + # output = self.model(data).numpy()q + + # extract outputs, resize, and remove padding + heatmap = np.transpose(np.squeeze(output), + (1, 2, 0)) # output 1 is heatmaps + heatmap = cv2.resize( + heatmap, (0, 0), + fx=stride, + fy=stride, + interpolation=cv2.INTER_CUBIC) + heatmap = heatmap[:imageToTest_padded.shape[0] + - pad[2], :imageToTest_padded.shape[1] + - pad[3], :] + heatmap = cv2.resize( + heatmap, (oriImg.shape[1], oriImg.shape[0]), + interpolation=cv2.INTER_CUBIC) + + heatmap_avg += heatmap / len(multiplier) + + all_peaks = [] + for part in range(21): + map_ori = heatmap_avg[:, :, part] + one_heatmap = gaussian_filter(map_ori, sigma=3) + binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8) + # 全部小于阈值 + if np.sum(binary) == 0: + all_peaks.append([0, 0]) + continue + label_img, label_numbers = label( + binary, return_num=True, connectivity=binary.ndim) + max_index = np.argmax([ + np.sum(map_ori[label_img == i]) + for i in range(1, label_numbers + 1) + ]) + 1 + label_img[label_img != max_index] = 0 + map_ori[label_img == 0] = 0 + + y, x = util.npmax(map_ori) + all_peaks.append([x, y]) + return np.array(all_peaks) + + +if __name__ == '__main__': + hand_estimation = Hand('../model/hand_pose_model.pth') + + # test_image = '../images/hand.jpg' + test_image = '../images/hand.jpg' + oriImg = cv2.imread(test_image) # B,G,R order + peaks = hand_estimation(oriImg) + canvas = util.draw_handpose(oriImg, peaks, True) + cv2.imshow('', canvas) + cv2.waitKey(0) diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/model.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/model.py new file mode 100644 index 0000000000000000000000000000000000000000..ce1f9f2e21681e179af3723f510a8bd7bf46ac63 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/model.py @@ -0,0 +1,217 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +from collections import OrderedDict + +import torch +import torch.nn as nn + + +def make_layers(block, no_relu_layers): + layers = [] + for layer_name, v in block.items(): + if 'pool' in layer_name: + layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2]) + layers.append((layer_name, layer)) + else: + conv2d = nn.Conv2d( + in_channels=v[0], + out_channels=v[1], + kernel_size=v[2], + stride=v[3], + padding=v[4]) + layers.append((layer_name, conv2d)) + if layer_name not in no_relu_layers: + layers.append(('relu_' + layer_name, nn.ReLU(inplace=True))) + + return nn.Sequential(OrderedDict(layers)) + + +class bodypose_model(nn.Module): + + def __init__(self): + super(bodypose_model, self).__init__() + + # these layers have no relu layer + no_relu_layers = [ + 'conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1', + 'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2', + 'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1', + 'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1' + ] + blocks = {} + block0 = OrderedDict([('conv1_1', [3, 64, 3, 1, 1]), + ('conv1_2', [64, 64, 3, 1, 1]), + ('pool1_stage1', [2, 2, 0]), + ('conv2_1', [64, 128, 3, 1, 1]), + ('conv2_2', [128, 128, 3, 1, 1]), + ('pool2_stage1', [2, 2, 0]), + ('conv3_1', [128, 256, 3, 1, 1]), + ('conv3_2', [256, 256, 3, 1, 1]), + ('conv3_3', [256, 256, 3, 1, 1]), + ('conv3_4', [256, 256, 3, 1, 1]), + ('pool3_stage1', [2, 2, 0]), + ('conv4_1', [256, 512, 3, 1, 1]), + ('conv4_2', [512, 512, 3, 1, 1]), + ('conv4_3_CPM', [512, 256, 3, 1, 1]), + ('conv4_4_CPM', [256, 128, 3, 1, 1])]) + + # Stage 1 + block1_1 = OrderedDict([('conv5_1_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L1', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L1', [512, 38, 1, 1, 0])]) + + block1_2 = OrderedDict([('conv5_1_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L2', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L2', [512, 19, 1, 1, 0])]) + blocks['block1_1'] = block1_1 + blocks['block1_2'] = block1_2 + + self.model0 = make_layers(block0, no_relu_layers) + + # Stages 2 - 6 + for i in range(2, 7): + blocks['block%d_1' % i] = OrderedDict([ + ('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0]) + ]) + + blocks['block%d_2' % i] = OrderedDict([ + ('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0]) + ]) + + for k in blocks.keys(): + blocks[k] = make_layers(blocks[k], no_relu_layers) + + self.model1_1 = blocks['block1_1'] + self.model2_1 = blocks['block2_1'] + self.model3_1 = blocks['block3_1'] + self.model4_1 = blocks['block4_1'] + self.model5_1 = blocks['block5_1'] + self.model6_1 = blocks['block6_1'] + + self.model1_2 = blocks['block1_2'] + self.model2_2 = blocks['block2_2'] + self.model3_2 = blocks['block3_2'] + self.model4_2 = blocks['block4_2'] + self.model5_2 = blocks['block5_2'] + self.model6_2 = blocks['block6_2'] + + def forward(self, x): + + out1 = self.model0(x) + + out1_1 = self.model1_1(out1) + out1_2 = self.model1_2(out1) + out2 = torch.cat([out1_1, out1_2, out1], 1) + + out2_1 = self.model2_1(out2) + out2_2 = self.model2_2(out2) + out3 = torch.cat([out2_1, out2_2, out1], 1) + + out3_1 = self.model3_1(out3) + out3_2 = self.model3_2(out3) + out4 = torch.cat([out3_1, out3_2, out1], 1) + + out4_1 = self.model4_1(out4) + out4_2 = self.model4_2(out4) + out5 = torch.cat([out4_1, out4_2, out1], 1) + + out5_1 = self.model5_1(out5) + out5_2 = self.model5_2(out5) + out6 = torch.cat([out5_1, out5_2, out1], 1) + + out6_1 = self.model6_1(out6) + out6_2 = self.model6_2(out6) + + return out6_1, out6_2 + + +class handpose_model(nn.Module): + + def __init__(self): + super(handpose_model, self).__init__() + + # these layers have no relu layer + no_relu_layers = [ + 'conv6_2_CPM', 'Mconv7_stage2', 'Mconv7_stage3', 'Mconv7_stage4', + 'Mconv7_stage5', 'Mconv7_stage6' + ] + # stage 1 + block1_0 = OrderedDict([('conv1_1', [3, 64, 3, 1, 1]), + ('conv1_2', [64, 64, 3, 1, 1]), + ('pool1_stage1', [2, 2, 0]), + ('conv2_1', [64, 128, 3, 1, 1]), + ('conv2_2', [128, 128, 3, 1, 1]), + ('pool2_stage1', [2, 2, 0]), + ('conv3_1', [128, 256, 3, 1, 1]), + ('conv3_2', [256, 256, 3, 1, 1]), + ('conv3_3', [256, 256, 3, 1, 1]), + ('conv3_4', [256, 256, 3, 1, 1]), + ('pool3_stage1', [2, 2, 0]), + ('conv4_1', [256, 512, 3, 1, 1]), + ('conv4_2', [512, 512, 3, 1, 1]), + ('conv4_3', [512, 512, 3, 1, 1]), + ('conv4_4', [512, 512, 3, 1, 1]), + ('conv5_1', [512, 512, 3, 1, 1]), + ('conv5_2', [512, 512, 3, 1, 1]), + ('conv5_3_CPM', [512, 128, 3, 1, 1])]) + + block1_1 = OrderedDict([('conv6_1_CPM', [128, 512, 1, 1, 0]), + ('conv6_2_CPM', [512, 22, 1, 1, 0])]) + + blocks = {} + blocks['block1_0'] = block1_0 + blocks['block1_1'] = block1_1 + + # stage 2-6 + for i in range(2, 7): + blocks['block%d' % i] = OrderedDict([ + ('Mconv1_stage%d' % i, [150, 128, 7, 1, 3]), + ('Mconv2_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d' % i, [128, 22, 1, 1, 0]) + ]) + + for k in blocks.keys(): + blocks[k] = make_layers(blocks[k], no_relu_layers) + + self.model1_0 = blocks['block1_0'] + self.model1_1 = blocks['block1_1'] + self.model2 = blocks['block2'] + self.model3 = blocks['block3'] + self.model4 = blocks['block4'] + self.model5 = blocks['block5'] + self.model6 = blocks['block6'] + + def forward(self, x): + out1_0 = self.model1_0(x) + out1_1 = self.model1_1(out1_0) + concat_stage2 = torch.cat([out1_1, out1_0], 1) + out_stage2 = self.model2(concat_stage2) + concat_stage3 = torch.cat([out_stage2, out1_0], 1) + out_stage3 = self.model3(concat_stage3) + concat_stage4 = torch.cat([out_stage3, out1_0], 1) + out_stage4 = self.model4(concat_stage4) + concat_stage5 = torch.cat([out_stage4, out1_0], 1) + out_stage5 = self.model5(concat_stage5) + concat_stage6 = torch.cat([out_stage5, out1_0], 1) + out_stage6 = self.model6(concat_stage6) + return out_stage6 diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/util.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/util.py new file mode 100644 index 0000000000000000000000000000000000000000..0830e343c8280653acbd1a43845ceec6c8e6b1fc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/annotator/openpose/util.py @@ -0,0 +1,193 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet +import math + +import cv2 +import matplotlib +import numpy as np + + +def padRightDownCorner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :] * 0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :] * 0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + +# transfer caffe model to pytorch which will match the layer name +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join( + weights_name.split('.')[1:])] + return transfered_model_weights + + +# draw the body keypoint and lims +def draw_bodypose(canvas, candidate, subset): + stickwidth = 4 + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], + [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], + [170, 255, 0], [85, 255, 0], [0, 255, 0], [0, 255, 85], + [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], + [0, 0, 255], [85, 0, 255], [170, 0, 255], [255, 0, 255], + [255, 0, 170], [255, 0, 85]] + for i in range(18): + for n in range(len(subset)): + index = int(subset[n][i]) + if index == -1: + continue + x, y = candidate[index][0:2] + cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1) + for i in range(17): + for n in range(len(subset)): + index = subset[n][np.array(limbSeq[i]) - 1] + if -1 in index: + continue + cur_canvas = canvas.copy() + Y = candidate[index.astype(int), 0] + X = candidate[index.astype(int), 1] + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly( + (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), + 0, 360, 1) + cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) + canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) + # plt.imsave("preview.jpg", canvas[:, :, [2, 1, 0]]) + # plt.imshow(canvas[:, :, [2, 1, 0]]) + return canvas + + +# image drawed by opencv is not good. +def draw_handpose(canvas, all_hand_peaks, show_number=False): + edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], + [0, 9], [9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], + [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] + + for peaks in all_hand_peaks: + for ie, e in enumerate(edges): + if np.sum(np.all(peaks[e], axis=1) == 0) == 0: + x1, y1 = peaks[e[0]] + x2, y2 = peaks[e[1]] + cv2.line( + canvas, (x1, y1), (x2, y2), + matplotlib.colors.hsv_to_rgb( + [ie / float(len(edges)), 1.0, 1.0]) * 255, + thickness=2) + + for i, keyponit in enumerate(peaks): + x, y = keyponit + cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1) + if show_number: + cv2.putText( + canvas, + str(i), (x, y), + cv2.FONT_HERSHEY_SIMPLEX, + 0.3, (0, 0, 0), + lineType=cv2.LINE_AA) + return canvas + + +# detect hand according to body pose keypoints +# please refer to https://github.com/CMU-Perceptual-Computing-Lab/ +# openpose/blob/master/src/openpose/hand/handDetector.cpp +def handDetect(candidate, subset, oriImg): + # right hand: wrist 4, elbow 3, shoulder 2 + # left hand: wrist 7, elbow 6, shoulder 5 + ratioWristElbow = 0.33 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + # if any of three not detected + has_left = np.sum(person[[5, 6, 7]] == -1) == 0 + has_right = np.sum(person[[2, 3, 4]] == -1) == 0 + if not (has_left or has_right): + continue + hands = [] + # left hand + if has_left: + left_shoulder_index, left_elbow_index, left_wrist_index = person[[ + 5, 6, 7 + ]] + x1, y1 = candidate[left_shoulder_index][:2] + x2, y2 = candidate[left_elbow_index][:2] + x3, y3 = candidate[left_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, True]) + # right hand + if has_right: + right_shoulder_index, right_elbow_index, right_wrist_index = person[ + [2, 3, 4]] + x1, y1 = candidate[right_shoulder_index][:2] + x2, y2 = candidate[right_elbow_index][:2] + x3, y3 = candidate[right_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, False]) + + for x1, y1, x2, y2, x3, y3, is_left in hands: + # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox + # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]); + # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]); + # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow); + # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder); + # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder); + x = x3 + ratioWristElbow * (x3 - x2) + y = y3 + ratioWristElbow * (y3 - y2) + distanceWristElbow = math.sqrt((x3 - x2)**2 + (y3 - y2)**2) + distanceElbowShoulder = math.sqrt((x2 - x1)**2 + (y2 - y1)**2) + width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder) + # x-y refers to the center --> offset to topLeft point + # handRectangle.x -= handRectangle.width / 2.f; + # handRectangle.y -= handRectangle.height / 2.f; + x -= width / 2 + y -= width / 2 # width = height + # overflow the image + if x < 0: + x = 0 + if y < 0: + y = 0 + width1 = width + width2 = width + if x + width > image_width: + width1 = image_width - x + if y + width > image_height: + width2 = image_height - y + width = min(width1, width2) + # the max hand box value is 20 pixels + if width >= 20: + detect_result.append([int(x), int(y), int(width), is_left]) + ''' + return value: [[x, y, w, True if left hand else False]]. + width=height since the network require squared input. + x, y is the coordinate of top left + ''' + return detect_result + + +# get max index of 2d array +def npmax(array): + arrayindex = array.argmax(1) + arrayvalue = array.max(1) + i = arrayvalue.argmax() + j = arrayindex[i] + return i, j diff --git a/runtime/Lib/site_packages/models/cv/controllable_image_generation/controlnet.py b/runtime/Lib/site_packages/models/cv/controllable_image_generation/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..5418a39e5431c5007f1c5d567c51947741b5f487 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/controllable_image_generation/controlnet.py @@ -0,0 +1,224 @@ +# Part of the implementation is borrowed and modified from ControlNet, +# publicly available at https://github.com/lllyasviel/ControlNet + +import math +import os +import random +import sys +import tempfile +from typing import Any, Dict, Optional, Union + +import cv2 +import einops +import numpy as np +import torch +import torch.nn as nn +from control_ldm.cldm.hack import disable_verbosity, enable_sliced_attention +from control_ldm.cldm.model import create_model, load_state_dict +from control_ldm.ldm.models.diffusion.ddim import DDIMSampler +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +__all__ = ['ControlNet'] + + +@MODELS.register_module( + Tasks.controllable_image_generation, + module_name=Models.controllable_image_generation) +class ControlNet(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize ControlNet from theq `model_dir` path. + ControlNet: + Adding Conditional Control to Text-to-Image Diffusion Models. + Paper: https://arxiv.org/abs/2302.05543 + Origin codes: https://github.com/lllyasviel/ControlNet + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + + enable_sa = self.config.modelsetting.get('enable_sliced_attention', + True) + self.image_resolution = self.config.modelsetting.image_resolution + + self.modelsetting = self.config.modelsetting + + disable_verbosity() + if enable_sa: + enable_sliced_attention() + init_control_type = kwargs.get('control_type', 'hed') + if init_control_type == 'scribble': + input_setting = self.modelsetting.scribble + elif init_control_type == 'canny': + input_setting = self.modelsetting.canny + elif init_control_type == 'hough': + input_setting = self.modelsetting.hough + elif init_control_type == 'hed': + input_setting = self.modelsetting.hed + elif init_control_type == 'depth': + input_setting = self.modelsetting.depth + elif init_control_type == 'normal': + input_setting = self.modelsetting.normal + elif init_control_type == 'pose': + input_setting = self.modelsetting.pose + elif init_control_type == 'seg': + input_setting = self.modelsetting.seg + elif init_control_type == 'fake_scribble': + input_setting = self.modelsetting.scribble + else: + print('Error input type, use HED for default!') + input_setting = self.modelsetting.hed + self.init_control_type = init_control_type + self.input_setting = input_setting + + yaml_path = os.path.join(self.model_dir, input_setting.yaml_path) + ckpt_path = os.path.join(self.model_dir, input_setting.ckpt_path) + device = kwargs.get('device', 'cuda') + if device == 'gpu': + device = 'cuda' + model = create_model(yaml_path).cpu() + state_dict = load_state_dict(ckpt_path, location=device) + compatible_position_ids( + state_dict, + 'cond_stage_model.transformer.text_model.embeddings.position_ids') + model.load_state_dict(state_dict) + self.model = model.to(device) + self.ddim_sampler = DDIMSampler(self.model) + + def get_resolution(self): + return self.image_resolution + + def get_config(self): + return self.modelsetting + + def get_model_dir(self): + return self.model_dir + + def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + """return the result by the model + Args: + inputs (Dict[str, Any]) should contains the keys: + - "image", a numpy array + - "prompt", string of prompt + - "detected_map", a numpy array of detected map + - "save_memory", boolean indicating whether to save memory + - "is_cat_img", boolean indicating whether to concatenate results + + inputs (Dict[str, Any]) can also contains the keys, but not required: + - "image_resolution", int + - "strength", float + - "guess_mode", bool + - "ddim_steps", int + - "scale", float + - "num_samples", int + - "eta", float + - "a_prompt", string of added prompt + - "n_prompt", string of negative prompt + + Returns: + Dict[str, Any]: A dict contains result, detected_map and boolean 'is_cat_img' + indicating whether to concatenate the result and the detected_map. + + """ + image = inputs['image'] + prompt = inputs['prompt'] + detected_map = inputs['detected_map'] # processed in preprocessor + save_memory = inputs.get('save_memory', False) + + num_samples = inputs.get('num_samples', self.input_setting.num_samples) + scale = inputs.get('scale', self.input_setting.scale) + ddim_steps = inputs.get('ddim_steps', self.input_setting.ddim_steps) + eta = inputs.get('eta', self.input_setting.eta) + a_prompt = inputs.get('a_prompt', self.input_setting.a_prompt) + n_prompt = inputs.get('n_prompt', self.input_setting.n_prompt) + guess_mode = inputs.get('guess_mode', self.input_setting.guess_mode) + strength = inputs.get('strength', self.input_setting.strength) + print(f'Process with guess_mode:{guess_mode},strength:{strength},') + print( + f'num_samples:{num_samples},scale:{scale},ddim_steps:{ddim_steps},eta:{eta}' + ) + print(f'a_prompt:\'{a_prompt}\',n_prompt:\'{n_prompt}\',') + + with torch.no_grad(): + H, W, C = image.shape + + control = torch.from_numpy( + detected_map.copy()).float().cuda() / 255.0 + control = torch.stack([control for _ in range(num_samples)], dim=0) + control = einops.rearrange(control, 'b h w c -> b c h w').clone() + + if save_memory: + self.model.low_vram_shift(is_diffusing=False) + + cond = { + 'c_concat': [control], + 'c_crossattn': [ + self.model.get_learned_conditioning( + [prompt + ', ' + a_prompt] * num_samples) + ] + } + un_cond = { + 'c_concat': + [torch.zeros_like(control) if guess_mode else control], + 'c_crossattn': [ + self.model.get_learned_conditioning([n_prompt] + * num_samples) + ] + } + shape = (4, H // 8, W // 8) + + if save_memory: + self.model.low_vram_shift(is_diffusing=True) + + self.model.control_scales = [ + strength * (0.825**float(12 - i)) for i in range(13) + ] if guess_mode else ([strength] * 13) + samples, intermediates = self.ddim_sampler.sample( + ddim_steps, + num_samples, + shape, + cond, + verbose=False, + eta=eta, + unconditional_guidance_scale=scale, + unconditional_conditioning=un_cond) + + if save_memory: + self.model.low_vram_shift(is_diffusing=False) + + x_samples = self.model.decode_first_stage(samples) + x_samples = ( + einops.rearrange(x_samples, 'b c h w -> b h w c') * 127.5 + + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8) + + results = [x_samples[i] for i in range(num_samples)] + + if self.init_control_type == 'hough': + show_det_map = cv2.dilate( + detected_map, + np.ones(shape=(3, 3), dtype=np.uint8), + iterations=1) + elif self.init_control_type == 'normal': + show_det_map = detected_map[:, :, ::-1] + elif self.init_control_type == 'fake_scribble' or self.init_control_type == 'scribble': + show_det_map = 255 - detected_map + else: + show_det_map = detected_map + return { + 'result': results, + 'detected_map': show_det_map, + 'is_cat_img': inputs['is_cat_img'] + } diff --git a/runtime/Lib/site_packages/models/cv/crowd_counting/__init__.py b/runtime/Lib/site_packages/models/cv/crowd_counting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b5eeb937cf66909043832272a309384ae7fe40cc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/crowd_counting/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .cc_model import HRNetCrowdCounting + +else: + _import_structure = { + 'cc_model': ['HRNetCrowdCounting'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/crowd_counting/cc_model.py b/runtime/Lib/site_packages/models/cv/crowd_counting/cc_model.py new file mode 100644 index 0000000000000000000000000000000000000000..16fbc26113239db0faca0f78191db55523f9250d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/crowd_counting/cc_model.py @@ -0,0 +1,36 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Any, Dict, Optional, Union + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.crowd_counting, module_name=Models.crowd_counting) +class HRNetCrowdCounting(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + super().__init__(model_dir, **kwargs) + + from .hrnet_aspp_relu import HighResolutionNet as HRNet_aspp_relu + + domain_center_model = os.path.join( + model_dir, 'average_clip_domain_center_54.97.npz') + net = HRNet_aspp_relu( + attn_weight=1.0, + fix_domain=0, + domain_center_model=domain_center_model) + net.load_state_dict( + torch.load( + os.path.join(model_dir, 'DCANet_final.pth'), + map_location='cpu')) + self.model = net + + def forward(self, inputs): + return self.model(inputs) diff --git a/runtime/Lib/site_packages/models/cv/crowd_counting/hrnet_aspp_relu.py b/runtime/Lib/site_packages/models/cv/crowd_counting/hrnet_aspp_relu.py new file mode 100644 index 0000000000000000000000000000000000000000..64f40da066b82aa4366a000194f6f5336fc72311 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/crowd_counting/hrnet_aspp_relu.py @@ -0,0 +1,638 @@ +""" +Copyright (c) Microsoft +Licensed under the MIT License. +Written by Bin Xiao (Bin.Xiao@microsoft.com) +Modified by Ke Sun (sunk@mail.ustc.edu.cn) +https://github.com/HRNet/HRNet-Image-Classification/blob/master/lib/models/cls_hrnet.py +""" + +import functools +import logging +import os + +import numpy as np +import torch +import torch._utils +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.utils.logger import get_logger + +BN_MOMENTUM = 0.01 # 0.01 for seg +logger = get_logger() + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d( + planes * self.expansion, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class HighResolutionModule(nn.Module): + + def __init__(self, + num_branches, + blocks, + num_blocks, + num_inchannels, + num_channels, + fuse_method, + multi_scale_output=True): + super(HighResolutionModule, self).__init__() + self._check_branches(num_branches, blocks, num_blocks, num_inchannels, + num_channels) + + self.num_inchannels = num_inchannels + self.fuse_method = fuse_method + self.num_branches = num_branches + + self.multi_scale_output = multi_scale_output + + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(False) + + def _check_branches(self, num_branches, blocks, num_blocks, num_inchannels, + num_channels): + if num_branches != len(num_blocks): + error_msg = 'NUM_BRANCHES({}) <> NUM_BLOCKS({})'.format( + num_branches, len(num_blocks)) + logger.info(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = 'NUM_BRANCHES({}) <> NUM_CHANNELS({})'.format( + num_branches, len(num_channels)) + logger.info(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_inchannels): + error_msg = 'NUM_BRANCHES({}) <> NUM_INCHANNELS({})'.format( + num_branches, len(num_inchannels)) + logger.info(error_msg) + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + downsample = None + if stride != 1 or \ + self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.num_inchannels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d( + num_channels[branch_index] * block.expansion, + momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index], stride, downsample)) + self.num_inchannels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index])) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + if self.num_branches == 1: + return None + + num_branches = self.num_branches + num_inchannels = self.num_inchannels + fuse_layers = [] + for i in range(num_branches if self.multi_scale_output else 1): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_inchannels[i], + 1, + 1, + 0, + bias=False), + nn.BatchNorm2d( + num_inchannels[i], momentum=BN_MOMENTUM), + nn.Upsample( + scale_factor=2**(j - i), mode='nearest'))) + elif j == i: + fuse_layer.append(None) + else: + conv3x3s = [] + for k in range(i - j): + if k == i - j - 1: + num_outchannels_conv3x3 = num_inchannels[i] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + nn.BatchNorm2d( + num_outchannels_conv3x3, + momentum=BN_MOMENTUM))) + else: + num_outchannels_conv3x3 = num_inchannels[j] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + nn.BatchNorm2d( + num_outchannels_conv3x3, + momentum=BN_MOMENTUM), nn.ReLU(False))) + fuse_layer.append(nn.Sequential(*conv3x3s)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def get_num_inchannels(self): + return self.num_inchannels + + def forward(self, x): + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = x[0] if i == 0 else self.fuse_layers[i][0](x[0]) + for j in range(1, self.num_branches): + if i == j: + y = y + x[j] + else: + y = y + self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + + return x_fuse + + +blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + +class HighResolutionNet(nn.Module): + + def __init__(self, + leaky_relu=False, + attn_weight=1, + fix_domain=1, + domain_center_model='', + **kwargs): + super(HighResolutionNet, self).__init__() + + self.criterion_attn = torch.nn.MSELoss(reduction='sum') + self.domain_center_model = domain_center_model + self.attn_weight = attn_weight + self.fix_domain = fix_domain + self.cosine = 1 + + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + 64, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + + num_channels = 64 + block = blocks_dict['BOTTLENECK'] + num_blocks = 4 + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + stage1_out_channel = block.expansion * num_channels + + # -- stage 2 + self.stage2_cfg = {} + self.stage2_cfg['NUM_MODULES'] = 1 + self.stage2_cfg['NUM_BRANCHES'] = 2 + self.stage2_cfg['BLOCK'] = 'BASIC' + self.stage2_cfg['NUM_BLOCKS'] = [4, 4] + self.stage2_cfg['NUM_CHANNELS'] = [40, 80] + self.stage2_cfg['FUSE_METHOD'] = 'SUM' + + num_channels = self.stage2_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage2_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition1 = self._make_transition_layer([stage1_out_channel], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + # -- stage 3 + self.stage3_cfg = {} + self.stage3_cfg['NUM_MODULES'] = 4 + self.stage3_cfg['NUM_BRANCHES'] = 3 + self.stage3_cfg['BLOCK'] = 'BASIC' + self.stage3_cfg['NUM_BLOCKS'] = [4, 4, 4] + self.stage3_cfg['NUM_CHANNELS'] = [40, 80, 160] + self.stage3_cfg['FUSE_METHOD'] = 'SUM' + + num_channels = self.stage3_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage3_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + last_inp_channels = int(np.sum(pre_stage_channels)) + 256 + self.redc_layer = nn.Sequential( + nn.Conv2d( + in_channels=last_inp_channels, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), + nn.BatchNorm2d(128, momentum=BN_MOMENTUM), + nn.ReLU(True), + ) + + self.aspp = nn.ModuleList(aspp(in_channel=128)) + + # additional layers specfic for Phase 3 + self.pred_conv = nn.Conv2d(128, 512, 3, padding=1) + self.pred_bn = nn.BatchNorm2d(512) + self.GAP = nn.AdaptiveAvgPool2d(1) + + # Specially for hidden domain + # Set the domain for learnable parameters + domain_center_src = np.load(self.domain_center_model) + G_SHA = torch.from_numpy(domain_center_src['G_SHA']).view(1, -1, 1, 1) + G_SHB = torch.from_numpy(domain_center_src['G_SHB']).view(1, -1, 1, 1) + G_QNRF = torch.from_numpy(domain_center_src['G_QNRF']).view( + 1, -1, 1, 1) + + self.n_domain = 3 + + self.G_all = torch.cat( + [G_SHA.clone(), G_SHB.clone(), + G_QNRF.clone()], dim=0) + + self.G_all = nn.Parameter(self.G_all) + + self.last_layer = nn.Sequential( + nn.Conv2d( + in_channels=128, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), + nn.BatchNorm2d(64, momentum=BN_MOMENTUM), + nn.ReLU(True), + nn.Conv2d( + in_channels=64, + out_channels=32, + kernel_size=3, + stride=1, + padding=1), + nn.BatchNorm2d(32, momentum=BN_MOMENTUM), + nn.ReLU(True), + nn.Conv2d( + in_channels=32, + out_channels=1, + kernel_size=1, + stride=1, + padding=0), + ) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + nn.Conv2d( + num_channels_pre_layer[i], + num_channels_cur_layer[i], + 3, + 1, + 1, + bias=False), + nn.BatchNorm2d( + num_channels_cur_layer[i], + momentum=BN_MOMENTUM), nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv3x3s = [] + for j in range(i + 1 - num_branches_pre): + inchannels = num_channels_pre_layer[-1] + outchannels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else inchannels + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + inchannels, outchannels, 3, 2, 1, bias=False), + nn.BatchNorm2d(outchannels, momentum=BN_MOMENTUM), + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv3x3s)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(inplanes, planes, stride, downsample)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_stage(self, + layer_config, + num_inchannels, + multi_scale_output=True): + num_modules = layer_config['NUM_MODULES'] + num_branches = layer_config['NUM_BRANCHES'] + num_blocks = layer_config['NUM_BLOCKS'] + num_channels = layer_config['NUM_CHANNELS'] + block = blocks_dict[layer_config['BLOCK']] + fuse_method = layer_config['FUSE_METHOD'] + + modules = [] + for i in range(num_modules): + # multi_scale_output is only used last module + if not multi_scale_output and i == num_modules - 1: + reset_multi_scale_output = False + else: + reset_multi_scale_output = True + + modules.append( + HighResolutionModule(num_branches, block, num_blocks, + num_inchannels, num_channels, fuse_method, + reset_multi_scale_output)) + num_inchannels = modules[-1].get_num_inchannels() + + return nn.Sequential(*modules), num_inchannels + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + x = self.layer1(x) + x_head_1 = x + + x_list = [] + for i in range(self.stage2_cfg['NUM_BRANCHES']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['NUM_BRANCHES']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + + x = self.stage3(x_list) + + # Replace the classification heaeder with custom setting + # Upsampling + x0_h, x0_w = x[0].size(2), x[0].size(3) + x1 = F.interpolate( + x[1], size=(x0_h, x0_w), mode='bilinear', align_corners=False) + x2 = F.interpolate( + x[2], size=(x0_h, x0_w), mode='bilinear', align_corners=False) + x = torch.cat([x[0], x1, x2, x_head_1], 1) + # first, reduce the channel down + x = self.redc_layer(x) + + pred_attn = self.GAP(F.relu_(self.pred_bn(self.pred_conv(x)))) + pred_attn = F.softmax(pred_attn, dim=1) + pred_attn_list = torch.chunk(pred_attn, 4, dim=1) + + aspp_out = [] + for k, v in enumerate(self.aspp): + if k % 2 == 0: + aspp_out.append(self.aspp[k + 1](v(x))) + else: + continue + # Using Aspp add, and relu inside + for i in range(4): + x = x + F.relu_(aspp_out[i] * 0.25) * pred_attn_list[i] + + bz = x.size(0) + # -- Besides, we also need to let the prediction attention be close to visable domain + # -- Calculate the domain distance and get the weights + # - First, detach domains + G_all_d = self.G_all.detach() # use detached G_all for calulcating + pred_attn_d = pred_attn.detach().view(bz, 512, 1, 1) + + if self.cosine == 1: + G_A, G_B, G_Q = torch.chunk(G_all_d, self.n_domain, dim=0) + + cos_dis_A = F.cosine_similarity(pred_attn_d, G_A, dim=1).view(-1) + cos_dis_B = F.cosine_similarity(pred_attn_d, G_B, dim=1).view(-1) + cos_dis_Q = F.cosine_similarity(pred_attn_d, G_Q, dim=1).view(-1) + + cos_dis_all = torch.stack([cos_dis_A, cos_dis_B, + cos_dis_Q]).view(bz, -1) # bz*3 + + cos_dis_all = F.softmax(cos_dis_all, dim=1) + + target_attn = cos_dis_all.view(bz, self.n_domain, 1, 1, 1).expand( + bz, self.n_domain, 512, 1, 1) * self.G_all.view( + 1, self.n_domain, 512, 1, 1).expand( + bz, self.n_domain, 512, 1, 1) + target_attn = torch.sum( + target_attn, dim=1, keepdim=False) # bz * 512 * 1 * 1 + + if self.fix_domain: + target_attn = target_attn.detach() + + else: + raise ValueError('Have not implemented not cosine distance yet') + + x = self.last_layer(x) + x = F.relu_(x) + + x = F.interpolate( + x, size=(x0_h * 2, x0_w * 2), mode='bilinear', align_corners=False) + + return x, pred_attn, target_attn + + def init_weights( + self, + pretrained='', + ): + logger.info('=> init weights from normal distribution') + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, std=0.01) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + if os.path.isfile(pretrained): + pretrained_dict = torch.load(pretrained) + logger.info(f'=> loading pretrained model {pretrained}') + model_dict = self.state_dict() + pretrained_dict = { + k: v + for k, v in pretrained_dict.items() if k in model_dict.keys() + } + for k, _ in pretrained_dict.items(): + logger.info(f'=> loading {k} pretrained model {pretrained}') + model_dict.update(pretrained_dict) + self.load_state_dict(model_dict) + else: + assert 1 == 2 + + +def aspp(aspp_num=4, aspp_stride=2, in_channel=512, use_bn=True): + aspp_list = [] + for i in range(aspp_num): + pad = (i + 1) * aspp_stride + dilate = pad + conv_aspp = nn.Conv2d( + in_channel, in_channel, 3, padding=pad, dilation=dilate) + aspp_list.append(conv_aspp) + if use_bn: + aspp_list.append(nn.BatchNorm2d(in_channel)) + + return aspp_list diff --git a/runtime/Lib/site_packages/models/cv/face_attribute_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d1592e7cd253856ccb52204f38db64d3d076fe0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .fair_face import FaceAttributeRecognition + +else: + _import_structure = {'fair_face': ['FaceAttributeRecognition']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/__init__.py b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5f2be336f0a5669cb8ead53106f72ce7f1c6789 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .face_attribute_recognition import FaceAttributeRecognition diff --git a/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/face_attribute_recognition.py b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/face_attribute_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..46441e9038ffb0bb76f7ceda5844df031b706aaa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_attribute_recognition/fair_face/face_attribute_recognition.py @@ -0,0 +1,79 @@ +# The implementation is based on FairFace, available at +# https://github.com/dchen236/FairFace +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from PIL import Image +from torch.autograd import Variable +from torchvision import datasets, models, transforms + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.face_attribute_recognition, module_name=Models.fairface) +class FaceAttributeRecognition(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.cfg_path = model_path.replace(ModelFile.TORCH_MODEL_FILE, + ModelFile.CONFIGURATION) + fair_face = torchvision.models.resnet34(pretrained=False) + fair_face.fc = nn.Linear(fair_face.fc.in_features, 18) + self.net = fair_face + self.load_model() + self.net = self.net.to(device) + self.trans = transforms.Compose([ + transforms.ToPILImage(), + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu')) + self.net.load_state_dict(pretrained_dict, strict=True) + self.net.eval() + + def forward(self, img): + """ FariFace model forward process. + + Args: + img: [h, w, c] + + Return: + list of attribute result: [gender_score, age_score] + """ + img = cv2.cvtColor(img.cpu().numpy(), cv2.COLOR_BGR2RGB) + img = img.astype(np.uint8) + + inputs = self.trans(img) + + c, h, w = inputs.shape + + inputs = inputs.view(-1, c, h, w) + inputs = inputs.to(self.device) + inputs = Variable(inputs, volatile=True) + outputs = self.net(inputs)[0] + + gender_outputs = outputs[7:9] + age_outputs = outputs[9:18] + + gender_score = F.softmax(gender_outputs).detach().cpu().tolist() + age_score = F.softmax(age_outputs).detach().cpu().tolist() + + return [gender_score, age_score] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b9700381082a9c5e762689707f2b1b2ba56b814 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mogface import MogFaceDetector + from .mtcnn import MtcnnFaceDetector + from .retinaface import RetinaFaceDetection + from .ulfd_slim import UlfdFaceDetector + from .scrfd import ScrfdDetect + from .scrfd import TinyMogDetect + from .scrfd import SCRFDPreprocessor +else: + _import_structure = { + 'ulfd_slim': ['UlfdFaceDetector'], + 'retinaface': ['RetinaFaceDetection'], + 'mtcnn': ['MtcnnFaceDetector'], + 'mogface': ['MogFaceDetector'], + 'scrfd': ['TinyMogDetect', 'ScrfdDetect', 'SCRFDPreprocessor'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a58268d0af8879ee9ced4a0a8f269ea020e57776 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .models.detectors import MogFaceDetector diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/detectors.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/detectors.py new file mode 100644 index 0000000000000000000000000000000000000000..9009abfbe40e45150c25f3041e9e44275b2c640a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/detectors.py @@ -0,0 +1,97 @@ +# The implementation is based on MogFace, available at +# https://github.com/damo-cv/MogFace +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from .mogface import MogFace +from .utils import MogPriorBox, mogdecode, py_cpu_nms + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.mogface) +class MogFaceDetector(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.net = MogFace() + self.load_model() + self.net = self.net.to(device) + + self.mean = np.array([[104, 117, 123]]) + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu')) + self.net.load_state_dict(pretrained_dict, strict=False) + self.net.eval() + + def forward(self, input): + img_raw = input['img'] + img = np.array(img_raw.cpu().detach()) + img = img[:, :, ::-1] + + im_height, im_width = img.shape[:2] + ss = 1.0 + # tricky + if max(im_height, im_width) > 1500: + ss = 1000.0 / max(im_height, im_width) + img = cv2.resize(img, (0, 0), fx=ss, fy=ss) + im_height, im_width = img.shape[:2] + + scale = torch.Tensor( + [img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) + img -= np.array([[103.53, 116.28, 123.675]]) + img /= np.array([[57.375, 57.120003, 58.395]]) + img /= 255 + img = img[:, :, ::-1].copy() + img = img.transpose(2, 0, 1) + img = torch.from_numpy(img).unsqueeze(0) + img = img.to(self.device) + scale = scale.to(self.device) + + conf, loc = self.net(img) # forward pass + + confidence_threshold = 0.82 + nms_threshold = 0.4 + top_k = 5000 + keep_top_k = 750 + + priorbox = MogPriorBox(scale_list=[0.68]) + priors = priorbox(im_height, im_width) + priors = torch.tensor(priors).to(self.device) + prior_data = priors.data + + boxes = mogdecode(loc.data.squeeze(0), prior_data) + boxes = boxes.cpu().numpy() + scores = conf.squeeze(0).data.cpu().numpy()[:, 0] + + # ignore low scores + inds = np.where(scores > confidence_threshold)[0] + boxes = boxes[inds] + scores = scores[inds] + + # keep top-K before NMS + order = scores.argsort()[::-1][:top_k] + boxes = boxes[order] + scores = scores[order] + + # do NMS + dets = np.hstack((boxes, scores[:, np.newaxis])).astype( + np.float32, copy=False) + keep = py_cpu_nms(dets, nms_threshold) + dets = dets[keep, :] + + # keep top-K faster NMS + dets = dets[:keep_top_k, :] + + return dets / ss diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogface.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogface.py new file mode 100644 index 0000000000000000000000000000000000000000..294c2c6b7ef56f0785f154efdccecb65a284e4d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogface.py @@ -0,0 +1,135 @@ +# -------------------------------------------------------- +# The implementation is also open-sourced by the authors as Yang Liu, and is available publicly on +# https://github.com/damo-cv/MogFace +# -------------------------------------------------------- +import torch.nn as nn +import torch.nn.functional as F + +from .mogprednet import MogPredNet +from .resnet import ResNet + + +class MogFace(nn.Module): + + def __init__(self): + super(MogFace, self).__init__() + self.backbone = ResNet(depth=101) + self.fpn = LFPN() + self.pred_net = MogPredNet() + + def forward(self, x): + feature_list = self.backbone(x) + fpn_list = self.fpn(feature_list) + pyramid_feature_list = fpn_list[0] + conf, loc = self.pred_net(pyramid_feature_list) + return conf, loc + + +class FeatureFusion(nn.Module): + + def __init__(self, lat_ch=256, **channels): + super(FeatureFusion, self).__init__() + self.main_conv = nn.Conv2d(channels['main'], lat_ch, kernel_size=1) + + def forward(self, up, main): + main = self.main_conv(main) + _, _, H, W = main.size() + res = F.upsample(up, scale_factor=2, mode='bilinear') + if res.size(2) != main.size(2) or res.size(3) != main.size(3): + res = res[:, :, 0:H, 0:W] + res = res + main + return res + + +class LFPN(nn.Module): + + def __init__(self, + c2_out_ch=256, + c3_out_ch=512, + c4_out_ch=1024, + c5_out_ch=2048, + c6_mid_ch=512, + c6_out_ch=512, + c7_mid_ch=128, + c7_out_ch=256, + out_dsfd_ft=True): + super(LFPN, self).__init__() + self.out_dsfd_ft = out_dsfd_ft + if self.out_dsfd_ft: + dsfd_module = [] + dsfd_module.append(nn.Conv2d(256, 256, kernel_size=3, padding=1)) + dsfd_module.append(nn.Conv2d(512, 256, kernel_size=3, padding=1)) + dsfd_module.append(nn.Conv2d(1024, 256, kernel_size=3, padding=1)) + dsfd_module.append(nn.Conv2d(2048, 256, kernel_size=3, padding=1)) + dsfd_module.append(nn.Conv2d(256, 256, kernel_size=3, padding=1)) + dsfd_module.append(nn.Conv2d(256, 256, kernel_size=3, padding=1)) + self.dsfd_modules = nn.ModuleList(dsfd_module) + + c6_input_ch = c5_out_ch + self.c6 = nn.Sequential(*[ + nn.Conv2d( + c6_input_ch, + c6_mid_ch, + kernel_size=1, + ), + nn.BatchNorm2d(c6_mid_ch), + nn.ReLU(inplace=True), + nn.Conv2d( + c6_mid_ch, c6_out_ch, kernel_size=3, padding=1, stride=2), + nn.BatchNorm2d(c6_out_ch), + nn.ReLU(inplace=True) + ]) + self.c7 = nn.Sequential(*[ + nn.Conv2d( + c6_out_ch, + c7_mid_ch, + kernel_size=1, + ), + nn.BatchNorm2d(c7_mid_ch), + nn.ReLU(inplace=True), + nn.Conv2d( + c7_mid_ch, c7_out_ch, kernel_size=3, padding=1, stride=2), + nn.BatchNorm2d(c7_out_ch), + nn.ReLU(inplace=True) + ]) + + self.p2_lat = nn.Conv2d(256, 256, kernel_size=3, padding=1) + self.p3_lat = nn.Conv2d(256, 256, kernel_size=3, padding=1) + self.p4_lat = nn.Conv2d(256, 256, kernel_size=3, padding=1) + + self.c5_lat = nn.Conv2d(c6_input_ch, 256, kernel_size=3, padding=1) + self.c6_lat = nn.Conv2d(c6_out_ch, 256, kernel_size=3, padding=1) + self.c7_lat = nn.Conv2d(c7_out_ch, 256, kernel_size=3, padding=1) + + self.ff_c5_c4 = FeatureFusion(main=c4_out_ch) + self.ff_c4_c3 = FeatureFusion(main=c3_out_ch) + self.ff_c3_c2 = FeatureFusion(main=c2_out_ch) + + def forward(self, feature_list): + c2, c3, c4, c5 = feature_list + c6 = self.c6(c5) + c7 = self.c7(c6) + + c5 = self.c5_lat(c5) + c6 = self.c6_lat(c6) + c7 = self.c7_lat(c7) + + if self.out_dsfd_ft: + dsfd_fts = [] + dsfd_fts.append(self.dsfd_modules[0](c2)) + dsfd_fts.append(self.dsfd_modules[1](c3)) + dsfd_fts.append(self.dsfd_modules[2](c4)) + dsfd_fts.append(self.dsfd_modules[3](feature_list[-1])) + dsfd_fts.append(self.dsfd_modules[4](c6)) + dsfd_fts.append(self.dsfd_modules[5](c7)) + + p4 = self.ff_c5_c4(c5, c4) + p3 = self.ff_c4_c3(p4, c3) + p2 = self.ff_c3_c2(p3, c2) + + p2 = self.p2_lat(p2) + p3 = self.p3_lat(p3) + p4 = self.p4_lat(p4) + + if self.out_dsfd_ft: + return ([p2, p3, p4, c5, c6, c7], dsfd_fts) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogprednet.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogprednet.py new file mode 100644 index 0000000000000000000000000000000000000000..31384976dcdee375002ef3d4175f5824011e94f0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/mogprednet.py @@ -0,0 +1,164 @@ +# -------------------------------------------------------- +# The implementation is also open-sourced by the authors as Yang Liu, and is available publicly on +# https://github.com/damo-cv/MogFace +# -------------------------------------------------------- +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class conv_bn(nn.Module): + """docstring for conv""" + + def __init__(self, in_plane, out_plane, kernel_size, stride, padding): + super(conv_bn, self).__init__() + self.conv1 = nn.Conv2d( + in_plane, + out_plane, + kernel_size=kernel_size, + stride=stride, + padding=padding) + self.bn1 = nn.BatchNorm2d(out_plane) + + def forward(self, x): + x = self.conv1(x) + return self.bn1(x) + + +class SSHContext(nn.Module): + + def __init__(self, channels, Xchannels=256): + super(SSHContext, self).__init__() + + self.conv1 = nn.Conv2d( + channels, Xchannels, kernel_size=3, stride=1, padding=1) + self.conv2 = nn.Conv2d( + channels, + Xchannels // 2, + kernel_size=3, + dilation=2, + stride=1, + padding=2) + self.conv2_1 = nn.Conv2d( + Xchannels // 2, Xchannels // 2, kernel_size=3, stride=1, padding=1) + self.conv2_2 = nn.Conv2d( + Xchannels // 2, + Xchannels // 2, + kernel_size=3, + dilation=2, + stride=1, + padding=2) + self.conv2_2_1 = nn.Conv2d( + Xchannels // 2, Xchannels // 2, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x1 = F.relu(self.conv1(x), inplace=True) + x2 = F.relu(self.conv2(x), inplace=True) + x2_1 = F.relu(self.conv2_1(x2), inplace=True) + x2_2 = F.relu(self.conv2_2(x2), inplace=True) + x2_2 = F.relu(self.conv2_2_1(x2_2), inplace=True) + + return torch.cat([x1, x2_1, x2_2], 1) + + +class DeepHead(nn.Module): + + def __init__(self, + in_channel=256, + out_channel=256, + use_gn=False, + num_conv=4): + super(DeepHead, self).__init__() + self.use_gn = use_gn + self.num_conv = num_conv + self.conv1 = nn.Conv2d(in_channel, out_channel, 3, 1, 1) + self.conv2 = nn.Conv2d(out_channel, out_channel, 3, 1, 1) + self.conv3 = nn.Conv2d(out_channel, out_channel, 3, 1, 1) + self.conv4 = nn.Conv2d(out_channel, out_channel, 3, 1, 1) + if self.use_gn: + self.gn1 = nn.GroupNorm(16, out_channel) + self.gn2 = nn.GroupNorm(16, out_channel) + self.gn3 = nn.GroupNorm(16, out_channel) + self.gn4 = nn.GroupNorm(16, out_channel) + + def forward(self, x): + if self.use_gn: + x1 = F.relu(self.gn1(self.conv1(x)), inplace=True) + x2 = F.relu(self.gn2(self.conv1(x1)), inplace=True) + x3 = F.relu(self.gn3(self.conv1(x2)), inplace=True) + x4 = F.relu(self.gn4(self.conv1(x3)), inplace=True) + else: + x1 = F.relu(self.conv1(x), inplace=True) + x2 = F.relu(self.conv1(x1), inplace=True) + if self.num_conv == 2: + return x2 + x3 = F.relu(self.conv1(x2), inplace=True) + x4 = F.relu(self.conv1(x3), inplace=True) + + return x4 + + +class MogPredNet(nn.Module): + + def __init__(self, + num_anchor_per_pixel=1, + num_classes=1, + input_ch_list=[256, 256, 256, 256, 256, 256], + use_deep_head=True, + deep_head_with_gn=True, + use_ssh=True, + deep_head_ch=512): + super(MogPredNet, self).__init__() + self.num_classes = num_classes + self.use_deep_head = use_deep_head + self.deep_head_with_gn = deep_head_with_gn + + self.use_ssh = use_ssh + + self.deep_head_ch = deep_head_ch + + if self.use_ssh: + self.conv_SSH = SSHContext(input_ch_list[0], + self.deep_head_ch // 2) + + if self.use_deep_head: + if self.deep_head_with_gn: + self.deep_loc_head = DeepHead( + self.deep_head_ch, self.deep_head_ch, use_gn=True) + self.deep_cls_head = DeepHead( + self.deep_head_ch, self.deep_head_ch, use_gn=True) + + self.pred_cls = nn.Conv2d(self.deep_head_ch, + 1 * num_anchor_per_pixel, 3, 1, 1) + self.pred_loc = nn.Conv2d(self.deep_head_ch, + 4 * num_anchor_per_pixel, 3, 1, 1) + + self.sigmoid = nn.Sigmoid() + + def forward(self, pyramid_feature_list, dsfd_ft_list=None): + loc = [] + conf = [] + + if self.use_deep_head: + for x in pyramid_feature_list: + if self.use_ssh: + x = self.conv_SSH(x) + x_cls = self.deep_cls_head(x) + x_loc = self.deep_loc_head(x) + + conf.append( + self.pred_cls(x_cls).permute(0, 2, 3, 1).contiguous()) + loc.append( + self.pred_loc(x_loc).permute(0, 2, 3, 1).contiguous()) + + loc = torch.cat([o.view(o.size(0), -1, 4) for o in loc], 1) + conf = torch.cat( + [o.view(o.size(0), -1, self.num_classes) for o in conf], 1) + output = ( + self.sigmoid(conf.view(conf.size(0), -1, self.num_classes)), + loc.view(loc.size(0), -1, 4), + ) + + return output diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/resnet.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..045f6fa37e2a3f7b0ca244430da19ff11547c1e3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/resnet.py @@ -0,0 +1,193 @@ +# The implementation is modified from original resent implementaiton, which is +# also open-sourced by the authors as Yang Liu, +# and is available publicly on https://github.com/damo-cv/MogFace + +import torch.nn as nn + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, + depth=50, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None, + inplanes=64, + shrink_ch_ratio=1): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + if depth == 50: + block = Bottleneck + layers = [3, 4, 6, 3] + elif depth == 101: + block = Bottleneck + layers = [3, 4, 23, 3] + elif depth == 152: + block = Bottleneck + layers = [3, 4, 36, 3] + elif depth == 18: + block = BasicBlock + layers = [2, 2, 2, 2] + else: + raise ValueError('only support depth in [18, 50, 101, 152]') + + shrink_input_ch = int(inplanes * shrink_ch_ratio) + self.inplanes = int(inplanes * shrink_ch_ratio) + if shrink_ch_ratio == 0.125: + layers = [2, 3, 3, 3] + + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, shrink_input_ch, layers[0]) + self.layer2 = self._make_layer( + block, + shrink_input_ch * 2, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + shrink_input_ch * 4, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + shrink_input_ch * 8, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + four_conv_layer = [] + x = self.layer1(x) + four_conv_layer.append(x) + x = self.layer2(x) + four_conv_layer.append(x) + x = self.layer3(x) + four_conv_layer.append(x) + x = self.layer4(x) + four_conv_layer.append(x) + + return four_conv_layer diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/utils.py b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..377ceb3d3b34acf4ec7a91b5ab460d8807c6f9f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mogface/models/utils.py @@ -0,0 +1,212 @@ +# Modified from https://github.com/biubug6/Pytorch_Retinaface + +import math +from itertools import product as product +from math import ceil + +import numpy as np +import torch + + +def transform_anchor(anchors): + """ + from [x0, x1, y0, y1] to [c_x, cy, w, h] + x1 = x0 + w - 1 + c_x = (x0 + x1) / 2 = (2x0 + w - 1) / 2 = x0 + (w - 1) / 2 + """ + return np.concatenate(((anchors[:, :2] + anchors[:, 2:]) / 2, + anchors[:, 2:] - anchors[:, :2] + 1), + axis=1) + + +def normalize_anchor(anchors): + """ + from [c_x, cy, w, h] to [x0, x1, y0, y1] + """ + item_1 = anchors[:, :2] - (anchors[:, 2:] - 1) / 2 + item_2 = anchors[:, :2] + (anchors[:, 2:] - 1) / 2 + return np.concatenate((item_1, item_2), axis=1) + + +class MogPriorBox(object): + """ + both for fpn and single layer, single layer need to test + return (np.array) [num_anchros, 4] [x0, y0, x1, y1] + """ + + def __init__(self, + scale_list=[1.], + aspect_ratio_list=[1.0], + stride_list=[4, 8, 16, 32, 64, 128], + anchor_size_list=[16, 32, 64, 128, 256, 512]): + self.scale_list = scale_list + self.aspect_ratio_list = aspect_ratio_list + self.stride_list = stride_list + self.anchor_size_list = anchor_size_list + + def __call__(self, img_height, img_width): + final_anchor_list = [] + + for idx, stride in enumerate(self.stride_list): + anchor_list = [] + cur_img_height = img_height + cur_img_width = img_width + tmp_stride = stride + + while tmp_stride != 1: + tmp_stride = tmp_stride // 2 + cur_img_height = (cur_img_height + 1) // 2 + cur_img_width = (cur_img_width + 1) // 2 + + for i in range(cur_img_height): + for j in range(cur_img_width): + for scale in self.scale_list: + cx = (j + 0.5) * stride + cy = (i + 0.5) * stride + side_x = self.anchor_size_list[idx] * scale + side_y = self.anchor_size_list[idx] * scale + for ratio in self.aspect_ratio_list: + anchor_list.append([ + cx, cy, side_x / math.sqrt(ratio), + side_y * math.sqrt(ratio) + ]) + + final_anchor_list.append(anchor_list) + final_anchor_arr = np.concatenate(final_anchor_list, axis=0) + normalized_anchor_arr = normalize_anchor(final_anchor_arr).astype( + 'float32') + transformed_anchor = transform_anchor(normalized_anchor_arr) + + return transformed_anchor + + +class PriorBox(object): + + def __init__(self, cfg, image_size=None, phase='train'): + super(PriorBox, self).__init__() + self.min_sizes = cfg['min_sizes'] + self.steps = cfg['steps'] + self.clip = cfg['clip'] + self.image_size = image_size + self.feature_maps = [[ + ceil(self.image_size[0] / step), + ceil(self.image_size[1] / step) + ] for step in self.steps] + self.name = 's' + + def forward(self): + anchors = [] + for k, f in enumerate(self.feature_maps): + min_sizes = self.min_sizes[k] + for i, j in product(range(f[0]), range(f[1])): + for min_size in min_sizes: + s_kx = min_size / self.image_size[1] + s_ky = min_size / self.image_size[0] + dense_cx = [ + x * self.steps[k] / self.image_size[1] + for x in [j + 0.5] + ] + dense_cy = [ + y * self.steps[k] / self.image_size[0] + for y in [i + 0.5] + ] + for cy, cx in product(dense_cy, dense_cx): + anchors += [cx, cy, s_kx, s_ky] + + # back to torch land + output = torch.Tensor(anchors).view(-1, 4) + if self.clip: + output.clamp_(max=1, min=0) + return output + + +def py_cpu_nms(dets, thresh): + """Pure Python NMS baseline.""" + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = dets[:, 2] + y2 = dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + + return keep + + +def mogdecode(loc, anchors): + """ + loc: torch.Tensor + anchors: 2-d, torch.Tensor (cx, cy, w, h) + boxes: 2-d, torch.Tensor (x0, y0, x1, y1) + """ + + boxes = torch.cat((anchors[:, :2] + loc[:, :2] * anchors[:, 2:], + anchors[:, 2:] * torch.exp(loc[:, 2:])), 1) + + boxes[:, 0] -= (boxes[:, 2] - 1) / 2 + boxes[:, 1] -= (boxes[:, 3] - 1) / 2 + boxes[:, 2] += boxes[:, 0] - 1 + boxes[:, 3] += boxes[:, 1] - 1 + + return boxes + + +# Adapted from https://github.com/Hakuyume/chainer-ssd +def decode(loc, priors, variances): + """Decode locations from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + loc (tensor): location predictions for loc layers, + Shape: [num_priors,4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat( + (priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def decode_landm(pre, priors, variances): + """Decode landm from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + pre (tensor): landm predictions for loc layers, + Shape: [num_priors,10] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded landm predictions + """ + a = priors[:, :2] + pre[:, :2] * variances[0] * priors[:, 2:] + b = priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:] + c = priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:] + d = priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:] + e = priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:] + landms = torch.cat((a, b, c, d, e), dim=1) + return landms diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9fddab9c1c3634d6542fe2d5b32b16ca3a2fb93c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .models.detector import MtcnnFaceDetector diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/box_utils.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/box_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a27b05d8c69a2e852d5d9180af2be7c46dbb59 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/box_utils.py @@ -0,0 +1,240 @@ +# The implementation is based on mtcnn, available at https://github.com/TropComplique/mtcnn-pytorch +import numpy as np +from PIL import Image + + +def nms(boxes, overlap_threshold=0.5, mode='union'): + """Non-maximum suppression. + + Arguments: + boxes: a float numpy array of shape [n, 5], + where each row is (xmin, ymin, xmax, ymax, score). + overlap_threshold: a float number. + mode: 'union' or 'min'. + + Returns: + list with indices of the selected boxes + """ + + # if there are no boxes, return the empty list + if len(boxes) == 0: + return [] + + # list of picked indices + pick = [] + + # grab the coordinates of the bounding boxes + x1, y1, x2, y2, score = [boxes[:, i] for i in range(5)] + + area = (x2 - x1 + 1.0) * (y2 - y1 + 1.0) + ids = np.argsort(score) # in increasing order + + while len(ids) > 0: + + # grab index of the largest value + last = len(ids) - 1 + i = ids[last] + pick.append(i) + + # compute intersections + # of the box with the largest score + # with the rest of boxes + + # left top corner of intersection boxes + ix1 = np.maximum(x1[i], x1[ids[:last]]) + iy1 = np.maximum(y1[i], y1[ids[:last]]) + + # right bottom corner of intersection boxes + ix2 = np.minimum(x2[i], x2[ids[:last]]) + iy2 = np.minimum(y2[i], y2[ids[:last]]) + + # width and height of intersection boxes + w = np.maximum(0.0, ix2 - ix1 + 1.0) + h = np.maximum(0.0, iy2 - iy1 + 1.0) + + # intersections' areas + inter = w * h + if mode == 'min': + overlap = inter / np.minimum(area[i], area[ids[:last]]) + elif mode == 'union': + # intersection over union (IoU) + overlap = inter / (area[i] + area[ids[:last]] - inter) + + # delete all boxes where overlap is too big + ids = np.delete( + ids, + np.concatenate([[last], + np.where(overlap > overlap_threshold)[0]])) + + return pick + + +def convert_to_square(bboxes): + """Convert bounding boxes to a square form. + + Arguments: + bboxes: a float numpy array of shape [n, 5]. + + Returns: + a float numpy array of shape [n, 5], + squared bounding boxes. + """ + + square_bboxes = np.zeros_like(bboxes) + x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] + h = y2 - y1 + 1.0 + w = x2 - x1 + 1.0 + max_side = np.maximum(h, w) + square_bboxes[:, 0] = x1 + w * 0.5 - max_side * 0.5 + square_bboxes[:, 1] = y1 + h * 0.5 - max_side * 0.5 + square_bboxes[:, 2] = square_bboxes[:, 0] + max_side - 1.0 + square_bboxes[:, 3] = square_bboxes[:, 1] + max_side - 1.0 + return square_bboxes + + +def calibrate_box(bboxes, offsets): + """Transform bounding boxes to be more like true bounding boxes. + 'offsets' is one of the outputs of the nets. + + Arguments: + bboxes: a float numpy array of shape [n, 5]. + offsets: a float numpy array of shape [n, 4]. + + Returns: + a float numpy array of shape [n, 5]. + """ + x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] + w = x2 - x1 + 1.0 + h = y2 - y1 + 1.0 + w = np.expand_dims(w, 1) + h = np.expand_dims(h, 1) + + # this is what happening here: + # tx1, ty1, tx2, ty2 = [offsets[:, i] for i in range(4)] + # x1_true = x1 + tx1*w + # y1_true = y1 + ty1*h + # x2_true = x2 + tx2*w + # y2_true = y2 + ty2*h + # below is just more compact form of this + + # are offsets always such that + # x1 < x2 and y1 < y2 ? + + translation = np.hstack([w, h, w, h]) * offsets + bboxes[:, 0:4] = bboxes[:, 0:4] + translation + return bboxes + + +def get_image_boxes(bounding_boxes, img, size=24): + """Cut out boxes from the image. + + Arguments: + bounding_boxes: a float numpy array of shape [n, 5]. + img: an instance of PIL.Image. + size: an integer, size of cutouts. + + Returns: + a float numpy array of shape [n, 3, size, size]. + """ + + num_boxes = len(bounding_boxes) + width, height = img.size + + [dy, edy, dx, edx, y, ey, x, ex, w, + h] = correct_bboxes(bounding_boxes, width, height) + img_boxes = np.zeros((num_boxes, 3, size, size), 'float32') + + for i in range(num_boxes): + img_box = np.zeros((h[i], w[i], 3), 'uint8') + + img_array = np.asarray(img, 'uint8') + img_box[dy[i]:(edy[i] + 1), dx[i]:(edx[i] + 1), :] =\ + img_array[y[i]:(ey[i] + 1), x[i]:(ex[i] + 1), :] + + # resize + img_box = Image.fromarray(img_box) + img_box = img_box.resize((size, size), Image.BILINEAR) + img_box = np.asarray(img_box, 'float32') + + img_boxes[i, :, :, :] = _preprocess(img_box) + + return img_boxes + + +def correct_bboxes(bboxes, width, height): + """Crop boxes that are too big and get coordinates + with respect to cutouts. + + Arguments: + bboxes: a float numpy array of shape [n, 5], + where each row is (xmin, ymin, xmax, ymax, score). + width: a float number. + height: a float number. + + Returns: + dy, dx, edy, edx: a int numpy arrays of shape [n], + coordinates of the boxes with respect to the cutouts. + y, x, ey, ex: a int numpy arrays of shape [n], + corrected ymin, xmin, ymax, xmax. + h, w: a int numpy arrays of shape [n], + just heights and widths of boxes. + + in the following order: + [dy, edy, dx, edx, y, ey, x, ex, w, h]. + """ + + x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] + w, h = x2 - x1 + 1.0, y2 - y1 + 1.0 + num_boxes = bboxes.shape[0] + + # 'e' stands for end + # (x, y) -> (ex, ey) + x, y, ex, ey = x1, y1, x2, y2 + + # we need to cut out a box from the image. + # (x, y, ex, ey) are corrected coordinates of the box + # in the image. + # (dx, dy, edx, edy) are coordinates of the box in the cutout + # from the image. + dx, dy = np.zeros((num_boxes, )), np.zeros((num_boxes, )) + edx, edy = w.copy() - 1.0, h.copy() - 1.0 + + # if box's bottom right corner is too far right + ind = np.where(ex > width - 1.0)[0] + edx[ind] = w[ind] + width - 2.0 - ex[ind] + ex[ind] = width - 1.0 + + # if box's bottom right corner is too low + ind = np.where(ey > height - 1.0)[0] + edy[ind] = h[ind] + height - 2.0 - ey[ind] + ey[ind] = height - 1.0 + + # if box's top left corner is too far left + ind = np.where(x < 0.0)[0] + dx[ind] = 0.0 - x[ind] + x[ind] = 0.0 + + # if box's top left corner is too high + ind = np.where(y < 0.0)[0] + dy[ind] = 0.0 - y[ind] + y[ind] = 0.0 + + return_list = [dy, edy, dx, edx, y, ey, x, ex, w, h] + return_list = [i.astype('int32') for i in return_list] + + return return_list + + +def _preprocess(img): + """Preprocessing step before feeding the network. + + Arguments: + img: a float numpy array of shape [h, w, c]. + + Returns: + a float numpy array of shape [1, c, h, w]. + """ + img = img.transpose((2, 0, 1)) + img = np.expand_dims(img, 0) + img = (img - 127.5) * 0.0078125 + return img diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/detector.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/detector.py new file mode 100644 index 0000000000000000000000000000000000000000..dde8a5e5f8ec9c47e1de42939e15dbbec65293fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/detector.py @@ -0,0 +1,148 @@ +# The implementation is based on mtcnn, available at https://github.com/TropComplique/mtcnn-pytorch +import os + +import numpy as np +import torch +import torch.backends.cudnn as cudnn +from PIL import Image +from torch.autograd import Variable + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from .box_utils import calibrate_box, convert_to_square, get_image_boxes, nms +from .first_stage import run_first_stage +from .get_nets import ONet, PNet, RNet + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.mtcnn) +class MtcnnFaceDetector(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + + self.pnet = PNet(model_path=os.path.join(self.model_path, 'pnet.npy')) + self.rnet = RNet(model_path=os.path.join(self.model_path, 'rnet.npy')) + self.onet = ONet(model_path=os.path.join(self.model_path, 'onet.npy')) + + self.pnet = self.pnet.to(device) + self.rnet = self.rnet.to(device) + self.onet = self.onet.to(device) + + def forward(self, input): + image = Image.fromarray(np.uint8(input['img'].cpu().numpy())) + pnet = self.pnet + rnet = self.rnet + onet = self.onet + onet.eval() + + min_face_size = 20.0 + thresholds = [0.7, 0.8, 0.9] + nms_thresholds = [0.7, 0.7, 0.7] + + # BUILD AN IMAGE PYRAMID + width, height = image.size + min_length = min(height, width) + + min_detection_size = 12 + factor = 0.707 # sqrt(0.5) + + # scales for scaling the image + scales = [] + + m = min_detection_size / min_face_size + min_length *= m + + factor_count = 0 + while min_length > min_detection_size: + scales.append(m * factor**factor_count) + min_length *= factor + factor_count += 1 + + # STAGE 1 + + # it will be returned + bounding_boxes = [] + + # run P-Net on different scales + for s in scales: + boxes = run_first_stage( + image, + pnet, + scale=s, + threshold=thresholds[0], + device=self.device) + bounding_boxes.append(boxes) + + # collect boxes (and offsets, and scores) from different scales + bounding_boxes = [i for i in bounding_boxes if i is not None] + bounding_boxes = np.vstack(bounding_boxes) + + keep = nms(bounding_boxes[:, 0:5], nms_thresholds[0]) + bounding_boxes = bounding_boxes[keep] + + # use offsets predicted by pnet to transform bounding boxes + bounding_boxes = calibrate_box(bounding_boxes[:, 0:5], + bounding_boxes[:, 5:]) + # shape [n_boxes, 5] + + bounding_boxes = convert_to_square(bounding_boxes) + bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) + + # STAGE 2 + + img_boxes = get_image_boxes(bounding_boxes, image, size=24) + img_boxes = Variable(torch.FloatTensor(img_boxes), volatile=True) + output = rnet(img_boxes.to(self.device)) + offsets = output[0].cpu().data.numpy() # shape [n_boxes, 4] + probs = output[1].cpu().data.numpy() # shape [n_boxes, 2] + + keep = np.where(probs[:, 1] > thresholds[1])[0] + bounding_boxes = bounding_boxes[keep] + bounding_boxes[:, 4] = probs[keep, 1].reshape((-1, )) + offsets = offsets[keep] + + keep = nms(bounding_boxes, nms_thresholds[1]) + bounding_boxes = bounding_boxes[keep] + bounding_boxes = calibrate_box(bounding_boxes, offsets[keep]) + bounding_boxes = convert_to_square(bounding_boxes) + bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) + + # STAGE 3 + + img_boxes = get_image_boxes(bounding_boxes, image, size=48) + if len(img_boxes) == 0: + return [], [] + img_boxes = Variable(torch.FloatTensor(img_boxes), volatile=True) + output = onet(img_boxes.to(self.device)) + landmarks = output[0].cpu().data.numpy() # shape [n_boxes, 10] + offsets = output[1].cpu().data.numpy() # shape [n_boxes, 4] + probs = output[2].cpu().data.numpy() # shape [n_boxes, 2] + + keep = np.where(probs[:, 1] > thresholds[2])[0] + bounding_boxes = bounding_boxes[keep] + bounding_boxes[:, 4] = probs[keep, 1].reshape((-1, )) + offsets = offsets[keep] + landmarks = landmarks[keep] + + # compute landmark points + width = bounding_boxes[:, 2] - bounding_boxes[:, 0] + 1.0 + height = bounding_boxes[:, 3] - bounding_boxes[:, 1] + 1.0 + xmin, ymin = bounding_boxes[:, 0], bounding_boxes[:, 1] + landmarks[:, 0:5] = np.expand_dims( + xmin, 1) + np.expand_dims(width, 1) * landmarks[:, 0:5] + landmarks[:, 5:10] = np.expand_dims( + ymin, 1) + np.expand_dims(height, 1) * landmarks[:, 5:10] + + bounding_boxes = calibrate_box(bounding_boxes, offsets) + keep = nms(bounding_boxes, nms_thresholds[2], mode='min') + bounding_boxes = bounding_boxes[keep] + landmarks = landmarks[keep] + landmarks = landmarks.reshape(-1, 2, 5).transpose( + (0, 2, 1)).reshape(-1, 10) + + return bounding_boxes, landmarks diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/first_stage.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/first_stage.py new file mode 100644 index 0000000000000000000000000000000000000000..e2aba47e0dcc4261053f95bc7210a3a10b5ce27f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/first_stage.py @@ -0,0 +1,100 @@ +# The implementation is based on mtcnn, available at https://github.com/TropComplique/mtcnn-pytorch +import math + +import numpy as np +import torch +from PIL import Image +from torch.autograd import Variable + +from .box_utils import _preprocess, nms + + +def run_first_stage(image, net, scale, threshold, device='cuda'): + """Run P-Net, generate bounding boxes, and do NMS. + + Arguments: + image: an instance of PIL.Image. + net: an instance of pytorch's nn.Module, P-Net. + scale: a float number, + scale width and height of the image by this number. + threshold: a float number, + threshold on the probability of a face when generating + bounding boxes from predictions of the net. + + Returns: + a float numpy array of shape [n_boxes, 9], + bounding boxes with scores and offsets (4 + 1 + 4). + """ + + # scale the image and convert it to a float array + width, height = image.size + sw, sh = math.ceil(width * scale), math.ceil(height * scale) + img = image.resize((sw, sh), Image.BILINEAR) + img = np.asarray(img, 'float32') + + img = Variable( + torch.FloatTensor(_preprocess(img)), volatile=True).to(device) + output = net(img) + probs = output[1].cpu().data.numpy()[0, 1, :, :] + offsets = output[0].cpu().data.numpy() + # probs: probability of a face at each sliding window + # offsets: transformations to true bounding boxes + + boxes = _generate_bboxes(probs, offsets, scale, threshold) + if len(boxes) == 0: + return None + + keep = nms(boxes[:, 0:5], overlap_threshold=0.5) + return boxes[keep] + + +def _generate_bboxes(probs, offsets, scale, threshold): + """Generate bounding boxes at places + where there is probably a face. + + Arguments: + probs: a float numpy array of shape [n, m]. + offsets: a float numpy array of shape [1, 4, n, m]. + scale: a float number, + width and height of the image were scaled by this number. + threshold: a float number. + + Returns: + a float numpy array of shape [n_boxes, 9] + """ + + # applying P-Net is equivalent, in some sense, to + # moving 12x12 window with stride 2 + stride = 2 + cell_size = 12 + + # indices of boxes where there is probably a face + inds = np.where(probs > threshold) + + if inds[0].size == 0: + return np.array([]) + + # transformations of bounding boxes + tx1, ty1, tx2, ty2 = [offsets[0, i, inds[0], inds[1]] for i in range(4)] + # they are defined as: + # w = x2 - x1 + 1 + # h = y2 - y1 + 1 + # x1_true = x1 + tx1*w + # x2_true = x2 + tx2*w + # y1_true = y1 + ty1*h + # y2_true = y2 + ty2*h + + offsets = np.array([tx1, ty1, tx2, ty2]) + score = probs[inds[0], inds[1]] + + # P-Net is applied to scaled images + # so we need to rescale bounding boxes back + bounding_boxes = np.vstack([ + np.round((stride * inds[1] + 1.0) / scale), + np.round((stride * inds[0] + 1.0) / scale), + np.round((stride * inds[1] + 1.0 + cell_size) / scale), + np.round((stride * inds[0] + 1.0 + cell_size) / scale), score, offsets + ]) + # why one is added? + + return bounding_boxes.T diff --git a/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/get_nets.py b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/get_nets.py new file mode 100644 index 0000000000000000000000000000000000000000..5fbbd33b75199d0e19864bc06c87f80530530cc2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/mtcnn/models/get_nets.py @@ -0,0 +1,160 @@ +# The implementation is based on mtcnn, available at https://github.com/TropComplique/mtcnn-pytorch +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Flatten(nn.Module): + + def __init__(self): + super(Flatten, self).__init__() + + def forward(self, x): + """ + Arguments: + x: a float tensor with shape [batch_size, c, h, w]. + Returns: + a float tensor with shape [batch_size, c*h*w]. + """ + + # without this pretrained model isn't working + x = x.transpose(3, 2).contiguous() + + return x.view(x.size(0), -1) + + +class PNet(nn.Module): + + def __init__(self, model_path=None): + + super(PNet, self).__init__() + + # suppose we have input with size HxW, then + # after first layer: H - 2, + # after pool: ceil((H - 2)/2), + # after second conv: ceil((H - 2)/2) - 2, + # after last conv: ceil((H - 2)/2) - 4, + # and the same for W + + self.features = nn.Sequential( + OrderedDict([('conv1', nn.Conv2d(3, 10, 3, 1)), + ('prelu1', nn.PReLU(10)), + ('pool1', nn.MaxPool2d(2, 2, ceil_mode=True)), + ('conv2', nn.Conv2d(10, 16, 3, 1)), + ('prelu2', nn.PReLU(16)), + ('conv3', nn.Conv2d(16, 32, 3, 1)), + ('prelu3', nn.PReLU(32))])) + + self.conv4_1 = nn.Conv2d(32, 2, 1, 1) + self.conv4_2 = nn.Conv2d(32, 4, 1, 1) + + weights = np.load(model_path, allow_pickle=True)[()] + for n, p in self.named_parameters(): + p.data = torch.FloatTensor(weights[n]) + + def forward(self, x): + """ + Arguments: + x: a float tensor with shape [batch_size, 3, h, w]. + Returns: + b: a float tensor with shape [batch_size, 4, h', w']. + a: a float tensor with shape [batch_size, 2, h', w']. + """ + x = self.features(x) + a = self.conv4_1(x) + b = self.conv4_2(x) + a = F.softmax(a) + return b, a + + +class RNet(nn.Module): + + def __init__(self, model_path=None): + + super(RNet, self).__init__() + + self.features = nn.Sequential( + OrderedDict([('conv1', nn.Conv2d(3, 28, 3, 1)), + ('prelu1', nn.PReLU(28)), + ('pool1', nn.MaxPool2d(3, 2, ceil_mode=True)), + ('conv2', nn.Conv2d(28, 48, 3, 1)), + ('prelu2', nn.PReLU(48)), + ('pool2', nn.MaxPool2d(3, 2, ceil_mode=True)), + ('conv3', nn.Conv2d(48, 64, 2, 1)), + ('prelu3', nn.PReLU(64)), ('flatten', Flatten()), + ('conv4', nn.Linear(576, 128)), + ('prelu4', nn.PReLU(128))])) + + self.conv5_1 = nn.Linear(128, 2) + self.conv5_2 = nn.Linear(128, 4) + + weights = np.load(model_path, allow_pickle=True)[()] + for n, p in self.named_parameters(): + p.data = torch.FloatTensor(weights[n]) + + def forward(self, x): + """ + Arguments: + x: a float tensor with shape [batch_size, 3, h, w]. + Returns: + b: a float tensor with shape [batch_size, 4]. + a: a float tensor with shape [batch_size, 2]. + """ + x = self.features(x) + a = self.conv5_1(x) + b = self.conv5_2(x) + a = F.softmax(a) + return b, a + + +class ONet(nn.Module): + + def __init__(self, model_path=None): + + super(ONet, self).__init__() + + self.features = nn.Sequential( + OrderedDict([ + ('conv1', nn.Conv2d(3, 32, 3, 1)), + ('prelu1', nn.PReLU(32)), + ('pool1', nn.MaxPool2d(3, 2, ceil_mode=True)), + ('conv2', nn.Conv2d(32, 64, 3, 1)), + ('prelu2', nn.PReLU(64)), + ('pool2', nn.MaxPool2d(3, 2, ceil_mode=True)), + ('conv3', nn.Conv2d(64, 64, 3, 1)), + ('prelu3', nn.PReLU(64)), + ('pool3', nn.MaxPool2d(2, 2, ceil_mode=True)), + ('conv4', nn.Conv2d(64, 128, 2, 1)), + ('prelu4', nn.PReLU(128)), + ('flatten', Flatten()), + ('conv5', nn.Linear(1152, 256)), + ('drop5', nn.Dropout(0.25)), + ('prelu5', nn.PReLU(256)), + ])) + + self.conv6_1 = nn.Linear(256, 2) + self.conv6_2 = nn.Linear(256, 4) + self.conv6_3 = nn.Linear(256, 10) + + weights = np.load(model_path, allow_pickle=True)[()] + for n, p in self.named_parameters(): + p.data = torch.FloatTensor(weights[n]) + + def forward(self, x): + """ + Arguments: + x: a float tensor with shape [batch_size, 3, h, w]. + Returns: + c: a float tensor with shape [batch_size, 10]. + b: a float tensor with shape [batch_size, 4]. + a: a float tensor with shape [batch_size, 2]. + """ + x = self.features(x) + a = self.conv6_1(x) + b = self.conv6_2(x) + c = self.conv6_3(x) + a = F.softmax(a) + return c, b, a diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/LK/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/LK/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/LK/lk.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/LK/lk.py new file mode 100644 index 0000000000000000000000000000000000000000..3b4a670c43857e488ca6e793ccf7f507276c2c68 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/LK/lk.py @@ -0,0 +1,97 @@ +# The implementation here is modified based on InsightFace_Pytorch, originally Apache License and publicly available +# at https://github.com/610265158/Peppa_Pig_Face_Engine +import numpy as np + + +class GroupTrack(): + + def __init__(self): + self.old_frame = None + self.previous_landmarks_set = None + self.with_landmark = True + self.thres = 1 + self.alpha = 0.95 + self.iou_thres = 0.5 + + def calculate(self, img, current_landmarks_set): + if self.previous_landmarks_set is None: + self.previous_landmarks_set = current_landmarks_set + result = current_landmarks_set + else: + previous_lm_num = self.previous_landmarks_set.shape[0] + if previous_lm_num == 0: + self.previous_landmarks_set = current_landmarks_set + result = current_landmarks_set + return result + else: + result = [] + for i in range(current_landmarks_set.shape[0]): + not_in_flag = True + for j in range(previous_lm_num): + if self.iou(current_landmarks_set[i], + self.previous_landmarks_set[j] + ) > self.iou_thres: + result.append( + self.smooth(current_landmarks_set[i], + self.previous_landmarks_set[j])) + not_in_flag = False + break + if not_in_flag: + result.append(current_landmarks_set[i]) + + result = np.array(result) + self.previous_landmarks_set = result + + return result + + def iou(self, p_set0, p_set1): + rec1 = [ + np.min(p_set0[:, 0]), + np.min(p_set0[:, 1]), + np.max(p_set0[:, 0]), + np.max(p_set0[:, 1]) + ] + rec2 = [ + np.min(p_set1[:, 0]), + np.min(p_set1[:, 1]), + np.max(p_set1[:, 0]), + np.max(p_set1[:, 1]) + ] + + # computing area of each rectangles + S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1]) + S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1]) + + # computing the sum_area + sum_area = S_rec1 + S_rec2 + + # find the each edge of intersect rectangle + x1 = max(rec1[0], rec2[0]) + y1 = max(rec1[1], rec2[1]) + x2 = min(rec1[2], rec2[2]) + y2 = min(rec1[3], rec2[3]) + + # judge if there is an intersect + intersect = max(0, x2 - x1) * max(0, y2 - y1) + + iou = intersect / (sum_area - intersect) + return iou + + def smooth(self, now_landmarks, previous_landmarks): + result = [] + for i in range(now_landmarks.shape[0]): + x = now_landmarks[i][0] - previous_landmarks[i][0] + y = now_landmarks[i][1] - previous_landmarks[i][1] + dis = np.sqrt(np.square(x) + np.square(y)) + if dis < self.thres: + result.append(previous_landmarks[i]) + else: + result.append( + self.do_moving_average(now_landmarks[i], + previous_landmarks[i])) + + return np.array(result) + + def do_moving_average(self, p_now, p_previous): + p = self.alpha * p_now + (1 - self.alpha) * p_previous + return p diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_detector.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..4deff9bf94779e488e2684310248c349a0ac91f8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_detector.py @@ -0,0 +1,115 @@ +# The implementation here is modified based on InsightFace_Pytorch, originally Apache License and publicly available +# at https://github.com/610265158/Peppa_Pig_Face_Engine +import cv2 +import numpy as np +import tensorflow as tf + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class FaceDetector: + + def __init__(self, dir): + + self.model_path = dir + '/detector.pb' + self.thres = 0.8 + self.input_shape = (512, 512, 3) + self.pixel_means = np.array([123., 116., 103.]) + + self._graph = tf.Graph() + + with self._graph.as_default(): + self._graph, self._sess = self.init_model(self.model_path) + + self.input_image = tf.get_default_graph().get_tensor_by_name( + 'tower_0/images:0') + self.training = tf.get_default_graph().get_tensor_by_name( + 'training_flag:0') + self.output_ops = [ + tf.get_default_graph().get_tensor_by_name('tower_0/boxes:0'), + tf.get_default_graph().get_tensor_by_name('tower_0/scores:0'), + tf.get_default_graph().get_tensor_by_name( + 'tower_0/num_detections:0'), + ] + + def __call__(self, image): + + image, scale_x, scale_y = self.preprocess( + image, + target_width=self.input_shape[1], + target_height=self.input_shape[0]) + + image = np.expand_dims(image, 0) + + boxes, scores, num_boxes = self._sess.run( + self.output_ops, + feed_dict={ + self.input_image: image, + self.training: False + }) + + num_boxes = num_boxes[0] + boxes = boxes[0][:num_boxes] + + scores = scores[0][:num_boxes] + + to_keep = scores > self.thres + boxes = boxes[to_keep] + scores = scores[to_keep] + + y1 = self.input_shape[0] / scale_y + x1 = self.input_shape[1] / scale_x + y2 = self.input_shape[0] / scale_y + x2 = self.input_shape[1] / scale_x + scaler = np.array([y1, x1, y2, x2], dtype='float32') + boxes = boxes * scaler + + scores = np.expand_dims(scores, 0).reshape([-1, 1]) + + for i in range(boxes.shape[0]): + boxes[i] = np.array( + [boxes[i][1], boxes[i][0], boxes[i][3], boxes[i][2]]) + return np.concatenate([boxes, scores], axis=1) + + def preprocess(self, image, target_height, target_width, label=None): + + h, w, c = image.shape + + bimage = np.zeros( + shape=[target_height, target_width, c], + dtype=image.dtype) + np.array( + self.pixel_means, dtype=image.dtype) + long_side = max(h, w) + + scale_x = scale_y = target_height / long_side + + image = cv2.resize(image, None, fx=scale_x, fy=scale_y) + + h_, w_, _ = image.shape + bimage[:h_, :w_, :] = image + + return bimage, scale_x, scale_y + + def init_model(self, *args): + pb_path = args[0] + + def init_pb(model_path): + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.2 + compute_graph = tf.Graph() + compute_graph.as_default() + sess = tf.Session(config=config) + with tf.gfile.GFile(model_path, 'rb') as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + tf.import_graph_def(graph_def, name='') + + return (compute_graph, sess) + + model = init_pb(pb_path) + + graph = model[0] + sess = model[1] + + return graph, sess diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_landmark.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_landmark.py new file mode 100644 index 0000000000000000000000000000000000000000..e7e2ddaf76265bbc653e28b96753a5dbaa4e64e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/face_landmark.py @@ -0,0 +1,154 @@ +# The implementation here is modified based on InsightFace_Pytorch, originally Apache License and publicly available +# at https://github.com/610265158/Peppa_Pig_Face_Engine +import cv2 +import numpy as np +import tensorflow as tf + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class FaceLandmark: + + def __init__(self, dir): + self.model_path = dir + '/keypoints.pb' + self.min_face = 60 + self.keypoint_num = 136 + self.pixel_means = np.array([123., 116., 103.]) + self.kp_extend_range = [0.2, 0.3] + self.kp_shape = (160, 160, 3) + + self._graph = tf.Graph() + + with self._graph.as_default(): + + self._graph, self._sess = self.init_model(self.model_path) + self.img_input = tf.get_default_graph().get_tensor_by_name( + 'tower_0/images:0') + self.embeddings = tf.get_default_graph().get_tensor_by_name( + 'tower_0/prediction:0') + self.training = tf.get_default_graph().get_tensor_by_name( + 'training_flag:0') + + self.landmark = self.embeddings[:, :self.keypoint_num] + self.headpose = self.embeddings[:, -7:-4] * 90. + self.state = tf.nn.sigmoid(self.embeddings[:, -4:]) + + def __call__(self, img, bboxes): + landmark_result = [] + state_result = [] + for i, bbox in enumerate(bboxes): + landmark, state = self._one_shot_run(img, bbox, i) + if landmark is not None: + landmark_result.append(landmark) + state_result.append(state) + return np.array(landmark_result), np.array(state_result) + + def simple_run(self, cropped_img): + with self._graph.as_default(): + + cropped_img = np.expand_dims(cropped_img, axis=0) + landmark, p, states = self._sess.run( + [self.landmark, self.headpose, self.state], + feed_dict={ + self.img_input: cropped_img, + self.training: False + }) + + return landmark, states + + def _one_shot_run(self, image, bbox, i): + + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + if (bbox_width <= self.min_face and bbox_height <= self.min_face): + return None, None + add = int(max(bbox_width, bbox_height)) + bimg = cv2.copyMakeBorder( + image, + add, + add, + add, + add, + borderType=cv2.BORDER_CONSTANT, + value=self.pixel_means) + bbox += add + + one_edge = (1 + 2 * self.kp_extend_range[0]) * bbox_width + center = [(bbox[0] + bbox[2]) // 2, (bbox[1] + bbox[3]) // 2] + + bbox[0] = center[0] - one_edge // 2 + bbox[1] = center[1] - one_edge // 2 + bbox[2] = center[0] + one_edge // 2 + bbox[3] = center[1] + one_edge // 2 + + bbox = bbox.astype(int) + crop_image = bimg[bbox[1]:bbox[3], bbox[0]:bbox[2], :] + h, w, _ = crop_image.shape + crop_image = cv2.resize(crop_image, + (self.kp_shape[1], self.kp_shape[0])) + crop_image = crop_image.astype(np.float32) + + keypoints, state = self.simple_run(crop_image) + + res = keypoints[0][:self.keypoint_num].reshape((-1, 2)) + res[:, 0] = res[:, 0] * w / self.kp_shape[1] + res[:, 1] = res[:, 1] * h / self.kp_shape[0] + + landmark = [] + for _index in range(res.shape[0]): + x_y = res[_index] + landmark.append([ + int(x_y[0] * self.kp_shape[0] + bbox[0] - add), + int(x_y[1] * self.kp_shape[1] + bbox[1] - add) + ]) + + landmark = np.array(landmark, np.float32) + + return landmark, state + + def init_model(self, *args): + + if len(args) == 1: + use_pb = True + pb_path = args[0] + else: + use_pb = False + meta_path = args[0] + restore_model_path = args[1] + + def ini_ckpt(): + graph = tf.Graph() + graph.as_default() + configProto = tf.ConfigProto() + configProto.gpu_options.allow_growth = True + sess = tf.Session(config=configProto) + # load_model(model_path, sess) + saver = tf.train.import_meta_graph(meta_path) + saver.restore(sess, restore_model_path) + + print('Model restored!') + return (graph, sess) + + def init_pb(model_path): + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.2 + compute_graph = tf.Graph() + compute_graph.as_default() + sess = tf.Session(config=config) + with tf.gfile.GFile(model_path, 'rb') as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + tf.import_graph_def(graph_def, name='') + + return (compute_graph, sess) + + if use_pb: + model = init_pb(pb_path) + else: + model = ini_ckpt() + + graph = model[0] + sess = model[1] + + return graph, sess diff --git a/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/facer.py b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/facer.py new file mode 100644 index 0000000000000000000000000000000000000000..16039d83eefe5a924cf0b89b53b08947d778989c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/peppa_pig_face/facer.py @@ -0,0 +1,138 @@ +# The implementation here is modified based on InsightFace_Pytorch, originally Apache License and publicly available +# at https://github.com/610265158/Peppa_Pig_Face_Engine +import cv2 +import numpy as np + +from .face_detector import FaceDetector +from .face_landmark import FaceLandmark +from .LK.lk import GroupTrack + + +class FaceAna(): + + def __init__(self, model_dir): + self.face_detector = FaceDetector(model_dir) + self.face_landmark = FaceLandmark(model_dir) + self.trace = GroupTrack() + + self.track_box = None + self.previous_image = None + self.previous_box = None + + self.diff_thres = 5 + self.top_k = 10 + self.iou_thres = 0.5 + self.alpha = 0.3 + + def run(self, image): + + boxes = self.face_detector(image) + + if boxes.shape[0] > self.top_k: + boxes = self.sort(boxes) + + boxes_return = np.array(boxes) + landmarks, states = self.face_landmark(image, boxes) + + if 1: + track = [] + for i in range(landmarks.shape[0]): + track.append([ + np.min(landmarks[i][:, 0]), + np.min(landmarks[i][:, 1]), + np.max(landmarks[i][:, 0]), + np.max(landmarks[i][:, 1]) + ]) + tmp_box = np.array(track) + + self.track_box = self.judge_boxs(boxes_return, tmp_box) + + self.track_box, landmarks = self.sort_res(self.track_box, landmarks) + return self.track_box, landmarks, states + + def sort_res(self, bboxes, points): + area = [] + for bbox in bboxes: + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + area.append(bbox_height * bbox_width) + + area = np.array(area) + picked = area.argsort()[::-1] + sorted_bboxes = [bboxes[x] for x in picked] + sorted_points = [points[x] for x in picked] + return np.array(sorted_bboxes), np.array(sorted_points) + + def diff_frames(self, previous_frame, image): + if previous_frame is None: + return True + else: + _diff = cv2.absdiff(previous_frame, image) + diff = np.sum( + _diff) / previous_frame.shape[0] / previous_frame.shape[1] / 3. + return diff > self.diff_thres + + def sort(self, bboxes): + if self.top_k > 100: + return bboxes + area = [] + for bbox in bboxes: + + bbox_width = bbox[2] - bbox[0] + bbox_height = bbox[3] - bbox[1] + area.append(bbox_height * bbox_width) + + area = np.array(area) + + picked = area.argsort()[-self.top_k:][::-1] + sorted_bboxes = [bboxes[x] for x in picked] + return np.array(sorted_bboxes) + + def judge_boxs(self, previuous_bboxs, now_bboxs): + + def iou(rec1, rec2): + + S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1]) + S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1]) + + sum_area = S_rec1 + S_rec2 + + x1 = max(rec1[0], rec2[0]) + y1 = max(rec1[1], rec2[1]) + x2 = min(rec1[2], rec2[2]) + y2 = min(rec1[3], rec2[3]) + + intersect = max(0, x2 - x1) * max(0, y2 - y1) + + return intersect / (sum_area - intersect) + + if previuous_bboxs is None: + return now_bboxs + + result = [] + + for i in range(now_bboxs.shape[0]): + contain = False + for j in range(previuous_bboxs.shape[0]): + if iou(now_bboxs[i], previuous_bboxs[j]) > self.iou_thres: + result.append( + self.smooth(now_bboxs[i], previuous_bboxs[j])) + contain = True + break + if not contain: + result.append(now_bboxs[i]) + + return np.array(result) + + def smooth(self, now_box, previous_box): + + return self.do_moving_average(now_box[:4], previous_box[:4]) + + def do_moving_average(self, p_now, p_previous): + p = self.alpha * p_now + (1 - self.alpha) * p_previous + return p + + def reset(self): + self.track_box = None + self.previous_image = None + self.previous_box = None diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7b589a1c5cca70d97a47a30c7d99e3fbfd6db9b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .detection import RetinaFaceDetection diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/detection.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/detection.py new file mode 100644 index 0000000000000000000000000000000000000000..7de2da585432a14041de4fbcdf93d4973f64dfe4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/detection.py @@ -0,0 +1,136 @@ +# The implementation is based on resnet, available at https://github.com/biubug6/Pytorch_Retinaface +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .models.retinaface import RetinaFace +from .utils import PriorBox, decode, decode_landm, py_cpu_nms + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.retinaface) +class RetinaFaceDetection(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.cfg = Config.from_file( + model_path.replace(ModelFile.TORCH_MODEL_FILE, + ModelFile.CONFIGURATION))['models'] + self.net = RetinaFace(cfg=self.cfg) + self.load_model() + self.device = device + self.net = self.net.to(self.device) + + self.mean = torch.tensor([[[[104]], [[117]], [[123]]]]).to(device) + + def check_keys(self, pretrained_state_dict): + ckpt_keys = set(pretrained_state_dict.keys()) + model_keys = set(self.net.state_dict().keys()) + used_pretrained_keys = model_keys & ckpt_keys + assert len( + used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint' + return True + + def remove_prefix(self, state_dict, prefix): + new_state_dict = dict() + for k, v in state_dict.items(): + if k.startswith(prefix): + new_state_dict[k[len(prefix):]] = v + else: + new_state_dict[k] = v + return new_state_dict + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu')) + if 'state_dict' in pretrained_dict.keys(): + pretrained_dict = self.remove_prefix(pretrained_dict['state_dict'], + 'module.') + else: + pretrained_dict = self.remove_prefix(pretrained_dict, 'module.') + self.check_keys(pretrained_dict) + self.net.load_state_dict(pretrained_dict, strict=False) + self.net.eval() + + def forward(self, input): + img_raw = input['img'].cpu().numpy() + img = np.float32(img_raw) + + im_height, im_width = img.shape[:2] + ss = 1.0 + # tricky + if max(im_height, im_width) > 1500: + ss = 1000.0 / max(im_height, im_width) + img = cv2.resize(img, (0, 0), fx=ss, fy=ss) + im_height, im_width = img.shape[:2] + + scale = torch.Tensor( + [img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) + img -= (104, 117, 123) + img = img.transpose(2, 0, 1) + img = torch.from_numpy(img).unsqueeze(0) + img = img.to(self.device) + scale = scale.to(self.device) + + loc, conf, landms = self.net(img) # forward pass + del img + + confidence_threshold = 0.9 + nms_threshold = 0.4 + top_k = 5000 + keep_top_k = 750 + + priorbox = PriorBox(self.cfg, image_size=(im_height, im_width)) + priors = priorbox.forward() + priors = priors.to(self.device) + prior_data = priors.data + boxes = decode(loc.data.squeeze(0), prior_data, self.cfg['variance']) + boxes = boxes * scale + boxes = boxes.cpu().numpy() + scores = conf.squeeze(0).data.cpu().numpy()[:, 1] + landms = decode_landm( + landms.data.squeeze(0), prior_data, self.cfg['variance']) + scale1 = torch.Tensor([ + im_width, im_height, im_width, im_height, im_width, im_height, + im_width, im_height, im_width, im_height + ]) + scale1 = scale1.to(self.device) + landms = landms * scale1 + landms = landms.cpu().numpy() + + # ignore low scores + inds = np.where(scores > confidence_threshold)[0] + boxes = boxes[inds] + landms = landms[inds] + scores = scores[inds] + + # keep top-K before NMS + order = scores.argsort()[::-1][:top_k] + boxes = boxes[order] + landms = landms[order] + scores = scores[order] + + # do NMS + dets = np.hstack((boxes, scores[:, np.newaxis])).astype( + np.float32, copy=False) + keep = py_cpu_nms(dets, nms_threshold) + dets = dets[keep, :] + landms = landms[keep] + + # keep top-K faster NMS + dets = dets[:keep_top_k, :] + landms = landms[:keep_top_k, :] + + landms = landms.reshape((-1, 5, 2)) + landms = landms.reshape( + -1, + 10, + ) + return dets / ss, landms / ss diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/net.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/net.py new file mode 100644 index 0000000000000000000000000000000000000000..3be7c4b9ab5e3dacc3fc07a3dd80e52c2c832b28 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/net.py @@ -0,0 +1,149 @@ +# The implementation is based on resnet, available at https://github.com/biubug6/Pytorch_Retinaface +import time + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models +import torchvision.models._utils as _utils +from torch.autograd import Variable + + +def conv_bn(inp, oup, stride=1, leaky=0): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True)) + + +def conv_bn_no_relu(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + ) + + +def conv_bn1X1(inp, oup, stride, leaky=0): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, stride, padding=0, bias=False), + nn.BatchNorm2d(oup), nn.LeakyReLU(negative_slope=leaky, inplace=True)) + + +def conv_dw(inp, oup, stride, leaky=0.1): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + ) + + +class SSH(nn.Module): + + def __init__(self, in_channel, out_channel): + super(SSH, self).__init__() + assert out_channel % 4 == 0 + leaky = 0 + if (out_channel <= 64): + leaky = 0.1 + self.conv3X3 = conv_bn_no_relu(in_channel, out_channel // 2, stride=1) + + self.conv5X5_1 = conv_bn( + in_channel, out_channel // 4, stride=1, leaky=leaky) + self.conv5X5_2 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + self.conv7X7_2 = conv_bn( + out_channel // 4, out_channel // 4, stride=1, leaky=leaky) + self.conv7x7_3 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + def forward(self, input): + conv3X3 = self.conv3X3(input) + + conv5X5_1 = self.conv5X5_1(input) + conv5X5 = self.conv5X5_2(conv5X5_1) + + conv7X7_2 = self.conv7X7_2(conv5X5_1) + conv7X7 = self.conv7x7_3(conv7X7_2) + + out = torch.cat([conv3X3, conv5X5, conv7X7], dim=1) + out = F.relu(out) + return out + + +class FPN(nn.Module): + + def __init__(self, in_channels_list, out_channels): + super(FPN, self).__init__() + leaky = 0 + if (out_channels <= 64): + leaky = 0.1 + self.output1 = conv_bn1X1( + in_channels_list[0], out_channels, stride=1, leaky=leaky) + self.output2 = conv_bn1X1( + in_channels_list[1], out_channels, stride=1, leaky=leaky) + self.output3 = conv_bn1X1( + in_channels_list[2], out_channels, stride=1, leaky=leaky) + + self.merge1 = conv_bn(out_channels, out_channels, leaky=leaky) + self.merge2 = conv_bn(out_channels, out_channels, leaky=leaky) + + def forward(self, input): + # names = list(input.keys()) + input = list(input.values()) + + output1 = self.output1(input[0]) + output2 = self.output2(input[1]) + output3 = self.output3(input[2]) + + up3 = F.interpolate( + output3, size=[output2.size(2), output2.size(3)], mode='nearest') + output2 = output2 + up3 + output2 = self.merge2(output2) + + up2 = F.interpolate( + output2, size=[output1.size(2), output1.size(3)], mode='nearest') + output1 = output1 + up2 + output1 = self.merge1(output1) + + out = [output1, output2, output3] + return out + + +class MobileNetV1(nn.Module): + + def __init__(self): + super(MobileNetV1, self).__init__() + self.stage1 = nn.Sequential( + conv_bn(3, 8, 2, leaky=0.1), # 3 + conv_dw(8, 16, 1), # 7 + conv_dw(16, 32, 2), # 11 + conv_dw(32, 32, 1), # 19 + conv_dw(32, 64, 2), # 27 + conv_dw(64, 64, 1), # 43 + ) + self.stage2 = nn.Sequential( + conv_dw(64, 128, 2), # 43 + 16 = 59 + conv_dw(128, 128, 1), # 59 + 32 = 91 + conv_dw(128, 128, 1), # 91 + 32 = 123 + conv_dw(128, 128, 1), # 123 + 32 = 155 + conv_dw(128, 128, 1), # 155 + 32 = 187 + conv_dw(128, 128, 1), # 187 + 32 = 219 + ) + self.stage3 = nn.Sequential( + conv_dw(128, 256, 2), # 219 +3 2 = 241 + conv_dw(256, 256, 1), # 241 + 64 = 301 + ) + self.avg = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(256, 1000) + + def forward(self, x): + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.avg(x) + x = x.view(-1, 256) + x = self.fc(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/retinaface.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/retinaface.py new file mode 100644 index 0000000000000000000000000000000000000000..8d2001dd7d3b034211d3ab6b91e2a3e42f0fd08a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/models/retinaface.py @@ -0,0 +1,145 @@ +# The implementation is based on resnet, available at https://github.com/biubug6/Pytorch_Retinaface +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models +import torchvision.models._utils as _utils +import torchvision.models.detection.backbone_utils as backbone_utils + +from .net import FPN, SSH, MobileNetV1 + + +class ClassHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(ClassHead, self).__init__() + self.num_anchors = num_anchors + self.conv1x1 = nn.Conv2d( + inchannels, + self.num_anchors * 2, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 2) + + +class BboxHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(BboxHead, self).__init__() + self.conv1x1 = nn.Conv2d( + inchannels, + num_anchors * 4, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 4) + + +class LandmarkHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(LandmarkHead, self).__init__() + self.conv1x1 = nn.Conv2d( + inchannels, + num_anchors * 10, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 10) + + +class RetinaFace(nn.Module): + + def __init__(self, cfg=None): + """ + :param cfg: Network related settings. + """ + super(RetinaFace, self).__init__() + backbone = None + if cfg['name'] == 'Resnet50': + backbone = models.resnet50(pretrained=cfg['pretrain']) + else: + raise Exception('Invalid name') + + self.body = _utils.IntermediateLayerGetter(backbone, + cfg['return_layers']) + in_channels_stage2 = cfg['in_channel'] + in_channels_list = [ + in_channels_stage2 * 2, + in_channels_stage2 * 4, + in_channels_stage2 * 8, + ] + out_channels = cfg['out_channel'] + self.fpn = FPN(in_channels_list, out_channels) + self.ssh1 = SSH(out_channels, out_channels) + self.ssh2 = SSH(out_channels, out_channels) + self.ssh3 = SSH(out_channels, out_channels) + + self.ClassHead = self._make_class_head( + fpn_num=3, inchannels=cfg['out_channel']) + self.BboxHead = self._make_bbox_head( + fpn_num=3, inchannels=cfg['out_channel']) + self.LandmarkHead = self._make_landmark_head( + fpn_num=3, inchannels=cfg['out_channel']) + + def _make_class_head(self, fpn_num=3, inchannels=64, anchor_num=2): + classhead = nn.ModuleList() + for i in range(fpn_num): + classhead.append(ClassHead(inchannels, anchor_num)) + return classhead + + def _make_bbox_head(self, fpn_num=3, inchannels=64, anchor_num=2): + bboxhead = nn.ModuleList() + for i in range(fpn_num): + bboxhead.append(BboxHead(inchannels, anchor_num)) + return bboxhead + + def _make_landmark_head(self, fpn_num=3, inchannels=64, anchor_num=2): + landmarkhead = nn.ModuleList() + for i in range(fpn_num): + landmarkhead.append(LandmarkHead(inchannels, anchor_num)) + return landmarkhead + + def forward(self, inputs): + out = self.body(inputs) + + # FPN + fpn = self.fpn(out) + + # SSH + feature1 = self.ssh1(fpn[0]) + feature2 = self.ssh2(fpn[1]) + feature3 = self.ssh3(fpn[2]) + features = [feature1, feature2, feature3] + + bbox_regressions = torch.cat( + [self.BboxHead[i](feature) for i, feature in enumerate(features)], + dim=1) + classifications = torch.cat( + [self.ClassHead[i](feature) for i, feature in enumerate(features)], + dim=1) + ldm_regressions = torch.cat( + [self.LandmarkHead[i](feat) for i, feat in enumerate(features)], + dim=1) + + output = (bbox_regressions, F.softmax(classifications, + dim=-1), ldm_regressions) + return output diff --git a/runtime/Lib/site_packages/models/cv/face_detection/retinaface/utils.py b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..60c9e2dded749ec77a905a60c1ac3e5535002cc2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/retinaface/utils.py @@ -0,0 +1,123 @@ +# -------------------------------------------------------- +# Modified from https://github.com/biubug6/Pytorch_Retinaface +# -------------------------------------------------------- + +from itertools import product as product +from math import ceil + +import numpy as np +import torch + + +class PriorBox(object): + + def __init__(self, cfg, image_size=None, phase='train'): + super(PriorBox, self).__init__() + self.min_sizes = cfg['min_sizes'] + self.steps = cfg['steps'] + self.clip = cfg['clip'] + self.image_size = image_size + self.feature_maps = [[ + ceil(self.image_size[0] / step), + ceil(self.image_size[1] / step) + ] for step in self.steps] + self.name = 's' + + def forward(self): + anchors = [] + for k, f in enumerate(self.feature_maps): + min_sizes = self.min_sizes[k] + for i, j in product(range(f[0]), range(f[1])): + for min_size in min_sizes: + s_kx = min_size / self.image_size[1] + s_ky = min_size / self.image_size[0] + dense_cx = [ + x * self.steps[k] / self.image_size[1] + for x in [j + 0.5] + ] + dense_cy = [ + y * self.steps[k] / self.image_size[0] + for y in [i + 0.5] + ] + for cy, cx in product(dense_cy, dense_cx): + anchors += [cx, cy, s_kx, s_ky] + + # back to torch land + output = torch.Tensor(anchors).view(-1, 4) + if self.clip: + output.clamp_(max=1, min=0) + return output + + +def py_cpu_nms(dets, thresh): + """Pure Python NMS baseline.""" + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = dets[:, 2] + y2 = dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + + return keep + + +# Adapted from https://github.com/Hakuyume/chainer-ssd +def decode(loc, priors, variances): + """Decode locations from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + loc (tensor): location predictions for loc layers, + Shape: [num_priors,4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat( + (priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def decode_landm(pre, priors, variances): + """Decode landm from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + pre (tensor): landm predictions for loc layers, + Shape: [num_priors,10] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded landm predictions + """ + a = priors[:, :2] + pre[:, :2] * variances[0] * priors[:, 2:] + b = priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:] + c = priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:] + d = priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:] + e = priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:] + landms = torch.cat((a, b, c, d, e), dim=1) + return landms diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..78ec7aa4214d130696eb97a3307e12cd02fad364 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .preprocessor import SCRFDPreprocessor +from .scrfd_detect import ScrfdDetect +from .tinymog_detect import TinyMogDetect diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/damofd_detect.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/damofd_detect.py new file mode 100644 index 0000000000000000000000000000000000000000..c91f32604bcce47eb105be92ae2fab106263b446 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/damofd_detect.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from copy import deepcopy +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .scrfd_detect import ScrfdDetect + +logger = get_logger() + +__all__ = ['DamoFdDetect'] + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.damofd) +class DamoFdDetect(ScrfdDetect): + + def __init__(self, model_dir, *args, **kwargs): + """ + initialize the tinymog face detection model from the `model_dir` path. + """ + config_file = 'DamoFD_lms.py' + kwargs['config_file'] = config_file + kwargs['model_file'] = ModelFile.TORCH_MODEL_FILE + super().__init__(model_dir, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..14fc23592f583a83e2b22e5f80623a68301e92ed --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/__init__.py @@ -0,0 +1,4 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet +""" diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e369df3284224f858b54dd5209e23f111614af7f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/core/bbox +""" +from .transforms import bbox2result, distance2kps, kps2distance + +__all__ = ['bbox2result', 'distance2kps', 'kps2distance'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/transforms.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..7d0f3070d640b747ea5dc8ba9fcd3fbb4da7aa8f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/bbox/transforms.py @@ -0,0 +1,87 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/core/bbox/transforms.py +""" +import numpy as np +import torch + + +def bbox2result(bboxes, labels, num_classes, kps=None, num_kps=5): + """Convert detection results to a list of numpy arrays. + + Args: + bboxes (torch.Tensor | np.ndarray): shape (n, 5) + labels (torch.Tensor | np.ndarray): shape (n, ) + num_classes (int): class number, including background class + + Returns: + list(ndarray): bbox results of each class + """ + bbox_len = 5 if kps is None else 5 + num_kps * 2 # if has kps, add num_kps*2 into bbox + if bboxes.shape[0] == 0: + return [ + np.zeros((0, bbox_len), dtype=np.float32) + for i in range(num_classes) + ] + else: + if isinstance(bboxes, torch.Tensor): + bboxes = bboxes.detach().cpu().numpy() + labels = labels.detach().cpu().numpy() + if kps is None: + return [bboxes[labels == i, :] for i in range(num_classes)] + else: # with kps + if isinstance(kps, torch.Tensor): + kps = kps.detach().cpu().numpy() + return [ + np.hstack([bboxes[labels == i, :], kps[labels == i, :]]) + for i in range(num_classes) + ] + + +def distance2kps(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + + Args: + points (Tensor): Shape (n, 2), [x, y]. + distance (Tensor): Distance from the given point to 4 + boundaries (left, top, right, bottom). + max_shape (tuple): Shape of the image. + + Returns: + Tensor: Decoded kps. + """ + preds = [] + for i in range(0, distance.shape[1], 2): + px = points[:, i % 2] + distance[:, i] + py = points[:, i % 2 + 1] + distance[:, i + 1] + if max_shape is not None: + px = px.clamp(min=0, max=max_shape[1]) + py = py.clamp(min=0, max=max_shape[0]) + preds.append(px) + preds.append(py) + return torch.stack(preds, -1) + + +def kps2distance(points, kps, max_dis=None, eps=0.1): + """Decode bounding box based on distances. + + Args: + points (Tensor): Shape (n, 2), [x, y]. + kps (Tensor): Shape (n, K), "xyxy" format + max_dis (float): Upper bound of the distance. + eps (float): a small value to ensure target < max_dis, instead <= + + Returns: + Tensor: Decoded distances. + """ + + preds = [] + for i in range(0, kps.shape[1], 2): + px = kps[:, i] - points[:, i % 2] + py = kps[:, i + 1] - points[:, i % 2 + 1] + if max_dis is not None: + px = px.clamp(min=0, max=max_dis - eps) + py = py.clamp(min=0, max=max_dis - eps) + preds.append(px) + preds.append(py) + return torch.stack(preds, -1) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7e0e85acef00d7772ab92e57d5864a06598f4a3b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/core/post_processing/bbox_nms.py +""" +from .bbox_nms import multiclass_nms + +__all__ = ['multiclass_nms'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/bbox_nms.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/bbox_nms.py new file mode 100644 index 0000000000000000000000000000000000000000..7f1a2fe95c1dd378787dfd530b6ff7989f51bf07 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/core/post_processing/bbox_nms.py @@ -0,0 +1,89 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/core/post_processing/bbox_nms.py +""" +import torch + + +def multiclass_nms(multi_bboxes, + multi_scores, + score_thr, + nms_cfg, + max_num=-1, + score_factors=None, + return_inds=False, + multi_kps=None): + """NMS for multi-class bboxes. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_kps (Tensor): shape (n, #class*num_kps*2) or (n, num_kps*2) + multi_scores (Tensor): shape (n, #class), where the last column + contains scores of the background class, but this will be ignored. + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + nms_thr (float): NMS IoU threshold + max_num (int, optional): if there are more than max_num bboxes after + NMS, only top max_num will be kept. Default to -1. + score_factors (Tensor, optional): The factors multiplied to scores + before applying NMS. Default to None. + return_inds (bool, optional): Whether return the indices of kept + bboxes. Default to False. + + Returns: + tuple: (bboxes, labels, indices (optional)), tensors of shape (k, 5), + (k), and (k). Labels are 0-based. + """ + num_classes = multi_scores.size(1) - 1 + # exclude background category + kps = None + if multi_kps is not None: + num_kps = int((multi_kps.shape[1] / num_classes) / 2) + if multi_bboxes.shape[1] > 4: + bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4) + if multi_kps is not None: + kps = multi_kps.view(multi_scores.size(0), -1, num_kps * 2) + else: + bboxes = multi_bboxes[:, None].expand( + multi_scores.size(0), num_classes, 4) + if multi_kps is not None: + kps = multi_kps[:, None].expand( + multi_scores.size(0), num_classes, num_kps * 2) + + scores = multi_scores[:, :-1] + if score_factors is not None: + scores = scores * score_factors[:, None] + + labels = torch.arange(num_classes, dtype=torch.long, device=scores.device) + labels = labels.view(1, -1).expand_as(scores) + + bboxes = bboxes.reshape(-1, 4) + if kps is not None: + kps = kps.reshape(-1, num_kps * 2) + scores = scores.reshape(-1) + labels = labels.reshape(-1) + + # remove low scoring boxes + valid_mask = scores > score_thr + inds = valid_mask.nonzero(as_tuple=False).squeeze(1) + bboxes, scores, labels = bboxes[inds], scores[inds], labels[inds] + if kps is not None: + kps = kps[inds] + if inds.numel() == 0: + if torch.onnx.is_in_onnx_export(): + raise RuntimeError('[ONNX Error] Can not record NMS ' + 'as it has not been executed this time') + return bboxes, labels, kps + + # TODO: add size check before feed into batched_nms + from mmcv.ops.nms import batched_nms + dets, keep = batched_nms(bboxes, scores, labels, nms_cfg) + + if max_num > 0: + dets = dets[:max_num] + keep = keep[:max_num] + + if return_inds: + return dets, labels[keep], kps[keep], keep + else: + return dets, labels[keep], kps[keep] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b31cb541843c44cbcbefa4fbca219be1a754fbbc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets +""" +from .retinaface import RetinaFaceDataset + +__all__ = ['RetinaFaceDataset'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..639e29df8bbf9214d9c2806d5bff980c1659d575 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/__init__.py @@ -0,0 +1,13 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/pipelines +""" +from .auto_augment import RotateV2 +from .formating import DefaultFormatBundleV2 +from .loading import LoadAnnotationsV2 +from .transforms import RandomSquareCrop + +__all__ = [ + 'RandomSquareCrop', 'LoadAnnotationsV2', 'RotateV2', + 'DefaultFormatBundleV2' +] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/auto_augment.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/auto_augment.py new file mode 100644 index 0000000000000000000000000000000000000000..93f826c73119ff8a6c55895d2a53ae5729b59377 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/auto_augment.py @@ -0,0 +1,271 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/pipelines/auto_augment.py +""" +import copy + +import cv2 +import mmcv +import numpy as np +from mmdet.datasets.builder import PIPELINES + +_MAX_LEVEL = 10 + + +def level_to_value(level, max_value): + """Map from level to values based on max_value.""" + return (level / _MAX_LEVEL) * max_value + + +def random_negative(value, random_negative_prob): + """Randomly negate value based on random_negative_prob.""" + return -value if np.random.rand() < random_negative_prob else value + + +def bbox2fields(): + """The key correspondence from bboxes to labels, masks and + segmentations.""" + bbox2label = { + 'gt_bboxes': 'gt_labels', + 'gt_bboxes_ignore': 'gt_labels_ignore' + } + bbox2mask = { + 'gt_bboxes': 'gt_masks', + 'gt_bboxes_ignore': 'gt_masks_ignore' + } + bbox2seg = { + 'gt_bboxes': 'gt_semantic_seg', + } + return bbox2label, bbox2mask, bbox2seg + + +@PIPELINES.register_module() +class RotateV2(object): + """Apply Rotate Transformation to image (and its corresponding bbox, mask, + segmentation). + + Args: + level (int | float): The level should be in range (0,_MAX_LEVEL]. + scale (int | float): Isotropic scale factor. Same in + ``mmcv.imrotate``. + center (int | float | tuple[float]): Center point (w, h) of the + rotation in the source image. If None, the center of the + image will be used. Same in ``mmcv.imrotate``. + img_fill_val (int | float | tuple): The fill value for image border. + If float, the same value will be used for all the three + channels of image. If tuple, the should be 3 elements (e.g. + equals the number of channels for image). + seg_ignore_label (int): The fill value used for segmentation map. + Note this value must equals ``ignore_label`` in ``semantic_head`` + of the corresponding config. Default 255. + prob (float): The probability for perform transformation and + should be in range 0 to 1. + max_rotate_angle (int | float): The maximum angles for rotate + transformation. + random_negative_prob (float): The probability that turns the + offset negative. + """ + + def __init__(self, + level, + scale=1, + center=None, + img_fill_val=128, + seg_ignore_label=255, + prob=0.5, + max_rotate_angle=30, + random_negative_prob=0.5): + assert isinstance(level, (int, float)), \ + f'The level must be type int or float. got {type(level)}.' + assert 0 <= level <= _MAX_LEVEL, \ + f'The level should be in range (0,{_MAX_LEVEL}]. got {level}.' + assert isinstance(scale, (int, float)), \ + f'The scale must be type int or float. got type {type(scale)}.' + if isinstance(center, (int, float)): + center = (center, center) + elif isinstance(center, tuple): + assert len(center) == 2, 'center with type tuple must have '\ + f'2 elements. got {len(center)} elements.' + else: + assert center is None, 'center must be None or type int, '\ + f'float or tuple, got type {type(center)}.' + if isinstance(img_fill_val, (float, int)): + img_fill_val = tuple([float(img_fill_val)] * 3) + elif isinstance(img_fill_val, tuple): + assert len(img_fill_val) == 3, 'img_fill_val as tuple must '\ + f'have 3 elements. got {len(img_fill_val)}.' + img_fill_val = tuple([float(val) for val in img_fill_val]) + else: + raise ValueError( + 'img_fill_val must be float or tuple with 3 elements.') + assert np.all([0 <= val <= 255 for val in img_fill_val]), \ + 'all elements of img_fill_val should between range [0,255]. '\ + f'got {img_fill_val}.' + assert 0 <= prob <= 1.0, 'The probability should be in range [0,1]. '\ + f'got {prob}.' + assert isinstance(max_rotate_angle, (int, float)), 'max_rotate_angle '\ + f'should be type int or float. got type {type(max_rotate_angle)}.' + self.level = level + self.scale = scale + # Rotation angle in degrees. Positive values mean + # clockwise rotation. + self.angle = level_to_value(level, max_rotate_angle) + self.center = center + self.img_fill_val = img_fill_val + self.seg_ignore_label = seg_ignore_label + self.prob = prob + self.max_rotate_angle = max_rotate_angle + self.random_negative_prob = random_negative_prob + + def _rotate_img(self, results, angle, center=None, scale=1.0): + """Rotate the image. + + Args: + results (dict): Result dict from loading pipeline. + angle (float): Rotation angle in degrees, positive values + mean clockwise rotation. Same in ``mmcv.imrotate``. + center (tuple[float], optional): Center point (w, h) of the + rotation. Same in ``mmcv.imrotate``. + scale (int | float): Isotropic scale factor. Same in + ``mmcv.imrotate``. + """ + for key in results.get('img_fields', ['img']): + img = results[key].copy() + img_rotated = mmcv.imrotate( + img, angle, center, scale, border_value=self.img_fill_val) + results[key] = img_rotated.astype(img.dtype) + results['img_shape'] = results[key].shape + + def _rotate_bboxes(self, results, rotate_matrix): + """Rotate the bboxes.""" + h, w, c = results['img_shape'] + for key in results.get('bbox_fields', []): + min_x, min_y, max_x, max_y = np.split( + results[key], results[key].shape[-1], axis=-1) + coordinates = np.stack([[min_x, min_y], [max_x, min_y], + [min_x, max_y], + [max_x, max_y]]) # [4, 2, nb_bbox, 1] + # pad 1 to convert from format [x, y] to homogeneous + # coordinates format [x, y, 1] + coordinates = np.concatenate( + (coordinates, + np.ones((4, 1, coordinates.shape[2], 1), coordinates.dtype)), + axis=1) # [4, 3, nb_bbox, 1] + coordinates = coordinates.transpose( + (2, 0, 1, 3)) # [nb_bbox, 4, 3, 1] + rotated_coords = np.matmul(rotate_matrix, + coordinates) # [nb_bbox, 4, 2, 1] + rotated_coords = rotated_coords[..., 0] # [nb_bbox, 4, 2] + min_x, min_y = np.min( + rotated_coords[:, :, 0], axis=1), np.min( + rotated_coords[:, :, 1], axis=1) + max_x, max_y = np.max( + rotated_coords[:, :, 0], axis=1), np.max( + rotated_coords[:, :, 1], axis=1) + results[key] = np.stack([min_x, min_y, max_x, max_y], + axis=-1).astype(results[key].dtype) + + def _rotate_keypoints90(self, results, angle): + """Rotate the keypoints, only valid when angle in [-90,90,-180,180]""" + if angle not in [-90, 90, 180, -180 + ] or self.scale != 1 or self.center is not None: + return + for key in results.get('keypoints_fields', []): + k = results[key] + if angle == 90: + w, h, c = results['img'].shape + new = np.stack([h - k[..., 1], k[..., 0], k[..., 2]], axis=-1) + elif angle == -90: + w, h, c = results['img'].shape + new = np.stack([k[..., 1], w - k[..., 0], k[..., 2]], axis=-1) + else: + h, w, c = results['img'].shape + new = np.stack([w - k[..., 0], h - k[..., 1], k[..., 2]], + axis=-1) + # a kps is invalid if thrid value is -1 + kps_invalid = new[..., -1][:, -1] == -1 + new[kps_invalid] = np.zeros(new.shape[1:]) - 1 + results[key] = new + + def _rotate_masks(self, + results, + angle, + center=None, + scale=1.0, + fill_val=0): + """Rotate the masks.""" + h, w, c = results['img_shape'] + for key in results.get('mask_fields', []): + masks = results[key] + results[key] = masks.rotate((h, w), angle, center, scale, fill_val) + + def _rotate_seg(self, + results, + angle, + center=None, + scale=1.0, + fill_val=255): + """Rotate the segmentation map.""" + for key in results.get('seg_fields', []): + seg = results[key].copy() + results[key] = mmcv.imrotate( + seg, angle, center, scale, + border_value=fill_val).astype(seg.dtype) + + def _filter_invalid(self, results, min_bbox_size=0): + """Filter bboxes and corresponding masks too small after rotate + augmentation.""" + bbox2label, bbox2mask, _ = bbox2fields() + for key in results.get('bbox_fields', []): + bbox_w = results[key][:, 2] - results[key][:, 0] + bbox_h = results[key][:, 3] - results[key][:, 1] + valid_inds = (bbox_w > min_bbox_size) & (bbox_h > min_bbox_size) + valid_inds = np.nonzero(valid_inds)[0] + results[key] = results[key][valid_inds] + # label fields. e.g. gt_labels and gt_labels_ignore + label_key = bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][valid_inds] + # mask fields, e.g. gt_masks and gt_masks_ignore + mask_key = bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][valid_inds] + + def __call__(self, results): + """Call function to rotate images, bounding boxes, masks and semantic + segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Rotated results. + """ + if np.random.rand() > self.prob: + return results + h, w = results['img'].shape[:2] + center = self.center + if center is None: + center = ((w - 1) * 0.5, (h - 1) * 0.5) + angle = random_negative(self.angle, self.random_negative_prob) + self._rotate_img(results, angle, center, self.scale) + rotate_matrix = cv2.getRotationMatrix2D(center, -angle, self.scale) + self._rotate_bboxes(results, rotate_matrix) + self._rotate_keypoints90(results, angle) + self._rotate_masks(results, angle, center, self.scale, fill_val=0) + self._rotate_seg( + results, angle, center, self.scale, fill_val=self.seg_ignore_label) + self._filter_invalid(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(level={self.level}, ' + repr_str += f'scale={self.scale}, ' + repr_str += f'center={self.center}, ' + repr_str += f'img_fill_val={self.img_fill_val}, ' + repr_str += f'seg_ignore_label={self.seg_ignore_label}, ' + repr_str += f'prob={self.prob}, ' + repr_str += f'max_rotate_angle={self.max_rotate_angle}, ' + repr_str += f'random_negative_prob={self.random_negative_prob})' + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/formating.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/formating.py new file mode 100644 index 0000000000000000000000000000000000000000..ccb8545ac6fa61fdc7982e81284af672d361b48d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/formating.py @@ -0,0 +1,113 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/pipelines/formating.py +""" +import numpy as np +import torch +from mmcv.parallel import DataContainer as DC +from mmdet.datasets.builder import PIPELINES + + +def to_tensor(data): + """Convert objects of various python types to :obj:`torch.Tensor`. + + Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`, + :class:`Sequence`, :class:`int` and :class:`float`. + + Args: + data (torch.Tensor | numpy.ndarray | Sequence | int | float): Data to + be converted. + """ + + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, np.ndarray): + return torch.from_numpy(data) + elif isinstance(data, Sequence) and not mmcv.is_str(data): + return torch.tensor(data) + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + else: + raise TypeError(f'type {type(data)} cannot be converted to tensor.') + + +@PIPELINES.register_module() +class DefaultFormatBundleV2(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img", + "proposals", "gt_bboxes", "gt_labels", "gt_masks" and "gt_semantic_seg". + These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - proposals: (1)to tensor, (2)to DataContainer + - gt_bboxes: (1)to tensor, (2)to DataContainer + - gt_bboxes_ignore: (1)to tensor, (2)to DataContainer + - gt_labels: (1)to tensor, (2)to DataContainer + - gt_masks: (1)to tensor, (2)to DataContainer (cpu_only=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, \ + (3)to DataContainer (stack=True) + """ + + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with \ + default bundle. + """ + + if 'img' in results: + img = results['img'] + # add default meta keys + results = self._add_default_meta_keys(results) + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + for key in [ + 'proposals', 'gt_bboxes', 'gt_bboxes_ignore', 'gt_keypointss', + 'gt_labels' + ]: + if key not in results: + continue + results[key] = DC(to_tensor(results[key])) + if 'gt_masks' in results: + results['gt_masks'] = DC(results['gt_masks'], cpu_only=True) + if 'gt_semantic_seg' in results: + results['gt_semantic_seg'] = DC( + to_tensor(results['gt_semantic_seg'][None, ...]), stack=True) + return results + + def _add_default_meta_keys(self, results): + """Add default meta keys. + + We set default meta keys including `pad_shape`, `scale_factor` and + `img_norm_cfg` to avoid the case where no `Resize`, `Normalize` and + `Pad` are implemented during the whole pipeline. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + results (dict): Updated result dict contains the data to convert. + """ + img = results['img'] + results.setdefault('pad_shape', img.shape) + results.setdefault('scale_factor', 1.0) + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results.setdefault( + 'img_norm_cfg', + dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False)) + return results + + def __repr__(self): + return self.__class__.__name__ diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/loading.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..a186ff6aa66809fd957216d3c25584ae81eb2cb4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/loading.py @@ -0,0 +1,225 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/pipelines/loading.py +""" +import os.path as osp + +import numpy as np +import pycocotools.mask as maskUtils +from mmdet.core import BitmapMasks, PolygonMasks +from mmdet.datasets.builder import PIPELINES + + +@PIPELINES.register_module() +class LoadAnnotationsV2(object): + """Load mutiple types of annotations. + + Args: + with_bbox (bool): Whether to parse and load the bbox annotation. + Default: True. + with_label (bool): Whether to parse and load the label annotation. + Default: True. + with_keypoints (bool): Whether to parse and load the keypoints annotation. + Default: False. + with_mask (bool): Whether to parse and load the mask annotation. + Default: False. + with_seg (bool): Whether to parse and load the semantic segmentation + annotation. Default: False. + poly2mask (bool): Whether to convert the instance masks from polygons + to bitmaps. Default: True. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + """ + + def __init__(self, + with_bbox=True, + with_label=True, + with_keypoints=False, + with_mask=False, + with_seg=False, + poly2mask=True, + file_client_args=dict(backend='disk')): + self.with_bbox = with_bbox + self.with_label = with_label + self.with_keypoints = with_keypoints + self.with_mask = with_mask + self.with_seg = with_seg + self.poly2mask = poly2mask + self.file_client_args = file_client_args.copy() + self.file_client = None + + def _load_bboxes(self, results): + """Private function to load bounding box annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded bounding box annotations. + """ + + ann_info = results['ann_info'] + results['gt_bboxes'] = ann_info['bboxes'].copy() + + gt_bboxes_ignore = ann_info.get('bboxes_ignore', None) + if gt_bboxes_ignore is not None: + results['gt_bboxes_ignore'] = gt_bboxes_ignore.copy() + results['bbox_fields'].append('gt_bboxes_ignore') + results['bbox_fields'].append('gt_bboxes') + return results + + def _load_keypoints(self, results): + """Private function to load bounding box annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded bounding box annotations. + """ + + ann_info = results['ann_info'] + results['gt_keypointss'] = ann_info['keypointss'].copy() + + results['keypoints_fields'] = ['gt_keypointss'] + return results + + def _load_labels(self, results): + """Private function to load label annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded label annotations. + """ + + results['gt_labels'] = results['ann_info']['labels'].copy() + return results + + def _poly2mask(self, mask_ann, img_h, img_w): + """Private function to convert masks represented with polygon to + bitmaps. + + Args: + mask_ann (list | dict): Polygon mask annotation input. + img_h (int): The height of output mask. + img_w (int): The width of output mask. + + Returns: + numpy.ndarray: The decode bitmap mask of shape (img_h, img_w). + """ + + if isinstance(mask_ann, list): + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(mask_ann, img_h, img_w) + rle = maskUtils.merge(rles) + elif isinstance(mask_ann['counts'], list): + # uncompressed RLE + rle = maskUtils.frPyObjects(mask_ann, img_h, img_w) + else: + # rle + rle = mask_ann + mask = maskUtils.decode(rle) + return mask + + def process_polygons(self, polygons): + """Convert polygons to list of ndarray and filter invalid polygons. + + Args: + polygons (list[list]): Polygons of one instance. + + Returns: + list[numpy.ndarray]: Processed polygons. + """ + + polygons = [np.array(p) for p in polygons] + valid_polygons = [] + for polygon in polygons: + if len(polygon) % 2 == 0 and len(polygon) >= 6: + valid_polygons.append(polygon) + return valid_polygons + + def _load_masks(self, results): + """Private function to load mask annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded mask annotations. + If ``self.poly2mask`` is set ``True``, `gt_mask` will contain + :obj:`PolygonMasks`. Otherwise, :obj:`BitmapMasks` is used. + """ + + h, w = results['img_info']['height'], results['img_info']['width'] + gt_masks = results['ann_info']['masks'] + if self.poly2mask: + gt_masks = BitmapMasks( + [self._poly2mask(mask, h, w) for mask in gt_masks], h, w) + else: + gt_masks = PolygonMasks( + [self.process_polygons(polygons) for polygons in gt_masks], h, + w) + results['gt_masks'] = gt_masks + results['mask_fields'].append('gt_masks') + return results + + def _load_semantic_seg(self, results): + """Private function to load semantic segmentation annotations. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: The dict contains loaded semantic segmentation annotations. + """ + import mmcv + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + filename = osp.join(results['seg_prefix'], + results['ann_info']['seg_map']) + img_bytes = self.file_client.get(filename) + results['gt_semantic_seg'] = mmcv.imfrombytes( + img_bytes, flag='unchanged').squeeze() + results['seg_fields'].append('gt_semantic_seg') + return results + + def __call__(self, results): + """Call function to load multiple types annotations. + + Args: + results (dict): Result dict from :obj:`mmdet.CustomDataset`. + + Returns: + dict: The dict contains loaded bounding box, label, mask and + semantic segmentation annotations. + """ + + if self.with_bbox: + results = self._load_bboxes(results) + if results is None: + return None + if self.with_label: + results = self._load_labels(results) + if self.with_keypoints: + results = self._load_keypoints(results) + if self.with_mask: + results = self._load_masks(results) + if self.with_seg: + results = self._load_semantic_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(with_bbox={self.with_bbox}, ' + repr_str += f'with_label={self.with_label}, ' + repr_str += f'with_keypoints={self.with_keypoints}, ' + repr_str += f'with_mask={self.with_mask}, ' + repr_str += f'with_seg={self.with_seg})' + repr_str += f'poly2mask={self.poly2mask})' + repr_str += f'poly2mask={self.file_client_args})' + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/transforms.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..45b060823cbf6f341163b77dbfcdb6e8f6ec45ea --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/pipelines/transforms.py @@ -0,0 +1,737 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/pipelines/transforms.py +""" +import mmcv +import numpy as np +from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps +from mmdet.datasets.builder import PIPELINES +from numpy import random + + +@PIPELINES.register_module() +class ResizeV2(object): + """Resize images & bbox & mask &kps. + + This transform resizes the input image to some scale. Bboxes and masks are + then resized with the same scale factor. If the input dict contains the key + "scale", then the scale in the input dict is used, otherwise the specified + scale in the init method is used. If the input dict contains the key + "scale_factor" (if MultiScaleFlipAug does not give img_scale but + scale_factor), the actual scale will be computed by image shape and + scale_factor. + + `img_scale` can either be a tuple (single-scale) or a list of tuple + (multi-scale). There are 3 multiscale modes: + + - ``ratio_range is not None``: randomly sample a ratio from the ratio \ + range and multiply it with the image scale. + - ``ratio_range is None`` and ``multiscale_mode == "range"``: randomly \ + sample a scale from the multiscale range. + - ``ratio_range is None`` and ``multiscale_mode == "value"``: randomly \ + sample a scale from multiple scales. + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + bbox_clip_border (bool, optional): Whether clip the objects outside + the border of the image. Defaults to True. + backend (str): Image resize backend, choices are 'cv2' and 'pillow'. + These two backends generates slightly different results. Defaults + to 'cv2'. + override (bool, optional): Whether to override `scale` and + `scale_factor` so as to call resize twice. Default False. If True, + after the first resizing, the existed `scale` and `scale_factor` + will be ignored so the second resizing can be allowed. + This option is a work-around for multiple times of resize in DETR. + Defaults to False. + """ + + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True, + bbox_clip_border=True, + backend='cv2', + override=False): + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given a scale and a range of image ratio + assert len(self.img_scale) == 1 + else: + # mode 2: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.backend = backend + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + # TODO: refactor the override option in Resize + self.override = override + self.bbox_clip_border = bbox_clip_border + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, \ + where ``img_scale`` is the selected image scale and \ + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and uper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where \ + ``img_scale`` is sampled scale and None is just a placeholder \ + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where \ + ``scale`` is sampled ratio multiplied with ``img_scale`` and \ + None is just a placeholder to be consistent with \ + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into \ + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + for key in results.get('img_fields', ['img']): + if self.keep_ratio: + img, scale_factor = mmcv.imrescale( + results[key], + results['scale'], + return_scale=True, + backend=self.backend) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results[key].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize( + results[key], + results['scale'], + return_scale=True, + backend=self.backend) + results[key] = img + + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img_shape'] = img.shape + # in case that there is no padding + results['pad_shape'] = img.shape + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_bboxes(self, results): + """Resize bounding boxes with ``results['scale_factor']``.""" + for key in results.get('bbox_fields', []): + bboxes = results[key] * results['scale_factor'] + if self.bbox_clip_border: + img_shape = results['img_shape'] + bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, img_shape[1]) + bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, img_shape[0]) + results[key] = bboxes + + def _resize_keypoints(self, results): + """Resize keypoints with ``results['scale_factor']``.""" + for key in results.get('keypoints_fields', []): + keypointss = results[key].copy() + factors = results['scale_factor'] + assert factors[0] == factors[2] + assert factors[1] == factors[3] + keypointss[:, :, 0] *= factors[0] + keypointss[:, :, 1] *= factors[1] + if self.bbox_clip_border: + img_shape = results['img_shape'] + keypointss[:, :, 0] = np.clip(keypointss[:, :, 0], 0, + img_shape[1]) + keypointss[:, :, 1] = np.clip(keypointss[:, :, 1], 0, + img_shape[0]) + results[key] = keypointss + + def _resize_masks(self, results): + """Resize masks with ``results['scale']``""" + for key in results.get('mask_fields', []): + if results[key] is None: + continue + if self.keep_ratio: + results[key] = results[key].rescale(results['scale']) + else: + results[key] = results[key].resize(results['img_shape'][:2]) + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale( + results[key], + results['scale'], + interpolation='nearest', + backend=self.backend) + else: + gt_seg = mmcv.imresize( + results[key], + results['scale'], + interpolation='nearest', + backend=self.backend) + results['gt_semantic_seg'] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', \ + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + if 'scale_factor' in results: + img_shape = results['img'].shape[:2] + scale_factor = results['scale_factor'] + assert isinstance(scale_factor, float) + results['scale'] = tuple( + [int(x * scale_factor) for x in img_shape][::-1]) + else: + self._random_scale(results) + else: + if not self.override: + assert 'scale_factor' not in results, ( + 'scale and scale_factor cannot be both set.') + else: + results.pop('scale') + if 'scale_factor' in results: + results.pop('scale_factor') + self._random_scale(results) + + self._resize_img(results) + self._resize_bboxes(results) + self._resize_keypoints(results) + self._resize_masks(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(img_scale={self.img_scale}, ' + repr_str += f'multiscale_mode={self.multiscale_mode}, ' + repr_str += f'ratio_range={self.ratio_range}, ' + repr_str += f'keep_ratio={self.keep_ratio})' + repr_str += f'bbox_clip_border={self.bbox_clip_border})' + return repr_str + + +@PIPELINES.register_module() +class RandomFlipV2(object): + """Flip the image & bbox & mask & kps. + + If the input dict contains the key "flip", then the flag will be used, + otherwise it will be randomly decided by a ratio specified in the init + method. + + When random flip is enabled, ``flip_ratio``/``direction`` can either be a + float/string or tuple of float/string. There are 3 flip modes: + + - ``flip_ratio`` is float, ``direction`` is string: the image will be + ``direction``ly flipped with probability of ``flip_ratio`` . + E.g., ``flip_ratio=0.5``, ``direction='horizontal'``, + then image will be horizontally flipped with probability of 0.5. + - ``flip_ratio`` is float, ``direction`` is list of string: the image wil + be ``direction[i]``ly flipped with probability of + ``flip_ratio/len(direction)``. + E.g., ``flip_ratio=0.5``, ``direction=['horizontal', 'vertical']``, + then image will be horizontally flipped with probability of 0.25, + vertically with probability of 0.25. + - ``flip_ratio`` is list of float, ``direction`` is list of string: + given ``len(flip_ratio) == len(direction)``, the image wil + be ``direction[i]``ly flipped with probability of ``flip_ratio[i]``. + E.g., ``flip_ratio=[0.3, 0.5]``, ``direction=['horizontal', + 'vertical']``, then image will be horizontally flipped with probability + of 0.3, vertically with probability of 0.5 + + Args: + flip_ratio (float | list[float], optional): The flipping probability. + Default: None. + direction(str | list[str], optional): The flipping direction. Options + are 'horizontal', 'vertical', 'diagonal'. Default: 'horizontal'. + If input is a list, the length must equal ``flip_ratio``. Each + element in ``flip_ratio`` indicates the flip probability of + corresponding direction. + """ + + def __init__(self, flip_ratio=None, direction='horizontal'): + if isinstance(flip_ratio, list): + assert mmcv.is_list_of(flip_ratio, float) + assert 0 <= sum(flip_ratio) <= 1 + elif isinstance(flip_ratio, float): + assert 0 <= flip_ratio <= 1 + elif flip_ratio is None: + pass + else: + raise ValueError('flip_ratios must be None, float, ' + 'or list of float') + self.flip_ratio = flip_ratio + + valid_directions = ['horizontal', 'vertical', 'diagonal'] + if isinstance(direction, str): + assert direction in valid_directions + elif isinstance(direction, list): + assert mmcv.is_list_of(direction, str) + assert set(direction).issubset(set(valid_directions)) + else: + raise ValueError('direction must be either str or list of str') + self.direction = direction + + if isinstance(flip_ratio, list): + assert len(self.flip_ratio) == len(self.direction) + self.count = 0 + + def bbox_flip(self, bboxes, img_shape, direction): + """Flip bboxes horizontally. + + Args: + bboxes (numpy.ndarray): Bounding boxes, shape (..., 4*k) + img_shape (tuple[int]): Image shape (height, width) + direction (str): Flip direction. Options are 'horizontal', + 'vertical'. + + Returns: + numpy.ndarray: Flipped bounding boxes. + """ + + assert bboxes.shape[-1] % 4 == 0 + flipped = bboxes.copy() + if direction == 'horizontal': + w = img_shape[1] + flipped[..., 0::4] = w - bboxes[..., 2::4] + flipped[..., 2::4] = w - bboxes[..., 0::4] + elif direction == 'vertical': + h = img_shape[0] + flipped[..., 1::4] = h - bboxes[..., 3::4] + flipped[..., 3::4] = h - bboxes[..., 1::4] + elif direction == 'diagonal': + w = img_shape[1] + h = img_shape[0] + flipped[..., 0::4] = w - bboxes[..., 2::4] + flipped[..., 1::4] = h - bboxes[..., 3::4] + flipped[..., 2::4] = w - bboxes[..., 0::4] + flipped[..., 3::4] = h - bboxes[..., 1::4] + else: + raise ValueError(f"Invalid flipping direction '{direction}'") + return flipped + + def keypoints_flip(self, keypointss, img_shape, direction): + """Flip keypoints horizontally.""" + + assert direction == 'horizontal' + assert keypointss.shape[-1] == 3 + num_kps = keypointss.shape[1] + assert num_kps in [4, 5], f'Only Support num_kps=4 or 5, got:{num_kps}' + assert keypointss.ndim == 3 + flipped = keypointss.copy() + if num_kps == 5: + flip_order = [1, 0, 2, 4, 3] + elif num_kps == 4: + flip_order = [3, 2, 1, 0] + for idx, a in enumerate(flip_order): + flipped[:, idx, :] = keypointss[:, a, :] + w = img_shape[1] + flipped[..., 0] = w - flipped[..., 0] + return flipped + + def __call__(self, results): + """Call function to flip bounding boxes, masks, semantic segmentation + maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Flipped results, 'flip', 'flip_direction' keys are added \ + into result dict. + """ + if 'flip' not in results: + if isinstance(self.direction, list): + # None means non-flip + direction_list = self.direction + [None] + else: + # None means non-flip + direction_list = [self.direction, None] + + if isinstance(self.flip_ratio, list): + non_flip_ratio = 1 - sum(self.flip_ratio) + flip_ratio_list = self.flip_ratio + [non_flip_ratio] + else: + non_flip_ratio = 1 - self.flip_ratio + # exclude non-flip + single_ratio = self.flip_ratio / (len(direction_list) - 1) + flip_ratio_list = [single_ratio] * (len(direction_list) + - 1) + [non_flip_ratio] + + cur_dir = np.random.choice(direction_list, p=flip_ratio_list) + + results['flip'] = cur_dir is not None + if 'flip_direction' not in results: + results['flip_direction'] = cur_dir + if results['flip']: + # flip image + for key in results.get('img_fields', ['img']): + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']) + # flip bboxes + for key in results.get('bbox_fields', []): + results[key] = self.bbox_flip(results[key], + results['img_shape'], + results['flip_direction']) + # flip kps + for key in results.get('keypoints_fields', []): + results[key] = self.keypoints_flip(results[key], + results['img_shape'], + results['flip_direction']) + # flip masks + for key in results.get('mask_fields', []): + results[key] = results[key].flip(results['flip_direction']) + + # flip segs + for key in results.get('seg_fields', []): + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(flip_ratio={self.flip_ratio})' + + +@PIPELINES.register_module() +class RandomSquareCrop(object): + """Random crop the image & bboxes, the cropped patches have minimum IoU + requirement with original image & bboxes, the IoU threshold is randomly + selected from min_ious. + + Args: + min_ious (tuple): minimum IoU threshold for all intersections with + bounding boxes + min_crop_size (float): minimum crop's size (i.e. h,w := a*h, a*w, + where a >= min_crop_size). + + Note: + The keys for bboxes, labels and masks should be paired. That is, \ + `gt_bboxes` corresponds to `gt_labels` and `gt_masks`, and \ + `gt_bboxes_ignore` to `gt_labels_ignore` and `gt_masks_ignore`. + """ + + def __init__(self, + crop_ratio_range=None, + crop_choice=None, + bbox_clip_border=True, + big_face_ratio=0, + big_face_crop_choice=None): + + self.crop_ratio_range = crop_ratio_range + self.crop_choice = crop_choice + self.big_face_crop_choice = big_face_crop_choice + self.bbox_clip_border = bbox_clip_border + + assert (self.crop_ratio_range is None) ^ (self.crop_choice is None) + if self.crop_ratio_range is not None: + self.crop_ratio_min, self.crop_ratio_max = self.crop_ratio_range + + self.bbox2label = { + 'gt_bboxes': 'gt_labels', + 'gt_bboxes_ignore': 'gt_labels_ignore' + } + self.bbox2mask = { + 'gt_bboxes': 'gt_masks', + 'gt_bboxes_ignore': 'gt_masks_ignore' + } + assert big_face_ratio >= 0 and big_face_ratio <= 1.0 + self.big_face_ratio = big_face_ratio + + def __call__(self, results): + """Call function to crop images and bounding boxes with minimum IoU + constraint. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images and bounding boxes cropped, \ + 'img_shape' key is updated. + """ + + if 'img_fields' in results: + assert results['img_fields'] == ['img'], \ + 'Only single img_fields is allowed' + img = results['img'] + assert 'bbox_fields' in results + assert 'gt_bboxes' in results + # try augment big face images + find_bigface = False + if np.random.random() < self.big_face_ratio: + min_size = 100 # h and w + expand_ratio = 0.3 # expand ratio of croped face alongwith both w and h + bbox = results['gt_bboxes'].copy() + lmks = results['gt_keypointss'].copy() + label = results['gt_labels'].copy() + # filter small faces + size_mask = ((bbox[:, 2] - bbox[:, 0]) > min_size) * ( + (bbox[:, 3] - bbox[:, 1]) > min_size) + bbox = bbox[size_mask] + lmks = lmks[size_mask] + label = label[size_mask] + # randomly choose a face that has no overlap with others + if len(bbox) > 0: + overlaps = bbox_overlaps(bbox, bbox) + overlaps -= np.eye(overlaps.shape[0]) + iou_mask = np.sum(overlaps, axis=1) == 0 + bbox = bbox[iou_mask] + lmks = lmks[iou_mask] + label = label[iou_mask] + if len(bbox) > 0: + choice = np.random.randint(len(bbox)) + bbox = bbox[choice] + lmks = lmks[choice] + label = [label[choice]] + w = bbox[2] - bbox[0] + h = bbox[3] - bbox[1] + x1 = bbox[0] - w * expand_ratio + x2 = bbox[2] + w * expand_ratio + y1 = bbox[1] - h * expand_ratio + y2 = bbox[3] + h * expand_ratio + x1, x2 = np.clip([x1, x2], 0, img.shape[1]) + y1, y2 = np.clip([y1, y2], 0, img.shape[0]) + bbox -= np.tile([x1, y1], 2) + lmks -= (x1, y1, 0) + + find_bigface = True + img = img[int(y1):int(y2), int(x1):int(x2), :] + results['gt_bboxes'] = np.expand_dims(bbox, axis=0) + results['gt_keypointss'] = np.expand_dims(lmks, axis=0) + results['gt_labels'] = np.array(label) + results['img'] = img + + boxes = results['gt_bboxes'] + h, w, c = img.shape + + if self.crop_ratio_range is not None: + max_scale = self.crop_ratio_max + else: + max_scale = np.amax(self.crop_choice) + scale_retry = 0 + while True: + scale_retry += 1 + if scale_retry == 1 or max_scale > 1.0: + if self.crop_ratio_range is not None: + scale = np.random.uniform(self.crop_ratio_min, + self.crop_ratio_max) + elif self.crop_choice is not None: + scale = np.random.choice(self.crop_choice) + else: + scale = scale * 1.2 + + if find_bigface: + # select a scale from big_face_crop_choice if in big_face mode + scale = np.random.choice(self.big_face_crop_choice) + + for i in range(250): + long_side = max(w, h) + cw = int(scale * long_side) + ch = cw + + # TODO +1 + if w == cw: + left = 0 + elif w > cw: + left = random.randint(0, w - cw) + else: + left = random.randint(w - cw, 0) + if h == ch: + top = 0 + elif h > ch: + top = random.randint(0, h - ch) + else: + top = random.randint(h - ch, 0) + + patch = np.array( + (int(left), int(top), int(left + cw), int(top + ch)), + dtype=np.int32) + + # center of boxes should inside the crop img + # only adjust boxes and instance masks when the gt is not empty + # adjust boxes + def is_center_of_bboxes_in_patch(boxes, patch): + # TODO >= + center = (boxes[:, :2] + boxes[:, 2:]) / 2 + mask = \ + ((center[:, 0] > patch[0]) + * (center[:, 1] > patch[1]) + * (center[:, 0] < patch[2]) + * (center[:, 1] < patch[3])) + return mask + + mask = is_center_of_bboxes_in_patch(boxes, patch) + if not mask.any(): + continue + for key in results.get('bbox_fields', []): + boxes = results[key].copy() + mask = is_center_of_bboxes_in_patch(boxes, patch) + boxes = boxes[mask] + if self.bbox_clip_border: + boxes[:, 2:] = boxes[:, 2:].clip(max=patch[2:]) + boxes[:, :2] = boxes[:, :2].clip(min=patch[:2]) + boxes -= np.tile(patch[:2], 2) + + results[key] = boxes + # labels + label_key = self.bbox2label.get(key) + if label_key in results: + results[label_key] = results[label_key][mask] + + # keypoints field + if key == 'gt_bboxes': + for kps_key in results.get('keypoints_fields', []): + keypointss = results[kps_key].copy() + keypointss = keypointss[mask, :, :] + if self.bbox_clip_border: + keypointss[:, :, : + 2] = keypointss[:, :, :2].clip( + max=patch[2:]) + keypointss[:, :, : + 2] = keypointss[:, :, :2].clip( + min=patch[:2]) + keypointss[:, :, 0] -= patch[0] + keypointss[:, :, 1] -= patch[1] + results[kps_key] = keypointss + + # mask fields + mask_key = self.bbox2mask.get(key) + if mask_key in results: + results[mask_key] = results[mask_key][mask.nonzero() + [0]].crop(patch) + + # adjust the img no matter whether the gt is empty before crop + rimg = np.ones((ch, cw, 3), dtype=img.dtype) * 128 + patch_from = patch.copy() + patch_from[0] = max(0, patch_from[0]) + patch_from[1] = max(0, patch_from[1]) + patch_from[2] = min(img.shape[1], patch_from[2]) + patch_from[3] = min(img.shape[0], patch_from[3]) + patch_to = patch.copy() + patch_to[0] = max(0, patch_to[0] * -1) + patch_to[1] = max(0, patch_to[1] * -1) + patch_to[2] = patch_to[0] + (patch_from[2] - patch_from[0]) + patch_to[3] = patch_to[1] + (patch_from[3] - patch_from[1]) + rimg[patch_to[1]:patch_to[3], + patch_to[0]:patch_to[2], :] = img[ + patch_from[1]:patch_from[3], + patch_from[0]:patch_from[2], :] + img = rimg + results['img'] = img + results['img_shape'] = img.shape + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(min_ious={self.min_iou}, ' + repr_str += f'crop_size={self.crop_size})' + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/retinaface.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/retinaface.py new file mode 100644 index 0000000000000000000000000000000000000000..4cfb61b5b82230a1797ee40b11579b356592c397 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/datasets/retinaface.py @@ -0,0 +1,153 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/datasets/retinaface.py +""" +import numpy as np +from mmdet.datasets.builder import DATASETS +from mmdet.datasets.custom import CustomDataset + + +@DATASETS.register_module() +class RetinaFaceDataset(CustomDataset): + + CLASSES = ('FG', ) + + def __init__(self, min_size=None, **kwargs): + self.NK = kwargs.pop('num_kps', 5) + self.cat2label = {cat: i for i, cat in enumerate(self.CLASSES)} + self.min_size = min_size + self.gt_path = kwargs.get('gt_path') + super(RetinaFaceDataset, self).__init__(**kwargs) + + def _parse_ann_line(self, line): + values = [float(x) for x in line.strip().split()] + bbox = np.array(values[0:4], dtype=np.float32) + kps = np.zeros((self.NK, 3), dtype=np.float32) + ignore = False + if self.min_size is not None: + assert not self.test_mode + w = bbox[2] - bbox[0] + h = bbox[3] - bbox[1] + if w < self.min_size or h < self.min_size: + ignore = True + if len(values) > 4: + if len(values) > 5: + kps = np.array( + values[4:4 + self.NK * 3], dtype=np.float32).reshape( + (self.NK, 3)) + for li in range(kps.shape[0]): + if (kps[li, :] == -1).all(): + kps[li][2] = 0.0 # weight = 0, ignore + else: + assert kps[li][2] >= 0 + kps[li][2] = 1.0 # weight + else: # len(values)==5 + if not ignore: + ignore = (values[4] == 1) + else: + assert self.test_mode + + return dict(bbox=bbox, kps=kps, ignore=ignore, cat='FG') + + def load_annotations(self, ann_file): + """Load annotation from COCO style annotation file. + + Args: + ann_file (str): Path of annotation file. + 20220711@tyx: ann_file is list of img paths is supported + + Returns: + list[dict]: Annotation info from COCO api. + """ + if isinstance(ann_file, list): + data_infos = [] + for line in ann_file: + name = line + objs = [0, 0, 0, 0] + data_infos.append( + dict(filename=name, width=0, height=0, objs=objs)) + else: + name = None + bbox_map = {} + for line in open(ann_file, 'r'): + line = line.strip() + if line.startswith('#'): + value = line[1:].strip().split() + name = value[0] + width = int(value[1]) + height = int(value[2]) + + bbox_map[name] = dict(width=width, height=height, objs=[]) + continue + assert name is not None + assert name in bbox_map + bbox_map[name]['objs'].append(line) + print('origin image size', len(bbox_map)) + data_infos = [] + for name in bbox_map: + item = bbox_map[name] + width = item['width'] + height = item['height'] + vals = item['objs'] + objs = [] + for line in vals: + data = self._parse_ann_line(line) + if data is None: + continue + objs.append(data) # data is (bbox, kps, cat) + if len(objs) == 0 and not self.test_mode: + continue + data_infos.append( + dict(filename=name, width=width, height=height, objs=objs)) + return data_infos + + def get_ann_info(self, idx): + """Get COCO annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + data_info = self.data_infos[idx] + + bboxes = [] + keypointss = [] + labels = [] + bboxes_ignore = [] + labels_ignore = [] + for obj in data_info['objs']: + label = self.cat2label[obj['cat']] + bbox = obj['bbox'] + keypoints = obj['kps'] + ignore = obj['ignore'] + if ignore: + bboxes_ignore.append(bbox) + labels_ignore.append(label) + else: + bboxes.append(bbox) + labels.append(label) + keypointss.append(keypoints) + if not bboxes: + bboxes = np.zeros((0, 4)) + labels = np.zeros((0, )) + keypointss = np.zeros((0, self.NK, 3)) + else: + # bboxes = np.array(bboxes, ndmin=2) - 1 + bboxes = np.array(bboxes, ndmin=2) + labels = np.array(labels) + keypointss = np.array(keypointss, ndmin=3) + if not bboxes_ignore: + bboxes_ignore = np.zeros((0, 4)) + labels_ignore = np.zeros((0, )) + else: + bboxes_ignore = np.array(bboxes_ignore, ndmin=2) + labels_ignore = np.array(labels_ignore) + ann = dict( + bboxes=bboxes.astype(np.float32), + labels=labels.astype(np.int64), + keypointss=keypointss.astype(np.float32), + bboxes_ignore=bboxes_ignore.astype(np.float32), + labels_ignore=labels_ignore.astype(np.int64)) + return ann diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1c86abf127aac5a21b283c58e502014f86d06b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/__init__.py @@ -0,0 +1,6 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models +""" +from .dense_heads import * # noqa: F401,F403 +from .detectors import * # noqa: F401,F403 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..65686d5568c0d341b1c05fc7dfab33f87019a6f4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/__init__.py @@ -0,0 +1,9 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/backbones +""" +from .master_net import MasterNet +from .mobilenet import MobileNetV1 +from .resnet import ResNetV1e + +__all__ = ['ResNetV1e', 'MobileNetV1', 'MasterNet'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/master_net.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/master_net.py new file mode 100644 index 0000000000000000000000000000000000000000..11a59302fe31574b99a07f0a9a1b61bb9a88c84c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/master_net.py @@ -0,0 +1,133 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import torch +import torch.nn.functional as F +from mmdet.models.builder import BACKBONES +from torch import nn + +from modelscope.models.cv.tinynas_classfication import (basic_blocks, + plain_net_utils) + + +@BACKBONES.register_module() +class MasterNet(plain_net_utils.PlainNet): + + def __init__(self, + argv=None, + opt=None, + num_classes=None, + plainnet_struct=None, + no_create=False, + no_reslink=None, + no_BN=None, + use_se=None, + dropout=None, + **kwargs): + """ + Any ReLU-CNN Backbone + Args: + plainet_struct: (obj: str): + Str of network topology structure. + no_reslink: (obj:bool): + no use residual structure. + no_BN: (obj:bool): + no use BN op. + no_se: (obj:bool): + no use se structure. + no_se: (obj:bool): + no use se structure. + """ + + module_opt = None + + if no_BN is None: + if module_opt is not None: + no_BN = module_opt.no_BN + else: + no_BN = False + + if no_reslink is None: + if module_opt is not None: + no_reslink = module_opt.no_reslink + else: + no_reslink = False + + if use_se is None: + if module_opt is not None: + use_se = module_opt.use_se + else: + use_se = False + + if dropout is None: + if module_opt is not None: + self.dropout = module_opt.dropout + else: + self.dropout = None + else: + self.dropout = dropout + + num_classes = 2048 + super(MasterNet, self).__init__( + argv=argv, + opt=opt, + num_classes=num_classes, + plainnet_struct=plainnet_struct, + no_create=no_create, + no_reslink=no_reslink, + no_BN=no_BN, + use_se=use_se, + **kwargs) + self.last_channels = self.block_list[-1].out_channels + + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + self.use_se = use_se + + for layer in self.modules(): + if isinstance(layer, nn.BatchNorm2d): + layer.eps = 1e-3 + + def extract_stage_features_and_logit(self, x, target_downsample_ratio=4): + stage_features_list = [] + image_size = x.shape[2] + output = x + block_id = 0 + for block_id, the_block in enumerate(self.block_list): + output = the_block(output) + dowsample_ratio = round(image_size / output.shape[2]) + if dowsample_ratio == target_downsample_ratio: + stage_features_list.append(output) + target_downsample_ratio *= 2 + pass + pass + + return stage_features_list + + def forward(self, x): + """ + Args: + The input image + Returns: + The list of stage-level feature map + """ + output = self.extract_stage_features_and_logit(x) + return tuple(output) + + def init_weights(self, pretrained=None): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.xavier_normal_(m.weight.data, gain=3.26033) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + hyper_para = m.weight.shape[0] + m.weight.shape[1] + nn.init.normal_(m.weight, 0, 3.26033 * np.sqrt(2 / hyper_para)) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.zeros_(m.bias) + else: + pass diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/mobilenet.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/mobilenet.py new file mode 100644 index 0000000000000000000000000000000000000000..7d0303e1fef35103c605ef9877f85cfafda1f685 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/mobilenet.py @@ -0,0 +1,99 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/backbones/mobilenet.py +""" + +import torch +import torch.nn as nn +from mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from mmdet.models.builder import BACKBONES +from mmdet.utils import get_root_logger +from torch.nn.modules.batchnorm import _BatchNorm + + +@BACKBONES.register_module() +class MobileNetV1(nn.Module): + + def __init__(self, + in_channels=3, + block_cfg=None, + num_stages=4, + out_indices=(0, 1, 2, 3)): + super(MobileNetV1, self).__init__() + self.out_indices = out_indices + + def conv_bn(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), nn.ReLU(inplace=True)) + + def conv_dw(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.ReLU(inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True), + ) + + if block_cfg is None: + stage_planes = [8, 16, 32, 64, 128, 256] + stage_blocks = [2, 4, 4, 2] + else: + stage_planes = block_cfg['stage_planes'] + stage_blocks = block_cfg['stage_blocks'] + assert len(stage_planes) == 6 + assert len(stage_blocks) == 4 + self.stem = nn.Sequential( + conv_bn(3, stage_planes[0], 2), + conv_dw(stage_planes[0], stage_planes[1], 1), + ) + self.stage_layers = [] + for i, num_blocks in enumerate(stage_blocks): + _layers = [] + for n in range(num_blocks): + if n == 0: + _layer = conv_dw(stage_planes[i + 1], stage_planes[i + 2], + 2) + else: + _layer = conv_dw(stage_planes[i + 2], stage_planes[i + 2], + 1) + _layers.append(_layer) + + _block = nn.Sequential(*_layers) + layer_name = f'layer{i + 1}' + self.add_module(layer_name, _block) + self.stage_layers.append(layer_name) + + def forward(self, x): + output = [] + x = self.stem(x) + for i, layer_name in enumerate(self.stage_layers): + stage_layer = getattr(self, layer_name) + x = stage_layer(x) + if i in self.out_indices: + output.append(x) + + return tuple(output) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/resnet.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..36ba93cd55bde772fd58dcbad84aa5a1761cb6dd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/backbones/resnet.py @@ -0,0 +1,414 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/backbones/resnet.py +""" +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from mmcv.runner import load_checkpoint +from mmdet.models.backbones.resnet import BasicBlock, Bottleneck +from mmdet.models.builder import BACKBONES +from mmdet.models.utils import ResLayer +from mmdet.utils import get_root_logger +from torch.nn.modules.batchnorm import _BatchNorm + + +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + stem_channels (int | None): Number of stem channels. If not specified, + it will be the same as `base_channels`. Default: None. + base_channels (int): Number of base channels of res layer. Default: 64. + in_channels (int): Number of input image channels. Default: 3. + num_stages (int): Resnet stages. Default: 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + - position (str, required): Position inside block to insert + plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages'. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Examples: + + >>> from mmdet.models import ResNet + >>> import torch + >>> self = ResNet(depth=18) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 64, 8, 8) + (1, 128, 4, 4) + (1, 256, 2, 2) + (1, 512, 1, 1) + """ + + arch_settings = { + 0: (BasicBlock, (2, 2, 2, 2)), + 18: (BasicBlock, (2, 2, 2, 2)), + 19: (BasicBlock, (2, 4, 4, 1)), + 20: (BasicBlock, (2, 3, 2, 2)), + 22: (BasicBlock, (2, 4, 3, 1)), + 24: (BasicBlock, (2, 4, 4, 1)), + 26: (BasicBlock, (2, 4, 4, 2)), + 28: (BasicBlock, (2, 5, 4, 2)), + 29: (BasicBlock, (2, 6, 3, 2)), + 30: (BasicBlock, (2, 5, 5, 2)), + 32: (BasicBlock, (2, 6, 5, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 35: (BasicBlock, (3, 6, 4, 3)), + 38: (BasicBlock, (3, 8, 4, 3)), + 40: (BasicBlock, (3, 8, 5, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 56: (Bottleneck, (3, 8, 4, 3)), + 68: (Bottleneck, (3, 10, 6, 3)), + 74: (Bottleneck, (3, 12, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + in_channels=3, + stem_channels=None, + base_channels=64, + num_stages=4, + block_cfg=None, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + no_pool33=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + self.depth = depth + if stem_channels is None: + stem_channels = base_channels + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.no_pool33 = no_pool33 + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.zero_init_residual = zero_init_residual + if block_cfg is None: + self.block, stage_blocks = self.arch_settings[depth] + else: + self.block = BasicBlock if block_cfg[ + 'block'] == 'BasicBlock' else Bottleneck + stage_blocks = block_cfg['stage_blocks'] + assert len(stage_blocks) >= num_stages + self.stage_blocks = stage_blocks[:num_stages] + self.inplanes = stem_channels + + self._make_stem_layer(in_channels, stem_channels) + if block_cfg is not None and 'stage_planes' in block_cfg: + stage_planes = block_cfg['stage_planes'] + else: + stage_planes = [base_channels * 2**i for i in range(num_stages)] + + # print('resnet cfg:', stage_blocks, stage_planes) + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = strides[i] + dilation = dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + planes = stage_planes[i] + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=stage_plugins) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = self.block.expansion * base_channels * 2**( + len(self.stage_blocks) - 1) + + def make_stage_plugins(self, plugins, stage_idx): + """Make plugins for ResNet ``stage_idx`` th stage. + + Currently we support to insert ``context_block``, + ``empirical_attention_block``, ``nonlocal_block`` into the backbone + like ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of + Bottleneck. + + An example of plugins format could be: + + Examples: + >>> plugins=[ + ... dict(cfg=dict(type='xxx', arg1='xxx'), + ... stages=(False, True, True, True), + ... position='after_conv2'), + ... dict(cfg=dict(type='yyy'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='1'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='2'), + ... stages=(True, True, True, True), + ... position='after_conv3') + ... ] + >>> self = ResNet(depth=18) + >>> stage_plugins = self.make_stage_plugins(plugins, 0) + >>> assert len(stage_plugins) == 3 + + Suppose ``stage_idx=0``, the structure of blocks in the stage would be: + + .. code-block:: none + + conv1-> conv2->conv3->yyy->zzz1->zzz2 + + Suppose 'stage_idx=1', the structure of blocks in the stage would be: + + .. code-block:: none + + conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 + + If stages is missing, the plugin would be applied to all stages. + + Args: + plugins (list[dict]): List of plugins cfg to build. The postfix is + required if multiple same type plugins are inserted. + stage_idx (int): Index of stage to build + + Returns: + list[dict]: Plugins for current stage + """ + stage_plugins = [] + for plugin in plugins: + plugin = plugin.copy() + stages = plugin.pop('stages', None) + assert stages is None or len(stages) == self.num_stages + # whether to insert plugin into current stage + if stages is None or stages[stage_idx]: + stage_plugins.append(plugin) + + return stage_plugins + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer(**kwargs) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def _make_stem_layer(self, in_channels, stem_channels): + if self.deep_stem: + self.stem = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels)[1], + nn.ReLU(inplace=True)) + else: + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, stem_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + if self.no_pool33: + assert self.deep_stem + self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0) + else: + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + if self.deep_stem: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + else: + self.norm1.eval() + for m in [self.conv1, self.norm1]: + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'layer{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m.conv2, 'conv_offset'): + constant_init(m.conv2.conv_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(ResNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + +@BACKBONES.register_module() +class ResNetV1e(ResNet): + r"""ResNetV1d variant described in `Bag of Tricks + `_. + + Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in + the input stem with three 3x3 convs. And in the downsampling block, a 2x2 + avg_pool with stride 2 is added before conv, whose stride is changed to 1. + + Compared with ResNetV1d, ResNetV1e change maxpooling from 3x3 to 2x2 pad=1 + """ + + def __init__(self, **kwargs): + super(ResNetV1e, self).__init__( + deep_stem=True, avg_down=True, no_pool33=True, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..55ddf654f560f6b0eff69c9cb94734b0c30235dc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/dense_heads +""" +from .scrfd_head import SCRFDHead + +__all__ = ['SCRFDHead'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/scrfd_head.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/scrfd_head.py new file mode 100644 index 0000000000000000000000000000000000000000..222d4297757dc297cb37ba9dea512a653bee17ca --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/dense_heads/scrfd_head.py @@ -0,0 +1,1071 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py +""" +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import (ConvModule, DepthwiseSeparableConvModule, Scale, + bias_init_with_prob, constant_init, kaiming_init, + normal_init) +from mmcv.runner import force_fp32 +from mmdet.core import (anchor_inside_flags, bbox2distance, bbox_overlaps, + build_assigner, build_sampler, distance2bbox, + images_to_levels, multi_apply, reduce_mean, unmap) +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.dense_heads.anchor_head import AnchorHead + +from ....mmdet_patch.core.bbox import distance2kps, kps2distance +from ....mmdet_patch.core.post_processing import multiclass_nms + + +class Integral(nn.Module): + """A fixed layer for calculating integral result from distribution. + + This layer calculates the target location by :math: `sum{P(y_i) * y_i}`, + P(y_i) denotes the softmax vector that represents the discrete distribution + y_i denotes the discrete set, usually {0, 1, 2, ..., reg_max} + + Args: + reg_max (int): The maximal value of the discrete set. Default: 16. You + may want to reset it according to your new dataset or related + settings. + """ + + def __init__(self, reg_max=16): + super(Integral, self).__init__() + self.reg_max = reg_max + self.register_buffer('project', + torch.linspace(0, self.reg_max, self.reg_max + 1)) + + def forward(self, x): + """Forward feature from the regression head to get integral result of + bounding box location. + + Args: + x (Tensor): Features of the regression head, shape (N, 4*(n+1)), + n is self.reg_max. + + Returns: + x (Tensor): Integral result of box locations, i.e., distance + offsets from the box center in four directions, shape (N, 4). + """ + x = F.softmax(x.reshape(-1, self.reg_max + 1), dim=1) + x = F.linear(x, self.project.type_as(x)).reshape(-1, 4) + return x + + +@HEADS.register_module() +class SCRFDHead(AnchorHead): + """Generalized Focal Loss: Learning Qualified and Distributed Bounding + Boxes for Dense Object Detection. + + GFL head structure is similar with ATSS, however GFL uses + 1) joint representation for classification and localization quality, and + 2) flexible General distribution for bounding box locations, + which are supervised by + Quality Focal Loss (QFL) and Distribution Focal Loss (DFL), respectively + + https://arxiv.org/abs/2006.04388 + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + stacked_convs (int): Number of conv layers in cls and reg tower. + Default: 4. + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None. + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='GN', num_groups=32, requires_grad=True). + loss_qfl (dict): Config of Quality Focal Loss (QFL). + reg_max (int): Max value of integral set :math: `{0, ..., reg_max}` + in QFL setting. Default: 16. + Example: + + >>> self = GFLHead(11, 7) + >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] + >>> cls_quality_score, bbox_pred = self.forward(feats) + >>> assert len(cls_quality_score) == len(self.scales) + """ + + def __init__(self, + num_classes, + in_channels, + stacked_convs=4, + feat_mults=None, + conv_cfg=None, + norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), + loss_dfl=None, + reg_max=8, + cls_reg_share=False, + strides_share=True, + scale_mode=1, + dw_conv=False, + use_kps=False, + num_kps=5, + loss_kps=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=0.1), + **kwargs): + self.stacked_convs = stacked_convs + self.feat_mults = feat_mults + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.reg_max = reg_max + self.cls_reg_share = cls_reg_share + self.strides_share = strides_share + self.scale_mode = scale_mode + self.use_dfl = True + self.dw_conv = dw_conv + self.NK = num_kps + self.extra_flops = 0.0 + if loss_dfl is None or not loss_dfl: + self.use_dfl = False + self.use_scale = False + self.use_kps = use_kps + if self.scale_mode > 0 and (self.strides_share + or self.scale_mode == 2): + self.use_scale = True + super(SCRFDHead, self).__init__(num_classes, in_channels, **kwargs) + + self.sampling = False + if self.train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + # SSD sampling=False so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.integral = Integral(self.reg_max) + if self.use_dfl: + self.loss_dfl = build_loss(loss_dfl) + self.loss_kps = build_loss(loss_kps) + self.loss_kps_std = 1.0 + self.train_step = 0 + self.pos_count = {} + self.gtgroup_count = {} + for stride in self.anchor_generator.strides: + self.pos_count[stride[0]] = 0 + + def _get_conv_module(self, in_channel, out_channel): + if not self.dw_conv: + conv = ConvModule( + in_channel, + out_channel, + 3, + stride=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg) + else: + conv = DepthwiseSeparableConvModule( + in_channel, + out_channel, + 3, + stride=1, + padding=1, + pw_norm_cfg=self.norm_cfg, + dw_norm_cfg=self.norm_cfg) + return conv + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + conv_strides = [0] if self.strides_share else \ + self.anchor_generator.strides + self.cls_stride_convs = nn.ModuleDict() + self.reg_stride_convs = nn.ModuleDict() + self.stride_cls = nn.ModuleDict() + self.stride_reg = nn.ModuleDict() + if self.use_kps: + self.stride_kps = nn.ModuleDict() + for stride_idx, conv_stride in enumerate(conv_strides): + key = str(conv_stride) + cls_convs = nn.ModuleList() + reg_convs = nn.ModuleList() + stacked_convs = self.stacked_convs[stride_idx] if \ + isinstance(self.stacked_convs, (list, tuple)) else \ + self.stacked_convs + feat_mult = self.feat_mults[stride_idx] if \ + self.feat_mults is not None else 1 + feat_ch = int(self.feat_channels * feat_mult) + last_feat_ch = 0 + for i in range(stacked_convs): + chn = self.in_channels if i == 0 else last_feat_ch + cls_convs.append(self._get_conv_module(chn, feat_ch)) + if not self.cls_reg_share: + reg_convs.append(self._get_conv_module(chn, feat_ch)) + last_feat_ch = feat_ch + self.cls_stride_convs[key] = cls_convs + self.reg_stride_convs[key] = reg_convs + self.stride_cls[key] = nn.Conv2d( + feat_ch, + self.cls_out_channels * self.num_anchors, + 3, + padding=1) + if not self.use_dfl: + self.stride_reg[key] = nn.Conv2d( + feat_ch, 4 * self.num_anchors, 3, padding=1) + else: + self.stride_reg[key] = nn.Conv2d( + feat_ch, + 4 * (self.reg_max + 1) * self.num_anchors, + 3, + padding=1) + if self.use_kps: + self.stride_kps[key] = nn.Conv2d( + feat_ch, self.NK * 2 * self.num_anchors, 3, padding=1) + if self.use_scale: + self.scales = nn.ModuleList( + [Scale(1.0) for _ in self.anchor_generator.strides]) + else: + self.scales = [None for _ in self.anchor_generator.strides] + + def init_weights(self): + """Initialize weights of the head.""" + for stride, cls_convs in self.cls_stride_convs.items(): + for m in cls_convs: + if not self.dw_conv: + try: + normal_init(m.conv, std=0.01) + except Exception: + pass + else: + normal_init(m.depthwise_conv.conv, std=0.01) + normal_init(m.pointwise_conv.conv, std=0.01) + for stride, reg_convs in self.reg_stride_convs.items(): + for m in reg_convs: + if not self.dw_conv: + normal_init(m.conv, std=0.01) + else: + normal_init(m.depthwise_conv.conv, std=0.01) + normal_init(m.pointwise_conv.conv, std=0.01) + bias_cls = -4.595 + for stride, conv in self.stride_cls.items(): + normal_init(conv, std=0.01, bias=bias_cls) + for stride, conv in self.stride_reg.items(): + normal_init(conv, std=0.01) + if self.use_kps: + for stride, conv in self.stride_kps.items(): + normal_init(conv, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: Usually a tuple of classification scores and bbox prediction + cls_scores (list[Tensor]): Classification and quality (IoU) + joint scores for all scale levels, each is a 4D-tensor, + the channel number is num_classes. + bbox_preds (list[Tensor]): Box distribution logits for all + scale levels, each is a 4D-tensor, the channel number is + 4*(n+1), n is max value of integral set. + """ + return multi_apply(self.forward_single, feats, self.scales, + self.anchor_generator.strides) + + def forward_single(self, x, scale, stride): + """Forward feature of a single scale level. + + Args: + x (Tensor): Features of a single scale level. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + + Returns: + tuple: + cls_score (Tensor): Cls and quality joint scores for a single + scale level the channel number is num_classes. + bbox_pred (Tensor): Box distribution logits for a single scale + level, the channel number is 4*(n+1), n is max value of + integral set. + """ + cls_feat = x + reg_feat = x + cls_convs = self.cls_stride_convs[ + '0'] if self.strides_share else self.cls_stride_convs[str(stride)] + for cls_conv in cls_convs: + cls_feat = cls_conv(cls_feat) + if not self.cls_reg_share: + reg_convs = self.reg_stride_convs[ + '0'] if self.strides_share else self.reg_stride_convs[str( + stride)] + for reg_conv in reg_convs: + reg_feat = reg_conv(reg_feat) + else: + reg_feat = cls_feat + cls_pred_module = self.stride_cls[ + '0'] if self.strides_share else self.stride_cls[str(stride)] + cls_score = cls_pred_module(cls_feat) + reg_pred_module = self.stride_reg[ + '0'] if self.strides_share else self.stride_reg[str(stride)] + _bbox_pred = reg_pred_module(reg_feat) + if self.use_scale: + bbox_pred = scale(_bbox_pred) + else: + bbox_pred = _bbox_pred + if self.use_kps: + kps_pred_module = self.stride_kps[ + '0'] if self.strides_share else self.stride_kps[str(stride)] + kps_pred = kps_pred_module(reg_feat) + else: + kps_pred = bbox_pred.new_zeros( + (bbox_pred.shape[0], self.NK * 2, bbox_pred.shape[2], + bbox_pred.shape[3])) + if torch.onnx.is_in_onnx_export(): + assert not self.use_dfl + print('in-onnx-export', cls_score.shape, bbox_pred.shape) + # Add output batch dim, based on pull request #1593 + batch_size = cls_score.shape[0] + cls_score = cls_score.permute(0, 2, 3, 1).reshape( + batch_size, -1, self.cls_out_channels).sigmoid() + bbox_pred = bbox_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, 4) + kps_pred = kps_pred.permute(0, 2, 3, + 1).reshape(batch_size, -1, self.NK * 2) + return cls_score, bbox_pred, kps_pred + + def forward_train(self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_keypointss=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs): + """ + Args: + x (list[Tensor]): Features from FPN. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + proposal_cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used + + Returns: + tuple: + losses: (dict[str, Tensor]): A dictionary of loss components. + proposal_list (list[Tensor]): Proposals of each image. + """ + outs = self(x) + if gt_labels is None: + loss_inputs = outs + (gt_bboxes, img_metas) + else: + loss_inputs = outs + (gt_bboxes, gt_labels, gt_keypointss, + img_metas) + losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + if proposal_cfg is None: + return losses + else: + proposal_list = self.get_bboxes(*outs, img_metas, cfg=proposal_cfg) + return losses, proposal_list + + def get_anchors(self, featmap_sizes, img_metas, device='cuda'): + """Get anchors according to feature map sizes. + + Args: + featmap_sizes (list[tuple]): Multi-level feature map sizes. + img_metas (list[dict]): Image meta info. + device (torch.device | str): Device for returned tensors + + Returns: + tuple: + anchor_list (list[Tensor]): Anchors of each image. + valid_flag_list (list[Tensor]): Valid flags of each image. + """ + num_imgs = len(img_metas) + + # since feature map sizes of all images are the same, we only compute + # anchors for one time + multi_level_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device) + anchor_list = [multi_level_anchors for _ in range(num_imgs)] + + # for each image, we compute valid flags of multi level anchors + valid_flag_list = [] + for img_id, img_meta in enumerate(img_metas): + multi_level_flags = self.anchor_generator.valid_flags( + featmap_sizes, img_meta['pad_shape'], device) + valid_flag_list.append(multi_level_flags) + + return anchor_list, valid_flag_list + + def anchor_center(self, anchors): + """Get anchor centers from anchors. + + Args: + anchors (Tensor): Anchor list with shape (N, 4), "xyxy" format. + + Returns: + Tensor: Anchor centers with shape (N, 2), "xy" format. + """ + anchors_cx = (anchors[:, 2] + anchors[:, 0]) / 2 + anchors_cy = (anchors[:, 3] + anchors[:, 1]) / 2 + return torch.stack([anchors_cx, anchors_cy], dim=-1) + + def loss_single(self, anchors, cls_score, bbox_pred, kps_pred, labels, + label_weights, bbox_targets, kps_targets, kps_weights, + stride, num_total_samples): + """Compute loss of a single scale level. + + Args: + anchors (Tensor): Box reference for each scale level with shape + (N, num_total_anchors, 4). + cls_score (Tensor): Cls and quality joint scores for each scale + level has shape (N, num_classes, H, W). + bbox_pred (Tensor): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + labels (Tensor): Labels of each anchors with shape + (N, num_total_anchors). + label_weights (Tensor): Label weights of each anchor with shape + (N, num_total_anchors) + bbox_targets (Tensor): BBox regression targets of each anchor wight + shape (N, num_total_anchors, 4). + stride (tuple): Stride in this scale level. + num_total_samples (int): Number of positive samples that is + reduced over all GPUs. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert stride[0] == stride[1], 'h stride is not equal to w stride!' + use_qscore = True + anchors = anchors.reshape(-1, 4) + cls_score = cls_score.permute(0, 2, 3, + 1).reshape(-1, self.cls_out_channels) + if not self.use_dfl: + bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) + else: + bbox_pred = bbox_pred.permute(0, 2, 3, 1) + bbox_pred = bbox_pred.reshape(-1, 4 * (self.reg_max + 1)) + bbox_targets = bbox_targets.reshape(-1, 4) + labels = labels.reshape(-1) + label_weights = label_weights.reshape(-1) + + if self.use_kps: + kps_pred = kps_pred.permute(0, 2, 3, 1).reshape(-1, self.NK * 2) + kps_targets = kps_targets.reshape((-1, self.NK * 2)) + kps_weights = kps_weights.reshape((-1, self.NK * 2)) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((labels >= 0) + & (labels < bg_class_ind)).nonzero().squeeze(1) + score = label_weights.new_zeros(labels.shape) + + if len(pos_inds) > 0: + pos_bbox_targets = bbox_targets[pos_inds] + pos_bbox_pred = bbox_pred[pos_inds] + pos_anchors = anchors[pos_inds] + pos_anchor_centers = self.anchor_center(pos_anchors) / stride[0] + + weight_targets = cls_score.detach().sigmoid() + weight_targets = weight_targets.max(dim=1)[0][pos_inds] + pos_decode_bbox_targets = pos_bbox_targets / stride[0] + + if self.use_dfl: + pos_bbox_pred_corners = self.integral(pos_bbox_pred) + pos_decode_bbox_pred = distance2bbox(pos_anchor_centers, + pos_bbox_pred_corners) + else: + pos_decode_bbox_pred = distance2bbox(pos_anchor_centers, + pos_bbox_pred) + if self.use_kps: + pos_kps_targets = kps_targets[pos_inds] + pos_kps_pred = kps_pred[pos_inds] + pos_kps_weights = kps_weights.max( + dim=1)[0][pos_inds] * weight_targets + pos_kps_weights = pos_kps_weights.reshape((-1, 1)) + pos_decode_kps_targets = kps2distance( + pos_anchor_centers, pos_kps_targets / stride[0]) + pos_decode_kps_pred = pos_kps_pred + if use_qscore: + score[pos_inds] = bbox_overlaps( + pos_decode_bbox_pred.detach(), + pos_decode_bbox_targets, + is_aligned=True) + else: + score[pos_inds] = 1.0 + + # regression loss + loss_bbox = self.loss_bbox( + pos_decode_bbox_pred, + pos_decode_bbox_targets, + weight=weight_targets, + avg_factor=1.0) + + if self.use_kps: + loss_kps = self.loss_kps( + pos_decode_kps_pred * self.loss_kps_std, + pos_decode_kps_targets * self.loss_kps_std, + weight=pos_kps_weights, + avg_factor=1.0) + else: + loss_kps = kps_pred.sum() * 0 + + # dfl loss + if self.use_dfl: + pred_corners = pos_bbox_pred.reshape(-1, self.reg_max + 1) + target_corners = bbox2distance(pos_anchor_centers, + pos_decode_bbox_targets, + self.reg_max).reshape(-1) + loss_dfl = self.loss_dfl( + pred_corners, + target_corners, + weight=weight_targets[:, None].expand(-1, 4).reshape(-1), + avg_factor=4.0) + else: + loss_dfl = bbox_pred.sum() * 0 + else: + loss_bbox = bbox_pred.sum() * 0 + loss_dfl = bbox_pred.sum() * 0 + loss_kps = kps_pred.sum() * 0 + weight_targets = torch.tensor(0).cuda() + + loss_cls = self.loss_cls( + cls_score, (labels, score), + weight=label_weights, + avg_factor=num_total_samples) + return loss_cls, loss_bbox, loss_dfl, loss_kps, weight_targets.sum() + + @force_fp32(apply_to=('cls_scores', 'bbox_preds')) + def loss(self, + cls_scores, + bbox_preds, + kps_preds, + gt_bboxes, + gt_labels, + gt_keypointss, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Cls and quality scores for each scale + level has shape (N, num_classes, H, W). + bbox_preds (list[Tensor]): Box distribution logits for each scale + level with shape (N, 4*(n+1), H, W), n is max value of integral + set. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (list[Tensor] | None): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + assert len(featmap_sizes) == self.anchor_generator.num_levels + + device = cls_scores[0].device + anchor_list, valid_flag_list = self.get_anchors( + featmap_sizes, img_metas, device=device) + label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 + + cls_reg_targets = self.get_targets( + anchor_list, + valid_flag_list, + gt_bboxes, + gt_keypointss, + img_metas, + gt_bboxes_ignore_list=gt_bboxes_ignore, + gt_labels_list=gt_labels, + label_channels=label_channels) + if cls_reg_targets is None: + return None + + (anchor_list, labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, keypoints_targets_list, keypoints_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + + num_total_samples = reduce_mean( + torch.tensor(num_total_pos, dtype=torch.float, + device=device)).item() + num_total_samples = max(num_total_samples, 1.0) + + losses_cls, losses_bbox, losses_dfl, losses_kps,\ + avg_factor = multi_apply( + self.loss_single, + anchor_list, + cls_scores, + bbox_preds, + kps_preds, + labels_list, + label_weights_list, + bbox_targets_list, + keypoints_targets_list, + keypoints_weights_list, + self.anchor_generator.strides, + num_total_samples=num_total_samples) + + avg_factor = sum(avg_factor) + avg_factor = reduce_mean(avg_factor).item() + losses_bbox = list(map(lambda x: x / avg_factor, losses_bbox)) + losses = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) + if self.use_kps: + losses_kps = list(map(lambda x: x / avg_factor, losses_kps)) + losses['loss_kps'] = losses_kps + if self.use_dfl: + losses_dfl = list(map(lambda x: x / avg_factor, losses_dfl)) + losses['loss_dfl'] = losses_dfl + return losses + + @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'kps_preds')) + def get_bboxes(self, + cls_scores, + bbox_preds, + kps_preds, + img_metas, + cfg=None, + rescale=False, + with_nms=True): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. The second item is a + (n,) tensor where each item is the predicted class labelof the + corresponding box. + + Example: + >>> import mmcv + >>> self = AnchorHead( + >>> num_classes=9, + >>> in_channels=1, + >>> anchor_generator=dict( + >>> type='AnchorGenerator', + >>> scales=[8], + >>> ratios=[0.5, 1.0, 2.0], + >>> strides=[4,])) + >>> img_metas = [{'img_shape': (32, 32, 3), 'scale_factor': 1}] + >>> cfg = mmcv.Config(dict( + >>> score_thr=0.00, + >>> nms=dict(type='nms', iou_thr=1.0), + >>> max_per_img=10)) + >>> feat = torch.rand(1, 1, 3, 3) + >>> cls_score, bbox_pred = self.forward_single(feat) + >>> # note the input lists are over different levels, not images + >>> cls_scores, bbox_preds = [cls_score], [bbox_pred] + >>> result_list = self.get_bboxes(cls_scores, bbox_preds, + >>> img_metas, cfg) + >>> det_bboxes, det_labels = result_list[0] + >>> assert len(result_list) == 1 + >>> assert det_bboxes.shape[1] == 5 + >>> assert len(det_bboxes) == len(det_labels) == cfg.max_per_img + """ + assert len(cls_scores) == len(bbox_preds) + num_levels = len(cls_scores) + + device = cls_scores[0].device + featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] + mlvl_anchors = self.anchor_generator.grid_anchors( + featmap_sizes, device=device) + + result_list = [] + # bbox_preds and kps_preds are list of 3 tensor, each tensor is NCHW + # corresponding to a stage, C is 8 for bbox and 20 for kps + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + if self.use_kps: + kps_pred_list = [ + kps_preds[i][img_id].detach() for i in range(num_levels) + ] + else: + kps_pred_list = [None for i in range(num_levels)] + img_shape = img_metas[img_id]['img_shape'] + scale_factor = img_metas[img_id]['scale_factor'] + if with_nms: + # some heads don't support with_nms argument + proposals = self._get_bboxes_single(cls_score_list, + bbox_pred_list, + kps_pred_list, + mlvl_anchors, img_shape, + scale_factor, cfg, rescale) + else: + proposals = self._get_bboxes_single(cls_score_list, + bbox_pred_list, + kps_pred_list, + mlvl_anchors, img_shape, + scale_factor, cfg, rescale, + with_nms) + result_list.append(proposals) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + kps_preds, + mlvl_anchors, + img_shape, + scale_factor, + cfg, + rescale=False, + with_nms=True): + """Transform outputs for a single batch item into labeled boxes. + + Args: + cls_scores (list[Tensor]): Box scores for a single scale level + has shape (num_classes, H, W). + bbox_preds (list[Tensor]): Box distribution logits for a single + scale level with shape (4*(n+1), H, W), n is max value of + integral set. + mlvl_anchors (list[Tensor]): Box reference for a single scale level + with shape (num_total_anchors, 4). + img_shape (tuple[int]): Shape of the input image, + (height, width, 3). + scale_factor (ndarray): Scale factor of the image arange as + (w_scale, h_scale, w_scale, h_scale). + cfg (mmcv.Config | None): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + tuple(Tensor): + det_bboxes (Tensor): Bbox predictions in shape (N, 5), where + the first 4 columns are bounding box positions + (tl_x, tl_y, br_x, br_y) and the 5-th column is a score + between 0 and 1. + det_labels (Tensor): A (N,) tensor where each item is the + predicted class label of the corresponding box. + """ + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_kps = [] + for cls_score, bbox_pred, kps_pred, stride, anchors in zip( + cls_scores, bbox_preds, kps_preds, + self.anchor_generator.strides, mlvl_anchors): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + assert stride[0] == stride[1] + + scores = cls_score.permute(1, 2, 0).reshape( + -1, self.cls_out_channels).sigmoid() + bbox_pred = bbox_pred.permute(1, 2, 0) + if self.use_dfl: + bbox_pred = self.integral(bbox_pred) * stride[0] + else: + bbox_pred = bbox_pred.reshape((-1, 4)) * stride[0] + if kps_pred is not None: + kps_pred = kps_pred.permute(1, 2, 0) + if self.use_dfl: + kps_pred = self.integral(kps_pred) * stride[0] + else: + kps_pred = kps_pred.reshape((-1, self.NK * 2)) * stride[0] + + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + max_scores, _ = scores.max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + anchors = anchors[topk_inds, :] + bbox_pred = bbox_pred[topk_inds, :] + scores = scores[topk_inds, :] + if kps_pred is not None: + kps_pred = kps_pred[topk_inds, :] + + bboxes = distance2bbox( + self.anchor_center(anchors), bbox_pred, max_shape=img_shape) + mlvl_bboxes.append(bboxes) + mlvl_scores.append(scores) + if kps_pred is not None: + kps = distance2kps(self.anchor_center(anchors), kps_pred) + mlvl_kps.append(kps) + + mlvl_bboxes = torch.cat(mlvl_bboxes) + if mlvl_kps is not None: + mlvl_kps = torch.cat(mlvl_kps) + if rescale: + mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) + if mlvl_kps is not None: + scale_factor2 = torch.tensor( + [scale_factor[0], scale_factor[1]] * self.NK) + mlvl_kps /= scale_factor2.to(mlvl_kps.device) + + mlvl_scores = torch.cat(mlvl_scores) + # Add a dummy background class to the backend when using sigmoid + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + + if with_nms: + det_bboxes, det_labels, det_kps = multiclass_nms( + mlvl_bboxes, + mlvl_scores, + cfg.score_thr, + cfg.nms, + cfg.max_per_img, + multi_kps=mlvl_kps) + if det_kps is not None: + return det_bboxes, det_labels, det_kps + else: + return det_bboxes, det_labels + else: + if mlvl_kps is not None: + return mlvl_bboxes, mlvl_scores, mlvl_kps + else: + return mlvl_bboxes, mlvl_scores + + def get_targets(self, + anchor_list, + valid_flag_list, + gt_bboxes_list, + gt_keypointss_list, + img_metas, + gt_bboxes_ignore_list=None, + gt_labels_list=None, + label_channels=1, + unmap_outputs=True): + """Get targets for GFL head. + + This method is almost the same as `AnchorHead.get_targets()`. Besides + returning the targets as the parent method does, it also returns the + anchors as the first element of the returned tuple. + """ + num_imgs = len(img_metas) + assert len(anchor_list) == len(valid_flag_list) == num_imgs + + # anchor number of multi levels + num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] + num_level_anchors_list = [num_level_anchors] * num_imgs + + # concat all level anchors and flags to a single tensor + for i in range(num_imgs): + assert len(anchor_list[i]) == len(valid_flag_list[i]) + anchor_list[i] = torch.cat(anchor_list[i]) + valid_flag_list[i] = torch.cat(valid_flag_list[i]) + + # compute targets for each image + if gt_bboxes_ignore_list is None: + gt_bboxes_ignore_list = [None for _ in range(num_imgs)] + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + if gt_keypointss_list is None: + gt_keypointss_list = [None for _ in range(num_imgs)] + (all_anchors, all_labels, all_label_weights, all_bbox_targets, + all_bbox_weights, all_keypoints_targets, all_keypoints_weights, + pos_inds_list, neg_inds_list) = multi_apply( + self._get_target_single, + anchor_list, + valid_flag_list, + num_level_anchors_list, + gt_bboxes_list, + gt_bboxes_ignore_list, + gt_labels_list, + gt_keypointss_list, + img_metas, + label_channels=label_channels, + unmap_outputs=unmap_outputs) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + num_total_pos = sum([max(inds.numel(), 1) for inds in pos_inds_list]) + num_total_neg = sum([max(inds.numel(), 1) for inds in neg_inds_list]) + # split targets to a list w.r.t. multiple levels + anchors_list = images_to_levels(all_anchors, num_level_anchors) + labels_list = images_to_levels(all_labels, num_level_anchors) + label_weights_list = images_to_levels(all_label_weights, + num_level_anchors) + bbox_targets_list = images_to_levels(all_bbox_targets, + num_level_anchors) + bbox_weights_list = images_to_levels(all_bbox_weights, + num_level_anchors) + keypoints_targets_list = images_to_levels(all_keypoints_targets, + num_level_anchors) + keypoints_weights_list = images_to_levels(all_keypoints_weights, + num_level_anchors) + return (anchors_list, labels_list, label_weights_list, + bbox_targets_list, bbox_weights_list, keypoints_targets_list, + keypoints_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, + flat_anchors, + valid_flags, + num_level_anchors, + gt_bboxes, + gt_bboxes_ignore, + gt_labels, + gt_keypointss, + img_meta, + label_channels=1, + unmap_outputs=True): + """Compute regression, classification targets for anchors in a single + image. + + Args: + flat_anchors (Tensor): Multi-level anchors of the image, which are + concatenated into a single tensor of shape (num_anchors, 4) + valid_flags (Tensor): Multi level valid flags of the image, + which are concatenated into a single tensor of + shape (num_anchors,). + num_level_anchors Tensor): Number of anchors of each scale level. + gt_bboxes (Tensor): Ground truth bboxes of the image, + shape (num_gts, 4). + gt_bboxes_ignore (Tensor): Ground truth bboxes to be + ignored, shape (num_ignored_gts, 4). + gt_labels (Tensor): Ground truth labels of each box, + shape (num_gts,). + img_meta (dict): Meta info of the image. + label_channels (int): Channel of label. + unmap_outputs (bool): Whether to map outputs back to the original + set of anchors. + + Returns: + tuple: N is the number of total anchors in the image. + anchors (Tensor): All anchors in the image with shape (N, 4). + labels (Tensor): Labels of all anchors in the image with shape + (N,). + label_weights (Tensor): Label weights of all anchor in the + image with shape (N,). + bbox_targets (Tensor): BBox targets of all anchors in the + image with shape (N, 4). + bbox_weights (Tensor): BBox weights of all anchors in the + image with shape (N, 4). + pos_inds (Tensor): Indices of postive anchor with shape + (num_pos,). + neg_inds (Tensor): Indices of negative anchor with shape + (num_neg,). + """ + inside_flags = anchor_inside_flags(flat_anchors, valid_flags, + img_meta['img_shape'][:2], + self.train_cfg.allowed_border) + if not inside_flags.any(): + return (None, ) * 7 + # assign gt and sample anchors + anchors = flat_anchors[inside_flags, :] + + num_level_anchors_inside = self.get_num_level_anchors_inside( + num_level_anchors, inside_flags) + if self.assigner.__class__.__name__ == 'ATSSAssigner': + assign_result = self.assigner.assign(anchors, + num_level_anchors_inside, + gt_bboxes, gt_bboxes_ignore, + gt_labels) + else: + assign_result = self.assigner.assign(anchors, gt_bboxes, + gt_bboxes_ignore, gt_labels) + + sampling_result = self.sampler.sample(assign_result, anchors, + gt_bboxes) + + num_valid_anchors = anchors.shape[0] + bbox_targets = torch.zeros_like(anchors) + bbox_weights = torch.zeros_like(anchors) + kps_targets = anchors.new_zeros(size=(anchors.shape[0], self.NK * 2)) + kps_weights = anchors.new_zeros(size=(anchors.shape[0], self.NK * 2)) + labels = anchors.new_full((num_valid_anchors, ), + self.num_classes, + dtype=torch.long) + label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) + + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + if len(pos_inds) > 0: + pos_bbox_targets = sampling_result.pos_gt_bboxes + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + if self.use_kps: + pos_assigned_gt_inds = sampling_result.pos_assigned_gt_inds + kps_targets[pos_inds, :] = gt_keypointss[ + pos_assigned_gt_inds, :, :2].reshape((-1, self.NK * 2)) + kps_weights[pos_inds, :] = torch.mean( + gt_keypointss[pos_assigned_gt_inds, :, 2], + dim=1, + keepdims=True) + if gt_labels is None: + # Only rpn gives gt_labels as None + # Foreground is the first class + labels[pos_inds] = 0 + else: + labels[pos_inds] = gt_labels[ + sampling_result.pos_assigned_gt_inds] + if self.train_cfg.pos_weight <= 0: + label_weights[pos_inds] = 1.0 + else: + label_weights[pos_inds] = self.train_cfg.pos_weight + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + + # map up to original set of anchors + if unmap_outputs: + num_total_anchors = flat_anchors.size(0) + anchors = unmap(anchors, num_total_anchors, inside_flags) + labels = unmap( + labels, num_total_anchors, inside_flags, fill=self.num_classes) + label_weights = unmap(label_weights, num_total_anchors, + inside_flags) + bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) + bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) + if self.use_kps: + kps_targets = unmap(kps_targets, num_total_anchors, + inside_flags) + kps_weights = unmap(kps_weights, num_total_anchors, + inside_flags) + + return (anchors, labels, label_weights, bbox_targets, bbox_weights, + kps_targets, kps_weights, pos_inds, neg_inds) + + def get_num_level_anchors_inside(self, num_level_anchors, inside_flags): + split_inside_flags = torch.split(inside_flags, num_level_anchors) + num_level_anchors_inside = [ + int(flags.sum()) for flags in split_inside_flags + ] + return num_level_anchors_inside + + def aug_test(self, feats, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + feats (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains features for all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[ndarray]: bbox results of each class + """ + return self.aug_test_bboxes(feats, img_metas, rescale=rescale) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ee67dd1a345beb05e13be9100b951c685292e7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/__init__.py @@ -0,0 +1,8 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/detectors +""" +from .scrfd import SCRFD +from .tinymog import TinyMog + +__all__ = ['SCRFD', 'TinyMog'] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/base.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/base.py new file mode 100644 index 0000000000000000000000000000000000000000..3bae34d83401b5dc022c58f48e585b8abdfce0de --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/base.py @@ -0,0 +1,271 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly avaialbe at +https://github.com/deepinsight/insightface/blob/master/detection/scrfd/mmdet/models/detectors/base.py +""" +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import mmcv +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from mmcv.runner import auto_fp16 +from mmcv.utils import print_log +from mmdet.utils import get_root_logger + + +class BaseDetector(nn.Module, metaclass=ABCMeta): + """Base class for detectors.""" + + def __init__(self): + super(BaseDetector, self).__init__() + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the detector has a neck""" + return hasattr(self, 'neck') and self.neck is not None + + # TODO: these properties need to be carefully handled + # for both single stage & two stage detectors + @property + def with_shared_head(self): + """bool: whether the detector has a shared head in the RoI Head""" + return hasattr(self, 'roi_head') and self.roi_head.with_shared_head + + @property + def with_bbox(self): + """bool: whether the detector has a bbox head""" + return ((hasattr(self, 'roi_head') and self.roi_head.with_bbox) + or (hasattr(self, 'bbox_head') and self.bbox_head is not None)) + + @property + def with_mask(self): + """bool: whether the detector has a mask head""" + return ((hasattr(self, 'roi_head') and self.roi_head.with_mask) + or (hasattr(self, 'mask_head') and self.mask_head is not None)) + + @abstractmethod + def extract_feat(self, imgs): + """Extract features from images.""" + pass + + def extract_feats(self, imgs): + """Extract features from multiple images. + + Args: + imgs (list[torch.Tensor]): A list of images. The images are + augmented from the same image but in different ways. + + Returns: + list[torch.Tensor]: Features of different images + """ + assert isinstance(imgs, list) + return [self.extract_feat(img) for img in imgs] + + def forward_train(self, imgs, img_metas, **kwargs): + """ + Args: + img (list[Tensor]): List of tensors of shape (1, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys, see + :class:`mmdet.datasets.pipelines.Collect`. + kwargs (keyword arguments): Specific to concrete implementation. + """ + # NOTE the batched image size information may be useful, e.g. + # in DETR, this is needed for the construction of masks, which is + # then used for the transformer_head. + batch_input_shape = tuple(imgs[0].size()[-2:]) + for img_meta in img_metas: + img_meta['batch_input_shape'] = batch_input_shape + + async def async_simple_test(self, img, img_metas, **kwargs): + raise NotImplementedError + + @abstractmethod + def simple_test(self, img, img_metas, **kwargs): + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Test function with test time augmentation.""" + pass + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if pretrained is not None: + logger = get_root_logger() + print_log(f'load model from: {pretrained}', logger=logger) + + async def aforward_test(self, *, img, img_metas, **kwargs): + for var, name in [(img, 'img'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got {type(var)}') + + num_augs = len(img) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(img)}) ' + f'!= num of image metas ({len(img_metas)})') + # TODO: remove the restriction of samples_per_gpu == 1 when prepared + samples_per_gpu = img[0].size(0) + assert samples_per_gpu == 1 + + if num_augs == 1: + return await self.async_simple_test(img[0], img_metas[0], **kwargs) + else: + raise NotImplementedError + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got {type(var)}') + + imgs = [imgs[0]] + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) ' + f'!= num of image meta ({len(img_metas)})') + + # NOTE the batched image size information may be useful, e.g. + # in DETR, this is needed for the construction of masks, which is + # then used for the transformer_head. + for img, img_meta in zip(imgs, img_metas): + batch_size = len(img_meta) + for img_id in range(batch_size): + img_meta[img_id]['batch_input_shape'] = tuple(img.size()[-2:]) + + if num_augs == 1: + # proposals (List[List[Tensor]]): the outer list indicates + # test-time augs (multiscale, flip, etc.) and the inner list + # indicates images in a batch. + # The Tensor should have a shape Px4, where P is the number of + # proposals. + if 'proposals' in kwargs: + kwargs['proposals'] = kwargs['proposals'][0] + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + assert imgs[0].size(0) == 1, 'aug test does not support ' \ + 'inference with batch size ' \ + f'{imgs[0].size(0)}' + # TODO: support test augmentation for predefined proposals + assert 'proposals' not in kwargs + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def _parse_losses(self, losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary infomation. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor \ + which may be a weighted sum of all losses, log_vars contains \ + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def train_step(self, data, optimizer): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, \ + ``num_samples``. + + - ``loss`` is a tensor for back propagation, which can be a \ + weighted sum of multiple losses. + - ``log_vars`` contains all the variables to be sent to the + logger. + - ``num_samples`` indicates the batch size (when the model is \ + DDP, it means the batch size on each GPU), which is used for \ + averaging the logs. + """ + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs + + def val_step(self, data, optimizer): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/scrfd.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/scrfd.py new file mode 100644 index 0000000000000000000000000000000000000000..fb6392d341d94ac67c769cc9241845f8de031b12 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/scrfd.py @@ -0,0 +1,151 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/detectors/scrfd.py +""" +import torch +from mmdet.models.builder import DETECTORS + +from ....mmdet_patch.core.bbox import bbox2result +from ....mmdet_patch.models.detectors.single_stage import \ + CustomSingleStageDetector + + +@DETECTORS.register_module() +class SCRFD(CustomSingleStageDetector): + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(SCRFD, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_keypointss=None, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + super(CustomSingleStageDetector, self).forward_train(img, img_metas) + x = self.extract_feat(img) + losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, + gt_labels, gt_keypointss, + gt_bboxes_ignore) + return losses + + def simple_test(self, + img, + img_metas, + rescale=False, + repeat_head=1, + output_kps_var=0, + output_results=2): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + repeat_head (int): repeat inference times in head + output_kps_var (int): whether output kps var to calculate quality + output_results (int): 0: nothing 1: bbox 2: both bbox and kps + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + x = self.extract_feat(img) + assert repeat_head >= 1 + kps_out0 = [] + kps_out1 = [] + kps_out2 = [] + for i in range(repeat_head): + outs = self.bbox_head(x) + kps_out0 += [outs[2][0].detach().cpu().numpy()] + kps_out1 += [outs[2][1].detach().cpu().numpy()] + kps_out2 += [outs[2][2].detach().cpu().numpy()] + if output_kps_var: + var0 = np.var(np.vstack(kps_out0), axis=0).mean() + var1 = np.var(np.vstack(kps_out1), axis=0).mean() + var2 = np.var(np.vstack(kps_out2), axis=0).mean() + var = np.mean([var0, var1, var2]) + else: + var = None + + if output_results > 0: + if torch.onnx.is_in_onnx_export(): + print('single_stage.py in-onnx-export') + print(outs.__class__) + cls_score, bbox_pred, kps_pred = outs + for c in cls_score: + print(c.shape) + for c in bbox_pred: + print(c.shape) + if self.bbox_head.use_kps: + for c in kps_pred: + print(c.shape) + return (cls_score, bbox_pred, kps_pred) + else: + return (cls_score, bbox_pred) + bbox_list = self.bbox_head.get_bboxes( + *outs, img_metas, rescale=rescale) + + # return kps if use_kps + if len(bbox_list[0]) == 2: + bbox_results = [ + bbox2result(det_bboxes, det_labels, + self.bbox_head.num_classes) + for det_bboxes, det_labels in bbox_list + ] + elif len(bbox_list[0]) == 3: + if output_results == 2: + bbox_results = [ + bbox2result( + det_bboxes, + det_labels, + self.bbox_head.num_classes, + kps=det_kps, + num_kps=self.bbox_head.NK) + for det_bboxes, det_labels, det_kps in bbox_list + ] + elif output_results == 1: + bbox_results = [ + bbox2result(det_bboxes, det_labels, + self.bbox_head.num_classes) + for det_bboxes, det_labels, _ in bbox_list + ] + else: + bbox_results = None + if var is not None: + return bbox_results, var + else: + return bbox_results + + def feature_test(self, img): + x = self.extract_feat(img) + outs = self.bbox_head(x) + return outs diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/single_stage.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/single_stage.py new file mode 100644 index 0000000000000000000000000000000000000000..117eaa82a8bb16b4f8e19980fb8027c2a79f850f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/single_stage.py @@ -0,0 +1,163 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly avaialbe at +https://github.com/deepinsight/insightface/blob/master/detection/scrfd/mmdet/models/detectors/single_stage.py +""" +import torch +import torch.nn as nn +from mmdet.models.builder import (DETECTORS, build_backbone, build_head, + build_neck) + +from ....mmdet_patch.core.bbox import bbox2result +from .base import BaseDetector + + +@DETECTORS.register_module() +class CustomSingleStageDetector(BaseDetector): + """Base class for single-stage detectors. + + Single-stage detectors directly and densely predict bounding boxes on the + output features of the backbone+neck. + """ + + def __init__(self, + backbone, + neck=None, + bbox_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(CustomSingleStageDetector, self).__init__() + self.backbone = build_backbone(backbone) + if neck is not None: + self.neck = build_neck(neck) + bbox_head.update(train_cfg=train_cfg) + bbox_head.update(test_cfg=test_cfg) + self.bbox_head = build_head(bbox_head) + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.init_weights(pretrained=pretrained) + + def init_weights(self, pretrained=None): + """Initialize the weights in detector. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + super(CustomSingleStageDetector, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + if self.with_neck: + if isinstance(self.neck, nn.Sequential): + for m in self.neck: + m.init_weights() + else: + self.neck.init_weights() + self.bbox_head.init_weights() + + def extract_feat(self, img): + """Directly extract features from the backbone+neck.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def forward_dummy(self, img): + """Used for computing network flops. + + See `mmdetection/tools/get_flops.py` + """ + x = self.extract_feat(img) + outs = self.bbox_head(x) + return outs + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + super(CustomSingleStageDetector, self).forward_train(img, img_metas) + x = self.extract_feat(img) + losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, + gt_labels, gt_bboxes_ignore) + return losses + + def simple_test(self, img, img_metas, rescale=False): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + x = self.extract_feat(img) + outs = self.bbox_head(x) + if torch.onnx.is_in_onnx_export(): + print('single_stage.py in-onnx-export') + print(outs.__class__) + cls_score, bbox_pred = outs + for c in cls_score: + print(c.shape) + for c in bbox_pred: + print(c.shape) + return outs + bbox_list = self.bbox_head.get_bboxes( + *outs, img_metas, rescale=rescale) + # skip post-processing when exporting to ONNX + if torch.onnx.is_in_onnx_export(): + return bbox_list + + bbox_results = [ + bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes) + for det_bboxes, det_labels in bbox_list + ] + return bbox_results + + def aug_test(self, imgs, img_metas, rescale=False): + """Test function with test time augmentation. + + Args: + imgs (list[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (list[list[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. each dict has image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + assert hasattr(self.bbox_head, 'aug_test'), \ + f'{self.bbox_head.__class__.__name__}' \ + ' does not support test-time augmentation' + print('aug-test:', len(imgs)) + feats = self.extract_feats(imgs) + return [self.bbox_head.aug_test(feats, img_metas, rescale=rescale)] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/tinymog.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/tinymog.py new file mode 100644 index 0000000000000000000000000000000000000000..7aa9d5b2ee815976bd8dff6baa717a48dcaa6e52 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/mmdet_patch/models/detectors/tinymog.py @@ -0,0 +1,148 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/tree/master/detection/scrfd/mmdet/models/detectors/scrfd.py +""" +import torch +from mmdet.models.builder import DETECTORS +from mmdet.models.detectors.single_stage import SingleStageDetector + +from ....mmdet_patch.core.bbox import bbox2result + + +@DETECTORS.register_module() +class TinyMog(SingleStageDetector): + + def __init__(self, + backbone, + neck, + bbox_head, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(TinyMog, self).__init__(backbone, neck, bbox_head, train_cfg, + test_cfg, pretrained) + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_keypointss=None, + gt_bboxes_ignore=None): + """ + Args: + img (Tensor): Input images of shape (N, C, H, W). + Typically these should be mean centered and std scaled. + img_metas (list[dict]): A List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + :class:`mmdet.datasets.pipelines.Collect`. + gt_bboxes (list[Tensor]): Each item are the truth boxes for each + image in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): Class indices corresponding to each box + gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + super(SingleStageDetector, self).forward_train(img, img_metas) + x = self.extract_feat(img) + losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, + gt_labels, gt_keypointss, + gt_bboxes_ignore) + return losses + + def simple_test(self, + img, + img_metas, + rescale=False, + repeat_head=1, + output_kps_var=0, + output_results=1): + """Test function without test time augmentation. + + Args: + imgs (list[torch.Tensor]): List of multiple images + img_metas (list[dict]): List of image information. + rescale (bool, optional): Whether to rescale the results. + Defaults to False. + repeat_head (int): repeat inference times in head + output_kps_var (int): whether output kps var to calculate quality + output_results (int): 0: nothing 1: bbox 2: both bbox and kps + + Returns: + list[list[np.ndarray]]: BBox results of each image and classes. + The outer list corresponds to each image. The inner list + corresponds to each class. + """ + x = self.extract_feat(img) + assert repeat_head >= 1 + kps_out0 = [] + kps_out1 = [] + kps_out2 = [] + for i in range(repeat_head): + outs = self.bbox_head(x) + kps_out0 += [outs[2][0].detach().cpu().numpy()] + kps_out1 += [outs[2][1].detach().cpu().numpy()] + kps_out2 += [outs[2][2].detach().cpu().numpy()] + if output_kps_var: + var0 = np.var(np.vstack(kps_out0), axis=0).mean() + var1 = np.var(np.vstack(kps_out1), axis=0).mean() + var2 = np.var(np.vstack(kps_out2), axis=0).mean() + var = np.mean([var0, var1, var2]) + else: + var = None + + if output_results > 0: + if torch.onnx.is_in_onnx_export(): + cls_score, bbox_pred, kps_pred = outs + for c in cls_score: + print(c.shape) + for c in bbox_pred: + print(c.shape) + if self.bbox_head.use_kps: + for c in kps_pred: + print(c.shape) + return (cls_score, bbox_pred, kps_pred) + else: + return (cls_score, bbox_pred) + bbox_list = self.bbox_head.get_bboxes( + *outs, img_metas, rescale=rescale) + + # return kps if use_kps + if len(bbox_list[0]) == 2: + bbox_results = [ + bbox2result(det_bboxes, det_labels, + self.bbox_head.num_classes) + for det_bboxes, det_labels in bbox_list + ] + elif len(bbox_list[0]) == 3: + if output_results == 2: + bbox_results = [ + bbox2result( + det_bboxes, + det_labels, + self.bbox_head.num_classes, + kps=det_kps, + num_kps=self.bbox_head.NK) + for det_bboxes, det_labels, det_kps in bbox_list + ] + elif output_results == 1: + bbox_results = [ + bbox2result(det_bboxes, det_labels, + self.bbox_head.num_classes) + for det_bboxes, det_labels, _ in bbox_list + ] + else: + bbox_results = None + if var is not None: + return bbox_results, var + else: + return bbox_results + + def feature_test(self, img): + x = self.extract_feat(img) + outs = self.bbox_head(x) + return outs diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/preprocessor.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..4e400b9518e1d4fdc557de8bf43b6a33db8466e5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/preprocessor.py @@ -0,0 +1,92 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict, Union + +import numpy as np +from PIL import Image + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.preprocessors.image import LoadImage +from modelscope.utils.constant import Fields, ModeKeys + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.object_detection_scrfd) +class SCRFDPreprocessor(Preprocessor): + + def __init__(self, model_dir: str = None, mode: str = ModeKeys.INFERENCE): + """The base constructor for all the fill-mask preprocessors. + + Args: + model_dir (str): model directory to initialize some resource + mode: The mode for the preprocessor. + """ + super().__init__(mode) + pre_pipeline = [ + dict( + type='MultiScaleFlipAug', + img_scale=(640, 640), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.0), + dict( + type='Normalize', + mean=[127.5, 127.5, 127.5], + std=[128.0, 128.0, 128.0], + to_rgb=False), + dict(type='Pad', size=(640, 640), pad_val=0), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) + ] + from mmdet.datasets.pipelines import Compose + self.pipeline = Compose(pre_pipeline) + + def __call__(self, data: Union[str, Dict], **kwargs) -> Dict[str, Any]: + """process the raw input data + Args: + data (str or dict): image path or data dict containing following info: + filename, ori_filename, img, img_shape, ori_shape, img_fields + Example: + >>> { + >>> "filename": "xxx.jpg" + >>> "ori_filename": "xxx.jpg", + >>> "img": np.ndarray, + >>> "img_shape": (300, 300, 3) + >>> "ori_shape": (300, 300, 3) + >>> "img_fields": "img" + >>> } + + Returns: + Dict[str, Any]: the preprocessed data + """ + if isinstance(data, str): + img = LoadImage.convert_to_ndarray(data) + img = img.astype(np.float32) + data_dict = {} + data_dict['filename'] = '' + data_dict['ori_filename'] = '' + data_dict['img'] = img + data_dict['img_shape'] = img.shape + data_dict['ori_shape'] = img.shape + data_dict['img_fields'] = ['img'] + elif isinstance(data, (np.ndarray, Image.Image)): + if isinstance(data, Image.Image): + data = LoadImage.convert_to_ndarray(data) + + data = data.astype(np.float32) + data_dict = {} + data_dict['filename'] = '' + data_dict['ori_filename'] = '' + data_dict['img'] = data + data_dict['img_shape'] = data.shape + data_dict['ori_shape'] = data.shape + data_dict['img_fields'] = ['img'] + + elif isinstance(data, dict): + data_dict = data + + return self.pipeline(data_dict) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/scrfd_detect.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/scrfd_detect.py new file mode 100644 index 0000000000000000000000000000000000000000..e382ec9ab0570a4ea5fbdcab24ac57281882debd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/scrfd_detect.py @@ -0,0 +1,96 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from copy import deepcopy +from typing import Any, Dict, List, Union + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.outputs.cv_outputs import DetectionOutput +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ScrfdDetect'] + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.scrfd) +@MODELS.register_module(Tasks.card_detection, module_name=Models.scrfd) +class ScrfdDetect(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the face detection model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + from mmcv import Config + from mmcv.parallel import MMDataParallel + from mmcv.runner import load_checkpoint + from mmdet.models import build_detector + from modelscope.models.cv.face_detection.scrfd.mmdet_patch.datasets import RetinaFaceDataset + from modelscope.models.cv.face_detection.scrfd.mmdet_patch.datasets.pipelines import RandomSquareCrop + from modelscope.models.cv.face_detection.scrfd.mmdet_patch.models.backbones import ResNetV1e + from modelscope.models.cv.face_detection.scrfd.mmdet_patch.models.dense_heads import SCRFDHead + from modelscope.models.cv.face_detection.scrfd.mmdet_patch.models.detectors import SCRFD + cfg_file = kwargs.get('config_file', 'mmcv_scrfd.py') + cfg = Config.fromfile(osp.join(model_dir, cfg_file)) + model_file = kwargs.get('model_file', ModelFile.TORCH_MODEL_BIN_FILE) + ckpt_path = osp.join(model_dir, model_file) + cfg.model.test_cfg.score_thr = kwargs.get('score_thr', 0.3) + detector = build_detector(cfg.model) + logger.info(f'loading model from {ckpt_path}') + load_checkpoint(detector, ckpt_path, map_location='cpu') + detector = MMDataParallel(detector, device_ids=[0]) + detector.eval() + self.detector = detector + logger.info('load model done') + + def forward( + self, img: Union[torch.Tensor, List[torch.Tensor]], + img_metas: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]] + ) -> DetectionOutput: + """ + Args: + img (`torch.Tensor` or `List[torch.Tensor]`): batched image tensor or list of + batched image tensor, shape of each tensor is [N, h, w, 3]. When input is + a list, each element is a different augmentation image to do multi-view + augmentation test. + img_metas (`List[List[Dict[str, Any]]]`): image meta info. + + Return: + `:obj:DetectionOutput` + """ + if isinstance(img, torch.Tensor): + img = [img] + img_metas = [img_metas] + + result = self.detector( + return_loss=False, + rescale=True, + img=img, + img_metas=img_metas, + output_results=2) + assert result is not None + result = result[0][0] + bboxes = result[:, :4] + kpss = result[:, 5:] + scores = result[:, 4] + return DetectionOutput(scores=scores, boxes=bboxes, keypoints=kpss) + + def postprocess(self, detection_out: DetectionOutput, + **kwargs) -> Dict[str, Any]: + scores = detection_out['scores'].tolist() + boxes = detection_out['boxes'].tolist() + kpss = detection_out['keypoints'].tolist() + return { + OutputKeys.SCORES: scores, + OutputKeys.BOXES: boxes, + OutputKeys.KEYPOINTS: kpss + } diff --git a/runtime/Lib/site_packages/models/cv/face_detection/scrfd/tinymog_detect.py b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/tinymog_detect.py new file mode 100644 index 0000000000000000000000000000000000000000..24c36d25207ce1108284ac323118fb38debd40f0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/scrfd/tinymog_detect.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from copy import deepcopy +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .scrfd_detect import ScrfdDetect + +logger = get_logger() + +__all__ = ['TinyMogDetect'] + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.tinymog) +class TinyMogDetect(ScrfdDetect): + + def __init__(self, model_dir, *args, **kwargs): + """ + initialize the tinymog face detection model from the `model_dir` path. + """ + config_file = 'mmcv_tinymog.py' + kwargs['config_file'] = config_file + kwargs['model_file'] = ModelFile.TORCH_MODEL_FILE + super().__init__(model_dir, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..af1e7b42a6be89d366a742c1019feb70e13fa59b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .detection import UlfdFaceDetector diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/detection.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/detection.py new file mode 100644 index 0000000000000000000000000000000000000000..25cad11559e9f0deff9ccc6a9b37218e16e937e8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/detection.py @@ -0,0 +1,43 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from .vision.ssd.fd_config import define_img_size +from .vision.ssd.mb_tiny_fd import (create_mb_tiny_fd, + create_mb_tiny_fd_predictor) + +define_img_size(640) + + +@MODELS.register_module(Tasks.face_detection, module_name=Models.ulfd) +class UlfdFaceDetector(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.net = create_mb_tiny_fd(2, is_test=True, device=device) + self.predictor = create_mb_tiny_fd_predictor( + self.net, candidate_size=1500, device=device) + self.net.load(model_path) + self.net = self.net.to(device) + + def forward(self, input): + img_raw = input['img'] + img = np.array(img_raw.cpu().detach()) + img = img[:, :, ::-1] + prob_th = 0.85 + keep_top_k = 750 + boxes, labels, probs = self.predictor.predict(img, keep_top_k, prob_th) + return boxes, probs diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/box_utils.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/box_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..46d3b890814e138865d2e1084f8ee0efe254078f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/box_utils.py @@ -0,0 +1,124 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import math + +import torch + + +def hard_nms(box_scores, iou_threshold, top_k=-1, candidate_size=200): + """ + + Args: + box_scores (N, 5): boxes in corner-form and probabilities. + iou_threshold: intersection over union threshold. + top_k: keep top_k results. If k <= 0, keep all the results. + candidate_size: only consider the candidates with the highest scores. + Returns: + picked: a list of indexes of the kept boxes + """ + scores = box_scores[:, -1] + boxes = box_scores[:, :-1] + picked = [] + _, indexes = scores.sort(descending=True) + indexes = indexes[:candidate_size] + while len(indexes) > 0: + current = indexes[0] + picked.append(current.item()) + if 0 < top_k == len(picked) or len(indexes) == 1: + break + current_box = boxes[current, :] + indexes = indexes[1:] + rest_boxes = boxes[indexes, :] + iou = iou_of( + rest_boxes, + current_box.unsqueeze(0), + ) + indexes = indexes[iou <= iou_threshold] + + return box_scores[picked, :] + + +def nms(box_scores, + nms_method=None, + score_threshold=None, + iou_threshold=None, + sigma=0.5, + top_k=-1, + candidate_size=200): + return hard_nms( + box_scores, iou_threshold, top_k, candidate_size=candidate_size) + + +def generate_priors(feature_map_list, + shrinkage_list, + image_size, + min_boxes, + clamp=True) -> torch.Tensor: + priors = [] + for index in range(0, len(feature_map_list[0])): + scale_w = image_size[0] / shrinkage_list[0][index] + scale_h = image_size[1] / shrinkage_list[1][index] + for j in range(0, feature_map_list[1][index]): + for i in range(0, feature_map_list[0][index]): + x_center = (i + 0.5) / scale_w + y_center = (j + 0.5) / scale_h + + for min_box in min_boxes[index]: + w = min_box / image_size[0] + h = min_box / image_size[1] + priors.append([x_center, y_center, w, h]) + priors = torch.tensor(priors) + if clamp: + torch.clamp(priors, 0.0, 1.0, out=priors) + return priors + + +def convert_locations_to_boxes(locations, priors, center_variance, + size_variance): + # priors can have one dimension less. + if priors.dim() + 1 == locations.dim(): + priors = priors.unsqueeze(0) + a = locations[..., :2] * center_variance * priors[..., + 2:] + priors[..., :2] + b = torch.exp(locations[..., 2:] * size_variance) * priors[..., 2:] + + return torch.cat([a, b], dim=locations.dim() - 1) + + +def center_form_to_corner_form(locations): + a = locations[..., :2] - locations[..., 2:] / 2 + b = locations[..., :2] + locations[..., 2:] / 2 + return torch.cat([a, b], locations.dim() - 1) + + +def iou_of(boxes0, boxes1, eps=1e-5): + """Return intersection-over-union (Jaccard index) of boxes. + + Args: + boxes0 (N, 4): ground truth boxes. + boxes1 (N or 1, 4): predicted boxes. + eps: a small number to avoid 0 as denominator. + Returns: + iou (N): IoU values. + """ + overlap_left_top = torch.max(boxes0[..., :2], boxes1[..., :2]) + overlap_right_bottom = torch.min(boxes0[..., 2:], boxes1[..., 2:]) + + overlap_area = area_of(overlap_left_top, overlap_right_bottom) + area0 = area_of(boxes0[..., :2], boxes0[..., 2:]) + area1 = area_of(boxes1[..., :2], boxes1[..., 2:]) + return overlap_area / (area0 + area1 - overlap_area + eps) + + +def area_of(left_top, right_bottom) -> torch.Tensor: + """Compute the areas of rectangles given two corners. + + Args: + left_top (N, 2): left top corner. + right_bottom (N, 2): right bottom corner. + + Returns: + area (N): return the area. + """ + hw = torch.clamp(right_bottom - left_top, min=0.0) + return hw[..., 0] * hw[..., 1] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/mb_tiny.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/mb_tiny.py new file mode 100644 index 0000000000000000000000000000000000000000..8bbcef416ea9fca6b655c91292343cd1940224a7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/mb_tiny.py @@ -0,0 +1,49 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import torch.nn as nn +import torch.nn.functional as F + + +class Mb_Tiny(nn.Module): + + def __init__(self, num_classes=2): + super(Mb_Tiny, self).__init__() + self.base_channel = 8 * 2 + + def conv_bn(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), nn.ReLU(inplace=True)) + + def conv_dw(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.ReLU(inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True), + ) + + self.model = nn.Sequential( + conv_bn(3, self.base_channel, 2), # 160*120 + conv_dw(self.base_channel, self.base_channel * 2, 1), + conv_dw(self.base_channel * 2, self.base_channel * 2, 2), # 80*60 + conv_dw(self.base_channel * 2, self.base_channel * 2, 1), + conv_dw(self.base_channel * 2, self.base_channel * 4, 2), # 40*30 + conv_dw(self.base_channel * 4, self.base_channel * 4, 1), + conv_dw(self.base_channel * 4, self.base_channel * 4, 1), + conv_dw(self.base_channel * 4, self.base_channel * 4, 1), + conv_dw(self.base_channel * 4, self.base_channel * 8, 2), # 20*15 + conv_dw(self.base_channel * 8, self.base_channel * 8, 1), + conv_dw(self.base_channel * 8, self.base_channel * 8, 1), + conv_dw(self.base_channel * 8, self.base_channel * 16, 2), # 10*8 + conv_dw(self.base_channel * 16, self.base_channel * 16, 1)) + self.fc = nn.Linear(1024, num_classes) + + def forward(self, x): + x = self.model(x) + x = F.avg_pool2d(x, 7) + x = x.view(-1, 1024) + x = self.fc(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/__init__.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/data_preprocessing.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/data_preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..9251d67fdc3985f1065ebd5b843d3ab59821fbba --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/data_preprocessing.py @@ -0,0 +1,18 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +from ..transforms import Compose, Resize, SubtractMeans, ToTensor + + +class PredictionTransform: + + def __init__(self, size, mean=0.0, std=1.0): + self.transform = Compose([ + Resize(size), + SubtractMeans(mean), lambda img, boxes=None, labels=None: + (img / std, boxes, labels), + ToTensor() + ]) + + def __call__(self, image): + image, _, _ = self.transform(image) + return image diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/fd_config.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/fd_config.py new file mode 100644 index 0000000000000000000000000000000000000000..495a2fcdba6792450264da979748e3d62d245391 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/fd_config.py @@ -0,0 +1,49 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import numpy as np + +from ..box_utils import generate_priors + +image_mean_test = image_mean = np.array([127, 127, 127]) +image_std = 128.0 +iou_threshold = 0.3 +center_variance = 0.1 +size_variance = 0.2 + +min_boxes = [[10, 16, 24], [32, 48], [64, 96], [128, 192, 256]] +shrinkage_list = [] +image_size = [320, 240] # default input size 320*240 +feature_map_w_h_list = [[40, 20, 10, 5], [30, 15, 8, + 4]] # default feature map size +priors = [] + + +def define_img_size(size): + global image_size, feature_map_w_h_list, priors + img_size_dict = { + 128: [128, 96], + 160: [160, 120], + 320: [320, 240], + 480: [480, 360], + 640: [640, 480], + 1280: [1280, 960] + } + image_size = img_size_dict[size] + + feature_map_w_h_list_dict = { + 128: [[16, 8, 4, 2], [12, 6, 3, 2]], + 160: [[20, 10, 5, 3], [15, 8, 4, 2]], + 320: [[40, 20, 10, 5], [30, 15, 8, 4]], + 480: [[60, 30, 15, 8], [45, 23, 12, 6]], + 640: [[80, 40, 20, 10], [60, 30, 15, 8]], + 1280: [[160, 80, 40, 20], [120, 60, 30, 15]] + } + feature_map_w_h_list = feature_map_w_h_list_dict[size] + + for i in range(0, len(image_size)): + item_list = [] + for k in range(0, len(feature_map_w_h_list[i])): + item_list.append(image_size[i] / feature_map_w_h_list[i][k]) + shrinkage_list.append(item_list) + priors = generate_priors(feature_map_w_h_list, shrinkage_list, image_size, + min_boxes) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/mb_tiny_fd.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/mb_tiny_fd.py new file mode 100644 index 0000000000000000000000000000000000000000..91ed268df862b2978d46064c44c7e73c0959c300 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/mb_tiny_fd.py @@ -0,0 +1,124 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +from torch.nn import Conv2d, ModuleList, ReLU, Sequential + +from ..mb_tiny import Mb_Tiny +from . import fd_config as config +from .predictor import Predictor +from .ssd import SSD + + +def SeperableConv2d(in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0): + """Replace Conv2d with a depthwise Conv2d and Pointwise Conv2d. + """ + return Sequential( + Conv2d( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=kernel_size, + groups=in_channels, + stride=stride, + padding=padding), + ReLU(), + Conv2d( + in_channels=in_channels, out_channels=out_channels, kernel_size=1), + ) + + +def create_mb_tiny_fd(num_classes, is_test=False, device='cuda'): + base_net = Mb_Tiny(2) + base_net_model = base_net.model # disable dropout layer + + source_layer_indexes = [8, 11, 13] + extras = ModuleList([ + Sequential( + Conv2d( + in_channels=base_net.base_channel * 16, + out_channels=base_net.base_channel * 4, + kernel_size=1), ReLU(), + SeperableConv2d( + in_channels=base_net.base_channel * 4, + out_channels=base_net.base_channel * 16, + kernel_size=3, + stride=2, + padding=1), ReLU()) + ]) + + regression_headers = ModuleList([ + SeperableConv2d( + in_channels=base_net.base_channel * 4, + out_channels=3 * 4, + kernel_size=3, + padding=1), + SeperableConv2d( + in_channels=base_net.base_channel * 8, + out_channels=2 * 4, + kernel_size=3, + padding=1), + SeperableConv2d( + in_channels=base_net.base_channel * 16, + out_channels=2 * 4, + kernel_size=3, + padding=1), + Conv2d( + in_channels=base_net.base_channel * 16, + out_channels=3 * 4, + kernel_size=3, + padding=1) + ]) + + classification_headers = ModuleList([ + SeperableConv2d( + in_channels=base_net.base_channel * 4, + out_channels=3 * num_classes, + kernel_size=3, + padding=1), + SeperableConv2d( + in_channels=base_net.base_channel * 8, + out_channels=2 * num_classes, + kernel_size=3, + padding=1), + SeperableConv2d( + in_channels=base_net.base_channel * 16, + out_channels=2 * num_classes, + kernel_size=3, + padding=1), + Conv2d( + in_channels=base_net.base_channel * 16, + out_channels=3 * num_classes, + kernel_size=3, + padding=1) + ]) + + return SSD( + num_classes, + base_net_model, + source_layer_indexes, + extras, + classification_headers, + regression_headers, + is_test=is_test, + config=config, + device=device) + + +def create_mb_tiny_fd_predictor(net, + candidate_size=200, + nms_method=None, + sigma=0.5, + device=None): + predictor = Predictor( + net, + config.image_size, + config.image_mean_test, + config.image_std, + nms_method=nms_method, + iou_threshold=config.iou_threshold, + candidate_size=candidate_size, + sigma=sigma, + device=device) + return predictor diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/predictor.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..e5a361c211733a97a143890b666383ec34f5c6dd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/predictor.py @@ -0,0 +1,79 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import torch + +from .. import box_utils +from .data_preprocessing import PredictionTransform + + +class Predictor: + + def __init__(self, + net, + size, + mean=0.0, + std=1.0, + nms_method=None, + iou_threshold=0.3, + filter_threshold=0.85, + candidate_size=200, + sigma=0.5, + device=None): + self.net = net + self.transform = PredictionTransform(size, mean, std) + self.iou_threshold = iou_threshold + self.filter_threshold = filter_threshold + self.candidate_size = candidate_size + self.nms_method = nms_method + + self.sigma = sigma + if device: + self.device = device + else: + self.device = torch.device( + 'cuda:0' if torch.cuda.is_available() else 'cpu') + + self.net.to(self.device) + self.net.eval() + + def predict(self, image, top_k=-1, prob_threshold=None): + height, width, _ = image.shape + image = self.transform(image) + images = image.unsqueeze(0) + images = images.to(self.device) + with torch.no_grad(): + scores, boxes = self.net.forward(images) + boxes = boxes[0] + scores = scores[0] + if not prob_threshold: + prob_threshold = self.filter_threshold + # this version of nms is slower on GPU, so we move data to CPU. + picked_box_probs = [] + picked_labels = [] + for class_index in range(1, scores.size(1)): + probs = scores[:, class_index] + mask = probs > prob_threshold + probs = probs[mask] + if probs.size(0) == 0: + continue + subset_boxes = boxes[mask, :] + box_probs = torch.cat([subset_boxes, probs.reshape(-1, 1)], dim=1) + box_probs = box_utils.nms( + box_probs, + self.nms_method, + score_threshold=prob_threshold, + iou_threshold=self.iou_threshold, + sigma=self.sigma, + top_k=top_k, + candidate_size=self.candidate_size) + picked_box_probs.append(box_probs) + picked_labels.extend([class_index] * box_probs.size(0)) + if not picked_box_probs: + return torch.tensor([]), torch.tensor([]), torch.tensor([]) + picked_box_probs = torch.cat(picked_box_probs) + picked_box_probs[:, 0] *= width + picked_box_probs[:, 1] *= height + picked_box_probs[:, 2] *= width + picked_box_probs[:, 3] *= height + return picked_box_probs[:, :4], torch.tensor( + picked_labels), picked_box_probs[:, 4] diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/ssd.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/ssd.py new file mode 100644 index 0000000000000000000000000000000000000000..08ff93a4f7dbbcb05f7dad296877296ad8e76636 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/ssd/ssd.py @@ -0,0 +1,129 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +from collections import namedtuple +from typing import List, Tuple + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .. import box_utils + +GraphPath = namedtuple('GraphPath', ['s0', 'name', 's1']) + + +class SSD(nn.Module): + + def __init__(self, + num_classes: int, + base_net: nn.ModuleList, + source_layer_indexes: List[int], + extras: nn.ModuleList, + classification_headers: nn.ModuleList, + regression_headers: nn.ModuleList, + is_test=False, + config=None, + device=None): + """Compose a SSD model using the given components. + """ + super(SSD, self).__init__() + + self.num_classes = num_classes + self.base_net = base_net + self.source_layer_indexes = source_layer_indexes + self.extras = extras + self.classification_headers = classification_headers + self.regression_headers = regression_headers + self.is_test = is_test + self.config = config + + # register layers in source_layer_indexes by adding them to a module list + self.source_layer_add_ons = nn.ModuleList([ + t[1] for t in source_layer_indexes + if isinstance(t, tuple) and not isinstance(t, GraphPath) + ]) + if device: + self.device = device + else: + self.device = torch.device( + 'cuda:0' if torch.cuda.is_available() else 'cpu') + if is_test: + self.config = config + self.priors = config.priors.to(self.device) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + confidences = [] + locations = [] + start_layer_index = 0 + header_index = 0 + end_layer_index = 0 + for end_layer_index in self.source_layer_indexes: + if isinstance(end_layer_index, GraphPath): + path = end_layer_index + end_layer_index = end_layer_index.s0 + added_layer = None + elif isinstance(end_layer_index, tuple): + added_layer = end_layer_index[1] + end_layer_index = end_layer_index[0] + path = None + else: + added_layer = None + path = None + for layer in self.base_net[start_layer_index:end_layer_index]: + x = layer(x) + if added_layer: + y = added_layer(x) + else: + y = x + if path: + sub = getattr(self.base_net[end_layer_index], path.name) + for layer in sub[:path.s1]: + x = layer(x) + y = x + for layer in sub[path.s1:]: + x = layer(x) + end_layer_index += 1 + start_layer_index = end_layer_index + confidence, location = self.compute_header(header_index, y) + header_index += 1 + confidences.append(confidence) + locations.append(location) + + for layer in self.base_net[end_layer_index:]: + x = layer(x) + + for layer in self.extras: + x = layer(x) + confidence, location = self.compute_header(header_index, x) + header_index += 1 + confidences.append(confidence) + locations.append(location) + + confidences = torch.cat(confidences, 1) + locations = torch.cat(locations, 1) + + if self.is_test: + confidences = F.softmax(confidences, dim=2) + boxes = box_utils.convert_locations_to_boxes( + locations, self.priors, self.config.center_variance, + self.config.size_variance) + boxes = box_utils.center_form_to_corner_form(boxes) + return confidences, boxes + else: + return confidences, locations + + def compute_header(self, i, x): + confidence = self.classification_headers[i](x) + confidence = confidence.permute(0, 2, 3, 1).contiguous() + confidence = confidence.view(confidence.size(0), -1, self.num_classes) + + location = self.regression_headers[i](x) + location = location.permute(0, 2, 3, 1).contiguous() + location = location.view(location.size(0), -1, 4) + + return confidence, location + + def load(self, model): + self.load_state_dict( + torch.load(model, map_location=lambda storage, loc: storage)) diff --git a/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/transforms.py b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..7c5331f13f880d8a3b2b59261c4798c966014d27 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_detection/ulfd_slim/vision/transforms.py @@ -0,0 +1,56 @@ +# The implementation is based on ULFD, available at +# https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB +import types + +import cv2 +import numpy as np +import torch +from numpy import random + + +class Compose(object): + """Composes several augmentations together. + Args: + transforms (List[Transform]): list of transforms to compose. + Example: + >>> augmentations.Compose([ + >>> transforms.CenterCrop(10), + >>> transforms.ToTensor(), + >>> ]) + """ + + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img, boxes=None, labels=None): + for t in self.transforms: + img, boxes, labels = t(img, boxes, labels) + return img, boxes, labels + + +class SubtractMeans(object): + + def __init__(self, mean): + self.mean = np.array(mean, dtype=np.float32) + + def __call__(self, image, boxes=None, labels=None): + image = image.astype(np.float32) + image -= self.mean + return image.astype(np.float32), boxes, labels + + +class Resize(object): + + def __init__(self, size=(300, 300)): + self.size = size + + def __call__(self, image, boxes=None, labels=None): + image = cv2.resize(image, (self.size[0], self.size[1])) + return image, boxes, labels + + +class ToTensor(object): + + def __call__(self, cvimage, boxes=None, labels=None): + return torch.from_numpy(cvimage.astype(np.float32)).permute( + 2, 0, 1), boxes, labels diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/__init__.py b/runtime/Lib/site_packages/models/cv/face_emotion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a13ea42628d6d77d9fdf4df0ae3a78bfd20da3d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .emotion_model import EfficientNetForFaceEmotion + +else: + _import_structure = {'emotion_model': ['EfficientNetForFaceEmotion']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/efficient/__init__.py b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3edaf9e212bd1821b1537bfcd854abdbdac4ec4d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/__init__.py @@ -0,0 +1,6 @@ +# The implementation here is modified based on EfficientNet, +# originally Apache 2.0 License and publicly available at https://github.com/lukemelas/EfficientNet-PyTorch + +from .model import VALID_MODELS, EfficientNet +from .utils import (BlockArgs, BlockDecoder, GlobalParams, efficientnet, + get_model_params) diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/efficient/model.py b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/model.py new file mode 100644 index 0000000000000000000000000000000000000000..aa0be250ab0254846d9ed451c6884b3aac36e93f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/model.py @@ -0,0 +1,380 @@ +# The implementation here is modified based on EfficientNet, +# originally Apache 2.0 License and publicly available at https://github.com/lukemelas/EfficientNet-PyTorch + +import torch +from torch import nn +from torch.nn import functional as F + +from .utils import (MemoryEfficientSwish, Swish, calculate_output_image_size, + drop_connect, efficientnet_params, get_model_params, + get_same_padding_conv2d, load_pretrained_weights, + round_filters, round_repeats) + +VALID_MODELS = ('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', + 'efficientnet-b3', 'efficientnet-b4', 'efficientnet-b5', + 'efficientnet-b6', 'efficientnet-b7', 'efficientnet-b8', + 'efficientnet-l2') + + +class MBConvBlock(nn.Module): + + def __init__(self, block_args, global_params, image_size=None): + super().__init__() + self._block_args = block_args + self._bn_mom = 1 - global_params.batch_norm_momentum + self._bn_eps = global_params.batch_norm_epsilon + self.has_se = (self._block_args.se_ratio + is not None) and (0 < self._block_args.se_ratio <= 1) + self.id_skip = block_args.id_skip + + inp = self._block_args.input_filters + oup = self._block_args.input_filters * self._block_args.expand_ratio + if self._block_args.expand_ratio != 1: + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._expand_conv = Conv2d( + in_channels=inp, out_channels=oup, kernel_size=1, bias=False) + self._bn0 = nn.BatchNorm2d( + num_features=oup, momentum=self._bn_mom, eps=self._bn_eps) + + k = self._block_args.kernel_size + s = self._block_args.stride + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._depthwise_conv = Conv2d( + in_channels=oup, + out_channels=oup, + groups=oup, + kernel_size=k, + stride=s, + bias=False) + self._bn1 = nn.BatchNorm2d( + num_features=oup, momentum=self._bn_mom, eps=self._bn_eps) + image_size = calculate_output_image_size(image_size, s) + + if self.has_se: + Conv2d = get_same_padding_conv2d(image_size=(1, 1)) + num_squeezed_channels = max( + 1, + int(self._block_args.input_filters + * self._block_args.se_ratio)) + self._se_reduce = Conv2d( + in_channels=oup, + out_channels=num_squeezed_channels, + kernel_size=1) + self._se_expand = Conv2d( + in_channels=num_squeezed_channels, + out_channels=oup, + kernel_size=1) + + final_oup = self._block_args.output_filters + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._project_conv = Conv2d( + in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False) + self._bn2 = nn.BatchNorm2d( + num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps) + self._swish = MemoryEfficientSwish() + + def forward(self, inputs, drop_connect_rate=None): + """MBConvBlock's forward function. + Args: + inputs (tensor): Input tensor. + drop_connect_rate (bool): Drop connect rate (float, between 0 and 1). + Returns: + Output of this block after processing. + """ + + x = inputs + if self._block_args.expand_ratio != 1: + x = self._expand_conv(inputs) + x = self._bn0(x) + x = self._swish(x) + + x = self._depthwise_conv(x) + x = self._bn1(x) + x = self._swish(x) + + if self.has_se: + x_squeezed = F.adaptive_avg_pool2d(x, 1) + x_squeezed = self._se_reduce(x_squeezed) + x_squeezed = self._swish(x_squeezed) + x_squeezed = self._se_expand(x_squeezed) + x = torch.sigmoid(x_squeezed) * x + + x = self._project_conv(x) + x = self._bn2(x) + + input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters + if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters: + if drop_connect_rate: + x = drop_connect( + x, p=drop_connect_rate, training=self.training) + x = x + inputs + return x + + def set_swish(self, memory_efficient=True): + """Sets swish function as memory efficient (for training) or standard (for export). + Args: + memory_efficient (bool): Whether to use memory-efficient version of swish. + """ + self._swish = MemoryEfficientSwish() if memory_efficient else Swish() + + +class EfficientNet(nn.Module): + """EfficientNet model. + Most easily loaded with the .from_name or .from_pretrained methods. + Args: + blocks_args (list[namedtuple]): A list of BlockArgs to construct blocks. + global_params (namedtuple): A set of GlobalParams shared between blocks. + References: + [1] https://arxiv.org/abs/1905.11946 (EfficientNet) + Example: + >>> import torch + >>> from efficientnet.model import EfficientNet + >>> inputs = torch.rand(1, 3, 224, 224) + >>> model = EfficientNet.from_pretrained('efficientnet-b0') + >>> model.eval() + >>> outputs = model(inputs) + """ + + def __init__(self, blocks_args=None, global_params=None): + super().__init__() + assert isinstance(blocks_args, list), 'blocks_args should be a list' + assert len(blocks_args) > 0, 'block args must be greater than 0' + self._global_params = global_params + self._blocks_args = blocks_args + + bn_mom = 1 - self._global_params.batch_norm_momentum + bn_eps = self._global_params.batch_norm_epsilon + image_size = global_params.image_size + Conv2d = get_same_padding_conv2d(image_size=image_size) + + in_channels = 3 + out_channels = round_filters(32, self._global_params) + self._conv_stem = Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=False) + self._bn0 = nn.BatchNorm2d( + num_features=out_channels, momentum=bn_mom, eps=bn_eps) + image_size = calculate_output_image_size(image_size, 2) + + self._blocks = nn.ModuleList([]) + for block_args in self._blocks_args: + + block_args = block_args._replace( + input_filters=round_filters(block_args.input_filters, + self._global_params), + output_filters=round_filters(block_args.output_filters, + self._global_params), + num_repeat=round_repeats(block_args.num_repeat, + self._global_params)) + + self._blocks.append( + MBConvBlock( + block_args, self._global_params, image_size=image_size)) + image_size = calculate_output_image_size(image_size, + block_args.stride) + if block_args.num_repeat > 1: + block_args = block_args._replace( + input_filters=block_args.output_filters, stride=1) + for _ in range(block_args.num_repeat - 1): + self._blocks.append( + MBConvBlock( + block_args, self._global_params, + image_size=image_size)) + + in_channels = block_args.output_filters + out_channels = round_filters(1280, self._global_params) + Conv2d = get_same_padding_conv2d(image_size=image_size) + self._conv_head = Conv2d( + in_channels, out_channels, kernel_size=1, bias=False) + self._bn1 = nn.BatchNorm2d( + num_features=out_channels, momentum=bn_mom, eps=bn_eps) + + self._avg_pooling = nn.AdaptiveAvgPool2d(1) + if self._global_params.include_top: + self._dropout = nn.Dropout(self._global_params.dropout_rate) + self._fc = nn.Linear(out_channels, self._global_params.num_classes) + + self._swish = MemoryEfficientSwish() + + def set_swish(self, memory_efficient=True): + """Sets swish function as memory efficient (for training) or standard (for export). + Args: + memory_efficient (bool): Whether to use memory-efficient version of swish. + """ + self._swish = MemoryEfficientSwish() if memory_efficient else Swish() + for block in self._blocks: + block.set_swish(memory_efficient) + + def extract_endpoints(self, inputs): + """Use convolution layer to extract features + from reduction levels i in [1, 2, 3, 4, 5]. + Args: + inputs (tensor): Input tensor. + Returns: + Dictionary of last intermediate features + with reduction levels i in [1, 2, 3, 4, 5]. + Example: + >>> import torch + >>> from efficientnet.model import EfficientNet + >>> inputs = torch.rand(1, 3, 224, 224) + >>> model = EfficientNet.from_pretrained('efficientnet-b0') + >>> endpoints = model.extract_endpoints(inputs) + >>> print(endpoints['reduction_1'].shape) # torch.Size([1, 16, 112, 112]) + >>> print(endpoints['reduction_2'].shape) # torch.Size([1, 24, 56, 56]) + >>> print(endpoints['reduction_3'].shape) # torch.Size([1, 40, 28, 28]) + >>> print(endpoints['reduction_4'].shape) # torch.Size([1, 112, 14, 14]) + >>> print(endpoints['reduction_5'].shape) # torch.Size([1, 320, 7, 7]) + >>> print(endpoints['reduction_6'].shape) # torch.Size([1, 1280, 7, 7]) + """ + endpoints = dict() + + x = self._swish(self._bn0(self._conv_stem(inputs))) + prev_x = x + + for idx, block in enumerate(self._blocks): + drop_connect_rate = self._global_params.drop_connect_rate + if drop_connect_rate: + drop_connect_rate *= float(idx) / len( + self._blocks) # scale drop connect_rate + x = block(x, drop_connect_rate=drop_connect_rate) + if prev_x.size(2) > x.size(2): + endpoints['reduction_{}'.format(len(endpoints) + 1)] = prev_x + elif idx == len(self._blocks) - 1: + endpoints['reduction_{}'.format(len(endpoints) + 1)] = x + prev_x = x + + x = self._swish(self._bn1(self._conv_head(x))) + endpoints['reduction_{}'.format(len(endpoints) + 1)] = x + + return endpoints + + def extract_features(self, inputs): + """use convolution layer to extract feature . + Args: + inputs (tensor): Input tensor. + Returns: + Output of the final convolution + layer in the efficientnet model. + """ + x = self._swish(self._bn0(self._conv_stem(inputs))) + + for idx, block in enumerate(self._blocks): + drop_connect_rate = self._global_params.drop_connect_rate + if drop_connect_rate: + drop_connect_rate *= float(idx) / len(self._blocks) + x = block(x, drop_connect_rate=drop_connect_rate) + x = self._swish(self._bn1(self._conv_head(x))) + + return x + + def forward(self, inputs): + """EfficientNet's forward function. + Calls extract_features to extract features, applies final linear layer, and returns logits. + Args: + inputs (tensor): Input tensor. + Returns: + Output of this model after processing. + """ + x = self.extract_features(inputs) + x = self._avg_pooling(x) + if self._global_params.include_top: + x = x.flatten(start_dim=1) + x = self._dropout(x) + x = self._fc(x) + return x + + @classmethod + def from_name(cls, model_name, in_channels=3, **override_params): + """Create an efficientnet model according to name. + Args: + model_name (str): Name for efficientnet. + in_channels (int): Input data's channel number. + override_params (other key word params): + Params to override model's global_params. + Optional key: + 'width_coefficient', 'depth_coefficient', + 'image_size', 'dropout_rate', + 'num_classes', 'batch_norm_momentum', + 'batch_norm_epsilon', 'drop_connect_rate', + 'depth_divisor', 'min_depth' + Returns: + An efficientnet model. + """ + cls._check_model_name_is_valid(model_name) + blocks_args, global_params = get_model_params(model_name, + override_params) + model = cls(blocks_args, global_params) + model._change_in_channels(in_channels) + return model + + @classmethod + def from_pretrained(cls, + model_name, + weights_path=None, + advprop=False, + in_channels=3, + num_classes=1000, + **override_params): + """Create an efficientnet model according to name. + Args: + model_name (str): Name for efficientnet. + weights_path (None or str): + str: path to pretrained weights file on the local disk. + None: use pretrained weights downloaded from the Internet. + advprop (bool): + Whether to load pretrained weights + trained with advprop (valid when weights_path is None). + in_channels (int): Input data's channel number. + num_classes (int): + Number of categories for classification. + It controls the output size for final linear layer. + override_params (other key word params): + Params to override model's global_params. + Optional key: + 'width_coefficient', 'depth_coefficient', + 'image_size', 'dropout_rate', + 'batch_norm_momentum', + 'batch_norm_epsilon', 'drop_connect_rate', + 'depth_divisor', 'min_depth' + Returns: + A pretrained efficientnet model. + """ + model = cls.from_name( + model_name, num_classes=num_classes, **override_params) + model._change_in_channels(in_channels) + return model + + @classmethod + def get_image_size(cls, model_name): + """Get the input image size for a given efficientnet model. + Args: + model_name (str): Name for efficientnet. + Returns: + Input image size (resolution). + """ + cls._check_model_name_is_valid(model_name) + _, _, res, _ = efficientnet_params(model_name) + return res + + @classmethod + def _check_model_name_is_valid(cls, model_name): + """Validates model name. + Args: + model_name (str): Name for efficientnet. + Returns: + bool: Is a valid name or not. + """ + if model_name not in VALID_MODELS: + raise ValueError('model_name should be one of: ' + + ', '.join(VALID_MODELS)) + + def _change_in_channels(self, in_channels): + """Adjust model's first convolution layer to in_channels, if in_channels not equals 3. + Args: + in_channels (int): Input data's channel number. + """ + if in_channels != 3: + Conv2d = get_same_padding_conv2d( + image_size=self._global_params.image_size) + out_channels = round_filters(32, self._global_params) + self._conv_stem = Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=False) diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/efficient/utils.py b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c1fcd9b3c3d7ff29db3e5c212cee854605bfd9f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/efficient/utils.py @@ -0,0 +1,559 @@ +# The implementation here is modified based on EfficientNet, +# originally Apache 2.0 License and publicly available at https://github.com/lukemelas/EfficientNet-PyTorch + +import collections +import math +import re +from functools import partial + +import torch +from torch import nn +from torch.nn import functional as F +from torch.utils import model_zoo + +GlobalParams = collections.namedtuple('GlobalParams', [ + 'width_coefficient', 'depth_coefficient', 'image_size', 'dropout_rate', + 'num_classes', 'batch_norm_momentum', 'batch_norm_epsilon', + 'drop_connect_rate', 'depth_divisor', 'min_depth', 'include_top' +]) + +BlockArgs = collections.namedtuple('BlockArgs', [ + 'num_repeat', 'kernel_size', 'stride', 'expand_ratio', 'input_filters', + 'output_filters', 'se_ratio', 'id_skip' +]) + +GlobalParams.__new__.__defaults__ = (None, ) * len(GlobalParams._fields) +BlockArgs.__new__.__defaults__ = (None, ) * len(BlockArgs._fields) + +if hasattr(nn, 'SiLU'): + Swish = nn.SiLU +else: + + class Swish(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(x) + + +class SwishImplementation(torch.autograd.Function): + + @staticmethod + def forward(ctx, i): + result = i * torch.sigmoid(i) + ctx.save_for_backward(i) + return result + + @staticmethod + def backward(ctx, grad_output): + i = ctx.saved_tensors[0] + sigmoid_i = torch.sigmoid(i) + return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i))) + + +class MemoryEfficientSwish(nn.Module): + + def forward(self, x): + return SwishImplementation.apply(x) + + +def round_filters(filters, global_params): + """Calculate and round number of filters based on width multiplier. + Use width_coefficient, depth_divisor and min_depth of global_params. + Args: + filters (int): Filters number to be calculated. + global_params (namedtuple): Global params of the model. + Returns: + new_filters: New filters number after calculating. + """ + multiplier = global_params.width_coefficient + if not multiplier: + return filters + + divisor = global_params.depth_divisor + min_depth = global_params.min_depth + filters *= multiplier + min_depth = min_depth or divisor + new_filters = max(min_depth, + int(filters + divisor / 2) // divisor * divisor) + if new_filters < 0.9 * filters: + new_filters += divisor + return int(new_filters) + + +def round_repeats(repeats, global_params): + """Calculate module's repeat number of a block based on depth multiplier. + Use depth_coefficient of global_params. + Args: + repeats (int): num_repeat to be calculated. + global_params (namedtuple): Global params of the model. + Returns: + new repeat: New repeat number after calculating. + """ + multiplier = global_params.depth_coefficient + if not multiplier: + return repeats + return int(math.ceil(multiplier * repeats)) + + +def drop_connect(inputs, p, training): + """Drop connect. + Args: + input (tensor: BCWH): Input of this structure. + p (float: 0.0~1.0): Probability of drop connection. + training (bool): The running mode. + Returns: + output: Output after drop connection. + """ + assert 0 <= p <= 1, 'p must be in range of [0,1]' + + if not training: + return inputs + + batch_size = inputs.shape[0] + keep_prob = 1 - p + + random_tensor = keep_prob + random_tensor += torch.rand([batch_size, 1, 1, 1], + dtype=inputs.dtype, + device=inputs.device) + binary_tensor = torch.floor(random_tensor) + + output = inputs / keep_prob * binary_tensor + return output + + +def get_width_and_height_from_size(x): + """Obtain height and width from x. + Args: + x (int, tuple or list): Data size. + Returns: + size: A tuple or list (H,W). + """ + if isinstance(x, int): + return x, x + if isinstance(x, list) or isinstance(x, tuple): + return x + else: + raise TypeError() + + +def calculate_output_image_size(input_image_size, stride): + """Calculates the output image size when using Conv2dSamePadding with a stride. + Necessary for static padding. Thanks to mannatsingh for pointing this out. + Args: + input_image_size (int, tuple or list): Size of input image. + stride (int, tuple or list): Conv2d operation's stride. + Returns: + output_image_size: A list [H,W]. + """ + if input_image_size is None: + return None + image_height, image_width = get_width_and_height_from_size( + input_image_size) + stride = stride if isinstance(stride, int) else stride[0] + image_height = int(math.ceil(image_height / stride)) + image_width = int(math.ceil(image_width / stride)) + return [image_height, image_width] + + +def get_same_padding_conv2d(image_size=None): + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models. + Args: + image_size (int or tuple): Size of the image. + Returns: + Conv2dDynamicSamePadding or Conv2dStaticSamePadding. + """ + if image_size is None: + return Conv2dDynamicSamePadding + else: + return partial(Conv2dStaticSamePadding, image_size=image_size) + + +class Conv2dDynamicSamePadding(nn.Conv2d): + """2D Convolutions like TensorFlow, for a dynamic image size. + The padding is operated in forward function by calculating dynamically. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=1, + groups=1, + bias=True): + super().__init__(in_channels, out_channels, kernel_size, stride, 0, + dilation, groups, bias) + self.stride = self.stride if len( + self.stride) == 2 else [self.stride[0]] * 2 + + def forward(self, x): + ih, iw = x.size()[-2:] + kh, kw = self.weight.size()[-2:] + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + a1 = (oh - 1) * self.stride[0] + pad_h = max(a1 + (kh - 1) * self.dilation[0] + 1 - ih, 0) + a2 = (ow - 1) * self.stride[1] + pad_w = max(a2 + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [ + pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 + ]) + return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) + + +class Conv2dStaticSamePadding(nn.Conv2d): + """2D Convolutions like TensorFlow's 'SAME' mode, with the given input image size. + The padding mudule is calculated in construction function, then used in forward. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + image_size=None, + **kwargs): + super().__init__(in_channels, out_channels, kernel_size, stride, + **kwargs) + self.stride = self.stride if len( + self.stride) == 2 else [self.stride[0]] * 2 + + assert image_size is not None + ih, iw = (image_size, + image_size) if isinstance(image_size, int) else image_size + kh, kw = self.weight.size()[-2:] + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + b1 = (oh - 1) * self.stride[0] + pad_h = max(b1 + (kh - 1) * self.dilation[0] + 1 - ih, 0) + b2 = (ow - 1) * self.stride[1] + pad_w = max(b2 + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + self.static_padding = nn.ZeroPad2d( + (pad_w // 2, pad_w - pad_w // 2, pad_h // 2, + pad_h - pad_h // 2)) + else: + self.static_padding = nn.Identity() + + def forward(self, x): + x = self.static_padding(x) + x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) + return x + + +def get_same_padding_maxPool2d(image_size=None): + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models. + Args: + image_size (int or tuple): Size of the image. + Returns: + MaxPool2dDynamicSamePadding or MaxPool2dStaticSamePadding. + """ + if image_size is None: + return MaxPool2dDynamicSamePadding + else: + return partial(MaxPool2dStaticSamePadding, image_size=image_size) + + +class MaxPool2dDynamicSamePadding(nn.MaxPool2d): + """2D MaxPooling like TensorFlow's 'SAME' mode, with a dynamic image size. + The padding is operated in forward function by calculating dynamically. + """ + + def __init__(self, + kernel_size, + stride, + padding=0, + dilation=1, + return_indices=False, + ceil_mode=False): + super().__init__(kernel_size, stride, padding, dilation, + return_indices, ceil_mode) + self.stride = [self.stride] * 2 if isinstance(self.stride, + int) else self.stride + self.kernel_size = [self.kernel_size] * 2 if isinstance( + self.kernel_size, int) else self.kernel_size + self.dilation = [self.dilation] * 2 if isinstance( + self.dilation, int) else self.dilation + + def forward(self, x): + ih, iw = x.size()[-2:] + kh, kw = self.kernel_size + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + c1 = (oh - 1) * self.stride[0] + pad_h = max(c1 + (kh - 1) * self.dilation[0] + 1 - ih, 0) + c2 = (ow - 1) * self.stride[1] + pad_w = max(c2 + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [ + pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 + ]) + return F.max_pool2d(x, self.kernel_size, self.stride, self.padding, + self.dilation, self.ceil_mode, self.return_indices) + + +class MaxPool2dStaticSamePadding(nn.MaxPool2d): + """2D MaxPooling like TensorFlow's 'SAME' mode, with the given input image size. + The padding mudule is calculated in construction function, then used in forward. + """ + + def __init__(self, kernel_size, stride, image_size=None, **kwargs): + super().__init__(kernel_size, stride, **kwargs) + self.stride = [self.stride] * 2 if isinstance(self.stride, + int) else self.stride + self.kernel_size = [self.kernel_size] * 2 if isinstance( + self.kernel_size, int) else self.kernel_size + self.dilation = [self.dilation] * 2 if isinstance( + self.dilation, int) else self.dilation + + assert image_size is not None + ih, iw = (image_size, + image_size) if isinstance(image_size, int) else image_size + kh, kw = self.kernel_size + sh, sw = self.stride + oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) + d1 = (oh - 1) * self.stride[0] + pad_h = max(d1 + (kh - 1) * self.dilation[0] + 1 - ih, 0) + d2 = (ow - 1) * self.stride[1] + pad_w = max(d2 + (kw - 1) * self.dilation[1] + 1 - iw, 0) + if pad_h > 0 or pad_w > 0: + self.static_padding = nn.ZeroPad2d( + (pad_w // 2, pad_w - pad_w // 2, pad_h // 2, + pad_h - pad_h // 2)) + else: + self.static_padding = nn.Identity() + + def forward(self, x): + x = self.static_padding(x) + x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding, + self.dilation, self.ceil_mode, self.return_indices) + return x + + +class BlockDecoder(object): + """Block Decoder for readability, + straight from the official TensorFlow repository. + """ + + @staticmethod + def _decode_block_string(block_string): + """Get a block through a string notation of arguments. + Args: + block_string (str): A string notation of arguments. + Examples: 'r1_k3_s11_e1_i32_o16_se0.25_noskip'. + Returns: + BlockArgs: The namedtuple defined at the top of this file. + """ + assert isinstance(block_string, str) + + ops = block_string.split('_') + options = {} + for op in ops: + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + # Check stride + assert (('s' in options and len(options['s']) == 1) + or (len(options['s']) == 2 + and options['s'][0] == options['s'][1])) + + return BlockArgs( + num_repeat=int(options['r']), + kernel_size=int(options['k']), + stride=[int(options['s'][0])], + expand_ratio=int(options['e']), + input_filters=int(options['i']), + output_filters=int(options['o']), + se_ratio=float(options['se']) if 'se' in options else None, + id_skip=('noskip' not in block_string)) + + @staticmethod + def _encode_block_string(block): + """Encode a block to a string. + Args: + block (namedtuple): A BlockArgs type argument. + Returns: + block_string: A String form of BlockArgs. + """ + args = [ + 'r%d' % block.num_repeat, + 'k%d' % block.kernel_size, + 's%d%d' % (block.strides[0], block.strides[1]), + 'e%s' % block.expand_ratio, + 'i%d' % block.input_filters, + 'o%d' % block.output_filters + ] + if 0 < block.se_ratio <= 1: + args.append('se%s' % block.se_ratio) + if block.id_skip is False: + args.append('noskip') + return '_'.join(args) + + @staticmethod + def decode(string_list): + """Decode a list of string notations to specify blocks inside the network. + Args: + string_list (list[str]): A list of strings, each string is a notation of block. + Returns: + blocks_args: A list of BlockArgs namedtuples of block args. + """ + assert isinstance(string_list, list) + blocks_args = [] + for block_string in string_list: + blocks_args.append(BlockDecoder._decode_block_string(block_string)) + return blocks_args + + @staticmethod + def encode(blocks_args): + """Encode a list of BlockArgs to a list of strings. + Args: + blocks_args (list[namedtuples]): A list of BlockArgs namedtuples of block args. + Returns: + block_strings: A list of strings, each string is a notation of block. + """ + block_strings = [] + for block in blocks_args: + block_strings.append(BlockDecoder._encode_block_string(block)) + return block_strings + + +def efficientnet_params(model_name): + """Map EfficientNet model name to parameter coefficients. + Args: + model_name (str): Model name to be queried. + Returns: + params_dict[model_name]: A (width,depth,res,dropout) tuple. + """ + params_dict = { + 'efficientnet-b0': (1.0, 1.0, 112, 0.2), + 'efficientnet-b1': (1.0, 1.1, 240, 0.2), + 'efficientnet-b2': (1.1, 1.2, 260, 0.3), + 'efficientnet-b3': (1.2, 1.4, 300, 0.3), + 'efficientnet-b4': (1.4, 1.8, 380, 0.4), + 'efficientnet-b5': (1.6, 2.2, 456, 0.4), + 'efficientnet-b6': (1.8, 2.6, 528, 0.5), + 'efficientnet-b7': (2.0, 3.1, 600, 0.5), + 'efficientnet-b8': (2.2, 3.6, 672, 0.5), + 'efficientnet-l2': (4.3, 5.3, 800, 0.5), + } + return params_dict[model_name] + + +def efficientnet(width_coefficient=None, + depth_coefficient=None, + image_size=None, + dropout_rate=0.2, + drop_connect_rate=0.2, + num_classes=1000, + include_top=True): + """Create BlockArgs and GlobalParams for efficientnet model. + Args: + width_coefficient (float) + depth_coefficient (float) + image_size (int) + dropout_rate (float) + drop_connect_rate (float) + num_classes (int) + Meaning as the name suggests. + Returns: + blocks_args, global_params. + """ + + blocks_args = [ + 'r1_k3_s11_e1_i32_o16_se0.25', + 'r2_k3_s22_e6_i16_o24_se0.25', + 'r2_k5_s22_e6_i24_o40_se0.25', + 'r3_k3_s22_e6_i40_o80_se0.25', + 'r3_k5_s11_e6_i80_o112_se0.25', + 'r4_k5_s22_e6_i112_o192_se0.25', + 'r1_k3_s11_e6_i192_o320_se0.25', + ] + blocks_args = BlockDecoder.decode(blocks_args) + + global_params = GlobalParams( + width_coefficient=width_coefficient, + depth_coefficient=depth_coefficient, + image_size=image_size, + dropout_rate=dropout_rate, + num_classes=num_classes, + batch_norm_momentum=0.99, + batch_norm_epsilon=1e-3, + drop_connect_rate=drop_connect_rate, + depth_divisor=8, + min_depth=None, + include_top=include_top, + ) + return blocks_args, global_params + + +def get_model_params(model_name, override_params): + """Get the block args and global params for a given model name. + Args: + model_name (str): Model's name. + override_params (dict): A dict to modify global_params. + Returns: + blocks_args, global_params + """ + if model_name.startswith('efficientnet'): + w, d, s, p = efficientnet_params(model_name) + blocks_args, global_params = efficientnet( + width_coefficient=w, + depth_coefficient=d, + dropout_rate=p, + image_size=s) + else: + raise NotImplementedError( + 'model name is not pre-defined: {}'.format(model_name)) + if override_params: + global_params = global_params._replace(**override_params) + return blocks_args, global_params + + +def load_pretrained_weights(model, + model_name, + weights_path=None, + load_fc=True, + advprop=False, + verbose=True): + """Loads pretrained weights from weights path or download using url. + Args: + model (Module): The whole model of efficientnet. + model_name (str): Model name of efficientnet. + weights_path (None or str): + str: path to pretrained weights file on the local disk. + None: use pretrained weights downloaded from the Internet. + load_fc (bool): Whether to load pretrained weights for fc layer at the end of the model. + advprop (bool): Whether to load pretrained weights + trained with advprop (valid when weights_path is None). + """ + if isinstance(weights_path, str): + state_dict = torch.load(weights_path) + else: + url_map_ = url_map_advprop if advprop else url_map + state_dict = model_zoo.load_url(url_map_[model_name]) + + if load_fc: + ret = model.load_state_dict(state_dict, strict=False) + assert not ret.missing_keys, 'Missing keys when loading pretrained weights: {}'.format( + ret.missing_keys) + else: + state_dict.pop('_fc.weight') + state_dict.pop('_fc.bias') + ret = model.load_state_dict(state_dict, strict=False) + assert set(ret.missing_keys) == set([ + '_fc.weight', '_fc.bias' + ]), 'Missing keys when loading pretrained weights: {}'.format( + ret.missing_keys) + assert not ret.unexpected_keys, 'Missing keys when loading pretrained weights: {}'.format( + ret.unexpected_keys) + + if verbose: + print('Loaded pretrained weights for {}'.format(model_name)) diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/emotion_infer.py b/runtime/Lib/site_packages/models/cv/face_emotion/emotion_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..22ebc11c3ffd8164701e25e1d3fd09641c9c5272 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/emotion_infer.py @@ -0,0 +1,67 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import torch +from PIL import Image +from torch import nn +from torchvision import transforms + +from modelscope.utils.logger import get_logger +from .face_alignment.face_align import face_detection_PIL_v2 + +logger = get_logger() + + +def transform_PIL(img_pil): + val_transforms = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + return val_transforms(img_pil) + + +index2AU = [1, 2, 4, 6, 7, 10, 12, 15, 23, 24, 25, 26] +emotion_list = [ + 'Neutral', 'Anger', 'Disgust', 'Fear', 'Happiness', 'Sadness', 'Surprise' +] + + +def inference(image, model, face_model, score_thre=0.5, GPU=0): + image = image.cpu().numpy() + image = Image.fromarray(image) + face, bbox = face_detection_PIL_v2(image, face_model) + if bbox is None: + logger.warning('no face detected!') + result = {'emotion_result': None, 'box': None} + return result + + face = transform_PIL(face) + face = face.unsqueeze(0) + if torch.cuda.is_available(): + face = face.cuda(GPU) + logits_AU, logits_emotion = model(face) + logits_AU = torch.sigmoid(logits_AU) + logits_emotion = nn.functional.softmax(logits_emotion, 1) + + _, index_list = logits_emotion.max(1) + emotion_index = index_list[0].data.item() + prob = logits_emotion[0][emotion_index] + if prob > score_thre and emotion_index != 3: + cur_emotion = emotion_list[emotion_index] + else: + cur_emotion = 'Neutral' + + logits_AU = logits_AU[0] + au_ouput = torch.zeros_like(logits_AU) + au_ouput[logits_AU >= score_thre] = 1 + au_ouput[logits_AU < score_thre] = 0 + + au_ouput = au_ouput.int() + + cur_au_list = [] + for idx in range(au_ouput.shape[0]): + if au_ouput[idx] == 1: + au = index2AU[idx] + cur_au_list.append(au) + cur_au_list.sort() + result = (cur_emotion, bbox) + return result diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/emotion_model.py b/runtime/Lib/site_packages/models/cv/face_emotion/emotion_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f8df9c37b4be417abe46f74beca25dfc52c85765 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/emotion_model.py @@ -0,0 +1,96 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import sys + +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.face_emotion.efficient import EfficientNet +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module(Tasks.face_emotion, module_name=Models.face_emotion) +class EfficientNetForFaceEmotion(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.model = FaceEmotionModel( + name='efficientnet-b0', num_embed=512, num_au=12, num_emotion=7) + + if torch.cuda.is_available(): + self.device = 'cuda' + logger.info('Use GPU') + else: + self.device = 'cpu' + logger.info('Use CPU') + pretrained_params = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location=self.device) + + state_dict = pretrained_params['model'] + new_state = {} + for k, v in state_dict.items(): + if k.startswith('module.'): + k = k[7:] + new_state[k] = v + + self.model.load_state_dict(new_state) + self.model.eval() + self.model.to(self.device) + + def forward(self, x): + logits_au, logits_emotion = self.model(x) + return logits_au, logits_emotion + + +class FaceEmotionModel(nn.Module): + + def __init__(self, + name='efficientnet-b0', + num_embed=512, + num_au=12, + num_emotion=7): + super(FaceEmotionModel, self).__init__() + self.backbone = EfficientNet.from_pretrained( + name, weights_path=None, advprop=True) + self.average_pool = nn.AdaptiveAvgPool2d(1) + self.embed = nn.Linear(self.backbone._fc.weight.data.shape[1], + num_embed) + self.features = nn.BatchNorm1d(num_embed) + nn.init.constant_(self.features.weight, 1.0) + self.features.weight.requires_grad = False + self.fc_au = nn.Sequential( + nn.Dropout(0.6), + nn.Linear(num_embed, num_au), + ) + self.fc_emotion = nn.Sequential( + nn.Dropout(0.6), + nn.Linear(num_embed, num_emotion), + ) + + def feat_single_img(self, x): + x = self.backbone.extract_features(x) + x = self.average_pool(x) + x = x.flatten(1) + x = self.embed(x) + x = self.features(x) + return x + + def forward(self, x): + x = self.feat_single_img(x) + logits_au = self.fc_au(x) + att_au = torch.sigmoid(logits_au).unsqueeze(-1) + x = x.unsqueeze(1) + emotion_vec_list = torch.matmul(att_au, x) + emotion_vec = emotion_vec_list.sum(1) + logits_emotion = self.fc_emotion(emotion_vec) + return logits_au, logits_emotion diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/__init__.py b/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face.py b/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face.py new file mode 100644 index 0000000000000000000000000000000000000000..a362bddcbc488fbf95cb01086f8829d2432f1658 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face.py @@ -0,0 +1,79 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os + +import cv2 +import numpy as np +import tensorflow as tf + + +def init(mod): + PATH_TO_CKPT = mod + net = tf.Graph() + with net.as_default(): + od_graph_def = tf.GraphDef() + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.6 + with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: + serialized_graph = fid.read() + od_graph_def.ParseFromString(serialized_graph) + tf.import_graph_def(od_graph_def, name='') + sess = tf.Session(graph=net, config=config) + return sess, net + + +def filter_bboxes_confs(shape, + imgsBboxes, + imgsConfs, + single=False, + thresh=0.5): + [w, h] = shape + if single: + bboxes, confs = [], [] + for y in range(len(imgsBboxes)): + if imgsConfs[y] >= thresh: + [x1, y1, x2, y2] = list(imgsBboxes[y]) + x1, y1, x2, y2 = int(w * x1), int(h * y1), int(w * x2), int( + h * y2) + bboxes.append([y1, x1, y2, x2]) + confs.append(imgsConfs[y]) + return bboxes, confs + else: + retImgsBboxes, retImgsConfs = [], [] + for x in range(len(imgsBboxes)): + bboxes, confs = [], [] + for y in range(len(imgsBboxes[x])): + if imgsConfs[x][y] >= thresh: + [x1, y1, x2, y2] = list(imgsBboxes[x][y]) + x1, y1, x2, y2 = int(w * x1), int(h * y1), int( + w * x2), int(h * y2) + bboxes.append([y1, x1, y2, x2]) + confs.append(imgsConfs[x][y]) + retImgsBboxes.append(bboxes) + retImgsConfs.append(confs) + return retImgsBboxes, retImgsConfs + + +def detect(im, sess, net): + image_np = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) + image_np_expanded = np.expand_dims(image_np, axis=0) + image_tensor = net.get_tensor_by_name('image_tensor:0') + bboxes = net.get_tensor_by_name('detection_boxes:0') + dConfs = net.get_tensor_by_name('detection_scores:0') + classes = net.get_tensor_by_name('detection_classes:0') + num_detections = net.get_tensor_by_name('num_detections:0') + (bboxes, dConfs, classes, + num_detections) = sess.run([bboxes, dConfs, classes, num_detections], + feed_dict={image_tensor: image_np_expanded}) + w, h, _ = im.shape + bboxes, confs = filter_bboxes_confs([w, h], bboxes[0], dConfs[0], True) + return bboxes, confs + + +class FaceDetector: + + def __init__(self, mod): + self.sess, self.net = init(mod) + + def do_detect(self, im): + bboxes, confs = detect(im, self.sess, self.net) + return bboxes, confs diff --git a/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face_align.py b/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face_align.py new file mode 100644 index 0000000000000000000000000000000000000000..71282b12092703e5411891c7e24db54e798f4598 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_emotion/face_alignment/face_align.py @@ -0,0 +1,59 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import sys + +import cv2 +import numpy as np +from PIL import Image, ImageFile + +from .face import FaceDetector + +ImageFile.LOAD_TRUNCATED_IMAGES = True + + +def adjust_bx_v2(box, w, h): + x1, y1, x2, y2 = box[0], box[1], box[2], box[3] + box_w = x2 - x1 + box_h = y2 - y1 + delta = abs(box_w - box_h) + if box_w > box_h: + if y1 >= delta: + y1 = y1 - delta + else: + delta_y1 = y1 + y1 = 0 + delta_y2 = delta - delta_y1 + y2 = y2 + delta_y2 if y2 < h - delta_y2 else h - 1 + else: + if x1 >= delta / 2 and x2 <= w - delta / 2: + x1 = x1 - delta / 2 + x2 = x2 + delta / 2 + elif x1 < delta / 2 and x2 <= w - delta / 2: + delta_x1 = x1 + x1 = 0 + delta_x2 = delta - delta_x1 + x2 = x2 + delta_x2 if x2 < w - delta_x2 else w - 1 + elif x1 >= delta / 2 and x2 > w - delta / 2: + delta_x2 = w - x2 + x2 = w - 1 + delta_x1 = delta - x1 + x1 = x1 - delta_x1 if x1 >= delta_x1 else 0 + + x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2) + return [x1, y1, x2, y2] + + +def face_detection_PIL_v2(image, face_model): + crop_size = 112 + face_detector = FaceDetector(face_model) + img = np.array(image) + h, w = img.shape[0:2] + bxs, conf = face_detector.do_detect(img) + bx = bxs[0] + bx = adjust_bx_v2(bx, w, h) + x1, y1, x2, y2 = bx + image = img[y1:y2, x1:x2, :] + img = Image.fromarray(image) + img = img.resize((crop_size, crop_size)) + bx = tuple(bx) + return img, bx diff --git a/runtime/Lib/site_packages/models/cv/face_generation/__init__.py b/runtime/Lib/site_packages/models/cv/face_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35a63f1c76b88bcb6ebb78f49d938f50c8b9182a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .stylegan2 import Generator + +else: + _import_structure = { + 'stylegan2': ['Generator'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/face_generation/op/__init__.py b/runtime/Lib/site_packages/models/cv/face_generation/op/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0918d92285955855be89f00096b888ee5597ce3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/op/__init__.py @@ -0,0 +1,2 @@ +from .fused_act import FusedLeakyReLU, fused_leaky_relu +from .upfirdn2d import upfirdn2d diff --git a/runtime/Lib/site_packages/models/cv/face_generation/op/conv2d_gradfix.py b/runtime/Lib/site_packages/models/cv/face_generation/op/conv2d_gradfix.py new file mode 100644 index 0000000000000000000000000000000000000000..a3aba91f36f33469672151dcb196565d65bf184d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/op/conv2d_gradfix.py @@ -0,0 +1,228 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# at https://github.com/rosinality/stylegan2-pytorch/blob/master/op/conv2d_gradfix.py +import contextlib +import warnings + +import torch +from torch import autograd +from torch.nn import functional as F + +enabled = True +weight_gradients_disabled = False + + +@contextlib.contextmanager +def no_weight_gradients(): + global weight_gradients_disabled + + old = weight_gradients_disabled + weight_gradients_disabled = True + yield + weight_gradients_disabled = old + + +def conv2d(input, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + if could_use_op(input): + return conv2d_gradfix( + transpose=False, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=0, + dilation=dilation, + groups=groups, + ).apply(input, weight, bias) + + return F.conv2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + +def conv_transpose2d( + input, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, +): + if could_use_op(input): + return conv2d_gradfix( + transpose=True, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + dilation=dilation, + ).apply(input, weight, bias) + + return F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + output_padding=output_padding, + dilation=dilation, + groups=groups, + ) + + +def could_use_op(input): + if (not enabled) or (not torch.backends.cudnn.enabled): + return False + + if input.device.type != 'cuda': + return False + + warnings.warn( + f'conv2d_gradfix not supported on PyTorch {torch.__version__}. Falling back to torch.nn.functional.conv2d().' + ) + + return False + + +def ensure_tuple(xs, ndim): + xs = tuple(xs) if isinstance(xs, (tuple, list)) else (xs, ) * ndim + + return xs + + +conv2d_gradfix_cache = dict() + + +def conv2d_gradfix(transpose, weight_shape, stride, padding, output_padding, + dilation, groups): + ndim = 2 + weight_shape = tuple(weight_shape) + stride = ensure_tuple(stride, ndim) + padding = ensure_tuple(padding, ndim) + output_padding = ensure_tuple(output_padding, ndim) + dilation = ensure_tuple(dilation, ndim) + + key = (transpose, weight_shape, stride, padding, output_padding, dilation, + groups) + if key in conv2d_gradfix_cache: + return conv2d_gradfix_cache[key] + + common_kwargs = dict( + stride=stride, padding=padding, dilation=dilation, groups=groups) + + def calc_output_padding(input_shape, output_shape): + if transpose: + return [0, 0] + + a = input_shape[i + 2] - (output_shape[i + 2] - 1) * stride[i] + return [ + a - (1 - 2 * padding[i]) - dilation[i] * (weight_shape[i + 2] - 1) + for i in range(ndim) + ] + + class Conv2d(autograd.Function): + + @staticmethod + def forward(ctx, input, weight, bias): + if not transpose: + out = F.conv2d( + input=input, weight=weight, bias=bias, **common_kwargs) + + else: + out = F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + output_padding=output_padding, + **common_kwargs, + ) + + ctx.save_for_backward(input, weight) + + return out + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input, grad_weight, grad_bias = None, None, None + + if ctx.needs_input_grad[0]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, weight, None) + + if ctx.needs_input_grad[1] and not weight_gradients_disabled: + grad_weight = Conv2dGradWeight.apply(grad_output, input) + + if ctx.needs_input_grad[2]: + grad_bias = grad_output.sum((0, 2, 3)) + + return grad_input, grad_weight, grad_bias + + class Conv2dGradWeight(autograd.Function): + + @staticmethod + def forward(ctx, grad_output, input): + op = torch._C._jit_get_operation( + 'aten::cudnn_convolution_backward_weight' if not transpose else + 'aten::cudnn_convolution_transpose_backward_weight') + flags = [ + torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, + torch.backends.cudnn.allow_tf32, + ] + grad_weight = op( + weight_shape, + grad_output, + input, + padding, + stride, + dilation, + groups, + *flags, + ) + ctx.save_for_backward(grad_output, input) + + return grad_weight + + @staticmethod + def backward(ctx, grad_grad_weight): + grad_output, input = ctx.saved_tensors + grad_grad_output, grad_grad_input = None, None + + if ctx.needs_input_grad[0]: + grad_grad_output = Conv2d.apply(input, grad_grad_weight, None) + + if ctx.needs_input_grad[1]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, grad_grad_weight, None) + + return grad_grad_output, grad_grad_input + + conv2d_gradfix_cache[key] = Conv2d + + return Conv2d diff --git a/runtime/Lib/site_packages/models/cv/face_generation/op/fused_act.py b/runtime/Lib/site_packages/models/cv/face_generation/op/fused_act.py new file mode 100644 index 0000000000000000000000000000000000000000..a24f597233032d9faceedf896cb05a4e6dbe7bd9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/op/fused_act.py @@ -0,0 +1,115 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# t https://github.com/rosinality/stylegan2-pytorch/blob/master/op/fused_act.py +import os + +import torch +from torch import nn +from torch.autograd import Function +from torch.nn import functional as F + +def_lib = False + + +class FusedLeakyReLUFunctionBackward(Function): + + @staticmethod + def forward(ctx, grad_output, out, bias, negative_slope, scale): + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + empty = grad_output.new_empty(0) + + grad_input = fused.fused_bias_act(grad_output.contiguous(), empty, out, + 3, 1, negative_slope, scale) + + dim = [0] + + if grad_input.ndim > 2: + dim += list(range(2, grad_input.ndim)) + + if bias: + grad_bias = grad_input.sum(dim).detach() + + else: + grad_bias = empty + + return grad_input, grad_bias + + @staticmethod + def backward(ctx, gradgrad_input, gradgrad_bias): + out, = ctx.saved_tensors + gradgrad_out = fused.fused_bias_act( + gradgrad_input.contiguous(), + gradgrad_bias, + out, + 3, + 1, + ctx.negative_slope, + ctx.scale, + ) + + return gradgrad_out, None, None, None, None + + +class FusedLeakyReLUFunction(Function): + + @staticmethod + def forward(ctx, input, bias, negative_slope, scale): + empty = input.new_empty(0) + + ctx.bias = bias is not None + + if bias is None: + bias = empty + + out = fused.fused_bias_act(input, bias, empty, 3, 0, negative_slope, + scale) + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + return out + + @staticmethod + def backward(ctx, grad_output): + out, = ctx.saved_tensors + + grad_input, grad_bias = FusedLeakyReLUFunctionBackward.apply( + grad_output, out, ctx.bias, ctx.negative_slope, ctx.scale) + + if not ctx.bias: + grad_bias = None + + return grad_input, grad_bias, None, None + + +class FusedLeakyReLU(nn.Module): + + def __init__(self, channel, bias=True, negative_slope=0.2, scale=2**0.5): + super().__init__() + + if bias: + self.bias = nn.Parameter(torch.zeros(channel)) + + else: + self.bias = None + + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + return fused_leaky_relu(input, self.bias, self.negative_slope, + self.scale) + + +def fused_leaky_relu(input, bias=None, negative_slope=0.2, scale=2**0.5): + if not def_lib: + if bias is not None: + rest_dim = [1] * (input.ndim - bias.ndim - 1) + return (F.leaky_relu( + input + bias.view(1, bias.shape[0], *rest_dim), + negative_slope=0.2) * scale) + + else: + return F.leaky_relu(input, negative_slope=0.2) * scale diff --git a/runtime/Lib/site_packages/models/cv/face_generation/op/upfirdn2d.py b/runtime/Lib/site_packages/models/cv/face_generation/op/upfirdn2d.py new file mode 100644 index 0000000000000000000000000000000000000000..95c987af79124e99b4d8ab52553424b703082bfd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/op/upfirdn2d.py @@ -0,0 +1,199 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# at https://github.com/rosinality/stylegan2-pytorch/blob/master/op/upfirdn2d.py +import os +from collections import abc + +import torch +from torch.autograd import Function +from torch.nn import functional as F + +def_lib = False + + +class UpFirDn2dBackward(Function): + + @staticmethod + def forward(ctx, grad_output, kernel, grad_kernel, up, down, pad, g_pad, + in_size, out_size): + + up_x, up_y = up + down_x, down_y = down + g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1 = g_pad + + grad_output = grad_output.reshape(-1, out_size[0], out_size[1], 1) + + grad_input = upfirdn2d_op.upfirdn2d( + grad_output, + grad_kernel, + down_x, + down_y, + up_x, + up_y, + g_pad_x0, + g_pad_x1, + g_pad_y0, + g_pad_y1, + ) + grad_input = grad_input.view(in_size[0], in_size[1], in_size[2], + in_size[3]) + + ctx.save_for_backward(kernel) + + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + ctx.up_x = up_x + ctx.up_y = up_y + ctx.down_x = down_x + ctx.down_y = down_y + ctx.pad_x0 = pad_x0 + ctx.pad_x1 = pad_x1 + ctx.pad_y0 = pad_y0 + ctx.pad_y1 = pad_y1 + ctx.in_size = in_size + ctx.out_size = out_size + + return grad_input + + @staticmethod + def backward(ctx, gradgrad_input): + kernel, = ctx.saved_tensors + + gradgrad_input = gradgrad_input.reshape(-1, ctx.in_size[2], + ctx.in_size[3], 1) + + gradgrad_out = upfirdn2d_op.upfirdn2d( + gradgrad_input, + kernel, + ctx.up_x, + ctx.up_y, + ctx.down_x, + ctx.down_y, + ctx.pad_x0, + ctx.pad_x1, + ctx.pad_y0, + ctx.pad_y1, + ) + # gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.out_size[0], ctx.out_size[1], ctx.in_size[3]) + gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.in_size[1], + ctx.out_size[0], ctx.out_size[1]) + + return gradgrad_out, None, None, None, None, None, None, None, None + + +class UpFirDn2d(Function): + + @staticmethod + def forward(ctx, input, kernel, up, down, pad): + up_x, up_y = up + down_x, down_y = down + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + kernel_h, kernel_w = kernel.shape + batch, channel, in_h, in_w = input.shape + ctx.in_size = input.shape + + input = input.reshape(-1, in_h, in_w, 1) + + ctx.save_for_backward(kernel, torch.flip(kernel, [0, 1])) + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + ctx.out_size = (out_h, out_w) + + ctx.up = (up_x, up_y) + ctx.down = (down_x, down_y) + ctx.pad = (pad_x0, pad_x1, pad_y0, pad_y1) + + g_pad_x0 = kernel_w - pad_x0 - 1 + g_pad_y0 = kernel_h - pad_y0 - 1 + g_pad_x1 = in_w * up_x - out_w * down_x + pad_x0 - up_x + 1 + g_pad_y1 = in_h * up_y - out_h * down_y + pad_y0 - up_y + 1 + + ctx.g_pad = (g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1) + + out = upfirdn2d_op.upfirdn2d(input, kernel, up_x, up_y, down_x, down_y, + pad_x0, pad_x1, pad_y0, pad_y1) + # out = out.view(major, out_h, out_w, minor) + out = out.view(-1, channel, out_h, out_w) + + return out + + @staticmethod + def backward(ctx, grad_output): + kernel, grad_kernel = ctx.saved_tensors + + grad_input = None + + if ctx.needs_input_grad[0]: + grad_input = UpFirDn2dBackward.apply( + grad_output, + kernel, + grad_kernel, + ctx.up, + ctx.down, + ctx.pad, + ctx.g_pad, + ctx.in_size, + ctx.out_size, + ) + + return grad_input, None, None, None, None + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + if not isinstance(up, abc.Iterable): + up = (up, up) + + if not isinstance(down, abc.Iterable): + down = (down, down) + + if len(pad) == 2: + pad = (pad[0], pad[1], pad[0], pad[1]) + + if not def_lib: + out = upfirdn2d_native(input, kernel, *up, *down, *pad) + + return out + + +def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, + pad_y0, pad_y1): + _, channel, in_h, in_w = input.shape + input = input.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad( + out, + [0, 0, + max(pad_x0, 0), + max(pad_x1, 0), + max(pad_y0, 0), + max(pad_y1, 0)]) + out = out[:, + max(-pad_y0, 0):out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0):out.shape[2] - max(-pad_x1, 0)] + + out = out.permute(0, 3, 1, 2) + out = out.reshape( + [-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + + return out.view(-1, channel, out_h, out_w) diff --git a/runtime/Lib/site_packages/models/cv/face_generation/stylegan2.py b/runtime/Lib/site_packages/models/cv/face_generation/stylegan2.py new file mode 100644 index 0000000000000000000000000000000000000000..4c650f549856fe344a95064b31d21e41b015a3bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_generation/stylegan2.py @@ -0,0 +1,733 @@ +# The implementation is adopted from stylegan2-pytorch, +# made public available under the MIT License at https://github.com/rosinality/stylegan2-pytorch/blob/master/model.py +import functools +import math +import operator +import random + +import torch +from torch import nn +from torch.autograd import Function +from torch.nn import functional as F + +from .op import FusedLeakyReLU, conv2d_gradfix, fused_leaky_relu, upfirdn2d + + +class PixelNorm(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, input): + return input * torch.rsqrt( + torch.mean(input**2, dim=1, keepdim=True) + 1e-8) + + +def make_kernel(k): + k = torch.tensor(k, dtype=torch.float32) + + if k.ndim == 1: + k = k[None, :] * k[:, None] + + k /= k.sum() + + return k + + +class Upsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) * (factor**2) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=self.factor, down=1, pad=self.pad) + + return out + + +class Downsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=1, down=self.factor, pad=self.pad) + + return out + + +class Blur(nn.Module): + + def __init__(self, kernel, pad, upsample_factor=1): + super().__init__() + + kernel = make_kernel(kernel) + + if upsample_factor > 1: + kernel = kernel * (upsample_factor**2) + + self.register_buffer('kernel', kernel) + + self.pad = pad + + def forward(self, input): + out = upfirdn2d(input, self.kernel, pad=self.pad) + + return out + + +class EqualConv2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + bias=True): + super().__init__() + + self.weight = nn.Parameter( + torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size**2) + + self.stride = stride + self.padding = padding + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + + else: + self.bias = None + + def forward(self, input): + out = conv2d_gradfix.conv2d( + input, + self.weight * self.scale, + bias=self.bias, + stride=self.stride, + padding=self.padding, + ) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class EqualLinear(nn.Module): + + def __init__(self, + in_dim, + out_dim, + bias=True, + bias_init=0, + lr_mul=1, + activation=None): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) + + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) + + else: + self.bias = None + + self.activation = activation + + self.scale = (1 / math.sqrt(in_dim)) * lr_mul + self.lr_mul = lr_mul + + def forward(self, input): + if self.activation: + out = F.linear(input, self.weight * self.scale) + out = fused_leaky_relu(out, self.bias * self.lr_mul) + + else: + out = F.linear( + input, self.weight * self.scale, bias=self.bias * self.lr_mul) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})' + ) + + +class ModulatedConv2d(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + demodulate=True, + upsample=False, + downsample=False, + blur_kernel=[1, 3, 3, 1], + fused=True, + ): + super().__init__() + + self.eps = 1e-8 + self.kernel_size = kernel_size + self.in_channel = in_channel + self.out_channel = out_channel + self.upsample = upsample + self.downsample = downsample + + if upsample: + factor = 2 + p = (len(blur_kernel) - factor) - (kernel_size - 1) + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + 1 + + self.blur = Blur( + blur_kernel, pad=(pad0, pad1), upsample_factor=factor) + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.blur = Blur(blur_kernel, pad=(pad0, pad1)) + + fan_in = in_channel * kernel_size**2 + self.scale = 1 / math.sqrt(fan_in) + self.padding = kernel_size // 2 + + self.weight = nn.Parameter( + torch.randn(1, out_channel, in_channel, kernel_size, kernel_size)) + + self.modulation = EqualLinear(style_dim, in_channel, bias_init=1) + + self.demodulate = demodulate + self.fused = fused + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, ' + f'upsample={self.upsample}, downsample={self.downsample})') + + def forward(self, input, style): + batch, in_channel, height, width = input.shape + + if not self.fused: + weight = self.scale * self.weight.squeeze(0) + style = self.modulation(style) + + if self.demodulate: + w = weight.unsqueeze(0) * style.view(batch, 1, in_channel, 1, + 1) + dcoefs = (w.square().sum((2, 3, 4)) + 1e-8).rsqrt() + + input = input * style.reshape(batch, in_channel, 1, 1) + + if self.upsample: + weight = weight.transpose(0, 1) + out = conv2d_gradfix.conv_transpose2d( + input, weight, padding=0, stride=2) + out = self.blur(out) + + elif self.downsample: + input = self.blur(input) + out = conv2d_gradfix.conv2d(input, weight, padding=0, stride=2) + + else: + out = conv2d_gradfix.conv2d( + input, weight, padding=self.padding) + + if self.demodulate: + out = out * dcoefs.view(batch, -1, 1, 1) + + return out + + style = self.modulation(style).view(batch, 1, in_channel, 1, 1) + weight = self.scale * self.weight * style + + if self.demodulate: + demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8) + weight = weight * demod.view(batch, self.out_channel, 1, 1, 1) + + weight = weight.view(batch * self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + + if self.upsample: + input = input.view(1, batch * in_channel, height, width) + weight = weight.view(batch, self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + weight = weight.transpose(1, 2).reshape(batch * in_channel, + self.out_channel, + self.kernel_size, + self.kernel_size) + out = conv2d_gradfix.conv_transpose2d( + input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + out = self.blur(out) + + elif self.downsample: + input = self.blur(input) + _, _, height, width = input.shape + input = input.view(1, batch * in_channel, height, width) + out = conv2d_gradfix.conv2d( + input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + else: + input = input.view(1, batch * in_channel, height, width) + out = conv2d_gradfix.conv2d( + input, weight, padding=self.padding, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + return out + + +class NoiseInjection(nn.Module): + + def __init__(self): + super().__init__() + + self.weight = nn.Parameter(torch.zeros(1)) + + def forward(self, image, noise=None): + if noise is None: + batch, _, height, width = image.shape + noise = image.new_empty(batch, 1, height, width).normal_() + + return image + self.weight * noise + + +class ConstantInput(nn.Module): + + def __init__(self, channel, size=4): + super().__init__() + + self.input = nn.Parameter(torch.randn(1, channel, size, size)) + + def forward(self, input): + batch = input.shape[0] + out = self.input.repeat(batch, 1, 1, 1) + + return out + + +class StyledConv(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=False, + blur_kernel=[1, 3, 3, 1], + demodulate=True, + ): + super().__init__() + + self.conv = ModulatedConv2d( + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=upsample, + blur_kernel=blur_kernel, + demodulate=demodulate, + ) + + self.noise = NoiseInjection() + # self.bias = nn.Parameter(torch.zeros(1, out_channel, 1, 1)) + # self.activate = ScaledLeakyReLU(0.2) + self.activate = FusedLeakyReLU(out_channel) + + def forward(self, input, style, noise=None): + out = self.conv(input, style) + out = self.noise(out, noise=noise) + # out = out + self.bias + out = self.activate(out) + + return out + + +class ToRGB(nn.Module): + + def __init__(self, + in_channel, + style_dim, + upsample=True, + blur_kernel=[1, 3, 3, 1]): + super().__init__() + + if upsample: + self.upsample = Upsample(blur_kernel) + + self.conv = ModulatedConv2d( + in_channel, 3, 1, style_dim, demodulate=False) + self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1)) + + def forward(self, input, style, skip=None): + out = self.conv(input, style) + out = out + self.bias + + if skip is not None: + skip = self.upsample(skip) + + out = out + skip + + return out + + +class Generator(nn.Module): + + def __init__( + self, + size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + ): + super().__init__() + + self.size = size + + self.style_dim = style_dim + + layers = [PixelNorm()] + + for i in range(n_mlp): + layers.append( + EqualLinear( + style_dim, + style_dim, + lr_mul=lr_mlp, + activation='fused_lrelu')) + + self.style = nn.Sequential(*layers) + + self.channels = { + 4: 512, + 8: 512, + 16: 512, + 32: 512, + 64: 256 * channel_multiplier, + 128: 128 * channel_multiplier, + 256: 64 * channel_multiplier, + 512: 32 * channel_multiplier, + 1024: 16 * channel_multiplier, + } + + self.input = ConstantInput(self.channels[4]) + self.conv1 = StyledConv( + self.channels[4], + self.channels[4], + 3, + style_dim, + blur_kernel=blur_kernel) + self.to_rgb1 = ToRGB(self.channels[4], style_dim, upsample=False) + + self.log_size = int(math.log(size, 2)) + self.num_layers = (self.log_size - 2) * 2 + 1 + + self.convs = nn.ModuleList() + self.upsamples = nn.ModuleList() + self.to_rgbs = nn.ModuleList() + self.noises = nn.Module() + + in_channel = self.channels[4] + + for layer_idx in range(self.num_layers): + res = (layer_idx + 5) // 2 + shape = [1, 1, 2**res, 2**res] + self.noises.register_buffer(f'noise_{layer_idx}', + torch.randn(*shape)) + + for i in range(3, self.log_size + 1): + out_channel = self.channels[2**i] + + self.convs.append( + StyledConv( + in_channel, + out_channel, + 3, + style_dim, + upsample=True, + blur_kernel=blur_kernel, + )) + + self.convs.append( + StyledConv( + out_channel, + out_channel, + 3, + style_dim, + blur_kernel=blur_kernel)) + + self.to_rgbs.append(ToRGB(out_channel, style_dim)) + + in_channel = out_channel + + self.n_latent = self.log_size * 2 - 2 + + def make_noise(self): + device = self.input.input.device + + noises = [torch.randn(1, 1, 2**2, 2**2, device=device)] + + for i in range(3, self.log_size + 1): + for _ in range(2): + noises.append(torch.randn(1, 1, 2**i, 2**i, device=device)) + + return noises + + def mean_latent(self, n_latent): + latent_in = torch.randn( + n_latent, self.style_dim, device=self.input.input.device) + latent = self.style(latent_in).mean(0, keepdim=True) + + return latent + + def get_latent(self, input): + return self.style(input) + + def forward( + self, + styles, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + noise=None, + randomize_noise=True, + ): + if not input_is_latent: + styles = [self.style(s) for s in styles] + + if noise is None: + if randomize_noise: + noise = [None] * self.num_layers + else: + noise = [ + getattr(self.noises, f'noise_{i}') + for i in range(self.num_layers) + ] + + if truncation < 1: + style_t = [] + + for style in styles: + style_t.append(truncation_latent + + truncation * (style - truncation_latent)) + + styles = style_t + + if len(styles) < 2: + inject_index = self.n_latent + + if styles[0].ndim < 3: + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + + else: + latent = styles[0] + + else: + if inject_index is None: + inject_index = random.randint(1, self.n_latent - 1) + + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + latent2 = styles[1].unsqueeze(1).repeat( + 1, self.n_latent - inject_index, 1) + + latent = torch.cat([latent, latent2], 1) + + out = self.input(latent) + out = self.conv1(out, latent[:, 0], noise=noise[0]) + + skip = self.to_rgb1(out, latent[:, 1]) + + i = 1 + for conv1, conv2, noise1, noise2, to_rgb in zip( + self.convs[::2], self.convs[1::2], noise[1::2], noise[2::2], + self.to_rgbs): + out = conv1(out, latent[:, i], noise=noise1) + out = conv2(out, latent[:, i + 1], noise=noise2) + skip = to_rgb(out, latent[:, i + 2], skip) + + i += 2 + + image = skip + + if return_latents: + return image, latent + + else: + return image, None + + +class ConvLayer(nn.Sequential): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + ): + layers = [] + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + layers.append(Blur(blur_kernel, pad=(pad0, pad1))) + + stride = 2 + self.padding = 0 + + else: + stride = 1 + self.padding = kernel_size // 2 + + layers.append( + EqualConv2d( + in_channel, + out_channel, + kernel_size, + padding=self.padding, + stride=stride, + bias=bias and not activate, + )) + + if activate: + layers.append(FusedLeakyReLU(out_channel, bias=bias)) + + super().__init__(*layers) + + +class ResBlock(nn.Module): + + def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + self.conv1 = ConvLayer(in_channel, in_channel, 3) + self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True) + + self.skip = ConvLayer( + in_channel, + out_channel, + 1, + downsample=True, + activate=False, + bias=False) + + def forward(self, input): + out = self.conv1(input) + out = self.conv2(out) + + skip = self.skip(input) + out = (out + skip) / math.sqrt(2) + + return out + + +class Discriminator(nn.Module): + + def __init__(self, size, channel_multiplier=2, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + channels = { + 4: 512, + 8: 512, + 16: 512, + 32: 512, + 64: 256 * channel_multiplier, + 128: 128 * channel_multiplier, + 256: 64 * channel_multiplier, + 512: 32 * channel_multiplier, + 1024: 16 * channel_multiplier, + } + + convs = [ConvLayer(3, channels[size], 1)] + + log_size = int(math.log(size, 2)) + + in_channel = channels[size] + + for i in range(log_size, 2, -1): + out_channel = channels[2**(i - 1)] + + convs.append(ResBlock(in_channel, out_channel, blur_kernel)) + + in_channel = out_channel + + self.convs = nn.Sequential(*convs) + + self.stddev_group = 4 + self.stddev_feat = 1 + + self.final_conv = ConvLayer(in_channel + 1, channels[4], 3) + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, channels[4], activation='fused_lrelu'), + EqualLinear(channels[4], 1), + ) + + def forward(self, input): + out = self.convs(input) + + batch, channel, height, width = out.shape + group = min(batch, self.stddev_group) + stddev = out.view(group, -1, self.stddev_feat, + channel // self.stddev_feat, height, width) + stddev = torch.sqrt(stddev.var(0, unbiased=False) + 1e-8) + stddev = stddev.mean([2, 3, 4], keepdims=True).squeeze(2) + stddev = stddev.repeat(group, 1, height, width) + out = torch.cat([out, stddev], 1) + + out = self.final_conv(out) + + out = out.view(batch, -1) + out = self.final_linear(out) + + return out diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/__init__.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..33a5fd2f5f9a8d5572c2e62295fe9b83f5b2e477 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .det_infer import NanoDetForFaceHumanHandDetection + +else: + _import_structure = {'det_infer': ['NanoDetForFaceHumanHandDetection']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/det_infer.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/det_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..6822bd9f89348ee1b7bd0fff43545eec0eb0f42a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/det_infer.py @@ -0,0 +1,138 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .one_stage_detector import OneStageDetector + +logger = get_logger() + + +def load_model_weight(model_dir, device): + checkpoint = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location=device) + state_dict = checkpoint['state_dict'].copy() + for k in checkpoint['state_dict']: + if k.startswith('avg_model.'): + v = state_dict.pop(k) + state_dict[k[4:]] = v + + return state_dict + + +@MODELS.register_module( + Tasks.face_human_hand_detection, + module_name=Models.face_human_hand_detection) +class NanoDetForFaceHumanHandDetection(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + self.model = OneStageDetector() + if torch.cuda.is_available(): + self.device = 'cuda' + logger.info('Use GPU ') + else: + self.device = 'cpu' + logger.info('Use CPU') + + self.state_dict = load_model_weight(model_dir, self.device) + self.model.load_state_dict(self.state_dict, strict=False) + self.model.eval() + self.model.to(self.device) + + def forward(self, x): + pred_result = self.model.inference(x) + return pred_result + + +def naive_collate(batch): + elem = batch[0] + if isinstance(elem, dict): + return {key: naive_collate([d[key] for d in batch]) for key in elem} + else: + return batch + + +def get_resize_matrix(raw_shape, dst_shape): + + r_w, r_h = raw_shape + d_w, d_h = dst_shape + Rs = np.eye(3) + + Rs[0, 0] *= d_w / r_w + Rs[1, 1] *= d_h / r_h + return Rs + + +def color_aug_and_norm(meta, mean, std): + img = meta['img'].astype(np.float32) / 255 + mean = np.array(mean, dtype=np.float32).reshape(1, 1, 3) / 255 + std = np.array(std, dtype=np.float32).reshape(1, 1, 3) / 255 + img = (img - mean) / std + meta['img'] = img + return meta + + +def img_process(meta, mean, std): + raw_img = meta['img'] + height = raw_img.shape[0] + width = raw_img.shape[1] + dst_shape = [320, 320] + M = np.array([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]) + ResizeM = get_resize_matrix((width, height), dst_shape) + M = ResizeM @ M + img = cv2.warpPerspective(raw_img, M, dsize=tuple(dst_shape)) + meta['img'] = img + meta['warp_matrix'] = M + meta = color_aug_and_norm(meta, mean, std) + return meta + + +def overlay_bbox_cv(dets, class_names, score_thresh): + all_box = [] + for label in dets: + for bbox in dets[label]: + score = bbox[-1] + if score > score_thresh: + x0, y0, x1, y1 = [int(i) for i in bbox[:4]] + all_box.append([label, x0, y0, x1, y1, score]) + all_box.sort(key=lambda v: v[5]) + return all_box + + +mean = [103.53, 116.28, 123.675] +std = [57.375, 57.12, 58.395] +class_names = ['person', 'face', 'hand'] + + +def inference(model, device, img): + img = img.cpu().numpy() + img_info = {'id': 0} + height, width = img.shape[:2] + img_info['height'] = height + img_info['width'] = width + meta = dict(img_info=img_info, raw_img=img, img=img) + + meta = img_process(meta, mean, std) + meta['img'] = torch.from_numpy(meta['img'].transpose(2, 0, 1)).to(device) + meta = naive_collate([meta]) + meta['img'] = (meta['img'][0]).reshape(1, 3, 320, 320) + with torch.no_grad(): + res = model(meta) + result = overlay_bbox_cv(res[0], class_names, score_thresh=0.35) + cls_list, bbox_list, score_list = [], [], [] + for pred in result: + cls_list.append(pred[0]) + bbox_list.append([pred[1], pred[2], pred[3], pred[4]]) + score_list.append(pred[5]) + return cls_list, bbox_list, score_list diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/ghost_pan.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/ghost_pan.py new file mode 100644 index 0000000000000000000000000000000000000000..cad6cfe00786a9bb07667045cc9b59f66ded9c40 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/ghost_pan.py @@ -0,0 +1,395 @@ +# The implementation here is modified based on nanodet, +# originally Apache 2.0 License and publicly available at https://github.com/RangiLyu/nanodet + +import math + +import torch +import torch.nn as nn + +from .utils import ConvModule, DepthwiseConvModule, act_layers + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +def hard_sigmoid(x, inplace: bool = False): + if inplace: + return x.add_(3.0).clamp_(0.0, 6.0).div_(6.0) + else: + return F.relu6(x + 3.0) / 6.0 + + +class SqueezeExcite(nn.Module): + + def __init__(self, + in_chs, + se_ratio=0.25, + reduced_base_chs=None, + activation='ReLU', + gate_fn=hard_sigmoid, + divisor=4, + **_): + super(SqueezeExcite, self).__init__() + self.gate_fn = gate_fn + reduced_chs = _make_divisible((reduced_base_chs or in_chs) * se_ratio, + divisor) + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True) + self.act1 = act_layers(activation) + self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True) + + def forward(self, x): + x_se = self.avg_pool(x) + x_se = self.conv_reduce(x_se) + x_se = self.act1(x_se) + x_se = self.conv_expand(x_se) + x = x * self.gate_fn(x_se) + return x + + +class GhostModule(nn.Module): + + def __init__(self, + inp, + oup, + kernel_size=1, + ratio=2, + dw_size=3, + stride=1, + activation='ReLU'): + super(GhostModule, self).__init__() + self.oup = oup + init_channels = math.ceil(oup / ratio) + new_channels = init_channels * (ratio - 1) + + self.primary_conv = nn.Sequential( + nn.Conv2d( + inp, + init_channels, + kernel_size, + stride, + kernel_size // 2, + bias=False), + nn.BatchNorm2d(init_channels), + act_layers(activation) if activation else nn.Sequential(), + ) + + self.cheap_operation = nn.Sequential( + nn.Conv2d( + init_channels, + new_channels, + dw_size, + 1, + dw_size // 2, + groups=init_channels, + bias=False, + ), + nn.BatchNorm2d(new_channels), + act_layers(activation) if activation else nn.Sequential(), + ) + + def forward(self, x): + x1 = self.primary_conv(x) + x2 = self.cheap_operation(x1) + out = torch.cat([x1, x2], dim=1) + return out + + +class GhostBottleneck(nn.Module): + """Ghost bottleneck w/ optional SE""" + + def __init__( + self, + in_chs, + mid_chs, + out_chs, + dw_kernel_size=3, + stride=1, + activation='ReLU', + se_ratio=0.0, + ): + super(GhostBottleneck, self).__init__() + has_se = se_ratio is not None and se_ratio > 0.0 + self.stride = stride + + # Point-wise expansion + self.ghost1 = GhostModule(in_chs, mid_chs, activation=activation) + + # Depth-wise convolution + if self.stride > 1: + self.conv_dw = nn.Conv2d( + mid_chs, + mid_chs, + dw_kernel_size, + stride=stride, + padding=(dw_kernel_size - 1) // 2, + groups=mid_chs, + bias=False, + ) + self.bn_dw = nn.BatchNorm2d(mid_chs) + + if has_se: + self.se = SqueezeExcite(mid_chs, se_ratio=se_ratio) + else: + self.se = None + + self.ghost2 = GhostModule(mid_chs, out_chs, activation=None) + + if in_chs == out_chs and self.stride == 1: + self.shortcut = nn.Sequential() + else: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_chs, + in_chs, + dw_kernel_size, + stride=stride, + padding=(dw_kernel_size - 1) // 2, + groups=in_chs, + bias=False, + ), + nn.BatchNorm2d(in_chs), + nn.Conv2d(in_chs, out_chs, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(out_chs), + ) + + def forward(self, x): + residual = x + + x = self.ghost1(x) + + if self.stride > 1: + x = self.conv_dw(x) + x = self.bn_dw(x) + + if self.se is not None: + x = self.se(x) + + x = self.ghost2(x) + + x += self.shortcut(residual) + return x + + +class GhostBlocks(nn.Module): + """Stack of GhostBottleneck used in GhostPAN. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + expand (int): Expand ratio of GhostBottleneck. Default: 1. + kernel_size (int): Kernel size of depthwise convolution. Default: 5. + num_blocks (int): Number of GhostBottlecneck blocks. Default: 1. + use_res (bool): Whether to use residual connection. Default: False. + activation (str): Name of activation function. Default: LeakyReLU. + """ + + def __init__( + self, + in_channels, + out_channels, + expand=1, + kernel_size=5, + num_blocks=1, + use_res=False, + activation='LeakyReLU', + ): + super(GhostBlocks, self).__init__() + self.use_res = use_res + if use_res: + self.reduce_conv = ConvModule( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + activation=activation, + ) + blocks = [] + for _ in range(num_blocks): + blocks.append( + GhostBottleneck( + in_channels, + int(out_channels * expand), + out_channels, + dw_kernel_size=kernel_size, + activation=activation, + )) + self.blocks = nn.Sequential(*blocks) + + def forward(self, x): + out = self.blocks(x) + if self.use_res: + out = out + self.reduce_conv(x) + return out + + +class GhostPAN(nn.Module): + """Path Aggregation Network with Ghost block. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_csp_blocks (int): Number of bottlenecks in CSPLayer. Default: 3 + use_depthwise (bool): Whether to depthwise separable convolution in + blocks. Default: False + kernel_size (int): Kernel size of depthwise convolution. Default: 5. + expand (int): Expand ratio of GhostBottleneck. Default: 1. + num_blocks (int): Number of GhostBottlecneck blocks. Default: 1. + use_res (bool): Whether to use residual connection. Default: False. + num_extra_level (int): Number of extra conv layers for more feature levels. + Default: 0. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(scale_factor=2, mode='nearest')` + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN') + activation (str): Activation layer name. + Default: LeakyReLU. + """ + + def __init__( + self, + in_channels, + out_channels, + use_depthwise=False, + kernel_size=5, + expand=1, + num_blocks=1, + use_res=False, + num_extra_level=0, + upsample_cfg=dict(scale_factor=2, mode='bilinear'), + norm_cfg=dict(type='BN'), + activation='LeakyReLU', + ): + super(GhostPAN, self).__init__() + assert num_extra_level >= 0 + assert num_blocks >= 1 + self.in_channels = in_channels + self.out_channels = out_channels + + conv = DepthwiseConvModule if use_depthwise else ConvModule + + # build top-down blocks + self.upsample = nn.Upsample(**upsample_cfg) + self.reduce_layers = nn.ModuleList() + for idx in range(len(in_channels)): + self.reduce_layers.append( + ConvModule( + in_channels[idx], + out_channels, + 1, + norm_cfg=norm_cfg, + activation=activation, + )) + self.top_down_blocks = nn.ModuleList() + for idx in range(len(in_channels) - 1, 0, -1): + self.top_down_blocks.append( + GhostBlocks( + out_channels * 2, + out_channels, + expand, + kernel_size=kernel_size, + num_blocks=num_blocks, + use_res=use_res, + activation=activation, + )) + + # build bottom-up blocks + self.downsamples = nn.ModuleList() + self.bottom_up_blocks = nn.ModuleList() + for idx in range(len(in_channels) - 1): + self.downsamples.append( + conv( + out_channels, + out_channels, + kernel_size, + stride=2, + padding=kernel_size // 2, + norm_cfg=norm_cfg, + activation=activation, + )) + self.bottom_up_blocks.append( + GhostBlocks( + out_channels * 2, + out_channels, + expand, + kernel_size=kernel_size, + num_blocks=num_blocks, + use_res=use_res, + activation=activation, + )) + + # extra layers + self.extra_lvl_in_conv = nn.ModuleList() + self.extra_lvl_out_conv = nn.ModuleList() + for i in range(num_extra_level): + self.extra_lvl_in_conv.append( + conv( + out_channels, + out_channels, + kernel_size, + stride=2, + padding=kernel_size // 2, + norm_cfg=norm_cfg, + activation=activation, + )) + self.extra_lvl_out_conv.append( + conv( + out_channels, + out_channels, + kernel_size, + stride=2, + padding=kernel_size // 2, + norm_cfg=norm_cfg, + activation=activation, + )) + + def forward(self, inputs): + """ + Args: + inputs (tuple[Tensor]): input features. + Returns: + tuple[Tensor]: multi level features. + """ + assert len(inputs) == len(self.in_channels) + inputs = [ + reduce(input_x) + for input_x, reduce in zip(inputs, self.reduce_layers) + ] + # top-down path + inner_outs = [inputs[-1]] + for idx in range(len(self.in_channels) - 1, 0, -1): + feat_heigh = inner_outs[0] + feat_low = inputs[idx - 1] + + inner_outs[0] = feat_heigh + + upsample_feat = self.upsample(feat_heigh) + + inner_out = self.top_down_blocks[len(self.in_channels) - 1 - idx]( + torch.cat([upsample_feat, feat_low], 1)) + inner_outs.insert(0, inner_out) + + # bottom-up path + outs = [inner_outs[0]] + for idx in range(len(self.in_channels) - 1): + feat_low = outs[-1] + feat_height = inner_outs[idx + 1] + downsample_feat = self.downsamples[idx](feat_low) + out = self.bottom_up_blocks[idx]( + torch.cat([downsample_feat, feat_height], 1)) + outs.append(out) + + # extra layers + for extra_in_layer, extra_out_layer in zip(self.extra_lvl_in_conv, + self.extra_lvl_out_conv): + outs.append(extra_in_layer(inputs[-1]) + extra_out_layer(outs[-1])) + + return tuple(outs) diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/nanodet_plus_head.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/nanodet_plus_head.py new file mode 100644 index 0000000000000000000000000000000000000000..cf0cdf3ccd141cafdafeaad9e7e4bee81f8e7606 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/nanodet_plus_head.py @@ -0,0 +1,427 @@ +# The implementation here is modified based on nanodet, +# originally Apache 2.0 License and publicly available at https://github.com/RangiLyu/nanodet + +import math + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.ops import nms + +from .utils import ConvModule, DepthwiseConvModule + + +class Integral(nn.Module): + """A fixed layer for calculating integral result from distribution. + This layer calculates the target location by :math: `sum{P(y_i) * y_i}`, + P(y_i) denotes the softmax vector that represents the discrete distribution + y_i denotes the discrete set, usually {0, 1, 2, ..., reg_max} + Args: + reg_max (int): The maximal value of the discrete set. Default: 16. You + may want to reset it according to your new dataset or related + settings. + """ + + def __init__(self, reg_max=16): + super(Integral, self).__init__() + self.reg_max = reg_max + self.register_buffer('project', + torch.linspace(0, self.reg_max, self.reg_max + 1)) + + def forward(self, x): + """Forward feature from the regression head to get integral result of + bounding box location. + Args: + x (Tensor): Features of the regression head, shape (N, 4*(n+1)), + n is self.reg_max. + Returns: + x (Tensor): Integral result of box locations, i.e., distance + offsets from the box center in four directions, shape (N, 4). + """ + shape = x.size() + x = F.softmax(x.reshape(*shape[:-1], 4, self.reg_max + 1), dim=-1) + x = F.linear(x, self.project.type_as(x)).reshape(*shape[:-1], 4) + return x + + +def batched_nms(boxes, scores, idxs, nms_cfg, class_agnostic=False): + """Performs non-maximum suppression in a batched fashion. + Modified from https://github.com/pytorch/vision/blob + /505cd6957711af790211896d32b40291bea1bc21/torchvision/ops/boxes.py#L39. + In order to perform NMS independently per class, we add an offset to all + the boxes. The offset is dependent only on the class idx, and is large + enough so that boxes from different classes do not overlap. + Arguments: + boxes (torch.Tensor): boxes in shape (N, 4). + scores (torch.Tensor): scores in shape (N, ). + idxs (torch.Tensor): each index value correspond to a bbox cluster, + and NMS will not be applied between elements of different idxs, + shape (N, ). + nms_cfg (dict): specify nms type and other parameters like iou_thr. + Possible keys includes the following. + - iou_thr (float): IoU threshold used for NMS. + - split_thr (float): threshold number of boxes. In some cases the + number of boxes is large (e.g., 200k). To avoid OOM during + training, the users could set `split_thr` to a small value. + If the number of boxes is greater than the threshold, it will + perform NMS on each group of boxes separately and sequentially. + Defaults to 10000. + class_agnostic (bool): if true, nms is class agnostic, + i.e. IoU thresholding happens over all boxes, + regardless of the predicted class. + Returns: + tuple: kept dets and indice. + """ + nms_cfg_ = nms_cfg.copy() + class_agnostic = nms_cfg_.pop('class_agnostic', class_agnostic) + if class_agnostic: + boxes_for_nms = boxes + else: + max_coordinate = boxes.max() + offsets = idxs.to(boxes) * (max_coordinate + 1) + boxes_for_nms = boxes + offsets[:, None] + nms_cfg_.pop('type', 'nms') + split_thr = nms_cfg_.pop('split_thr', 10000) + if len(boxes_for_nms) < split_thr: + keep = nms(boxes_for_nms, scores, **nms_cfg_) + boxes = boxes[keep] + scores = scores[keep] + else: + total_mask = scores.new_zeros(scores.size(), dtype=torch.bool) + for id in torch.unique(idxs): + mask = (idxs == id).nonzero(as_tuple=False).view(-1) + keep = nms(boxes_for_nms[mask], scores[mask], **nms_cfg_) + total_mask[mask[keep]] = True + + keep = total_mask.nonzero(as_tuple=False).view(-1) + keep = keep[scores[keep].argsort(descending=True)] + boxes = boxes[keep] + scores = scores[keep] + + return torch.cat([boxes, scores[:, None]], -1), keep + + +def multiclass_nms(multi_bboxes, + multi_scores, + score_thr, + nms_cfg, + max_num=-1, + score_factors=None): + """NMS for multi-class bboxes. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_scores (Tensor): shape (n, #class), where the last column + contains scores of the background class, but this will be ignored. + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + nms_thr (float): NMS IoU threshold + max_num (int): if there are more than max_num bboxes after NMS, + only top max_num will be kept. + score_factors (Tensor): The factors multiplied to scores before + applying NMS + + Returns: + tuple: (bboxes, labels), tensors of shape (k, 5) and (k, 1). Labels \ + are 0-based. + """ + num_classes = multi_scores.size(1) - 1 + if multi_bboxes.shape[1] > 4: + bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4) + else: + bboxes = multi_bboxes[:, None].expand( + multi_scores.size(0), num_classes, 4) + scores = multi_scores[:, :-1] + + valid_mask = scores > score_thr + + bboxes = torch.masked_select( + bboxes, + torch.stack((valid_mask, valid_mask, valid_mask, valid_mask), + -1)).view(-1, 4) + if score_factors is not None: + scores = scores * score_factors[:, None] + scores = torch.masked_select(scores, valid_mask) + labels = valid_mask.nonzero(as_tuple=False)[:, 1] + + if bboxes.numel() == 0: + bboxes = multi_bboxes.new_zeros((0, 5)) + labels = multi_bboxes.new_zeros((0, ), dtype=torch.long) + + if torch.onnx.is_in_onnx_export(): + raise RuntimeError('[ONNX Error] Can not record NMS ' + 'as it has not been executed this time') + return bboxes, labels + + dets, keep = batched_nms(bboxes, scores, labels, nms_cfg) + + if max_num > 0: + dets = dets[:max_num] + keep = keep[:max_num] + + return dets, labels[keep] + + +def distance2bbox(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + + Args: + points (Tensor): Shape (n, 2), [x, y]. + distance (Tensor): Distance from the given point to 4 + boundaries (left, top, right, bottom). + max_shape (tuple): Shape of the image. + + Returns: + Tensor: Decoded bboxes. + """ + x1 = points[..., 0] - distance[..., 0] + y1 = points[..., 1] - distance[..., 1] + x2 = points[..., 0] + distance[..., 2] + y2 = points[..., 1] + distance[..., 3] + if max_shape is not None: + x1 = x1.clamp(min=0, max=max_shape[1]) + y1 = y1.clamp(min=0, max=max_shape[0]) + x2 = x2.clamp(min=0, max=max_shape[1]) + y2 = y2.clamp(min=0, max=max_shape[0]) + return torch.stack([x1, y1, x2, y2], -1) + + +def warp_boxes(boxes, M, width, height): + n = len(boxes) + if n: + xy = np.ones((n * 4, 3)) + xy[:, :2] = boxes[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(n * 4, 2) + xy = xy @ M.T + xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) + x = xy[:, [0, 2, 4, 6]] + y = xy[:, [1, 3, 5, 7]] + xy = np.concatenate( + (x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T + xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width) + xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height) + return xy.astype(np.float32) + else: + return boxes + + +class NanoDetPlusHead(nn.Module): + """Detection head used in NanoDet-Plus. + + Args: + num_classes (int): Number of categories excluding the background + category. + loss (dict): Loss config. + input_channel (int): Number of channels of the input feature. + feat_channels (int): Number of channels of the feature. + Default: 96. + stacked_convs (int): Number of conv layers in the stacked convs. + Default: 2. + kernel_size (int): Size of the convolving kernel. Default: 5. + strides (list[int]): Strides of input multi-level feature maps. + Default: [8, 16, 32]. + conv_type (str): Type of the convolution. + Default: "DWConv". + norm_cfg (dict): Dictionary to construct and config norm layer. + Default: dict(type='BN'). + reg_max (int): The maximal value of the discrete set. Default: 7. + activation (str): Type of activation function. Default: "LeakyReLU". + assigner_cfg (dict): Config dict of the assigner. Default: dict(topk=13). + """ + + def __init__(self, + num_classes, + input_channel, + feat_channels=96, + stacked_convs=2, + kernel_size=5, + strides=[8, 16, 32], + conv_type='DWConv', + norm_cfg=dict(type='BN'), + reg_max=7, + activation='LeakyReLU', + assigner_cfg=dict(topk=13), + **kwargs): + super(NanoDetPlusHead, self).__init__() + self.num_classes = num_classes + self.in_channels = input_channel + self.feat_channels = feat_channels + self.stacked_convs = stacked_convs + self.kernel_size = kernel_size + self.strides = strides + self.reg_max = reg_max + self.activation = activation + self.ConvModule = ConvModule if conv_type == 'Conv' else DepthwiseConvModule + + self.norm_cfg = norm_cfg + self.distribution_project = Integral(self.reg_max) + + self._init_layers() + + def _init_layers(self): + self.cls_convs = nn.ModuleList() + for _ in self.strides: + cls_convs = self._buid_not_shared_head() + self.cls_convs.append(cls_convs) + + self.gfl_cls = nn.ModuleList([ + nn.Conv2d( + self.feat_channels, + self.num_classes + 4 * (self.reg_max + 1), + 1, + padding=0, + ) for _ in self.strides + ]) + + def _buid_not_shared_head(self): + cls_convs = nn.ModuleList() + for i in range(self.stacked_convs): + chn = self.in_channels if i == 0 else self.feat_channels + cls_convs.append( + self.ConvModule( + chn, + self.feat_channels, + self.kernel_size, + stride=1, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg, + bias=self.norm_cfg is None, + activation=self.activation, + )) + return cls_convs + + def forward(self, feats): + if torch.onnx.is_in_onnx_export(): + return self._forward_onnx(feats) + outputs = [] + for feat, cls_convs, gfl_cls in zip( + feats, + self.cls_convs, + self.gfl_cls, + ): + for conv in cls_convs: + feat = conv(feat) + output = gfl_cls(feat) + outputs.append(output.flatten(start_dim=2)) + outputs = torch.cat(outputs, dim=2).permute(0, 2, 1) + return outputs + + def post_process(self, preds, meta): + """Prediction results post processing. Decode bboxes and rescale + to original image size. + Args: + preds (Tensor): Prediction output. + meta (dict): Meta info. + """ + cls_scores, bbox_preds = preds.split( + [self.num_classes, 4 * (self.reg_max + 1)], dim=-1) + result_list = self.get_bboxes(cls_scores, bbox_preds, meta) + det_results = {} + warp_matrixes = ( + meta['warp_matrix'] + if isinstance(meta['warp_matrix'], list) else meta['warp_matrix']) + img_heights = ( + meta['img_info']['height'].cpu().numpy() if isinstance( + meta['img_info']['height'], torch.Tensor) else + meta['img_info']['height']) + img_widths = ( + meta['img_info']['width'].cpu().numpy() if isinstance( + meta['img_info']['width'], torch.Tensor) else + meta['img_info']['width']) + img_ids = ( + meta['img_info']['id'].cpu().numpy() if isinstance( + meta['img_info']['id'], torch.Tensor) else + meta['img_info']['id']) + + for result, img_width, img_height, img_id, warp_matrix in zip( + result_list, img_widths, img_heights, img_ids, warp_matrixes): + det_result = {} + det_bboxes, det_labels = result + det_bboxes = det_bboxes.detach().cpu().numpy() + det_bboxes[:, :4] = warp_boxes(det_bboxes[:, :4], + np.linalg.inv(warp_matrix), + img_width, img_height) + classes = det_labels.detach().cpu().numpy() + for i in range(self.num_classes): + inds = classes == i + det_result[i] = np.concatenate( + [ + det_bboxes[inds, :4].astype(np.float32), + det_bboxes[inds, 4:5].astype(np.float32), + ], + axis=1, + ).tolist() + det_results[img_id] = det_result + return det_results + + def get_bboxes(self, cls_preds, reg_preds, img_metas): + """Decode the outputs to bboxes. + Args: + cls_preds (Tensor): Shape (num_imgs, num_points, num_classes). + reg_preds (Tensor): Shape (num_imgs, num_points, 4 * (regmax + 1)). + img_metas (dict): Dict of image info. + + Returns: + results_list (list[tuple]): List of detection bboxes and labels. + """ + device = cls_preds.device + b = cls_preds.shape[0] + input_height, input_width = img_metas['img'].shape[2:] + input_shape = (input_height, input_width) + + featmap_sizes = [(math.ceil(input_height / stride), + math.ceil(input_width) / stride) + for stride in self.strides] + mlvl_center_priors = [ + self.get_single_level_center_priors( + b, + featmap_sizes[i], + stride, + dtype=torch.float32, + device=device, + ) for i, stride in enumerate(self.strides) + ] + center_priors = torch.cat(mlvl_center_priors, dim=1) + dis_preds = self.distribution_project(reg_preds) * center_priors[..., + 2, + None] + bboxes = distance2bbox( + center_priors[..., :2], dis_preds, max_shape=input_shape) + scores = cls_preds.sigmoid() + result_list = [] + for i in range(b): + score, bbox = scores[i], bboxes[i] + padding = score.new_zeros(score.shape[0], 1) + score = torch.cat([score, padding], dim=1) + results = multiclass_nms( + bbox, + score, + score_thr=0.05, + nms_cfg=dict(type='nms', iou_threshold=0.6), + max_num=100, + ) + result_list.append(results) + return result_list + + def get_single_level_center_priors(self, batch_size, featmap_size, stride, + dtype, device): + """Generate centers of a single stage feature map. + Args: + batch_size (int): Number of images in one batch. + featmap_size (tuple[int]): height and width of the feature map + stride (int): down sample stride of the feature map + dtype (obj:`torch.dtype`): data type of the tensors + device (obj:`torch.device`): device of the tensors + Return: + priors (Tensor): center priors of a single level feature map. + """ + h, w = featmap_size + x_range = (torch.arange(w, dtype=dtype, device=device)) * stride + y_range = (torch.arange(h, dtype=dtype, device=device)) * stride + y, x = torch.meshgrid(y_range, x_range) + y = y.flatten() + x = x.flatten() + strides = x.new_full((x.shape[0], ), stride) + proiors = torch.stack([x, y, strides, strides], dim=-1) + return proiors.unsqueeze(0).repeat(batch_size, 1, 1) diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/one_stage_detector.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/one_stage_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..a8d376fdaca080d153f706afd827e06afa4508f5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/one_stage_detector.py @@ -0,0 +1,61 @@ +# The implementation here is modified based on nanodet, +# originally Apache 2.0 License and publicly available at https://github.com/RangiLyu/nanodet + +import torch +import torch.nn as nn + +from .ghost_pan import GhostPAN +from .nanodet_plus_head import NanoDetPlusHead +from .shufflenetv2 import ShuffleNetV2 + + +class OneStageDetector(nn.Module): + + def __init__(self): + super(OneStageDetector, self).__init__() + self.backbone = ShuffleNetV2( + model_size='1.0x', + out_stages=(2, 3, 4), + with_last_conv=False, + kernal_size=3, + activation='LeakyReLU', + pretrain=False) + self.fpn = GhostPAN( + in_channels=[116, 232, 464], + out_channels=96, + use_depthwise=True, + kernel_size=5, + expand=1, + num_blocks=1, + use_res=False, + num_extra_level=1, + upsample_cfg=dict(scale_factor=2, mode='bilinear'), + norm_cfg=dict(type='BN'), + activation='LeakyReLU') + self.head = NanoDetPlusHead( + num_classes=3, + input_channel=96, + feat_channels=96, + stacked_convs=2, + kernel_size=5, + strides=[8, 16, 32, 64], + conv_type='DWConv', + norm_cfg=dict(type='BN'), + reg_max=7, + activation='LeakyReLU', + assigner_cfg=dict(topk=13)) + self.epoch = 0 + + def forward(self, x): + x = self.backbone(x) + if hasattr(self, 'fpn'): + x = self.fpn(x) + if hasattr(self, 'head'): + x = self.head(x) + return x + + def inference(self, meta): + with torch.no_grad(): + preds = self(meta['img']) + results = self.head.post_process(preds, meta) + return results diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/shufflenetv2.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/shufflenetv2.py new file mode 100644 index 0000000000000000000000000000000000000000..5038ef5a60b96048f1e639a116630161c65b043c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/shufflenetv2.py @@ -0,0 +1,182 @@ +# The implementation here is modified based on nanodet, +# originally Apache 2.0 License and publicly available at https://github.com/RangiLyu/nanodet + +import torch +import torch.nn as nn + +from .utils import act_layers + + +def channel_shuffle(x, groups): + batchsize, num_channels, height, width = x.data.size() + channels_per_group = num_channels // groups + + x = x.view(batchsize, groups, channels_per_group, height, width) + + x = torch.transpose(x, 1, 2).contiguous() + + x = x.view(batchsize, -1, height, width) + + return x + + +class ShuffleV2Block(nn.Module): + + def __init__(self, inp, oup, stride, activation='ReLU'): + super(ShuffleV2Block, self).__init__() + + if not (1 <= stride <= 3): + raise ValueError('illegal stride value') + self.stride = stride + + branch_features = oup // 2 + assert (self.stride != 1) or (inp == branch_features << 1) + + if self.stride > 1: + self.branch1 = nn.Sequential( + self.depthwise_conv( + inp, inp, kernel_size=3, stride=self.stride, padding=1), + nn.BatchNorm2d(inp), + nn.Conv2d( + inp, + branch_features, + kernel_size=1, + stride=1, + padding=0, + bias=False), + nn.BatchNorm2d(branch_features), + act_layers(activation), + ) + else: + self.branch1 = nn.Sequential() + + self.branch2 = nn.Sequential( + nn.Conv2d( + inp if (self.stride > 1) else branch_features, + branch_features, + kernel_size=1, + stride=1, + padding=0, + bias=False, + ), + nn.BatchNorm2d(branch_features), + act_layers(activation), + self.depthwise_conv( + branch_features, + branch_features, + kernel_size=3, + stride=self.stride, + padding=1, + ), + nn.BatchNorm2d(branch_features), + nn.Conv2d( + branch_features, + branch_features, + kernel_size=1, + stride=1, + padding=0, + bias=False, + ), + nn.BatchNorm2d(branch_features), + act_layers(activation), + ) + + @staticmethod + def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False): + return nn.Conv2d( + i, o, kernel_size, stride, padding, bias=bias, groups=i) + + def forward(self, x): + if self.stride == 1: + x1, x2 = x.chunk(2, dim=1) + out = torch.cat((x1, self.branch2(x2)), dim=1) + else: + out = torch.cat((self.branch1(x), self.branch2(x)), dim=1) + + out = channel_shuffle(out, 2) + + return out + + +class ShuffleNetV2(nn.Module): + + def __init__( + self, + model_size='1.5x', + out_stages=(2, 3, 4), + with_last_conv=False, + kernal_size=3, + activation='ReLU', + pretrain=True, + ): + super(ShuffleNetV2, self).__init__() + assert set(out_stages).issubset((2, 3, 4)) + + print('model size is ', model_size) + + self.stage_repeats = [4, 8, 4] + self.model_size = model_size + self.out_stages = out_stages + self.with_last_conv = with_last_conv + self.kernal_size = kernal_size + self.activation = activation + if model_size == '0.5x': + self._stage_out_channels = [24, 48, 96, 192, 1024] + elif model_size == '1.0x': + self._stage_out_channels = [24, 116, 232, 464, 1024] + elif model_size == '1.5x': + self._stage_out_channels = [24, 176, 352, 704, 1024] + elif model_size == '2.0x': + self._stage_out_channels = [24, 244, 488, 976, 2048] + else: + raise NotImplementedError + + # building first layer + input_channels = 3 + output_channels = self._stage_out_channels[0] + self.conv1 = nn.Sequential( + nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False), + nn.BatchNorm2d(output_channels), + act_layers(activation), + ) + input_channels = output_channels + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + stage_names = ['stage{}'.format(i) for i in [2, 3, 4]] + for name, repeats, output_channels in zip( + stage_names, self.stage_repeats, self._stage_out_channels[1:]): + seq = [ + ShuffleV2Block( + input_channels, output_channels, 2, activation=activation) + ] + for i in range(repeats - 1): + seq.append( + ShuffleV2Block( + output_channels, + output_channels, + 1, + activation=activation)) + setattr(self, name, nn.Sequential(*seq)) + input_channels = output_channels + output_channels = self._stage_out_channels[-1] + if self.with_last_conv: + conv5 = nn.Sequential( + nn.Conv2d( + input_channels, output_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(output_channels), + act_layers(activation), + ) + self.stage4.add_module('conv5', conv5) + + def forward(self, x): + x = self.conv1(x) + x = self.maxpool(x) + output = [] + + for i in range(2, 5): + stage = getattr(self, 'stage{}'.format(i)) + x = stage(x) + if i in self.out_stages: + output.append(x) + return tuple(output) diff --git a/runtime/Lib/site_packages/models/cv/face_human_hand_detection/utils.py b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..478dd7c2bdb8c3db8c8cc7347f73fc0f2fb4fc75 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_human_hand_detection/utils.py @@ -0,0 +1,277 @@ +# The implementation here is modified based on nanodet, +# originally Apache 2.0 License and publicly available at https://github.com/RangiLyu/nanodet + +import torch +import torch.nn as nn + +activations = { + 'ReLU': nn.ReLU, + 'LeakyReLU': nn.LeakyReLU, + 'ReLU6': nn.ReLU6, + 'SELU': nn.SELU, + 'ELU': nn.ELU, + 'GELU': nn.GELU, + 'PReLU': nn.PReLU, + 'SiLU': nn.SiLU, + 'HardSwish': nn.Hardswish, + 'Hardswish': nn.Hardswish, + None: nn.Identity, +} + + +def act_layers(name): + assert name in activations.keys() + if name == 'LeakyReLU': + return nn.LeakyReLU(negative_slope=0.1, inplace=True) + elif name == 'GELU': + return nn.GELU() + elif name == 'PReLU': + return nn.PReLU() + else: + return activations[name](inplace=True) + + +norm_cfg = { + 'BN': ('bn', nn.BatchNorm2d), + 'SyncBN': ('bn', nn.SyncBatchNorm), + 'GN': ('gn', nn.GroupNorm), +} + + +def build_norm_layer(cfg, num_features, postfix=''): + """Build normalization layer + + Args: + cfg (dict): cfg should contain: + type (str): identify norm layer type. + layer args: args needed to instantiate a norm layer. + requires_grad (bool): [optional] whether stop gradient updates + num_features (int): number of channels from input. + postfix (int, str): appended into norm abbreviation to + create named layer. + + Returns: + name (str): abbreviation + postfix + layer (nn.Module): created norm layer + """ + assert isinstance(cfg, dict) and 'type' in cfg + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in norm_cfg: + raise KeyError('Unrecognized norm type {}'.format(layer_type)) + else: + abbr, norm_layer = norm_cfg[layer_type] + if norm_layer is None: + raise NotImplementedError + + assert isinstance(postfix, (int, str)) + name = abbr + str(postfix) + + requires_grad = cfg_.pop('requires_grad', True) + cfg_.setdefault('eps', 1e-5) + if layer_type != 'GN': + layer = norm_layer(num_features, **cfg_) + if layer_type == 'SyncBN' and hasattr(layer, '_specify_ddp_gpu_num'): + layer._specify_ddp_gpu_num(1) + else: + assert 'num_groups' in cfg_ + layer = norm_layer(num_channels=num_features, **cfg_) + + for param in layer.parameters(): + param.requires_grad = requires_grad + + return name, layer + + +class ConvModule(nn.Module): + """A conv block that contains conv/norm/activation layers. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int or tuple[int]): Same as nn.Conv2d. + stride (int or tuple[int]): Same as nn.Conv2d. + padding (int or tuple[int]): Same as nn.Conv2d. + dilation (int or tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + bias (bool or str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if norm_cfg is None, otherwise + False. + conv_cfg (dict): Config dict for convolution layer. + norm_cfg (dict): Config dict for normalization layer. + activation (str): activation layer, "ReLU" by default. + inplace (bool): Whether to use inplace mode for activation. + order (tuple[str]): The order of conv/norm/activation layers. It is a + sequence of "conv", "norm" and "act". Examples are + ("conv", "norm", "act") and ("act", "conv", "norm"). + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias='auto', + conv_cfg=None, + norm_cfg=None, + activation='ReLU', + inplace=True, + order=('conv', 'norm', 'act'), + ): + super(ConvModule, self).__init__() + assert conv_cfg is None or isinstance(conv_cfg, dict) + assert norm_cfg is None or isinstance(norm_cfg, dict) + assert activation is None or isinstance(activation, str) + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.activation = activation + self.inplace = inplace + self.order = order + assert isinstance(self.order, tuple) and len(self.order) == 3 + assert set(order) == {'conv', 'norm', 'act'} + + self.with_norm = norm_cfg is not None + if bias == 'auto': + bias = False if self.with_norm else True + self.with_bias = bias + + if self.with_norm and self.with_bias: + warnings.warn('ConvModule has norm and bias at the same time') + + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + ) + self.in_channels = self.conv.in_channels + self.out_channels = self.conv.out_channels + self.kernel_size = self.conv.kernel_size + self.stride = self.conv.stride + self.padding = self.conv.padding + self.dilation = self.conv.dilation + self.transposed = self.conv.transposed + self.output_padding = self.conv.output_padding + self.groups = self.conv.groups + + if self.with_norm: + if order.index('norm') > order.index('conv'): + norm_channels = out_channels + else: + norm_channels = in_channels + self.norm_name, norm = build_norm_layer(norm_cfg, norm_channels) + self.add_module(self.norm_name, norm) + else: + self.norm_name = None + + if self.activation: + self.act = act_layers(self.activation) + + @property + def norm(self): + if self.norm_name: + return getattr(self, self.norm_name) + else: + return None + + def forward(self, x, norm=True): + for layer in self.order: + if layer == 'conv': + x = self.conv(x) + elif layer == 'norm' and norm and self.with_norm: + x = self.norm(x) + elif layer == 'act' and self.activation: + x = self.act(x) + return x + + +class DepthwiseConvModule(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + bias='auto', + norm_cfg=dict(type='BN'), + activation='ReLU', + inplace=True, + order=('depthwise', 'dwnorm', 'act', 'pointwise', 'pwnorm', 'act'), + ): + super(DepthwiseConvModule, self).__init__() + assert activation is None or isinstance(activation, str) + self.activation = activation + self.inplace = inplace + self.order = order + assert isinstance(self.order, tuple) and len(self.order) == 6 + assert set(order) == { + 'depthwise', + 'dwnorm', + 'act', + 'pointwise', + 'pwnorm', + 'act', + } + + self.with_norm = norm_cfg is not None + if bias == 'auto': + bias = False if self.with_norm else True + self.with_bias = bias + + if self.with_norm and self.with_bias: + warnings.warn('ConvModule has norm and bias at the same time') + + self.depthwise = nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=in_channels, + bias=bias, + ) + self.pointwise = nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias) + + self.in_channels = self.depthwise.in_channels + self.out_channels = self.pointwise.out_channels + self.kernel_size = self.depthwise.kernel_size + self.stride = self.depthwise.stride + self.padding = self.depthwise.padding + self.dilation = self.depthwise.dilation + self.transposed = self.depthwise.transposed + self.output_padding = self.depthwise.output_padding + + if self.with_norm: + _, self.dwnorm = build_norm_layer(norm_cfg, in_channels) + _, self.pwnorm = build_norm_layer(norm_cfg, out_channels) + + if self.activation: + self.act = act_layers(self.activation) + + def forward(self, x, norm=True): + for layer_name in self.order: + if layer_name != 'act': + layer = self.__getattr__(layer_name) + x = layer(x) + elif layer_name == 'act' and self.activation: + x = self.act(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/face_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/align_face.py b/runtime/Lib/site_packages/models/cv/face_recognition/align_face.py new file mode 100644 index 0000000000000000000000000000000000000000..b5fa95b6c144d033e2d143ace5ddf9548124c4ae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/align_face.py @@ -0,0 +1,54 @@ +""" +The implementation here is modified based on insightface, originally MIT license and publicly available at +https://github.com/deepinsight/insightface/blob/master/python-package/insightface/utils/face_align.py +""" +import cv2 +import numpy as np +from skimage import transform as trans + + +def align_face(image, size, lmks): + dst_w = size[1] + dst_h = size[0] + # landmark calculation of dst images + base_w = 96 + base_h = 112 + assert (dst_w >= base_w) + assert (dst_h >= base_h) + base_lmk = [ + 30.2946, 51.6963, 65.5318, 51.5014, 48.0252, 71.7366, 33.5493, 92.3655, + 62.7299, 92.2041 + ] + + dst_lmk = np.array(base_lmk).reshape((5, 2)).astype(np.float32) + if dst_w != base_w: + slide = (dst_w - base_w) / 2 + dst_lmk[:, 0] += slide + + if dst_h != base_h: + slide = (dst_h - base_h) / 2 + dst_lmk[:, 1] += slide + + src_lmk = lmks + # using skimage method + tform = trans.SimilarityTransform() + tform.estimate(src_lmk, dst_lmk) + t = tform.params[0:2, :] + + assert (image.shape[2] == 3) + + dst_image = cv2.warpAffine(image.copy(), t, (dst_w, dst_h)) + dst_pts = GetAffinePoints(src_lmk, t) + return dst_image, dst_pts + + +def GetAffinePoints(pts_in, trans): + pts_out = pts_in.copy() + assert (pts_in.shape[1] == 2) + + for k in range(pts_in.shape[0]): + pts_out[k, 0] = pts_in[k, 0] * trans[0, 0] + pts_in[k, 1] * trans[ + 0, 1] + trans[0, 2] + pts_out[k, 1] = pts_in[k, 0] * trans[1, 0] + pts_in[k, 1] * trans[ + 1, 1] + trans[1, 2] + return pts_out diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/__init__.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5531670a1acc1292bf3c943c812cdcfe944c614b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .rts_backbone import RTSBackbone +else: + _import_structure = {'rts_backbone': ['RTSBackbone']} + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__) diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..afe8996328e32dde8aca8093f304c0e4980debe6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/__init__.py @@ -0,0 +1,33 @@ +# The implementation is adopted from TFace,made pubicly available under the Apache-2.0 license at +# https://github.com/Tencent/TFace/blob/master/recognition/torchkit/backbone +from .model_irse import (IR_18, IR_34, IR_50, IR_101, IR_152, IR_200, IR_SE_50, + IR_SE_101, IR_SE_152, IR_SE_200) +from .model_resnet import ResNet_50, ResNet_101, ResNet_152 + +_model_dict = { + 'ResNet_50': ResNet_50, + 'ResNet_101': ResNet_101, + 'ResNet_152': ResNet_152, + 'IR_18': IR_18, + 'IR_34': IR_34, + 'IR_50': IR_50, + 'IR_101': IR_101, + 'IR_152': IR_152, + 'IR_200': IR_200, + 'IR_SE_50': IR_SE_50, + 'IR_SE_101': IR_SE_101, + 'IR_SE_152': IR_SE_152, + 'IR_SE_200': IR_SE_200 +} + + +def get_model(key): + """ Get different backbone network by key, + support ResNet50, ResNet_101, ResNet_152 + IR_18, IR_34, IR_50, IR_101, IR_152, IR_200, + IR_SE_50, IR_SE_101, IR_SE_152, IR_SE_200. + """ + if key in _model_dict.keys(): + return _model_dict[key] + else: + raise KeyError('not support model {}'.format(key)) diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/arcface_backbone.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/arcface_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..25b9fe33290fe7bd230711fb910c70bf4f59a4d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/arcface_backbone.py @@ -0,0 +1,200 @@ +# The implementation is adopted from TFace,made pubicly available under the Apache-2.0 license at +# https://github.com/deepinsight/insightface/blob/master/recognition/arcface_torch/backbones/iresnet.py +import torch +from torch import nn +from torch.utils.checkpoint import checkpoint + +using_ckpt = False + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class IBasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1): + super(IBasicBlock, self).__init__() + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + self.bn1 = nn.BatchNorm2d( + inplanes, + eps=1e-05, + ) + self.conv1 = conv3x3(inplanes, planes) + self.bn2 = nn.BatchNorm2d( + planes, + eps=1e-05, + ) + self.prelu = nn.PReLU(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn3 = nn.BatchNorm2d( + planes, + eps=1e-05, + ) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + out = self.bn1(x) + out = self.conv1(out) + out = self.bn2(out) + out = self.prelu(out) + out = self.conv2(out) + out = self.bn3(out) + if self.downsample is not None: + identity = self.downsample(x) + out += identity + return out + + +class IResNet(nn.Module): + fc_scale = 7 * 7 + + def __init__(self, + block, + layers, + dropout=0, + num_features=512, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + fp16=False): + super(IResNet, self).__init__() + self.extra_gflops = 0.0 + self.fp16 = fp16 + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05) + self.prelu = nn.PReLU(self.inplanes) + self.layer1 = self._make_layer(block, 64, layers[0], stride=2) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.bn2 = nn.BatchNorm2d( + 512 * block.expansion, + eps=1e-05, + ) + self.dropout = nn.Dropout(p=dropout, inplace=True) + self.fc = nn.Linear(512 * block.expansion * self.fc_scale, + num_features) + self.features = nn.BatchNorm1d(num_features, eps=1e-05) + nn.init.constant_(self.features.weight, 1.0) + self.features.weight.requires_grad = False + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, 0, 0.1) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if zero_init_residual: + for m in self.modules(): + if isinstance(m, IBasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + nn.BatchNorm2d( + planes * block.expansion, + eps=1e-05, + ), + ) + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation)) + + return nn.Sequential(*layers) + + def forward(self, x): + with torch.cuda.amp.autocast(self.fp16): + x = self.conv1(x) + x = self.bn1(x) + x = self.prelu(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.bn2(x) + x = torch.flatten(x, 1) + x = self.dropout(x) + x = self.fc(x.float() if self.fp16 else x) + x = self.features(x) + return x + + +def _iresnet(arch, layers): + model = IResNet(IBasicBlock, layers) + return model diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/common.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/common.py new file mode 100644 index 0000000000000000000000000000000000000000..a1683225ef27d87e22ea69a7f79b34b7e09341df --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/common.py @@ -0,0 +1,70 @@ +# The implementation is adopted from TFace,made pubicly available under the Apache-2.0 license at +# https://github.com/Tencent/TFace/blob/master/recognition/torchkit/backbone/common.py +import torch +import torch.nn as nn +from torch.nn import (BatchNorm1d, BatchNorm2d, Conv2d, Linear, Module, ReLU, + Sigmoid) + + +def initialize_weights(modules): + """ Weight initilize, conv2d and linear is initialized with kaiming_normal + """ + for m in modules: + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + + +class Flatten(Module): + """ Flat tensor + """ + + def forward(self, input): + return input.view(input.size(0), -1) + + +class SEModule(Module): + """ SE block + """ + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + + nn.init.xavier_uniform_(self.fc1.weight.data) + + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + + return module_input * x diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/facemask_backbone.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/facemask_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..c9e01367ef4bbda869a0b46c8e3ab35eec558f48 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/facemask_backbone.py @@ -0,0 +1,213 @@ +# The implementation is adopted from InsightFace, made pubicly available under the Apache-2.0 license at +# https://github.com/TreB1eN/InsightFace_Pytorch/blob/master/model.py + +from collections import namedtuple + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import (AdaptiveAvgPool2d, AvgPool2d, BatchNorm1d, BatchNorm2d, + Conv2d, Dropout, Dropout2d, Linear, MaxPool2d, Module, + Parameter, PReLU, ReLU, Sequential, Sigmoid) + + +class Flatten(Module): + + def forward(self, input): + return input.view(input.size(0), -1) + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + return module_input * x + + +class BottleneckIR(Module): + + def __init__(self, in_channel, depth, stride): + super(BottleneckIR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + '''A named tuple describing a ResNet block.''' + + +def get_block(in_channel, depth, num_units, stride=2): + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 252: + blocks = [ + get_block(in_channel=64, depth=64, num_units=6), + get_block(in_channel=64, depth=128, num_units=21), + get_block(in_channel=128, depth=256, num_units=66), + get_block(in_channel=256, depth=512, num_units=6) + ] + return blocks + + +class IResNet(Module): + + def __init__(self, + dropout=0, + num_features=512, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + fp16=False, + with_wcd=False, + wrs_M=400, + wrs_q=0.9): + super(IResNet, self).__init__() + num_layers = 252 + mode = 'ir' + assert num_layers in [50, 100, 152, + 252], 'num_layers should be 50,100, or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + self.fc_scale = 7 * 7 + num_features = 512 + self.fp16 = fp16 + drop_ratio = 0.0 + self.with_wcd = with_wcd + if self.with_wcd: + self.wrs_M = wrs_M + self.wrs_q = wrs_q + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = BottleneckIR + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + self.bn2 = nn.BatchNorm2d( + 512, + eps=1e-05, + ) + self.dropout = nn.Dropout(p=drop_ratio, inplace=True) + self.fc = nn.Linear(512 * self.fc_scale, num_features) + self.features = nn.BatchNorm1d(num_features, eps=1e-05) + nn.init.constant_(self.features.weight, 1.0) + self.features.weight.requires_grad = False + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + def forward(self, x): + with torch.cuda.amp.autocast(self.fp16): + x = self.input_layer(x) + x = self.body(x) + x = self.bn2(x) + if self.with_wcd: + B = x.size()[0] + C = x.size()[1] + x_abs = torch.abs(x) + score = torch.nn.functional.adaptive_avg_pool2d(x_abs, + 1).reshape( + (B, C)) + r = torch.rand((B, C), device=x.device) + key = torch.pow(r, 1. / score) + _, topidx = torch.topk(key, self.wrs_M, dim=1) + mask = torch.zeros_like(key, dtype=torch.float32) + mask.scatter_(1, topidx, 1.) + maskq = torch.rand((B, C), device=x.device) + maskq_ones = torch.ones_like(maskq, dtype=torch.float32) + maskq_zeros = torch.zeros_like(maskq, dtype=torch.float32) + maskq_m = torch.where(maskq < self.wrs_q, maskq_ones, + maskq_zeros) + new_mask = mask * maskq_m + score_sum = torch.sum(score, dim=1, keepdim=True) + selected_score_sum = torch.sum( + new_mask * score, dim=1, keepdim=True) + alpha = score_sum / (selected_score_sum + 1e-6) + alpha = alpha.reshape((B, 1, 1, 1)) + new_mask = new_mask.reshape((B, C, 1, 1)) + x = x * new_mask * alpha + x = torch.flatten(x, 1) + x = self.dropout(x) + x = self.fc(x.float() if self.fp16 else x) + x = self.features(x) + return x + + +def iresnet286(pretrained=False, progress=True, **kwargs): + model = IResNet( + dropout=0, + num_features=512, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + fp16=False, + with_wcd=False, + wrs_M=400, + wrs_q=0.9) + return model diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_irse.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_irse.py new file mode 100644 index 0000000000000000000000000000000000000000..1982ca059a7853e70058615811ebec5551f2b728 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_irse.py @@ -0,0 +1,279 @@ +# The implementation is adopted from TFace,made pubicly available under the Apache-2.0 license at +# https://github.com/Tencent/TFace/blob/master/recognition/torchkit/backbone/model_irse.py +from collections import namedtuple + +from torch.nn import (BatchNorm1d, BatchNorm2d, Conv2d, Dropout, Linear, + MaxPool2d, Module, PReLU, Sequential) + +from .common import Flatten, SEModule, initialize_weights + + +class BasicBlockIR(Module): + """ BasicBlock for IRNet + """ + + def __init__(self, in_channel, depth, stride): + super(BasicBlockIR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + BatchNorm2d(depth), PReLU(depth), + Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + + return res + shortcut + + +class BottleneckIR(Module): + """ BasicBlock with bottleneck for IRNet + """ + + def __init__(self, in_channel, depth, stride): + super(BottleneckIR, self).__init__() + reduction_channel = depth // 4 + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d( + in_channel, reduction_channel, (1, 1), (1, 1), 0, bias=False), + BatchNorm2d(reduction_channel), PReLU(reduction_channel), + Conv2d( + reduction_channel, + reduction_channel, (3, 3), (1, 1), + 1, + bias=False), BatchNorm2d(reduction_channel), + PReLU(reduction_channel), + Conv2d(reduction_channel, depth, (1, 1), stride, 0, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + + return res + shortcut + + +class BasicBlockIRSE(BasicBlockIR): + + def __init__(self, in_channel, depth, stride): + super(BasicBlockIRSE, self).__init__(in_channel, depth, stride) + self.res_layer.add_module('se_block', SEModule(depth, 16)) + + +class BottleneckIRSE(BottleneckIR): + + def __init__(self, in_channel, depth, stride): + super(BottleneckIRSE, self).__init__(in_channel, depth, stride) + self.res_layer.add_module('se_block', SEModule(depth, 16)) + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + '''A named tuple describing a ResNet block.''' + + +def get_block(in_channel, depth, num_units, stride=2): + + return [Bottleneck(in_channel, depth, stride)] +\ + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 18: + blocks = [ + get_block(in_channel=64, depth=64, num_units=2), + get_block(in_channel=64, depth=128, num_units=2), + get_block(in_channel=128, depth=256, num_units=2), + get_block(in_channel=256, depth=512, num_units=2) + ] + elif num_layers == 34: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=6), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=256, num_units=3), + get_block(in_channel=256, depth=512, num_units=8), + get_block(in_channel=512, depth=1024, num_units=36), + get_block(in_channel=1024, depth=2048, num_units=3) + ] + elif num_layers == 200: + blocks = [ + get_block(in_channel=64, depth=256, num_units=3), + get_block(in_channel=256, depth=512, num_units=24), + get_block(in_channel=512, depth=1024, num_units=36), + get_block(in_channel=1024, depth=2048, num_units=3) + ] + + return blocks + + +class Backbone(Module): + + def __init__(self, input_size, num_layers, mode='ir'): + """ Args: + input_size: input_size of backbone + num_layers: num_layers of backbone + mode: support ir or irse + """ + super(Backbone, self).__init__() + assert input_size[0] in [112, 224], \ + 'input_size should be [112, 112] or [224, 224]' + assert num_layers in [18, 34, 50, 100, 152, 200], \ + 'num_layers should be 18, 34, 50, 100 or 152' + assert mode in ['ir', 'ir_se'], \ + 'mode should be ir or ir_se' + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + blocks = get_blocks(num_layers) + if num_layers <= 100: + if mode == 'ir': + unit_module = BasicBlockIR + elif mode == 'ir_se': + unit_module = BasicBlockIRSE + output_channel = 512 + else: + if mode == 'ir': + unit_module = BottleneckIR + elif mode == 'ir_se': + unit_module = BottleneckIRSE + output_channel = 2048 + + if input_size[0] == 112: + self.output_layer = Sequential( + BatchNorm2d(output_channel), Dropout(0.4), Flatten(), + Linear(output_channel * 7 * 7, 512), + BatchNorm1d(512, affine=False)) + else: + self.output_layer = Sequential( + BatchNorm2d(output_channel), Dropout(0.4), Flatten(), + Linear(output_channel * 14 * 14, 512), + BatchNorm1d(512, affine=False)) + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + initialize_weights(self.modules()) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return x + + +def IR_18(input_size): + """ Constructs a ir-18 model. + """ + model = Backbone(input_size, 18, 'ir') + + return model + + +def IR_34(input_size): + """ Constructs a ir-34 model. + """ + model = Backbone(input_size, 34, 'ir') + + return model + + +def IR_50(input_size): + """ Constructs a ir-50 model. + """ + model = Backbone(input_size, 50, 'ir') + + return model + + +def IR_101(input_size): + """ Constructs a ir-101 model. + """ + model = Backbone(input_size, 100, 'ir') + + return model + + +def IR_152(input_size): + """ Constructs a ir-152 model. + """ + model = Backbone(input_size, 152, 'ir') + + return model + + +def IR_200(input_size): + """ Constructs a ir-200 model. + """ + model = Backbone(input_size, 200, 'ir') + + return model + + +def IR_SE_50(input_size): + """ Constructs a ir_se-50 model. + """ + model = Backbone(input_size, 50, 'ir_se') + + return model + + +def IR_SE_101(input_size): + """ Constructs a ir_se-101 model. + """ + model = Backbone(input_size, 100, 'ir_se') + + return model + + +def IR_SE_152(input_size): + """ Constructs a ir_se-152 model. + """ + model = Backbone(input_size, 152, 'ir_se') + + return model + + +def IR_SE_200(input_size): + """ Constructs a ir_se-200 model. + """ + model = Backbone(input_size, 200, 'ir_se') + + return model diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_resnet.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..568e24ffc900a63b92d303f2c8bfe9f31f03bae6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/backbone/model_resnet.py @@ -0,0 +1,162 @@ +# The implementation is adopted from TFace,made pubicly available under the Apache-2.0 license at +# https://github.com/Tencent/TFace/blob/master/recognition/torchkit/backbone/model_resnet.py +import torch.nn as nn +from torch.nn import (BatchNorm1d, BatchNorm2d, Conv2d, Dropout, Linear, + MaxPool2d, Module, ReLU, Sequential) + +from .common import initialize_weights + + +def conv3x3(in_planes, out_planes, stride=1): + """ 3x3 convolution with padding + """ + return Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +def conv1x1(in_planes, out_planes, stride=1): + """ 1x1 convolution + """ + return Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class Bottleneck(Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = conv1x1(inplanes, planes) + self.bn1 = BatchNorm2d(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn2 = BatchNorm2d(planes) + self.conv3 = conv1x1(planes, planes * self.expansion) + self.bn3 = BatchNorm2d(planes * self.expansion) + self.relu = ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(Module): + """ ResNet backbone + """ + + def __init__(self, input_size, block, layers, zero_init_residual=True): + """ Args: + input_size: input_size of backbone + block: block function + layers: layers in each block + """ + super(ResNet, self).__init__() + assert input_size[0] in [112, 224],\ + 'input_size should be [112, 112] or [224, 224]' + self.inplanes = 64 + self.conv1 = Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = BatchNorm2d(64) + self.relu = ReLU(inplace=True) + self.maxpool = MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + + self.bn_o1 = BatchNorm2d(2048) + self.dropout = Dropout() + if input_size[0] == 112: + self.fc = Linear(2048 * 4 * 4, 512) + else: + self.fc = Linear(2048 * 7 * 7, 512) + self.bn_o2 = BatchNorm1d(512) + + initialize_weights(self.modules) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.bn_o1(x) + x = self.dropout(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + x = self.bn_o2(x) + + return x + + +def ResNet_50(input_size, **kwargs): + """ Constructs a ResNet-50 model. + """ + model = ResNet(input_size, Bottleneck, [3, 4, 6, 3], **kwargs) + + return model + + +def ResNet_101(input_size, **kwargs): + """ Constructs a ResNet-101 model. + """ + model = ResNet(input_size, Bottleneck, [3, 4, 23, 3], **kwargs) + + return model + + +def ResNet_152(input_size, **kwargs): + """ Constructs a ResNet-152 model. + """ + model = ResNet(input_size, Bottleneck, [3, 8, 36, 3], **kwargs) + + return model diff --git a/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/rts_backbone.py b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/rts_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..812f8af8abc7f3a9629c6203ee59af645c9fbd80 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_recognition/torchkit/rts_backbone.py @@ -0,0 +1,223 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import namedtuple +from math import lgamma + +import torch +import torch.nn as nn +from torch.nn import (AdaptiveAvgPool2d, BatchNorm1d, BatchNorm2d, Conv2d, + Dropout, Linear, MaxPool2d, Module, PReLU, ReLU, + Sequential, Sigmoid) +from torch.nn.modules.flatten import Flatten + +from modelscope.models import MODELS +from modelscope.models.base import TorchModel +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module('face-recognition', 'rts-backbone') +class RTSBackbone(TorchModel): + + def __init__(self, *args, **kwargs): + super(RTSBackbone, self).__init__() + # model initialization + self.alpha = kwargs.get('alpha') + self.rts_plus = kwargs.get('rts_plus') + resnet = Backbone([112, 112], 64, mode='ir_se') + + self.features = nn.Sequential( + resnet.input_layer, resnet.body, + Sequential( + BatchNorm2d(512), + Dropout(), + Flatten(), + )) + + self.features_backbone = nn.Sequential( + Linear(512 * 7 * 7, 512), + BatchNorm1d(512), + ) + + self.logvar_rts_backbone = nn.Sequential( + Linear(512 * 7 * 7, 1), + BatchNorm1d(1), + ) + + self.logvar_rts_plus_backbone = nn.Sequential( + Linear(512 * 7 * 7, self.alpha), + BatchNorm1d(self.alpha), + ) + + def forward(self, img): + x = self.features(img) + image_features = self.features_backbone(x) + if not self.rts_plus: + logvar = self.logvar_rts_backbone(x) + else: + logvar = self.logvar_rts_plus_backbone(x) + return image_features, logvar + + @classmethod + def _instantiate(cls, **kwargs): + model_file = kwargs.get('am_model_name', ModelFile.TORCH_MODEL_FILE) + ckpt_path = os.path.join(kwargs['model_dir'], model_file) + logger.info(f'loading model from {ckpt_path}') + model_dir = kwargs.pop('model_dir') + model = cls(**kwargs) + ckpt_path = os.path.join(model_dir, model_file) + model.load_state_dict(torch.load(ckpt_path, map_location='cpu')) + return model + + +def l2_norm(input, axis=1): + norm = torch.norm(input, 2, axis, True) + output = torch.div(input, norm) + + return output + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + + nn.init.xavier_uniform_(self.fc1.weight.data) + + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + + return module_input * x + + +class bottleneck_IR_SE(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR_SE, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth), SEModule(depth, 16)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + + return res + shortcut + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + '''A named tuple describing a ResNet block.''' + + +def get_block(in_channel, depth, num_units, stride=2): + + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 64: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=16), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + + return blocks + + +class Backbone(Module): + + def __init__(self, input_size, num_layers, mode='ir'): + super(Backbone, self).__init__() + assert input_size[0] in [ + 112, 224 + ], 'input_size should be [112, 112] or [224, 224]' + assert num_layers in [50, 64, 100, + 152], 'num_layers should be 50, 64, 100 or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = bottleneck_IR + elif mode == 'ir_se': + unit_module = bottleneck_IR_SE + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + if input_size[0] == 112: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(), Flatten(), + Linear(512 * 7 * 7, 512), BatchNorm1d(512)) + else: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(), Flatten(), + Linear(512 * 14 * 14, 512), BatchNorm1d(512)) + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/__init__.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/__init__.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/bfm.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/bfm.py new file mode 100644 index 0000000000000000000000000000000000000000..bd2b7c250df4a77698a9a9d710ff0698d9ff707b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/bfm.py @@ -0,0 +1,742 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch + +import os + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.io import loadmat + +from .. import utils +from ..utils import read_obj, transferBFM09 +from .renderer import SRenderY, set_rasterizer + + +def perspective_projection(focal, center): + # return p.T (N, 3) @ (3, 3) + return np.array([focal, 0, center, 0, focal, center, 0, 0, + 1]).reshape([3, 3]).astype(np.float32).transpose() + + +class SH: + + def __init__(self): + self.a = [np.pi, 2 * np.pi / np.sqrt(3.), 2 * np.pi / np.sqrt(8.)] + self.c = [ + 1 / np.sqrt(4 * np.pi), + np.sqrt(3.) / np.sqrt(4 * np.pi), + 3 * np.sqrt(5.) / np.sqrt(12 * np.pi) + ] + + +class ParametricFaceModel: + + def __init__(self, + assets_folder='assets', + recenter=True, + camera_distance=10., + init_lit=np.array([0.8, 0, 0, 0, 0, 0, 0, 0, 0]), + focal=1015., + center=112., + is_train=True, + default_name='BFM_model_front.mat'): + + if not os.path.isfile(os.path.join(assets_folder, default_name)): + transferBFM09(assets_folder) + model = loadmat(os.path.join(assets_folder, default_name)) + # mean face shape. [3*N,1] + self.mean_shape = model['meanshape'].astype(np.float32) # (1, 107127) + + # identity basis. [3*N,80] + self.id_base = model['idBase'].astype(np.float32) # (107127, 80) + + # expression basis. [3*N,64] + self.exp_base = model['exBase'].astype(np.float32) # (107127, 64) + + # mean face texture. [3*N,1] (0-255) + self.mean_tex = model['meantex'].astype(np.float32) # (1, 107127) + + # texture basis. [3*N,80] + self.tex_base = model['texBase'].astype(np.float32) # (107127, 80) + + mean_tex_uv_path = os.path.join(assets_folder, 'bfm_tex_mean2.npy') + tex_base_uv_path = os.path.join(assets_folder, 'bfm_texmap_base2.npy') + self.mean_tex_uv = np.load(mean_tex_uv_path) + self.mean_tex_uv = self.mean_tex_uv.reshape((1, -1)) + self.tex_base_uv = np.load(tex_base_uv_path) + self.tex_base_uv = self.tex_base_uv.reshape((-1, 80)) + set_rasterizer() + template_obj_path = os.path.join(assets_folder, 'template_bfm.obj') + uvcoords_path = os.path.join(assets_folder, 'bfm_uvs2.npy') + self.render = SRenderY( + 224, template_obj_path, uvcoords_path, uv_size=256) + + # face indices for each vertex that lies in. starts from 0. [N,8] + self.point_buf = model['point_buf'].astype(np.int64) - 1 # (35709, 8) + + # vertex indices for each face. starts from 0. [F,3] + self.face_buf = model['tri'].astype(np.int64) - 1 # (70789, 3) + + # vertex indices for 68 landmarks. starts from 0. [68,1] + self.keypoints = np.squeeze(model['keypoints']).astype(np.int64) - 1 + + if recenter: + mean_shape = self.mean_shape.reshape([-1, 3]) + mean_shape = mean_shape - np.mean( + mean_shape[:35709, ...], axis=0, keepdims=True) + self.mean_shape = mean_shape.reshape([-1, 1]) + + self.center = center + self.persc_proj = perspective_projection(focal, self.center) + self.camera_distance = camera_distance + self.SH = SH() + self.init_lit = init_lit.reshape([1, 1, -1]).astype(np.float32) + + def to(self, device): + self.device = device + for key, value in self.__dict__.items(): + if type(value).__module__ == np.__name__: + setattr(self, key, torch.tensor(value).to(device)) + self.render = self.render.to(device) + + def compute_shape(self, id_coeff, exp_coeff): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) + + Parameters: + id_coeff -- torch.tensor, size (B, 80), identity coeffs + exp_coeff -- torch.tensor, size (B, 64), expression coeffs + """ + batch_size = id_coeff.shape[0] + id_part = torch.einsum('ij,aj->ai', self.id_base, id_coeff) + exp_part = torch.einsum('ij,aj->ai', self.exp_base, exp_coeff) + face_shape = id_part + exp_part + self.mean_shape.reshape([1, -1]) + + return face_shape.reshape([batch_size, -1, 3]) + + def compute_albedo(self, tex_coeff, normalize=True): + """ + Return: + face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.) + + Parameters: + tex_coeff -- torch.tensor, size (B, 80) + """ + batch_size = tex_coeff.shape[0] + face_texture = torch.einsum('ij,aj->ai', self.tex_base, + tex_coeff) + self.mean_tex + if normalize: + face_texture = face_texture / 255. + return face_texture.reshape([batch_size, -1, 3]) + + def compute_albedo_map(self, tex_coeff, normalize=True): + """ + Return: + face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.) + + Parameters: + tex_coeff -- torch.tensor, size (B, 80) + """ + batch_size = tex_coeff.shape[0] + face_texture = torch.einsum('ij,aj->ai', self.tex_base_uv, + tex_coeff) + self.mean_tex_uv + if normalize: + face_texture = face_texture / 255. + return face_texture.reshape([batch_size, 512, 512, 3]) + + def compute_norm(self, face_shape): + """ + Return: + vertex_norm -- torch.tensor, size (B, N, 3) + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + v1 = face_shape[:, self.face_buf[:, 0]] + v2 = face_shape[:, self.face_buf[:, 1]] + v3 = face_shape[:, self.face_buf[:, 2]] + e1 = v1 - v2 + e2 = v2 - v3 + face_norm = torch.cross(e1, e2, dim=-1) + face_norm = F.normalize(face_norm, dim=-1, p=2) + face_norm = torch.cat( + [face_norm, + torch.zeros(face_norm.shape[0], 1, 3).to(self.device)], + dim=1) + + vertex_norm = torch.sum(face_norm[:, self.point_buf], dim=2) + vertex_norm = F.normalize(vertex_norm, dim=-1, p=2) + return vertex_norm + + def compute_color(self, face_texture, face_norm, gamma): + """ + Return: + face_color -- torch.tensor, size (B, N, 3), range (0, 1.) + + Parameters: + face_texture -- torch.tensor, size (B, N, 3), from texture model, range (0, 1.) + face_norm -- torch.tensor, size (B, N, 3), rotated face normal + gamma -- torch.tensor, size (B, 27), SH coeffs + """ + batch_size = gamma.shape[0] + a, c = self.SH.a, self.SH.c + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + self.init_lit + gamma = gamma.permute(0, 2, 1) + + y1 = a[0] * c[0] * torch.ones_like(face_norm[..., :1]).to(self.device) + y2 = -a[1] * c[1] * face_norm[..., 1:2] + y3 = a[1] * c[1] * face_norm[..., 2:] + y4 = -a[1] * c[1] * face_norm[..., :1] + y5 = a[2] * c[2] * face_norm[..., :1] * face_norm[..., 1:2] + y6 = -a[2] * c[2] * face_norm[..., 1:2] * face_norm[..., 2:] + y7 = 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm[..., 2:]**2 - 1) + y8 = -a[2] * c[2] * face_norm[..., :1] * face_norm[..., 2:] + y9 = 0.5 * a[2] * c[2] * ( + face_norm[..., :1]**2 - face_norm[..., 1:2]**2) + Y = torch.cat([y1, y2, y3, y4, y5, y6, y7, y8, y9], dim=-1) + r = Y @ gamma[..., :1] + g = Y @ gamma[..., 1:2] + b = Y @ gamma[..., 2:] + face_color = torch.cat([r, g, b], dim=-1) * face_texture + return face_color + + def compute_color_map(self, face_texture_uv, face_norm_uv, gamma): + """ + Return: + face_color -- torch.tensor, size (B, N, 3), range (0, 1.) + + Parameters: + face_texture_uv -- torch.tensor, (B, 3, 256, 256) + face_norm_uv -- torch.tensor, (B, 3, 256, 256) + gamma -- torch.tensor, size (B, 27), SH coeffs + """ + face_texture_uv = face_texture_uv.permute( + 0, 2, 3, 1).contiguous() # (B, 256, 256, 3) + face_norm_uv = face_norm_uv.permute(0, 2, 3, + 1).contiguous() # (B, 256, 256, 3) + size = face_texture_uv.shape[1] + + batch_size = gamma.shape[0] + a, c = self.SH.a, self.SH.c + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + self.init_lit + gamma = gamma.permute(0, 2, 1) # (B, 9, 3) + y1 = a[0] * c[0] * torch.ones_like(face_norm_uv[..., :1]).to( + self.device) + y2 = -a[1] * c[1] * face_norm_uv[..., 1:2] + y3 = a[1] * c[1] * face_norm_uv[..., 2:] + y4 = -a[1] * c[1] * face_norm_uv[..., :1] + y5 = a[2] * c[2] * face_norm_uv[..., :1] * face_norm_uv[..., 1:2] + y6 = -a[2] * c[2] * face_norm_uv[..., 1:2] * face_norm_uv[..., 2:] + y7 = 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm_uv[..., 2:]**2 + - 1) + y8 = -a[2] * c[2] * face_norm_uv[..., :1] * face_norm_uv[..., 2:] + y9 = 0.5 * a[2] * c[2] * ( + face_norm_uv[..., :1]**2 - face_norm_uv[..., 1:2]**2) + Y = torch.cat([y1, y2, y3, y4, y5, y6, y7, y8, y9], + dim=-1) # (B, 256, 256, 9) + Y = Y.reshape(batch_size, -1, 9) + r = Y @ gamma[..., :1] + g = Y @ gamma[..., 1:2] + b = Y @ gamma[..., 2:] + face_color = torch.cat([r, g, b], dim=-1).reshape( + batch_size, size, size, 3) * face_texture_uv # (B, 256, 256, 3) + face_color = face_color.permute(0, 3, 1, + 2).contiguous() # (B, 3, 256, 256) + return face_color + + def displacement2normal(self, uv_z, coarse_verts, coarse_normals): + ''' Convert displacement map into detail normal map + ''' + batch_size = uv_z.shape[0] + uv_coarse_vertices = self.render.world2uv(coarse_verts) + uv_coarse_normals = self.render.world2uv(coarse_normals) + + uv_detail_vertices = uv_coarse_vertices + dense_vertices = uv_detail_vertices.permute(0, 2, 3, 1).reshape( + [batch_size, -1, 3]) + uv_detail_normals = utils.vertex_normals( + dense_vertices, self.render.dense_faces.expand(batch_size, -1, -1)) + uv_detail_normals = uv_detail_normals.reshape([ + batch_size, uv_coarse_vertices.shape[2], + uv_coarse_vertices.shape[3], 3 + ]).permute(0, 3, 1, 2) + uv_detail_normals[:, :2, ...] = -uv_detail_normals[:, :2, ...] + offset = uv_coarse_normals - uv_detail_normals + + uv_detail_vertices = uv_coarse_vertices + uv_z * uv_coarse_normals + dense_vertices = uv_detail_vertices.permute(0, 2, 3, 1).reshape( + [batch_size, -1, 3]) + uv_detail_normals = utils.vertex_normals( + dense_vertices, self.render.dense_faces.expand(batch_size, -1, -1)) + uv_detail_normals = uv_detail_normals.reshape([ + batch_size, uv_coarse_vertices.shape[2], + uv_coarse_vertices.shape[3], 3 + ]).permute(0, 3, 1, 2) + uv_detail_normals[:, :2, ...] = -uv_detail_normals[:, :2, ...] + uv_detail_normals = uv_detail_normals + offset + return uv_detail_normals + + def compute_color_with_displacement(self, face_texture_uv, verts, normals, + displacement_uv, gamma): + """ + Return: + face_color -- torch.tensor, size (B, N, 3), range (0, 1.) + + Parameters: + face_texture -- torch.tensor, size (B, N, 3), from texture model, range (0, 1.) + face_norm -- torch.tensor, size (B, N, 3), rotated face normal + gamma -- torch.tensor, size (B, 27), SH coeffs + """ + uv_detail_normals = self.displacement2normal( + displacement_uv, verts, normals + ) # verts: (B, n, 3), ops['normals']: (B, n, 3), uv_detail_normals: (B, 3, 256, 256) + uv_texture = self.compute_color_map(face_texture_uv, uv_detail_normals, + gamma) + return uv_texture + + def compute_rotation(self, angles): + """ + Return: + rot -- torch.tensor, size (B, 3, 3) pts @ trans_mat + + Parameters: + angles -- torch.tensor, size (B, 3), radian + """ + + batch_size = angles.shape[0] + ones = torch.ones([batch_size, 1]).to(self.device) + zeros = torch.zeros([batch_size, 1]).to(self.device) + x, y, z = angles[:, :1], angles[:, 1:2], angles[:, 2:], + + value_list = [ + ones, zeros, zeros, zeros, + torch.cos(x), -torch.sin(x), zeros, + torch.sin(x), + torch.cos(x) + ] + rot_x = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + value_list = [ + torch.cos(y), zeros, + torch.sin(y), zeros, ones, zeros, -torch.sin(y), zeros, + torch.cos(y) + ] + rot_y = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + value_list = [ + torch.cos(z), -torch.sin(z), zeros, + torch.sin(z), + torch.cos(z), zeros, zeros, zeros, ones + ] + rot_z = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + rot = rot_z @ rot_y @ rot_x + return rot.permute(0, 2, 1) + + def to_camera(self, face_shape): + face_shape[..., -1] = self.camera_distance - face_shape[..., -1] + return face_shape + + def to_image(self, face_shape): + """ + Return: + face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + # to image_plane + face_proj = face_shape @ self.persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + def transform(self, face_shape, rot, trans): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) pts @ rot + trans + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + rot -- torch.tensor, size (B, 3, 3) + trans -- torch.tensor, size (B, 3) + """ + return face_shape @ rot + trans.unsqueeze(1) + + def get_landmarks(self, face_proj): + """ + Return: + face_lms -- torch.tensor, size (B, 68, 2) + + Parameters: + face_proj -- torch.tensor, size (B, N, 2) + """ + return face_proj[:, self.keypoints] + + def split_coeff(self, coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 257) + """ + if type(coeffs) == dict and 'id' in coeffs: + return coeffs + + id_coeffs = coeffs[:, :80] + exp_coeffs = coeffs[:, 80:144] + tex_coeffs = coeffs[:, 144:224] + angles = coeffs[:, 224:227] + gammas = coeffs[:, 227:254] + translations = coeffs[:, 254:] + return { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + + def merge_coeff(self, coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 256) + """ + names = ['id', 'exp', 'tex', 'angle', 'gamma', 'trans'] + coeffs_merge = [] + for name in names: + coeffs_merge.append(coeffs[name].detach()) + coeffs_merge = torch.cat(coeffs_merge, dim=1) + + return coeffs_merge + + def get_texture_map(self, face_vertex, input_img): + batch_size = input_img.shape[0] + h, w = input_img.shape[2:] + face_vertex_uv = self.render.world2uv(face_vertex) # (B, 3, 256, 256) + face_vertex_uv = face_vertex_uv.reshape(batch_size, 3, -1).permute( + 0, 2, 1) # (B, N, 3), N: 256*256 + face_vertex_uv_proj = self.to_image( + face_vertex_uv) # (B, N, 2) , project to image (size=224) + face_vertex_uv_proj[..., 0] *= w / 224 + face_vertex_uv_proj[..., 1] *= h / 224 + face_vertex_uv_proj[torch.isnan(face_vertex_uv_proj)] = 0 + + face_vertex_uv_proj[..., -1] = h - 1 - face_vertex_uv_proj[..., -1] + + input_img = input_img.permute(0, 2, 3, 1) # (B, h, w, 3) + + face_vertex_uv_proj_int = torch.floor(face_vertex_uv_proj) + face_vertex_uv_proj_float = face_vertex_uv_proj - face_vertex_uv_proj_int # (B, N, 2) + face_vertex_uv_proj_float = face_vertex_uv_proj_float.reshape( + -1, 2) # (B * N, 2) + face_vertex_uv_proj_int = face_vertex_uv_proj_int.long() # (B, N, 2) + + batch_indices = torch.arange(0, batch_size)[:, None, None].repeat( + 1, face_vertex_uv_proj_int.shape[1], + 1).long().to(face_vertex_uv_proj_int.device) # (B, N, 1) + indices = torch.cat([face_vertex_uv_proj_int, batch_indices], dim=2) + indices = indices.reshape(-1, 3) # (B * N, 3) + + face_vertex_uv_proj_lt = input_img[indices[:, 2], indices[:, 1].clamp( + 0, h - 1), indices[:, 0].clamp(0, w - 1)] # (B * N, 3) + face_vertex_uv_proj_lb = input_img[indices[:, 2], + (indices[:, 1] + 1).clamp(0, h - 1), + indices[:, 0].clamp(0, w - 1)] + face_vertex_uv_proj_rt = input_img[indices[:, 2], + indices[:, 1].clamp(0, h - 1), + (indices[:, 0] + 1).clamp(0, w - 1)] + face_vertex_uv_proj_rb = input_img[indices[:, 2], + (indices[:, 1] + 1).clamp(0, h - 1), + (indices[:, 0] + 1).clamp(0, w - 1)] + + value_1 = face_vertex_uv_proj_lt * ( + 1 - face_vertex_uv_proj_float[:, :1] + ) * face_vertex_uv_proj_float[:, 1:] + value_2 = face_vertex_uv_proj_lb * ( + 1 - face_vertex_uv_proj_float[:, :1]) * ( + 1 - face_vertex_uv_proj_float[:, 1:]) + value_3 = face_vertex_uv_proj_rt * face_vertex_uv_proj_float[:, : + 1] * face_vertex_uv_proj_float[:, + 1:] + value_4 = face_vertex_uv_proj_rb * face_vertex_uv_proj_float[:, :1] * ( + 1 - face_vertex_uv_proj_float[:, 1:]) + + texture_map = value_1 + value_2 + value_3 + value_4 # (B * N, 3) + + texture_map = texture_map.reshape(batch_size, self.render.uv_size, + self.render.uv_size, + -1) # (B, 256, 256, 3) + + return texture_map + + def compute_for_render(self, coeffs): + if type(coeffs) == dict: + coef_dict = coeffs + elif type(coeffs) == torch.Tensor: + coef_dict = self.split_coeff(coeffs) + + face_shape = self.compute_shape(coef_dict['id'], coef_dict['exp']) + + rotation = self.compute_rotation(coef_dict['angle']) + + face_shape_transformed = self.transform(face_shape, rotation, + coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed.clone()) + face_vertex_noTrans = self.to_camera(face_shape.clone()) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + + face_albedo_map = self.compute_albedo_map( + coef_dict['tex']) # (1, 512, 512, 3) + face_albedo_map = face_albedo_map.permute(0, 3, 1, 2) + face_albedo_map = torch.nn.functional.interpolate( + face_albedo_map, [self.render.uv_size, self.render.uv_size], + mode='bilinear') + face_norm_roted_uv = self.render.world2uv(face_norm_roted) + face_color_map = self.compute_color_map(face_albedo_map, + face_norm_roted_uv, + coef_dict['gamma']) + + position_map = self.render.world2uv(face_shape) + + return face_vertex, face_albedo_map, face_color_map, landmark, face_vertex_noTrans, position_map + + def recolor_texture(self, shaded_texture): + rgb_mean = torch.mean(shaded_texture, dim=(2, 3), keepdim=True) + target_mean = torch.ones_like(rgb_mean) * 0.6 + shaded_texture = shaded_texture * target_mean / rgb_mean + return shaded_texture + + def compute_for_render_hierarchical_mid(self, + coeffs, + deformation_map, + UVs, + visualize=False, + de_retouched_albedo_map=None): + if type(coeffs) == dict: + coef_dict = coeffs + elif type(coeffs) == torch.Tensor: + coef_dict = self.split_coeff(coeffs) + + face_shape = self.compute_shape(coef_dict['id'], + coef_dict['exp']) # (B, n, 3) + face_shape_base = face_shape.clone() + + face_shape, shape_offset = self.add_nonlinear_offset( + face_shape, deformation_map, UVs) # (B, n, 3) + + rotation = self.compute_rotation(coef_dict['angle']) + + face_shape_transformed = self.transform(face_shape, rotation, + coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed.clone()) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + + face_albedo_map = self.compute_albedo_map( + coef_dict['tex']) # (B, 512, 512, 3) + face_albedo_map = face_albedo_map.permute(0, 3, 1, 2) + face_albedo_map = torch.nn.functional.interpolate( + face_albedo_map, [self.render.uv_size, self.render.uv_size], + mode='bilinear') + face_norm_roted_uv = self.render.world2uv(face_norm_roted) + + albedo_for_render = face_albedo_map if de_retouched_albedo_map is None else de_retouched_albedo_map + face_color_map = self.compute_color_map(albedo_for_render, + face_norm_roted_uv, + coef_dict['gamma']) + + extra_results = None + if visualize: + extra_results = {} + extra_results['tex_mid_color'] = face_color_map + + face_shape_transformed_base = self.transform( + face_shape_base, rotation, coef_dict['trans']) + face_vertex_base = self.to_camera( + face_shape_transformed_base.clone()) + extra_results['pred_vertex_base'] = face_vertex_base + face_norm_base = self.compute_norm(face_shape_base) + face_norm_roted_base = face_norm_base @ rotation + + batch_size = albedo_for_render.shape[0] + size = albedo_for_render.shape[2] + gray_tex = torch.ones((batch_size, 3, size, size), + dtype=torch.float32).to(self.device) * 0.8 + zero_displacement = torch.zeros( + (batch_size, 1, size, size), + dtype=torch.float32).to(self.device) + + tex_mid_gray = self.compute_color_with_displacement( + gray_tex.detach(), face_shape_transformed, face_norm_roted, + zero_displacement, coef_dict['gamma']) + tex_mid_gray = self.recolor_texture(tex_mid_gray) + extra_results['tex_mid_gray'] = tex_mid_gray + + tex_base_color = self.compute_color_with_displacement( + albedo_for_render, face_shape_transformed_base, + face_norm_roted_base, zero_displacement, coef_dict['gamma']) + extra_results['tex_base_color'] = tex_base_color + + tex_base_gray = self.compute_color_with_displacement( + gray_tex.detach(), face_shape_transformed_base, + face_norm_roted_base, zero_displacement, coef_dict['gamma']) + tex_base_gray = self.recolor_texture(tex_base_gray) + extra_results['tex_base_gray'] = tex_base_gray + + # to export rotate video + init_angle = torch.zeros_like(coef_dict['angle']).to( + coef_dict['angle'].device) + + pi = 3.14 + n_frame = 30 + y_angles = torch.linspace(-pi / 6, pi / 6, steps=n_frame).float() + + extra_results['face_shape_transformed_list'] = [] + extra_results['face_norm_roted_list'] = [] + extra_results['face_vertex_list'] = [] + for y_angle in y_angles: + cur_angle = init_angle.clone() + cur_angle[:, 1] = y_angle + cur_angle[:, 0] = pi / 36 + cur_rotation = self.compute_rotation(cur_angle) + cur_face_shape_transformed = self.transform( + face_shape, cur_rotation, coef_dict['trans'] * 0) + cur_face_norm_roted = face_norm @ cur_rotation + cur_face_vertex = self.to_camera( + cur_face_shape_transformed.clone()) + + extra_results['face_shape_transformed_list'].append( + cur_face_shape_transformed) + extra_results['face_norm_roted_list'].append( + cur_face_norm_roted) + extra_results['face_vertex_list'].append(cur_face_vertex) + + return (face_vertex, face_color_map, landmark, face_proj, + face_albedo_map, face_shape_transformed, face_norm_roted, + extra_results) + + def compute_for_render_hierarchical_high(self, + coeffs, + displacement_uv, + face_albedo_map, + face_shape_transformed, + face_norm_roted, + extra_results=None): + if type(coeffs) == dict: + coef_dict = coeffs + elif type(coeffs) == torch.Tensor: + coef_dict = self.split_coeff(coeffs) + + face_color_map = self.compute_color_with_displacement( + face_albedo_map, face_shape_transformed, face_norm_roted, + displacement_uv, coef_dict['gamma']) + + if extra_results is not None: + extra_results['tex_high_color'] = face_color_map + + batch_size = face_albedo_map.shape[0] + size = face_albedo_map.shape[2] + gray_tex = torch.ones((batch_size, 3, size, size), + dtype=torch.float32).to(self.device) * 0.8 + + tex_high_gray = self.compute_color_with_displacement( + gray_tex.detach(), face_shape_transformed, face_norm_roted, + displacement_uv, coef_dict['gamma']) + tex_high_gray = self.recolor_texture(tex_high_gray) + extra_results['tex_high_gray'] = tex_high_gray + + if 'face_shape_transformed_list' in extra_results: + extra_results['tex_high_gray_list'] = [] + extra_results['tex_high_color_list'] = [] + for i in range( + len(extra_results['face_shape_transformed_list'])): + tex_high_gray_i = self.compute_color_with_displacement( + gray_tex.detach(), + extra_results['face_shape_transformed_list'][i], + extra_results['face_norm_roted_list'][i], + displacement_uv, coef_dict['gamma']) + extra_results['tex_high_gray_list'].append(tex_high_gray_i) + + tex_high_color_i = self.compute_color_with_displacement( + face_albedo_map.detach(), + extra_results['face_shape_transformed_list'][i], + extra_results['face_norm_roted_list'][i], + displacement_uv, coef_dict['gamma']) + extra_results['tex_high_color_list'].append( + tex_high_color_i) + + return face_color_map, extra_results + + def reverse_recenter(self, face_shape): + batch_size = face_shape.shape[0] + face_shape = face_shape.reshape([-1, 3]) + mean_shape_ori = self.mean_shape_ori.reshape([-1, 3]) + face_shape = face_shape + torch.mean( + mean_shape_ori[:35709, ...], dim=0, keepdim=True) + face_shape = face_shape.reshape([batch_size, -1, 3]) + return face_shape + + def add_nonlinear_offset(self, face_shape, shape_offset_uv, UVs): + """ + + Args: + face_shape: torch.tensor, size (B, N, 3) + shape_offset_uv: torch.tensor, size (B, h, w, 3) + UVs: torch.tensor, size (N, 2) + + Returns: + + """ + h, w = shape_offset_uv.shape[1:3] + UVs_coords = UVs.clone() + UVs_coords[:, 0] *= w + UVs_coords[:, 1] *= h + UVs_coords_int = torch.floor(UVs_coords) + UVs_coords_float = UVs_coords - UVs_coords_int + UVs_coords_int = UVs_coords_int.long() + + shape_lt = shape_offset_uv[:, (h - 1 - UVs_coords_int[:, 1]).clamp( + 0, h - 1), UVs_coords_int[:, 0].clamp(0, w - 1)] # (B, N, 3) + shape_lb = shape_offset_uv[:, + (h - UVs_coords_int[:, 1]).clamp(0, h - 1), + UVs_coords_int[:, 0].clamp(0, w - 1)] + shape_rt = shape_offset_uv[:, (h - 1 + - UVs_coords_int[:, 1]).clamp(0, h - 1), + (UVs_coords_int[:, 0] + 1).clamp(0, w - 1)] + shape_rb = shape_offset_uv[:, + (h - UVs_coords_int[:, 1]).clamp(0, h - 1), + (UVs_coords_int[:, 0] + 1).clamp(0, w - 1)] + + value_1 = shape_lt * ( + 1 - UVs_coords_float[:, :1]) * UVs_coords_float[:, 1:] + value_2 = shape_lb * (1 - UVs_coords_float[:, :1]) * ( + 1 - UVs_coords_float[:, 1:]) + value_3 = shape_rt * UVs_coords_float[:, :1] * UVs_coords_float[:, 1:] + value_4 = shape_rb * UVs_coords_float[:, :1] * ( + 1 - UVs_coords_float[:, 1:]) + + offset_shape = value_1 + value_2 + value_3 + value_4 # (B, N, 3) + + face_shape = face_shape + offset_shape + + return face_shape, offset_shape diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/de_retouching_module.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/de_retouching_module.py new file mode 100644 index 0000000000000000000000000000000000000000..92711f6eafa035425eaf2fe7ff79f9c40503a112 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/de_retouching_module.py @@ -0,0 +1,47 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch + +from .unet import UNet + + +class DeRetouchingModule(): + + def __init__(self, model_path): + self.retouching_network = UNet(3, 3).to('cuda') + self.retouching_network.load_state_dict( + torch.load(model_path, map_location='cpu')['generator']) + self.retouching_network.eval() + + def run(self, face_albedo_map, texture_map): + """ + + :param face_albedo_map: tensor, (1, 3, 256, 256), 0~1, rgb + :param texture_map: tensor, (1, 3, 256, 256), -1~1, rgb + :return: + """ + h, w = texture_map.shape[2:] + retouch_input = torch.nn.functional.interpolate( + texture_map, (512, 512), mode='bilinear') + + # predict blend layer + blend_layer = self.retouching_network(retouch_input) # value: 0~1 + blend_layer = torch.nn.functional.interpolate( + blend_layer, (h, w), mode='bilinear') + + # retouch texture map + tex = (texture_map + 1.0) / 2 + retouched_tex = (1 - 2 * blend_layer + ) * tex * tex + 2 * blend_layer * tex # value: 0~1 + + # de-retouch albedo map + A_0 = face_albedo_map + T_0 = retouched_tex + T_ = tex + + phi = 1e6 * T_0 * T_0 * T_0 + 1e-6 + B = (T_ + 1 / phi) / (T_0 + 1 / phi) + A_ = A_0 * B + + de_retouched_albedo = A_ + + return de_retouched_albedo diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/__init__.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/large_base_lmks_infer.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/large_base_lmks_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..b9e329ee31c3beb04cc6f8a2fcc395cd6d5f4a50 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/large_base_lmks_infer.py @@ -0,0 +1,91 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch + +from .nets.large_base_lmks_net import LargeBaseLmksNet + +BASE_LANDMARK_NUM = 106 +INPUT_SIZE = 224 +ENLARGE_RATIO = 1.35 + + +class LargeBaseLmkInfer: + + @staticmethod + def model_preload(model_path, use_gpu=True): + model = LargeBaseLmksNet(infer=False) + # using gpu + if use_gpu: + model = model.cuda() + + checkpoint = [] + if use_gpu: + checkpoint = torch.load(model_path, map_location='cuda') + else: + checkpoint = torch.load(model_path, map_location='cpu') + + model.load_state_dict( + { + k.replace('module.', ''): v + for k, v in checkpoint['state_dict'].items() + }, + strict=False) + model.eval() + return model + + @staticmethod + def process_img(model, image, use_gpu=True): + img_resize = image + + img_resize = (img_resize + - [103.94, 116.78, 123.68]) / 255.0 # important + img_resize = img_resize.transpose([2, 0, 1]) + + if use_gpu: + img_resize = torch.from_numpy(img_resize).cuda() + else: + img_resize = torch.from_numpy(img_resize) + + w_new = INPUT_SIZE + h_new = INPUT_SIZE + img_in = torch.zeros([1, 3, h_new, w_new], dtype=torch.float32) + if use_gpu: + img_in = img_in.cuda() + + img_in[0, :] = img_resize + + with torch.no_grad(): + output = model(img_in) + output = output * INPUT_SIZE + + if use_gpu: + output = output.cpu().numpy() + else: + output = output.numpy() + + return output + + @staticmethod + def smooth(cur_lmks, prev_lmks): + smooth_lmks = np.zeros((106, 2)) + + cur_rect_x1 = np.min(cur_lmks[:, 0]) + cur_rect_x2 = np.max(cur_lmks[:, 0]) + + smooth_param = 60.0 + factor = smooth_param / (cur_rect_x1 - cur_rect_x2) + for i in range(BASE_LANDMARK_NUM): + weightX = np.exp(factor * np.abs(cur_lmks[i][0] - prev_lmks[i][0])) + weightY = np.exp(factor * np.abs(cur_lmks[i][1] - prev_lmks[i][1])) + + smooth_lmks[i][0] = ( + 1 - weightX) * cur_lmks[i][0] + weightX * prev_lmks[i][0] + smooth_lmks[i][1] = ( + 1 - weightY) * cur_lmks[i][1] + weightY * prev_lmks[i][1] + + return smooth_lmks + + @staticmethod + def infer_img(img, model, use_gpu=True): + lmks = LargeBaseLmkInfer.process_img(model, img, use_gpu) + return lmks diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/__init__.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_base_lmks_net.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_base_lmks_net.py new file mode 100644 index 0000000000000000000000000000000000000000..f81fea1bde269aade48fa804d3a05555f3b10ad5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_base_lmks_net.py @@ -0,0 +1,201 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +from torch.nn import functional as F + +INPUT_SIZE = 224 + + +def constant_init(module, val, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.constant_(module.weight, val) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def kaiming_init(module, + a=0, + mode='fan_out', + nonlinearity='relu', + bias=0, + distribution='normal'): + assert distribution in ['uniform', 'normal'] + if distribution == 'uniform': + nn.init.kaiming_uniform_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + else: + nn.init.kaiming_normal_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def conv_bn(inp, oup, kernel, stride, padding=1): + return nn.Sequential( + nn.Conv2d(inp, oup, kernel, stride, padding, bias=False), + nn.BatchNorm2d(oup), nn.PReLU(oup)) + + +def conv_1x1_bn(inp, oup): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), nn.BatchNorm2d(oup), + nn.ReLU(inplace=True)) + + +class InvertedResidual(nn.Module): + + def __init__(self, + inp, + oup, + stride, + padding, + use_res_connect, + expand_ratio=6): + + super(InvertedResidual, self).__init__() + + self.stride = stride + assert stride in [1, 2] + + self.use_res_connect = use_res_connect + hid_channels = inp * expand_ratio + self.conv = nn.Sequential( + nn.Conv2d(inp, hid_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(hid_channels), + nn.PReLU(hid_channels), + nn.Conv2d( + hid_channels, + hid_channels, + 3, + stride, + padding, + groups=hid_channels, + bias=False), + nn.BatchNorm2d(hid_channels), + nn.PReLU(hid_channels), + nn.Conv2d(hid_channels, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class SoftArgmax(nn.Module): + + def __init__(self, beta: int = 1, infer=False): + if not 0.0 <= beta: + raise ValueError(f'Invalid beta: {beta}') + super().__init__() + self.beta = beta + self.infer = infer + + def forward(self, heatmap: torch.Tensor) -> torch.Tensor: + heatmap = heatmap.mul(self.beta) + batch_size, num_channel, height, width = heatmap.size() + device: str = heatmap.device + + if not self.infer: + softmax: torch.Tensor = F.softmax( + heatmap.view(batch_size, num_channel, height * width), + dim=2).view(batch_size, num_channel, height, width) + + xx, yy = torch.meshgrid(list(map(torch.arange, [width, height]))) + + approx_x = ( + softmax.mul(xx.float().to(device)).view( + batch_size, num_channel, + height * width).sum(2).unsqueeze(2)) + approx_y = ( + softmax.mul(yy.float().to(device)).view( + batch_size, num_channel, + height * width).sum(2).unsqueeze(2)) + + output = [approx_x / width, approx_y / height] + output = torch.cat(output, 2) + output = output.view(-1, output.size(1) * output.size(2)) + return output + else: + softmax: torch.Tensor = F.softmax( + heatmap.view(batch_size, num_channel, height * width), dim=2) + + return softmax + + +class LargeBaseLmksNet(nn.Module): + + def __init__(self, er=1.0, infer=False): + + super(LargeBaseLmksNet, self).__init__() + + self.infer = infer + + self.block1 = conv_bn(3, int(64 * er), 3, 2, 1) + self.block2 = InvertedResidual( + int(64 * er), int(64 * er), 1, 1, False, 2) + + self.block3 = InvertedResidual( + int(64 * er), int(64 * er), 2, 1, False, 2) + self.block4 = InvertedResidual( + int(64 * er), int(64 * er), 1, 1, True, 2) + self.block5 = InvertedResidual( + int(64 * er), int(64 * er), 1, 1, True, 2) + self.block6 = InvertedResidual( + int(64 * er), int(64 * er), 1, 1, True, 2) + self.block7 = InvertedResidual( + int(64 * er), int(64 * er), 1, 1, True, 2) + + self.block8 = InvertedResidual( + int(64 * er), int(128 * er), 2, 1, False, 2) + + self.block9 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, False, 4) + self.block10 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, True, 4) + self.block11 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, True, 4) + self.block12 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, True, 4) + self.block13 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, True, 4) + self.block14 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, True, 4) + + self.block15 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, False, 2) # [128, 14, 14] + self.block16 = InvertedResidual( + int(128 * er), int(128 * er), 2, 1, False, 2) + self.block17 = InvertedResidual( + int(128 * er), int(128 * er), 1, 1, False, 2) + + self.block18 = conv_bn(int(128 * er), int(256 * er), 3, 1, 1) + self.block19 = nn.Conv2d(int(256 * er), 106, 3, 1, 1, bias=False) + self.softargmax = SoftArgmax(infer=infer) + + def forward(self, x): # x: 3, 224, 224 + + x = self.block1(x) + x = self.block2(x) + x = self.block3(x) + x = self.block4(x) + x = self.block5(x) + x = self.block6(x) + x = self.block7(x) + x = self.block8(x) + x = self.block9(x) + x = self.block10(x) + x = self.block11(x) + x = self.block12(x) + x = self.block13(x) + x = self.block14(x) + x = self.block15(x) + x = self.block16(x) + x = self.block17(x) + x = self.block18(x) + x = self.block19(x) + x = self.softargmax(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_eyeball_net.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_eyeball_net.py new file mode 100644 index 0000000000000000000000000000000000000000..97d36b2c1f297078839c609816ec3b947e18585e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facelandmark/nets/large_eyeball_net.py @@ -0,0 +1,160 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch.nn as nn + +FACE_PART_SIZE = 56 + + +class InvertedResidual(nn.Module): + + def __init__(self, + inp, + oup, + kernel_size, + stride, + padding, + expand_ratio=2, + use_connect=False, + activation='relu'): + super(InvertedResidual, self).__init__() + + hid_channels = int(inp * expand_ratio) + if activation == 'relu': + self.conv = nn.Sequential( + nn.Conv2d(inp, hid_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(hid_channels), nn.ReLU(inplace=True), + nn.Conv2d( + hid_channels, + hid_channels, + kernel_size, + stride, + padding, + groups=hid_channels, + bias=False), nn.BatchNorm2d(hid_channels), + nn.ReLU(inplace=True), + nn.Conv2d(hid_channels, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup)) + elif activation == 'prelu': + self.conv = nn.Sequential( + nn.Conv2d(inp, hid_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(hid_channels), nn.PReLU(hid_channels), + nn.Conv2d( + hid_channels, + hid_channels, + kernel_size, + stride, + padding, + groups=hid_channels, + bias=False), nn.BatchNorm2d(hid_channels), + nn.PReLU(hid_channels), + nn.Conv2d(hid_channels, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup)) + self.use_connect = use_connect + + def forward(self, x): + if self.use_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class Residual(nn.Module): + + def __init__(self, + inp, + oup, + kernel_size, + stride, + padding, + use_connect=False, + activation='relu'): + super(Residual, self).__init__() + + self.use_connect = use_connect + + if activation == 'relu': + self.conv = nn.Sequential( + nn.Conv2d( + inp, + inp, + kernel_size, + stride, + padding, + groups=inp, + bias=False), nn.BatchNorm2d(inp), nn.ReLU(inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), nn.BatchNorm2d(oup), + nn.ReLU(inplace=True)) + elif activation == 'prelu': + self.conv = nn.Sequential( + nn.Conv2d( + inp, + inp, + kernel_size, + stride, + padding, + groups=inp, + bias=False), nn.BatchNorm2d(inp), nn.PReLU(inp), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), nn.BatchNorm2d(oup), + nn.PReLU(oup)) + + def forward(self, x): + if self.use_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +def conv_bn(inp, oup, kernel, stride, padding=1): + return nn.Sequential( + nn.Conv2d(inp, oup, kernel, stride, padding, bias=False), + nn.BatchNorm2d(oup), nn.PReLU(oup)) + + +def conv_no_relu(inp, oup, kernel, stride, padding=1): + return nn.Sequential( + nn.Conv2d(inp, oup, kernel, stride, padding, bias=False), + nn.BatchNorm2d(oup)) + + +class View(nn.Module): + + def __init__(self, shape): + super(View, self).__init__() + self.shape = shape + + def forward(self, x): + return x.view(*self.shape) + + +class Softmax(nn.Module): + + def __init__(self, dim): + super(Softmax, self).__init__() + self.softmax = nn.Softmax(dim) + + def forward(self, x): + return self.softmax(x) + + +class LargeEyeballNet(nn.Module): + + def __init__(self): + super(LargeEyeballNet, self).__init__() + + # v6/v7/v9 + # iris : -1*2, 3, FACE_PART_SIZE, FACE_PART_SIZE + self.net = nn.Sequential( + conv_bn(3, 16, 3, 2, 0), + InvertedResidual(16, 16, 3, 1, 1, 2, True, activation='prelu'), + InvertedResidual(16, 32, 3, 2, 0, 2, False, activation='prelu'), + InvertedResidual(32, 32, 3, 1, 1, 2, True, activation='prelu'), + InvertedResidual(32, 64, 3, 2, 1, 2, False, activation='prelu'), + InvertedResidual(64, 64, 3, 1, 1, 2, True, activation='prelu'), + InvertedResidual(64, 64, 3, 2, 0, 2, False, activation='prelu'), + InvertedResidual(64, 64, 3, 1, 1, 2, True, activation='prelu'), + View((-1, 64 * 3 * 3, 1, 1)), conv_bn(64 * 3 * 3, 64, 1, 1, 0), + conv_no_relu(64, 40, 1, 1, 0), View((-1, 40))) + + def forward(self, x): # x: -1, 3, FACE_PART_SIZE, FACE_PART_SIZE + iris = self.net(x) + + return iris diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facerecon_model.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facerecon_model.py new file mode 100644 index 0000000000000000000000000000000000000000..d753b4163ecaba412eb95059b8849aa0cdd31077 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/facerecon_model.py @@ -0,0 +1,690 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import OrderedDict + +import cv2 +import numpy as np +import torch + +from modelscope.models import MODELS, TorchModel +from modelscope.models.cv.face_reconstruction.models import opt +from .. import utils +from . import networks +from .bfm import ParametricFaceModel +from .de_retouching_module import DeRetouchingModule +from .losses import TVLoss, photo_loss +from .nv_diffrast import MeshRenderer +from .pix2pix.pix2pix_model import Pix2PixModel +from .pix2pix.pix2pix_options import Pix2PixOptions + + +@MODELS.register_module('face-reconstruction', 'face_reconstruction') +class FaceReconModel(TorchModel): + + def __init__(self, + model_dir, + w_color=1.92, + tex_iters=100, + w_tex_smooth=10, + *args, + **kwargs): + """The FaceReconModel is implemented based on Deep3DFaceRecon_pytorch, publicly available at + https://github.com/sicxu/Deep3DFaceRecon_pytorch + + Args: + model_dir: the root directory of the model files + w_color: the weight of color loss + """ + super().__init__(model_dir, *args, **kwargs) + + opt.bfm_folder = os.path.join(model_dir, 'assets') + self.opt = opt + self.w_color = w_color + self.w_tex_smooth = w_tex_smooth + self.tex_iters = tex_iters + self.isTrain = opt.isTrain + self.visual_names = ['output_vis'] + self.model_names = ['net_recon', 'mid_net', 'high_net'] + self.parallel_names = self.model_names + [ + 'renderer', 'renderer_high_res' + ] + + # networks + self.net_recon = networks.define_net_recon( + net_recon=opt.net_recon, + use_last_fc=opt.use_last_fc, + init_path=None) + + de_retouching_model_path = os.path.join(model_dir, + 'de_retouching_model.pth') + self.de_retouching_module = DeRetouchingModule( + de_retouching_model_path) + + self.mid_opt = Pix2PixOptions() + self.mid_opt.input_nc = 6 + self.mid_opt.output_nc = 3 + self.mid_opt.name = 'mid_net' + self.mid_net = Pix2PixModel(self.mid_opt).netG + + self.high_opt = Pix2PixOptions() + self.high_opt.input_nc = 9 + self.high_opt.output_nc = 1 + self.high_opt.name = 'high_net' + self.high_net = Pix2PixModel(self.high_opt).netG + + self.alpha = (torch.ones(1, dtype=torch.float32) * 0.01) + self.alpha.requires_grad = True + self.beta = (torch.ones(1, dtype=torch.float32) * 0.01) + self.beta.requires_grad = True + + # assets + self.facemodel_front = ParametricFaceModel( + assets_folder=opt.bfm_folder, + camera_distance=opt.camera_d, + focal=opt.focal, + center=opt.center, + is_train=self.isTrain, + default_name='BFM_model_front.mat') + + bfm_uvs_path = os.path.join(opt.bfm_folder, 'bfm_uvs2.npy') + self.bfm_UVs = np.load(bfm_uvs_path) + self.bfm_UVs = torch.from_numpy(self.bfm_UVs).float() + + # renderer + fov = 2 * np.arctan(opt.center / opt.focal) * 180 / np.pi + self.renderer_high_res = MeshRenderer( + rasterize_fov=fov, + znear=opt.z_near, + zfar=opt.z_far, + rasterize_size=int(2 * opt.center)) + + self.renderer = MeshRenderer( + rasterize_fov=fov, + znear=opt.z_near, + zfar=opt.z_far, + rasterize_size=int(2 * opt.center)) + + self.comupte_color_loss = photo_loss + + def set_device(self, device): + self.device = device + self.mid_net = self.mid_net.to(self.device) + self.high_net = self.high_net.to(self.device) + self.alpha = self.alpha.to(self.device) + self.beta = self.beta.to(self.device) + self.bfm_UVs = self.bfm_UVs.to(self.device) + self.facemodel_front.to(self.device) + + def load_networks(self, load_path): + state_dict = torch.load(load_path, map_location=self.device) + print('loading the model from %s' % load_path) + + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + if isinstance(net, torch.nn.DataParallel): + net = net.module + net.load_state_dict(state_dict[name], strict=False) + + self.alpha = state_dict['alpha'] + self.beta = state_dict['beta'] + + if self.opt.phase != 'test': + if self.opt.continue_train: + + try: + for i, sched in enumerate(self.schedulers): + sched.load_state_dict(state_dict['sched_%02d' % i]) + except Exception as e: + print(e) + for i, sched in enumerate(self.schedulers): + sched.last_epoch = self.opt.epoch_count - 1 + + def setup(self, checkpoint_path): + """Load and print networks; create schedulers + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + self.load_networks(checkpoint_path) + + def parallelize(self, convert_sync_batchnorm=True): + if not self.opt.use_ddp: + for name in self.parallel_names: + if isinstance(name, str): + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + else: + for name in self.model_names: + if isinstance(name, str): + module = getattr(self, name) + if convert_sync_batchnorm: + module = torch.nn.SyncBatchNorm.convert_sync_batchnorm( + module) + setattr( + self, name, + torch.nn.parallel.DistributedDataParallel( + module.to(self.device), + device_ids=[self.device.index], + find_unused_parameters=True, + broadcast_buffers=True)) + + # DistributedDataParallel is not needed when a module doesn't have any parameter that requires a gradient. + for name in self.parallel_names: + if isinstance(name, str) and name not in self.model_names: + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + + # put state_dict of optimizer to gpu device + if self.opt.phase != 'test': + if self.opt.continue_train: + for optim in self.optimizers: + for state in optim.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(self.device) + + def eval(self): + """Make models eval mode""" + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + net.eval() + + self.alpha.requires_grad = False + self.beta.requires_grad = False + + def set_render(self, image_res): + fov = 2 * np.arctan(self.opt.center / self.opt.focal) * 180 / np.pi + if image_res is None: + image_res = int(2 * self.opt.center) + + self.renderer_high_res = MeshRenderer( + rasterize_fov=fov, + znear=self.opt.z_near, + zfar=self.opt.z_far, + rasterize_size=image_res) + + def set_input_base(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input: a dictionary that contains the data itself and its metadata information. + """ + self.input_img = input['imgs'].to(self.device) + self.input_img_hd = input['imgs_hd'].to( + self.device) if 'imgs_hd' in input else None + self.gt_lm = input['lms'].to(self.device) if 'lms' in input else None + self.gt_lm_hd = input['lms_hd'].to( + self.device) if 'lms_hd' in input else None + self.face_mask = input['face_mask'].to( + self.device) if 'face_mask' in input else None + + def set_input_hrn(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input: a dictionary that contains the data itself and its metadata information. + """ + self.input_img = input['input_img'].to(self.device) + self.input_img_for_tex = input['input_img_for_tex'].to(self.device) + self.input_img_hd = input['input_img_hd'].to(self.device) + self.face_mask = input['face_mask'].to(self.device) + self.gt_lm = input['gt_lm'].to(self.device) + self.coeffs = input['coeffs'].to(self.device) + self.position_map = input['position_map'].to(self.device) + self.texture_map = input['texture_map'].to(self.device) + self.tex_valid_mask = input['tex_valid_mask'].to(self.device) + self.de_retouched_albedo_map = input['de_retouched_albedo_map'].to( + self.device) + + def predict_results_base(self): + # predict low-frequency coefficients + with torch.no_grad(): + output_coeff = self.net_recon(self.input_img) + + # 3DMM + face_vertex, face_albedo_map, face_color_map, landmark, face_vertex_noTrans, position_map = \ + self.facemodel_front.compute_for_render(output_coeff) + + # get texture map + texture_map = self.facemodel_front.get_texture_map( + face_vertex, self.input_img_hd) + + # de-retouch + texture_map_input_high = texture_map.permute( + 0, 3, 1, 2).detach() # (1, 3, 256, 256) + texture_map_input_high = (texture_map_input_high - 0.5) * 2 + de_retouched_face_albedo_map = self.de_retouching_module.run( + face_albedo_map, texture_map_input_high) + + # get valid texture mask to deal with occlusion + valid_mask = self.facemodel_front.get_texture_map( + face_vertex, self.face_mask) # (256, 256, 1) + valid_mask = valid_mask.permute(0, 3, 1, + 2).detach() # (1, 1, 256, 256) + + # render + pred_mask, _, pred_face = self.renderer.render_uv_texture( + face_vertex, self.facemodel_front.face_buf, self.bfm_UVs.clone(), + face_color_map) + + input_img_numpy = 255. * (self.input_img).detach().cpu().permute( + 0, 2, 3, 1).numpy() + input_img_numpy = np.squeeze(input_img_numpy) + output_vis = pred_face * pred_mask + (1 - pred_mask) * self.input_img + output_vis_numpy_raw = 255. * output_vis.detach().cpu().permute( + 0, 2, 3, 1).numpy() + output_vis_numpy_raw = np.squeeze(output_vis_numpy_raw) + output_vis_numpy = np.concatenate( + (input_img_numpy, output_vis_numpy_raw), axis=-2) + output_vis = np.squeeze(output_vis_numpy) + output_vis = output_vis[..., ::-1] # rgb->bgr + output_face_mask = pred_mask.detach().cpu().permute( + 0, 2, 3, 1).squeeze().numpy() * 255.0 + output_vis = np.column_stack( + (output_vis, cv2.cvtColor(output_face_mask, cv2.COLOR_GRAY2BGR))) + output_input_vis = output_vis[:, :224] + output_pred_vis = output_vis[:, 224:448] + + input_img_hd = 255. * (self.input_img_hd).detach().cpu().permute( + 0, 2, 3, 1).numpy()[..., ::-1] + input_img_hd = np.squeeze(input_img_hd) + + # from camera space to world space + recon_vertices = face_vertex # [1, 35709, 3] + recon_vertices[..., -1] = 10 - recon_vertices[..., -1] + + recon_shape = face_vertex_noTrans # [1, 35709, 3] + recon_shape[..., -1] = 10 - recon_shape[..., -1] + + tri = self.facemodel_front.face_buf + + # output + output = {} + output['coeffs'] = output_coeff.detach() # [B, 257] + output['vertices'] = recon_vertices.detach() # [B, 35709, 3] + output['vertices_noTrans'] = recon_shape.detach() # [B, 35709, 3] + output['triangles'] = tri.detach() # [n_faces, 3], start from 0 + output['UVs'] = self.bfm_UVs.detach() # [35709, 2] + output['texture_map'] = texture_map.detach() # [B, h, w, 3], RGB + output['albedo_map'] = face_albedo_map.detach() # [B, 3, h, w] + output['color_map'] = face_color_map.detach() # [B, 3, h, w] + output['position_map'] = position_map.detach() # [B, 3, h, w] + + output['input_face'] = output_input_vis + output['pred_face'] = output_pred_vis + output['input_face_hd'] = input_img_hd + + output['input_img'] = self.input_img + output['input_img_hd'] = self.input_img_hd + output['gt_lm'] = self.gt_lm + output['face_mask'] = self.face_mask + output['tex_valid_mask'] = valid_mask + + output[ + 'de_retouched_albedo_map'] = de_retouched_face_albedo_map.detach() + + return output + + def forward(self, visualize=False): + self.bfm_UVs = self.bfm_UVs + + # get valid mask to deal with occlusion + tex_valid_mask = self.tex_valid_mask # (B, 1, 256, 256) + if visualize: + tex_valid_mask = self.smooth_valid_mask(tex_valid_mask) + tex_valid_mask_mid = torch.nn.functional.interpolate( + tex_valid_mask, (64, 64), mode='bilinear') + + # mid frequency + texture_map_input = self.texture_map.permute(0, 3, 1, + 2).to(self.device) + texture_map_input = (texture_map_input - 0.5) * 2 + texture_map_input_mid = torch.nn.functional.interpolate( + texture_map_input, (64, 64), mode='bilinear') + position_map_input_mid = torch.nn.functional.interpolate( + self.position_map, (64, 64), mode='bilinear') + input_mid = torch.cat([position_map_input_mid, texture_map_input_mid], + dim=1) + self.deformation_map = self.mid_net( + input_mid) * 0.1 * self.alpha # ori * 0.1 * self.alpha + self.deformation_map = self.deformation_map * tex_valid_mask_mid + self.deformation_map = self.deformation_map.permute(0, 2, 3, 1) + + # render + self.pred_vertex, self.pred_color, self.pred_lm, self.verts_proj, self.face_albedo_map, \ + face_shape_transformed, face_norm_roted, self.extra_results = \ + self.facemodel_front.compute_for_render_hierarchical_mid(self.coeffs, self.deformation_map, self.bfm_UVs, + visualize=visualize, + de_retouched_albedo_map= + self.de_retouched_albedo_map) + self.pred_mask, _, self.pred_face_mid = self.renderer.render_uv_texture( + self.pred_vertex, self.facemodel_front.face_buf, + self.bfm_UVs.clone(), self.pred_color) + self.deformation_map = self.deformation_map.permute(0, 3, 1, 2) + + # get re-aligned texture + texture_map_input_high = self.facemodel_front.get_texture_map( + self.pred_vertex, self.input_img_hd) # (1, 256, 256, 3) + texture_map_input_high = texture_map_input_high.permute( + 0, 3, 1, 2).detach() # (1, 3, 256, 256) + texture_map_input_high = (texture_map_input_high - 0.5) * 2 + + # high frequency + position_map_input_high = torch.nn.functional.interpolate( + self.position_map, (256, 256), mode='bilinear') + deformation_map_input_high = torch.nn.functional.interpolate( + self.deformation_map, (256, 256), mode='bilinear') + value = [ + position_map_input_high, texture_map_input_high, + deformation_map_input_high + ] + input_high = torch.cat(value, dim=1) + self.displacement_map = self.high_net( + input_high) * 0.1 * self.beta # ori * 0.1 * self.alpha + self.displacement_map = self.displacement_map * tex_valid_mask + + # render + self.pred_color_high, self.extra_results = self.facemodel_front.compute_for_render_hierarchical_high( + self.coeffs, + self.displacement_map, + self.de_retouched_albedo_map, + face_shape_transformed, + face_norm_roted, + extra_results=self.extra_results) + _, _, self.pred_face_high = self.renderer.render_uv_texture( + self.pred_vertex, self.facemodel_front.face_buf, + self.bfm_UVs.clone(), self.pred_color_high) + + self.pred_coeffs_dict = self.facemodel_front.split_coeff(self.coeffs) + + if visualize: + # high + self.extra_results['pred_mask_high'] = self.pred_mask + self.extra_results['pred_face_high_color'] = self.pred_face_high + _, _, self.extra_results[ + 'pred_face_high_gray'] = self.renderer.render_uv_texture( + self.pred_vertex, self.facemodel_front.face_buf, + self.bfm_UVs.clone(), self.extra_results['tex_high_gray']) + + # mid + self.extra_results['pred_mask_mid'] = self.pred_mask + self.extra_results['pred_face_mid_color'] = self.pred_face_mid + _, _, self.extra_results[ + 'pred_face_mid_gray'] = self.renderer.render_uv_texture( + self.pred_vertex, self.facemodel_front.face_buf, + self.bfm_UVs.clone(), self.extra_results['tex_mid_gray']) + + # base + self.extra_results['pred_mask_base'], _, self.extra_results[ + 'pred_face_base_color'] = self.renderer.render_uv_texture( + self.extra_results['pred_vertex_base'], + self.facemodel_front.face_buf, self.bfm_UVs.clone(), + self.extra_results['tex_base_color']) + _, _, self.extra_results[ + 'pred_face_base_gray'] = self.renderer.render_uv_texture( + self.extra_results['pred_vertex_base'], + self.facemodel_front.face_buf, self.bfm_UVs.clone(), + self.extra_results['tex_base_gray']) + + # fit texture + with torch.enable_grad(): + texture_offset = torch.zeros( + (1, 3, 256, 256), dtype=torch.float32).to(self.device) + texture_offset.requires_grad = True + + optim = torch.optim.Adam([texture_offset], lr=1e-2) + + for i in range(self.tex_iters): + pred_color_high = self.pred_color_high.detach( + ) + texture_offset + _, _, pred_face_high = self.renderer.render_uv_texture( + self.pred_vertex.detach(), + self.facemodel_front.face_buf, self.bfm_UVs.clone(), + pred_color_high) + + loss_color_high = self.w_color * self.comupte_color_loss( + pred_face_high, self.input_img_for_tex, + self.pred_mask.detach()) + loss_smooth = TVLoss()(texture_offset) * self.w_tex_smooth + loss_all = loss_color_high + loss_smooth + optim.zero_grad() + loss_all.backward() + optim.step() + + self.pred_color_high = (self.pred_color_high + + texture_offset).detach() + + # for video + self.extra_results['pred_face_high_gray_list'] = [] + self.extra_results['pred_face_high_color_list'] = [] + for i in range(len(self.extra_results['tex_high_gray_list'])): + _, _, pred_face_high_gray_i = self.renderer_high_res.render_uv_texture( + self.extra_results['face_vertex_list'][i], + self.facemodel_front.face_buf, self.bfm_UVs.clone(), + self.extra_results['tex_high_gray_list'][i]) + self.extra_results['pred_face_high_gray_list'].append( + pred_face_high_gray_i) + + _, _, pred_face_high_color_i = self.renderer_high_res.render_uv_texture( + self.extra_results['face_vertex_list'][i], + self.facemodel_front.face_buf, self.bfm_UVs.clone(), + self.pred_color_high) + self.extra_results['pred_face_high_color_list'].append( + pred_face_high_color_i) + + def get_edge_points_horizontal(self): + left_points_list = [] + right_points_list = [] + for k in range(self.face_mask.shape[0]): + left_points = [] + right_points = [] + for i in range(self.face_mask.shape[2]): + inds = torch.where(self.face_mask[k, 0, i, :] > 0.5) # 0.9 + if len(inds[0]) > 0: # i > 112 and len(inds[0]) > 0 + left_points.append(int(inds[0][0]) + 1) + right_points.append(int(inds[0][-1])) + else: + left_points.append(0) + right_points.append(self.face_mask.shape[3] - 1) + left_points_list.append( + torch.tensor(left_points).long().to(self.device)) + right_points_list.append( + torch.tensor(right_points).long().to(self.device)) + self.left_points = torch.stack( + left_points_list, dim=0).long().to(self.device) + self.right_points = torch.stack( + right_points_list, dim=0).long().to(self.device) + + def smooth_valid_mask(self, tex_valid_mask): + """ + + :param tex_valid_mask: torch.tensor, (B, 1, 256, 256), value: 0~1 + :return: + """ + batch_size = tex_valid_mask.shape[0] + tex_valid_mask_ = tex_valid_mask.detach().cpu().numpy() + result_list = [] + for i in range(batch_size): + mask = tex_valid_mask_[i, 0] + mask = cv2.erode(mask, np.ones(shape=(3, 3), dtype=np.float32)) + mask = cv2.blur(mask, (11, 11), 0) + result_list.append( + torch.from_numpy(mask)[None].float().to(tex_valid_mask.device)) + smoothed_mask = torch.stack(result_list, dim=0) + return smoothed_mask + + def compute_visuals_hrn(self): + with torch.no_grad(): + + input_img_vis = 255. * self.input_img.detach().cpu().permute( + 0, 2, 3, 1).numpy() + output_vis_mid = self.pred_face_mid * self.pred_mask + ( + 1 - self.pred_mask) * self.input_img + output_vis_mid = 255. * output_vis_mid.detach().cpu().permute( + 0, 2, 3, 1).numpy() + output_vis_high = self.pred_face_high * self.pred_mask + ( + 1 - self.pred_mask) * self.input_img + output_vis_high = 255. * output_vis_high.detach().cpu().permute( + 0, 2, 3, 1).numpy() + + deformation_map_vis = torch.nn.functional.interpolate( + self.deformation_map, + input_img_vis.shape[1:3], + mode='bilinear').permute(0, 2, 3, 1) + deformation_map_vis = ( + deformation_map_vis - deformation_map_vis.min()) / ( + deformation_map_vis.max() - deformation_map_vis.min()) + deformation_map_vis = 255. * deformation_map_vis.detach().cpu( + ).numpy() + + displacement_map_vis = torch.nn.functional.interpolate( + self.displacement_map, + input_img_vis.shape[1:3], + mode='bilinear').permute(0, 2, 3, 1) + displacement_vis = ( + displacement_map_vis - displacement_map_vis.min()) / ( + displacement_map_vis.max() - displacement_map_vis.min()) + displacement_vis = 255. * displacement_vis.detach().cpu().numpy() + displacement_vis = np.concatenate( + [displacement_vis, displacement_vis, displacement_vis], + axis=-1) + + face_albedo_map_vis = torch.nn.functional.interpolate( + self.face_albedo_map, + input_img_vis.shape[1:3], + mode='bilinear').permute(0, 2, 3, 1) + face_albedo_map_vis = 255. * face_albedo_map_vis.detach().cpu( + ).numpy() + + de_retouched_face_albedo_map_vis = torch.nn.functional.interpolate( + self.de_retouched_albedo_map, + input_img_vis.shape[1:3], + mode='bilinear').permute(0, 2, 3, 1) + de_retouched_face_albedo_map_vis = 255. * de_retouched_face_albedo_map_vis.detach( + ).cpu().numpy() + + if self.extra_results is not None: + pred_mask_base = self.extra_results['pred_mask_base'] + output_vis_base = self.extra_results[ + 'pred_face_base_color'] * pred_mask_base + ( + 1 - pred_mask_base) * self.input_img + output_vis_base = 255. * output_vis_base.detach().cpu( + ).permute(0, 2, 3, 1).numpy() + + output_vis_base_gray = self.extra_results[ + 'pred_face_base_gray'] * pred_mask_base + ( + 1 - pred_mask_base) * self.input_img + output_vis_base_gray = 255. * output_vis_base_gray.detach( + ).cpu().permute(0, 2, 3, 1).numpy() + output_vis_mid_gray = self.extra_results[ + 'pred_face_mid_gray'] * self.pred_mask + ( + 1 - self.pred_mask) * self.input_img + output_vis_mid_gray = 255. * output_vis_mid_gray.detach().cpu( + ).permute(0, 2, 3, 1).numpy() + output_vis_high_gray = self.extra_results[ + 'pred_face_high_gray'] * self.pred_mask + ( + 1 - self.pred_mask) * self.input_img + output_vis_high_gray = 255. * output_vis_high_gray.detach( + ).cpu().permute(0, 2, 3, 1).numpy() + + output_vis_numpy = np.concatenate( + (input_img_vis, output_vis_high, output_vis_base_gray, + output_vis_mid_gray, output_vis_high_gray, + deformation_map_vis, displacement_vis, + face_albedo_map_vis, de_retouched_face_albedo_map_vis), + axis=-2) + + elif self.gt_lm is not None: + gt_lm_numpy = self.gt_lm.cpu().numpy() + pred_lm_numpy = self.pred_lm.detach().cpu().numpy() + output_vis_high_lm = utils.draw_landmarks( + output_vis_high, gt_lm_numpy, 'b') + output_vis_high_lm = utils.draw_landmarks( + output_vis_high_lm, pred_lm_numpy, 'r') + + output_vis_numpy = np.concatenate( + (input_img_vis, output_vis_mid, output_vis_high, + output_vis_high_lm, deformation_map_vis, + displacement_vis), + axis=-2) + else: + output_vis_numpy = np.concatenate( + (input_img_vis, output_vis_mid, output_vis_high, + deformation_map_vis, displacement_vis), + axis=-2) + + self.output_vis = torch.tensor( + output_vis_numpy / 255., + dtype=torch.float32).permute(0, 3, 1, 2).to(self.device) + + def get_current_visuals(self): + """Return visualization images. train.py will display these images with visdom, and save the images to a HTML""" + visual_ret = OrderedDict() + for name in self.visual_names: + if isinstance(name, str): + visual_ret[name] = getattr(self, name)[:, :3, ...] + return visual_ret + + def save_results_hrn(self): + self.compute_visuals_hrn() + results = self.get_current_visuals() + + hrn_output_vis_batch = (255.0 * results['output_vis']).permute( + 0, 2, 3, 1).detach().cpu().numpy()[..., ::-1] + + vertices_batch = self.pred_vertex.detach( + ) # get reconstructed shape, [1, 35709, 3] + vertices_batch[..., -1] = 10 - vertices_batch[ + ..., -1] # from camera space to world space + vertices_batch = vertices_batch.cpu().numpy() + + texture_map_batch = (255.0 * self.pred_color_high).permute( + 0, 2, 3, 1).detach().cpu().numpy()[..., ::-1] + + output = {} + + output['vis_image'] = hrn_output_vis_batch[0] + + texture_map = texture_map_batch[0] + vertices = vertices_batch[0] + normals = utils.estimate_normals( + vertices, + self.facemodel_front.face_buf.cpu().numpy()) + face_mesh = { + 'vertices': vertices, + 'faces': self.facemodel_front.face_buf.cpu().numpy() + 1, + 'UVs': self.bfm_UVs.detach().cpu().numpy(), + 'faces_uv': self.facemodel_front.face_buf.cpu().numpy() + 1, + 'normals': normals, + 'faces_normal': self.facemodel_front.face_buf.cpu().numpy() + 1, + } + output['face_mesh'] = face_mesh + output['texture_map'] = texture_map + + frame_list = [] + if 'pred_face_high_gray_list' in self.extra_results: + input_img_vis = 255. * self.input_img_hd.detach().cpu().permute( + 0, 2, 3, 1).numpy()[0] + for j in range( + len(self.extra_results['pred_face_high_gray_list'])): + pred_face_high_gray_j = self.extra_results[ + 'pred_face_high_gray_list'][j][0, ...] + pred_face_high_gray_j = 255. * pred_face_high_gray_j.detach( + ).cpu().permute(1, 2, 0).numpy() + pred_face_high_color_j = self.extra_results[ + 'pred_face_high_color_list'][j][0, ...] + pred_face_high_color_j = 255. * pred_face_high_color_j.detach( + ).cpu().permute(1, 2, 0).numpy() + + value = [ + input_img_vis, pred_face_high_gray_j, + pred_face_high_color_j + ] + vis_j = np.concatenate(value, axis=1) + + frame_list.append(vis_j.clip(0, 255).astype(np.uint8)) + output['frame_list'] = frame_list + + return output diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/losses.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..6d4af4e8d875f6809f8defbcb6ad94d6e7a56777 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/losses.py @@ -0,0 +1,367 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from kornia.geometry import warp_affine + + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize)) + + +# perceptual level loss +class PerceptualLoss(nn.Module): + + def __init__(self, recog_net, input_size=112): + super(PerceptualLoss, self).__init__() + self.recog_net = recog_net + self.preprocess = lambda x: 2 * x - 1 + self.input_size = input_size + + def forward(self, imageA, imageB, M): + """ + 1 - cosine distance + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1) , RGB order + imageB --same as imageA + """ + + imageA = self.preprocess(resize_n_crop(imageA, M, self.input_size)) + imageB = self.preprocess(resize_n_crop(imageB, M, self.input_size)) + + # freeze bn + self.recog_net.eval() + + id_featureA = F.normalize(self.recog_net(imageA), dim=-1, p=2) + id_featureB = F.normalize(self.recog_net(imageB), dim=-1, p=2) + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + + +def perceptual_loss(id_featureA, id_featureB): + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + + +# image level loss +def photo_loss(imageA, imageB, mask, eps=1e-6): + """ + l2 norm (with sqrt, to ensure backward stabililty, use eps, otherwise Nan may occur) + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1), RGB order + imageB --same as imageA + """ + loss = torch.sqrt(eps + torch.sum( + (imageA - imageB)**2, dim=1, keepdims=True)) * mask + loss = torch.sum(loss) / torch.max( + torch.sum(mask), + torch.tensor(1.0).to(mask.device)) + return loss + + +def landmark_loss(predict_lm, gt_lm, weight=None): + """ + weighted mse loss + Parameters: + predict_lm --torch.tensor (B, 68, 2) + gt_lm --torch.tensor (B, 68, 2) + weight --numpy.array (1, 68) + """ + if not weight: + weight = np.ones([68]) + weight[28:31] = 20 + weight[-8:] = 20 + weight = np.expand_dims(weight, 0) + weight = torch.tensor(weight).to(predict_lm.device) + loss = torch.sum((predict_lm - gt_lm)**2, dim=-1) * weight + loss = torch.sum(loss) / (predict_lm.shape[0] * predict_lm.shape[1]) + return loss + + +# regulization +def reg_loss(coeffs_dict, w_id=1, w_exp=1, w_tex=1): + """ + l2 norm without the sqrt, from yu's implementation (mse) + tf.nn.l2_loss https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss + Parameters: + coeffs_dict -- a dict of torch.tensors , keys: id, exp, tex, angle, gamma, trans + + """ + # coefficient regularization to ensure plausible 3d faces + value_1 = w_id * torch.sum(coeffs_dict['id']**2) + value_2 = w_exp * torch.sum(coeffs_dict['exp']**2) + value_3 = w_tex * torch.sum(coeffs_dict['tex']**2) + creg_loss = value_1 + value_2 + value_3 + creg_loss = creg_loss / coeffs_dict['id'].shape[0] + + # gamma regularization to ensure a nearly-monochromatic light + gamma = coeffs_dict['gamma'].reshape([-1, 3, 9]) + gamma_mean = torch.mean(gamma, dim=1, keepdims=True) + gamma_loss = torch.mean((gamma - gamma_mean)**2) + + return creg_loss, gamma_loss + + +def reflectance_loss(texture, mask): + """ + minimize texture variance (mse), albedo regularization to ensure an uniform skin albedo + Parameters: + texture --torch.tensor, (B, N, 3) + mask --torch.tensor, (N), 1 or 0 + + """ + mask = mask.reshape([1, mask.shape[0], 1]) + texture_mean = torch.sum( + mask * texture, dim=1, keepdims=True) / torch.sum(mask) + loss = torch.sum(((texture - texture_mean) * mask)**2) / ( + texture.shape[0] * torch.sum(mask)) + return loss + + +def lm_3d_loss(pred_lm_3d, gt_lm_3d, mask): + loss = torch.abs(pred_lm_3d - gt_lm_3d)[mask, :] + loss = torch.mean(loss) + return loss + + +class TVLoss(nn.Module): + + def __init__(self, TVLoss_weight=1): + super(TVLoss, self).__init__() + self.TVLoss_weight = TVLoss_weight + + def forward(self, x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + count_h = self._tensor_size(x[:, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, 1:]) + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, :h_x - 1, :]), 2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, :w_x - 1]), 2).sum() + return self.TVLoss_weight * 2 * (h_tv / count_h + + w_tv / count_w) / batch_size + + def _tensor_size(self, t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +class TVLoss_std(nn.Module): + + def __init__(self, TVLoss_weight=1): + super(TVLoss_std, self).__init__() + self.TVLoss_weight = TVLoss_weight + + def forward(self, x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, :h_x - 1, :]), 2) + h_tv = ((h_tv - torch.mean(h_tv))**2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, :w_x - 1]), 2) + w_tv = ((w_tv - torch.mean(w_tv))**2).sum() + return self.TVLoss_weight * 2 * (h_tv + w_tv) / batch_size + + def _tensor_size(self, t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +def photo_loss_sum(imageA, imageB, mask, eps=1e-6): + """ + l2 norm (with sqrt, to ensure backward stabililty, use eps, otherwise Nan may occur) + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1), RGB order + imageB --same as imageA + """ + loss = torch.sqrt(eps + torch.sum( + (imageA - imageB)**2, dim=1, keepdims=True)) * mask + loss = torch.sum(loss) / ( + imageA.shape[0] * imageA.shape[2] * imageA.shape[3]) + return loss + + +def points_loss_horizontal(verts, left_points, right_points, width=224): + verts_int = torch.ceil(verts[0]).long().clamp(0, width - 1) # (n, 2) + verts_left = left_points[width - 1 - verts_int[:, 1]].float() + verts_right = right_points[width - 1 - verts_int[:, 1]].float() + verts_x = verts[0, :, 0] + dist = (verts_left - verts_x) / width * (verts_right - verts_x) / width + dist /= torch.max( + torch.abs((verts_left - verts_x) / width), + torch.abs((verts_right - verts_x) / width)) + edge_inds = torch.where(dist > 0)[0] + dist += 0.01 + dist = torch.nn.functional.relu(dist).clone() + dist -= 0.01 + dist = torch.abs(dist) + loss = torch.mean(dist) + return loss, edge_inds + + +class LaplacianLoss(nn.Module): + + def __init__(self): + super(LaplacianLoss, self).__init__() + + def forward(self, x): + batch_size, slice_num = x.size()[:2] + z_x = x.size()[2] + h_x = x.size()[3] + w_x = x.size()[4] + count_z = self._tensor_size(x[:, :, 1:, :, :]) + count_h = self._tensor_size(x[:, :, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, :, 1:]) + z_tv = torch.pow((x[:, :, 1:, :, :] - x[:, :, :z_x - 1, :, :]), + 2).sum() + h_tv = torch.pow((x[:, :, :, 1:, :] - x[:, :, :, :h_x - 1, :]), + 2).sum() + w_tv = torch.pow((x[:, :, :, :, 1:] - x[:, :, :, :, :w_x - 1]), + 2).sum() + return 2 * (z_tv / count_z + h_tv / count_h + w_tv / count_w) / ( + batch_size * slice_num) + + def _tensor_size(self, t): + return t.size()[2] * t.size()[3] * t.size()[4] + + +class LaplacianLoss_L1(nn.Module): + + def __init__(self): + super(LaplacianLoss_L1, self).__init__() + + def forward(self, x): + batch_size, slice_num = x.size()[:2] + z_x = x.size()[2] + h_x = x.size()[3] + w_x = x.size()[4] + count_z = self._tensor_size(x[:, :, 1:, :, :]) + count_h = self._tensor_size(x[:, :, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, :, 1:]) + z_tv = torch.abs((x[:, :, 1:, :, :] - x[:, :, :z_x - 1, :, :])).sum() + h_tv = torch.abs((x[:, :, :, 1:, :] - x[:, :, :, :h_x - 1, :])).sum() + w_tv = torch.abs((x[:, :, :, :, 1:] - x[:, :, :, :, :w_x - 1])).sum() + return 2 * (z_tv / count_z + h_tv / count_h + w_tv / count_w) / ( + batch_size * slice_num) + + def _tensor_size(self, t): + return t.size()[2] * t.size()[3] * t.size()[4] + + +class GANLoss(nn.Module): + + def __init__(self, + gan_mode, + target_real_label=1.0, + target_fake_label=0.0, + tensor=torch.FloatTensor): + super(GANLoss, self).__init__() + self.real_label = target_real_label + self.fake_label = target_fake_label + self.real_label_tensor = None + self.fake_label_tensor = None + self.zero_tensor = None + self.Tensor = tensor + self.gan_mode = gan_mode + if gan_mode == 'ls': + pass + elif gan_mode == 'original': + pass + elif gan_mode == 'w': + pass + elif gan_mode == 'hinge': + pass + else: + raise ValueError('Unexpected gan_mode {}'.format(gan_mode)) + + def get_target_tensor(self, input, target_is_real): + if target_is_real: + if self.real_label_tensor is None: + self.real_label_tensor = self.Tensor(1).fill_(self.real_label) + self.real_label_tensor.requires_grad_(False) + return self.real_label_tensor.expand_as(input) + else: + if self.fake_label_tensor is None: + self.fake_label_tensor = self.Tensor(1).fill_(self.fake_label) + self.fake_label_tensor.requires_grad_(False) + return self.fake_label_tensor.expand_as(input) + + def get_zero_tensor(self, input): + if self.zero_tensor is None: + self.zero_tensor = self.Tensor(1).fill_(0) + self.zero_tensor.requires_grad_(False) + return self.zero_tensor.expand_as(input) + + def loss(self, input, target_is_real, for_discriminator=True): + if self.gan_mode == 'original': # cross entropy loss + target_tensor = self.get_target_tensor(input, target_is_real) + loss = F.binary_cross_entropy_with_logits(input, target_tensor) + return loss + elif self.gan_mode == 'ls': + target_tensor = self.get_target_tensor(input, target_is_real) + return F.mse_loss(input, target_tensor) + elif self.gan_mode == 'hinge': + if for_discriminator: + if target_is_real: + minval = torch.min(input - 1, self.get_zero_tensor(input)) + loss = -torch.mean(minval) + else: + minval = torch.min(-input - 1, self.get_zero_tensor(input)) + loss = -torch.mean(minval) + else: + assert target_is_real, "The generator's hinge loss must be aiming for real" + loss = -torch.mean(input) + return loss + else: + # wgan + if target_is_real: + return -input.mean() + else: + return input.mean() + + def __call__(self, input, target_is_real, for_discriminator=True): + # computing loss is a bit complicated because |input| may not be + # a tensor, but list of tensors in case of multiscale discriminator + if isinstance(input, list): + loss = 0 + for pred_i in input: + if isinstance(pred_i, list): + pred_i = pred_i[-1] + loss_tensor = self.loss(pred_i, target_is_real, + for_discriminator) + bs = 1 if len(loss_tensor.size()) == 0 else loss_tensor.size(0) + new_loss = torch.mean(loss_tensor.view(bs, -1), dim=1) + loss += new_loss + return loss / len(input) + else: + return self.loss(input, target_is_real, for_discriminator) + + +class BinaryDiceLoss(nn.Module): + + def __init__(self, smooth=1, p=1, reduction='mean'): + super(BinaryDiceLoss, self).__init__() + self.smooth = smooth + self.p = p + self.reduction = reduction + + def forward(self, predict, target): + assert predict.shape[0] == target.shape[ + 0], "predict & target batch size don't match" + predict = predict.contiguous().view(predict.shape[0], -1) + target = target.contiguous().view(target.shape[0], -1) + + num = torch.sum(torch.mul(predict, target), dim=1) + den = torch.sum(predict + target, dim=1) + + loss = 1 - (2 * num + self.smooth) / (den + self.smooth) + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + elif self.reduction == 'none': + return loss + else: + raise Exception('Unexpected reduction {}'.format(self.reduction)) diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/networks.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb5770b2e935fefff93953927170ee422b7f789 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/networks.py @@ -0,0 +1,577 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch +import os +from typing import Any, Callable, List, Optional, Type, Union + +import torch +import torch.nn as nn +from kornia.geometry import warp_affine +from torch import Tensor +from torch.optim import lr_scheduler + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize)) + + +def filter_state_dict(state_dict, remove_name='fc'): + new_state_dict = {} + for key in state_dict: + if remove_name in key: + continue + new_state_dict[key] = state_dict[key] + return new_state_dict + + +def define_net_recon(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper( + net_recon, use_last_fc=use_last_fc, init_path=init_path) + + +def define_net_recon2(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper2( + net_recon, use_last_fc=use_last_fc, init_path=init_path) + + +class ReconNetWrapper(nn.Module): + fc_dim = 257 + + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', + net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict( + torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print('loading init net_recon %s from %s' % (net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 64, bias=True), # exp layer + conv1x1(last_dim, 80, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +class ReconNetWrapper2(nn.Module): + fc_dim = 264 + + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper2, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', + net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict( + torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print('loading init net_recon %s from %s' % (net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers2 = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 51, bias=True), # exp layer + conv1x1(last_dim, 100, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers2: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers2: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +# adapted from https://github.com/pytorch/vision/edit/master/torchvision/models/resnet.py +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + +model_urls = { + 'resnet18': + 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': + 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': + 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': + 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': + 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': + 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': + 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': + 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': + 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes: int, + out_planes: int, + stride: int = 1, + groups: int = 1, + dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes: int, + out_planes: int, + stride: int = 1, + bias: bool = False) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=bias) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + use_last_fc: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.use_last_fc = use_last_fc + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + if self.use_last_fc: + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, + 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, + 0) # type: ignore[arg-type] + + def _make_layer(self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + if self.use_last_fc: + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet(arch: str, block: Type[Union[BasicBlock, + Bottleneck]], layers: List[int], + pretrained: bool, progress: bool, **kwargs: Any) -> ResNet: + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url( + model_urls[arch], progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +func_dict = {'resnet18': (resnet18, 512), 'resnet50': (resnet50, 2048)} diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/nv_diffrast.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/nv_diffrast.py new file mode 100644 index 0000000000000000000000000000000000000000..348c811257e04a93e562af73e922287013b47690 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/nv_diffrast.py @@ -0,0 +1,234 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch +import warnings +from typing import List + +import numpy as np +import nvdiffrast +import nvdiffrast.torch as dr +import torch +import torch.nn.functional as F +from torch import nn + +from .losses import TVLoss, TVLoss_std + +warnings.filterwarnings('ignore') + + +def ndc_projection(x=0.1, n=1.0, f=50.0): + return np.array([[n / x, 0, 0, 0], [0, n / -x, 0, 0], + [0, 0, -(f + n) / (f - n), -(2 * f * n) / (f - n)], + [0, 0, -1, 0]]).astype(np.float32) + + +def to_image(face_shape): + """ + Return: + face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + focal = 1015. + center = 112. + persc_proj = np.array([focal, 0, center, 0, focal, center, 0, 0, + 1]).reshape([3, 3]).astype(np.float32).transpose() + + persc_proj = torch.tensor(persc_proj).to(face_shape.device) + + face_proj = face_shape @ persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + +class MeshRenderer(nn.Module): + + def __init__(self, rasterize_fov, znear=0.1, zfar=10, rasterize_size=224): + super(MeshRenderer, self).__init__() + + x = np.tan(np.deg2rad(rasterize_fov * 0.5)) * znear + self.ndc_proj = torch.tensor(ndc_projection( + x=x, n=znear, + f=zfar)).matmul(torch.diag(torch.tensor([1., -1, -1, 1]))) + self.rasterize_size = rasterize_size + self.glctx = None + + def forward(self, vertex, tri, feat=None): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (B, M, 3) or (M, 3), triangles + feat(optional) -- torch.tensor, size (B, C), features + """ + device = vertex.device + rsize = int(self.rasterize_size) + ndc_proj = self.ndc_proj.to(device) + verts_proj = to_image(vertex) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat( + [vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], + dim=-1) + vertex[..., 1] = -vertex[..., 1] + + vertex_ndc = vertex @ ndc_proj.t() + if self.glctx is None: + if nvdiffrast.__version__ == '0.2.7': + self.glctx = dr.RasterizeGLContext(device=device) + else: + self.glctx = dr.RasterizeCudaContext(device=device) + + ranges = None + if isinstance(tri, List) or len(tri.shape) == 3: + vum = vertex_ndc.shape[1] + fnum = torch.tensor([f.shape[0] + for f in tri]).unsqueeze(1).to(device) + + print('fnum shape:{}'.format(fnum.shape)) + + fstartidx = torch.cumsum(fnum, dim=0) - fnum + ranges = torch.cat([fstartidx, fnum], + axis=1).type(torch.int32).cpu() + for i in range(tri.shape[0]): + tri[i] = tri[i] + i * vum + vertex_ndc = torch.cat(vertex_ndc, dim=0) + tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + rast_out, _ = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + depth, _ = dr.interpolate( + vertex.reshape([-1, 4])[..., 2].unsqueeze(1).contiguous(), + rast_out, tri) + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out[..., 3] > 0).float().unsqueeze(1) + depth = mask * depth + + image = None + + verts_x = verts_proj[0, :, 0] + verts_y = 224 - verts_proj[0, :, 1] + verts_int = torch.ceil(verts_proj[0]).long() # (n, 2) + verts_xr_int = verts_int[:, 0].clamp(1, 224 - 1) + verts_yt_int = 224 - verts_int[:, 1].clamp(2, 224) + verts_right_float = verts_xr_int - verts_x + verts_left_float = 1 - verts_right_float + verts_top_float = verts_y - verts_yt_int + verts_bottom_float = 1 - verts_top_float + + rast_lt = rast_out[0, verts_yt_int, verts_xr_int - 1, 3] + rast_lb = rast_out[0, verts_yt_int + 1, verts_xr_int - 1, 3] + rast_rt = rast_out[0, verts_yt_int, verts_xr_int, 3] + rast_rb = rast_out[0, verts_yt_int + 1, verts_xr_int, 3] + + occ_feat = (rast_lt > 0) * 1.0 * (verts_left_float + verts_top_float) + \ + (rast_lb > 0) * 1.0 * (verts_left_float + verts_bottom_float) + \ + (rast_rt > 0) * 1.0 * (verts_right_float + verts_top_float) + \ + (rast_rb > 0) * 1.0 * (verts_right_float + verts_bottom_float) + occ_feat = occ_feat[None, :, None] / 4.0 + + occ, _ = dr.interpolate(occ_feat, rast_out, tri) + occ = occ.permute(0, 3, 1, 2) + + if feat is not None: + image, _ = dr.interpolate(feat, rast_out, tri) + image = image.permute(0, 3, 1, 2) + image = mask * image + + return mask, depth, image, occ + + def render_uv_texture(self, vertex, tri, uv, uv_texture): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (M, 3), triangles + uv -- torch.tensor, size (B,N, 2), uv mapping + base_tex -- torch.tensor, size (B,H,W,C) + """ + device = vertex.device + rsize = int(self.rasterize_size) + ndc_proj = self.ndc_proj.to(device) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat( + [vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], + dim=-1) + vertex[..., 1] = -vertex[..., 1] + + vertex_ndc = vertex @ ndc_proj.t() + if self.glctx is None: + if nvdiffrast.__version__ == '0.2.7': + self.glctx = dr.RasterizeGLContext(device=device) + else: + self.glctx = dr.RasterizeCudaContext(device=device) + + ranges = None + if isinstance(tri, List) or len(tri.shape) == 3: + vum = vertex_ndc.shape[1] + fnum = torch.tensor([f.shape[0] + for f in tri]).unsqueeze(1).to(device) + + print('fnum shape:{}'.format(fnum.shape)) + + fstartidx = torch.cumsum(fnum, dim=0) - fnum + ranges = torch.cat([fstartidx, fnum], + axis=1).type(torch.int32).cpu() + for i in range(tri.shape[0]): + tri[i] = tri[i] + i * vum + vertex_ndc = torch.cat(vertex_ndc, dim=0) + tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + rast_out, _ = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + depth, _ = dr.interpolate( + vertex.reshape([-1, 4])[..., 2].unsqueeze(1).contiguous(), + rast_out, tri) + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out[..., 3] > 0).float().unsqueeze(1) + depth = mask * depth + uv[..., -1] = 1.0 - uv[..., -1] + + rast_out, rast_db = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + interp_out, uv_da = dr.interpolate( + uv, rast_out, tri, rast_db, diff_attrs='all') + + uv_texture = uv_texture.permute(0, 2, 3, 1).contiguous() + img = dr.texture( + uv_texture, interp_out, filter_mode='linear') # , uv_da) + img = img * torch.clamp(rast_out[..., -1:], 0, + 1) # Mask out background. + + image = img.permute(0, 3, 1, 2) + + return mask, depth, image diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/opt.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/opt.py new file mode 100644 index 0000000000000000000000000000000000000000..b8d97faf72beeceeaeebc4563e60217592d97724 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/opt.py @@ -0,0 +1,14 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +bfm_folder = '' +bfm_model = 'head_model_for_maas.mat' +camera_d = 10.0 +center = 112.0 +focal = 1015.0 +isTrain = False +net_recon = 'resnet50' +phase = 'test' +use_ddp = False +use_last_fc = False +z_far = 15.0 +z_near = 5.0 +lr = 0.001 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/__init__.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/networks.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..c18881edc1cf2763c136408f706289e1555431f5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/networks.py @@ -0,0 +1,848 @@ +# Part of the implementation is borrowed and modified from pix2pix, +# publicly available at https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix +import functools + +import torch +import torch.nn as nn +from torch.nn import init +from torch.optim import lr_scheduler + +############################################################################### +# Helper Functions +############################################################################### + + +class Identity(nn.Module): + + def forward(self, x): + return x + + +def get_norm_layer(norm_type='instance'): + """Return a normalization layer + + Parameters: + norm_type (str) -- the name of the normalization layer: batch | instance | none + + For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev). + For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics. + """ + if norm_type == 'batch': + norm_layer = functools.partial( + nn.BatchNorm2d, affine=True, track_running_stats=True) + elif norm_type == 'instance': + norm_layer = functools.partial( + nn.InstanceNorm2d, affine=False, track_running_stats=False) + elif norm_type == 'none': + + def norm_layer(x): + return Identity() + else: + raise NotImplementedError('normalization layer [%s] is not found' + % norm_type) + return norm_layer + + +def get_scheduler(optimizer, opt): + """Return a learning rate scheduler + + Parameters: + optimizer -- the optimizer of the network + opt (option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions.  + opt.lr_policy is the name of learning rate policy: linear | step | plateau | cosine + + For 'linear', we keep the same learning rate for the first epochs + and linearly decay the rate to zero over the next epochs. + For other schedulers (step, plateau, and cosine), we use the default PyTorch schedulers. + See https://pytorch.org/docs/stable/optim.html for more details. + """ + if opt.lr_policy == 'linear': + + def lambda_rule(epoch): + lr_l = 1.0 - max(0, epoch + opt.epoch_count + - opt.n_epochs) / float(opt.n_epochs_decay + 1) + return lr_l + + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) + elif opt.lr_policy == 'step': + scheduler = lr_scheduler.StepLR( + optimizer, step_size=opt.lr_decay_iters, gamma=0.1) + elif opt.lr_policy == 'plateau': + scheduler = lr_scheduler.ReduceLROnPlateau( + optimizer, mode='min', factor=0.2, threshold=0.01, patience=5) + elif opt.lr_policy == 'cosine': + scheduler = lr_scheduler.CosineAnnealingLR( + optimizer, T_max=opt.n_epochs, eta_min=0) + else: + return NotImplementedError( + 'learning rate policy [%s] is not implemented', opt.lr_policy) + return scheduler + + +def init_weights(net, init_type='normal', init_gain=0.02): + """Initialize network weights. + + Parameters: + net (network) -- network to be initialized + init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + + We use 'normal' in the original pix2pix and CycleGAN paper. But xavier and kaiming might + work better for some applications. Feel free to try yourself. + """ + + def init_func(m): # define the initialization function + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + if init_type == 'normal': + init.normal_(m.weight.data, 0.0, init_gain) + elif init_type == 'xavier': + init.xavier_normal_(m.weight.data, gain=init_gain) + elif init_type == 'kaiming': + init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + init.orthogonal_(m.weight.data, gain=init_gain) + else: + raise NotImplementedError( + 'initialization method [%s] is not implemented' + % init_type) + if hasattr(m, 'bias') and m.bias is not None: + init.constant_(m.bias.data, 0.0) + elif classname.find( + 'BatchNorm2d' + ) != -1: # BatchNorm Layer's weight is not a matrix; only normal distribution applies. + init.normal_(m.weight.data, 1.0, init_gain) + init.constant_(m.bias.data, 0.0) + + print('initialize network with %s' % init_type) + net.apply(init_func) # apply the initialization function + + +def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]): + """Initialize a network: 1. register CPU/GPU device (with multi-GPU support); 2. initialize the network weights + Parameters: + net (network) -- the network to be initialized + init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal + gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Return an initialized network. + """ + if len(gpu_ids) > 0: + assert (torch.cuda.is_available()) + net.to(gpu_ids[0]) + net = torch.nn.DataParallel(net, gpu_ids) # multi-GPUs + init_weights(net, init_type, init_gain=init_gain) + return net + + +def define_G(input_nc, + output_nc, + ngf, + netG, + norm='batch', + use_dropout=False, + init_type='normal', + init_gain=0.02, + gpu_ids=[]): + """Create a generator + + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + ngf (int) -- the number of filters in the last conv layer + netG (str) -- the architecture's name: resnet_9blocks | resnet_6blocks | unet_256 | unet_128 + norm (str) -- the name of normalization layers used in the network: batch | instance | none + use_dropout (bool) -- if use dropout layers. + init_type (str) -- the name of our initialization method. + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Returns a generator + + Our current implementation provides two types of generators: + U-Net: [unet_128] (for 128x128 input images) and [unet_256] (for 256x256 input images) + The original U-Net paper: https://arxiv.org/abs/1505.04597 + + Resnet-based generator: [resnet_6blocks] (with 6 Resnet blocks) and [resnet_9blocks] (with 9 Resnet blocks) + Resnet-based generator consists of several Resnet blocks between a few downsampling/upsampling operations. + We adapt Torch code from Justin Johnson's neural style transfer project + (https://github.com/jcjohnson/fast-neural-style). + + + The generator has been initialized by . It uses RELU for non-linearity. + """ + net = None + norm_layer = get_norm_layer(norm_type=norm) + + if netG == 'resnet_9blocks': + net = ResnetGenerator( + input_nc, + output_nc, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout, + n_blocks=9) + elif netG == 'resnet_6blocks': + net = ResnetGenerator( + input_nc, + output_nc, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout, + n_blocks=6) + elif netG == 'unet_32': + net = UnetGenerator( + input_nc, + output_nc, + 5, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout) + elif netG == 'unet_64': + net = UnetGenerator( + input_nc, + output_nc, + 6, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout) + elif netG == 'unet_128': + net = UnetGenerator( + input_nc, + output_nc, + 7, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout) + elif netG == 'unet_256': + net = UnetGenerator( + input_nc, + output_nc, + 8, + ngf, + norm_layer=norm_layer, + use_dropout=use_dropout) + else: + raise NotImplementedError('Generator model name [%s] is not recognized' + % netG) + return init_net(net, init_type, init_gain, gpu_ids) + + +def define_D(input_nc, + ndf, + netD, + n_layers_D=3, + norm='batch', + init_type='normal', + init_gain=0.02, + gpu_ids=[]): + """Create a discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the first conv layer + netD (str) -- the architecture's name: basic | n_layers | pixel + n_layers_D (int) -- the number of conv layers in the discriminator; effective when netD=='n_layers' + norm (str) -- the type of normalization layers used in the network. + init_type (str) -- the name of the initialization method. + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Returns a discriminator + + Our current implementation provides three types of discriminators: + [basic]: 'PatchGAN' classifier described in the original pix2pix paper. + It can classify whether 70×70 overlapping patches are real or fake. + Such a patch-level discriminator architecture has fewer parameters + than a full-image discriminator and can work on arbitrarily-sized images + in a fully convolutional fashion. + + [n_layers]: With this mode, you can specify the number of conv layers in the discriminator + with the parameter (default=3 as used in [basic] (PatchGAN).) + + [pixel]: 1x1 PixelGAN discriminator can classify whether a pixel is real or not. + It encourages greater color diversity but has no effect on spatial statistics. + + The discriminator has been initialized by . It uses Leakly RELU for non-linearity. + """ + net = None + norm_layer = get_norm_layer(norm_type=norm) + + if netD == 'basic': # default PatchGAN classifier + net = NLayerDiscriminator( + input_nc, ndf, n_layers=3, norm_layer=norm_layer) + elif netD == 'n_layers': # more options + net = NLayerDiscriminator( + input_nc, ndf, n_layers_D, norm_layer=norm_layer) + elif netD == 'pixel': # classify if each pixel is real or fake + net = PixelDiscriminator(input_nc, ndf, norm_layer=norm_layer) + else: + raise NotImplementedError( + 'Discriminator model name [%s] is not recognized' % netD) + return init_net(net, init_type, init_gain, gpu_ids) + + +############################################################################## +# Classes +############################################################################## +class GANLoss(nn.Module): + """Define different GAN objectives. + + The GANLoss class abstracts away the need to create the target label tensor + that has the same size as the input. + """ + + def __init__(self, gan_mode, target_real_label=1.0, target_fake_label=0.0): + """ Initialize the GANLoss class. + + Parameters: + gan_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp. + target_real_label (bool) - - label for a real image + target_fake_label (bool) - - label of a fake image + + Note: Do not use sigmoid as the last layer of Discriminator. + LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss. + """ + super(GANLoss, self).__init__() + self.register_buffer('real_label', torch.tensor(target_real_label)) + self.register_buffer('fake_label', torch.tensor(target_fake_label)) + self.gan_mode = gan_mode + if gan_mode == 'lsgan': + self.loss = nn.MSELoss() + elif gan_mode == 'vanilla': + self.loss = nn.BCEWithLogitsLoss() + elif gan_mode in ['wgangp']: + self.loss = None + else: + raise NotImplementedError('gan mode %s not implemented' % gan_mode) + + def get_target_tensor(self, prediction, target_is_real): + """Create label tensors with the same size as the input. + + Parameters: + prediction (tensor) - - tpyically the prediction from a discriminator + target_is_real (bool) - - if the ground truth label is for real images or fake images + + Returns: + A label tensor filled with ground truth label, and with the size of the input + """ + + if target_is_real: + target_tensor = self.real_label + else: + target_tensor = self.fake_label + return target_tensor.expand_as(prediction) + + def __call__(self, prediction, target_is_real): + """Calculate loss given Discriminator's output and grount truth labels. + + Parameters: + prediction (tensor) - - tpyically the prediction output from a discriminator + target_is_real (bool) - - if the ground truth label is for real images or fake images + + Returns: + the calculated loss. + """ + if self.gan_mode in ['lsgan', 'vanilla']: + target_tensor = self.get_target_tensor(prediction, target_is_real) + loss = self.loss(prediction, target_tensor) + elif self.gan_mode == 'wgangp': + if target_is_real: + loss = -prediction.mean() + else: + loss = prediction.mean() + return loss + + +def cal_gradient_penalty(netD, + real_data, + fake_data, + device, + type='mixed', + constant=1.0, + lambda_gp=10.0): + """Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028 + + Arguments: + netD (network) -- discriminator network + real_data (tensor array) -- real images + fake_data (tensor array) -- generated images from the generator + device (str) -- GPU / CPU: from torch.device('cuda:{}'.format(self.gpu_ids[0])) + if self.gpu_ids else torch.device('cpu') + type (str) -- if we mix real and fake data or not [real | fake | mixed]. + constant (float) -- the constant used in formula ( ||gradient||_2 - constant)^2 + lambda_gp (float) -- weight for this loss + + Returns the gradient penalty loss + """ + if lambda_gp > 0.0: + if type == 'real': # either use real images, fake images, or a linear interpolation of two. + interpolatesv = real_data + elif type == 'fake': + interpolatesv = fake_data + elif type == 'mixed': + alpha = torch.rand(real_data.shape[0], 1, device=device) + alpha = alpha.expand( + real_data.shape[0], + real_data.nelement() + // real_data.shape[0]).contiguous().view(*real_data.shape) + interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) + else: + raise NotImplementedError('{} not implemented'.format(type)) + interpolatesv.requires_grad_(True) + disc_interpolates = netD(interpolatesv) + gradients = torch.autograd.grad( + outputs=disc_interpolates, + inputs=interpolatesv, + grad_outputs=torch.ones(disc_interpolates.size()).to(device), + create_graph=True, + retain_graph=True, + only_inputs=True) + gradients = gradients[0].view(real_data.size(0), -1) # flat the data + gradient_penalty = (((gradients + 1e-16).norm(2, dim=1) + - constant)**2).mean() + gradient_penalty = gradient_penalty * lambda_gp + return gradient_penalty, gradients + else: + return 0.0, None + + +class ResnetGenerator(nn.Module): + """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations. + + We adapt Torch code and idea from Justin Johnson's neural style transfer + project(https://github.com/jcjohnson/fast-neural-style) + """ + + def __init__(self, + input_nc, + output_nc, + ngf=64, + norm_layer=nn.BatchNorm2d, + use_dropout=False, + n_blocks=6, + padding_type='reflect'): + """Construct a Resnet-based generator + + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + ngf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers + n_blocks (int) -- the number of ResNet blocks + padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero + """ + assert (n_blocks >= 0) + super(ResnetGenerator, self).__init__() + if type(norm_layer) == functools.partial: + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + model = [ + nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias), + norm_layer(ngf), + nn.ReLU(True) + ] + + n_downsampling = 2 + for i in range(n_downsampling): # add downsampling layers + mult = 2**i + model += [ + nn.Conv2d( + ngf * mult, + ngf * mult * 2, + kernel_size=3, + stride=2, + padding=1, + bias=use_bias), + norm_layer(ngf * mult * 2), + nn.ReLU(True) + ] + + mult = 2**n_downsampling + for i in range(n_blocks): # add ResNet blocks + + model += [ + ResnetBlock( + ngf * mult, + padding_type=padding_type, + norm_layer=norm_layer, + use_dropout=use_dropout, + use_bias=use_bias) + ] + + for i in range(n_downsampling): # add upsampling layers + mult = 2**(n_downsampling - i) + model += [ + nn.ConvTranspose2d( + ngf * mult, + int(ngf * mult / 2), + kernel_size=3, + stride=2, + padding=1, + output_padding=1, + bias=use_bias), + norm_layer(int(ngf * mult / 2)), + nn.ReLU(True) + ] + model += [nn.ReflectionPad2d(3)] + model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + model += [nn.Tanh()] + + self.model = nn.Sequential(*model) + + def forward(self, input): + """Standard forward""" + return self.model(input) + + +class ResnetBlock(nn.Module): + """Define a Resnet block""" + + def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias): + """Initialize the Resnet block + + A resnet block is a conv block with skip connections + We construct a conv block with build_conv_block function, + and implement skip connections in function. + Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf + """ + super(ResnetBlock, self).__init__() + self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, + use_dropout, use_bias) + + def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, + use_bias): + """Construct a convolutional block. + + Parameters: + dim (int) -- the number of channels in the conv layer. + padding_type (str) -- the name of padding layer: reflect | replicate | zero + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers. + use_bias (bool) -- if the conv layer uses bias or not + + Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU)) + """ + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' + % padding_type) + + conv_block += [ + nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), + norm_layer(dim), + nn.ReLU(True) + ] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' + % padding_type) + conv_block += [ + nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), + norm_layer(dim) + ] + + return nn.Sequential(*conv_block) + + def forward(self, x): + """Forward function (with skip connections)""" + out = x + self.conv_block(x) # add skip connections + return out + + +class UnetGenerator(nn.Module): + """Create a Unet-based generator""" + + def __init__(self, + input_nc, + output_nc, + num_downs, + ngf=64, + norm_layer=nn.BatchNorm2d, + use_dropout=False): + """Construct a Unet generator + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7, + image of size 128x128 will become of size 1x1 # at the bottleneck + ngf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + + We construct the U-Net from the innermost layer to the outermost layer. + It is a recursive process. + """ + super(UnetGenerator, self).__init__() + # construct unet structure + unet_block = UnetSkipConnectionBlock( + ngf * 8, + ngf * 8, + input_nc=None, + submodule=None, + norm_layer=norm_layer, + innermost=True) # add the innermost layer + for i in range(num_downs + - 5): # add intermediate layers with ngf * 8 filters + unet_block = UnetSkipConnectionBlock( + ngf * 8, + ngf * 8, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer, + use_dropout=use_dropout) + # gradually reduce the number of filters from ngf * 8 to ngf + unet_block = UnetSkipConnectionBlock( + ngf * 4, + ngf * 8, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock( + ngf * 2, + ngf * 4, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock( + ngf, + ngf * 2, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + self.model = UnetSkipConnectionBlock( + output_nc, + ngf, + input_nc=input_nc, + submodule=unet_block, + outermost=True, + norm_layer=norm_layer) # add the outermost layer + + def forward(self, input): + """Standard forward""" + return self.model(input) + + +class UnetSkipConnectionBlock(nn.Module): + """Defines the Unet submodule with skip connection. + X -------------------identity---------------------- + |-- downsampling -- |submodule| -- upsampling --| + """ + + def __init__(self, + outer_nc, + inner_nc, + input_nc=None, + submodule=None, + outermost=False, + innermost=False, + norm_layer=nn.BatchNorm2d, + use_dropout=False): + """Construct a Unet submodule with skip connections. + + Parameters: + outer_nc (int) -- the number of filters in the outer conv layer + inner_nc (int) -- the number of filters in the inner conv layer + input_nc (int) -- the number of channels in input images/features + submodule (UnetSkipConnectionBlock) -- previously defined submodules + outermost (bool) -- if this module is the outermost module + innermost (bool) -- if this module is the innermost module + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers. + """ + super(UnetSkipConnectionBlock, self).__init__() + self.outermost = outermost + if type(norm_layer) == functools.partial: + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + if input_nc is None: + input_nc = outer_nc + downconv = nn.Conv2d( + input_nc, + inner_nc, + kernel_size=4, + stride=2, + padding=1, + bias=use_bias) + downrelu = nn.LeakyReLU(0.2, True) + downnorm = norm_layer(inner_nc) + uprelu = nn.ReLU(True) + upnorm = norm_layer(outer_nc) + + if outermost: + upconv = nn.ConvTranspose2d( + inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1) + down = [downconv] + up = [uprelu, upconv, nn.Tanh()] + model = down + [submodule] + up + elif innermost: + upconv = nn.ConvTranspose2d( + inner_nc, + outer_nc, + kernel_size=4, + stride=2, + padding=1, + bias=use_bias) + down = [downrelu, downconv] + up = [uprelu, upconv, upnorm] + model = down + up + else: + upconv = nn.ConvTranspose2d( + inner_nc * 2, + outer_nc, + kernel_size=4, + stride=2, + padding=1, + bias=use_bias) + down = [downrelu, downconv, downnorm] + up = [uprelu, upconv, upnorm] + + if use_dropout: + model = down + [submodule] + up + [nn.Dropout(0.5)] + else: + model = down + [submodule] + up + + self.model = nn.Sequential(*model) + + def forward(self, x): + if self.outermost: + return self.model(x) + else: # add skip connections + return torch.cat([x, self.model(x)], 1) + + +class NLayerDiscriminator(nn.Module): + """Defines a PatchGAN discriminator""" + + def __init__(self, + input_nc, + ndf=64, + n_layers=3, + norm_layer=nn.BatchNorm2d): + """Construct a PatchGAN discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + n_layers (int) -- the number of conv layers in the discriminator + norm_layer -- normalization layer + """ + super(NLayerDiscriminator, self).__init__() + if type( + norm_layer + ) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + kw = 4 + padw = 1 + sequence = [ + nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), + nn.LeakyReLU(0.2, True) + ] + nf_mult = 1 + nf_mult_prev = 1 + for n in range(1, + n_layers): # gradually increase the number of filters + nf_mult_prev = nf_mult + nf_mult = min(2**n, 8) + sequence += [ + nn.Conv2d( + ndf * nf_mult_prev, + ndf * nf_mult, + kernel_size=kw, + stride=2, + padding=padw, + bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + nf_mult_prev = nf_mult + nf_mult = min(2**n_layers, 8) + sequence += [ + nn.Conv2d( + ndf * nf_mult_prev, + ndf * nf_mult, + kernel_size=kw, + stride=1, + padding=padw, + bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + sequence += [ + nn.Conv2d( + ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw) + ] # output 1 channel prediction map + self.model = nn.Sequential(*sequence) + + def forward(self, input): + """Standard forward.""" + return self.model(input) + + +class PixelDiscriminator(nn.Module): + """Defines a 1x1 PatchGAN discriminator (pixelGAN)""" + + def __init__(self, input_nc, ndf=64, norm_layer=nn.BatchNorm2d): + """Construct a 1x1 PatchGAN discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + """ + super(PixelDiscriminator, self).__init__() + if type( + norm_layer + ) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + self.net = [ + nn.Conv2d(input_nc, ndf, kernel_size=1, stride=1, padding=0), + nn.LeakyReLU(0.2, True), + nn.Conv2d( + ndf, + ndf * 2, + kernel_size=1, + stride=1, + padding=0, + bias=use_bias), + norm_layer(ndf * 2), + nn.LeakyReLU(0.2, True), + nn.Conv2d( + ndf * 2, 1, kernel_size=1, stride=1, padding=0, bias=use_bias) + ] + + self.net = nn.Sequential(*self.net) + + def forward(self, input): + """Standard forward.""" + return self.net(input) diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_model.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_model.py new file mode 100644 index 0000000000000000000000000000000000000000..54768fc1c28cd4385e539bdc2fc7793ce03144e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_model.py @@ -0,0 +1,125 @@ +# Part of the implementation is borrowed and modified from pix2pix, +# publicly available at https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix +import torch +from torch import nn + +from . import networks + + +class Pix2PixModel(nn.Module): + """ This class implements the pix2pix model, for learning a mapping from input + images to output images given paired data. + + The model training requires '--dataset_mode aligned' dataset. + By default, it uses a '--netG unet256' U-Net generator, + a '--netD basic' discriminator (PatchGAN), + and a '--gan_mode' vanilla GAN loss (the cross-entropy objective used in the orignal GAN paper). + + pix2pix paper: https://arxiv.org/pdf/1611.07004.pdf + """ + + def __init__(self, opt): + """Initialize the pix2pix class. + + Parameters: + opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + super(Pix2PixModel, self).__init__() + # specify the training losses you want to print out. + # The training/test scripts will call + self.loss_names = ['G_GAN', 'G_L1', 'D_real', 'D_fake'] + # specify the images you want to save/display. + # The training/test scripts will call + self.visual_names = ['real_A', 'fake_B', 'real_B'] + # specify the models you want to save to the disk. The training/test scripts will call + # and + if opt.isTrain: + self.model_names = ['G', 'D'] + else: # during test time, only load G + self.model_names = ['G'] + # define networks (both generator and discriminator) + self.netG = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf, + opt.netG, opt.norm, not opt.no_dropout, + opt.init_type, opt.init_gain, + opt.gpu_ids) + # self.netG = UNet(opt.input_nc, opt.output_nc) + + # define a discriminator; conditional GANs need to take both input and output images; + # Therefore, #channels for D is input_nc + output_nc + if opt.isTrain: + self.netD = networks.define_D(opt.input_nc + opt.output_nc, + opt.ndf, opt.netD, opt.n_layers_D, + opt.norm, opt.init_type, + opt.init_gain, opt.gpu_ids) + + if opt.isTrain: + # define loss functions + self.criterionGAN = networks.GANLoss(opt.gan_mode).to(self.device) + self.criterionL1 = torch.nn.L1Loss() + # initialize optimizers; schedulers will be automatically created by function . + self.optimizer_G = torch.optim.Adam( + self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + self.optimizer_D = torch.optim.Adam( + self.netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + self.optimizers.append(self.optimizer_G) + self.optimizers.append(self.optimizer_D) + + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input (dict): include the data itself and its metadata information. + + The option 'direction' can be used to swap images in domain A and domain B. + """ + AtoB = self.opt.direction == 'AtoB' + self.real_A = input['A' if AtoB else 'B'].to(self.device) + self.real_B = input['B' if AtoB else 'A'].to(self.device) + self.image_paths = input['A_paths' if AtoB else 'B_paths'] + + def forward(self): + """Run forward pass; called by both functions and .""" + self.fake_B = self.netG(self.real_A) # G(A) + + def backward_D(self): + """Calculate GAN loss for the discriminator""" + # Fake; stop backprop to the generator by detaching fake_B + fake_AB = torch.cat( + (self.real_A, self.fake_B), 1 + ) # we use conditional GANs; we need to feed both input and output to the discriminator + pred_fake = self.netD(fake_AB.detach()) + self.loss_D_fake = self.criterionGAN(pred_fake, False) + # Real + real_AB = torch.cat((self.real_A, self.real_B), 1) + pred_real = self.netD(real_AB) + self.loss_D_real = self.criterionGAN(pred_real, True) + # combine loss and calculate gradients + self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5 + self.loss_D.backward() + + def backward_G(self): + """Calculate GAN and L1 loss for the generator""" + # First, G(A) should fake the discriminator + fake_AB = torch.cat((self.real_A, self.fake_B), 1) + pred_fake = self.netD(fake_AB) + self.loss_G_GAN = self.criterionGAN(pred_fake, True) + # Second, G(A) = B + self.loss_G_L1 = self.criterionL1(self.fake_B, + self.real_B) * self.opt.lambda_L1 + # combine loss and calculate gradients + self.loss_G = self.loss_G_GAN + self.loss_G_L1 + self.loss_G.backward() + + def optimize_parameters(self): + self.forward() # compute fake images: G(A) + # update D + self.set_requires_grad(self.netD, True) # enable backprop for D + self.optimizer_D.zero_grad() # set D's gradients to zero + self.backward_D() # calculate gradients for D + self.optimizer_D.step() # update D's weights + # update G + self.set_requires_grad( + self.netD, False) # D requires no gradients when optimizing G + self.optimizer_G.zero_grad() # set G's gradients to zero + self.backward_G() # calculate graidents for G + self.optimizer_G.step() # udpate G's weights diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_options.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_options.py new file mode 100644 index 0000000000000000000000000000000000000000..27eb4c074ed24ab4e1b9ea626e413450d0650e96 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/pix2pix/pix2pix_options.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +class Pix2PixOptions(): + + def __init__(self): + self.gpu_ids = [] + self.input_nc = 3 + self.output_nc = 3 + self.ngf = 64 + self.ndf = 64 + self.netG = 'resnet_9blocks' + self.netD = 'basic' + self.norm = 'instance' + self.no_dropout = False + self.init_type = 'normal' + self.init_gain = 0.02 + self.n_layers_D = 3 + self.gan_mode = 'lsgan' + self.lr = 0.0002 + self.beta1 = 0.5 + self.isTrain = False + self.checkpoints_dir = './pix2pix_checkpoints' + self.name = 'mid_net' + self.lr_policy = 'linear' + self.direction = 'AtoB' + self.lambda_L1 = 100.0 + self.preprocess = 'resize_and_crop' diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/renderer.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..d10fd560448f637dc64d5fddbc0675db9534e8e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/renderer.py @@ -0,0 +1,325 @@ +# Part of the implementation is borrowed and modified from pytorch3d, +# publicly available at https://github.com/facebookresearch/pytorch3d + +import imageio +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from skimage.io import imread + +from .. import utils +from ..utils import read_obj + + +def set_rasterizer(): + global Meshes, load_obj, rasterize_meshes + from pytorch3d.structures import Meshes + from pytorch3d.io import load_obj + from pytorch3d.renderer.mesh import rasterize_meshes + + +class Pytorch3dRasterizer(nn.Module): + # TODO: add support for rendering non-squared images, since pytorc3d supports this now + """ Borrowed from https://github.com/facebookresearch/pytorch3d + Notice: + x,y,z are in image space, normalized + can only render squared image now + """ + + def __init__(self, image_size=224): + """ + use fixed raster_settings for rendering faces + """ + super().__init__() + raster_settings = { + 'image_size': image_size, + 'blur_radius': 0.0, + 'faces_per_pixel': 1, + 'bin_size': None, + 'max_faces_per_bin': None, + 'perspective_correct': False, + } + raster_settings = utils.dict2obj(raster_settings) + self.raster_settings = raster_settings + + def forward(self, vertices, faces, attributes=None, h=None, w=None): + fixed_vertices = vertices.clone() + fixed_vertices[..., :2] = -fixed_vertices[..., :2] + raster_settings = self.raster_settings + if h is None and w is None: + image_size = raster_settings.image_size + else: + image_size = [h, w] + if h > w: + fixed_vertices[..., 1] = fixed_vertices[..., 1] * h / w + else: + fixed_vertices[..., 0] = fixed_vertices[..., 0] * w / h + + meshes_screen = Meshes( + verts=fixed_vertices.float(), faces=faces.long()) + pix_to_face, zbuf, bary_coords, dists = rasterize_meshes( + meshes_screen, + image_size=image_size, + blur_radius=raster_settings.blur_radius, + faces_per_pixel=raster_settings.faces_per_pixel, + bin_size=raster_settings.bin_size, + max_faces_per_bin=raster_settings.max_faces_per_bin, + perspective_correct=raster_settings.perspective_correct, + ) + vismask = (pix_to_face > -1).float() + D = attributes.shape[-1] + attributes = attributes.clone() + attributes = attributes.view(attributes.shape[0] * attributes.shape[1], + 3, attributes.shape[-1]) + N, H, W, K, _ = bary_coords.shape + mask = pix_to_face == -1 + pix_to_face = pix_to_face.clone() + pix_to_face[mask] = 0 + idx = pix_to_face.view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) + pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) + pixel_vals = (bary_coords[..., None] * pixel_face_vals).sum(dim=-2) + pixel_vals[mask] = 0 # Replace masked values in output. + pixel_vals = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) + pixel_vals = torch.cat( + [pixel_vals, vismask[:, :, :, 0][:, None, :, :]], dim=1) + return pixel_vals + + +class SRenderY(nn.Module): + + def __init__(self, image_size, obj_filename, uvcoords_path, uv_size=256): + super(SRenderY, self).__init__() + self.image_size = image_size + self.uv_size = uv_size + + self.rasterizer = Pytorch3dRasterizer(image_size) + self.uv_rasterizer = Pytorch3dRasterizer(uv_size) + + mesh = read_obj(obj_filename) + uvcoords = np.load(uvcoords_path)[None, ...] + uvcoords = torch.from_numpy(uvcoords) + verts = mesh['vertices'] + verts = torch.from_numpy(verts) + uvfaces = mesh['faces'][None, ...] - 1 + uvfaces = torch.from_numpy(uvfaces) + faces = mesh['faces'][None, ...] - 1 + faces = torch.from_numpy(faces) + + # faces + dense_triangles = utils.generate_triangles(uv_size, uv_size) + self.register_buffer( + 'dense_faces', + torch.from_numpy(dense_triangles).long()[None, :, :]) + self.register_buffer('faces', faces) + self.register_buffer('raw_uvcoords', uvcoords) + + # uv coords + uvcoords = torch.cat([uvcoords, uvcoords[:, :, 0:1] * 0. + 1.], + -1) # [bz, ntv, 3] + uvcoords = uvcoords * 2 - 1 + uvcoords[..., 1] = -uvcoords[..., 1] + face_uvcoords = utils.face_vertices(uvcoords, uvfaces) + self.register_buffer('uvcoords', uvcoords) + self.register_buffer('uvfaces', uvfaces) + self.register_buffer('face_uvcoords', face_uvcoords) + + # shape colors, for rendering shape overlay + colors = torch.tensor([180, 180, 180])[None, None, :].repeat( + 1, + faces.max() + 1, 1).float() / 255. + face_colors = utils.face_vertices(colors, faces) + self.register_buffer('face_colors', face_colors) + + # SH factors for lighting + pi = np.pi + value_1 = 1 / np.sqrt(4 * pi) + value_2 = ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))) + value_3 = ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))) + value_4 = ((2 * pi) / 3) * (np.sqrt(3 / (4 * pi))) + value_5 = (pi / 4) * 3 * (np.sqrt(5 / (12 * pi))) + value_6 = (pi / 4) * 3 * (np.sqrt(5 / (12 * pi))) + value_7 = (pi / 4) * 3 * (np.sqrt(5 / (12 * pi))) + value_8 = (pi / 4) * (3 / 2) * (np.sqrt(5 / (12 * pi))) + value_9 = (pi / 4) * (1 / 2) * (np.sqrt(5 / (4 * pi))) + constant_factor = torch.tensor([ + value_1, value_2, value_3, value_4, value_5, value_6, value_7, + value_8, value_9 + ]).float() + self.register_buffer('constant_factor', constant_factor) + + def forward(self, + vertices, + transformed_vertices, + albedos, + lights=None, + light_type='point'): + ''' + -- Texture Rendering + vertices: [batch_size, V, 3], vertices in world space, for calculating normals, then shading + transformed_vertices: [batch_size, V, 3], range:normalized to [-1,1], projected vertices in image space + (that is aligned to the iamge pixel), for rasterization + albedos: [batch_size, 3, h, w], uv map + lights: + spherical homarnic: [N, 9(shcoeff), 3(rgb)] + points/directional lighting: [N, n_lights, 6(xyzrgb)] + light_type: + point or directional + ''' + batch_size = vertices.shape[0] + # rasterizer near 0 far 100. move mesh so minz larger than 0 + transformed_vertices[:, :, 2] = transformed_vertices[:, :, 2] + 10 + # attributes + face_vertices = utils.face_vertices( + vertices, self.faces.expand(batch_size, -1, -1)) + normals = utils.vertex_normals(vertices, + self.faces.expand(batch_size, -1, -1)) + face_normals = utils.face_vertices( + normals, self.faces.expand(batch_size, -1, -1)) + transformed_normals = utils.vertex_normals( + transformed_vertices, self.faces.expand(batch_size, -1, -1)) + transformed_face_normals = utils.face_vertices( + transformed_normals, self.faces.expand(batch_size, -1, -1)) + + attributes = torch.cat([ + self.face_uvcoords.expand(batch_size, -1, -1, -1), + transformed_face_normals.detach(), + face_vertices.detach(), face_normals + ], -1) + # rasterize + rendering = self.rasterizer(transformed_vertices, + self.faces.expand(batch_size, -1, -1), + attributes) + + #### + # vis mask + alpha_images = rendering[:, -1, :, :][:, None, :, :].detach() + + # albedo + uvcoords_images = rendering[:, :3, :, :] + grid = (uvcoords_images).permute(0, 2, 3, 1)[:, :, :, :2] + albedo_images = F.grid_sample(albedos, grid, align_corners=False) + + # visible mask for pixels with positive normal direction + transformed_normal_map = rendering[:, 3:6, :, :].detach() + pos_mask = (transformed_normal_map[:, 2:, :, :] < -0.05).float() + + # shading + normal_images = rendering[:, 9:12, :, :] + if lights is not None: + if lights.shape[1] == 9: + shading_images = self.add_SHlight(normal_images, lights) + else: + if light_type == 'point': + vertice_images = rendering[:, 6:9, :, :].detach() + shading = self.add_pointlight( + vertice_images.permute(0, 2, 3, + 1).reshape([batch_size, -1, 3]), + normal_images.permute(0, 2, 3, + 1).reshape([batch_size, -1, 3]), + lights) + shading_images = shading.reshape([ + batch_size, albedo_images.shape[2], + albedo_images.shape[3], 3 + ]).permute(0, 3, 1, 2) + else: + shading = self.add_directionlight( + normal_images.permute(0, 2, 3, + 1).reshape([batch_size, -1, 3]), + lights) + shading_images = shading.reshape([ + batch_size, albedo_images.shape[2], + albedo_images.shape[3], 3 + ]).permute(0, 3, 1, 2) + images = albedo_images * shading_images + else: + images = albedo_images + shading_images = images.detach() * 0. + + outputs = { + 'images': images * alpha_images, + 'albedo_images': albedo_images * alpha_images, + 'alpha_images': alpha_images, + 'pos_mask': pos_mask, + 'shading_images': shading_images, + 'grid': grid, + 'normals': normals, + 'normal_images': normal_images * alpha_images, + 'transformed_normals': transformed_normals, + } + + return outputs + + def add_SHlight(self, normal_images, gamma, init_lit): + ''' + sh_coeff: [bz, 9, 3] + ''' + batch_size = gamma.shape[0] + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + init_lit + sh_coeff = gamma.permute(0, 2, 1) + + N = normal_images + tmp_value = 3 * (N[:, 2]**2) - 1 + sh = torch.stack([ + N[:, 0] * 0. + 1., N[:, 0], N[:, 1], N[:, 2], N[:, 0] * N[:, 1], + N[:, 0] * N[:, 2], N[:, 1] * N[:, 2], N[:, 0]**2 - N[:, 1]**2, + tmp_value + ], 1) # [bz, 9, h, w] + sh = sh * self.constant_factor[None, :, None, None] + shading = torch.sum(sh_coeff[:, :, :, None, None] + * sh[:, :, None, :, :], 1) # [bz, 9, 3, h, w] + return shading + + def add_pointlight(self, vertices, normals, lights): + ''' + vertices: [bz, nv, 3] + lights: [bz, nlight, 6] + returns: + shading: [bz, nv, 3] + ''' + light_positions = lights[:, :, :3] + light_intensities = lights[:, :, 3:] + directions_to_lights = F.normalize( + light_positions[:, :, None, :] - vertices[:, None, :, :], dim=3) + # normals_dot_lights = torch.clamp((normals[:,None,:,:]*directions_to_lights).sum(dim=3), 0., 1.) + normals_dot_lights = (normals[:, None, :, :] + * directions_to_lights).sum(dim=3) + shading = normals_dot_lights[:, :, :, + None] * light_intensities[:, :, None, :] + return shading.mean(1) + + def add_directionlight(self, normals, lights): + ''' + normals: [bz, nv, 3] + lights: [bz, nlight, 6] + returns: + shading: [bz, nv, 3] + ''' + light_direction = lights[:, :, :3] + light_intensities = lights[:, :, 3:] + directions_to_lights = F.normalize( + light_direction[:, :, None, :].expand(-1, -1, normals.shape[1], + -1), + dim=3) + # normals_dot_lights = torch.clamp((normals[:,None,:,:]*directions_to_lights).sum(dim=3), 0., 1.) + # normals_dot_lights = (normals[:,None,:,:]*directions_to_lights).sum(dim=3) + normals_dot_lights = torch.clamp( + (normals[:, None, :, :] * directions_to_lights).sum(dim=3), 0., 1.) + shading = normals_dot_lights[:, :, :, + None] * light_intensities[:, :, None, :] + return shading.mean(1) + + def world2uv(self, vertices): + ''' + warp vertices from world space to uv space + vertices: [bz, V, 3] + uv_vertices: [bz, 3, h, w] + ''' + batch_size = vertices.shape[0] + face_vertices = utils.face_vertices( + vertices, self.faces.expand(batch_size, -1, -1)) + uv_vertices = self.uv_rasterizer( + self.uvcoords.expand(batch_size, -1, -1), + self.uvfaces.expand(batch_size, -1, -1), face_vertices)[:, :3] + return uv_vertices diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/models/unet.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..3ceed26d71a610e9f19053a7f0a26c725722bbeb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/models/unet.py @@ -0,0 +1,173 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F + +warnings.filterwarnings(action='ignore') + + +def weights_init(init_type='kaiming', gain=0.02): + + def init_func(m): + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + + if init_type == 'normal': + nn.init.normal_(m.weight.data, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight.data, gain=gain) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight.data, gain=gain) + + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias.data, 0.0) + + elif classname.find('BatchNorm2d') != -1: + nn.init.normal_(m.weight.data, 1.0, gain) + nn.init.constant_(m.bias.data, 0.0) + + return init_func + + +class double_conv(nn.Module): + '''(conv => BN => ReLU) * 2''' + + def __init__(self, in_ch, out_ch): + super(double_conv, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(in_ch, out_ch, 3, padding=1), + nn.BatchNorm2d(out_ch), + # nn.InstanceNorm2d(out_ch), + nn.ReLU(inplace=True), + nn.Conv2d(out_ch, out_ch, 3, padding=1), + nn.BatchNorm2d(out_ch), + # nn.InstanceNorm2d(out_ch), + nn.ReLU(inplace=True)) + + def forward(self, x): + x = self.conv(x) + return x + + +class inconv(nn.Module): + + def __init__(self, in_ch, out_ch): + super(inconv, self).__init__() + self.conv = double_conv(in_ch, out_ch) + + def forward(self, x): + x = self.conv(x) + return x + + +class down(nn.Module): + + def __init__(self, in_ch, out_ch): + super(down, self).__init__() + self.mpconv = nn.Sequential( + nn.MaxPool2d(2), double_conv(in_ch, out_ch)) + + def forward(self, x): + x = self.mpconv(x) + return x + + +class up(nn.Module): + + def __init__(self, in_ch, out_ch, bilinear=True): + super(up, self).__init__() + + if bilinear: + self.up = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=True) + else: + self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2) + + self.conv = double_conv(in_ch, out_ch) + + def forward(self, x1, x2): + x1 = self.up(x1) + + diffY = x2.size()[2] - x1.size()[2] + diffX = x2.size()[3] - x1.size()[3] + + x1 = F.pad( + x1, + (diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2)) + + x = torch.cat([x2, x1], dim=1) + x = self.conv(x) + return x + + +class outconv(nn.Module): + + def __init__(self, in_ch, out_ch): + super(outconv, self).__init__() + self.conv = nn.Conv2d(in_ch, out_ch, 1) + + def forward(self, x): + x = self.conv(x) + return x + + +class UNet(nn.Module): + + def __init__(self, + n_channels, + n_classes, + deep_supervision=False, + init_weights=True): + super(UNet, self).__init__() + self.deep_supervision = deep_supervision + self.inc = inconv(n_channels, 64) + self.down1 = down(64, 128) + self.down2 = down(128, 256) + self.down3 = down(256, 512) + self.down4 = down(512, 512) + self.up1 = up(1024, 256) + self.up2 = up(512, 128) + self.up3 = up(256, 64) + self.up4 = up(128, 64) + self.outc = outconv(64, n_classes) + + self.dsoutc4 = outconv(256, n_classes) + self.dsoutc3 = outconv(128, n_classes) + self.dsoutc2 = outconv(64, n_classes) + self.dsoutc1 = outconv(64, n_classes) + + self.sigmoid = nn.Sigmoid() + + if init_weights: + self.apply(weights_init()) + + def forward(self, x): + x1 = self.inc(x) + x2 = self.down1(x1) + x3 = self.down2(x2) + x4 = self.down3(x3) + x5 = self.down4(x4) + x44 = self.up1(x5, x4) + x33 = self.up2(x44, x3) + x22 = self.up3(x33, x2) + x11 = self.up4(x22, x1) + x0 = self.outc(x11) + x0 = self.sigmoid(x0) + if self.deep_supervision: + x11 = F.interpolate( + self.dsoutc1(x11), x0.shape[2:], mode='bilinear') + x22 = F.interpolate( + self.dsoutc2(x22), x0.shape[2:], mode='bilinear') + x33 = F.interpolate( + self.dsoutc3(x33), x0.shape[2:], mode='bilinear') + x44 = F.interpolate( + self.dsoutc4(x44), x0.shape[2:], mode='bilinear') + + return x0, x11, x22, x33, x44 + else: + return x0 diff --git a/runtime/Lib/site_packages/models/cv/face_reconstruction/utils.py b/runtime/Lib/site_packages/models/cv/face_reconstruction/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f23b2f707cb5a3b795462f14cbf6eb3ff60ec806 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/face_reconstruction/utils.py @@ -0,0 +1,1001 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import argparse +import math +import os +import os.path as osp +from array import array + +import cv2 +import numba +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from scipy.io import loadmat, savemat + + +def img_value_rescale(img, old_range: list, new_range: list): + assert len(old_range) == 2 + assert len(new_range) == 2 + img = (img - old_range[0]) / (old_range[1] - old_range[0]) * ( + new_range[1] - new_range[0]) + new_range[0] + return img + + +def resize_on_long_side(img, long_side=800): + src_height = img.shape[0] + src_width = img.shape[1] + + if src_height > src_width: + scale = long_side * 1.0 / src_height + _img = cv2.resize( + img, (int(src_width * scale), long_side), + interpolation=cv2.INTER_CUBIC) + + else: + scale = long_side * 1.0 / src_width + _img = cv2.resize( + img, (long_side, int(src_height * scale)), + interpolation=cv2.INTER_CUBIC) + + return _img, scale + + +def get_mg_layer(src, gt, skin_mask=None): + """ + src, gt shape: [h, w, 3] value: [0, 1] + return: mg, shape: [h, w, 1] value: [0, 1] + """ + mg = (src * src - gt + 1e-10) / (2 * src * src - 2 * src + 2e-10) + mg[mg < 0] = 0.5 + mg[mg > 1] = 0.5 + + diff_abs = np.abs(gt - src) + mg[diff_abs < (1 / 255.0)] = 0.5 + + if skin_mask is not None: + mg[skin_mask == 0] = 0.5 + + return mg + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +def spread_flow(length, spread_ratio=2): + Flow = np.zeros(shape=(length, length, 2), dtype=np.float32) + mag = np.zeros(shape=(length, length), dtype=np.float32) + + radius = length * 0.5 + for h in range(Flow.shape[0]): + for w in range(Flow.shape[1]): + + if (h - length // 2)**2 + (w - length // 2)**2 <= radius**2: + Flow[h, w, 0] = -(w - length // 2) + Flow[h, w, 1] = -(h - length // 2) + + distance = np.sqrt((w - length // 2)**2 + (h - length // 2)**2) + + if distance <= radius / 2.0: + mag[h, w] = 2.0 / radius * distance + else: + mag[h, w] = -2.0 / radius * distance + 2.0 + + _, ang = cv2.cartToPolar(Flow[..., 0] + 1e-8, Flow[..., 1] + 1e-8) + + mag *= spread_ratio + + x, y = cv2.polarToCart(mag, ang, angleInDegrees=False) + Flow = np.dstack((x, y)) + + return Flow + + +@numba.jit(nopython=True, parallel=True) +def bilinear_interp(x, y, v11, v12, v21, v22): + t = 0.2 + + if x < t and y < t: + return v11 + elif x < t and y > 1 - t: + return v12 + elif x > 1 - t and y < t: + return v21 + elif x > 1 - t and y > 1 - t: + return v22 + else: + result = (v11 * (1 - y) + v12 * y) * (1 - x) + \ + (v21 * (1 - y) + v22 * y) * x + if result < 0: + result = 0 + + if result > 255: + result = 255 + return result + + +@numba.jit(nopython=True, parallel=True) +def image_warp_grid1(rDx, rDy, oriImg, transRatio, pads): + # assert oriImg.dtype == np.uint8 + srcW = oriImg.shape[1] + srcH = oriImg.shape[0] + + padTop, padBottom, padLeft, padRight = pads + + left_bound = padLeft + 1 + right_bound = srcW - padRight + bottom_bound = srcH - padBottom + top_bound = padTop + 1 + + newImg = oriImg.copy() + + for i in range(srcH): + for j in range(srcW): + _i = i + _j = j + + deltaX = rDx[_i, _j] + deltaY = rDy[_i, _j] + + if abs(deltaX) < 0.2 and abs(deltaY) < 0.2: + continue + + nx = _j + deltaX * transRatio + ny = _i + deltaY * transRatio + + if nx >= srcW - padRight: + if nx > srcW - 1: + nx = srcW - 1 + + if _j < right_bound: + right_bound = _j + + if ny >= srcH - padBottom: + if ny > srcH - 1: + ny = srcH - 1 + + if _i < bottom_bound: + bottom_bound = _i + + if nx < padLeft: + if nx < 0: + nx = 0 + + if _j + 1 > left_bound: + left_bound = _j + 1 + + if ny < padTop: + if ny < 0: + ny = 0 + + if _i + 1 > top_bound: + top_bound = _i + 1 + + nxi = int(math.floor(nx)) + nyi = int(math.floor(ny)) + nxi1 = int(math.ceil(nx)) + nyi1 = int(math.ceil(ny)) + + if nxi < 0: + nxi = 0 + if nxi > oriImg.shape[1] - 1: + nxi = oriImg.shape[1] - 1 + + if nxi1 < 0: + nxi1 = 0 + if nxi1 > oriImg.shape[1] - 1: + nxi1 = oriImg.shape[1] - 1 + + if nyi < 0: + nyi = 0 + if nyi > oriImg.shape[0] - 1: + nyi = oriImg.shape[0] - 1 + + if nyi1 < 0: + nyi1 = 0 + if nyi1 > oriImg.shape[0] - 1: + nyi1 = oriImg.shape[0] - 1 + + for ll in range(3): + newImg[_i, _j, + ll] = bilinear_interp(ny - nyi, nx - nxi, + oriImg[nyi, nxi, + ll], oriImg[nyi, nxi1, ll], + oriImg[nyi1, nxi, + ll], oriImg[nyi1, nxi1, + ll]) + + return newImg, top_bound, bottom_bound, left_bound, right_bound + + +def warp(x, flow, mode='bilinear', padding_mode='zeros', coff=0.1): + """ + + Args: + x: [n, c, h, w] + flow: [n, h, w, 2] + mode: + padding_mode: + coff: + + Returns: + + """ + n, c, h, w = x.size() + yv, xv = torch.meshgrid([torch.arange(h), torch.arange(w)]) + xv = xv.float() / (w - 1) * 2.0 - 1 + yv = yv.float() / (h - 1) * 2.0 - 1 + ''' + grid[0,:,:,0] = + -1, .....1 + -1, .....1 + -1, .....1 + + grid[0,:,:,1] = + -1, -1, -1 + ; ; + 1, 1, 1 + + ''' + + if torch.cuda.is_available(): + grid = torch.cat((xv.unsqueeze(-1), yv.unsqueeze(-1)), + -1).unsqueeze(0).cuda() + else: + grid = torch.cat((xv.unsqueeze(-1), yv.unsqueeze(-1)), -1).unsqueeze(0) + grid_x = grid + 2 * flow * coff + warp_x = F.grid_sample(x, grid_x, mode=mode, padding_mode=padding_mode) + return warp_x + + +# load expression basis +def LoadExpBasis(bfm_folder='asset/BFM'): + n_vertex = 53215 + Expbin = open(osp.join(bfm_folder, 'Exp_Pca.bin'), 'rb') + exp_dim = array('i') + exp_dim.fromfile(Expbin, 1) + expMU = array('f') + expPC = array('f') + expMU.fromfile(Expbin, 3 * n_vertex) + expPC.fromfile(Expbin, 3 * exp_dim[0] * n_vertex) + Expbin.close() + + expPC = np.array(expPC) + expPC = np.reshape(expPC, [exp_dim[0], -1]) + expPC = np.transpose(expPC) + + expEV = np.loadtxt(osp.join(bfm_folder, 'std_exp.txt')) + + return expPC, expEV + + +# transfer original BFM09 to our face model +def transferBFM09(bfm_folder='BFM'): + print('Transfer BFM09 to BFM_model_front......') + original_BFM = loadmat(osp.join(bfm_folder, '01_MorphableModel.mat')) + shapePC = original_BFM['shapePC'] # shape basis, 160470*199 + shapeEV = original_BFM['shapeEV'] # corresponding eigen value, 199*1 + shapeMU = original_BFM['shapeMU'] # mean face, 160470*1 + texPC = original_BFM['texPC'] # texture basis, 160470*199 + texEV = original_BFM['texEV'] # eigen value, 199*1 + texMU = original_BFM['texMU'] # mean texture, 160470*1 + + expPC, expEV = LoadExpBasis() + + # transfer BFM09 to our face model + + idBase = shapePC * np.reshape(shapeEV, [-1, 199]) + idBase = idBase / 1e5 # unify the scale to decimeter + idBase = idBase[:, :80] # use only first 80 basis + + exBase = expPC * np.reshape(expEV, [-1, 79]) + exBase = exBase / 1e5 # unify the scale to decimeter + exBase = exBase[:, :64] # use only first 64 basis + + texBase = texPC * np.reshape(texEV, [-1, 199]) + texBase = texBase[:, :80] # use only first 80 basis + + # our face model is cropped along face landmarks and contains only 35709 vertex. + # original BFM09 contains 53490 vertex, and expression basis provided by Guo et al. contains 53215 vertex. + # thus we select corresponding vertex to get our face model. + + index_exp = loadmat(osp.join(bfm_folder, 'BFM_front_idx.mat')) + index_exp = index_exp['idx'].astype( + np.int32) - 1 # starts from 0 (to 53215) + + index_shape = loadmat(osp.join(bfm_folder, 'BFM_exp_idx.mat')) + index_shape = index_shape['trimIndex'].astype( + np.int32) - 1 # starts from 0 (to 53490) + index_shape = index_shape[index_exp] + + idBase = np.reshape(idBase, [-1, 3, 80]) + idBase = idBase[index_shape, :, :] + idBase = np.reshape(idBase, [-1, 80]) + + texBase = np.reshape(texBase, [-1, 3, 80]) + texBase = texBase[index_shape, :, :] + texBase = np.reshape(texBase, [-1, 80]) + + exBase = np.reshape(exBase, [-1, 3, 64]) + exBase = exBase[index_exp, :, :] + exBase = np.reshape(exBase, [-1, 64]) + + meanshape = np.reshape(shapeMU, [-1, 3]) / 1e5 + meanshape = meanshape[index_shape, :] + meanshape = np.reshape(meanshape, [1, -1]) + + meantex = np.reshape(texMU, [-1, 3]) + meantex = meantex[index_shape, :] + meantex = np.reshape(meantex, [1, -1]) + + # other info contains triangles, region used for computing photometric loss, + # region used for skin texture regularization, and 68 landmarks index etc. + other_info = loadmat(osp.join(bfm_folder, 'facemodel_info.mat')) + frontmask2_idx = other_info['frontmask2_idx'] + skinmask = other_info['skinmask'] + keypoints = other_info['keypoints'] + point_buf = other_info['point_buf'] + tri = other_info['tri'] + tri_mask2 = other_info['tri_mask2'] + + # save our face model + savemat( + osp.join(bfm_folder, 'BFM_model_front.mat'), { + 'meanshape': meanshape, + 'meantex': meantex, + 'idBase': idBase, + 'exBase': exBase, + 'texBase': texBase, + 'tri': tri, + 'point_buf': point_buf, + 'tri_mask2': tri_mask2, + 'keypoints': keypoints, + 'frontmask2_idx': frontmask2_idx, + 'skinmask': skinmask + }) + + +# load landmarks for standard face, which is used for image preprocessing +def load_lm3d(bfm_folder): + + Lm3D = loadmat(osp.join(bfm_folder, 'similarity_Lm3D_all.mat')) + Lm3D = Lm3D['lm'] + + # calculate 5 facial landmarks using 68 landmarks + lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 + value_list = [ + Lm3D[lm_idx[0], :], + np.mean(Lm3D[lm_idx[[1, 2]], :], 0), + np.mean(Lm3D[lm_idx[[3, 4]], :], 0), Lm3D[lm_idx[5], :], + Lm3D[lm_idx[6], :] + ] + Lm3D = np.stack(value_list, axis=0) + Lm3D = Lm3D[[1, 2, 0, 3, 4], :] + + return Lm3D + + +def mesh_to_string(mesh): + out_string = '' + out_string += '# Create by HRN\n' + + if 'colors' in mesh: + for i, v in enumerate(mesh['vertices']): + out_string += \ + 'v {:.6f} {:.6f} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + v[0], v[1], v[2], mesh['colors'][i][0], + mesh['colors'][i][1], mesh['colors'][i][2]) + else: + for v in mesh['vertices']: + out_string += 'v {:.6f} {:.6f} {:.6f}\n'.format(v[0], v[1], v[2]) + + if 'UVs' in mesh: + for uv in mesh['UVs']: + out_string += 'vt {:.6f} {:.6f}\n'.format(uv[0], uv[1]) + + if 'normals' in mesh: + for vn in mesh['normals']: + out_string += 'vn {:.6f} {:.6f} {:.6f}\n'.format( + vn[0], vn[1], vn[2]) + + if 'faces' in mesh: + for ind, face in enumerate(mesh['faces']): + if 'faces_uv' in mesh or 'faces_normal' in mesh or 'UVs' in mesh: + if 'faces_uv' in mesh: + face_uv = mesh['faces_uv'][ind] + else: + face_uv = face + if 'faces_normal' in mesh: + face_normal = mesh['faces_normal'][ind] + else: + face_normal = face + row = 'f ' + ' '.join([ + '{}/{}/{}'.format(face[i], face_uv[i], face_normal[i]) + for i in range(len(face)) + ]) + '\n' + else: + row = 'f ' + ' '.join( + ['{}'.format(face[i]) for i in range(len(face))]) + '\n' + out_string += row + + return out_string + + +def write_obj(save_path, mesh): + save_dir = os.path.dirname(save_path) + save_name = os.path.splitext(os.path.basename(save_path))[0] + + if 'texture_map' in mesh: + cv2.imwrite( + os.path.join(save_dir, save_name + '.jpg'), mesh['texture_map']) + + with open(os.path.join(save_dir, save_name + '.mtl'), 'w') as wf: + wf.write('# Created by HRN\n') + wf.write('newmtl material_0\n') + wf.write('Ka 1.000000 0.000000 0.000000\n') + wf.write('Kd 1.000000 1.000000 1.000000\n') + wf.write('Ks 0.000000 0.000000 0.000000\n') + wf.write('Tr 0.000000\n') + wf.write('illum 0\n') + wf.write('Ns 0.000000\n') + wf.write('map_Kd {}\n'.format(save_name + '.jpg')) + + with open(save_path, 'w') as wf: + if 'texture_map' in mesh: + wf.write('# Create by HRN\n') + wf.write('mtllib ./{}.mtl\n'.format(save_name)) + + if 'colors' in mesh: + for i, v in enumerate(mesh['vertices']): + wf.write( + 'v {:.6f} {:.6f} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + v[0], v[1], v[2], mesh['colors'][i][0], + mesh['colors'][i][1], mesh['colors'][i][2])) + else: + for v in mesh['vertices']: + wf.write('v {:.6f} {:.6f} {:.6f}\n'.format(v[0], v[1], v[2])) + + if 'UVs' in mesh: + for uv in mesh['UVs']: + wf.write('vt {:.6f} {:.6f}\n'.format(uv[0], uv[1])) + + if 'normals' in mesh: + for vn in mesh['normals']: + wf.write('vn {:.6f} {:.6f} {:.6f}\n'.format( + vn[0], vn[1], vn[2])) + + if 'faces' in mesh: + for ind, face in enumerate(mesh['faces']): + if 'faces_uv' in mesh or 'faces_normal' in mesh or 'UVs' in mesh: + if 'faces_uv' in mesh: + face_uv = mesh['faces_uv'][ind] + else: + face_uv = face + if 'faces_normal' in mesh: + face_normal = mesh['faces_normal'][ind] + else: + face_normal = face + row = 'f ' + ' '.join([ + '{}/{}/{}'.format(face[i], face_uv[i], face_normal[i]) + for i in range(len(face)) + ]) + '\n' + else: + row = 'f ' + ' '.join( + ['{}'.format(face[i]) + for i in range(len(face))]) + '\n' + wf.write(row) + + +def read_obj(obj_path, print_shape=True): + with open(obj_path, 'r') as f: + bfm_lines = f.readlines() + + vertices = [] + faces = [] + uvs = [] + vns = [] + faces_uv = [] + faces_normal = [] + max_face_length = 0 + for line in bfm_lines: + if line[:2] == 'v ': + vertex = [ + float(a) for a in line.strip().split(' ')[1:] if len(a) > 0 + ] + vertices.append(vertex) + + if line[:2] == 'f ': + items = line.strip().split(' ')[1:] + face = [int(a.split('/')[0]) for a in items if len(a) > 0] + max_face_length = max(max_face_length, len(face)) + if len(faces) > 0 and len(face) != len(faces[0]): + continue + faces.append(face) + + if '/' in items[0] and len(items[0].split('/')[1]) > 0: + face_uv = [int(a.split('/')[1]) for a in items if len(a) > 0] + faces_uv.append(face_uv) + + if '/' in items[0] and len(items[0].split('/')) >= 3 and len( + items[0].split('/')[2]) > 0: + face_normal = [ + int(a.split('/')[2]) for a in items if len(a) > 0 + ] + faces_normal.append(face_normal) + + if line[:3] == 'vt ': + items = line.strip().split(' ')[1:] + uv = [float(a) for a in items if len(a) > 0] + uvs.append(uv) + + if line[:3] == 'vn ': + items = line.strip().split(' ')[1:] + vn = [float(a) for a in items if len(a) > 0] + vns.append(vn) + + vertices = np.array(vertices).astype(np.float32) + if max_face_length <= 3: + faces = np.array(faces).astype(np.int32) + + if vertices.shape[1] == 3: + mesh = { + 'vertices': vertices, + 'faces': faces, + } + else: + mesh = { + 'vertices': vertices[:, :3], + 'colors': vertices[:, 3:], + 'faces': faces, + } + + if len(uvs) > 0: + uvs = np.array(uvs).astype(np.float32) + mesh['uvs'] = uvs + + if len(vns) > 0: + vns = np.array(vns).astype(np.float32) + mesh['vns'] = vns + + if len(faces_uv) > 0: + if max_face_length <= 3: + faces_uv = np.array(faces_uv).astype(np.int32) + mesh['faces_uv'] = faces_uv + + if len(faces_normal) > 0: + if max_face_length <= 3: + faces_normal = np.array(faces_normal).astype(np.int32) + mesh['faces_normal'] = faces_normal + + return mesh + + +# calculating least square problem for image alignment +def POS(xp, x): + npts = xp.shape[1] + + A = np.zeros([2 * npts, 8]) + + A[0:2 * npts - 1:2, 0:3] = x.transpose() + A[0:2 * npts - 1:2, 3] = 1 + + A[1:2 * npts:2, 4:7] = x.transpose() + A[1:2 * npts:2, 7] = 1 + + b = np.reshape(xp.transpose(), [2 * npts, 1]) + + k, _, _, _ = np.linalg.lstsq(A, b) + + R1 = k[0:3] + R2 = k[4:7] + sTx = k[3] + sTy = k[7] + s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2 + t = np.stack([sTx, sTy], axis=0) + + return t, s + + +# bounding box for 68 landmark detection +def BBRegression(points, params): + w1 = params['W1'] + b1 = params['B1'] + w2 = params['W2'] + b2 = params['B2'] + data = points.copy() + data = data.reshape([5, 2]) + data_mean = np.mean(data, axis=0) + x_mean = data_mean[0] + y_mean = data_mean[1] + data[:, 0] = data[:, 0] - x_mean + data[:, 1] = data[:, 1] - y_mean + + rms = np.sqrt(np.sum(data**2) / 5) + data = data / rms + data = data.reshape([1, 10]) + data = np.transpose(data) + inputs = np.matmul(w1, data) + b1 + inputs = 2 / (1 + np.exp(-2 * inputs)) - 1 + inputs = np.matmul(w2, inputs) + b2 + inputs = np.transpose(inputs) + x = inputs[:, 0] * rms + x_mean + y = inputs[:, 1] * rms + y_mean + w = 224 / inputs[:, 2] * rms + rects = [x, y, w, w] + return np.array(rects).reshape([4]) + + +# utils for landmark detection +def img_padding(img, box): + success = True + bbox = box.copy() + res = np.zeros([2 * img.shape[0], 2 * img.shape[1], 3]) + res[img.shape[0] // 2:img.shape[0] + img.shape[0] // 2, + img.shape[1] // 2:img.shape[1] + img.shape[1] // 2] = img + + bbox[0] = bbox[0] + img.shape[1] // 2 + bbox[1] = bbox[1] + img.shape[0] // 2 + if bbox[0] < 0 or bbox[1] < 0: + success = False + return res, bbox, success + + +# utils for landmark detection +def crop(img, bbox): + padded_img, padded_bbox, flag = img_padding(img, bbox) + if flag: + crop_img = padded_img[padded_bbox[1]:padded_bbox[1] + padded_bbox[3], + padded_bbox[0]:padded_bbox[0] + padded_bbox[2]] + crop_img = cv2.resize( + crop_img.astype(np.uint8), (224, 224), + interpolation=cv2.INTER_CUBIC) + scale = 224 / padded_bbox[3] + return crop_img, scale + else: + return padded_img, 0 + + +# utils for landmark detection +def scale_trans(img, lm, t, s): + imgw = img.shape[1] + imgh = img.shape[0] + M_s = np.array( + [[1, 0, -t[0] + imgw // 2 + 0.5], [0, 1, -imgh // 2 + t[1]]], + dtype=np.float32) + img = cv2.warpAffine(img, M_s, (imgw, imgh)) + w = int(imgw / s * 100) + h = int(imgh / s * 100) + img = cv2.resize(img, (w, h)) + lm = np.stack([lm[:, 0] - t[0] + imgw // 2, lm[:, 1] - t[1] + imgh // 2], + axis=1) / s * 100 + + left = w // 2 - 112 + up = h // 2 - 112 + bbox = [left, up, 224, 224] + cropped_img, scale2 = crop(img, bbox) + assert (scale2 != 0) + t1 = np.array([bbox[0], bbox[1]]) + + # back to raw img s * crop + s * t1 + t2 + t1 = np.array([w // 2 - 112, h // 2 - 112]) + scale = s / 100 + t2 = np.array([t[0] - imgw / 2, t[1] - imgh / 2]) + inv = (scale / scale2, scale * t1 + t2.reshape([2])) + return cropped_img, inv + + +# utils for landmark detection +def align_for_lm(img, five_points, params): + five_points = np.array(five_points).reshape([1, 10]) + bbox = BBRegression(five_points, params) + assert (bbox[2] != 0) + bbox = np.round(bbox).astype(np.int32) + crop_img, scale = crop(img, bbox) + return crop_img, scale, bbox + + +# resize and crop images for face reconstruction +def resize_n_crop_img(img, lm, t, s, target_size=224., mask=None): + w0, h0 = img.size + w = (w0 * s).astype(np.int32) + h = (h0 * s).astype(np.int32) + left = (w / 2 - target_size / 2 + float( + (t[0] - w0 / 2) * s)).astype(np.int32) + right = left + target_size + up = (h / 2 - target_size / 2 + float( + (h0 / 2 - t[1]) * s)).astype(np.int32) + below = up + target_size + + new_img = img.resize((w, h), resample=Image.BICUBIC) + new_img = new_img.crop((left, up, right, below)) + + if mask is not None: + mask = mask.resize((w, h), resample=Image.BICUBIC) + mask = mask.crop((left, up, right, below)) + + new_lm = np.stack([lm[:, 0] - t[0] + w0 / 2, lm[:, 1] - t[1] + h0 / 2], + axis=1) * s + new_lm = new_lm - np.reshape( + np.array([(w / 2 - target_size / 2), + (h / 2 - target_size / 2)]), [1, 2]) + + return new_img, new_lm, mask + + +# utils for face reconstruction +def extract_5p(lm): + lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 + value_list = [ + lm[lm_idx[0], :], + np.mean(lm[lm_idx[[1, 2]], :], 0), + np.mean(lm[lm_idx[[3, 4]], :], 0), lm[lm_idx[5], :], lm[lm_idx[6], :] + ] + lm5p = np.stack(value_list, axis=0) + lm5p = lm5p[[1, 2, 0, 3, 4], :] + return lm5p + + +# utils for face reconstruction +def align_img(img, lm, lm3D, mask=None, target_size=224., rescale_factor=102.): + """ + Return: + transparams --numpy.array (raw_W, raw_H, scale, tx, ty) + img_new --PIL.Image (target_size, target_size, 3) + lm_new --numpy.array (68, 2), y direction is opposite to v direction + mask_new --PIL.Image (target_size, target_size) + + Parameters: + img --PIL.Image (raw_H, raw_W, 3) + lm --numpy.array (68, 2), y direction is opposite to v direction + lm3D --numpy.array (5, 3) + mask --PIL.Image (raw_H, raw_W, 3) + """ + + w0, h0 = img.size + if lm.shape[0] != 5: + lm5p = extract_5p(lm) + else: + lm5p = lm + + # calculate translation and scale factors using 5 facial landmarks and standard landmarks of a 3D face + t, s = POS(lm5p.transpose(), lm3D.transpose()) + t = t.squeeze() + s = rescale_factor / s + + # processing the image + img_new, lm_new, mask_new = resize_n_crop_img( + img, lm, t, s, target_size=target_size, mask=mask) + trans_params = np.array([w0, h0, s, t[0], t[1]]) + + return trans_params, img_new, lm_new, mask_new + + +def normalize_v3(arr): + ''' Normalize a numpy array of 3 component vectors shape=(n,3) ''' + lens = np.sqrt(arr[:, 0]**2 + arr[:, 1]**2 + arr[:, 2]**2)[:, None] + arr /= lens + return arr + + +def estimate_normals(vertices, faces): + norm = np.zeros(vertices.shape, dtype=vertices.dtype) + tris = vertices[faces] + n = np.cross(tris[::, 1] - tris[::, 0], tris[::, 2] - tris[::, 0]) + n[(n[:, 0] == 0) * (n[:, 1] == 0) * (n[:, 2] == 0)] = [0, 0, 1.0] + n = normalize_v3(n) + for i in range(3): + for j in range(faces.shape[0]): + norm[faces[j, i]] += n[j] + + inds = (norm[:, 0] == 0) * (norm[:, 1] == 0) * (norm[:, 2] == 0) + norm[inds] = [0, 0, 1.0] + result = normalize_v3(norm) + return result + + +def draw_landmarks(img, landmark, color='r', step=2): + """ + Return: + img -- numpy.array, (B, H, W, 3) img with landmark, RGB order, range (0, 255) + + + Parameters: + img -- numpy.array, (B, H, W, 3), RGB order, range (0, 255) + landmark -- numpy.array, (B, 68, 2), y direction is opposite to v direction + color -- str, 'r' or 'b' (red or blue) + """ + if color == 'r': + c = np.array([255., 0, 0]) + else: + c = np.array([0, 0, 255.]) + + _, H, W, _ = img.shape + img, landmark = img.copy(), landmark.copy() + landmark[..., 1] = H - 1 - landmark[..., 1] + landmark = np.round(landmark).astype(np.int32) + for i in range(landmark.shape[1]): + x, y = landmark[:, i, 0], landmark[:, i, 1] + for j in range(-step, step): + for k in range(-step, step): + u = np.clip(x + j, 0, W - 1) + v = np.clip(y + k, 0, H - 1) + for m in range(landmark.shape[0]): + img[m, v[m], u[m]] = c + return img + + +def split_vis(img_path, target_dir=None): + img = cv2.imread(img_path) + h, w = img.shape[:2] + n_split = w // h + if target_dir is None: + target_dir = os.path.dirname(img_path) + base_name = os.path.splitext(os.path.basename(img_path))[0] + for i in range(n_split): + img_i = img[:, i * h:(i + 1) * h, :] + cv2.imwrite( + os.path.join(target_dir, '{}_{:0>2d}.jpg'.format(base_name, + i + 1)), img_i) + + +def write_video(image_list, save_path, fps=20.0): + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + # fourcc = cv2.VideoWriter_fourcc(*'MJPG') # avi格式 + + h, w = image_list[0].shape[:2] + + out = cv2.VideoWriter(save_path, fourcc, fps, (w, h), True) + + for frame in image_list: + out.write(frame) + + out.release() + + +# ---------------------------- process/generate vertices, normals, faces +def generate_triangles(h, w, margin_x=2, margin_y=5, mask=None): + # quad layout: + # 0 1 ... w-1 + # w w+1 + # . + # w*h + triangles = [] + for x in range(margin_x, w - 1 - margin_x): + for y in range(margin_y, h - 1 - margin_y): + triangle0 = [y * w + x, y * w + x + 1, (y + 1) * w + x] + triangle1 = [y * w + x + 1, (y + 1) * w + x + 1, (y + 1) * w + x] + triangles.append(triangle0) + triangles.append(triangle1) + triangles = np.array(triangles) + triangles = triangles[:, [0, 2, 1]] + return triangles + + +def face_vertices(vertices, faces): + """ + :param vertices: [batch size, number of vertices, 3] + :param faces: [batch size, number of faces, 3] + :return: [batch size, number of faces, 3, 3] + """ + assert (vertices.ndimension() == 3) + assert (faces.ndimension() == 3) + assert (vertices.shape[0] == faces.shape[0]) + assert (vertices.shape[2] == 3) + assert (faces.shape[2] == 3) + + bs, nv = vertices.shape[:2] + bs, nf = faces.shape[:2] + device = vertices.device + faces = faces + (torch.arange(bs, dtype=torch.int32).to(device) + * nv)[:, None, None] + vertices = vertices.reshape((bs * nv, 3)) + # pytorch only supports long and byte tensors for indexing + return vertices[faces.long()] + + +def vertex_normals(vertices, faces): + """ + :param vertices: [batch size, number of vertices, 3] + :param faces: [batch size, number of faces, 3] + :return: [batch size, number of vertices, 3] + """ + assert (vertices.ndimension() == 3) + assert (faces.ndimension() == 3) + assert (vertices.shape[0] == faces.shape[0]) + assert (vertices.shape[2] == 3) + assert (faces.shape[2] == 3) + bs, nv = vertices.shape[:2] + bs, nf = faces.shape[:2] + device = vertices.device + normals = torch.zeros(bs * nv, 3).to(device) + + faces = faces + (torch.arange(bs, dtype=torch.int32).to(device) + * nv)[:, None, None] # expanded faces + vertices_faces = vertices.reshape((bs * nv, 3))[faces.long()] + + faces = faces.reshape(-1, 3) + vertices_faces = vertices_faces.reshape(-1, 3, 3) + + normals.index_add_( + 0, faces[:, 1].long(), + torch.cross(vertices_faces[:, 2] - vertices_faces[:, 1], + vertices_faces[:, 0] - vertices_faces[:, 1])) + normals.index_add_( + 0, faces[:, 2].long(), + torch.cross(vertices_faces[:, 0] - vertices_faces[:, 2], + vertices_faces[:, 1] - vertices_faces[:, 2])) + normals.index_add_( + 0, faces[:, 0].long(), + torch.cross(vertices_faces[:, 1] - vertices_faces[:, 0], + vertices_faces[:, 2] - vertices_faces[:, 0])) + + normals = F.normalize(normals, eps=1e-6, dim=1) + normals = normals.reshape((bs, nv, 3)) + # pytorch only supports long and byte tensors for indexing + return normals + + +def dict2obj(d): + # if isinstance(d, list): + # d = [dict2obj(x) for x in d] + if not isinstance(d, dict): + return d + + class C(object): + pass + + o = C() + for k in d: + o.__dict__[k] = dict2obj(d[k]) + return o + + +def enlarged_bbox(bbox, img_width, img_height, enlarge_ratio=0.2): + ''' + :param bbox: [xmin,ymin,xmax,ymax] + :return: bbox: [xmin,ymin,xmax,ymax] + ''' + + left = bbox[0] + top = bbox[1] + + right = bbox[2] + bottom = bbox[3] + + roi_width = right - left + roi_height = bottom - top + + new_left = left - int(roi_width * enlarge_ratio) + new_left = 0 if new_left < 0 else new_left + + new_top = top - int(roi_height * enlarge_ratio) + new_top = 0 if new_top < 0 else new_top + + new_right = right + int(roi_width * enlarge_ratio) + new_right = img_width if new_right > img_width else new_right + + new_bottom = bottom + int(roi_height * enlarge_ratio) + new_bottom = img_height if new_bottom > img_height else new_bottom + + bbox = [new_left, new_top, new_right, new_bottom] + + bbox = [int(x) for x in bbox] + + return bbox + + +def draw_line(im, points, color, stroke_size=2, closed=False): + points = points.astype(np.int32) + for i in range(len(points) - 1): + cv2.line(im, tuple(points[i]), tuple(points[i + 1]), color, + stroke_size) + if closed: + cv2.line(im, tuple(points[0]), tuple(points[-1]), color, stroke_size) diff --git a/runtime/Lib/site_packages/models/cv/facial_expression_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35a15d18d804948c50240663367a5bb53ae2b6aa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .fer import FacialExpressionRecognition + +else: + _import_structure = {'fer': ['FacialExpressionRecognition']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/__init__.py b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2546035b1ae96d9d83af6a74143b68f158c4e51a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .facial_expression_recognition import FacialExpressionRecognition diff --git a/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/facial_expression_recognition.py b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/facial_expression_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..ee08f5271e9461d6bf22c01777948ee980439843 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/facial_expression_recognition.py @@ -0,0 +1,71 @@ +# The implementation is based on Facial-Expression-Recognition, available at +# https://github.com/WuJie1010/Facial-Expression-Recognition.Pytorch +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from PIL import Image +from torch.autograd import Variable + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from . import transforms +from .vgg import VGG + + +@MODELS.register_module( + Tasks.facial_expression_recognition, module_name=Models.fer) +class FacialExpressionRecognition(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.cfg_path = model_path.replace(ModelFile.TORCH_MODEL_FILE, + ModelFile.CONFIGURATION) + self.net = VGG('VGG19', cfg_path=self.cfg_path) + self.load_model() + self.net = self.net.to(device) + self.transform_test = transforms.Compose([ + transforms.TenCrop(44), + transforms.Lambda(lambda crops: torch.stack( + [transforms.ToTensor()(crop) for crop in crops])), + ]) + + self.mean = np.array([[104, 117, 123]]) + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu')) + self.net.load_state_dict(pretrained_dict['net'], strict=True) + self.net.eval() + + def forward(self, input): + img = input['img'] + img = cv2.cvtColor(img.cpu().numpy(), cv2.COLOR_BGR2GRAY) + img = cv2.resize(img, (48, 48)) + img = img[:, :, np.newaxis] + img = np.concatenate((img, img, img), axis=2) + + img = Image.fromarray(np.uint8(img)) + inputs = self.transform_test(img) + + ncrops, c, h, w = inputs.shape + + inputs = inputs.view(-1, c, h, w) + inputs = inputs.to(self.device) + inputs = Variable(inputs, volatile=True) + outputs = self.net(inputs) + + outputs_avg = outputs.view(ncrops, -1).mean(0) # avg over crops + + score = F.softmax(outputs_avg) + _, predicted = torch.max(outputs_avg.data, 0) + + return score, predicted diff --git a/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/transforms.py b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..a1448c4900fd0c88327aac2010d403df6e97aa6d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/transforms.py @@ -0,0 +1,118 @@ +# The implementation is based on Facial-Expression-Recognition, available at +# https://github.com/WuJie1010/Facial-Expression-Recognition.Pytorch +import numbers +import types + +import numpy as np +import torch +from PIL import Image + + +def to_tensor(pic): + + # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor(torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + # put it from HWC to CHW format + # yikes, this transpose takes 80% of the loading time/CPU + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float().div(255) + else: + return img + + +def center_crop(img, output_size): + if isinstance(output_size, numbers.Number): + output_size = (int(output_size), int(output_size)) + w, h = img.size + th, tw = output_size + i = int(round((h - th) / 2.)) + j = int(round((w - tw) / 2.)) + return img.crop((j, i, j + tw, i + th)) + + +def five_crop(img, size): + if isinstance(size, numbers.Number): + size = (int(size), int(size)) + else: + assert len( + size) == 2, 'Please provide only two dimensions (h, w) for size.' + + w, h = img.size + crop_h, crop_w = size + if crop_w > w or crop_h > h: + raise ValueError( + 'Requested crop size {} is bigger than input size {}'.format( + size, (h, w))) + tl = img.crop((0, 0, crop_w, crop_h)) + tr = img.crop((w - crop_w, 0, w, crop_h)) + bl = img.crop((0, h - crop_h, crop_w, h)) + br = img.crop((w - crop_w, h - crop_h, w, h)) + center = center_crop(img, (crop_h, crop_w)) + return (tl, tr, bl, br, center) + + +class TenCrop(object): + + def __init__(self, size, vertical_flip=False): + self.size = size + if isinstance(size, numbers.Number): + self.size = (int(size), int(size)) + else: + assert len( + size + ) == 2, 'Please provide only two dimensions (h, w) for size.' + self.size = size + self.vertical_flip = vertical_flip + + def __call__(self, img): + first_five = five_crop(img, self.size) + + if self.vertical_flip: + img = img.transpose(Image.FLIP_TOP_BOTTOM) + else: + img = img.transpose(Image.FLIP_LEFT_RIGHT) + + second_five = five_crop(img, self.size) + + return first_five + second_five + + +class Compose(object): + + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img): + for t in self.transforms: + img = t(img) + return img + + +class ToTensor(object): + + def __call__(self, pic): + return to_tensor(pic) + + +class Lambda(object): + + def __init__(self, lambd): + assert isinstance(lambd, types.LambdaType) + self.lambd = lambd + + def __call__(self, img): + return self.lambd(img) diff --git a/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/vgg.py b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/vgg.py new file mode 100644 index 0000000000000000000000000000000000000000..8120b6cc30a899c4fe72f7877deb0492632d5cde --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_expression_recognition/fer/vgg.py @@ -0,0 +1,40 @@ +# The implementation is based on Facial-Expression-Recognition, available at +# https://github.com/WuJie1010/Facial-Expression-Recognition.Pytorch +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +from modelscope.utils.config import Config + + +class VGG(nn.Module): + + def __init__(self, vgg_name, cfg_path): + super(VGG, self).__init__() + model_cfg = Config.from_file(cfg_path)['models'] + self.features = self._make_layers(model_cfg[vgg_name]) + self.classifier = nn.Linear(512, 7) + + def forward(self, x): + out = self.features(x) + out = out.view(out.size(0), -1) + out = F.dropout(out, p=0.5, training=self.training) + out = self.classifier(out) + return out + + def _make_layers(self, cfg): + layers = [] + in_channels = 3 + for x in cfg: + if x == 'M': + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + layers += [ + nn.Conv2d(in_channels, x, kernel_size=3, padding=1), + nn.BatchNorm2d(x), + nn.ReLU(inplace=True) + ] + in_channels = x + layers += [nn.AvgPool2d(kernel_size=1, stride=1)] + return nn.Sequential(*layers) diff --git a/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/__init__.py b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..594e9aeb16b7bd8f069e584cc93fed0fc5d941c9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .flc import FacialLandmarkConfidence + +else: + _import_structure = {'flc': ['FacialLandmarkConfidence']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/__init__.py b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eaf7e3e2affefdc217175447748ca8f7c32b1a98 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .facial_landmark_confidence import FacialLandmarkConfidence diff --git a/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/facial_landmark_confidence.py b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/facial_landmark_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..152903c11224f01f085b7cef4773cb8f6c8aacee --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/facial_landmark_confidence.py @@ -0,0 +1,93 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from PIL import Image +from torch.autograd import Variable + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from .manual_landmark_net import LandmarkConfidence + + +@MODELS.register_module(Tasks.face_2d_keypoints, module_name=Models.flc) +class FacialLandmarkConfidence(TorchModel): + + def __init__(self, model_path, device='cuda'): + super().__init__(model_path) + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.cfg_path = model_path.replace(ModelFile.TORCH_MODEL_FILE, + ModelFile.CONFIGURATION) + self.landmark_count = 5 + self.net = LandmarkConfidence(landmark_count=self.landmark_count) + self.load_model() + self.net = self.net.to(device) + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu'))['state_dict'] + pretrained_dict['rp_net.binary_cls.weight'] = 32.0 * F.normalize( + pretrained_dict['rp_net.binary_cls.weight'], dim=1).t() + self.net.load_state_dict(pretrained_dict, strict=True) + self.net.eval() + + def forward(self, input): + img_org = input['orig_img'] + bbox = input['bbox'] + img_org = img_org.cpu().numpy() + + image_height = img_org.shape[0] + image_width = img_org.shape[1] + x1 = max(0, int(bbox[0])) + y1 = max(0, int(bbox[1])) + x2 = min(image_width, int(bbox[2])) + y2 = min(image_height, int(bbox[3])) + box_w = x2 - x1 + 1 + box_h = y2 - y1 + 1 + if box_h > box_w: + delta = box_h - box_w + dy = edy = 0 + dx = delta // 2 + edx = delta - dx + else: + dx = edx = 0 + delta = box_w - box_h + dy = delta // 2 + edy = delta - dy + + cv_img = img_org[y1:y2, x1:x2] + if dx > 0 or dy > 0 or edx > 0 or edy > 0: + cv_img = cv2.copyMakeBorder(cv_img, dy, edy, dx, edx, + cv2.BORDER_CONSTANT, 0) + inter_x = cv_img.shape[1] + inter_y = cv_img.shape[0] + + cv_img = cv2.resize(cv_img, (120, 120)) + + cv_img = cv_img.transpose((2, 0, 1)) + + input_blob = torch.from_numpy(cv_img[np.newaxis, :, :, :].astype( + np.float32)) + + tmp_conf_lms, tmp_feat, tmp_conf_resp, tmp_nose = self.net( + input_blob.to(self.device)) + conf_lms = tmp_conf_lms.cpu().numpy().squeeze() + feat = tmp_feat.cpu().numpy().squeeze() + + pts5pt = [] + for i in range(feat.shape[0]): + if i < self.landmark_count: + pts5pt.append(feat[i] * inter_x - dx + x1) + else: + pts5pt.append(feat[i] * inter_y - dy + y1) + + lm5pt = np.array(pts5pt).reshape(2, 5).T + return lm5pt, conf_lms diff --git a/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/manual_landmark_net.py b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/manual_landmark_net.py new file mode 100644 index 0000000000000000000000000000000000000000..92136689530be20d8f8d7635e164f03fa45ada18 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/facial_landmark_confidence/flc/manual_landmark_net.py @@ -0,0 +1,152 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + +import torch +import torch.nn.functional as F +from torch.nn import (AdaptiveAvgPool2d, BatchNorm2d, Conv2d, Linear, + MaxPool2d, Module, Parameter, ReLU, Sequential) + + +class LandmarkConfidence(Module): + + def __init__(self, landmark_count=5): + super(LandmarkConfidence, self).__init__() + self.landmark_net = LandmarkNetD(landmark_count) + self.landmark_net.eval() + self.cls_net = ClassNet() + self.cls_net.eval() + self.rp_net = RespiratorNet() + + def forward(self, x): + feat, nose_feat, lms = self.landmark_net(x) + cls_respirator, nose = self.rp_net(feat, nose_feat) + confidence = self.cls_net(feat) + return confidence, lms, cls_respirator, nose + + +class FC(Module): + + def __init__(self, feat_dim=256, num_class=2): + super(FC, self).__init__() + self.weight = Parameter( + torch.zeros(num_class, feat_dim, dtype=torch.float32)) + + def forward(self, x): + cos_theta = F.linear(x, self.weight) + return F.softmax(cos_theta, dim=1) + + +class Flatten(Module): + + def forward(self, x): + return torch.flatten(x, 1) + + +class RespiratorNet(Module): + + def __init__(self): + super(RespiratorNet, self).__init__() + self.conv1 = Sequential( + Conv2d(48, 48, 3, 2, 1), BatchNorm2d(48), ReLU(True)) + self.conv2 = AdaptiveAvgPool2d( + (1, 1) + ) # Sequential(Conv2d(48, 48, 5, 1, 0), BatchNorm2d(48), ReLU(True)) + self.binary_cls = FC(feat_dim=48, num_class=2) + self.nose_layer = Sequential( + Conv2d(48, 64, 3, 1, 0), BatchNorm2d(64), ReLU(True), + Conv2d(64, 64, 3, 1, 0), BatchNorm2d(64), ReLU(True), Flatten(), + Linear(64, 96), ReLU(True), Linear(96, 6)) + + def train(self, mode=True): + self.conv1.train(mode) + self.conv2.train(mode) + # self.nose_feat.train(mode) + self.nose_layer.train(mode) + self.binary_cls.train(mode) + + def forward(self, x, y): + x = self.conv1(x) + x = self.conv2(x) + cls = self.binary_cls(torch.flatten(x, 1)) + # loc = self.nose_feat(y) + loc = self.nose_layer(y) + return cls, loc + + +class ClassNet(Module): + + def __init__(self): + super(ClassNet, self).__init__() + self.conv1 = Sequential( + Conv2d(48, 48, 3, 1, 1), BatchNorm2d(48), ReLU(True)) + self.conv2 = Sequential( + Conv2d(48, 54, 3, 2, 1), BatchNorm2d(54), ReLU(True)) + self.conv3 = Sequential( + Conv2d(54, 54, 5, 1, 0), BatchNorm2d(54), ReLU(True)) + self.fc1 = Sequential(Flatten(), Linear(54, 54), ReLU(True)) + self.fc2 = Linear(54, 1) + + def forward(self, x): + y = self.conv1(x) + y = self.conv2(y) + y = self.conv3(y) + y = self.fc1(y) + y = self.fc2(y) + return y + + +class LandmarkNetD(Module): + + def __init__(self, landmark_count=5): + super(LandmarkNetD, self).__init__() + self.conv_pre = Sequential( + Conv2d(3, 16, 5, 2, 0), BatchNorm2d(16), ReLU(True)) + self.pool_pre = MaxPool2d(2, 2) # output is 29 + + self.conv1 = Sequential( + Conv2d(16, 32, 3, 1, 1), BatchNorm2d(32), ReLU(True), + Conv2d(32, 32, 3, 1, 1), BatchNorm2d(32), ReLU(True)) + self.pool1 = MaxPool2d(2, 2) # 14 + + self.conv2 = Sequential( + Conv2d(32, 48, 3, 1, 0), BatchNorm2d(48), ReLU(True), + Conv2d(48, 48, 3, 1, 0), BatchNorm2d(48), ReLU(True)) + self.pool2 = MaxPool2d(2, 2) # 5 + + self.conv3 = Sequential( + Conv2d(48, 80, 3, 1, 0), BatchNorm2d(80), ReLU(True), + Conv2d(80, 80, 3, 1, 0), BatchNorm2d(80), ReLU(True)) + + self.fc1 = Sequential(Linear(80, 128), ReLU(True)) + self.fc2 = Sequential(Linear(128, 128), ReLU(True)) + + self.output = Linear(128, landmark_count * 2) + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, Linear): + n = m.weight.size(1) + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() + + def forward(self, x): + y = self.conv_pre(x) + y = self.pool_pre(y) + y = self.conv1(y) + y = self.pool1(y[:, :, :28, :28]) + feat = self.conv2(y) + y2 = self.pool2(feat) + y = self.conv3(y2) + y = torch.flatten(y, 1) + y = self.fc1(y) + y = self.fc2(y) + y = self.output(y) + return feat, y2, y diff --git a/runtime/Lib/site_packages/models/cv/hand_static/__init__.py b/runtime/Lib/site_packages/models/cv/hand_static/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..654d2acb0e28f88b0da60d1ce3420e4dc14d0d4b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/hand_static/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .hand_model import HandStatic + +else: + _import_structure = {'hand_model': ['HandStatic']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/hand_static/hand_model.py b/runtime/Lib/site_packages/models/cv/hand_static/hand_model.py new file mode 100644 index 0000000000000000000000000000000000000000..7a8a323e6de300ca85bb457c34327dcc77cf296d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/hand_static/hand_model.py @@ -0,0 +1,93 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import sys + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torch import nn +from torchvision import transforms + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .networks import StaticGestureNet + +logger = get_logger() + +map_idx = { + 0: 'unrecog', + 1: 'one', + 2: 'two', + 3: 'bixin', + 4: 'yaogun', + 5: 'zan', + 6: 'fist', + 7: 'ok', + 8: 'tuoju', + 9: 'd_bixin', + 10: 'd_fist_left', + 11: 'd_fist_right', + 12: 'd_hand', + 13: 'fashe', + 14: 'five', + 15: 'nohand' +} + +img_size = [112, 112] + +spatial_transform = transforms.Compose([ + transforms.Resize(img_size), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) +]) + + +@MODELS.register_module(Tasks.hand_static, module_name=Models.hand_static) +class HandStatic(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + self.model = StaticGestureNet() + if torch.cuda.is_available(): + self.device = 'cuda' + else: + self.device = 'cpu' + self.params = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location=self.device) + + self.model.load_state_dict(self.params) + self.model.to(self.device) + self.model.eval() + self.device_id = device_id + if self.device_id >= 0 and self.device == 'cuda': + self.model.to('cuda:{}'.format(self.device_id)) + logger.info('Use GPU: {}'.format(self.device_id)) + else: + self.device_id = -1 + logger.info('Use CPU for inference') + + def forward(self, x): + pred_result = self.model(x) + return pred_result + + +def infer(img, model, device): + img = img.cpu().numpy() + img = Image.fromarray(img) + clip = spatial_transform(img) + clip = clip.unsqueeze(0).to(device).float() + outputs = model(clip) + predicted = int(outputs.max(1)[1]) + pred_result = map_idx.get(predicted) + logger.info('pred result: {}'.format(pred_result)) + + return pred_result diff --git a/runtime/Lib/site_packages/models/cv/hand_static/networks.py b/runtime/Lib/site_packages/models/cv/hand_static/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..b9db7946eff676629acc67eb4fda60f4715b529f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/hand_static/networks.py @@ -0,0 +1,358 @@ +""" HandStatic +The implementation here is modified based on MobileFaceNet, +originally Apache 2.0 License and publicly available at https://github.com/xuexingyu24/MobileFaceNet_Tutorial_Pytorch +""" + +import os + +import torch +import torch.nn as nn +import torchvision +import torchvision.models as models +from torch.nn import (AdaptiveAvgPool2d, BatchNorm1d, BatchNorm2d, Conv2d, + Dropout, Linear, MaxPool2d, Module, PReLU, ReLU, + Sequential, Sigmoid) + + +class StaticGestureNet(torch.nn.Module): + + def __init__(self, train=True): + super().__init__() + + model = MobileFaceNet(512) + self.feature_extractor = model + self.fc_layer = torch.nn.Sequential( + nn.Linear(512, 128), nn.Softplus(), nn.Linear(128, 15)) + self.sigmoid = nn.Sigmoid() + + def forward(self, inputs): + out = self.feature_extractor(inputs) + out = self.fc_layer(out) + out = self.sigmoid(out) + return out + + +class Flatten(Module): + + def forward(self, input): + return input.view(input.size(0), -1) + + +def l2_norm(input, axis=1): + norm = torch.norm(input, 2, axis, True) + output = torch.div(input, norm) + return output + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + return module_input * x + + +class BottleneckIR(Module): + + def __init__(self, in_channel, depth, stride): + super(BottleneckIR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut + + +class BottleneckIRSE(Module): + + def __init__(self, in_channel, depth, stride): + super(BottleneckIRSE, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth), SEModule(depth, 16)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut + + +def get_block(in_channel, depth, num_units, stride=2): + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + return blocks + + +class Backbone(Module): + + def __init__(self, num_layers, drop_ratio, mode='ir'): + super(Backbone, self).__init__() + assert num_layers in [50, 100, + 152], 'num_layers should be 50,100, or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = BottleneckIR + elif mode == 'ir_se': + unit_module = BottleneckIRSE + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(drop_ratio), Flatten(), + Linear(512 * 7 * 7, 512), BatchNorm1d(512)) + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return l2_norm(x) + + +class ConvBlock(Module): + + def __init__(self, + in_c, + out_c, + kernel=(1, 1), + stride=(1, 1), + padding=(0, 0), + groups=1): + super(ConvBlock, self).__init__() + self.conv = Conv2d( + in_c, + out_channels=out_c, + kernel_size=kernel, + groups=groups, + stride=stride, + padding=padding, + bias=False) + self.bn = BatchNorm2d(out_c) + self.prelu = PReLU(out_c) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.prelu(x) + return x + + +class LinearBlock(Module): + + def __init__(self, + in_c, + out_c, + kernel=(1, 1), + stride=(1, 1), + padding=(0, 0), + groups=1): + super(LinearBlock, self).__init__() + self.conv = Conv2d( + in_c, + out_channels=out_c, + kernel_size=kernel, + groups=groups, + stride=stride, + padding=padding, + bias=False) + self.bn = BatchNorm2d(out_c) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class DepthWise(Module): + + def __init__(self, + in_c, + out_c, + residual=False, + kernel=(3, 3), + stride=(2, 2), + padding=(1, 1), + groups=1): + super(DepthWise, self).__init__() + self.conv = ConvBlock( + in_c, out_c=groups, kernel=(1, 1), padding=(0, 0), stride=(1, 1)) + self.conv_dw = ConvBlock( + groups, + groups, + groups=groups, + kernel=kernel, + padding=padding, + stride=stride) + self.project = LinearBlock( + groups, out_c, kernel=(1, 1), padding=(0, 0), stride=(1, 1)) + self.residual = residual + + def forward(self, x): + if self.residual: + short_cut = x + x = self.conv(x) + x = self.conv_dw(x) + x = self.project(x) + if self.residual: + output = short_cut + x + else: + output = x + return output + + +class Residual(Module): + + def __init__(self, + c, + num_block, + groups, + kernel=(3, 3), + stride=(1, 1), + padding=(1, 1)): + super(Residual, self).__init__() + modules = [] + for _ in range(num_block): + modules.append( + DepthWise( + c, + c, + residual=True, + kernel=kernel, + padding=padding, + stride=stride, + groups=groups)) + self.model = Sequential(*modules) + + def forward(self, x): + return self.model(x) + + +class MobileFaceNet(Module): + + def __init__(self, embedding_size): + super(MobileFaceNet, self).__init__() + self.conv1 = ConvBlock( + 3, 64, kernel=(3, 3), stride=(2, 2), padding=(1, 1)) + self.conv2_dw = ConvBlock( + 64, 64, kernel=(3, 3), stride=(1, 1), padding=(1, 1), groups=64) + self.conv_23 = DepthWise( + 64, 64, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=128) + self.conv_3 = Residual( + 64, + num_block=4, + groups=128, + kernel=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.conv_34 = DepthWise( + 64, 128, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=256) + self.conv_4 = Residual( + 128, + num_block=6, + groups=256, + kernel=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.conv_45 = DepthWise( + 128, 128, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=512) + self.conv_5 = Residual( + 128, + num_block=2, + groups=256, + kernel=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.conv_6_sep = ConvBlock( + 128, 512, kernel=(1, 1), stride=(1, 1), padding=(0, 0)) + self.conv_6_dw = LinearBlock( + 512, 512, groups=512, kernel=(7, 7), stride=(1, 1), padding=(0, 0)) + self.conv_6_flatten = Flatten() + self.linear = Linear(512, embedding_size, bias=False) + self.bn = BatchNorm1d(embedding_size) + + def forward(self, x): + out = self.conv1(x) + out = self.conv2_dw(out) + out = self.conv_23(out) + out = self.conv_3(out) + out = self.conv_34(out) + out = self.conv_4(out) + out = self.conv_45(out) + out = self.conv_5(out) + out = self.conv_6_sep(out) + out = self.conv_6_dw(out) + out = self.conv_6_flatten(out) + out = self.linear(out) + return l2_norm(out) diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/__init__.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/__init__.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/bfm.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/bfm.py new file mode 100644 index 0000000000000000000000000000000000000000..d0aebd8eb326000c25546da3b8b1ddc47243059f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/bfm.py @@ -0,0 +1,673 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch + +import os + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.io import loadmat + +from modelscope.models.cv.face_reconstruction.utils import read_obj + + +def perspective_projection(focal, center): + # return p.T (N, 3) @ (3, 3) + return np.array([focal, 0, center, 0, focal, center, 0, 0, + 1]).reshape([3, 3]).astype(np.float32).transpose() + + +class SH: + + def __init__(self): + self.a = [np.pi, 2 * np.pi / np.sqrt(3.), 2 * np.pi / np.sqrt(8.)] + self.c = [ + 1 / np.sqrt(4 * np.pi), + np.sqrt(3.) / np.sqrt(4 * np.pi), + 3 * np.sqrt(5.) / np.sqrt(12 * np.pi) + ] + + +class ParametricFaceModel: + + def __init__(self, + assets_root='assets', + recenter=True, + camera_distance=10., + init_lit=np.array([0.8, 0, 0, 0, 0, 0, 0, 0, 0]), + focal=1015., + center=112., + is_train=True, + default_name='BFM_model_front.mat'): + + model = loadmat(os.path.join(assets_root, '3dmm/BFM', default_name)) + model_bfm_front = loadmat( + os.path.join(assets_root, '3dmm/BFM/BFM_model_front.mat')) + self.mean_shape_ori = model_bfm_front['meanshape'].astype(np.float32) + # mean face shape. [3*N,1] + self.mean_shape = model['meanshape'].astype(np.float32) # (1, 107127) + + # identity basis. [3*N,80] + self.id_base = model['idBase'].astype(np.float32) # (107127, 80) + + # expression basis. [3*N,64] + self.exp_base = model['exBase'].astype(np.float32) # (107127, 64) + + # mean face texture. [3*N,1] (0-255) + self.mean_tex = model['meantex'].astype(np.float32) # (1, 107127) + + # texture basis. [3*N,80] + self.tex_base = model['texBase'].astype(np.float32) # (107127, 80) + + self.bfm_keep_inds = np.load( + os.path.join(assets_root, '3dmm/inds/bfm_keep_inds.npy')) + + self.ours_hair_area_inds = np.load( + os.path.join(assets_root, '3dmm/inds/ours_hair_area_inds.npy')) + + if default_name == 'ourRefineFull_model.mat': + self.mean_tex = self.mean_tex.reshape(1, -1, 3) + mean_tex_keep = self.mean_tex[:, self.bfm_keep_inds] + self.mean_tex[:, :len(self.bfm_keep_inds)] = mean_tex_keep + self.mean_tex[:, + len(self.bfm_keep_inds):] = np.array([200, 146, + 118])[None, + None] + self.mean_tex[:, self.ours_hair_area_inds] = 40.0 + self.mean_tex = self.mean_tex.reshape(1, -1) + self.mean_tex = np.ascontiguousarray(self.mean_tex) + + self.tex_base = self.tex_base.reshape(-1, 3, 80) + tex_base_keep = self.tex_base[self.bfm_keep_inds] + self.tex_base[:len(self.bfm_keep_inds)] = tex_base_keep + self.tex_base[len(self.bfm_keep_inds):] = 0.0 + self.tex_base = self.tex_base.reshape(-1, 80) + self.tex_base = np.ascontiguousarray(self.tex_base) + + # face indices for each vertex that lies in. starts from 0. [N,8] + self.point_buf = model['point_buf'].astype(np.int64) - 1 # (35709, 8) + + # vertex indices for each face. starts from 0. [F,3] + self.face_buf = model['tri'].astype(np.int64) - 1 # (70789, 3) + + # vertex indices for 68 landmarks. starts from 0. [68,1] + self.keypoints = np.squeeze(model['keypoints']).astype(np.int64) - 1 + + if default_name == 'ourRefineFull_model.mat': + self.keypoints = np.load( + os.path.join( + assets_root, + '3dmm/inds/our_refine0223_basis_withoutEyes_withUV_keypoints_inds.npy' + )).astype(np.int64) + self.point_buf = self.point_buf[:, :8] + 1 + + if is_train: + # vertex indices for small face region to compute photometric error. starts from 0. + self.front_mask = np.squeeze(model['frontmask2_idx']).astype( + np.int64) - 1 + # vertex indices for each face from small face region. starts from 0. [f,3] + self.front_face_buf = model['tri_mask2'].astype(np.int64) - 1 + # vertex indices for pre-defined skin region to compute reflectance loss + self.skin_mask = np.squeeze(model['skinmask']) + + if default_name == 'ourRefineFull_model.mat': + nose_reduced_mesh = read_obj( + os.path.join(assets_root, + '3dmm/adjust_part/our_full/145_nose.obj')) + self.nose_reduced_part = nose_reduced_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + neck_mesh = read_obj( + os.path.join(assets_root, + '3dmm/adjust_part/our_full/154_neck.obj')) + self.neck_adjust_part = neck_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + eyes_mesh = read_obj( + os.path.join( + assets_root, + '3dmm/adjust_part/our_full/our_mean_adjust_eyes.obj')) + self.eyes_adjust_part = eyes_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + self.neck_slim_part = None + self.neck_stretch_part = None + elif default_name == 'ourRefineBFMEye0504_model.mat': + nose_reduced_mesh = read_obj( + os.path.join(assets_root, + '3dmm/adjust_part/our_full_bfmEyes/145_nose.obj')) + self.nose_reduced_part = nose_reduced_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + neck_mesh = read_obj( + os.path.join(assets_root, + '3dmm/adjust_part/our_full_bfmEyes/146_neck.obj')) + self.neck_adjust_part = neck_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + self.eyes_adjust_part = None + + neck_slim_mesh = read_obj( + os.path.join( + assets_root, + '3dmm/adjust_part/our_full_bfmEyes/147_neckSlim2.obj')) + self.neck_slim_part = neck_slim_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + + neck_stretch_mesh = read_obj( + os.path.join( + assets_root, + '3dmm/adjust_part/our_full_bfmEyes/148_neckLength.obj')) + self.neck_stretch_part = neck_stretch_mesh['vertices'].reshape( + (1, -1)) - self.mean_shape + else: + self.nose_reduced_part = None + + self.neck_adjust_part = None + self.eyes_adjust_part = None + self.neck_slim_part = None + self.neck_stretch_part = None + + if recenter: + mean_shape = self.mean_shape.reshape([-1, 3]) + mean_shape_ori = self.mean_shape_ori.reshape([-1, 3]) + mean_shape = mean_shape - np.mean( + mean_shape_ori[:35709, ...], axis=0, keepdims=True) + self.mean_shape = mean_shape.reshape([-1, 1]) + + eye_corner_inds = np.load( + os.path.join(assets_root, '3dmm/inds/eye_corner_inds.npy')) + self.eye_corner_inds = torch.from_numpy(eye_corner_inds).long() + eye_lines = np.load( + os.path.join(assets_root, '3dmm/inds/eye_corner_lines.npy')) + self.eye_lines = torch.from_numpy(eye_lines).long() + + self.center = center + self.persc_proj = perspective_projection(focal, self.center) + self.camera_distance = camera_distance + self.SH = SH() + self.init_lit = init_lit.reshape([1, 1, -1]).astype(np.float32) + + def to(self, device): + self.device = device + for key, value in self.__dict__.items(): + if type(value).__module__ == np.__name__: + setattr(self, key, torch.tensor(value).to(device)) + + def compute_shape(self, + id_coeff, + exp_coeff, + nose_coeff=0.0, + neck_coeff=0.0, + eyes_coeff=0.0, + neckSlim_coeff=0.0, + neckStretch_coeff=0.0): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) + + Parameters: + id_coeff -- torch.tensor, size (B, 80), identity coeffs + exp_coeff -- torch.tensor, size (B, 64), expression coeffs + """ + batch_size = id_coeff.shape[0] + id_part = torch.einsum('ij,aj->ai', self.id_base, id_coeff) + exp_part = torch.einsum('ij,aj->ai', self.exp_base, exp_coeff) + face_shape = id_part + exp_part + self.mean_shape.reshape([1, -1]) + + if nose_coeff != 0: + face_shape = face_shape + nose_coeff * self.nose_reduced_part + if neck_coeff != 0: + face_shape = face_shape + neck_coeff * self.neck_adjust_part + if eyes_coeff != 0 and self.eyes_adjust_part is not None: + face_shape = face_shape + eyes_coeff * self.eyes_adjust_part + if neckSlim_coeff != 0 and self.neck_slim_part is not None: + face_shape = face_shape + neckSlim_coeff * self.neck_slim_part + if neckStretch_coeff != 0 and self.neck_stretch_part is not None: + + neck_stretch_part = self.neck_stretch_part.reshape(1, -1, 3) + neck_stretch_part_top = neck_stretch_part[0, 37476, 1] + neck_stretch_part_bottom = neck_stretch_part[0, 37357, 1] + neck_stretch_height = neck_stretch_part_top - neck_stretch_part_bottom + + face_shape_ = face_shape.reshape(1, -1, 3) + face_shape_top = face_shape_[0, 37476, 1] + face_shape_bottom = face_shape_[0, 37357, 1] + face_shape_height = face_shape_top - face_shape_bottom + + target_neck_height = 0.72 # top ind 37476, bottom ind 37357 + neckStretch_coeff = (target_neck_height + - face_shape_height) / neck_stretch_height + + face_shape = face_shape + neckStretch_coeff * self.neck_stretch_part + + return face_shape.reshape([batch_size, -1, 3]) + + def compute_texture(self, tex_coeff, normalize=True): + """ + Return: + face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.) + + Parameters: + tex_coeff -- torch.tensor, size (B, 80) + """ + batch_size = tex_coeff.shape[0] + face_texture = torch.einsum('ij,aj->ai', self.tex_base, + tex_coeff) + self.mean_tex + if normalize: + face_texture = face_texture / 255. + return face_texture.reshape([batch_size, -1, 3]) + + def compute_norm(self, face_shape): + """ + Return: + vertex_norm -- torch.tensor, size (B, N, 3) + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + v1 = face_shape[:, self.face_buf[:, 0]] + v2 = face_shape[:, self.face_buf[:, 1]] + v3 = face_shape[:, self.face_buf[:, 2]] + e1 = v1 - v2 + e2 = v2 - v3 + face_norm = torch.cross(e1, e2, dim=-1) + face_norm = F.normalize(face_norm, dim=-1, p=2) + face_norm = torch.cat( + [face_norm, + torch.zeros(face_norm.shape[0], 1, 3).to(self.device)], + dim=1) + + vertex_norm = torch.sum(face_norm[:, self.point_buf], dim=2) + vertex_norm = F.normalize(vertex_norm, dim=-1, p=2) + return vertex_norm + + def compute_color(self, face_texture, face_norm, gamma): + """ + Return: + face_color -- torch.tensor, size (B, N, 3), range (0, 1.) + + Parameters: + face_texture -- torch.tensor, size (B, N, 3), from texture model, range (0, 1.) + face_norm -- torch.tensor, size (B, N, 3), rotated face normal + gamma -- torch.tensor, size (B, 27), SH coeffs + """ + batch_size = gamma.shape[0] + a, c = self.SH.a, self.SH.c + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + self.init_lit + gamma = gamma.permute(0, 2, 1) + + y1 = a[0] * c[0] * torch.ones_like(face_norm[..., :1]).to(self.device) + y2 = -a[1] * c[1] * face_norm[..., 1:2] + y3 = a[1] * c[1] * face_norm[..., 2:] + y4 = -a[1] * c[1] * face_norm[..., :1] + y5 = a[2] * c[2] * face_norm[..., :1] * face_norm[..., 1:2] + y6 = -a[2] * c[2] * face_norm[..., 1:2] * face_norm[..., 2:] + y7 = 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm[..., 2:]**2 - 1) + y8 = -a[2] * c[2] * face_norm[..., :1] * face_norm[..., 2:] + y9 = 0.5 * a[2] * c[2] * ( + face_norm[..., :1]**2 - face_norm[..., 1:2]**2) + Y = torch.cat([y1, y2, y3, y4, y5, y6, y7, y8, y9], dim=-1) + r = Y @ gamma[..., :1] + g = Y @ gamma[..., 1:2] + b = Y @ gamma[..., 2:] + face_color = torch.cat([r, g, b], dim=-1) * face_texture + return face_color + + def compute_rotation(self, angles): + """ + Return: + rot -- torch.tensor, size (B, 3, 3) pts @ trans_mat + + Parameters: + angles -- torch.tensor, size (B, 3), radian + """ + + batch_size = angles.shape[0] + ones = torch.ones([batch_size, 1]).to(self.device) + zeros = torch.zeros([batch_size, 1]).to(self.device) + x, y, z = angles[:, :1], angles[:, 1:2], angles[:, 2:], + + value_list = [ + ones, zeros, zeros, zeros, + torch.cos(x), -torch.sin(x), zeros, + torch.sin(x), + torch.cos(x) + ] + rot_x = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + value_list = [ + torch.cos(y), zeros, + torch.sin(y), zeros, ones, zeros, -torch.sin(y), zeros, + torch.cos(y) + ] + rot_y = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + value_list = [ + torch.cos(z), -torch.sin(z), zeros, + torch.sin(z), + torch.cos(z), zeros, zeros, zeros, ones + ] + rot_z = torch.cat(value_list, dim=1).reshape([batch_size, 3, 3]) + + rot = rot_z @ rot_y @ rot_x + return rot.permute(0, 2, 1) + + def to_camera(self, face_shape): + face_shape[..., -1] = self.camera_distance - face_shape[..., -1] + return face_shape + + def to_image(self, face_shape): + """ + Return: + face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + # to image_plane + face_proj = face_shape @ self.persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + def transform(self, face_shape, rot, trans): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) pts @ rot + trans + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + rot -- torch.tensor, size (B, 3, 3) + trans -- torch.tensor, size (B, 3) + """ + return face_shape @ rot + trans.unsqueeze(1) + + def get_landmarks(self, face_proj): + """ + Return: + face_lms -- torch.tensor, size (B, 68, 2) + + Parameters: + face_proj -- torch.tensor, size (B, N, 2) + """ + return face_proj[:, self.keypoints] + + def split_coeff(self, coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 256) + """ + if type(coeffs) == dict and 'id' in coeffs: + return coeffs + + id_coeffs = coeffs[:, :80] + exp_coeffs = coeffs[:, 80:144] + tex_coeffs = coeffs[:, 144:224] + angles = coeffs[:, 224:227] + gammas = coeffs[:, 227:254] + translations = coeffs[:, 254:] + return { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + + def merge_coeff(self, coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 256) + """ + names = ['id', 'exp', 'tex', 'angle', 'gamma', 'trans'] + coeffs_merge = [] + for name in names: + coeffs_merge.append(coeffs[name].detach()) + coeffs_merge = torch.cat(coeffs_merge, dim=1) + + return coeffs_merge + + def reverse_recenter(self, face_shape): + batch_size = face_shape.shape[0] + face_shape = face_shape.reshape([-1, 3]) + mean_shape_ori = self.mean_shape_ori.reshape([-1, 3]) + face_shape = face_shape + torch.mean( + mean_shape_ori[:35709, ...], dim=0, keepdim=True) + face_shape = face_shape.reshape([batch_size, -1, 3]) + return face_shape + + def add_nonlinear_offset_eyes(self, face_shape, shape_offset): + assert face_shape.shape[0] == 1 and shape_offset.shape[0] == 1 + face_shape = face_shape[0] + shape_offset = shape_offset[0] + + corner_shape = face_shape[-625:, :] + corner_offset = shape_offset[self.eye_corner_inds] + for i in range(len(self.eye_lines)): + corner_shape[self.eye_lines[i]] += corner_offset[i][None, ...] + face_shape[-625:, :] = corner_shape + + l_eye_landmarks = [11540, 11541] + r_eye_landmarks = [4271, 4272] + + l_eye_offset = torch.mean( + shape_offset[l_eye_landmarks], dim=0, keepdim=True) + face_shape[37082:37082 + 609] += l_eye_offset + + r_eye_offset = torch.mean( + shape_offset[r_eye_landmarks], dim=0, keepdim=True) + face_shape[37082 + 609:37082 + 609 + 608] += r_eye_offset + + face_shape = face_shape[None, ...] + + return face_shape + + def add_nonlinear_offset(self, face_shape, shape_offset_uv, UVs): + """ + + Args: + face_shape: torch.tensor, size (1, N, 3) + shape_offset_uv: torch.tensor, size (1, h, w, 3) + UVs: torch.tensor, size (N, 2) + + Returns: + + """ + assert face_shape.shape[0] == 1 and shape_offset_uv.shape[0] == 1 + face_shape = face_shape[0] + shape_offset_uv = shape_offset_uv[0] + + h, w = shape_offset_uv.shape[:2] + UVs_coords = UVs.clone() + UVs_coords[:, 0] *= w + UVs_coords[:, 1] *= h + UVs_coords_int = torch.floor(UVs_coords) + UVs_coords_float = UVs_coords - UVs_coords_int + UVs_coords_int = UVs_coords_int.long() + + shape_lt = shape_offset_uv[(h - 1 - UVs_coords_int[:, 1]).clamp( + 0, h - 1), UVs_coords_int[:, 0].clamp(0, w - 1)] # (N, 3) + shape_lb = shape_offset_uv[(h - UVs_coords_int[:, 1]).clamp(0, h - 1), + UVs_coords_int[:, 0].clamp(0, w - 1)] + shape_rt = shape_offset_uv[(h - 1 + - UVs_coords_int[:, 1]).clamp(0, h - 1), + (UVs_coords_int[:, 0] + 1).clamp(0, w - 1)] + shape_rb = shape_offset_uv[(h - UVs_coords_int[:, 1]).clamp(0, h - 1), + (UVs_coords_int[:, 0] + 1).clamp(0, w - 1)] + + value_1 = shape_lt * ( + 1 - UVs_coords_float[:, :1]) * UVs_coords_float[:, 1:] + value_2 = shape_lb * (1 - UVs_coords_float[:, :1]) * ( + 1 - UVs_coords_float[:, 1:]) + value_3 = shape_rt * UVs_coords_float[:, :1] * UVs_coords_float[:, 1:] + value_4 = shape_rb * UVs_coords_float[:, :1] * ( + 1 - UVs_coords_float[:, 1:]) + + offset_shape = value_1 + value_2 + value_3 + value_4 # (B, N, 3) + + face_shape = (face_shape + offset_shape)[None, ...] + + return face_shape, offset_shape[None, ...] + + def compute_for_render_head_fitting(self, + coeffs, + shape_offset_uv, + texture_offset_uv, + shape_offset_uv_head, + texture_offset_uv_head, + UVs, + reverse_recenter=True, + get_eyes=False, + get_neck=False, + nose_coeff=0.0, + neck_coeff=0.0, + eyes_coeff=0.0): + if type(coeffs) == dict: + coef_dict = coeffs + elif type(coeffs) == torch.Tensor: + coef_dict = self.split_coeff(coeffs) + + face_shape = self.compute_shape( + coef_dict['id'], + coef_dict['exp'], + nose_coeff=nose_coeff, + neck_coeff=neck_coeff, + eyes_coeff=eyes_coeff) # (1, n, 3) + if reverse_recenter: + face_shape_ori_noRecenter = self.reverse_recenter( + face_shape.clone()) + else: + face_shape_ori_noRecenter = face_shape.clone() + face_vertex_ori = self.to_camera(face_shape_ori_noRecenter) + + face_shape[:, :35241, :], shape_offset = self.add_nonlinear_offset( + face_shape[:, :35241, :], shape_offset_uv, + UVs[:35709, ...][self.bfm_keep_inds]) # (1, n, 3) + if get_eyes: + face_shape = self.add_nonlinear_offset_eyes( + face_shape, shape_offset) + if get_neck: + face_shape[:, 35241:37082, ...], _ = self.add_nonlinear_offset( + face_shape[:, 35241:37082, ...], shape_offset_uv_head, + UVs[35709:, ...]) # (1, n, 3) + else: + face_shape[:, self.ours_hair_area_inds, + ...], _ = self.add_nonlinear_offset( + face_shape[:, self.ours_hair_area_inds, + ...], shape_offset_uv_head, + UVs[self.ours_hair_area_inds + (35709 - 35241), + ...]) # (1, n, 3) + + if reverse_recenter: + face_shape_offset_noRecenter = self.reverse_recenter( + face_shape.clone()) + else: + face_shape_offset_noRecenter = face_shape.clone() + face_vertex_offset = self.to_camera(face_shape_offset_noRecenter) + + rotation = self.compute_rotation(coef_dict['angle']) + + face_shape_transformed = self.transform(face_shape, rotation, + coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_texture = self.compute_texture(coef_dict['tex']) # (1, n, 3) + face_texture[:, :35241, :], texture_offset = self.add_nonlinear_offset( + face_texture[:, :35241, :], texture_offset_uv, + UVs[:35709, ...][self.bfm_keep_inds]) + face_texture[:, 35241:37082, :], _ = self.add_nonlinear_offset( + face_texture[:, 35241:37082, :], texture_offset_uv_head, + UVs[35709:, ...]) + + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + face_color = self.compute_color(face_texture, face_norm_roted, + coef_dict['gamma']) + + return face_vertex, face_texture, face_color, landmark, face_vertex_ori, face_vertex_offset, face_proj + + def compute_for_render_head(self, + coeffs, + shape_offset_uv, + texture_offset_uv, + shape_offset_uv_head, + texture_offset_uv_head, + UVs, + reverse_recenter=True, + nose_coeff=0.0, + neck_coeff=0.0, + eyes_coeff=0.0, + neckSlim_coeff=0.0, + neckStretch_coeff=0.0): + if type(coeffs) == dict: + coef_dict = coeffs + elif type(coeffs) == torch.Tensor: + coef_dict = self.split_coeff(coeffs) + + face_shape = self.compute_shape( + coef_dict['id'], + coef_dict['exp'], + nose_coeff=nose_coeff, + neck_coeff=neck_coeff, + eyes_coeff=eyes_coeff, + neckSlim_coeff=neckSlim_coeff, + neckStretch_coeff=neckStretch_coeff) # (1, n, 3) + if reverse_recenter: + face_shape_ori_noRecenter = self.reverse_recenter( + face_shape.clone()) + else: + face_shape_ori_noRecenter = face_shape.clone() + face_vertex_ori = self.to_camera(face_shape_ori_noRecenter) + + face_shape[:, :35709, :], shape_offset = self.add_nonlinear_offset( + face_shape[:, :35709, :], shape_offset_uv, UVs[:35709, + ...]) # (1, n, 3) + face_shape[:, 35709:, + ...], _ = self.add_nonlinear_offset(face_shape[:, 35709:, + ...], + shape_offset_uv_head, + UVs[35709:, + ...]) # (1, n, 3) + + if reverse_recenter: + face_shape_offset_noRecenter = self.reverse_recenter( + face_shape.clone()) + else: + face_shape_offset_noRecenter = face_shape.clone() + face_vertex_offset = self.to_camera(face_shape_offset_noRecenter) + + rotation = self.compute_rotation(coef_dict['angle']) + + face_shape_transformed = self.transform(face_shape, rotation, + coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_texture = self.compute_texture(coef_dict['tex']) # (1, n, 3) + face_texture[:, :35709, :], texture_offset = self.add_nonlinear_offset( + face_texture[:, :35709, :], texture_offset_uv, UVs[:35709, ...]) + face_texture[:, 35709:, :], _ = self.add_nonlinear_offset( + face_texture[:, 35709:, :], texture_offset_uv_head, UVs[35709:, + ...]) + + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + face_color = self.compute_color(face_texture, face_norm_roted, + coef_dict['gamma']) + + return face_vertex, face_texture, face_color, landmark, face_vertex_ori, face_vertex_offset, face_proj diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/head_segmentation.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/head_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..ef784a7d5ad5b40beb9b5c521c2583379006685c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/head_segmentation.py @@ -0,0 +1,196 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import cv2 +import json +import numpy as np +import tensorflow as tf + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + tf.disable_eager_execution() + + +class HeadSegmentor(): + + def __init__(self, model_root): + """The HeadSegmentor is implemented based on https://arxiv.org/abs/2004.04955 + Args: + model_root: the root directory of the model files + """ + self.sess = self.load_sess( + os.path.join(model_root, 'head_segmentation', + 'Matting_headparser_6_18.pb')) + self.sess_detect = self.load_sess( + os.path.join(model_root, 'head_segmentation', 'face_detect.pb')) + self.sess_face = self.load_sess( + os.path.join(model_root, 'head_segmentation', 'segment_face.pb')) + + def load_sess(self, model_path): + config = tf.ConfigProto(allow_soft_placement=True) + config.gpu_options.allow_growth = True + sess = tf.Session(config=config) + with tf.gfile.FastGFile(model_path, 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + sess.graph.as_default() + tf.import_graph_def(graph_def, name='') + sess.run(tf.global_variables_initializer()) + return sess + + def process(self, image): + """ image: bgr + """ + + h, w, c = image.shape + faceRects = self.detect_face(image) + face_num = len(faceRects) + all_head_alpha = [] + all_face_mask = [] + for i in range(face_num): + y1 = faceRects[i][0] + y2 = faceRects[i][1] + x1 = faceRects[i][2] + x2 = faceRects[i][3] + pad_y1, pad_y2, pad_x1, pad_x2 = self.pad_box( + y1, y2, x1, x2, 0.15, 0.15, 0.15, 0.15, h, w) + temp_img = image.copy() + roi_img = temp_img[pad_y1:pad_y2, pad_x1:pad_x2] + output_alpha = self.sess_face.run( + self.sess_face.graph.get_tensor_by_name('output_alpha_face:0'), + feed_dict={'input_image_face:0': roi_img[:, :, ::-1]}) + face_mask = np.zeros((h, w, 3)) + face_mask[pad_y1:pad_y2, pad_x1:pad_x2] = output_alpha + all_face_mask.append(face_mask) + cv2.imwrite(str(i) + 'face.jpg', face_mask) + cv2.imwrite(str(i) + 'face_roi.jpg', roi_img) + + for i in range(face_num): + y1 = faceRects[i][0] + y2 = faceRects[i][1] + x1 = faceRects[i][2] + x2 = faceRects[i][3] + pad_y1, pad_y2, pad_x1, pad_x2 = self.pad_box( + y1, y2, x1, x2, 1.47, 1.47, 1.3, 2.0, h, w) + temp_img = image.copy() + for j in range(face_num): + y1 = faceRects[j][0] + y2 = faceRects[j][1] + x1 = faceRects[j][2] + x2 = faceRects[j][3] + small_y1, small_y2, small_x1, small_x2 = self.pad_box( + y1, y2, x1, x2, -0.1, -0.1, -0.1, -0.1, h, w) + small_width = small_x2 - small_x1 + small_height = small_y2 - small_y1 + if (small_x1 < 0 or small_y1 < 0 or small_width < 3 + or small_height < 3 or small_x2 > w or small_y2 > h): + continue + # if(i!=j): + # temp_img[small_y1:small_y2,small_x1:small_x2]=0 + if (i != j): + temp_img = temp_img * (1.0 - all_face_mask[j] / 255.0) + + roi_img = temp_img[pad_y1:pad_y2, pad_x1:pad_x2] + output_alpha = self.sess.run( + self.sess.graph.get_tensor_by_name('output_alpha:0'), + feed_dict={'input_image:0': roi_img[:, :, ::-1]}) + head_alpha = np.zeros((h, w)) + head_alpha[pad_y1:pad_y2, pad_x1:pad_x2] = output_alpha[:, :, 0] + if np.sum(head_alpha) > 255 * w * h * 0.01 * 0.01: + all_head_alpha.append(head_alpha) + + head_num = len(all_head_alpha) + head_elements = [] + if head_num == 0: + return head_elements + + for i in range(head_num): + head_alpha = all_head_alpha[i] + head_elements.append(head_alpha) + + return head_elements + + def pad_box(self, y1, y2, x1, x2, left_ratio, right_ratio, top_ratio, + bottom_ratio, h, w): + box_w = x2 - x1 + box_h = y2 - y1 + pad_y1 = np.maximum(np.int32(y1 - top_ratio * box_h), 0) + pad_y2 = np.minimum(np.int32(y2 + bottom_ratio * box_h), h - 1) + pad_x1 = np.maximum(np.int32(x1 - left_ratio * box_w), 0) + pad_x2 = np.minimum(np.int32(x2 + right_ratio * box_w), w - 1) + return pad_y1, pad_y2, pad_x1, pad_x2 + + def detect_face(self, img): + h, w, c = img.shape + input_img = cv2.resize(img[:, :, ::-1], (512, 512)) + boxes, scores, num_detections = self.sess_detect.run( + [ + self.sess_detect.graph.get_tensor_by_name('tower_0/boxes:0'), + self.sess_detect.graph.get_tensor_by_name('tower_0/scores:0'), + self.sess_detect.graph.get_tensor_by_name( + 'tower_0/num_detections:0') + ], + feed_dict={ + 'tower_0/images:0': input_img[np.newaxis], + 'training_flag:0': False + }) + faceRects = [] + for i in range(num_detections[0]): + if scores[0, i] < 0.5: + continue + y1 = np.int32(boxes[0, i, 0] * h) + x1 = np.int32(boxes[0, i, 1] * w) + y2 = np.int32(boxes[0, i, 2] * h) + x2 = np.int32(boxes[0, i, 3] * w) + if x2 <= x1 + 3 or y2 <= y1 + 3: + continue + faceRects.append((y1, y2, x1, x2, y2 - y1, x2 - x1)) + sorted(faceRects, key=lambda x: x[4] * x[5], reverse=True) + return faceRects + + def generate_json(self, status_code, status_msg, ori_url, result_element, + track_id): + data = {} + data['originUri'] = ori_url + data['elements'] = result_element + data['statusCode'] = status_code + data['statusMessage'] = status_msg + data['requestId'] = track_id + return json.dumps(data) + + def get_box(self, alpha): + h, w = alpha.shape + start_h = 0 + end_h = 0 + start_w = 0 + end_w = 0 + for i in range(0, h, 3): + line = alpha[i, :] + if np.max(line) >= 1: + start_h = i + break + + for i in range(0, w, 3): + line = alpha[:, i] + if np.max(line) >= 1: + start_w = i + break + + for i in range(0, h, 3): + i = h - 1 - i + line = alpha[i, :] + if np.max(line) >= 1: + end_h = i + if end_h < h - 1: + end_h = end_h + 1 + break + for i in range(0, w, 3): + i = w - 1 - i + line = alpha[:, i] + if np.max(line) >= 1: + end_w = i + if end_w < w - 1: + end_w = end_w + 1 + break + + return start_h, start_w, end_h, end_w diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/headrecon_model.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/headrecon_model.py new file mode 100644 index 0000000000000000000000000000000000000000..e515421c1eb58469c65bf8c17bd508638cd95d5c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/headrecon_model.py @@ -0,0 +1,564 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import OrderedDict + +import cv2 +import numpy as np +import torch + +from modelscope.models import MODELS, TorchModel +from modelscope.models.cv.face_reconstruction.utils import (estimate_normals, + read_obj) +from . import networks, opt +from .bfm import ParametricFaceModel +from .losses import (BinaryDiceLoss, TVLoss, TVLoss_std, landmark_loss, + perceptual_loss, photo_loss, points_loss_horizontal, + reflectance_loss, reg_loss) +from .nv_diffrast import MeshRenderer + + +@MODELS.register_module('head-reconstruction', 'head_reconstruction') +class HeadReconModel(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + """The HeadReconModel is implemented based on HRN, publicly available at + https://github.com/youngLBW/HRN + + Args: + model_dir: the root directory of the model files + """ + super().__init__(model_dir, *args, **kwargs) + + self.model_dir = model_dir + opt.bfm_folder = os.path.join(model_dir, 'assets') + self.opt = opt + self.isTrain = opt.isTrain + self.visual_names = ['output_vis'] + self.model_names = ['net_recon'] + self.parallel_names = self.model_names + [ + 'renderer', 'renderer_fitting' + ] + + # networks + self.net_recon = networks.define_net_recon( + net_recon=opt.net_recon, + use_last_fc=opt.use_last_fc, + init_path=None) + + # assets + self.headmodel = ParametricFaceModel( + assets_root=opt.bfm_folder, + camera_distance=opt.camera_d, + focal=opt.focal, + center=opt.center, + is_train=self.isTrain, + default_name='ourRefineBFMEye0504_model.mat') + + self.headmodel_for_fitting = ParametricFaceModel( + assets_root=opt.bfm_folder, + camera_distance=opt.camera_d, + focal=opt.focal, + center=opt.center, + is_train=self.isTrain, + default_name='ourRefineFull_model.mat') + + # renderer + fov = 2 * np.arctan(opt.center / opt.focal) * 180 / np.pi + self.renderer = MeshRenderer( + rasterize_fov=fov, + znear=opt.z_near, + zfar=opt.z_far, + rasterize_size=int(2 * opt.center)) + + self.renderer_fitting = MeshRenderer( + rasterize_fov=fov, + znear=opt.z_near, + zfar=opt.z_far, + rasterize_size=int(2 * opt.center)) + + template_obj_path = os.path.join( + model_dir, + 'assets/3dmm/template_mesh/template_ourFull_bfmEyes.obj') + self.template_output_mesh = read_obj(template_obj_path) + + self.nonlinear_UVs = self.template_output_mesh['uvs'] + self.nonlinear_UVs = torch.from_numpy(self.nonlinear_UVs) + + self.jaw_edge_mask = cv2.imread( + os.path.join(model_dir, + 'assets/texture/jaw_edge_mask2.png'))[..., 0].astype( + np.float32) / 255.0 + self.jaw_edge_mask = cv2.resize(self.jaw_edge_mask, (300, 300))[..., + None] + + self.input_imgs = [] + self.input_img_hds = [] + self.input_fat_img_hds = [] + self.atten_masks = [] + self.gt_lms = [] + self.gt_lm_hds = [] + self.trans_ms = [] + self.img_names = [] + self.face_masks = [] + self.head_masks = [] + self.input_imgs_coeff = [] + self.gt_lms_coeff = [] + + self.loss_names = [ + 'all', 'feat', 'color', 'lm', 'reg', 'gamma', 'reflc' + ] + + self.compute_feat_loss = perceptual_loss + self.comupte_color_loss = photo_loss + self.compute_lm_loss = landmark_loss + self.compute_reg_loss = reg_loss + self.compute_reflc_loss = reflectance_loss + + if opt.isTrain: + self.optimizer = torch.optim.Adam( + self.net_recon.parameters(), lr=opt.lr) + self.optimizers = [self.optimizer] + self.parallel_names += ['net_recog'] + + def set_device(self, device): + self.device = device + self.net_recon = self.net_recon.to(self.device) + self.headmodel.to(self.device) + self.headmodel_for_fitting.to(self.device) + self.nonlinear_UVs = self.nonlinear_UVs.to(self.device) + + def load_networks(self, load_path): + state_dict = torch.load(load_path, map_location=self.device) + print('loading the model from %s' % load_path) + + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + if isinstance(net, torch.nn.DataParallel): + net = net.module + net.load_state_dict(state_dict[name], strict=False) + + def setup(self, checkpoint_path): + """Load and print networks; create schedulers + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + self.load_networks(checkpoint_path) + + def parallelize(self, convert_sync_batchnorm=True): + if not self.opt.use_ddp: + for name in self.parallel_names: + if isinstance(name, str): + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + else: + for name in self.model_names: + if isinstance(name, str): + module = getattr(self, name) + if convert_sync_batchnorm: + module = torch.nn.SyncBatchNorm.convert_sync_batchnorm( + module) + setattr( + self, name, + torch.nn.parallel.DistributedDataParallel( + module.to(self.device), + device_ids=[self.device.index], + find_unused_parameters=True, + broadcast_buffers=True)) + + # DistributedDataParallel is not needed when a module doesn't have any parameter that requires a gradient. + for name in self.parallel_names: + if isinstance(name, str) and name not in self.model_names: + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + + # put state_dict of optimizer to gpu device + if self.opt.phase != 'test': + if self.opt.continue_train: + for optim in self.optimizers: + for state in optim.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(self.device) + + def eval(self): + """Make models eval mode""" + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + net.eval() + + def set_render(self, image_res=1024): + fov = 2 * np.arctan(self.opt.center / self.opt.focal) * 180 / np.pi + if image_res is None: + image_res = int(2 * self.opt.center) + + self.renderer = MeshRenderer( + rasterize_fov=fov, + znear=self.opt.z_near, + zfar=self.opt.z_far, + rasterize_size=image_res) + + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input: a dictionary that contains the data itself and its metadata information. + """ + self.input_img = input['imgs'].to(self.device) + self.input_img_hd = input['imgs_hd'].to( + self.device) if 'imgs_hd' in input else None + + if 'imgs_fat_hd' not in input or input['imgs_fat_hd'] is None: + self.input_fat_img_hd = self.input_img_hd + else: + self.input_fat_img_hd = input['imgs_fat_hd'].to(self.device) + + self.atten_mask = input['msks'].to( + self.device) if 'msks' in input else None + self.gt_lm = input['lms'].to(self.device) if 'lms' in input else None + self.gt_lm_hd = input['lms_hd'].to( + self.device) if 'lms_hd' in input else None + self.trans_m = input['M'].to(self.device) if 'M' in input else None + self.image_paths = input['im_paths'] if 'im_paths' in input else None + self.img_name = input['img_name'] if 'img_name' in input else None + self.face_mask = input['face_mask'].to( + self.device) if 'face_mask' in input else None + self.head_mask = input['head_mask'].to( + self.device) if 'head_mask' in input else None + self.gt_normals = input['normals'].to( + self.device) if 'normals' in input else None + self.input_img_coeff = input['imgs_coeff'].to( + self.device) if 'imgs_coeff' in input else None + self.gt_lm_coeff = input['lms_coeff'].to( + self.device) if 'lms_coeff' in input else None + + def check_head_pose(self, coeffs): + pi = 3.14 + if coeffs[0, 225] > pi / 6 or coeffs[0, 225] < -pi / 6: + return False + elif coeffs[0, 224] > pi / 6 or coeffs[0, 224] < -pi / 6: + return False + elif coeffs[0, 226] > pi / 6 or coeffs[0, 226] < -pi / 6: + return False + else: + return True + + def get_fusion_mask(self, keep_forehead=True): + self.without_forehead_inds = torch.from_numpy( + np.load( + os.path.join(self.model_dir, + 'assets/3dmm/inds/bfm_withou_forehead_inds.npy')) + ).long().to(self.device) + + h, w = self.shape_offset_uv.shape[1:3] + self.fusion_mask = torch.zeros((h, w)).to(self.device).float() + if keep_forehead: + UVs_coords = self.nonlinear_UVs.clone()[:35709][ + self.without_forehead_inds] + else: + UVs_coords = self.nonlinear_UVs.clone()[:35709] + UVs_coords[:, 0] *= w + UVs_coords[:, 1] *= h + UVs_coords_int = torch.floor(UVs_coords) + UVs_coords_int = UVs_coords_int.long() + + self.fusion_mask[h - 1 - UVs_coords_int[:, 1], UVs_coords_int[:, + 0]] = 1 + + # blur mask + self.fusion_mask = self.fusion_mask.cpu().numpy() + new_kernel1 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + new_kernel2 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (8, 8)) + self.fusion_mask = cv2.dilate(self.fusion_mask, new_kernel1, 1) + self.fusion_mask = cv2.erode(self.fusion_mask, new_kernel2, 1) + self.fusion_mask = cv2.blur(self.fusion_mask, (17, 17)) + self.fusion_mask = torch.from_numpy(self.fusion_mask).float().to( + self.device) + + def get_edge_mask(self): + + h, w = self.shape_offset_uv.shape[1:3] + self.edge_mask = torch.zeros((h, w)).to(self.device).float() + UVs_coords = self.nonlinear_UVs.clone()[self.edge_points_inds] + UVs_coords[:, 0] *= w + UVs_coords[:, 1] *= h + UVs_coords_int = torch.floor(UVs_coords) + UVs_coords_int = UVs_coords_int.long() + + self.edge_mask[h - 1 - UVs_coords_int[:, 1], UVs_coords_int[:, 0]] = 1 + + # blur mask + self.edge_mask = self.edge_mask.cpu().numpy() + new_kernel1 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (8, 8)) + self.edge_mask = cv2.dilate(self.edge_mask, new_kernel1, 1) + self.edge_mask = cv2.blur(self.edge_mask, (5, 5)) + self.edge_mask = torch.from_numpy(self.edge_mask).float().to( + self.device) + + def blur_shape_offset_uv(self, global_blur=False, blur_size=3): + if self.edge_mask is not None: + shape_offset_uv_blur = self.shape_offset_uv[0].detach().cpu( + ).numpy() + shape_offset_uv_blur = cv2.blur(shape_offset_uv_blur, (15, 15)) + shape_offset_uv_blur = torch.from_numpy( + shape_offset_uv_blur).float().to(self.device)[None, ...] + self.shape_offset_uv = shape_offset_uv_blur * self.edge_mask[ + None, ..., None] + self.shape_offset_uv * ( + 1 - self.edge_mask[None, ..., None]) + + self.shape_offset_uv = self.shape_offset_uv * self.fusion_mask[None, + ..., + None] + + if global_blur and blur_size > 0: + shape_offset_uv_blur = self.shape_offset_uv[0].detach().cpu( + ).numpy() + shape_offset_uv_blur = cv2.blur(shape_offset_uv_blur, + (blur_size, blur_size)) + shape_offset_uv_blur = torch.from_numpy( + shape_offset_uv_blur).float().to(self.device)[None, ...] + self.shape_offset_uv = shape_offset_uv_blur + + def blur_offset_edge(self): + shape_offset_uv = self.shape_offset_uv[0].detach().cpu().numpy() + shape_offset_uv_head = self.shape_offset_uv_head[0].detach().cpu( + ).numpy() + shape_offset_uv_head = cv2.resize(shape_offset_uv_head, (300, 300)) + shape_offset_uv_head = shape_offset_uv_head * ( + 1 - self.jaw_edge_mask) + shape_offset_uv * self.jaw_edge_mask + shape_offset_uv_head = cv2.resize(shape_offset_uv_head, (100, 100)) + + self.shape_offset_uv_head = torch.from_numpy( + shape_offset_uv_head).float().to(self.device)[None, ...] + + def fitting_nonlinear(self, coeff, n_iters=250): + output_coeff = coeff.detach().clone() + + output_coeff = self.headmodel_for_fitting.split_coeff(output_coeff) + output_coeff['id'].requires_grad = True + output_coeff['exp'].requires_grad = True + output_coeff['tex'].requires_grad = True + output_coeff['angle'].requires_grad = True + output_coeff['gamma'].requires_grad = True + output_coeff['trans'].requires_grad = True + + self.shape_offset_uv = torch.zeros((1, 300, 300, 3), + dtype=torch.float32).to(self.device) + self.shape_offset_uv.requires_grad = True + + self.texture_offset_uv = torch.zeros( + (1, 300, 300, 3), dtype=torch.float32).to(self.device) + self.texture_offset_uv.requires_grad = True + + self.shape_offset_uv_head = torch.zeros( + (1, 100, 100, 3), dtype=torch.float32).to(self.device) + self.shape_offset_uv_head.requires_grad = True + + self.texture_offset_uv_head = torch.zeros( + (1, 100, 100, 3), dtype=torch.float32).to(self.device) + self.texture_offset_uv_head.requires_grad = True + + head_face_inds = np.load( + os.path.join(self.model_dir, + 'assets/3dmm/inds/ours_head_face_inds.npy')) + head_face_inds = torch.from_numpy(head_face_inds).to(self.device) + head_faces = self.headmodel_for_fitting.face_buf[head_face_inds] + + # print('before fitting', output_coeff) + + opt_parameters = [ + self.shape_offset_uv, self.texture_offset_uv, + self.shape_offset_uv_head, self.texture_offset_uv_head, + output_coeff['id'], output_coeff['exp'], output_coeff['tex'], + output_coeff['gamma'] + ] + optim = torch.optim.Adam(opt_parameters, lr=1e-3) + + optim_pose = torch.optim.Adam([output_coeff['trans']], lr=1e-1) + + self.get_edge_points_horizontal() + + for i in range(n_iters): # 500 + self.pred_vertex_head, self.pred_tex, self.pred_color_head, self.pred_lm, face_shape, \ + face_shape_offset, self.verts_proj_head = \ + self.headmodel_for_fitting.compute_for_render_head_fitting(output_coeff, self.shape_offset_uv, + self.texture_offset_uv, + self.shape_offset_uv_head, + self.texture_offset_uv_head, + self.nonlinear_UVs) + self.pred_vertex = self.pred_vertex_head[:, :35241] + self.pred_color = self.pred_color_head[:, :35241] + self.verts_proj = self.verts_proj_head[:, :35241] + self.pred_mask_head, _, self.pred_head, self.occ_head = self.renderer_fitting( + self.pred_vertex_head, head_faces, feat=self.pred_color_head) + self.pred_mask, _, self.pred_face, self.occ_face = self.renderer_fitting( + self.pred_vertex, + self.headmodel_for_fitting.face_buf[:69732], + feat=self.pred_color) + + self.pred_coeffs_dict = self.headmodel_for_fitting.split_coeff( + output_coeff) + self.compute_losses_fitting() + + if i < 150: + optim_pose.zero_grad() + (self.loss_lm + self.loss_color * 0.1).backward() + optim_pose.step() + else: + optim.zero_grad() + self.loss_all.backward() + optim.step() + + output_coeff = self.headmodel_for_fitting.merge_coeff(output_coeff) + + self.get_edge_mask() + self.get_fusion_mask(keep_forehead=False) + self.blur_shape_offset_uv(global_blur=True) + self.blur_offset_edge() + return output_coeff + + def forward(self): + with torch.no_grad(): + output_coeff = self.net_recon(self.input_img_coeff) + + if not self.check_head_pose(output_coeff): + return None + + with torch.enable_grad(): + output_coeff = self.fitting_nonlinear(output_coeff) + + output_coeff = self.headmodel.split_coeff(output_coeff) + eye_coeffs = output_coeff['exp'][0, 16] + output_coeff['exp'][ + 0, 17] + output_coeff['exp'][0, 19] + if eye_coeffs > 1.0: + degree = 0.5 + else: + degree = 1.0 + # degree = 0.5 + output_coeff['exp'][0, 16] += 1 * degree + output_coeff['exp'][0, 17] += 1 * degree + output_coeff['exp'][0, 19] += 1.5 * degree + output_coeff = self.headmodel.merge_coeff(output_coeff) + + self.pred_vertex, _, _, _, face_shape_ori, face_shape, _ = \ + self.headmodel.compute_for_render_head(output_coeff, + self.shape_offset_uv.detach(), + self.texture_offset_uv.detach(), + self.shape_offset_uv_head.detach() * 0, + self.texture_offset_uv_head.detach(), + self.nonlinear_UVs, + nose_coeff=0.1, + neck_coeff=0.3, + neckSlim_coeff=0.5, + neckStretch_coeff=0.5) + + UVs = np.array(self.template_output_mesh['uvs']) + UVs_tensor = torch.tensor(UVs, dtype=torch.float32) + UVs_tensor = torch.unsqueeze(UVs_tensor, 0).to(self.pred_vertex.device) + + target_img = self.input_fat_img_hd + target_img = target_img.permute(0, 2, 3, 1) + face_buf = self.headmodel.face_buf + # get texture map + with torch.enable_grad(): + pred_mask, _, pred_face, texture_map, texture_mask = self.renderer.pred_shape_and_texture( + self.pred_vertex, face_buf, UVs_tensor, target_img, None) + self.pred_coeffs_dict = self.headmodel.split_coeff(output_coeff) + + recon_shape = face_shape # get reconstructed shape, [1, 35709, 3] + recon_shape[ + ..., + -1] = 10 - recon_shape[..., -1] # from camera space to world space + recon_shape = recon_shape.cpu().numpy()[0] + tri = self.headmodel.face_buf.cpu().numpy() + + output = {} + output['flag'] = 0 + + output['tex_map'] = texture_map + output['tex_mask'] = texture_mask * 255.0 + ''' + coeffs + { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + ''' + output['coeffs'] = self.pred_coeffs_dict + + normals = estimate_normals(recon_shape, tri) + + output['vertices'] = recon_shape + output['triangles'] = tri + output['uvs'] = UVs + output['faces_uv'] = self.template_output_mesh['faces_uv'] + output['normals'] = normals + + return output + + def get_edge_points_horizontal(self): + left_points = [] + right_points = [] + for i in range(self.face_mask.shape[2]): + inds = torch.where(self.face_mask[0, 0, i, :] > 0.5) # 0.9 + if len(inds[0]) > 0: # i > 112 and len(inds[0]) > 0 + left_points.append(int(inds[0][0]) + 1) + right_points.append(int(inds[0][-1])) + else: + left_points.append(0) + right_points.append(self.face_mask.shape[3] - 1) + self.left_points = torch.tensor(left_points).long().to(self.device) + self.right_points = torch.tensor(right_points).long().to(self.device) + + def compute_losses_fitting(self): + face_mask = self.pred_mask + face_mask = face_mask.detach() + self.loss_color = self.opt.w_color * self.comupte_color_loss( + self.pred_face, self.input_img, face_mask) # 1.0 + + loss_reg, loss_gamma = self.compute_reg_loss( + self.pred_coeffs_dict, + w_id=self.opt.w_id, + w_exp=self.opt.w_exp, + w_tex=self.opt.w_tex) + self.loss_reg = self.opt.w_reg * loss_reg # 1.0 + self.loss_gamma = self.opt.w_gamma * loss_gamma # 1.0 + + self.loss_lm = self.opt.w_lm * self.compute_lm_loss( + self.pred_lm, self.gt_lm) * 0.1 # 0.1 + + self.loss_smooth_offset = TVLoss()(self.shape_offset_uv.permute( + 0, 3, 1, 2)) * 10000 # 10000 + + self.loss_reg_textureOff = torch.mean( + torch.abs(self.texture_offset_uv)) * 10 # 10 + + self.loss_smooth_offset_std = TVLoss_std()( + self.shape_offset_uv.permute(0, 3, 1, 2)) * 50000 # 50000 + + self.loss_points_horizontal, self.edge_points_inds = points_loss_horizontal( + self.verts_proj, self.left_points, self.right_points) # 20 + self.loss_points_horizontal *= 20 + + self.loss_all = self.loss_color + self.loss_lm + self.loss_reg + self.loss_gamma + self.loss_all += self.loss_smooth_offset + self.loss_smooth_offset_std + self.loss_reg_textureOff + self.loss_all += self.loss_points_horizontal + + head_mask = self.pred_mask_head + head_mask = head_mask.detach() + self.loss_color_head = self.opt.w_color * self.comupte_color_loss( + self.pred_head, self.input_img, head_mask) # 1.0 + self.loss_smooth_offset_head = TVLoss()( + self.shape_offset_uv_head.permute(0, 3, 1, 2)) * 100 # 10000 + self.loss_smooth_offset_std_head = TVLoss_std()( + self.shape_offset_uv_head.permute(0, 3, 1, 2)) * 500 # 50000 + self.loss_mask = BinaryDiceLoss()(self.occ_head, self.head_mask) * 20 + + self.loss_all += self.loss_mask + self.loss_color_head + self.loss_all += self.loss_smooth_offset_head + self.loss_smooth_offset_std_head diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/losses.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..6d4af4e8d875f6809f8defbcb6ad94d6e7a56777 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/losses.py @@ -0,0 +1,367 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from kornia.geometry import warp_affine + + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize)) + + +# perceptual level loss +class PerceptualLoss(nn.Module): + + def __init__(self, recog_net, input_size=112): + super(PerceptualLoss, self).__init__() + self.recog_net = recog_net + self.preprocess = lambda x: 2 * x - 1 + self.input_size = input_size + + def forward(self, imageA, imageB, M): + """ + 1 - cosine distance + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1) , RGB order + imageB --same as imageA + """ + + imageA = self.preprocess(resize_n_crop(imageA, M, self.input_size)) + imageB = self.preprocess(resize_n_crop(imageB, M, self.input_size)) + + # freeze bn + self.recog_net.eval() + + id_featureA = F.normalize(self.recog_net(imageA), dim=-1, p=2) + id_featureB = F.normalize(self.recog_net(imageB), dim=-1, p=2) + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + + +def perceptual_loss(id_featureA, id_featureB): + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + + +# image level loss +def photo_loss(imageA, imageB, mask, eps=1e-6): + """ + l2 norm (with sqrt, to ensure backward stabililty, use eps, otherwise Nan may occur) + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1), RGB order + imageB --same as imageA + """ + loss = torch.sqrt(eps + torch.sum( + (imageA - imageB)**2, dim=1, keepdims=True)) * mask + loss = torch.sum(loss) / torch.max( + torch.sum(mask), + torch.tensor(1.0).to(mask.device)) + return loss + + +def landmark_loss(predict_lm, gt_lm, weight=None): + """ + weighted mse loss + Parameters: + predict_lm --torch.tensor (B, 68, 2) + gt_lm --torch.tensor (B, 68, 2) + weight --numpy.array (1, 68) + """ + if not weight: + weight = np.ones([68]) + weight[28:31] = 20 + weight[-8:] = 20 + weight = np.expand_dims(weight, 0) + weight = torch.tensor(weight).to(predict_lm.device) + loss = torch.sum((predict_lm - gt_lm)**2, dim=-1) * weight + loss = torch.sum(loss) / (predict_lm.shape[0] * predict_lm.shape[1]) + return loss + + +# regulization +def reg_loss(coeffs_dict, w_id=1, w_exp=1, w_tex=1): + """ + l2 norm without the sqrt, from yu's implementation (mse) + tf.nn.l2_loss https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss + Parameters: + coeffs_dict -- a dict of torch.tensors , keys: id, exp, tex, angle, gamma, trans + + """ + # coefficient regularization to ensure plausible 3d faces + value_1 = w_id * torch.sum(coeffs_dict['id']**2) + value_2 = w_exp * torch.sum(coeffs_dict['exp']**2) + value_3 = w_tex * torch.sum(coeffs_dict['tex']**2) + creg_loss = value_1 + value_2 + value_3 + creg_loss = creg_loss / coeffs_dict['id'].shape[0] + + # gamma regularization to ensure a nearly-monochromatic light + gamma = coeffs_dict['gamma'].reshape([-1, 3, 9]) + gamma_mean = torch.mean(gamma, dim=1, keepdims=True) + gamma_loss = torch.mean((gamma - gamma_mean)**2) + + return creg_loss, gamma_loss + + +def reflectance_loss(texture, mask): + """ + minimize texture variance (mse), albedo regularization to ensure an uniform skin albedo + Parameters: + texture --torch.tensor, (B, N, 3) + mask --torch.tensor, (N), 1 or 0 + + """ + mask = mask.reshape([1, mask.shape[0], 1]) + texture_mean = torch.sum( + mask * texture, dim=1, keepdims=True) / torch.sum(mask) + loss = torch.sum(((texture - texture_mean) * mask)**2) / ( + texture.shape[0] * torch.sum(mask)) + return loss + + +def lm_3d_loss(pred_lm_3d, gt_lm_3d, mask): + loss = torch.abs(pred_lm_3d - gt_lm_3d)[mask, :] + loss = torch.mean(loss) + return loss + + +class TVLoss(nn.Module): + + def __init__(self, TVLoss_weight=1): + super(TVLoss, self).__init__() + self.TVLoss_weight = TVLoss_weight + + def forward(self, x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + count_h = self._tensor_size(x[:, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, 1:]) + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, :h_x - 1, :]), 2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, :w_x - 1]), 2).sum() + return self.TVLoss_weight * 2 * (h_tv / count_h + + w_tv / count_w) / batch_size + + def _tensor_size(self, t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +class TVLoss_std(nn.Module): + + def __init__(self, TVLoss_weight=1): + super(TVLoss_std, self).__init__() + self.TVLoss_weight = TVLoss_weight + + def forward(self, x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, :h_x - 1, :]), 2) + h_tv = ((h_tv - torch.mean(h_tv))**2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, :w_x - 1]), 2) + w_tv = ((w_tv - torch.mean(w_tv))**2).sum() + return self.TVLoss_weight * 2 * (h_tv + w_tv) / batch_size + + def _tensor_size(self, t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +def photo_loss_sum(imageA, imageB, mask, eps=1e-6): + """ + l2 norm (with sqrt, to ensure backward stabililty, use eps, otherwise Nan may occur) + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1), RGB order + imageB --same as imageA + """ + loss = torch.sqrt(eps + torch.sum( + (imageA - imageB)**2, dim=1, keepdims=True)) * mask + loss = torch.sum(loss) / ( + imageA.shape[0] * imageA.shape[2] * imageA.shape[3]) + return loss + + +def points_loss_horizontal(verts, left_points, right_points, width=224): + verts_int = torch.ceil(verts[0]).long().clamp(0, width - 1) # (n, 2) + verts_left = left_points[width - 1 - verts_int[:, 1]].float() + verts_right = right_points[width - 1 - verts_int[:, 1]].float() + verts_x = verts[0, :, 0] + dist = (verts_left - verts_x) / width * (verts_right - verts_x) / width + dist /= torch.max( + torch.abs((verts_left - verts_x) / width), + torch.abs((verts_right - verts_x) / width)) + edge_inds = torch.where(dist > 0)[0] + dist += 0.01 + dist = torch.nn.functional.relu(dist).clone() + dist -= 0.01 + dist = torch.abs(dist) + loss = torch.mean(dist) + return loss, edge_inds + + +class LaplacianLoss(nn.Module): + + def __init__(self): + super(LaplacianLoss, self).__init__() + + def forward(self, x): + batch_size, slice_num = x.size()[:2] + z_x = x.size()[2] + h_x = x.size()[3] + w_x = x.size()[4] + count_z = self._tensor_size(x[:, :, 1:, :, :]) + count_h = self._tensor_size(x[:, :, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, :, 1:]) + z_tv = torch.pow((x[:, :, 1:, :, :] - x[:, :, :z_x - 1, :, :]), + 2).sum() + h_tv = torch.pow((x[:, :, :, 1:, :] - x[:, :, :, :h_x - 1, :]), + 2).sum() + w_tv = torch.pow((x[:, :, :, :, 1:] - x[:, :, :, :, :w_x - 1]), + 2).sum() + return 2 * (z_tv / count_z + h_tv / count_h + w_tv / count_w) / ( + batch_size * slice_num) + + def _tensor_size(self, t): + return t.size()[2] * t.size()[3] * t.size()[4] + + +class LaplacianLoss_L1(nn.Module): + + def __init__(self): + super(LaplacianLoss_L1, self).__init__() + + def forward(self, x): + batch_size, slice_num = x.size()[:2] + z_x = x.size()[2] + h_x = x.size()[3] + w_x = x.size()[4] + count_z = self._tensor_size(x[:, :, 1:, :, :]) + count_h = self._tensor_size(x[:, :, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, :, 1:]) + z_tv = torch.abs((x[:, :, 1:, :, :] - x[:, :, :z_x - 1, :, :])).sum() + h_tv = torch.abs((x[:, :, :, 1:, :] - x[:, :, :, :h_x - 1, :])).sum() + w_tv = torch.abs((x[:, :, :, :, 1:] - x[:, :, :, :, :w_x - 1])).sum() + return 2 * (z_tv / count_z + h_tv / count_h + w_tv / count_w) / ( + batch_size * slice_num) + + def _tensor_size(self, t): + return t.size()[2] * t.size()[3] * t.size()[4] + + +class GANLoss(nn.Module): + + def __init__(self, + gan_mode, + target_real_label=1.0, + target_fake_label=0.0, + tensor=torch.FloatTensor): + super(GANLoss, self).__init__() + self.real_label = target_real_label + self.fake_label = target_fake_label + self.real_label_tensor = None + self.fake_label_tensor = None + self.zero_tensor = None + self.Tensor = tensor + self.gan_mode = gan_mode + if gan_mode == 'ls': + pass + elif gan_mode == 'original': + pass + elif gan_mode == 'w': + pass + elif gan_mode == 'hinge': + pass + else: + raise ValueError('Unexpected gan_mode {}'.format(gan_mode)) + + def get_target_tensor(self, input, target_is_real): + if target_is_real: + if self.real_label_tensor is None: + self.real_label_tensor = self.Tensor(1).fill_(self.real_label) + self.real_label_tensor.requires_grad_(False) + return self.real_label_tensor.expand_as(input) + else: + if self.fake_label_tensor is None: + self.fake_label_tensor = self.Tensor(1).fill_(self.fake_label) + self.fake_label_tensor.requires_grad_(False) + return self.fake_label_tensor.expand_as(input) + + def get_zero_tensor(self, input): + if self.zero_tensor is None: + self.zero_tensor = self.Tensor(1).fill_(0) + self.zero_tensor.requires_grad_(False) + return self.zero_tensor.expand_as(input) + + def loss(self, input, target_is_real, for_discriminator=True): + if self.gan_mode == 'original': # cross entropy loss + target_tensor = self.get_target_tensor(input, target_is_real) + loss = F.binary_cross_entropy_with_logits(input, target_tensor) + return loss + elif self.gan_mode == 'ls': + target_tensor = self.get_target_tensor(input, target_is_real) + return F.mse_loss(input, target_tensor) + elif self.gan_mode == 'hinge': + if for_discriminator: + if target_is_real: + minval = torch.min(input - 1, self.get_zero_tensor(input)) + loss = -torch.mean(minval) + else: + minval = torch.min(-input - 1, self.get_zero_tensor(input)) + loss = -torch.mean(minval) + else: + assert target_is_real, "The generator's hinge loss must be aiming for real" + loss = -torch.mean(input) + return loss + else: + # wgan + if target_is_real: + return -input.mean() + else: + return input.mean() + + def __call__(self, input, target_is_real, for_discriminator=True): + # computing loss is a bit complicated because |input| may not be + # a tensor, but list of tensors in case of multiscale discriminator + if isinstance(input, list): + loss = 0 + for pred_i in input: + if isinstance(pred_i, list): + pred_i = pred_i[-1] + loss_tensor = self.loss(pred_i, target_is_real, + for_discriminator) + bs = 1 if len(loss_tensor.size()) == 0 else loss_tensor.size(0) + new_loss = torch.mean(loss_tensor.view(bs, -1), dim=1) + loss += new_loss + return loss / len(input) + else: + return self.loss(input, target_is_real, for_discriminator) + + +class BinaryDiceLoss(nn.Module): + + def __init__(self, smooth=1, p=1, reduction='mean'): + super(BinaryDiceLoss, self).__init__() + self.smooth = smooth + self.p = p + self.reduction = reduction + + def forward(self, predict, target): + assert predict.shape[0] == target.shape[ + 0], "predict & target batch size don't match" + predict = predict.contiguous().view(predict.shape[0], -1) + target = target.contiguous().view(target.shape[0], -1) + + num = torch.sum(torch.mul(predict, target), dim=1) + den = torch.sum(predict + target, dim=1) + + loss = 1 - (2 * num + self.smooth) / (den + self.smooth) + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + elif self.reduction == 'none': + return loss + else: + raise Exception('Unexpected reduction {}'.format(self.reduction)) diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/networks.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb5770b2e935fefff93953927170ee422b7f789 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/networks.py @@ -0,0 +1,577 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch +import os +from typing import Any, Callable, List, Optional, Type, Union + +import torch +import torch.nn as nn +from kornia.geometry import warp_affine +from torch import Tensor +from torch.optim import lr_scheduler + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize)) + + +def filter_state_dict(state_dict, remove_name='fc'): + new_state_dict = {} + for key in state_dict: + if remove_name in key: + continue + new_state_dict[key] = state_dict[key] + return new_state_dict + + +def define_net_recon(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper( + net_recon, use_last_fc=use_last_fc, init_path=init_path) + + +def define_net_recon2(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper2( + net_recon, use_last_fc=use_last_fc, init_path=init_path) + + +class ReconNetWrapper(nn.Module): + fc_dim = 257 + + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', + net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict( + torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print('loading init net_recon %s from %s' % (net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 64, bias=True), # exp layer + conv1x1(last_dim, 80, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +class ReconNetWrapper2(nn.Module): + fc_dim = 264 + + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper2, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', + net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict( + torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print('loading init net_recon %s from %s' % (net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers2 = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 51, bias=True), # exp layer + conv1x1(last_dim, 100, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers2: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers2: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +# adapted from https://github.com/pytorch/vision/edit/master/torchvision/models/resnet.py +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + +model_urls = { + 'resnet18': + 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': + 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': + 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': + 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': + 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': + 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': + 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': + 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': + 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes: int, + out_planes: int, + stride: int = 1, + groups: int = 1, + dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes: int, + out_planes: int, + stride: int = 1, + bias: bool = False) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=bias) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + use_last_fc: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.use_last_fc = use_last_fc + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + if self.use_last_fc: + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, + 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, + 0) # type: ignore[arg-type] + + def _make_layer(self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + if self.use_last_fc: + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet(arch: str, block: Type[Union[BasicBlock, + Bottleneck]], layers: List[int], + pretrained: bool, progress: bool, **kwargs: Any) -> ResNet: + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url( + model_urls[arch], progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +func_dict = {'resnet18': (resnet18, 512), 'resnet50': (resnet50, 2048)} diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/nv_diffrast.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/nv_diffrast.py new file mode 100644 index 0000000000000000000000000000000000000000..f765c6c710e78e6be634fadc31ae19482293e765 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/nv_diffrast.py @@ -0,0 +1,414 @@ +# Part of the implementation is borrowed and modified from Deep3DFaceRecon_pytorch, +# publicly available at https://github.com/sicxu/Deep3DFaceRecon_pytorch +import warnings +from typing import List + +import numpy as np +import nvdiffrast +import nvdiffrast.torch as dr +import torch +import torch.nn.functional as F +from torch import nn + +from .losses import TVLoss, TVLoss_std + +warnings.filterwarnings('ignore') + + +def ndc_projection(x=0.1, n=1.0, f=50.0): + return np.array([[n / x, 0, 0, 0], [0, n / -x, 0, 0], + [0, 0, -(f + n) / (f - n), -(2 * f * n) / (f - n)], + [0, 0, -1, 0]]).astype(np.float32) + + +def to_image(face_shape): + """ + Return: + face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + focal = 1015. + center = 112. + persc_proj = np.array([focal, 0, center, 0, focal, center, 0, 0, + 1]).reshape([3, 3]).astype(np.float32).transpose() + + persc_proj = torch.tensor(persc_proj).to(face_shape.device) + + face_proj = face_shape @ persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + +class MeshRenderer(nn.Module): + + def __init__(self, rasterize_fov, znear=0.1, zfar=10, rasterize_size=224): + super(MeshRenderer, self).__init__() + + x = np.tan(np.deg2rad(rasterize_fov * 0.5)) * znear + self.ndc_proj = torch.tensor(ndc_projection( + x=x, n=znear, + f=zfar)).matmul(torch.diag(torch.tensor([1., -1, -1, 1]))) + self.rasterize_size = rasterize_size + self.glctx = None + + def forward(self, vertex, tri, feat=None): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (B, M, 3) or (M, 3), triangles + feat(optional) -- torch.tensor, size (B, C), features + """ + device = vertex.device + rsize = int(self.rasterize_size) + ndc_proj = self.ndc_proj.to(device) + verts_proj = to_image(vertex) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat( + [vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], + dim=-1) + vertex[..., 1] = -vertex[..., 1] + + vertex_ndc = vertex @ ndc_proj.t() + if self.glctx is None: + if nvdiffrast.__version__ == '0.2.7': + self.glctx = dr.RasterizeGLContext(device=device) + else: + self.glctx = dr.RasterizeCudaContext(device=device) + + ranges = None + if isinstance(tri, List) or len(tri.shape) == 3: + vum = vertex_ndc.shape[1] + fnum = torch.tensor([f.shape[0] + for f in tri]).unsqueeze(1).to(device) + + print('fnum shape:{}'.format(fnum.shape)) + + fstartidx = torch.cumsum(fnum, dim=0) - fnum + ranges = torch.cat([fstartidx, fnum], + axis=1).type(torch.int32).cpu() + for i in range(tri.shape[0]): + tri[i] = tri[i] + i * vum + vertex_ndc = torch.cat(vertex_ndc, dim=0) + tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + rast_out, _ = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + depth, _ = dr.interpolate( + vertex.reshape([-1, 4])[..., 2].unsqueeze(1).contiguous(), + rast_out, tri) + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out[..., 3] > 0).float().unsqueeze(1) + depth = mask * depth + + image = None + + verts_x = verts_proj[0, :, 0] + verts_y = 224 - verts_proj[0, :, 1] + verts_int = torch.ceil(verts_proj[0]).long() # (n, 2) + verts_xr_int = verts_int[:, 0].clamp(1, 224 - 1) + verts_yt_int = 224 - verts_int[:, 1].clamp(2, 224) + verts_right_float = verts_xr_int - verts_x + verts_left_float = 1 - verts_right_float + verts_top_float = verts_y - verts_yt_int + verts_bottom_float = 1 - verts_top_float + + rast_lt = rast_out[0, verts_yt_int, verts_xr_int - 1, 3] + rast_lb = rast_out[0, verts_yt_int + 1, verts_xr_int - 1, 3] + rast_rt = rast_out[0, verts_yt_int, verts_xr_int, 3] + rast_rb = rast_out[0, verts_yt_int + 1, verts_xr_int, 3] + + occ_feat = (rast_lt > 0) * 1.0 * (verts_left_float + verts_top_float) + \ + (rast_lb > 0) * 1.0 * (verts_left_float + verts_bottom_float) + \ + (rast_rt > 0) * 1.0 * (verts_right_float + verts_top_float) + \ + (rast_rb > 0) * 1.0 * (verts_right_float + verts_bottom_float) + occ_feat = occ_feat[None, :, None] / 4.0 + + occ, _ = dr.interpolate(occ_feat, rast_out, tri) + occ = occ.permute(0, 3, 1, 2) + + if feat is not None: + image, _ = dr.interpolate(feat, rast_out, tri) + image = image.permute(0, 3, 1, 2) + image = mask * image + + return mask, depth, image, occ + + def render_uv_texture(self, vertex, tri, uv, uv_texture): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (M, 3), triangles + uv -- torch.tensor, size (B,N, 2), uv mapping + base_tex -- torch.tensor, size (B,H,W,C) + """ + device = vertex.device + rsize = int(self.rasterize_size) + ndc_proj = self.ndc_proj.to(device) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat( + [vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], + dim=-1) + vertex[..., 1] = -vertex[..., 1] + + vertex_ndc = vertex @ ndc_proj.t() + if self.glctx is None: + if nvdiffrast.__version__ == '0.2.7': + self.glctx = dr.RasterizeGLContext(device=device) + else: + self.glctx = dr.RasterizeCudaContext(device=device) + + ranges = None + if isinstance(tri, List) or len(tri.shape) == 3: + vum = vertex_ndc.shape[1] + fnum = torch.tensor([f.shape[0] + for f in tri]).unsqueeze(1).to(device) + + print('fnum shape:{}'.format(fnum.shape)) + + fstartidx = torch.cumsum(fnum, dim=0) - fnum + ranges = torch.cat([fstartidx, fnum], + axis=1).type(torch.int32).cpu() + for i in range(tri.shape[0]): + tri[i] = tri[i] + i * vum + vertex_ndc = torch.cat(vertex_ndc, dim=0) + tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + rast_out, _ = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + depth, _ = dr.interpolate( + vertex.reshape([-1, 4])[..., 2].unsqueeze(1).contiguous(), + rast_out, tri) + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out[..., 3] > 0).float().unsqueeze(1) + depth = mask * depth + uv[..., -1] = 1.0 - uv[..., -1] + + rast_out, rast_db = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + interp_out, uv_da = dr.interpolate( + uv, rast_out, tri, rast_db, diff_attrs='all') + + uv_texture = uv_texture.permute(0, 2, 3, 1).contiguous() + img = dr.texture( + uv_texture, interp_out, filter_mode='linear') # , uv_da) + img = img * torch.clamp(rast_out[..., -1:], 0, + 1) # Mask out background. + + image = img.permute(0, 3, 1, 2) + + return mask, depth, image + + def pred_shape_and_texture(self, + vertex, + tri, + uv, + target_img, + base_tex=None): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (B, M, 3) or (M, 3), triangles + uv -- torch.tensor, size (B,N, 2), uv mapping + base_tex -- torch.tensor, size (B,H,W,C) + """ + uv = uv.clone() + + device = vertex.device + rsize = int(self.rasterize_size) + ndc_proj = self.ndc_proj.to(device) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat( + [vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], + dim=-1) + vertex[..., 1] = -vertex[..., 1] + + vertex_ndc = vertex @ ndc_proj.t() + if self.glctx is None: + if nvdiffrast.__version__ == '0.2.7': + self.glctx = dr.RasterizeGLContext(device=device) + else: + self.glctx = dr.RasterizeCudaContext(device=device) + # print("create glctx on device cuda:%d" % device.index) + + # print('vertex_ndc shape:{}'.format(vertex_ndc.shape)) # Size([1, 35709, 4]) + # print('tri shape:{}'.format(tri.shape)) # Size([70789, 3]) + + ranges = None + if isinstance(tri, List) or len(tri.shape) == 3: + vum = vertex_ndc.shape[1] + fnum = torch.tensor([f.shape[0] + for f in tri]).unsqueeze(1).to(device) + + # print('fnum shape:{}'.format(fnum.shape)) + + fstartidx = torch.cumsum(fnum, dim=0) - fnum + ranges = torch.cat([fstartidx, fnum], + axis=1).type(torch.int32).cpu() + for i in range(tri.shape[0]): + tri[i] = tri[i] + i * vum + vertex_ndc = torch.cat(vertex_ndc, dim=0) + tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + rast_out, _ = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + depth, _ = dr.interpolate( + vertex.reshape([-1, 4])[..., 2].unsqueeze(1).contiguous(), + rast_out, tri) + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out[..., 3] > 0).float().unsqueeze(1) + depth = mask * depth + uv[..., -1] = 1.0 - uv[..., -1] + + rast_out, rast_db = dr.rasterize( + self.glctx, + vertex_ndc.contiguous(), + tri, + resolution=[rsize, rsize], + ranges=ranges) + + interp_out, uv_da = dr.interpolate( + uv, rast_out, tri, rast_db, diff_attrs='all') + + mask_3c = mask.permute(0, 2, 3, 1) + mask_3c = torch.cat((mask_3c, mask_3c, mask_3c), dim=-1) + maskout_img = mask_3c * target_img + mean_color = torch.sum(maskout_img, dim=(1, 2)) + valid_pixel_count = torch.sum(mask) + + mean_color = mean_color / valid_pixel_count + + tex = torch.zeros((1, int(128), 128, 3), dtype=torch.float32) + # tex = torch.zeros((1, 128, 128, 3), dtype=torch.float32) + tex[:, :, :, 0] = mean_color[0, 0] + tex[:, :, :, 1] = mean_color[0, 1] + tex[:, :, :, 2] = mean_color[0, 2] + + tex = tex.cuda() + + tex_mask = torch.zeros((1, int(2048), 2048, 3), dtype=torch.float32) + # tex_mask = torch.zeros((1, 2048, 2048, 3), dtype=torch.float32) + tex_mask[:, :, :, 1] = 1.0 + tex_mask = tex_mask.cuda() + tex_mask.requires_grad = True + tex_mask = tex_mask.contiguous() + + criterionTV = TVLoss() + + if base_tex is not None: + base_tex = base_tex.cuda() + + for tex_resolution in [64, 128, 256, 512, 1024, 2048]: + tex = tex.detach() + tex = tex.permute(0, 3, 1, 2) + tex = F.interpolate(tex, (int(tex_resolution), tex_resolution)) + # tex = F.interpolate(tex, (tex_resolution, tex_resolution)) + tex = tex.permute(0, 2, 3, 1).contiguous() + + if base_tex is not None: + _base_tex = base_tex.permute(0, 3, 1, 2) + _base_tex = F.interpolate( + _base_tex, (int(tex_resolution), tex_resolution)) + # _base_tex = F.interpolate(_base_tex, (tex_resolution, tex_resolution)) + _base_tex = _base_tex.permute(0, 2, 3, 1).contiguous() + tex += _base_tex + + tex.requires_grad = True + + optim = torch.optim.Adam([tex], lr=1e-2) + + texture_opt_iters = 200 + + if tex_resolution == 2048: + optim_mask = torch.optim.Adam([tex_mask], lr=1e-2) + + for i in range(int(texture_opt_iters)): + + if tex_resolution == 2048: + optim_mask.zero_grad() + rendered = dr.texture( + tex_mask, interp_out, filter_mode='linear') # , uv_da) + rendered = rendered * torch.clamp( + rast_out[..., -1:], 0, 1) # Mask out background. + tex_loss = torch.mean((target_img - rendered)**2) + + tex_loss.backward() + optim_mask.step() + + optim.zero_grad() + + img = dr.texture( + tex, interp_out, filter_mode='linear') # , uv_da) + img = img * torch.clamp(rast_out[..., -1:], 0, + 1) # Mask out background. + recon_loss = torch.mean((target_img - img)**2) + + if tex_resolution < 2048: + tv_loss = criterionTV(tex.permute(0, 3, 1, 2)) + + total_loss = recon_loss + tv_loss * 0.01 + else: + + total_loss = recon_loss + + total_loss.backward() + optim.step() + + tex_map = tex[0].detach().cpu().numpy()[..., ::-1] * 255.0 + + image = img.permute(0, 3, 1, 2) + + tex_mask = tex_mask[0].detach().cpu().numpy() * 255.0 + tex_mask = np.where(tex_mask[..., 1] > 250, 1.0, 0.0) * np.where( + tex_mask[..., 0] < 10, 1.0, 0) * np.where(tex_mask[..., 2] < 10, + 1.0, 0) + tex_mask = 1.0 - tex_mask + + return mask, depth, image, tex_map, tex_mask diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/opt.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/opt.py new file mode 100644 index 0000000000000000000000000000000000000000..d6be64b38952b6c264ccd71fbc04f6c8888e0838 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/opt.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +bfm_folder = '' +bfm_model = 'head_model_for_maas.mat' +camera_d = 10.0 +center = 112.0 +focal = 1015.0 +isTrain = False +net_recon = 'resnet50' +phase = 'test' +use_ddp = False +use_last_fc = False +z_far = 15.0 +z_near = 5.0 +lr = 0.001 +w_color = 1.92 +w_reg = 3.0e-4 +w_gamma = 10.0 +w_lm = 1.6e-3 +w_id = 1.0 +w_exp = 0.8 +w_tex = 1.7e-2 diff --git a/runtime/Lib/site_packages/models/cv/head_reconstruction/models/tex_processor.py b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/tex_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..e202ab7d97c11e14adf8b01ab8b4fd0fff90216d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/head_reconstruction/models/tex_processor.py @@ -0,0 +1,145 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import cv2 +import numpy as np + + +def get_fade_out_mask(length, start_value, end_value, fade_start_ratio, + fade_end_ratio): + fade_start_ind = int(length * fade_start_ratio) + fade_end_ind = int(length * fade_end_ratio) + + left_part = np.array([start_value] * fade_start_ind) + fade_part = np.linspace(start_value, end_value, + fade_end_ind - fade_start_ind) + len_right = length - len(left_part) - len(fade_part) + right_part = np.array([end_value] * len_right) + + fade_out_mask = np.concatenate([left_part, fade_part, right_part], axis=0) + return fade_out_mask + + +class TexProcesser(): + + def __init__(self, model_root): + + self.tex_size = 4096 + + self.bald_tex_bg = cv2.imread( + os.path.join(model_root, + 'assets/texture/template_bald_tex_2.jpg')).astype( + np.float32) + self.hair_tex_bg = cv2.imread( + os.path.join(model_root, + 'assets/texture/template_withHair_tex.jpg')).astype( + np.float32) + + self.hair_mask = cv2.imread( + os.path.join(model_root, + 'assets/texture/hair_mask_male.png'))[..., 0].astype( + np.float32) / 255.0 + self.hair_mask = cv2.resize(self.hair_mask, (4096, 4096 + 1024)) + + front_mask = cv2.imread( + os.path.join(model_root, + 'assets/texture/face_mask_singleview.jpg')).astype( + np.float32) / 255 + front_mask = cv2.resize(front_mask, (1024, 1024)) + front_mask = cv2.resize(front_mask, (0, 0), fx=0.1, fy=0.1) + front_mask = cv2.erode(front_mask, + np.ones(shape=(7, 7), dtype=np.float32)) + front_mask = cv2.GaussianBlur(front_mask, (13, 13), 0) + self.front_mask = cv2.resize(front_mask, + (self.tex_size, self.tex_size)) + self.binary_front_mask = self.front_mask.copy() + self.binary_front_mask[(self.front_mask < 0.3) + + (self.front_mask > 0.7)] = 0 + self.binary_front_mask[self.binary_front_mask != 0] = 1.0 + self.binary_front_mask_ = self.binary_front_mask.copy() + self.binary_front_mask_[:int(4096 * 375 / 950)] = 0 + self.binary_front_mask_[int(4096 * 600 / 950):] = 0 + self.binary_front_mask = np.zeros((4096 + 1024, 4096, 3), + dtype=np.float32) + self.binary_front_mask[:4096, :] = self.binary_front_mask_ + self.front_mask_ = self.front_mask.copy() + self.front_mask = np.zeros((4096 + 1024, 4096, 3), dtype=np.float32) + self.front_mask[:4096, :] = self.front_mask_ + + self.fg_mask = cv2.imread( + os.path.join(model_root, + 'assets/texture/fg_mask.png'))[..., 0].astype( + np.float32) / 255.0 + self.fg_mask = cv2.resize(self.fg_mask, (256, 256)) + self.fg_mask = cv2.dilate(self.fg_mask, + np.ones(shape=(13, 13), dtype=np.float32)) + self.fg_mask = cv2.blur(self.fg_mask, (27, 27), 0) + self.fg_mask = cv2.resize(self.fg_mask, (4096, 4096 + 1024)) + self.fg_mask = self.fg_mask[..., None] + + self.cheek_mask = cv2.imread( + os.path.join(model_root, + 'assets/texture/cheek_area_mask.png'))[..., 0].astype( + np.float32) / 255.0 + self.cheek_mask = cv2.resize(self.cheek_mask, (4096, 4096 + 1024)) + self.cheek_mask = self.cheek_mask[..., None] + + self.bald_tex_bg = self.bald_tex_bg[:4096] + self.hair_tex_bg = self.hair_tex_bg[:4096] + self.fg_mask = self.fg_mask[:4096] + self.hair_mask = self.hair_mask[:4096] + self.front_mask = self.front_mask[:4096] + self.binary_front_mask = self.binary_front_mask[:4096] + self.front_mask_ = self.front_mask_[:4096] + + self.cheek_mask_left = self.cheek_mask[:4096] + self.cheek_mask_right = self.cheek_mask[:4096].copy()[:, ::-1] + + def post_process_texture(self, tex_map, hair_tex=True): + tex_map = cv2.resize(tex_map, (self.tex_size, self.tex_size)) + + # if hair_tex is true and there is a dark side, use the mirror texture + if hair_tex: + left_cheek_light_mean = np.mean( + tex_map[self.cheek_mask_left[..., 0] == 1.0]) + right_cheek_light_mean = np.mean( + tex_map[self.cheek_mask_right[..., 0] == 1.0]) + + tex_map_flip = tex_map[:, ::-1, :] + w = tex_map.shape[1] + half_w = w // 2 + if left_cheek_light_mean > right_cheek_light_mean * 1.5: + tex_map[:, half_w:, :] = tex_map_flip[:, half_w:, :] + elif right_cheek_light_mean > left_cheek_light_mean * 2: + tex_map[:, :half_w, :] = tex_map_flip[:, :half_w, :] + + # change the color of template texture + bg_mean_rgb = np.mean( + self.bald_tex_bg[self.binary_front_mask[..., 0] == 1.0], + axis=0)[None, None] + pred_tex_mean_rgb = np.mean( + tex_map[self.binary_front_mask[..., 0] == 1.0], axis=0)[None, + None] * 1.1 + _bald_tex_bg = self.bald_tex_bg.copy() + _bald_tex_bg = self.bald_tex_bg + (pred_tex_mean_rgb - bg_mean_rgb) + + if hair_tex: + # inpaint hair + tex_gray = cv2.cvtColor( + tex_map.astype(np.uint8), + cv2.COLOR_BGR2GRAY).astype(np.float32) + hair_mask = (self.hair_mask == 1.0) * (tex_gray < 120) + hair_bgr = np.mean(tex_map[hair_mask, :], axis=0) * 0.5 + if hair_bgr is None: + hair_bgr = 20.0 + _bald_tex_bg[self.hair_mask == 1.0] = hair_bgr + + # fuse + tex_map = _bald_tex_bg * (1. + - self.fg_mask) + tex_map * self.fg_mask + else: + # fuse + tex_map = _bald_tex_bg * ( + 1. - self.front_mask) + tex_map * self.front_mask + + return tex_map diff --git a/runtime/Lib/site_packages/models/cv/human3d_animation/__init__.py b/runtime/Lib/site_packages/models/cv/human3d_animation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..07f94b108c73566bbde886b781e7fce34ec93c01 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human3d_animation/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .generate_skeleton import gen_skeleton_bvh + from .utils import (read_obj, write_obj, render, rotate_x, rotate_y, + translate, projection) + +else: + _import_structure = { + 'generate_skeleton': ['gen_skeleton_bvh'], + 'utils': [ + 'read_obj', 'write_obj', 'render', 'rotate_x', 'rotate_y', + 'translate', 'projection' + ], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/human3d_animation/bvh_writer.py b/runtime/Lib/site_packages/models/cv/human3d_animation/bvh_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..beacdffe4f6047d564754efed30598b0cdaa60d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human3d_animation/bvh_writer.py @@ -0,0 +1,184 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch + +from .transforms import aa2quat, batch_rodrigues, mat2aa, quat2euler + + +def write_bvh(parent, + offset, + rotation, + position, + names, + frametime, + order, + path, + endsite=None): + file = open(path, 'w') + frame = rotation.shape[0] + joint_num = rotation.shape[1] + order = order.upper() + + file_string = 'HIERARCHY\n' + + seq = [] + + def write_static(idx, prefix): + nonlocal parent, offset, rotation, names + nonlocal order, endsite, file_string, seq + seq.append(idx) + if idx == 0: + name_label = 'ROOT ' + names[idx] + channel_label = 'CHANNELS 6 Xposition Yposition Zposition \ + {}rotation {}rotation {}rotation'.format(*order) + else: + name_label = 'JOINT ' + names[idx] + channel_label = 'CHANNELS 3 {}rotation {}rotation \ + {}rotation'.format(*order) + offset_label = 'OFFSET %.6f %.6f %.6f' % ( + offset[idx][0], offset[idx][1], offset[idx][2]) + + file_string += prefix + name_label + '\n' + file_string += prefix + '{\n' + file_string += prefix + '\t' + offset_label + '\n' + file_string += prefix + '\t' + channel_label + '\n' + + has_child = False + for y in range(idx + 1, rotation.shape[1]): + if parent[y] == idx: + has_child = True + write_static(y, prefix + '\t') + if not has_child: + file_string += prefix + '\t' + 'End Site\n' + file_string += prefix + '\t' + '{\n' + file_string += prefix + '\t\t' + 'OFFSET 0 0 0\n' + file_string += prefix + '\t' + '}\n' + + file_string += prefix + '}\n' + + write_static(0, '') + + file_string += 'MOTION\n' + 'Frames: {}\n'.format( + frame) + 'Frame Time: %.8f\n' % frametime + for i in range(frame): + file_string += '%.6f %.6f %.6f ' % (position[i][0], position[i][1], + position[i][2]) + + for j in range(joint_num): + idx = seq[j] + file_string += '%.6f %.6f %.6f ' % ( + rotation[i][idx][0], rotation[i][idx][1], rotation[i][idx][2]) + + file_string += '\n' + + file.write(file_string) + return file_string + + +class WriterWrapper: + + def __init__(self, parents): + self.parents = parents + + def axis2euler(self, rot): + rot = rot.reshape(rot.shape[0], -1, 3) # 45, 24, 3 + quat = aa2quat(rot) + euler = quat2euler(quat, order='xyz') + rot = euler + return rot + + def mapper_rot_mixamo(self, rot, n_bone): + rot = rot.reshape(rot.shape[0], -1, 3) + + smpl_mapper = [ + 0, 1, 6, 11, 2, 7, 12, 3, 8, 13, 4, 9, 14, 17, 21, 15, 18, 22, 19, + 23, 20, 24 + ] + + if n_bone > 24: + hand_mapper = list(range(25, 65)) + smpl_mapper += hand_mapper + + new_rot = torch.zeros((rot.shape[0], n_bone, 3)) # n, 24, 3 + new_rot[:, :len(smpl_mapper), :] = rot[:, smpl_mapper, :] + + return new_rot + + def transform_rot_with_restpose(self, rot, rest_pose, node_list, n_bone): + + rest_pose = batch_rodrigues(rest_pose.reshape(-1, 3)).reshape( + 1, n_bone, 3, 3) # N*3-> N*3*3 + + frame_num = rot.shape[0] + rot = rot.reshape(rot.shape[0], -1, 3) + new_rot = rot.clone() + for k in range(frame_num): + action_rot = batch_rodrigues(rot[k].reshape(-1, 3)).reshape( + 1, n_bone, 3, 3) + for i in node_list: + rot1 = rest_pose[0, i, :, :] + rot2 = action_rot[0, i, :, :] + nrot = torch.matmul(rot2, torch.inverse(rot1)) + nvec = mat2aa(nrot) + new_rot[k, i, :] = nvec + + new_rot = self.axis2euler(new_rot) # =# 45,24,3 + return new_rot + + def transform_rot_with_stdApose(self, rot, rest_pose): + print('transform_rot_with_stdApose') + rot = rot.reshape(rot.shape[0], -1, 3) + rest_pose = self.axis2euler(rest_pose) + print(rot.shape) + print(rest_pose.shape) + smpl_left_arm_idx = 18 + smpl_right_arm_idx = 19 + std_arm_rot = torch.tensor([[21.7184, -4.8148, 16.3985], + [-20.1108, 10.7190, -8.9279]]) + x = rest_pose[:, smpl_left_arm_idx:smpl_right_arm_idx + 1, :] + delta = (x - std_arm_rot) + rot[:, smpl_left_arm_idx:smpl_right_arm_idx + 1, :] -= delta + return rot + + def write(self, + filename, + offset, + rot=None, + action_loc=None, + rest_pose=None, + correct_arm=0): # offset: [24,3], rot:[45,72] + if not isinstance(offset, torch.Tensor): + offset = torch.tensor(offset) + n_bone = offset.shape[0] # 24 + pos = offset[0].unsqueeze(0) # 1,3 + + if rot is None: + rot = np.zeros((1, n_bone, 3)) + else: # rot: 45, 72 + if rest_pose is None: + rot = self.mapper_rot_mixamo(rot, n_bone) + else: + if correct_arm == 1: + rot = self.mapper_rot_mixamo(rot, n_bone) + print(rot.shape) + node_list_chage = [16, 17] + n_bone = rot.shape[1] + print(rot[0, 19, :]) + else: + node_list_chage = [1, 2, 3, 6, 9, 12, 13, 14, 15, 16, 17] + rot = self.transform_rot_with_restpose( + rot, rest_pose, node_list_chage, n_bone) + + rest = torch.zeros((1, n_bone * 3)) + rest = self.axis2euler(rest) + frames_add = 1 + rest = rest.repeat(frames_add, 1, 1) + rot = torch.cat((rest, rot), 0) + + pos = pos.repeat(rot.shape[0], 1) + action_len = action_loc.shape[0] + pos[-action_len:, :] = action_loc[..., :] + + names = ['%02d' % i for i in range(n_bone)] + write_bvh(self.parents, offset, rot, pos, names, 0.0333, 'xyz', + filename) diff --git a/runtime/Lib/site_packages/models/cv/human3d_animation/generate_skeleton.py b/runtime/Lib/site_packages/models/cv/human3d_animation/generate_skeleton.py new file mode 100644 index 0000000000000000000000000000000000000000..556cdbd372f620d80ed851b3e5f6888052522bf9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human3d_animation/generate_skeleton.py @@ -0,0 +1,167 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import pickle + +import numpy as np +import torch + +from .bvh_writer import WriterWrapper +from .utils import matrix_to_axis_angle, rotation_6d_to_matrix + + +def laod_smpl_params(pose_fname): + with open(pose_fname, 'rb') as f: + data = pickle.load(f) + pose = torch.from_numpy(data['pose']) + beta = torch.from_numpy(data['betas']) + trans = torch.from_numpy(data['trans']) + if 'joints' in data: + joints = torch.from_numpy(data['joints']) + joints = joints.reshape(1, -1, 3) + else: + joints = None + trans = trans.reshape(1, 3) + beta = beta.reshape(1, -1)[:, :10] + pose = pose.reshape(-1, 24 * 3) + return pose, beta, trans, joints + + +def set_pose_param(pose, start, end): + pose[:, start * 3:(end + 1) * 3] = 0 + return pose + + +def load_test_anim(filename, device, mode='move'): + anim = np.load(filename) + anim = torch.tensor(anim, device=device, dtype=torch.float) + poses = anim[:, :-3] + loc = anim[:, -3:] + if os.path.basename(filename)[:5] == 'comb_': + loc = loc / 100 + repeat = 0 + idx = -1 + for i in range(poses.shape[0]): + if i == 0: + continue + if repeat >= 5: + idx = i + break + if poses[i].equal(poses[i - 1]): + repeat += 1 + else: + repeat = 0 + poses = poses[:idx - 5, :] + loc = loc[:idx - 5, :] + + if mode == 'inplace': + loc[1:, :] = loc[0, :] + + return poses, loc + + +def load_syn_motion(filename, device, mode='move'): + data = np.load(filename, allow_pickle=True).item() + anim = data['thetas'] + n_joint, c, t = anim.shape + + anim = torch.tensor(anim, device=device, dtype=torch.float) + anim = anim.permute(2, 0, 1) # 180, 24, 6 + poses = anim.reshape(-1, 6) + poses = rotation_6d_to_matrix(poses) + poses = matrix_to_axis_angle(poses) + poses = poses.reshape(-1, 24, 3) + + loc = data['root_translation'] + loc = torch.tensor(loc, device=device, dtype=torch.float) + loc = loc.permute(1, 0) + + if mode == 'inplace': + loc = torch.zeros((t, 3)) + + print('load %s' % filename) + + return poses, loc + + +def load_action(action_name, + model_dir, + action_dir, + mode='move', + device=torch.device('cpu')): + action_path = os.path.join(action_dir, action_name + '.npy') + if not os.path.exists(action_path): + print('can not find action %s, use default action instead' % + (action_name)) + action_path = os.path.join(model_dir, '3D-assets', 'SwingDancing.npy') + print('load action %s' % action_path) + test_pose, test_loc = load_test_anim( + action_path, device, mode=mode) # pose:[45,72], loc:[45,1,3] + + return test_pose, test_loc + + +def load_action_list(action, + model_dir, + action_dir, + mode='move', + device=torch.device('cpu')): + action_list = action.split(',') + test_pose, test_loc = load_action( + action_list[0], model_dir, action_dir, mode=mode, device=device) + final_loc = test_loc[-1, :] + idx = 0 + if len(action_list) > 1: + for action in action_list: + if idx == 0: + idx += 1 + continue + print('load action %s' % action) + pose, loc = load_action( + action, model_dir, action_dir, mode=mode, device=device) + delta_loc = final_loc - loc[0, :] + loc += delta_loc + final_loc = loc[-1, :] + test_pose = torch.cat([test_pose, pose], 0) + test_loc = torch.cat([test_loc, loc], 0) + idx += 1 + return test_pose, test_loc + + +def gen_skeleton_bvh(model_dir, action_dir, case_dir, action, mode='move'): + outpath_a = os.path.join(case_dir, 'skeleton_a.bvh') + device = torch.device('cpu') + assets_dir = os.path.join(model_dir, '3D-assets') + pkl_path = os.path.join(assets_dir, 'smpl.pkl') + poses, shapes, trans, joints = laod_smpl_params(pkl_path) + if action.endswith('.npy'): + skeleton_path = os.path.join(assets_dir, 'skeleton_nohand.npy') + else: + skeleton_path = os.path.join(assets_dir, 'skeleton.npy') + data = np.load(skeleton_path, allow_pickle=True).item() + skeleton = data['skeleton'] + parent = data['parent'] + skeleton = skeleton.squeeze(0) + bvh_writer = WriterWrapper(parent) + + if action.endswith('.npy'): + action_path = action + print('load action %s' % action_path) + test_pose, test_loc = load_syn_motion(action_path, device, mode=mode) + bvh_writer.write( + outpath_a, + skeleton, + test_pose, + action_loc=test_loc, + rest_pose=poses) + + else: + print('load action %s' % action) + test_pose, test_loc = load_action_list( + action, model_dir, action_dir, mode='move', device=device) + std_y = torch.tensor(0.99) + test_loc = test_loc + (skeleton[0, 1] - std_y) + bvh_writer.write(outpath_a, skeleton, test_pose, action_loc=test_loc) + + print('save %s' % outpath_a) + + return 0 diff --git a/runtime/Lib/site_packages/models/cv/human3d_animation/transforms.py b/runtime/Lib/site_packages/models/cv/human3d_animation/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..388c34ad995c5958f8077090b27baea668178b68 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human3d_animation/transforms.py @@ -0,0 +1,316 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/facebookresearch/pytorch3d +# All Rights Reserved. +# ------------------------------------------------------------------------ +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor + + +def batch_mm(matrix, matrix_batch): + """ + https://github.com/pytorch/pytorch/issues/14489#issuecomment-607730242 + :param matrix: Sparse or dense matrix, size (m, n). + :param matrix_batch: Batched dense matrices, size (b, n, k). + :return: The batched matrix-matrix product, + size (m, n) x (b, n, k) = (b, m, k). + """ + batch_size = matrix_batch.shape[0] + # Stack the vector batch into columns. (b, n, k) -> (n, b, k) -> (n, b*k) + vectors = matrix_batch.transpose(0, 1).reshape(matrix.shape[1], -1) + + # A matrix-matrix product is a batched matrix-vector + # product of the columns. + # And then reverse the reshaping. + # (m, n) x (n, b*k) = (m, b*k) -> (m, b, k) -> (b, m, k) + return matrix.mm(vectors).reshape(matrix.shape[0], batch_size, + -1).transpose(1, 0) + + +def aa2quat(rots, form='wxyz', unified_orient=True): + """ + Convert angle-axis representation to wxyz quaternion + and to the half plan (w >= 0) + @param rots: angle-axis rotations, (*, 3) + @param form: quaternion format, either 'wxyz' or 'xyzw' + @param unified_orient: Use unified orientation for quaternion + (quaternion is dual cover of SO3) + :return: + """ + angles = rots.norm(dim=-1, keepdim=True) + norm = angles.clone() + norm[norm < 1e-8] = 1 + axis = rots / norm + quats = torch.empty( + rots.shape[:-1] + (4, ), device=rots.device, dtype=rots.dtype) + angles = angles * 0.5 + if form == 'wxyz': + quats[..., 0] = torch.cos(angles.squeeze(-1)) + quats[..., 1:] = torch.sin(angles) * axis + elif form == 'xyzw': + quats[..., :3] = torch.sin(angles) * axis + quats[..., 3] = torch.cos(angles.squeeze(-1)) + + if unified_orient: + idx = quats[..., 0] < 0 + quats[idx, :] *= -1 + + return quats + + +def quat2aa(quats): + """ + Convert wxyz quaternions to angle-axis representation + :param quats: + :return: + """ + _cos = quats[..., 0] + xyz = quats[..., 1:] + _sin = xyz.norm(dim=-1) + norm = _sin.clone() + norm[norm < 1e-7] = 1 + axis = xyz / norm.unsqueeze(-1) + angle = torch.atan2(_sin, _cos) * 2 + return axis * angle.unsqueeze(-1) + + +def quat2mat(quats: torch.Tensor): + """ + Convert (w, x, y, z) quaternions to 3x3 rotation matrix + :param quats: quaternions of shape (..., 4) + :return: rotation matrices of shape (..., 3, 3) + """ + qw = quats[..., 0] + qx = quats[..., 1] + qy = quats[..., 2] + qz = quats[..., 3] + + x2 = qx + qx + y2 = qy + qy + z2 = qz + qz + xx = qx * x2 + yy = qy * y2 + wx = qw * x2 + xy = qx * y2 + yz = qy * z2 + wy = qw * y2 + xz = qx * z2 + zz = qz * z2 + wz = qw * z2 + + m = torch.empty( + quats.shape[:-1] + (3, 3), device=quats.device, dtype=quats.dtype) + m[..., 0, 0] = 1.0 - (yy + zz) + m[..., 0, 1] = xy - wz + m[..., 0, 2] = xz + wy + m[..., 1, 0] = xy + wz + m[..., 1, 1] = 1.0 - (xx + zz) + m[..., 1, 2] = yz - wx + m[..., 2, 0] = xz - wy + m[..., 2, 1] = yz + wx + m[..., 2, 2] = 1.0 - (xx + yy) + + return m + + +def quat2euler(q, order='xyz', degrees=True): + """ + Convert (w, x, y, z) quaternions to xyz euler angles. + This is used for bvh output. + """ + q0 = q[..., 0] + q1 = q[..., 1] + q2 = q[..., 2] + q3 = q[..., 3] + es = torch.empty(q0.shape + (3, ), device=q.device, dtype=q.dtype) + + if order == 'xyz': + es[..., 2] = torch.atan2(2 * (q0 * q3 - q1 * q2), + q0 * q0 + q1 * q1 - q2 * q2 - q3 * q3) + es[..., 1] = torch.asin((2 * (q1 * q3 + q0 * q2)).clip(-1, 1)) + es[..., 0] = torch.atan2(2 * (q0 * q1 - q2 * q3), + q0 * q0 - q1 * q1 - q2 * q2 + q3 * q3) + else: + raise NotImplementedError('Cannot convert to ordering %s' % order) + + if degrees: + es = es * 180 / np.pi + + return es + + +def aa2mat(rots): + """ + Convert angle-axis representation to rotation matrix + :param rots: angle-axis representation + :return: + """ + quat = aa2quat(rots) + mat = quat2mat(quat) + return mat + + +def inv_affine(mat): + """ + Calculate the inverse of any affine transformation + """ + affine = torch.zeros((mat.shape[:2] + (1, 4))) + affine[..., 3] = 1 + vert_mat = torch.cat((mat, affine), dim=2) + vert_mat_inv = torch.inverse(vert_mat) + return vert_mat_inv[..., :3, :] + + +def inv_rigid_affine(mat): + """ + Calculate the inverse of a rigid affine transformation + """ + res = mat.clone() + res[..., :3] = mat[..., :3].transpose(-2, -1) + res[..., + 3] = -torch.matmul(res[..., :3], mat[..., 3].unsqueeze(-1)).squeeze(-1) + return res + + +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + ret[positive_mask] = torch.sqrt(x[positive_mask]) + return ret + + +def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to quaternions. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f'Invalid rotation matrix shape {matrix.shape}.') + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( + matrix.reshape(batch_dim + (9, )), dim=-1) + + q_abs = _sqrt_positive_part( + torch.stack( + [ + 1.0 + m00 + m11 + m22, + 1.0 + m00 - m11 - m22, + 1.0 - m00 + m11 - m22, + 1.0 - m00 - m11 + m22, + ], + dim=-1, + )) + + # we produce the desired quaternion multiplied by each of r, i, j, k + quat_by_rijk = torch.stack( + [ + torch.stack([q_abs[..., 0]**2, m21 - m12, m02 - m20, m10 - m01], + dim=-1), + torch.stack([m21 - m12, q_abs[..., 1]**2, m10 + m01, m02 + m20], + dim=-1), + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2]**2, m12 + m21], + dim=-1), + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3]**2], + dim=-1), + ], + dim=-2, + ) + + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + return quat_candidates[F.one_hot(q_abs.argmax( + dim=-1), num_classes=4) > 0.5, :].reshape(batch_dim + (4, )) + + +def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to axis/angle. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True) + half_angles = torch.atan2(norms, quaternions[..., :1]) + angles = 2 * half_angles + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles]) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48) + return quaternions[..., 1:] / sin_half_angles_over_angles + + +def mat2aa(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to axis/angle. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + return quaternion_to_axis_angle(matrix_to_quaternion(matrix)) + + +def batch_rodrigues(rot_vecs: Tensor, epsilon: float = 1e-8) -> Tensor: + ''' Calculates the rotation matrices for a batch of rotation vectors + Parameters + ---------- + rot_vecs: torch.tensor Nx3 + array of N axis-angle vectors + Returns + ------- + R: torch.tensor Nx3x3 + The rotation matrices for the given axis-angle parameters + ''' + assert len(rot_vecs.shape) == 2, ( + f'Expects an array of size Bx3, but received {rot_vecs.shape}') + + batch_size = rot_vecs.shape[0] + device = rot_vecs.device + dtype = rot_vecs.dtype + + angle = torch.norm(rot_vecs + epsilon, dim=1, keepdim=True, p=2) + rot_dir = rot_vecs / angle + + cos = torch.unsqueeze(torch.cos(angle), dim=1) + sin = torch.unsqueeze(torch.sin(angle), dim=1) + + # Bx1 arrays + rx, ry, rz = torch.split(rot_dir, 1, dim=1) + K = torch.zeros((batch_size, 3, 3), dtype=dtype, device=device) + + zeros = torch.zeros((batch_size, 1), dtype=dtype, device=device) + K = torch.cat([zeros, -rz, ry, rz, zeros, -rx, -ry, rx, zeros], dim=1) \ + .view((batch_size, 3, 3)) + + ident = torch.eye(3, dtype=dtype, device=device).unsqueeze(dim=0) + rot_mat = ident + sin * K + (1 - cos) * torch.bmm(K, K) + return rot_mat diff --git a/runtime/Lib/site_packages/models/cv/human3d_animation/utils.py b/runtime/Lib/site_packages/models/cv/human3d_animation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6be9fb253b8c9a0d7833f74243786fff215fe3fb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human3d_animation/utils.py @@ -0,0 +1,375 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import cv2 +import numpy as np +import nvdiffrast.torch as dr +import torch +import torch.nn.functional as F + + +def read_obj(obj_path, print_shape=False): + with open(obj_path, 'r') as f: + bfm_lines = f.readlines() + + vertices = [] + faces = [] + uvs = [] + vns = [] + faces_uv = [] + faces_normal = [] + max_face_length = 0 + for line in bfm_lines: + if line[:2] == 'v ': + vertex = [ + float(a) for a in line.strip().split(' ')[1:] if len(a) > 0 + ] + vertices.append(vertex) + + if line[:2] == 'f ': + items = line.strip().split(' ')[1:] + face = [int(a.split('/')[0]) for a in items if len(a) > 0] + max_face_length = max(max_face_length, len(face)) + faces.append(face) + + if '/' in items[0] and len(items[0].split('/')[1]) > 0: + face_uv = [int(a.split('/')[1]) for a in items if len(a) > 0] + faces_uv.append(face_uv) + + if '/' in items[0] and len(items[0].split('/')) >= 3 and len( + items[0].split('/')[2]) > 0: + face_normal = [ + int(a.split('/')[2]) for a in items if len(a) > 0 + ] + faces_normal.append(face_normal) + + if line[:3] == 'vt ': + items = line.strip().split(' ')[1:] + uv = [float(a) for a in items if len(a) > 0] + uvs.append(uv) + + if line[:3] == 'vn ': + items = line.strip().split(' ')[1:] + vn = [float(a) for a in items if len(a) > 0] + vns.append(vn) + + vertices = np.array(vertices).astype(np.float32) + if max_face_length <= 3: + faces = np.array(faces).astype(np.int32) + else: + print('not a triangle face mesh!') + + if vertices.shape[1] == 3: + mesh = { + 'vertices': vertices, + 'faces': faces, + } + else: + mesh = { + 'vertices': vertices[:, :3], + 'colors': vertices[:, 3:], + 'faces': faces, + } + + if len(uvs) > 0: + uvs = np.array(uvs).astype(np.float32) + mesh['uvs'] = uvs + + if len(vns) > 0: + vns = np.array(vns).astype(np.float32) + mesh['normals'] = vns + + if len(faces_uv) > 0: + if max_face_length <= 3: + faces_uv = np.array(faces_uv).astype(np.int32) + mesh['faces_uv'] = faces_uv + + if len(faces_normal) > 0: + if max_face_length <= 3: + faces_normal = np.array(faces_normal).astype(np.int32) + mesh['faces_normal'] = faces_normal + + if print_shape: + print('num of vertices', len(vertices)) + print('num of faces', len(faces)) + return mesh + + +def write_obj(save_path, mesh): + save_dir = os.path.dirname(save_path) + save_name = os.path.splitext(os.path.basename(save_path))[0] + + if 'texture_map' in mesh: + cv2.imwrite( + os.path.join(save_dir, save_name + '.png'), mesh['texture_map']) + + with open(os.path.join(save_dir, save_name + '.mtl'), 'w') as wf: + wf.write('newmtl material_0\n') + wf.write('Ka 1.000000 0.000000 0.000000\n') + wf.write('Kd 1.000000 1.000000 1.000000\n') + wf.write('Ks 0.000000 0.000000 0.000000\n') + wf.write('Tr 0.000000\n') + wf.write('illum 0\n') + wf.write('Ns 0.000000\n') + wf.write('map_Kd {}\n'.format(save_name + '.png')) + + with open(save_path, 'w') as wf: + if 'texture_map' in mesh: + wf.write('# Create by ModelScope\n') + wf.write('mtllib ./{}.mtl\n'.format(save_name)) + + if 'colors' in mesh: + for i, v in enumerate(mesh['vertices']): + wf.write('v {} {} {} {} {} {}\n'.format( + v[0], v[1], v[2], mesh['colors'][i][0], + mesh['colors'][i][1], mesh['colors'][i][2])) + else: + for v in mesh['vertices']: + wf.write('v {} {} {}\n'.format(v[0], v[1], v[2])) + + if 'uvs' in mesh: + for uv in mesh['uvs']: + wf.write('vt {} {}\n'.format(uv[0], uv[1])) + + if 'normals' in mesh: + for vn in mesh['normals']: + wf.write('vn {} {} {}\n'.format(vn[0], vn[1], vn[2])) + + if 'faces' in mesh: + for ind, face in enumerate(mesh['faces']): + if 'faces_uv' in mesh or 'faces_normal' in mesh: + if 'faces_uv' in mesh: + face_uv = mesh['faces_uv'][ind] + else: + face_uv = face + if 'faces_normal' in mesh: + face_normal = mesh['faces_normal'][ind] + else: + face_normal = face + row = 'f ' + ' '.join([ + '{}/{}/{}'.format(face[i], face_uv[i], face_normal[i]) + for i in range(len(face)) + ]) + '\n' + else: + row = 'f ' + ' '.join( + ['{}'.format(face[i]) + for i in range(len(face))]) + '\n' + wf.write(row) + + +def projection(x=0.1, n=1.0, f=50.0): + return np.array([[n / x, 0, 0, 0], [0, n / x, 0, 0], + [0, 0, -(f + n) / (f - n), -(2 * f * n) / (f - n)], + [0, 0, -1, 0]]).astype(np.float32) + + +def translate(x, y, z): + return np.array([[1, 0, 0, x], [0, 1, 0, y], [0, 0, 1, z], + [0, 0, 0, 1]]).astype(np.float32) + + +def rotate_x(a): + s, c = np.sin(a), np.cos(a) + return np.array([[1, 0, 0, 0], [0, c, s, 0], [0, -s, c, 0], + [0, 0, 0, 1]]).astype(np.float32) + + +def rotate_y(a): + s, c = np.sin(a), np.cos(a) + return np.array([[c, 0, s, 0], [0, 1, 0, 0], [-s, 0, c, 0], + [0, 0, 0, 1]]).astype(np.float32) + + +def dot(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + return torch.sum(x * y, -1, keepdim=True) + + +def reflect(x: torch.Tensor, n: torch.Tensor) -> torch.Tensor: + return 2 * dot(x, n) * n - x + + +def length(x: torch.Tensor, eps: float = 1e-20) -> torch.Tensor: + return torch.sqrt(torch.clamp( + dot(x, x), + min=eps)) # Clamp to avoid nan gradients because grad(sqrt(0)) = NaN + + +def safe_normalize(x: torch.Tensor, eps: float = 1e-20) -> torch.Tensor: + return x / length(x, eps) + + +def transform_pos(mtx, pos): + t_mtx = torch.from_numpy(mtx).cuda() if isinstance(mtx, + np.ndarray) else mtx + posw = torch.cat([pos, torch.ones([pos.shape[0], 1]).cuda()], axis=1) + return torch.matmul(posw, t_mtx.t())[None, ...] + + +def render(glctx, mtx, pos, pos_idx, uv, uv_idx, tex, resolution, enable_mip, + max_mip_level): + pos_clip = transform_pos(mtx, pos) + rast_out, rast_out_db = dr.rasterize( + glctx, pos_clip, pos_idx, resolution=[resolution, resolution]) + + if enable_mip: + texc, texd = dr.interpolate( + uv[None, ...], + rast_out, + uv_idx, + rast_db=rast_out_db, + diff_attrs='all') + color = dr.texture( + tex[None, ...], + texc, + texd, + filter_mode='linear-mipmap-linear', + max_mip_level=max_mip_level) + else: + texc, _ = dr.interpolate(uv[None, ...], rast_out, uv_idx) + color = dr.texture(tex[None, ...], texc, filter_mode='linear') + + pos_idx = pos_idx.type(torch.long) + v0 = pos[pos_idx[:, 0], :] + v1 = pos[pos_idx[:, 1], :] + v2 = pos[pos_idx[:, 2], :] + face_normals = safe_normalize(torch.cross(v1 - v0, v2 - v0)) + face_normal_indices = (torch.arange( + 0, face_normals.shape[0], dtype=torch.int64, + device='cuda')[:, None]).repeat(1, 3) + gb_geometric_normal, _ = dr.interpolate(face_normals[None, ...], rast_out, + face_normal_indices.int()) + normal = (gb_geometric_normal + 1) * 0.5 + mask = torch.clamp(rast_out[..., -1:], 0, 1) + color = color * mask + (1 - mask) * torch.ones_like(color) + normal = normal * mask + (1 - mask) * torch.ones_like(normal) + + return color, mask, normal + + +# The following code is based on https://github.com/Mathux/ACTOR.git +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +# Check PYTORCH3D_LICENCE before use + + +def _copysign(a, b): + """ + Return a tensor where each element has the absolute value taken from the, + corresponding element of a, with sign taken from the corresponding + element of b. This is like the standard copysign floating-point operation, + but is not careful about negative 0 and NaN. + + Args: + a: source tensor. + b: tensor whose signs will be used, of the same shape as a. + + Returns: + Tensor of the same shape as a with the signs of b. + """ + signs_differ = (a < 0) != (b < 0) + return torch.where(signs_differ, -a, a) + + +def _sqrt_positive_part(x): + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + ret[positive_mask] = torch.sqrt(x[positive_mask]) + return ret + + +def matrix_to_quaternion(matrix): + """ + Convert rotations given as rotation matrices to quaternions. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f'Invalid rotation matrix shape f{matrix.shape}.') + m00 = matrix[..., 0, 0] + m11 = matrix[..., 1, 1] + m22 = matrix[..., 2, 2] + o0 = 0.5 * _sqrt_positive_part(1 + m00 + m11 + m22) + x = 0.5 * _sqrt_positive_part(1 + m00 - m11 - m22) + y = 0.5 * _sqrt_positive_part(1 - m00 + m11 - m22) + z = 0.5 * _sqrt_positive_part(1 - m00 - m11 + m22) + o1 = _copysign(x, matrix[..., 2, 1] - matrix[..., 1, 2]) + o2 = _copysign(y, matrix[..., 0, 2] - matrix[..., 2, 0]) + o3 = _copysign(z, matrix[..., 1, 0] - matrix[..., 0, 1]) + return torch.stack((o0, o1, o2, o3), -1) + + +def quaternion_to_axis_angle(quaternions): + """ + Convert rotations given as quaternions to axis/angle. + + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True) + half_angles = torch.atan2(norms, quaternions[..., :1]) + angles = 2 * half_angles + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles]) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48) + return quaternions[..., 1:] / sin_half_angles_over_angles + + +def matrix_to_axis_angle(matrix): + """ + Convert rotations given as rotation matrices to axis/angle. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + return quaternion_to_axis_angle(matrix_to_quaternion(matrix)) + + +def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor: + """ + Converts 6D rotation representation by Zhou et al. [1] to rotation matrix + using Gram--Schmidt orthogonalisation per Section B of [1]. + Args: + d6: 6D rotation representation, of size (*, 6) + + Returns: + batch of rotation matrices of size (*, 3, 3) + + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + + a1, a2 = d6[..., :3], d6[..., 3:] + b1 = F.normalize(a1, dim=-1) + b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1 + b2 = F.normalize(b2, dim=-1) + b3 = torch.cross(b1, b2, dim=-1) + return torch.stack((b1, b2, b3), dim=-2) diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/__init__.py b/runtime/Lib/site_packages/models/cv/human_image_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a8d5b81b9a61b2a0d0b492e2c1a9a1d03b32c22 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .human_image_generation_infer import FreqHPTForHumanImageGeneration + +else: + _import_structure = { + 'human_image_generation_infer': ['FreqHPTForHumanImageGeneration'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/__init__.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_function.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_function.py new file mode 100644 index 0000000000000000000000000000000000000000..e6eca6f5db3daf81918e8e0fd6cd95f5d831de70 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_function.py @@ -0,0 +1,717 @@ +import collections +import math +import sys + +import torch +from pytorch_wavelets import DWTForward, DWTInverse +from torch import kl_div, nn +from torch.nn import functional as F + +from modelscope.ops.human_image_generation.fused_act import (FusedLeakyReLU, + fused_leaky_relu) +from modelscope.ops.human_image_generation.upfirdn2d import upfirdn2d +from .conv2d_gradfix import conv2d, conv_transpose2d +from .wavelet_module import * + + +# add flow +class ExtractionOperation_flow(nn.Module): + + def __init__(self, in_channel, num_label, match_kernel): + super(ExtractionOperation_flow, self).__init__() + self.value_conv = EqualConv2d( + in_channel, + in_channel, + match_kernel, + 1, + match_kernel // 2, + bias=True) + self.semantic_extraction_filter = EqualConv2d( + in_channel, + num_label, + match_kernel, + 1, + match_kernel // 2, + bias=False) + + self.softmax = nn.Softmax(dim=-1) + self.num_label = num_label + + def forward(self, value, recoder): + key = value + b, c, h, w = value.shape + key = self.semantic_extraction_filter(self.feature_norm(key)) + extraction_softmax = self.softmax(key.view(b, -1, h * w)) + values_flatten = self.value_conv(value).view(b, -1, h * w) + neural_textures = torch.einsum('bkm,bvm->bvk', extraction_softmax, + values_flatten) + recoder['extraction_softmax'].insert(0, extraction_softmax) + recoder['neural_textures'].insert(0, neural_textures) + return neural_textures, extraction_softmax + + def feature_norm(self, input_tensor): + input_tensor = input_tensor - input_tensor.mean(dim=1, keepdim=True) + norm = torch.norm( + input_tensor, 2, 1, keepdim=True) + sys.float_info.epsilon + out = torch.div(input_tensor, norm) + return out + + +class DistributionOperation_flow(nn.Module): + + def __init__(self, num_label, input_dim, match_kernel=3): + super(DistributionOperation_flow, self).__init__() + self.semantic_distribution_filter = EqualConv2d( + input_dim, + num_label, + kernel_size=match_kernel, + stride=1, + padding=match_kernel // 2) + self.num_label = num_label + + def forward(self, query, extracted_feature, recoder): + b, c, h, w = query.shape + + query = self.semantic_distribution_filter(query) + query_flatten = query.view(b, self.num_label, -1) + query_softmax = F.softmax(query_flatten, 1) + values_q = torch.einsum('bkm,bkv->bvm', query_softmax, + extracted_feature.permute(0, 2, 1)) + attn_out = values_q.view(b, -1, h, w) + recoder['semantic_distribution'].append(query) + return attn_out + + +class EncoderLayer_flow(nn.Sequential): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + use_extraction=False, + num_label=None, + match_kernel=None, + num_extractions=2): + super().__init__() + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + self.blur = Blur(blur_kernel, pad=(pad0, pad1)) + + stride = 2 + padding = 0 + + else: + self.blur = None + stride = 1 + padding = kernel_size // 2 + + self.conv = EqualConv2d( + in_channel, + out_channel, + kernel_size, + padding=padding, + stride=stride, + bias=bias and not activate, + ) + + self.activate = FusedLeakyReLU( + out_channel, bias=bias) if activate else None + self.use_extraction = use_extraction + if self.use_extraction: + self.extraction_operations = nn.ModuleList() + for _ in range(num_extractions): + self.extraction_operations.append( + ExtractionOperation_flow(out_channel, num_label, + match_kernel)) + + def forward(self, input, recoder=None): + out = self.blur(input) if self.blur is not None else input + out = self.conv(out) + out = self.activate(out) if self.activate is not None else out + if self.use_extraction: + for extraction_operation in self.extraction_operations: + extraction_operation(out, recoder) + return out + + +class DecoderLayer_flow_wavelet_fuse24(nn.Module): + + # add fft refinement and tps + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + upsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + use_distribution=True, + num_label=16, + match_kernel=3, + wavelet_down_level=False, + window_size=8, + ): + super().__init__() + if upsample: + factor = 2 + p = (len(blur_kernel) - factor) - (kernel_size - 1) + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + 1 + + self.blur = Blur( + blur_kernel, pad=(pad0, pad1), upsample_factor=factor) + self.conv = EqualTransposeConv2d( + in_channel, + out_channel, + kernel_size, + stride=2, + padding=0, + bias=bias and not activate, + ) + else: + self.conv = EqualConv2d( + in_channel, + out_channel, + kernel_size, + stride=1, + padding=kernel_size // 2, + bias=bias and not activate, + ) + self.blur = None + + self.distribution_operation = DistributionOperation_flow( + num_label, out_channel, + match_kernel=match_kernel) if use_distribution else None + self.activate = FusedLeakyReLU( + out_channel, bias=bias) if activate else None + self.use_distribution = use_distribution + + # mask prediction network + if use_distribution: + self.conv_mask_lf = nn.Sequential(*[ + EqualConv2d( + out_channel, 1, 3, stride=1, padding=3 // 2, bias=False), + nn.Sigmoid() + ]) + self.conv_mask_dict = nn.ModuleDict() + for level in range(wavelet_down_level): + conv_mask = nn.Sequential(*[ + EqualConv2d( + out_channel, + 1, + 3, + stride=1, + padding=3 // 2, + bias=False), + nn.Sigmoid() + ]) + self.conv_mask_dict[str(level)] = conv_mask + + self.wavelet_down_level = wavelet_down_level + if wavelet_down_level: + self.dwt = DWTForward( + J=self.wavelet_down_level, mode='zero', wave='haar') + self.idwt = DWTInverse(mode='zero', wave='haar') + + # for mask input channel squeeze and expand + self.conv_l_squeeze = EqualConv2d( + 2 * out_channel, out_channel, 1, 1, 0, bias=False) + self.conv_h_squeeze = EqualConv2d( + 6 * out_channel, out_channel, 1, 1, 0, bias=False) + + self.conv_l = EqualConv2d( + out_channel, out_channel, 3, 1, 3 // 2, bias=False) + + self.hf_modules = nn.ModuleDict() + for level in range(wavelet_down_level): + hf_module = nn.Module() + prev_channel = out_channel if level == self.wavelet_down_level - 1 else 3 * out_channel + hf_module.conv_prev = EqualConv2d( + prev_channel, 3 * out_channel, 3, 1, 3 // 2, bias=False) + hf_module.conv_hf = GatedConv2dWithActivation( + 3 * out_channel, 3 * out_channel, 3, 1, 3 // 2, bias=False) + hf_module.conv_out = GatedConv2dWithActivation( + 3 * out_channel, 3 * out_channel, 3, 1, 3 // 2, bias=False) + self.hf_modules[str(level)] = hf_module + + self.amp_fuse = nn.Sequential( + EqualConv2d(2 * out_channel, out_channel, 1, 1, 0), + FusedLeakyReLU(out_channel, bias=False), + EqualConv2d(out_channel, out_channel, 1, 1, 0)) + self.pha_fuse = nn.Sequential( + EqualConv2d(2 * out_channel, out_channel, 1, 1, 0), + FusedLeakyReLU(out_channel, bias=False), + EqualConv2d(out_channel, out_channel, 1, 1, 0)) + self.post = EqualConv2d(out_channel, out_channel, 1, 1, 0) + self.eps = 1e-8 + + def forward(self, + input, + neural_texture=None, + recoder=None, + warped_texture=None, + style_net=None, + gstyle=None): + out = self.conv(input) + out = self.blur(out) if self.blur is not None else out + + mask_l, mask_h = None, None + out_attn = None + if self.use_distribution and neural_texture is not None: + out_ori = out + out_attn = self.distribution_operation(out, neural_texture, + recoder) + # wavelet fusion + if self.wavelet_down_level: + assert out.shape[2] % 2 == 0, \ + f'out shape {out.shape} is not appropriate for processing' + b, c, h, w = out.shape + + # wavelet decomposition + LF_attn, HF_attn = self.dwt(out_attn) + LF_warp, HF_warp = self.dwt(warped_texture) + LF_out, HF_out = self.dwt(out) + + # generate mask + hf_dict = {} + l_mask_input = torch.cat([LF_attn, LF_warp], dim=1) + l_mask_input = self.conv_l_squeeze(l_mask_input) + l_mask_input = style_net(l_mask_input, gstyle) + ml = self.conv_mask_lf(l_mask_input) + mask_l = ml + + for level in range(self.wavelet_down_level): + # level up, feature size down + scale = 2**(level + 1) + hfa = HF_attn[level].view(b, c * 3, h // scale, w // scale) + hfw = HF_warp[level].view(b, c * 3, h // scale, w // scale) + hfg = HF_out[level].view(b, c * 3, h // scale, w // scale) + + h_mask_input = torch.cat([hfa, hfw], dim=1) + h_mask_input = self.conv_h_squeeze(h_mask_input) + h_mask_input = style_net(h_mask_input, gstyle) + mh = self.conv_mask_dict[str(level)](h_mask_input) + if level == 0: + mask_h = mh + + # fuse high frequency + xh = (mh * hfa + (1 - mh) * hfw + hfg) / math.sqrt(2) + hf_dict[str(level)] = xh + + temp_result = (1 - ml) * LF_warp + LF_out + out_l = (ml * LF_attn + temp_result) / math.sqrt(2) + out_h_list = [] + for level in range(self.wavelet_down_level - 1, -1, -1): + xh = hf_dict[str(level)] + b, c, h, w = xh.shape + out_h_list.append(xh.view(b, c // 3, 3, h, w)) + out_h_list = ( + out_h_list)[::-1] # the h list from large to small size + # + out = self.idwt((out_l, out_h_list)) + else: + out = (out + out_attn) / math.sqrt(2) + + # fourier refinement + _, _, H, W = out.shape + fuseF = torch.fft.rfft2(out + self.eps, norm='backward') + outF = torch.fft.rfft2(out_ori + self.eps, norm='backward') + amp = self.amp_fuse( + torch.cat([torch.abs(fuseF), torch.abs(outF)], 1)) + pha = self.pha_fuse( + torch.cat( + [torch.angle(fuseF), torch.angle(outF)], 1)) + out_fft = torch.fft.irfft2( + amp * torch.exp(1j * pha) + self.eps, + s=(H, W), + dim=(-2, -1), + norm='backward') + + out = out + self.post(out_fft) + + out = self.activate( + out.contiguous()) if self.activate is not None else out + return out, mask_h, mask_l + + +# base functions + + +class EqualConv2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + bias=True, + dilation=1): + super().__init__() + + self.weight = nn.Parameter( + torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size**2) + + self.stride = stride + self.padding = padding + self.dilation = dilation + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + + else: + self.bias = None + + def forward(self, input): + out = conv2d( + input, + self.weight * self.scale, + bias=self.bias, + stride=self.stride, + padding=self.padding, + dilation=self.dilation) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class EqualTransposeConv2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + bias=True): + super().__init__() + + self.weight = nn.Parameter( + torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size**2) + + self.stride = stride + self.padding = padding + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + + else: + self.bias = None + + def forward(self, input): + weight = self.weight.transpose(0, 1) + out = conv_transpose2d( + input, + weight * self.scale, + bias=self.bias, + stride=self.stride, + padding=self.padding, + ) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class ToRGB(nn.Module): + + def __init__(self, in_channel, upsample=True, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + if upsample: + self.upsample = Upsample(blur_kernel) + self.conv = EqualConv2d(in_channel, 3, 3, stride=1, padding=1) + + def forward(self, input, skip=None): + out = self.conv(input) + if skip is not None: + skip = self.upsample(skip) + out = out + skip + return out + + +class EqualLinear(nn.Module): + + def __init__(self, + in_dim, + out_dim, + bias=True, + bias_init=0, + lr_mul=1, + activation=None): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) + + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) + + else: + self.bias = None + + self.activation = activation + + self.scale = (1 / math.sqrt(in_dim)) * lr_mul + self.lr_mul = lr_mul + + def forward(self, input): + if self.activation: + out = F.linear(input, self.weight * self.scale) + out = fused_leaky_relu(out, self.bias * self.lr_mul) + + else: + out = F.linear( + input, self.weight * self.scale, bias=self.bias * self.lr_mul) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})' + ) + + +class Upsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) * (factor**2) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=self.factor, down=1, pad=self.pad) + + return out + + +class ResBlock(nn.Module): + + def __init__(self, + in_channel, + out_channel, + blur_kernel=[1, 3, 3, 1], + downsample=True): + super().__init__() + + self.conv1 = ConvLayer(in_channel, in_channel, 3) + self.conv2 = ConvLayer( + in_channel, out_channel, 3, downsample=downsample) + + self.skip = ConvLayer( + in_channel, + out_channel, + 1, + downsample=downsample, + activate=False, + bias=False) + + def forward(self, input): + out = self.conv1(input) + out = self.conv2(out) + + skip = self.skip(input) + out = (out + skip) / math.sqrt(2) + + return out + + +class ConvLayer(nn.Sequential): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + ): + layers = [] + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + layers.append(Blur(blur_kernel, pad=(pad0, pad1))) + + stride = 2 + self.padding = 0 + + else: + stride = 1 + self.padding = kernel_size // 2 + + layers.append( + EqualConv2d( + in_channel, + out_channel, + kernel_size, + padding=self.padding, + stride=stride, + bias=bias and not activate, + )) + + if activate: + layers.append(FusedLeakyReLU(out_channel, bias=bias)) + + super().__init__(*layers) + + +class Blur(nn.Module): + + def __init__(self, kernel, pad, upsample_factor=1): + super().__init__() + + kernel = make_kernel(kernel) + + if upsample_factor > 1: + kernel = kernel * (upsample_factor**2) + + self.register_buffer('kernel', kernel) + + self.pad = pad + + def forward(self, input): + out = upfirdn2d(input, self.kernel, pad=self.pad) + + return out + + +class GatedConv2dWithActivation(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + bias=True, + activation=None): + super(GatedConv2dWithActivation, self).__init__() + self.activation = FusedLeakyReLU(out_channels, bias=False) + self.conv2d = EqualConv2d(in_channels, out_channels, kernel_size, + stride, padding, bias, dilation) + self.mask_conv2d = EqualConv2d(in_channels, out_channels, kernel_size, + stride, padding, bias, dilation) + self.sigmoid = nn.Sigmoid() + + def gated(self, mask): + return self.sigmoid(mask) + + def forward(self, input): + x = self.conv2d(input) + mask = self.mask_conv2d(input) + if self.activation is not None: + x = self.activation(x) * self.gated(mask) + else: + x = x * self.gated(mask) + + return x + + +def make_kernel(k): + k = torch.tensor(k, dtype=torch.float32) + + if k.ndim == 1: + k = k[None, :] * k[:, None] + + k /= k.sum() + + return k + + +class SPDNorm(nn.Module): + + def __init__(self, + norm_channel, + label_nc, + norm_type='position', + use_equal=False): + super().__init__() + param_free_norm_type = norm_type + ks = 3 + if param_free_norm_type == 'instance': + self.param_free_norm = nn.InstanceNorm2d( + norm_channel, affine=False) + elif param_free_norm_type == 'batch': + self.param_free_norm = nn.BatchNorm2d(norm_channel, affine=False) + elif param_free_norm_type == 'position': + self.param_free_norm = PositionalNorm2d + else: + raise ValueError( + '%s is not a recognized param-free norm type in SPADE' + % param_free_norm_type) + + # The dimension of the intermediate embedding space. Yes, hardcoded. + pw = ks // 2 + nhidden = 128 + if not use_equal: + self.mlp_activate = nn.Sequential( + nn.Conv2d(label_nc, nhidden, kernel_size=ks, padding=pw), + nn.ReLU()) + self.mlp_gamma = nn.Conv2d( + nhidden, norm_channel, kernel_size=ks, padding=pw) + self.mlp_beta = nn.Conv2d( + nhidden, norm_channel, kernel_size=ks, padding=pw) + else: + self.mlp_activate = nn.Sequential(*[ + EqualConv2d(label_nc, nhidden, kernel_size=ks, padding=pw), + FusedLeakyReLU(nhidden, bias=False) + ]) + self.mlp_gamma = EqualConv2d( + nhidden, norm_channel, kernel_size=ks, padding=pw) + self.mlp_beta = EqualConv2d( + nhidden, norm_channel, kernel_size=ks, padding=pw) + + def forward(self, x, prior_f, weight=1.0): + normalized = self.param_free_norm(x) + # Part 2. produce scaling and bias conditioned on condition feature + actv = self.mlp_activate(prior_f) + gamma = self.mlp_gamma(actv) * weight + beta = self.mlp_beta(actv) * weight + # apply scale and bias + out = normalized * (1 + gamma) + beta + return out + + +def PositionalNorm2d(x, epsilon=1e-5): + # x: B*C*W*H normalize in C dim + mean = x.mean(dim=1, keepdim=True) + std = x.var(dim=1, keepdim=True).add(epsilon).sqrt() + output = (x - mean) / std + return output diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_module.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_module.py new file mode 100644 index 0000000000000000000000000000000000000000..a774b37e178ccbe2c28aad958da5eeec5a216612 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/base_module.py @@ -0,0 +1,358 @@ +import collections +import functools +import math +from tkinter.ttk import Style + +import numpy as np +import torch.nn as nn +import torch.nn.functional as F + +from .base_function import * +from .flow_module import MaskStyle, StyleFlow +from .tps import TPS + + +# adding flow version +class Encoder_wiflow(nn.Module): + + def __init__( + self, + size, + input_dim, + channels, + num_labels=None, + match_kernels=None, + blur_kernel=[1, 3, 3, 1], + ): + super().__init__() + self.first = EncoderLayer_flow(input_dim, channels[size], 1) + self.convs = nn.ModuleList() + self.num_labels = num_labels + self.match_kernels = match_kernels + + log_size = int(math.log(size, 2)) + self.log_size = log_size + + in_channel = channels[size] + for i in range(log_size - 1, 3, -1): + out_channel = channels[2**i] + num_label = num_labels[2**i] if num_labels is not None else None + match_kernel = match_kernels[ + 2**i] if match_kernels is not None else None + use_extraction = num_label and match_kernel + conv = EncoderLayer_flow( + in_channel, + out_channel, + kernel_size=3, + downsample=True, + blur_kernel=blur_kernel, + use_extraction=use_extraction, + num_label=num_label, + match_kernel=match_kernel) + + self.convs.append(conv) + in_channel = out_channel + + def forward(self, input, recoder=None, out_list=None): + out = self.first(input) + for layer in self.convs: + out = layer(out, recoder) + if out_list is not None: + out_list.append(out) + return out + + +class Decoder_wiflow_wavelet_fuse25(nn.Module): + + def __init__( + self, + size, + channels, + num_labels, + match_kernels, + blur_kernel=[1, 3, 3, 1], + wavelet_down_levels={'16': 3}, + window_size=8, + ): + super().__init__() + + self.convs = nn.ModuleList() + # input at resolution 16*16 + in_channel = channels[16] + self.log_size = int(math.log(size, 2)) + self.conv_mask_dict = nn.ModuleDict() + self.conv_mask_fuse_dict = nn.ModuleDict() + + flow_fusion = False + + for i in range(4, self.log_size + 1): + out_channel = channels[2**i] + num_label, match_kernel = num_labels[2**i], match_kernels[2**i] + use_distribution = num_label and match_kernel + upsample = (i != 4) + wavelet_down_level = wavelet_down_levels[(2**i)] + base_layer = functools.partial( + DecoderLayer_flow_wavelet_fuse24, + out_channel=out_channel, + kernel_size=3, + blur_kernel=blur_kernel, + use_distribution=use_distribution, + num_label=num_label, + match_kernel=match_kernel, + wavelet_down_level=wavelet_down_level, + window_size=window_size) + # mask head for fusion + if use_distribution: + conv_mask = [ + EqualConv2d( + 2 * out_channel, + 3, + 3, + stride=1, + padding=3 // 2, + bias=False), + nn.Sigmoid() + ] + conv_mask = nn.Sequential(*conv_mask) + self.conv_mask_dict[str(2**i)] = conv_mask + + if not i == 4: + conv_mask_fuse = nn.Sequential(*[ + EqualConv2d( + 2, 1, 3, stride=1, padding=3 // 2, bias=False), + nn.Sigmoid() + ]) + self.conv_mask_fuse_dict[str(2**i)] = conv_mask_fuse + + if not flow_fusion: + self.conv_flow_fusion = nn.Sequential( + EqualConv2d( + 2 * out_channel, + 1, + kernel_size=7, + stride=1, + padding=3, + bias=False), nn.Sigmoid()) + flow_fusion = True + + up = nn.Module() + up.conv0 = base_layer(in_channel=in_channel, upsample=upsample) + up.conv1 = base_layer(in_channel=out_channel, upsample=False) + up.to_rgb = ToRGB(out_channel, upsample=upsample) + self.convs.append(up) + in_channel = out_channel + + style_in_channels = channels[16] + self.style_out_channel = 128 + self.cond_style = nn.Sequential( + nn.Conv2d( + style_in_channels, + self.style_out_channel, + kernel_size=3, + stride=1, + padding=1), nn.LeakyReLU(inplace=False, negative_slope=0.1), + nn.AdaptiveAvgPool2d(1)) + self.image_style = nn.Sequential( + nn.Conv2d( + style_in_channels, + self.style_out_channel, + kernel_size=3, + stride=1, + padding=1), nn.LeakyReLU(inplace=False, negative_slope=0.1), + nn.AdaptiveAvgPool2d(1)) + self.flow_model = StyleFlow( + channels, self.log_size, style_in=2 * self.style_out_channel) + + self.num_labels, self.match_kernels = num_labels, match_kernels + + # for mask prediction + self.mask_style = MaskStyle( + channels, + self.log_size, + style_in=2 * self.style_out_channel, + channels_multiplier=1) + + # tps transformation + self.tps = TPS() + + def forward(self, + input, + neural_textures, + skeleton_features, + source_features, + kp_skeleton, + recoder, + add_nted=True): + source_features = source_features[::-1] + skeleton_features = skeleton_features[::-1] + + counter = 0 + out, skip = input, None + + last_flow = None + mask_all_h, mask_all_l = [], [] + delta_list = [] + delta_x_all = [] + delta_y_all = [] + last_flow_all = [] + filter_x = [[0, 0, 0], [1, -2, 1], [0, 0, 0]] + filter_y = [[0, 1, 0], [0, -2, 0], [0, 1, 0]] + filter_diag1 = [[1, 0, 0], [0, -2, 0], [0, 0, 1]] + filter_diag2 = [[0, 0, 1], [0, -2, 0], [1, 0, 0]] + weight_array = np.ones([3, 3, 1, 4]) + weight_array[:, :, 0, 0] = filter_x + weight_array[:, :, 0, 1] = filter_y + weight_array[:, :, 0, 2] = filter_diag1 + weight_array[:, :, 0, 3] = filter_diag2 + weight_array = torch.FloatTensor(weight_array).permute(3, 2, 0, 1).to( + input.device) + self.weight = nn.Parameter(data=weight_array, requires_grad=False) + + B = source_features[0].shape[0] + source_style = self.cond_style(source_features[0]).view(B, -1) + target_style = self.image_style(skeleton_features[0]).view(B, -1) + style = torch.cat([source_style, target_style], 1) + + for i, up in enumerate(self.convs): + use_distribution = ( + self.num_labels[2**(i + 4)] and self.match_kernels[2**(i + 4)]) + if use_distribution: + # warp features with styleflow + source_feature = source_features[i] + skeleton_feature = skeleton_features[i] + if last_flow is not None: + last_flow = F.interpolate( + last_flow, scale_factor=2, mode='bilinear') + s_warp_after = F.grid_sample( + source_feature, + last_flow.detach().permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + else: + s_warp_after = source_feature + scale = str(2**(i + 4)) + + # use tps transformation to estimate flow at the very beginning + if last_flow is not None: + style_map = self.flow_model.netStyle[scale](s_warp_after, + style) + flow = self.flow_model.netF[scale](style_map, style) + flow = apply_offset(flow) + + else: + style_map = self.flow_model.netStyle[scale](s_warp_after, + style) + flow = self.flow_model.netF[scale](style_map, style) + flow_dense = apply_offset(flow) + flow_tps = self.tps(source_feature, kp_skeleton) + warped_dense = F.grid_sample( + source_feature, + flow_dense, + mode='bilinear', + padding_mode='border') + warped_tps = F.grid_sample( + source_feature, + flow_tps, + mode='bilinear', + padding_mode='border') + contribution_map = self.conv_flow_fusion( + torch.cat([warped_dense, warped_tps], 1)) + flow = contribution_map * flow_tps.permute(0, 3, 1, 2) + ( + 1 - contribution_map) * flow_dense.permute(0, 3, 1, 2) + flow = flow.permute(0, 2, 3, 1).contiguous() + + if last_flow is not None: + # update flow according to the last scale flow + flow = F.grid_sample( + last_flow, + flow, + mode='bilinear', + padding_mode='border') + else: + flow = flow.permute(0, 3, 1, 2) + + last_flow = flow + s_warp = F.grid_sample( + source_feature, + flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + + # refine flow according to the original flow + flow = self.flow_model.netRefine[scale]( + torch.cat([s_warp, skeleton_feature], 1)) + + delta_list.append(flow) + flow = apply_offset(flow) + flow = F.grid_sample( + last_flow, flow, mode='bilinear', padding_mode='border') + last_flow_all.append(flow) + + last_flow = flow + flow_x, flow_y = torch.split(last_flow, 1, dim=1) + delta_x = F.conv2d(flow_x, self.weight) + delta_y = F.conv2d(flow_y, self.weight) + delta_x_all.append(delta_x) + delta_y_all.append(delta_y) + + s_warp = F.grid_sample( + source_feature, + last_flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + + # nted attention + neural_texture_conv0 = neural_textures[counter] + neural_texture_conv1 = neural_textures[counter + 1] + counter += 2 + + if not add_nted: # turn off the nted attention + neural_texture_conv0, neural_texture_conv1 = None, None + else: + neural_texture_conv0, neural_texture_conv1 = None, None + s_warp = None + + mask_style_net = self.mask_style.netM[ + scale] if use_distribution else None + out, mask_h, mask_l = up.conv0( + out, + neural_texture=neural_texture_conv0, + recoder=recoder, + warped_texture=s_warp, + style_net=mask_style_net, + gstyle=style) + out, mask_h, mask_l = up.conv1( + out, + neural_texture=neural_texture_conv1, + recoder=recoder, + warped_texture=s_warp, + style_net=mask_style_net, + gstyle=style) + if use_distribution: + if mask_h is not None: + mask_all_h.append(mask_h) + if mask_l is not None: + mask_all_l.append(mask_l) + skip = up.to_rgb(out, skip) + + image = skip + return image, delta_x_all, delta_y_all, delta_list, last_flow_all, mask_all_h, mask_all_l + + +def apply_offset(offset): + sizes = list(offset.size()[2:]) + grid_list = torch.meshgrid( + [torch.arange(size, device=offset.device) for size in sizes]) + grid_list = reversed(grid_list) + # apply offset + grid_list = [ + grid.float().unsqueeze(0) + offset[:, dim, ...] + for dim, grid in enumerate(grid_list) + ] + # normalize + grid_list = [ + grid / ((size - 1.0) / 2.0) - 1.0 + for grid, size in zip(grid_list, reversed(sizes)) + ] + + return torch.stack(grid_list, dim=-1) diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/conv2d_gradfix.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/conv2d_gradfix.py new file mode 100644 index 0000000000000000000000000000000000000000..c2452cdc12224db7f633b9dacb4de5812250d24a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/conv2d_gradfix.py @@ -0,0 +1,227 @@ +import contextlib +import warnings + +import torch +from torch import autograd +from torch.nn import functional as F + +enabled = True +weight_gradients_disabled = False + + +@contextlib.contextmanager +def no_weight_gradients(): + global weight_gradients_disabled + + old = weight_gradients_disabled + weight_gradients_disabled = True + yield + weight_gradients_disabled = old + + +def conv2d(input, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + if could_use_op(input): + return conv2d_gradfix( + transpose=False, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=0, + dilation=dilation, + groups=groups, + ).apply(input, weight, bias) + + return F.conv2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + +def conv_transpose2d( + input, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, +): + if could_use_op(input): + return conv2d_gradfix( + transpose=True, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + dilation=dilation, + ).apply(input, weight, bias) + + return F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + output_padding=output_padding, + dilation=dilation, + groups=groups, + ) + + +def could_use_op(input): + if (not enabled) or (not torch.backends.cudnn.enabled): + return False + + if input.device.type != 'cuda': + return False + + if any(torch.__version__.startswith(x) for x in ['1.7.', '1.8.']): + return True + + warnings.warn( + f'conv2d_gradfix not supported on PyTorch {torch.__version__}. Falling back to torch.nn.functional.conv2d().' + ) + + return False + + +def ensure_tuple(xs, ndim): + xs = tuple(xs) if isinstance(xs, (tuple, list)) else (xs, ) * ndim + + return xs + + +conv2d_gradfix_cache = dict() + + +def conv2d_gradfix(transpose, weight_shape, stride, padding, output_padding, + dilation, groups): + ndim = 2 + weight_shape = tuple(weight_shape) + stride = ensure_tuple(stride, ndim) + padding = ensure_tuple(padding, ndim) + output_padding = ensure_tuple(output_padding, ndim) + dilation = ensure_tuple(dilation, ndim) + + key = (transpose, weight_shape, stride, padding, output_padding, dilation, + groups) + if key in conv2d_gradfix_cache: + return conv2d_gradfix_cache[key] + + common_kwargs = dict( + stride=stride, padding=padding, dilation=dilation, groups=groups) + + def calc_output_padding(input_shape, output_shape): + if transpose: + return [0, 0] + + shape1 = (output_shape[i + 2] - 1) * stride[i] + shape2 = (1 - 2 * padding[i]) - dilation[i] * (weight_shape[i + 2] - 1) + return [input_shape[i + 2] - shape1 - shape2 for i in range(ndim)] + + class Conv2d(autograd.Function): + + @staticmethod + def forward(ctx, input, weight, bias): + if not transpose: + out = F.conv2d( + input=input, weight=weight, bias=bias, **common_kwargs) + + else: + out = F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + output_padding=output_padding, + **common_kwargs, + ) + + ctx.save_for_backward(input, weight) + + return out + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input, grad_weight, grad_bias = None, None, None + + if ctx.needs_input_grad[0]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, weight, None) + + if ctx.needs_input_grad[1] and not weight_gradients_disabled: + grad_weight = Conv2dGradWeight.apply(grad_output, input) + + if ctx.needs_input_grad[2]: + grad_bias = grad_output.sum((0, 2, 3)) + + return grad_input, grad_weight, grad_bias + + class Conv2dGradWeight(autograd.Function): + + @staticmethod + def forward(ctx, grad_output, input): + op = torch._C._jit_get_operation( + 'aten::cudnn_convolution_backward_weight' if not transpose else + 'aten::cudnn_convolution_transpose_backward_weight') + flags = [ + torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, + torch.backends.cudnn.allow_tf32, + ] + grad_weight = op( + weight_shape, + grad_output, + input, + padding, + stride, + dilation, + groups, + *flags, + ) + ctx.save_for_backward(grad_output, input) + + return grad_weight + + @staticmethod + def backward(ctx, grad_grad_weight): + grad_output, input = ctx.saved_tensors + grad_grad_output, grad_grad_input = None, None + + if ctx.needs_input_grad[0]: + grad_grad_output = Conv2d.apply(input, grad_grad_weight, None) + + if ctx.needs_input_grad[1]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, grad_grad_weight, None) + + return grad_grad_output, grad_grad_input + + conv2d_gradfix_cache[key] = Conv2d + + return Conv2d diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/extraction_distribution_model_flow25.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/extraction_distribution_model_flow25.py new file mode 100644 index 0000000000000000000000000000000000000000..10b82e2c5f57eec35409a558974160d6d54387f0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/extraction_distribution_model_flow25.py @@ -0,0 +1,64 @@ +import collections +import os +import sys + +import torch +from torch import nn + +from .base_module import * + +sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__)))) + + +class Generator(nn.Module): + + def __init__( + self, + size, + semantic_dim, + channels, + num_labels, + match_kernels, + blur_kernel=[1, 3, 3, 1], + wavelet_down_levels={'16': 3}, + window_size=8, + ): + super().__init__() + self.size = size + self.reference_encoder = Encoder_wiflow(size, 3, channels, num_labels, + match_kernels, blur_kernel) + + self.skeleton_encoder = Encoder_wiflow( + size, + semantic_dim, + channels, + ) + + self.target_image_renderer = Decoder_wiflow_wavelet_fuse25( + size, channels, num_labels, match_kernels, blur_kernel, + wavelet_down_levels, window_size) + + def _cal_temp(self, module): + return sum(p.numel() for p in module.parameters() if p.requires_grad) + + def forward(self, source_image, skeleton, kp_skeleton): + output_dict = {} + recoder = collections.defaultdict(list) + skeleton_feature_list, source_feature_list = [], [] + skeleton_feature = self.skeleton_encoder( + skeleton, out_list=skeleton_feature_list) + _ = self.reference_encoder( + source_image, recoder, out_list=source_feature_list) + neural_textures = recoder['neural_textures'] + + output_dict['fake_image'], delta_x_all, delta_y_all, delta_list, last_flow_all, mask_all_h, mask_all_l = \ + self.target_image_renderer(skeleton_feature, neural_textures, skeleton_feature_list, + source_feature_list, kp_skeleton, recoder) + output_dict['info'] = recoder + output_dict['delta_x'] = delta_x_all + output_dict['delta_y'] = delta_y_all + output_dict['delta_list'] = delta_list + output_dict['last_flow_all'] = last_flow_all + output_dict['mask_all_h'] = mask_all_h + output_dict['mask_all_l'] = mask_all_l + return output_dict diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/flow_module.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/flow_module.py new file mode 100644 index 0000000000000000000000000000000000000000..59c404b66b86a02f984a333a5d68d01707514d11 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/flow_module.py @@ -0,0 +1,346 @@ +from math import sqrt + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .base_function import EqualConv2d, EqualLinear + + +def TVLoss(x): + tv_h = x[:, :, 1:, :] - x[:, :, :-1, :] + tv_w = x[:, :, :, 1:] - x[:, :, :, :-1] + + return torch.mean(torch.abs(tv_h)) + torch.mean(torch.abs(tv_w)) + + +class MaskStyle(nn.Module): + + def __init__(self, channels, log_size, style_in, channels_multiplier=2): + super().__init__() + self.log_size = log_size + padding_type = 'zero' + actvn = 'lrelu' + normalize_mlp = False + modulated_conv = True + + self.netM = nn.ModuleDict() + + for i in range(4, self.log_size + 1): + out_channel = channels[2**i] + + style_mask = StyledConvBlock( + channels_multiplier * out_channel, + channels_multiplier * out_channel, + latent_dim=style_in, + padding=padding_type, + actvn=actvn, + normalize_affine_output=normalize_mlp, + modulated_conv=modulated_conv) + + scale = str(2**i) + self.netM[scale] = style_mask + + +class StyleFlow(nn.Module): + + def __init__(self, channels, log_size, style_in): + super().__init__() + self.log_size = log_size + padding_type = 'zero' + actvn = 'lrelu' + normalize_mlp = False + modulated_conv = True + + self.netRefine = nn.ModuleDict() + self.netStyle = nn.ModuleDict() + self.netF = nn.ModuleDict() + + for i in range(4, self.log_size + 1): + out_channel = channels[2**i] + + netRefine_layer = torch.nn.Sequential( + torch.nn.Conv2d( + 2 * out_channel, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d( + in_channels=128, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d( + in_channels=64, + out_channels=32, + kernel_size=3, + stride=1, + padding=1), + torch.nn.LeakyReLU(inplace=False, negative_slope=0.1), + torch.nn.Conv2d( + in_channels=32, + out_channels=2, + kernel_size=3, + stride=1, + padding=1)) + + style_block = StyledConvBlock( + out_channel, + 49, + latent_dim=style_in, + padding=padding_type, + actvn=actvn, + normalize_affine_output=normalize_mlp, + modulated_conv=modulated_conv) + + style_F_block = Styled_F_ConvBlock( + 49, + 2, + latent_dim=style_in, + padding=padding_type, + actvn=actvn, + normalize_affine_output=normalize_mlp, + modulated_conv=modulated_conv) + + scale = str(2**i) + self.netRefine[scale] = (netRefine_layer) + self.netStyle[scale] = (style_block) + self.netF[scale] = (style_F_block) + + +class StyledConvBlock(nn.Module): + + def __init__(self, + fin, + fout, + latent_dim=256, + padding='zero', + actvn='lrelu', + normalize_affine_output=False, + modulated_conv=False): + super(StyledConvBlock, self).__init__() + if not modulated_conv: + if padding == 'reflect': + padding_layer = nn.ReflectionPad2d + else: + padding_layer = nn.ZeroPad2d + + if modulated_conv: + conv2d = ModulatedConv2d + else: + conv2d = EqualConv2d + + if modulated_conv: + self.actvn_gain = sqrt(2) + else: + self.actvn_gain = 1.0 + + self.modulated_conv = modulated_conv + + if actvn == 'relu': + activation = nn.ReLU(True) + else: + activation = nn.LeakyReLU(0.2, True) + + if self.modulated_conv: + self.conv0 = conv2d( + fin, + fout, + kernel_size=3, + padding_type=padding, + upsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv0 = conv2d(fin, fout, kernel_size=3) + + seq0 = [padding_layer(1), conv0] + self.conv0 = nn.Sequential(*seq0) + + self.actvn0 = activation + + if self.modulated_conv: + self.conv1 = conv2d( + fout, + fout, + kernel_size=3, + padding_type=padding, + downsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv1 = conv2d(fout, fout, kernel_size=3) + seq1 = [padding_layer(1), conv1] + self.conv1 = nn.Sequential(*seq1) + + self.actvn1 = activation + + def forward(self, input, latent=None): + if self.modulated_conv: + out = self.conv0(input, latent) + else: + out = self.conv0(input) + + out = self.actvn0(out) * self.actvn_gain + + if self.modulated_conv: + out = self.conv1(out, latent) + else: + out = self.conv1(out) + + out = self.actvn1(out) * self.actvn_gain + + return out + + +class Styled_F_ConvBlock(nn.Module): + + def __init__(self, + fin, + fout, + latent_dim=256, + padding='zero', + actvn='lrelu', + normalize_affine_output=False, + modulated_conv=False): + super(Styled_F_ConvBlock, self).__init__() + if not modulated_conv: + if padding == 'reflect': + padding_layer = nn.ReflectionPad2d + else: + padding_layer = nn.ZeroPad2d + + if modulated_conv: + conv2d = ModulatedConv2d + else: + conv2d = EqualConv2d + + if modulated_conv: + self.actvn_gain = sqrt(2) + else: + self.actvn_gain = 1.0 + + self.modulated_conv = modulated_conv + + if actvn == 'relu': + activation = nn.ReLU(True) + else: + activation = nn.LeakyReLU(0.2, True) + + if self.modulated_conv: + self.conv0 = conv2d( + fin, + 128, + kernel_size=3, + padding_type=padding, + upsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv0 = conv2d(fin, 128, kernel_size=3) + + seq0 = [padding_layer(1), conv0] + self.conv0 = nn.Sequential(*seq0) + + self.actvn0 = activation + + if self.modulated_conv: + self.conv1 = conv2d( + 128, + fout, + kernel_size=3, + padding_type=padding, + downsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv1 = conv2d(128, fout, kernel_size=3) + seq1 = [padding_layer(1), conv1] + self.conv1 = nn.Sequential(*seq1) + + def forward(self, input, latent=None): + if self.modulated_conv: + out = self.conv0(input, latent) + else: + out = self.conv0(input) + + out = self.actvn0(out) * self.actvn_gain + + if self.modulated_conv: + out = self.conv1(out, latent) + else: + out = self.conv1(out) + + return out + + +class ModulatedConv2d(nn.Module): + + def __init__(self, + fin, + fout, + kernel_size, + padding_type='zero', + upsample=False, + downsample=False, + latent_dim=512, + normalize_mlp=False): + super(ModulatedConv2d, self).__init__() + self.in_channels = fin + self.out_channels = fout + self.kernel_size = kernel_size + padding_size = kernel_size // 2 + + if kernel_size == 1: + self.demudulate = False + else: + self.demudulate = True + + self.weight = nn.Parameter( + torch.Tensor(fout, fin, kernel_size, kernel_size)) + self.bias = nn.Parameter(torch.Tensor(1, fout, 1, 1)) + + if normalize_mlp: + self.mlp_class_std = nn.Sequential( + EqualLinear(latent_dim, fin), PixelNorm()) + else: + self.mlp_class_std = EqualLinear(latent_dim, fin) + + if padding_type == 'reflect': + self.padding = nn.ReflectionPad2d(padding_size) + else: + self.padding = nn.ZeroPad2d(padding_size) + + self.weight.data.normal_() + self.bias.data.zero_() + + def forward(self, input, latent): + fan_in = self.weight.data.size(1) * self.weight.data[0][0].numel() + weight = self.weight * sqrt(2 / fan_in) + weight = weight.view(1, self.out_channels, self.in_channels, + self.kernel_size, self.kernel_size) + + s = self.mlp_class_std(latent).view(-1, 1, self.in_channels, 1, 1) + weight = s * weight + if self.demudulate: + d = torch.rsqrt((weight**2).sum(4).sum(3).sum(2) + 1e-5).view( + -1, self.out_channels, 1, 1, 1) + weight = (d * weight).view(-1, self.in_channels, self.kernel_size, + self.kernel_size) + else: + weight = weight.view(-1, self.in_channels, self.kernel_size, + self.kernel_size) + + batch, _, height, width = input.shape + + input = input.reshape(1, -1, height, width) + input = self.padding(input) + out = F.conv2d( + input, weight, groups=batch).view(batch, self.out_channels, height, + width) + self.bias + + return out diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/tps.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/tps.py new file mode 100644 index 0000000000000000000000000000000000000000..71a72e74e1e6ac839b4c31941d3040a7281b9c7c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/tps.py @@ -0,0 +1,121 @@ +import torch +import torch.nn.functional as F +from torch import nn + + +class TPS(nn.Module): + + def __init__(self, mode='kp'): + super().__init__() + self.mode = mode + + def trans(self, kp_1): + if self.mode == 'kp': + device = kp_1.device + kp_type = kp_1.type() + self.gs = kp_1.shape[1] + n = kp_1.shape[2] + K = torch.norm( + kp_1[:, :, :, None] - kp_1[:, :, None, :], dim=4, p=2) + K = K**2 + K = K * torch.log(K + 1e-9) + + one1 = torch.ones(self.bs, kp_1.shape[1], kp_1.shape[2], + 1).to(device).type(kp_type) + kp_1p = torch.cat([kp_1, one1], 3) + + zero = torch.zeros(self.bs, kp_1.shape[1], 3, + 3).to(device).type(kp_type) + P = torch.cat([kp_1p, zero], 2) + L = torch.cat([K, kp_1p.permute(0, 1, 3, 2)], 2) + L = torch.cat([L, P], 3) + + zero = torch.zeros(self.bs, kp_1.shape[1], 3, + 2).to(device).type(kp_type) + kp_substitute = torch.zeros(kp_1.shape).to(device).type(kp_type) + Y = torch.cat([kp_substitute, zero], 2) + one = torch.eye(L.shape[2]).expand( + L.shape).to(device).type(kp_type) * 0.01 + L = L + one + + param = torch.matmul(torch.inverse(L), Y) + self.theta = param[:, :, n:, :].permute(0, 1, 3, 2) + + self.control_points = kp_1 + self.control_params = param[:, :, :n, :] + else: + raise Exception('Error TPS mode') + + def transform_frame(self, frame): + grid = make_coordinate_grid( + frame.shape[2:], type=frame.type()).unsqueeze(0).to(frame.device) + grid = grid.view(1, frame.shape[2] * frame.shape[3], 2) + shape = [self.bs, frame.shape[2], frame.shape[3], 2] + if self.mode == 'kp': + shape.insert(1, self.gs) + grid = self.warp_coordinates(grid).view(*shape) + return grid + + def warp_coordinates(self, coordinates): + theta = self.theta.type(coordinates.type()).to(coordinates.device) + control_points = self.control_points.type(coordinates.type()).to( + coordinates.device) + control_params = self.control_params.type(coordinates.type()).to( + coordinates.device) + + if self.mode == 'kp': + transformed = torch.matmul(theta[:, :, :, :2], + coordinates.permute( + 0, 2, 1)) + theta[:, :, :, 2:] + + distances = coordinates.view( + coordinates.shape[0], 1, 1, -1, 2) - control_points.view( + self.bs, control_points.shape[1], -1, 1, 2) + distances = distances**2 + result = distances.sum(-1) + result = result * torch.log(result + 1e-9) + result = torch.matmul(result.permute(0, 1, 3, 2), control_params) + transformed = transformed.permute(0, 1, 3, 2) + result + + else: + raise Exception('Error TPS mode') + + return transformed + + def preprocess_kp(self, kp_1): + ''' + kp_1: (b, ntps*nkp, 2) + ''' + kp_mask = (kp_1 == -1) + num_keypoints = kp_1.shape[1] + kp_1 = kp_1.masked_fill(kp_mask, -1.) + return kp_1, num_keypoints + + def forward(self, source_image, kp_driving): + bs, _, h, w = source_image.shape + self.bs = bs + kp_driving, num_keypoints = self.preprocess_kp(kp_driving) + kp_1 = kp_driving.view(bs, -1, num_keypoints, 2) + self.trans(kp_1) + grid = self.transform_frame(source_image) + grid = grid.view(bs, h, w, 2) + return grid + + +def make_coordinate_grid(spatial_size, type): + """ + Create a meshgrid [-1,1] x [-1,1] of given spatial_size. + """ + h, w = spatial_size + x = torch.arange(w).type(type) + y = torch.arange(h).type(type) + + x = (2 * (x / (w - 1)) - 1) + y = (2 * (y / (h - 1)) - 1) + + yy = y.view(-1, 1).repeat(1, w) + xx = x.view(1, -1).repeat(h, 1) + + meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2) + + return meshed diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/generators/wavelet_module.py b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/wavelet_module.py new file mode 100644 index 0000000000000000000000000000000000000000..8c3efaf2d3eca8b84eafbc070b558ac32c4acead --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/generators/wavelet_module.py @@ -0,0 +1,182 @@ +import numpy as np +import torch +import torch.nn as nn + + +def get_wav(in_channels, pool=True): + harr_wav_L = 1 / np.sqrt(2) * np.ones((1, 2)) + harr_wav_H = 1 / np.sqrt(2) * np.ones((1, 2)) + harr_wav_H[0, 0] = -1 * harr_wav_H[0, 0] + harr_wav_LL = np.transpose(harr_wav_L) * harr_wav_L + harr_wav_LH = np.transpose(harr_wav_L) * harr_wav_H + harr_wav_HL = np.transpose(harr_wav_H) * harr_wav_L + harr_wav_HH = np.transpose(harr_wav_H) * harr_wav_H + filter_LL = torch.from_numpy(harr_wav_LL).unsqueeze(0) + filter_LH = torch.from_numpy(harr_wav_LH).unsqueeze(0) + filter_HL = torch.from_numpy(harr_wav_HL).unsqueeze(0) + filter_HH = torch.from_numpy(harr_wav_HH).unsqueeze(0) + if pool: + net = nn.Conv2d + else: + net = nn.ConvTranspose2d + LL = net( + in_channels, + in_channels * 2, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + LH = net( + in_channels, + in_channels * 2, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + HL = net( + in_channels, + in_channels * 2, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + HH = net( + in_channels, + in_channels * 2, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + LL.weight.requires_grad = False + LH.weight.requires_grad = False + HL.weight.requires_grad = False + HH.weight.requires_grad = False + LL.weight.data = filter_LL.float().unsqueeze(0).expand( + in_channels * 2, -1, -1, -1) + LH.weight.data = filter_LH.float().unsqueeze(0).expand( + in_channels * 2, -1, -1, -1) + HL.weight.data = filter_HL.float().unsqueeze(0).expand( + in_channels * 2, -1, -1, -1) + HH.weight.data = filter_HH.float().unsqueeze(0).expand( + in_channels * 2, -1, -1, -1) + return LL, LH, HL, HH + + +class WavePool(nn.Module): + + def __init__(self, in_channels): + super(WavePool, self).__init__() + self.LL, self.LH, self.HL, self.HH = get_wav(in_channels) + + def forward(self, x): + return self.LL(x), self.LH(x), self.HL(x), self.HH(x) + + +def get_wav_two(in_channels, out_channels=None, pool=True): + """wavelet decomposition using conv2d""" + harr_wav_L = 1 / np.sqrt(2) * np.ones((1, 2)) + harr_wav_H = 1 / np.sqrt(2) * np.ones((1, 2)) + harr_wav_H[0, 0] = -1 * harr_wav_H[0, 0] + + harr_wav_LL = np.transpose(harr_wav_L) * harr_wav_L + harr_wav_LH = np.transpose(harr_wav_L) * harr_wav_H + harr_wav_HL = np.transpose(harr_wav_H) * harr_wav_L + harr_wav_HH = np.transpose(harr_wav_H) * harr_wav_H + + filter_LL = torch.from_numpy(harr_wav_LL).unsqueeze(0) + filter_LH = torch.from_numpy(harr_wav_LH).unsqueeze(0) + filter_HL = torch.from_numpy(harr_wav_HL).unsqueeze(0) + filter_HH = torch.from_numpy(harr_wav_HH).unsqueeze(0) + + if pool: + net = nn.Conv2d + else: + net = nn.ConvTranspose2d + if out_channels is None: + out_channels = in_channels + LL = net( + in_channels, + out_channels, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + LH = net( + in_channels, + out_channels, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + HL = net( + in_channels, + out_channels, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + HH = net( + in_channels, + out_channels, + kernel_size=2, + stride=2, + padding=0, + bias=False, + groups=in_channels) + + LL.weight.requires_grad = False + LH.weight.requires_grad = False + HL.weight.requires_grad = False + HH.weight.requires_grad = False + + LL.weight.data = filter_LL.float().unsqueeze(0).expand( + in_channels, -1, -1, -1) + LH.weight.data = filter_LH.float().unsqueeze(0).expand( + in_channels, -1, -1, -1) + HL.weight.data = filter_HL.float().unsqueeze(0).expand( + in_channels, -1, -1, -1) + HH.weight.data = filter_HH.float().unsqueeze(0).expand( + in_channels, -1, -1, -1) + + return LL, LH, HL, HH + + +class WavePool2(nn.Module): + + def __init__(self, in_channels, out_channels=None): + super(WavePool2, self).__init__() + self.LL, self.LH, self.HL, self.HH = get_wav_two( + in_channels, out_channels) + + def forward(self, x): + return self.LL(x), self.LH(x), self.HL(x), self.HH(x) + + +class WaveUnpool(nn.Module): + + def __init__(self, in_channels, out_channels=None, option_unpool='cat5'): + super(WaveUnpool, self).__init__() + self.in_channels = in_channels + self.option_unpool = option_unpool + self.LL, self.LH, self.HL, self.HH = get_wav_two( + self.in_channels, out_channels, pool=False) + + def forward(self, LL, LH, HL, HH, original=None): + if self.option_unpool == 'sum': + return self.LL(LL) + self.LH(LH) + self.HL(HL) + self.HH(HH) + elif self.option_unpool == 'cat5' and original is not None: + return torch.cat( + [self.LL(LL), + self.LH(LH), + self.HL(HL), + self.HH(HH), original], + dim=1) + else: + raise NotImplementedError diff --git a/runtime/Lib/site_packages/models/cv/human_image_generation/human_image_generation_infer.py b/runtime/Lib/site_packages/models/cv/human_image_generation/human_image_generation_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..0781d8930591aa07b325c19f2ccf79342f07181b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_image_generation/human_image_generation_infer.py @@ -0,0 +1,268 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math +import random +from ast import Global +from pickle import GLOBAL + +import cv2 +import numpy as np +import torch +import torchvision +import torchvision.transforms as transforms +import torchvision.transforms.functional as F +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .generators.extraction_distribution_model_flow25 import \ + Generator as Generator + +tv_version = int(torchvision.__version__.split('.')[1]) +if tv_version > 8: + from torchvision.transforms.functional import InterpolationMode + resize_method = InterpolationMode.BICUBIC + resize_nearest = InterpolationMode.NEAREST +else: + resize_method = Image.BICUBIC + resize_nearest = Image.NEAREST + +logger = get_logger() + + +def get_random_params(size, scale_param, use_flip=False): + w, h = size + scale = random.random() * scale_param + + if use_flip: + use_flip = random.random() > 0.9 + + new_w = int(w * (1.0 + scale)) + new_h = int(h * (1.0 + scale)) + x = random.randint(0, np.maximum(0, new_w - w)) + y = random.randint(0, np.maximum(0, new_h - h)) + return { + 'crop_param': (x, y, w, h), + 'scale_size': (new_h, new_w), + 'use_flip': use_flip + } + + +def get_transform(param, method=resize_method, normalize=True, toTensor=True): + transform_list = [] + if 'scale_size' in param and param['scale_size'] is not None: + osize = param['scale_size'] + transform_list.append(transforms.Resize(osize, interpolation=method)) + + if 'crop_param' in param and param['crop_param'] is not None: + transform_list.append( + transforms.Lambda(lambda img: __crop(img, param['crop_param']))) + + if param['use_flip']: + transform_list.append(transforms.Lambda(lambda img: __flip(img))) + + if toTensor: + transform_list += [transforms.ToTensor()] + + if normalize: + transform_list += [ + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ] + return transforms.Compose(transform_list) + + +def __crop(img, pos): + x1, y1, tw, th = pos + return img.crop((x1, y1, x1 + tw, y1 + th)) + + +def __flip(img): + return F.hflip(img) + + +def normalize(): + return transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + + +def load_checkpoint(model, checkpoint_path, device): + params = torch.load(checkpoint_path, map_location=device) + if 'target_image_renderer.weight' in params['net_G_ema'].keys(): + params['net_G_ema'].pop('target_image_renderer.weight') + model.load_state_dict(params['net_G_ema']) + model.to(device) + model.eval() + return model + + +@MODELS.register_module( + Tasks.human_image_generation, module_name=Models.human_image_generation) +class FreqHPTForHumanImageGeneration(TorchModel): + """initialize the human image generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + if torch.cuda.is_available(): + self.device = 'cuda' + logger.info('Use GPU') + else: + self.device = 'cpu' + logger.info('Use CPU') + + size = 512 + semantic_dim = 20 + channels = { + 16: 256, + 32: 256, + 64: 256, + 128: 128, + 256: 128, + 512: 64, + 1024: 32 + } + num_labels = {16: 16, 32: 32, 64: 64, 128: 64, 256: 64, 512: False} + match_kernels = {16: False, 32: 3, 64: 3, 128: 3, 256: 3, 512: False} + wavelet_down_levels = {16: False, 32: 1, 64: 2, 128: 3, 256: 3, 512: 3} + self.model = Generator( + size, + semantic_dim, + channels, + num_labels, + match_kernels, + wavelet_down_levels=wavelet_down_levels) + self.model = load_checkpoint( + self.model, model_dir + '/' + ModelFile.TORCH_MODEL_BIN_FILE, + self.device) + + def forward(self, x, y, z): + pred_result = self.model(x, y, z) + return pred_result + + +def trans_keypoins(keypoints, param, img_size, offset=None): + missing_keypoint_index = keypoints == -1 + + # crop the white line in the original dataset + if not offset == 40: + keypoints[:, 0] = (keypoints[:, 0] - 40) + + # resize the dataset + img_h, img_w = img_size + scale_w = 1.0 / 176.0 * img_w + scale_h = 1.0 / 256.0 * img_h + + if 'scale_size' in param and param['scale_size'] is not None: + new_h, new_w = param['scale_size'] + scale_w = scale_w / img_w * new_w + scale_h = scale_h / img_h * new_h + + if 'crop_param' in param and param['crop_param'] is not None: + w, h, _, _ = param['crop_param'] + else: + w, h = 0, 0 + + keypoints[:, 0] = keypoints[:, 0] * scale_w - w + keypoints[:, 1] = keypoints[:, 1] * scale_h - h + + normalized_kp = keypoints.copy() + normalized_kp[:, 0] = (normalized_kp[:, 0]) / img_w * 2 - 1 + normalized_kp[:, 1] = (normalized_kp[:, 1]) / img_h * 2 - 1 + normalized_kp[missing_keypoint_index] = -1 + + keypoints[missing_keypoint_index] = -1 + return keypoints, normalized_kp + + +def get_label_tensor(path, img, param): + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], + [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], + [170, 255, 0], [85, 255, 0], [0, 255, 0], [0, 255, 85], + [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], + [0, 0, 255], [85, 0, 255], [170, 0, 255], [255, 0, 255], + [255, 0, 170], [255, 0, 85]] + canvas = np.zeros((img.shape[1], img.shape[2], 3)).astype(np.uint8) + keypoint = np.loadtxt(path) + keypoint, normalized_kp = trans_keypoins(keypoint, param, img.shape[1:]) + stickwidth = 4 + for i in range(18): + x, y = keypoint[i, 0:2] + if x == -1 or y == -1: + continue + cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1) + joints = [] + for i in range(17): + Y = keypoint[np.array(limbSeq[i]) - 1, 0] + X = keypoint[np.array(limbSeq[i]) - 1, 1] + cur_canvas = canvas.copy() + if -1 in Y or -1 in X: + joints.append(np.zeros_like(cur_canvas[:, :, 0])) + continue + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1])**2 + (Y[0] - Y[1])**2)**0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly( + (int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, + 360, 1) + cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) + canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) + + joint = np.zeros_like(cur_canvas[:, :, 0]) + cv2.fillConvexPoly(joint, polygon, 255) + joint = cv2.addWeighted(joint, 0.4, joint, 0.6, 0) + joints.append(joint) + pose = F.to_tensor( + Image.fromarray(cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB))) + + tensors_dist = 0 + e = 1 + for i in range(len(joints)): + im_dist = cv2.distanceTransform(255 - joints[i], cv2.DIST_L1, 3) + im_dist = np.clip((im_dist / 3), 0, 255).astype(np.uint8) + tensor_dist = F.to_tensor(Image.fromarray(im_dist)) + tensors_dist = tensor_dist if e == 1 else torch.cat( + [tensors_dist, tensor_dist]) + e += 1 + + label_tensor = torch.cat((pose, tensors_dist), dim=0) + return label_tensor, normalized_kp + + +def get_image_tensor(path): + img = Image.open(path) + param = get_random_params(img.size, 0) + trans = get_transform(param, normalize=True, toTensor=True) + img = trans(img) + return img, param + + +def infer(genmodel, image_path, target_label_path, device): + ref_tensor, param = get_image_tensor(image_path) + target_label_tensor, target_kp = get_label_tensor(target_label_path, + ref_tensor, param) + + ref_tensor = ref_tensor.unsqueeze(0).to(device) + target_label_tensor = target_label_tensor.unsqueeze(0).to(device) + target_kp = torch.from_numpy(target_kp).unsqueeze(0).to(device) + output_dict = genmodel(ref_tensor, target_label_tensor, target_kp) + output_image = output_dict['fake_image'][0] + + output_image = output_image.clamp_(-1, 1) + image = (output_image + 1) * 0.5 + image = image.detach().cpu().squeeze().numpy() + image = np.transpose(image, (1, 2, 0)) * 255 + image = np.uint8(image) + bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + return bgr diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/Reconstruction.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/Reconstruction.py new file mode 100644 index 0000000000000000000000000000000000000000..4140565e934db3fc66148c648679c4b4df14c90e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/Reconstruction.py @@ -0,0 +1,137 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Optional + +import cv2 +import numpy as np +import PIL.Image as Image +import torch +import torchvision.transforms as transforms +from skimage.io import imread +from skimage.transform import estimate_transform, warp + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.human_reconstruction.models.detectors import \ + FasterRCNN +from modelscope.models.cv.human_reconstruction.models.human_segmenter import \ + human_segmenter +from modelscope.models.cv.human_reconstruction.models.networks import define_G +from modelscope.models.cv.human_reconstruction.models.PixToMesh import \ + Pixto3DNet +from modelscope.models.cv.human_reconstruction.utils import create_grid +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module( + Tasks.human_reconstruction, module_name=Models.human_reconstruction) +class HumanReconstruction(TorchModel): + + def __init__(self, model_dir, modelconfig, *args, **kwargs): + """The HumanReconstruction is modified based on PiFuHD and pix2pixhd, publicly available at + https://shunsukesaito.github.io/PIFuHD/ & + https://github.com/NVIDIA/pix2pixHD + + Args: + model_dir: the root directory of the model files + modelconfig: the config param path of the model + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + if torch.cuda.is_available(): + self.device = torch.device('cuda') + logger.info('Use GPU: {}'.format(self.device)) + else: + self.device = torch.device('cpu') + logger.info('Use CPU: {}'.format(self.device)) + + model_path = '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_FILE) + normal_back_model = '{}/{}'.format(model_dir, 'Norm_B_GAN.pth') + normal_front_model = '{}/{}'.format(model_dir, 'Norm_F_GAN.pth') + human_seg_model = '{}/{}'.format(model_dir, ModelFile.TF_GRAPH_FILE) + fastrcnn_ckpt = '{}/{}'.format(model_dir, 'fasterrcnn_resnet50.pth') + self.meshmodel = Pixto3DNet(**modelconfig['model']) + self.detector = FasterRCNN(ckpt=fastrcnn_ckpt, device=self.device) + self.meshmodel.load_state_dict( + torch.load(model_path, map_location='cpu')) + self.netB = define_G(3, 3, 64, 'global', 4, 9, 1, 3, 'instance') + self.netF = define_G(3, 3, 64, 'global', 4, 9, 1, 3, 'instance') + self.netF.load_state_dict(torch.load(normal_front_model)) + self.netB.load_state_dict(torch.load(normal_back_model)) + self.netF = self.netF.to(self.device) + self.netB = self.netB.to(self.device) + self.netF.eval() + self.netB.eval() + self.meshmodel = self.meshmodel.to(self.device).eval() + self.portrait_matting = human_segmenter(model_path=human_seg_model) + b_min = np.array([-1, -1, -1]) + b_max = np.array([1, 1, 1]) + self.coords, self.mat = create_grid(modelconfig['resolution'], b_min, + b_max) + projection_matrix = np.identity(4) + projection_matrix[1, 1] = -1 + self.calib = torch.Tensor(projection_matrix).float().to(self.device) + self.calib = self.calib[:3, :4].unsqueeze(0) + logger.info('model load over') + + def get_mask(self, img): + result = self.portrait_matting.run(img) + result = result[..., None] + mask = result.repeat(3, axis=2) + return img, mask + + @torch.no_grad() + def crop_img(self, img_url): + image = imread(img_url)[:, :, :3] / 255. + h, w, _ = image.shape + image_size = 512 + image_tensor = torch.tensor( + image.transpose(2, 0, 1), dtype=torch.float32)[None, ...] + bbox = self.detector.run(image_tensor) + left = bbox[0] + right = bbox[2] + top = bbox[1] + bottom = bbox[3] + + old_size = max(right - left, bottom - top) + center = np.array( + [right - (right - left) / 2.0, bottom - (bottom - top) / 2.0]) + size = int(old_size * 1.1) + src_pts = np.array([[center[0] - size / 2, center[1] - size / 2], + [center[0] - size / 2, center[1] + size / 2], + [center[0] + size / 2, center[1] - size / 2]]) + DST_PTS = np.array([[0, 0], [0, image_size - 1], [image_size - 1, 0]]) + tform = estimate_transform('similarity', src_pts, DST_PTS) + dst_image = warp( + image, tform.inverse, output_shape=(image_size, image_size)) + dst_image = (dst_image[:, :, ::-1] * 255).astype(np.uint8) + return dst_image + + @torch.no_grad() + def generation_normal(self, img, mask): + to_tensor = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + im_512 = cv2.resize(img, (512, 512)) + image_512 = Image.fromarray(im_512).convert('RGB') + image_512 = to_tensor(image_512).unsqueeze(0) + img = image_512.to(self.device) + nml_f = self.netF.forward(img) + nml_b = self.netB.forward(img) + mask = cv2.resize(mask, (512, 512)) + mask = transforms.ToTensor()(mask).unsqueeze(0) + nml_f = (nml_f.cpu() * mask).detach().cpu().numpy()[0] + nml_f = (np.transpose(nml_f, + (1, 2, 0)) * 0.5 + 0.5)[:, :, ::-1] * 255.0 + nml_b = (nml_b.cpu() * mask).detach().cpu().numpy()[0] + nml_b = (np.transpose(nml_b, + (1, 2, 0)) * 0.5 + 0.5)[:, :, ::-1] * 255.0 + nml_f = nml_f.astype(np.uint8) + nml_b = nml_b.astype(np.uint8) + return nml_f, nml_b + + # def forward(self, img, mask, normal_f, normal_b): diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/__init__.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Embedding.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..a2ec18774d223b0285e6cce1624616de76b03bec --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Embedding.py @@ -0,0 +1,32 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +from torch import nn + + +class Embedding(nn.Module): + + def __init__(self, in_channels, N_freqs, logscale=True): + """ + Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...) + in_channels: number of input channels (3 for both xyz and direction) + """ + super(Embedding, self).__init__() + self.N_freqs = N_freqs + self.in_channels = in_channels + self.name = 'Embedding' + self.funcs = [torch.sin, torch.cos] + self.out_channels = in_channels * (len(self.funcs) * N_freqs + 1) + self.input_para = dict(in_channels=in_channels, N_freqs=N_freqs) + + if logscale: + self.freq_bands = 2**torch.linspace(0, N_freqs - 1, N_freqs) + else: + self.freq_bands = torch.linspace(1, 2**(N_freqs - 1), N_freqs) + + def forward(self, x): + out = [x] + for freq in self.freq_bands: + for func in self.funcs: + out += [func(freq * x)] + + return torch.cat(out, 1) diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/PixToMesh.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/PixToMesh.py new file mode 100644 index 0000000000000000000000000000000000000000..0299bf82538cd6111e8b7bca110062b5d12f0e79 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/PixToMesh.py @@ -0,0 +1,142 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from .Embedding import Embedding +from .geometry import index, orthogonal, perspective +from .Res_backbone import Res_hournet +from .Surface_head import Surface_Head + + +class Pixto3DNet(nn.Module): + + def __init__(self, + backbone, + head, + rgbhead, + embedding, + projection_mode: str = 'orthogonal', + error_term: str = 'mse', + num_views: int = 1): + """ + Parameters: + backbone: parameter of networks to extract image features + head: parameter of networks to predict value in surface + rgbhead: parameter of networks to predict rgb of point + embedding: parameter of networks to normalize depth of camera coordinate + projection_mode: how to render your 3d model to images + error_term: train loss + num_view: how many images from which you want to reconstruct model + """ + super(Pixto3DNet, self).__init__() + + self.backbone = Res_hournet(**backbone) + self.head = Surface_Head(**head) + self.rgbhead = Surface_Head(**rgbhead) + self.depth = Embedding(**embedding) + + if error_term == 'mse': + self.error_term = nn.MSELoss(reduction='none') + elif error_term == 'bce': + self.error_term = nn.BCELoss(reduction='none') + elif error_term == 'l1': + self.error_term = nn.L1Loss(reduction='none') + else: + raise NotImplementedError + + self.index = index + self.projection = orthogonal if projection_mode == 'orthogonal' else perspective + + self.num_views = num_views + self.im_feat_list = [] + self.intermediate_preds_list = [] + + def extract_features(self, images: torch.Tensor): + self.im_feat_list = self.backbone(images) + + def query(self, points, calibs, transforms=None, labels=None): + if labels is not None: + self.labels = labels + + xyz = self.projection(points, calibs, transforms) + + xy = xyz[:, :2, :] + xyz_feat = self.depth(xyz) + + self.intermediate_preds_list = [] + + im_feat_256 = self.im_feat_list[0] + im_feat_512 = self.im_feat_list[1] + + point_local_feat_list = [ + self.index(im_feat_256, xy), + self.index(im_feat_512, xy), xyz_feat + ] + point_local_feat = torch.cat(point_local_feat_list, 1) + + pred, phi = self.head(point_local_feat) + self.intermediate_preds_list.append(pred) + self.phi = phi + + self.preds = self.intermediate_preds_list[-1] + + def get_preds(self): + return self.preds + + def query_rgb(self, points, calibs, transforms=None): + xyz = self.projection(points, calibs, transforms) + + xy = xyz[:, :2, :] + xyz_feat = self.depth(xyz) + + self.intermediate_preds_list = [] + + im_feat_256 = self.im_feat_list[0] + im_feat_512 = self.im_feat_list[1] + + point_local_feat_list = [ + self.index(im_feat_256, xy), + self.index(im_feat_512, xy), xyz_feat + ] + point_local_feat = torch.cat(point_local_feat_list, 1) + + pred, phi = self.head(point_local_feat) + rgb_point_feat = torch.cat([point_local_feat, phi], 1) + rgb, phi = self.rgbhead(rgb_point_feat) + return rgb + + def get_error(self): + error = 0 + lc = torch.tensor(self.labels.shape[0] * self.labels.shape[1] + * self.labels.shape[2]) + inw = torch.sum(self.labels) + weight_in = inw / lc + weight = torch.abs(self.labels - weight_in) + lamda = 1 / torch.mean(weight) + for preds in self.intermediate_preds_list: + error += lamda * torch.mean( + self.error_term(preds, self.labels) * weight) + error /= len(self.intermediate_preds_list) + + return error + + def forward(self, + images, + points, + calibs, + surpoint=None, + transforms=None, + labels=None): + self.extract_features(images) + + self.query( + points=points, calibs=calibs, transforms=transforms, labels=labels) + + if surpoint is not None: + rgb = self.query_rgb( + points=surpoint, calibs=calibs, transforms=transforms) + else: + rgb = None + res = self.preds + + return res, rgb diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Res_backbone.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Res_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..b14ae7724558792e424a113931895b8ed35ff8b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Res_backbone.py @@ -0,0 +1,330 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BlurPool(nn.Module): + + def __init__(self, + channels, + pad_type='reflect', + filt_size=4, + stride=2, + pad_off=0): + super(BlurPool, self).__init__() + self.filt_size = filt_size + self.pad_off = pad_off + self.pad_sizes = [ + int(1. * (filt_size - 1) / 2), + int(np.ceil(1. * (filt_size - 1) / 2)), + int(1. * (filt_size - 1) / 2), + int(np.ceil(1. * (filt_size - 1) / 2)) + ] + self.pad_sizes = [pad_size + pad_off for pad_size in self.pad_sizes] + self.stride = stride + self.off = int((self.stride - 1) / 2.) + self.channels = channels + + if (self.filt_size == 1): + a = np.array([ + 1., + ]) + elif (self.filt_size == 2): + a = np.array([1., 1.]) + elif (self.filt_size == 3): + a = np.array([1., 2., 1.]) + elif (self.filt_size == 4): + a = np.array([1., 3., 3., 1.]) + elif (self.filt_size == 5): + a = np.array([1., 4., 6., 4., 1.]) + elif (self.filt_size == 6): + a = np.array([1., 5., 10., 10., 5., 1.]) + elif (self.filt_size == 7): + a = np.array([1., 6., 15., 20., 15., 6., 1.]) + + filt = torch.Tensor(a[:, None] * a[None, :]) + filt = filt / torch.sum(filt) + self.register_buffer( + 'filt', filt[None, None, :, :].repeat((self.channels, 1, 1, 1))) + + self.pad = get_pad_layer(pad_type)(self.pad_sizes) + + def forward(self, inp): + if (self.filt_size == 1): + if (self.pad_off == 0): + return inp[:, :, ::self.stride, ::self.stride] + else: + return self.pad(inp)[:, :, ::self.stride, ::self.stride] + else: + return F.conv2d( + self.pad(inp), + self.filt, + stride=self.stride, + groups=inp.shape[1]) + + +def get_pad_layer(pad_type): + if (pad_type in ['refl', 'reflect']): + PadLayer = nn.ReflectionPad2d + elif (pad_type in ['repl', 'replicate']): + PadLayer = nn.ReplicationPad2d + elif (pad_type == 'zero'): + PadLayer = nn.ZeroPad2d + else: + print('Pad type [%s] not recognized' % pad_type) + return PadLayer + + +class ConvBlockv1(nn.Module): + + def __init__(self, in_planes, out_planes, norm='batch'): + super(ConvBlockv1, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, + int(out_planes / 2), + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.conv2 = nn.Conv2d( + int(out_planes / 2), + int(out_planes / 4), + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.conv3 = nn.Conv2d( + int(out_planes / 4), + int(out_planes / 4), + kernel_size=3, + stride=1, + padding=1, + bias=False) + + if norm == 'batch': + self.bn2 = nn.BatchNorm2d(int(out_planes / 2)) + self.bn3 = nn.BatchNorm2d(int(out_planes / 4)) + self.bn4 = nn.BatchNorm2d(out_planes) + elif norm == 'group': + self.bn2 = nn.GroupNorm(32, int(out_planes / 2)) + self.bn3 = nn.GroupNorm(32, int(out_planes / 4)) + self.bn4 = nn.GroupNorm(32, out_planes) + + if in_planes != out_planes: + self.downsample = nn.Sequential( + nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=1, + bias=False), ) + else: + self.downsample = None + + def forward(self, x): + residual = x + out1 = self.conv1(x) + out2 = self.bn2(out1) + out2 = F.relu(out2, True) + out2 = self.conv2(out2) + + out3 = self.bn3(out2) + out3 = F.relu(out3, True) + out3 = self.conv3(out3) + out3 = torch.cat((out1, out2, out3), 1) + + if self.downsample is not None: + residual = self.downsample(residual) + out3 += residual + out4 = self.bn4(out3) + out4 = F.relu(out4, True) + return out4 + + +class Conv2(nn.Module): + + def __init__(self, in_planes, out_planes, norm='batch'): + super(Conv2, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, + int(out_planes / 4), + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.conv2 = nn.Conv2d( + in_planes, + int(out_planes / 4), + kernel_size=5, + stride=1, + padding=2, + bias=False) + self.conv3 = nn.Conv2d( + in_planes, + int(out_planes / 2), + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.conv4 = nn.Conv2d( + out_planes, + out_planes, + kernel_size=3, + stride=1, + padding=1, + bias=False) + + if norm == 'batch': + self.bn1 = nn.BatchNorm2d(int(out_planes / 4)) + self.bn2 = nn.BatchNorm2d(int(out_planes / 4)) + self.bn3 = nn.BatchNorm2d(int(out_planes / 2)) + self.bn4 = nn.BatchNorm2d(out_planes) + elif norm == 'group': + self.bn1 = nn.GroupNorm(32, int(out_planes / 4)) + self.bn2 = nn.GroupNorm(32, int(out_planes / 4)) + self.bn3 = nn.GroupNorm(32, int(out_planes / 2)) + self.bn4 = nn.GroupNorm(32, out_planes) + + if in_planes != out_planes: + self.downsample = nn.Sequential( + nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=1, + bias=False), ) + else: + self.downsample = None + + def forward(self, x): + residual = x + out1 = self.conv1(x) + out1 = self.bn1(out1) + out1 = F.relu(out1, True) + + out2 = self.conv2(x) + out2 = self.bn2(out2) + out2 = F.relu(out2, True) + + out3 = self.conv3(x) + out3 = self.bn3(out3) + out3 = F.relu(out3, True) + out3 = torch.cat((out1, out2, out3), 1) + if self.downsample is not None: + residual = self.downsample(residual) + out = out3 + residual + out = self.conv4(out) + out = self.bn4(out) + out = F.relu(out, True) + return out + + +class Res_hournet(nn.Module): + + def __init__(self, norm: str = 'group', use_front=False, use_back=False): + """ + Defines a backbone of human reconstruction + use_front & use_back is the normal map of input image + """ + super(Res_hournet, self).__init__() + self.name = 'Res Backbone' + self.norm = norm + inc = 3 + self.use_front = use_front + self.use_back = use_back + if self.use_front: + inc += 3 + if self.use_back: + inc += 3 + self.conv1 = nn.Conv2d(inc, 64, kernel_size=7, stride=1, padding=3) + if self.norm == 'batch': + self.bn1 = nn.BatchNorm2d(64) + elif self.norm == 'group': + self.bn1 = nn.GroupNorm(32, 64) + self.down_conv1 = BlurPool( + 64, pad_type='reflect', filt_size=7, stride=2) + self.conv2 = ConvBlockv1(64, 128, self.norm) + self.down_conv2 = BlurPool( + 128, pad_type='reflect', filt_size=7, stride=2) + self.conv3 = ConvBlockv1(128, 128, self.norm) + self.conv5 = ConvBlockv1(128, 256, self.norm) + self.conv6 = ConvBlockv1(256, 256, self.norm) + self.down_conv3 = BlurPool( + 256, pad_type='reflect', filt_size=5, stride=2) + self.conv7 = ConvBlockv1(256, 256, self.norm) + self.conv8 = ConvBlockv1(256, 256, self.norm) + self.conv9 = ConvBlockv1(256, 256, self.norm) + self.conv10 = ConvBlockv1(256, 256, self.norm) + self.conv10_1 = ConvBlockv1(256, 512, self.norm) + self.conv10_2 = Conv2(512, 512, self.norm) + self.down_conv4 = BlurPool( + 512, pad_type='reflect', filt_size=5, stride=2) + self.conv11 = Conv2(512, 512, self.norm) + self.conv12 = ConvBlockv1(512, 512, self.norm) + self.conv13 = Conv2(512, 512, self.norm) + self.conv14 = ConvBlockv1(512, 512, self.norm) + self.conv15 = Conv2(512, 512, self.norm) + self.conv16 = ConvBlockv1(512, 512, self.norm) + self.conv17 = Conv2(512, 512, self.norm) + self.conv18 = ConvBlockv1(512, 512, self.norm) + self.conv19 = Conv2(512, 512, self.norm) + self.conv20 = ConvBlockv1(512, 512, self.norm) + self.conv21 = Conv2(512, 512, self.norm) + self.conv22 = ConvBlockv1(512, 512, self.norm) + + self.up_down1 = nn.Conv2d(1024, 512, 3, 1, 1, bias=False) + self.upconv1 = ConvBlockv1(512, 512, self.norm) + self.upconv1_1 = ConvBlockv1(512, 512, self.norm) + self.up_down2 = nn.Conv2d(768, 512, 3, 1, 1, bias=False) + self.upconv2 = ConvBlockv1(512, 256, self.norm) + self.upconv2_1 = ConvBlockv1(256, 256, self.norm) + self.up_down3 = nn.Conv2d(384, 256, 3, 1, 1, bias=False) + self.upconv3 = ConvBlockv1(256, 256, self.norm) + self.upconv3_4 = nn.Conv2d(256, 128, 3, 1, 1, bias=False) + self.up_down4 = nn.Conv2d(192, 64, 3, 1, 1, bias=False) + self.upconv4 = ConvBlockv1(64, 64, 'batch') + + def forward(self, x): + out0 = self.bn1(self.conv1(x)) + out1 = self.down_conv1(out0) + out1 = self.conv2(out1) + out2 = self.down_conv2(out1) + out2 = self.conv3(out2) + out2 = self.conv5(out2) + out2 = self.conv6(out2) + out3 = self.down_conv3(out2) + out3 = self.conv7(out3) + out3 = self.conv9(self.conv8(out3)) + out3 = self.conv10(out3) + out3 = self.conv10_2(self.conv10_1(out3)) + out4 = self.down_conv4(out3) + out4 = self.conv12(self.conv11(out4)) + out4 = self.conv14(self.conv13(out4)) + out4 = self.conv16(self.conv15(out4)) + out4 = self.conv18(self.conv17(out4)) + out4 = self.conv20(self.conv19(out4)) + out4 = self.conv22(self.conv21(out4)) + + up1 = F.interpolate( + out4, scale_factor=2, mode='bicubic', align_corners=True) + up1 = torch.cat((up1, out3), 1) + up1 = self.up_down1(up1) + up1 = self.upconv1(up1) + up1 = self.upconv1_1(up1) + + up2 = F.interpolate( + up1, scale_factor=2, mode='bicubic', align_corners=True) + up2 = torch.cat((up2, out2), 1) + up2 = self.up_down2(up2) + up2 = self.upconv2(up2) + up2 = self.upconv2_1(up2) + + up3 = F.interpolate( + up2, scale_factor=2, mode='bicubic', align_corners=True) + up3 = torch.cat((up3, out1), 1) + up3 = self.up_down3(up3) + up3 = self.upconv3(up3) + + up34 = self.upconv3_4(up3) + up4 = F.interpolate( + up34, scale_factor=2, mode='bicubic', align_corners=True) + up4 = torch.cat((up4, out0), 1) + up4 = self.up_down4(up4) + up4 = self.upconv4(up4) + return up3, up4 diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Surface_head.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Surface_head.py new file mode 100644 index 0000000000000000000000000000000000000000..47c0cccb5be38fccbc1a0f769265389bfcba88ff --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/Surface_head.py @@ -0,0 +1,73 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Surface_Head(nn.Module): + """ + MLP: aims at learn iso-surface function Implicit function + """ + + def __init__(self, + filter_channels, + merge_layer=0, + res_layers=[], + norm='group', + last_op=None): + super(Surface_Head, self).__init__() + if last_op == 'sigmoid': + self.last_op = nn.Sigmoid() + elif last_op == 'tanh': + self.last_op = nn.Tanh() + else: + raise NotImplementedError( + 'only sigmoid/tanh function could be used') + + self.filters = nn.ModuleList() + self.norms = nn.ModuleList() + self.merge_layer = merge_layer if merge_layer > 0 else len( + filter_channels) // 2 + + self.res_layers = res_layers + self.norm = norm + + for i in range(0, len(filter_channels) - 1): + if i in self.res_layers: + self.filters.append( + nn.Conv1d(filter_channels[i] + filter_channels[0], + filter_channels[i + 1], 1)) + else: + self.filters.append( + nn.Conv1d(filter_channels[i], filter_channels[i + 1], 1)) + if i != len(filter_channels) - 2: + if norm == 'group': + self.norms.append(nn.GroupNorm(32, filter_channels[i + 1])) + elif norm == 'batch': + self.norms.append(nn.BatchNorm1d(filter_channels[i + 1])) + + def forward(self, feature): + """feature may include multiple view inputs + Parameters: + feature: [B, C_in, N] + return: + prediction: [B, C_out, N] and merge layer features + """ + + y = feature + tmpy = feature + phi = None + + for i, f in enumerate(self.filters): + y = f(y if i not in self.res_layers else torch.cat([y, tmpy], 1)) + if i != len(self.filters) - 1: + if self.norm not in ['batch', 'group']: + y = F.leaky_relu(y) + else: + y = F.leaky_relu(self.norms[i](y)) + if i == self.merge_layer: + phi = y.clone() + + if self.last_op is not None: + y = self.last_op(y) + return y, phi diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/__init__.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/detectors.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/detectors.py new file mode 100644 index 0000000000000000000000000000000000000000..4f63dd8c7b0409d949567af5b33d2c5ed178c6fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/detectors.py @@ -0,0 +1,66 @@ +# The implementation here is modified based on Pytorch, originally BSD License and publicly avaialbe at +# https://github.com/pytorch/pytorch +import numpy as np +import torch + + +class FasterRCNN(object): + ''' detect body + COCO_INSTANCE_CATEGORY_NAMES = [ + '__background__', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', + 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'N/A', 'stop sign', + 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', + 'elephant', 'bear', 'zebra', 'giraffe', 'N/A', 'backpack', 'umbrella', 'N/A', 'N/A', + 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', + 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', + 'bottle', 'N/A', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', + 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', + 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'N/A', 'dining table', + 'N/A', 'N/A', 'toilet', 'N/A', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'N/A', 'book', + 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' + ] + ''' + + def __init__(self, ckpt=None, device='cuda:0'): + """ + https://pytorch.org/docs/stable/torchvision/models.html#faster-r-cnn + """ + import torchvision + if ckpt is None: + self.model = torchvision.models.detection.fasterrcnn_resnet50_fpn( + pretrained=True) + else: + self.model = torchvision.models.detection.fasterrcnn_resnet50_fpn( + pretrained=False) + state_dict = torch.load(ckpt, map_location='cpu') + self.model.load_state_dict(state_dict) + self.model.to(device) + self.model.eval() + self.device = device + + @torch.no_grad() + def run(self, input): + """ + return: detected box, [x1, y1, x2, y2] + """ + prediction = self.model(input.to(self.device))[0] + inds = (prediction['labels'] == 1) * (prediction['scores'] > 0.5) + if len(inds) < 1: + return None + else: + bbox = prediction['boxes'][inds][0].cpu().numpy() + return bbox + + @torch.no_grad() + def run_multi(self, input): + """ + return: detected box, [x1, y1, x2, y2] + """ + prediction = self.model(input.to(self.device))[0] + inds = (prediction['labels'] == 1) * (prediction['scores'] > 0.9) + if len(inds) < 1: + return None + else: + bbox = prediction['boxes'][inds].cpu().numpy() + return bbox diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/geometry.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..fa4a00a6ba63591f42b5b3443948cca74e6cbc37 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/geometry.py @@ -0,0 +1,61 @@ +# The implementation here is modified based on PIFU, originally MIT License and publicly avaialbe at +# https://github.com/shunsukesaito/PIFu/blob/master/lib/geometry.py +import torch + + +def index(feat, uv): + """ + extract image features at floating coordinates with bilinear interpolation + args: + feat: [B, C, H, W] image features + uv: [B, 2, N] normalized image coordinates ranged in [-1, 1] + return: + [B, C, N] sampled pixel values + """ + uv = uv.transpose(1, 2) + uv = uv.unsqueeze(2) + samples = torch.nn.functional.grid_sample(feat, uv, align_corners=True) + return samples[:, :, :, 0] + + +def orthogonal(points, calib, transform=None): + """ + project points onto screen space using orthogonal projection + args: + points: [B, 3, N] 3d points in world coordinates + calib: [B, 3, 4] projection matrix + transform: [B, 2, 3] screen space transformation + return: + [B, 3, N] 3d coordinates in screen space + """ + rot = calib[:, :3, :3] + trans = calib[:, :3, 3:4] + pts = torch.baddbmm(trans, rot, points) + if transform is not None: + scale = transform[:2, :2] + shift = transform[:2, 2:3] + pts[:, :2, :] = torch.baddbmm(shift, scale, pts[:, :2, :]) + return pts + + +def perspective(points, calib, transform=None): + """ + project points onto screen space using perspective projection + args: + points: [B, 3, N] 3d points in world coordinates + calib: [B, 3, 4] projection matrix + transform: [B, 2, 3] screen space trasnformation + return: + [B, 3, N] 3d coordinates in screen space + """ + rot = calib[:, :3, :3] + trans = calib[:, :3, 3:4] + homo = torch.baddbmm(trans, rot, points) + xy = homo[:, :2, :] / homo[:, 2:3, :] + if transform is not None: + scale = transform[:2, :2] + shift = transform[:2, 2:3] + xy = torch.baddbmm(shift, scale, xy) + + xyz = torch.cat([xy, homo[:, 2:3, :]], 1) + return xyz diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/human_segmenter.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/human_segmenter.py new file mode 100644 index 0000000000000000000000000000000000000000..29bf6f701602ff019aef2b662adb658671434170 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/human_segmenter.py @@ -0,0 +1,60 @@ +# The implementation is also open-sourced by the authors, and available at +# https://www.modelscope.cn/models/damo/cv_unet_image-matting/summary +import cv2 +import numpy as np +import tensorflow as tf + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class human_segmenter(object): + + def __init__(self, model_path): + super(human_segmenter, self).__init__() + f = tf.gfile.FastGFile(model_path, 'rb') + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + persisted_graph = tf.import_graph_def(graph_def, name='') + + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.3 # 占用GPU 30%的显存 + self.sess = tf.InteractiveSession(graph=persisted_graph, config=config) + + self.image_node = self.sess.graph.get_tensor_by_name('input_image:0') + self.output_node = self.sess.graph.get_tensor_by_name('output_png:0') + self.logits_node = self.sess.graph.get_tensor_by_name('if_person:0') + print('human_segmenter init done') + + def image_preprocess(self, img): + if len(img.shape) == 2: + img = np.dstack((img, img, img)) + elif img.shape[2] == 4: + img = img[:, :, :3] + img = img.astype(float) + return img + + def run(self, img): + image_feed = self.image_preprocess(img) + output_img_value, logits_value = self.sess.run( + [self.output_node, self.logits_node], + feed_dict={self.image_node: image_feed}) + mask = output_img_value[:, :, -1] + return mask + + def get_human_bbox(self, mask): + print('dtype:{}, max:{},shape:{}'.format(mask.dtype, np.max(mask), + mask.shape)) + ret, thresh = cv2.threshold(mask, 127, 255, 0) + contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, + cv2.CHAIN_APPROX_SIMPLE) + if len(contours) == 0: + return None + + contoursArea = [cv2.contourArea(c) for c in contours] + max_area_index = contoursArea.index(max(contoursArea)) + bbox = cv2.boundingRect(contours[max_area_index]) + return bbox + + def release(self): + self.sess.close() diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/models/networks.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..266237b6b75d65b38c83c8b294d86a04b542b1d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/models/networks.py @@ -0,0 +1,366 @@ +# The implementation here is modified based on Pix2PixHD, originally BSD License and publicly avaialbe at +# https://github.com/NVIDIA/pix2pixHD +import functools + +import numpy as np +import torch +import torch.nn as nn + + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + m.weight.data.normal_(0.0, 0.02) + elif classname.find('BatchNorm2d') != -1: + m.weight.data.normal_(1.0, 0.02) + m.bias.data.fill_(0) + + +def get_norm_layer(norm_type='instance'): + if norm_type == 'batch': + norm_layer = functools.partial(nn.BatchNorm2d, affine=True) + elif norm_type == 'instance': + norm_layer = functools.partial(nn.InstanceNorm2d, affine=False) + else: + raise NotImplementedError('normalization layer [%s] is not found' + % norm_type) + return norm_layer + + +def define_G(input_nc, + output_nc, + ngf, + netG, + n_downsample_global=3, + n_blocks_global=9, + n_local_enhancers=1, + n_blocks_local=3, + norm='instance', + gpu_ids=[], + last_op=nn.Tanh()): + norm_layer = get_norm_layer(norm_type=norm) + if netG == 'global': + netG = GlobalGenerator( + input_nc, + output_nc, + ngf, + n_downsample_global, + n_blocks_global, + norm_layer, + last_op=last_op) + elif netG == 'local': + netG = LocalEnhancer(input_nc, output_nc, ngf, n_downsample_global, + n_blocks_global, n_local_enhancers, + n_blocks_local, norm_layer) + elif netG == 'encoder': + netG = Encoder(input_nc, output_nc, ngf, n_downsample_global, + norm_layer) + else: + raise ('generator not implemented!') + if len(gpu_ids) > 0: + assert (torch.cuda.is_available()) + netG.cuda(gpu_ids[0]) + netG.apply(weights_init) + return netG + + +def print_network(net): + if isinstance(net, list): + net = net[0] + num_params = 0 + for param in net.parameters(): + num_params += param.numel() + print(net) + print('Total number of parameters: %d' % num_params) + + +""" + Generator code +""" + + +class LocalEnhancer(nn.Module): + + def __init__(self, + input_nc, + output_nc, + ngf=32, + n_downsample_global=3, + n_blocks_global=9, + n_local_enhancers=1, + n_blocks_local=3, + norm_layer=nn.BatchNorm2d, + padding_type='reflect'): + super(LocalEnhancer, self).__init__() + self.n_local_enhancers = n_local_enhancers + + ngf_global = ngf * (2**n_local_enhancers) + model_global = GlobalGenerator(input_nc, output_nc, ngf_global, + n_downsample_global, n_blocks_global, + norm_layer).model + model_global = [model_global[i] for i in range(len(model_global) - 3) + ] # get rid of final convolution layers + self.model = nn.Sequential(*model_global) + + for n in range(1, n_local_enhancers + 1): + ngf_global = ngf * (2**(n_local_enhancers - n)) + model_downsample = [ + nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, ngf_global, kernel_size=7, padding=0), + norm_layer(ngf_global), + nn.ReLU(True), + nn.Conv2d( + ngf_global, + ngf_global * 2, + kernel_size=3, + stride=2, + padding=1), + norm_layer(ngf_global * 2), + nn.ReLU(True) + ] + model_upsample = [] + for i in range(n_blocks_local): + model_upsample += [ + ResnetBlock( + ngf_global * 2, + padding_type=padding_type, + norm_layer=norm_layer) + ] + + model_upsample += [ + nn.ConvTranspose2d( + ngf_global * 2, + ngf_global, + kernel_size=3, + stride=2, + padding=1, + output_padding=1), + norm_layer(ngf_global), + nn.ReLU(True) + ] + + if n == n_local_enhancers: + model_upsample += [ + nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0), + nn.Tanh() + ] + + setattr(self, 'model' + str(n) + '_1', + nn.Sequential(*model_downsample)) + setattr(self, 'model' + str(n) + '_2', + nn.Sequential(*model_upsample)) + + self.downsample = nn.AvgPool2d( + 3, stride=2, padding=[1, 1], count_include_pad=False) + + def forward(self, input): + input_downsampled = [input] + for i in range(self.n_local_enhancers): + input_downsampled.append(self.downsample(input_downsampled[-1])) + + output_prev = self.model(input_downsampled[-1]) + for n_local_enhancers in range(1, self.n_local_enhancers + 1): + model_downsample = getattr(self, + 'model' + str(n_local_enhancers) + '_1') + model_upsample = getattr(self, + 'model' + str(n_local_enhancers) + '_2') + input_i = input_downsampled[self.n_local_enhancers + - n_local_enhancers] + output_prev = model_upsample( + model_downsample(input_i) + output_prev) + return output_prev + + +class GlobalGenerator(nn.Module): + + def __init__(self, + input_nc, + output_nc, + ngf=64, + n_downsampling=3, + n_blocks=9, + norm_layer=nn.BatchNorm2d, + padding_type='reflect', + last_op=nn.Tanh()): + assert (n_blocks >= 0) + super(GlobalGenerator, self).__init__() + activation = nn.ReLU(True) + + model = [ + nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), activation + ] + for i in range(n_downsampling): + mult = 2**i + model += [ + nn.Conv2d( + ngf * mult, + ngf * mult * 2, + kernel_size=3, + stride=2, + padding=1), + norm_layer(ngf * mult * 2), activation + ] + + mult = 2**n_downsampling + for i in range(n_blocks): + model += [ + ResnetBlock( + ngf * mult, + padding_type=padding_type, + activation=activation, + norm_layer=norm_layer) + ] + + for i in range(n_downsampling): + mult = 2**(n_downsampling - i) + model += [ + nn.ConvTranspose2d( + ngf * mult, + int(ngf * mult / 2), + kernel_size=3, + stride=2, + padding=1, + output_padding=1), + norm_layer(int(ngf * mult / 2)), activation + ] + model += [ + nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0) + ] + if last_op is not None: + model += [last_op] + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + + +""" + Define a resnet block +""" + + +class ResnetBlock(nn.Module): + + def __init__(self, + dim, + padding_type, + norm_layer, + activation=nn.ReLU(True), + use_dropout=False): + super(ResnetBlock, self).__init__() + self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, + activation, use_dropout) + + def build_conv_block(self, dim, padding_type, norm_layer, activation, + use_dropout): + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' + % padding_type) + + conv_block += [ + nn.Conv2d(dim, dim, kernel_size=3, padding=p), + norm_layer(dim), activation + ] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' + % padding_type) + conv_block += [ + nn.Conv2d(dim, dim, kernel_size=3, padding=p), + norm_layer(dim) + ] + + return nn.Sequential(*conv_block) + + def forward(self, x): + out = x + self.conv_block(x) + return out + + +class Encoder(nn.Module): + + def __init__(self, + input_nc, + output_nc, + ngf=32, + n_downsampling=4, + norm_layer=nn.BatchNorm2d): + super(Encoder, self).__init__() + self.output_nc = output_nc + + model = [ + nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), + nn.ReLU(True) + ] + for i in range(n_downsampling): + mult = 2**i + model += [ + nn.Conv2d( + ngf * mult, + ngf * mult * 2, + kernel_size=3, + stride=2, + padding=1), + norm_layer(ngf * mult * 2), + nn.ReLU(True) + ] + + for i in range(n_downsampling): + mult = 2**(n_downsampling - i) + model += [ + nn.ConvTranspose2d( + ngf * mult, + int(ngf * mult / 2), + kernel_size=3, + stride=2, + padding=1, + output_padding=1), + norm_layer(int(ngf * mult / 2)), + nn.ReLU(True) + ] + + model += [ + nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0), + nn.Tanh() + ] + self.model = nn.Sequential(*model) + + def forward(self, input, inst): + outputs = self.model(input) + + outputs_mean = outputs.clone() + inst_list = np.unique(inst.cpu().numpy().astype(int)) + for i in inst_list: + for b in range(input.size()[0]): + indices = (inst[b:b + 1] == int(i)).nonzero() + for j in range(self.output_nc): + output_ins = outputs[indices[:, 0] + b, indices[:, 1] + j, + indices[:, 2], indices[:, 3]] + mean_feat = torch.mean(output_ins).expand_as(output_ins) + outputs_mean[indices[:, 0] + b, indices[:, 1] + j, + indices[:, 2], indices[:, 3]] = mean_feat + return outputs_mean diff --git a/runtime/Lib/site_packages/models/cv/human_reconstruction/utils.py b/runtime/Lib/site_packages/models/cv/human_reconstruction/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..67e1efdba1518cfd9d353bfe4f20820cd17a1f2d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/human_reconstruction/utils.py @@ -0,0 +1,178 @@ +import os + +import mcubes +import numpy as np +import torch + + +def save_obj_mesh_with_color(mesh_path, verts, faces, colors): + file = open(mesh_path, 'w') + for idx, v in enumerate(verts): + c = colors[idx] + file.write('v %.4f %.4f %.4f %.4f %.4f %.4f\n' % + (v[0], v[1], v[2], c[0], c[1], c[2])) + for f in faces: + f_plus = f + 1 + file.write('f %d %d %d\n' % (f_plus[0], f_plus[2], f_plus[1])) + file.close() + + +def save_obj_mesh(mesh_path, verts, faces): + file = open(mesh_path, 'w') + for idx, v in enumerate(verts): + file.write('v %.4f %.4f %.4f\n' % (v[0], v[1], v[2])) + for f in faces: + f_plus = f + 1 + file.write('f %d %d %d\n' % (f_plus[0], f_plus[2], f_plus[1])) + file.close() + + +def to_tensor(img): + if len(img.shape) == 2: + img = img[:, :, np.newaxis] + img = torch.from_numpy(img.transpose(2, 0, 1)).float() + img = img / 255. + return img + + +def reconstruction(net, calib_tensor, coords, mat, num_samples=50000): + + def eval_func(points): + points = np.expand_dims(points, axis=0) + points = np.repeat(points, 1, axis=0) + samples = torch.from_numpy(points).cuda().float() + net.query(samples, calib_tensor) + pred = net.get_preds() + pred = pred[0] + return pred.detach().cpu().numpy() + + sdf = eval_grid(coords, eval_func, num_samples=num_samples) + vertices, faces = mcubes.marching_cubes(sdf, 0.5) + verts = np.matmul(mat[:3, :3], vertices.T) + mat[:3, 3:4] + verts = verts.T + return verts, faces + + +def keep_largest(mesh_big): + mesh_lst = mesh_big.split(only_watertight=False) + keep_mesh = mesh_lst[0] + for mesh in mesh_lst: + if mesh.vertices.shape[0] > keep_mesh.vertices.shape[0]: + keep_mesh = mesh + return keep_mesh + + +def eval_grid(coords, + eval_func, + init_resolution=64, + threshold=0.01, + num_samples=512 * 512 * 512): + resolution = coords.shape[1:4] + sdf = np.zeros(resolution) + dirty = np.ones(resolution, dtype=bool) + grid_mask = np.zeros(resolution, dtype=bool) + reso = resolution[0] // init_resolution + + while reso > 0: + grid_mask[0:resolution[0]:reso, 0:resolution[1]:reso, + 0:resolution[2]:reso] = True + test_mask = np.logical_and(grid_mask, dirty) + points = coords[:, test_mask] + + sdf[test_mask] = batch_eval(points, eval_func, num_samples=num_samples) + dirty[test_mask] = False + + if reso <= 1: + break + for x in range(0, resolution[0] - reso, reso): + for y in range(0, resolution[1] - reso, reso): + for z in range(0, resolution[2] - reso, reso): + if not dirty[x + reso // 2, y + reso // 2, z + reso // 2]: + continue + v0 = sdf[x, y, z] + v1 = sdf[x, y, z + reso] + v2 = sdf[x, y + reso, z] + v3 = sdf[x, y + reso, z + reso] + v4 = sdf[x + reso, y, z] + v5 = sdf[x + reso, y, z + reso] + v6 = sdf[x + reso, y + reso, z] + v7 = sdf[x + reso, y + reso, z + reso] + v = np.array([v0, v1, v2, v3, v4, v5, v6, v7]) + v_min = v.min() + v_max = v.max() + if (v_max - v_min) < threshold: + sdf[x:x + reso, y:y + reso, + z:z + reso] = (v_max + v_min) / 2 + dirty[x:x + reso, y:y + reso, z:z + reso] = False + reso //= 2 + + return sdf.reshape(resolution) + + +def batch_eval(points, eval_func, num_samples=512 * 512 * 512): + num_pts = points.shape[1] + sdf = np.zeros(num_pts) + + num_batches = num_pts // num_samples + for i in range(num_batches): + sdf[i * num_samples:i * num_samples + num_samples] = eval_func( + points[:, i * num_samples:i * num_samples + num_samples]) + if num_pts % num_samples: + sdf[num_batches * num_samples:] = eval_func(points[:, num_batches + * num_samples:]) + return sdf + + +def create_grid(res, + b_min=np.array([0, 0, 0]), + b_max=np.array([1, 1, 1]), + transform=None): + coords = np.mgrid[:res, :res, :res] + + coords = coords.reshape(3, -1) + coords_matrix = np.eye(4) + length = b_max - b_min + + coords_matrix[0, 0] = length[0] / res + coords_matrix[1, 1] = length[1] / res + coords_matrix[2, 2] = length[2] / res + coords_matrix[0:3, 3] = b_min + + coords = np.matmul(coords_matrix[:3, :3], coords) + coords_matrix[:3, 3:4] + if transform is not None: + coords = np.matmul(transform[:3, :3], coords) + transform[:3, 3:4] + coords_matrix = np.matmul(transform, coords_matrix) + coords = coords.reshape(3, res, res, res) + return coords, coords_matrix + + +def get_submesh(verts, + faces, + color, + verts_retained=None, + faces_retained=None, + min_vert_in_face=2): + verts = verts + faces = faces + colors = color + if verts_retained is not None: + if verts_retained.dtype != 'bool': + vert_mask = np.zeros(len(verts), dtype=bool) + vert_mask[verts_retained] = True + else: + vert_mask = verts_retained + bool_faces = np.sum( + vert_mask[faces.ravel()].reshape(-1, 3), axis=1) > min_vert_in_face + elif faces_retained is not None: + if faces_retained.dtype != 'bool': + bool_faces = np.zeros(len(faces_retained), dtype=bool) + else: + bool_faces = faces_retained + new_faces = faces[bool_faces] + vertex_ids = list(set(new_faces.ravel())) + oldtonew = -1 * np.ones([len(verts)]) + oldtonew[vertex_ids] = range(0, len(vertex_ids)) + new_verts = verts[vertex_ids] + new_colors = colors[vertex_ids] + new_faces = oldtonew[new_faces].astype('int32') + return (new_verts, new_faces, new_colors, bool_faces, vertex_ids) diff --git a/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/__init__.py b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8891177387ab16d2f7fa6f93450aa188fcc0374d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .binary_quant_model import BinaryQuantClassificationModel + +else: + _import_structure = { + 'binary_quant_model': ['BinaryQuantClassificationModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/binary_quant_model.py b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/binary_quant_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0afcbe1d245a39c32f765ca7735e4ce5ab55018b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/binary_quant_model.py @@ -0,0 +1,82 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import OrderedDict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.hub import read_config +from modelscope.utils.logger import get_logger +from .bnext import BNext + +logger = get_logger() + +__all__ = ['BinaryQuantClassificationModel'] + + +@MODELS.register_module(Tasks.image_classification, module_name=Models.bnext) +class BinaryQuantClassificationModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + if torch.cuda.is_available(): + self._device = torch.device('cuda') + logger.info('Use GPU: {}'.format(self._device)) + else: + self._device = torch.device('cpu') + logger.info('Use CPU: {}'.format(self._device)) + + self.model = BNext(num_classes=1000) + self.model = self.model.to(self._device) + + self.model_dir = model_dir + + self._load_pretrained_checkpoint() + + def forward(self, inputs, return_loss=False): + + return self.model(**inputs) + + def _convert_state_dict(self, state_dict): + """Converts a state dict saved from a dataParallel module to normal + module state_dict inplace + :param state_dict is the loaded DataParallel model_state + """ + if not next(iter(state_dict)).startswith('module.'): + return state_dict # abort if dict is not a DataParallel model_state + new_state_dict = OrderedDict() + + split_index = 0 + for cur_key, _ in state_dict.items(): + if cur_key.startswith('module.model'): + split_index = 13 + elif cur_key.startswith('module'): + split_index = 7 + break + + for k, v in state_dict.items(): + name = k[split_index:] # remove `module.` + new_state_dict[name] = v + return new_state_dict + + def _load_pretrained_checkpoint(self): + model_path = os.path.join(self.model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(model_path) + if os.path.exists(model_path): + ckpt = torch.load(model_path, 'cpu') + model_state = self._convert_state_dict(ckpt['state_dict']) + + if ckpt.get('meta', None): + self.CLASSES = ckpt['meta'] + self.config_type = 'ms_config' + self.model.load_state_dict(model_state) + self.model.to(self._device) + + else: + logger.error( + '[checkModelPath]:model path dose not exits!!! model Path:' + + model_path) + raise Exception('[checkModelPath]:model path dose not exits!') diff --git a/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/bnext.py b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/bnext.py new file mode 100644 index 0000000000000000000000000000000000000000..fd1f73446c1c1071e26e84ed584dab5748f42baf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_binary_quant_classification/bnext.py @@ -0,0 +1,664 @@ +# Part of the implementation is borrowed and modified from BNext, +# publicly available at https://github.com/hpi-xnor/BNext +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +# stage ratio: 1:1:3:1 +stage_out_channel_tiny = [32] + [ + 64 +] + [128] * 2 + [256] * 2 + [512] * 6 + [1024] * 2 + +# stage ratio 1:1:3:1 +stage_out_channel_small = [48] + [ + 96 +] + [192] * 2 + [384] * 2 + [768] * 6 + [1536] * 2 + +# stage ratio 2:2:4:2 +stage_out_channel_middle = [48] + [ + 96 +] + [192] * 4 + [384] * 4 + [768] * 8 + [1536] * 4 + +# stage ratio 2:2:8:2 +stage_out_channel_large = [64] + [ + 128 +] + [256] * 4 + [512] * 4 + [1024] * 16 + [2048] * 4 + + +def conv3x3(in_planes, + out_planes, + kernel_size=3, + stride=1, + groups=1, + dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=kernel_size, + stride=stride, + padding=kernel_size // 2, + dilation=dilation, + groups=groups, + bias=False) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class HardSigmoid(nn.Module): + + def __init__(self, ): + super(HardSigmoid, self).__init__() + + def forward(self, x): + return F.relu6(x + 3) / 6 + + +class firstconv3x3(nn.Module): + + def __init__(self, inp, oup, stride): + super(firstconv3x3, self).__init__() + + self.conv1 = nn.Conv2d(inp, oup, 3, stride, 1, bias=False) + self.bn1 = nn.BatchNorm2d(oup) + self.prelu = nn.PReLU(oup, oup) + + def forward(self, x): + + out = self.conv1(x) + out = self.bn1(out) + + return out + + +class LearnableBias(nn.Module): + + def __init__(self, out_chn): + super(LearnableBias, self).__init__() + self.bias = nn.Parameter( + torch.zeros(1, out_chn, 1, 1), requires_grad=True) + + def forward(self, x): + out = x + self.bias.expand_as(x) + return out + + +class HardSign(nn.Module): + + def __init__(self, range=[-1, 1], progressive=False): + super(HardSign, self).__init__() + self.range = range + self.progressive = progressive + self.register_buffer('temperature', torch.ones(1)) + + def adjust(self, x, scale=0.1): + self.temperature.mul_(scale) + + def forward(self, x): + replace = x.clamp(self.range[0], self.range[1]) + x = x.div(self.temperature.clamp(min=1e-8)).clamp(-1, 1) + if not self.progressive: + sign = x.sign() + else: + sign = x + return (sign - replace).detach() + replace + + +class HardBinaryConv(nn.Module): + + def __init__(self, + in_chn, + out_chn, + kernel_size=3, + stride=1, + padding=1, + groups=1): + super(HardBinaryConv, self).__init__() + self.stride = stride + self.padding = kernel_size // 2 + self.groups = groups + self.number_of_weights = in_chn // groups * out_chn * kernel_size * kernel_size + self.shape = (out_chn, in_chn // groups, kernel_size, kernel_size) + self.weight = nn.Parameter( + torch.randn((self.shape)) * 0.001, requires_grad=True) + + self.register_buffer('temperature', torch.ones(1)) + + def forward(self, x): + if self.training: + self.weight.data.clamp_(-1.5, 1.5) + + real_weights = self.weight + + if self.temperature < 1e-7: + binary_weights_no_grad = real_weights.sign() + else: + binary_weights_no_grad = ( + real_weights / self.temperature.clamp(min=1e-8)).clamp(-1, 1) + cliped_weights = real_weights + + if self.training: + binary_weights = binary_weights_no_grad.detach( + ) - cliped_weights.detach() + cliped_weights + else: + binary_weights = binary_weights_no_grad + + y = F.conv2d( + x, + binary_weights, + stride=self.stride, + padding=self.padding, + groups=self.groups) + + return y + + +class SqueezeAndExpand(nn.Module): + + def __init__(self, + channels, + planes, + ratio=8, + attention_mode='hard_sigmoid'): + super(SqueezeAndExpand, self).__init__() + self.se = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(channels, channels // ratio, kernel_size=1, padding=0), + nn.ReLU(channels // ratio), + nn.Conv2d(channels // ratio, planes, kernel_size=1, padding=0), + ) + + if attention_mode == 'sigmoid': + self.attention = nn.Sigmoid() + + elif attention_mode == 'hard_sigmoid': + self.attention = HardSigmoid() + + else: + self.attention = nn.Softmax(dim=1) + + def forward(self, x): + x = self.se(x) + x = self.attention(x) + return x + + +class Attention(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + infor_recoupling=True, + groups=1): + super(Attention, self).__init__() + + self.inplanes = inplanes + self.planes = planes + self.infor_recoupling = infor_recoupling + + self.move = LearnableBias(inplanes) + self.binary_activation = HardSign(range=[-1.5, 1.5]) + self.binary_conv = HardBinaryConv( + inplanes, planes, kernel_size=3, stride=stride, groups=groups) + + self.norm1 = nn.BatchNorm2d(planes) + self.norm2 = nn.BatchNorm2d(planes) + + self.activation1 = nn.PReLU(inplanes) + self.activation2 = nn.PReLU(planes) + + self.downsample = downsample + self.stride = stride + if stride == 2: + self.pooling = nn.AvgPool2d(2, 2) + + if self.infor_recoupling: + self.se = SqueezeAndExpand( + planes, planes, attention_mode='sigmoid') + self.scale = nn.Parameter(torch.ones(1, planes, 1, 1) * 0.5) + + def forward(self, input): + + residual = self.activation1(input) + + if self.stride == 2: + residual = self.pooling(residual) + + x = self.move(input) + x = self.binary_activation(x) + x = self.binary_conv(x) + x = self.norm1(x) + x = self.activation2(x) + + if self.infor_recoupling: + if self.training: + self.scale.data.clamp_(0, 1) + if self.stride == 2: + input = self.pooling(input) + mix = self.scale * input + x * (1 - self.scale) + x = self.se(mix) * x + else: + pass + x = x * residual + x = self.norm2(x) + x = x + residual + + return x + + +class FFN_3x3(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + infor_recoupling=True, + groups=1): + super(FFN_3x3, self).__init__() + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.infor_recoupling = infor_recoupling + + self.move = LearnableBias(inplanes) + self.binary_activation = HardSign(range=[-1.5, 1.5]) + self.binary_conv = HardBinaryConv( + inplanes, planes, kernel_size=3, stride=stride, groups=groups) + + self.norm1 = nn.BatchNorm2d(planes) + self.norm2 = nn.BatchNorm2d(planes) + + self.activation1 = nn.PReLU(inplanes) + self.activation2 = nn.PReLU(planes) + + if stride == 2: + self.pooling = nn.AvgPool2d(2, 2) + + if self.infor_recoupling: + self.se = SqueezeAndExpand( + inplanes, planes, attention_mode='sigmoid') + self.scale = nn.Parameter(torch.ones(1, planes, 1, 1) * 0.5) + + def forward(self, input): + + residual = input + + if self.stride == 2: + residual = self.pooling(residual) + + x = self.move(input) + x = self.binary_activation(x) + x = self.binary_conv(x) + x = self.norm1(x) + x = self.activation2(x) + + if self.infor_recoupling: + if self.training: + self.scale.data.clamp_(0, 1) + if self.stride == 2: + input = self.pooling(input) + mix = self.scale * input + (1 - self.scale) * x + x = self.se(mix) * x + x = self.norm2(x) + else: + pass + + x = x + residual + + return x + + +class FFN_1x1(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + attention=True, + drop_rate=0.1, + infor_recoupling=True): + super(FFN_1x1, self).__init__() + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.infor_recoupling = infor_recoupling + + self.move = LearnableBias(inplanes) + self.binary_activation = HardSign(range=[-1.5, 1.5]) + self.binary_conv = HardBinaryConv( + inplanes, planes, kernel_size=1, stride=stride, padding=0) + + self.norm1 = nn.BatchNorm2d(planes) + self.norm2 = nn.BatchNorm2d(planes) + + self.activation1 = nn.PReLU(inplanes) + self.activation2 = nn.PReLU(planes) + + if stride == 2: + self.pooling = nn.AvgPool2d(2, 2) + + if self.infor_recoupling: + self.se = SqueezeAndExpand( + inplanes, planes, attention_mode='sigmoid') + self.scale = nn.Parameter(torch.ones(1, planes, 1, 1) * 0.5) + + def forward(self, input): + + residual = input + + if self.stride == 2: + residual = self.pooling(residual) + + x = self.move(input) + x = self.binary_activation(x) + x = self.binary_conv(x) + x = self.norm1(x) + x = self.activation2(x) + if self.infor_recoupling: + self.scale.data.clamp_(0, 1) + mix = self.scale * input + (1 - self.scale) * x + x = self.se(mix) * x + x = self.norm2(x) + else: + pass + + x = x + residual + + return x + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + mode='scale'): + super(BasicBlock, self).__init__() + self.inplanes = inplanes + self.planes = planes + + if mode == 'scale': + self.Attention = Attention( + inplanes, + inplanes, + stride, + None, + drop_rate=drop_rate, + groups=1) + else: + self.Attention = FFN_3x3( + inplanes, + inplanes, + stride, + None, + drop_rate=drop_rate, + groups=1) + + if inplanes == planes: + self.FFN = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + + else: + self.FFN_1 = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + + self.FFN_2 = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + + def forward(self, input): + x = self.Attention(input) + + if self.inplanes == self.planes: + y = self.FFN(x) + + else: + y_1 = self.FFN_1(x) + y_2 = self.FFN_2(x) + y = torch.cat((y_1, y_2), dim=1) + + return y + + +class BasicBlock_No_ELM_Attention(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + mode='scale'): + super(BasicBlock_No_ELM_Attention, self).__init__() + self.inplanes = inplanes + self.planes = planes + + self.FFN_3x3 = FFN_3x3( + inplanes, inplanes, stride, None, drop_rate=drop_rate, groups=1) + + if self.inplanes == self.planes: + self.FFN = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + else: + self.FFN_1 = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + self.FFN_2 = FFN_1x1(inplanes, inplanes, drop_rate=drop_rate) + + def forward(self, input): + x = self.FFN_3x3(input) + if self.inplanes == self.planes: + y = self.FFN(x) + else: + y_1 = self.FFN_1(x) + y_2 = self.FFN_2(x) + y = torch.cat((y_1, y_2), dim=1) + + return y + + +class BasicBlock_No_Infor_Recoupling(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + mode='scale'): + super(BasicBlock_No_Infor_Recoupling, self).__init__() + self.inplanes = inplanes + self.planes = planes + + if mode == 'scale': + self.Attention = Attention( + inplanes, + inplanes, + stride, + None, + drop_rate, + infor_recoupling=False, + groups=1) + else: + self.Attention = FFN_3x3( + inplanes, + inplanes, + stride, + None, + drop_rate=drop_rate, + infor_recoupling=False, + groups=1) + + if self.inplanes == self.planes: + self.FFN = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + else: + self.FFN_1 = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + self.FFN_2 = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + + def forward(self, input): + x = self.Attention(input) + if self.inplanes == self.planes: + y = self.FFN(x) + else: + y_1 = self.FFN_1(x) + y_2 = self.FFN_2(x) + y = torch.cat((y_1, y_2), dim=1) + + return y + + +class BasicBlock_No_Extra_Design(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + drop_rate=0.1, + mode='scale'): + super(BasicBlock_No_Extra_Design, self).__init__() + self.inplanes = inplanes + self.planes = planes + + self.FFN_3x3 = FFN_3x3( + inplanes, + inplanes, + stride, + None, + drop_rate, + infor_recoupling=False, + groups=1) + if self.inplanes == self.planes: + self.FFN = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + else: + self.FFN_1 = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + self.FFN_2 = FFN_1x1( + inplanes, + inplanes, + drop_rate=drop_rate, + infor_recoupling=False) + + def forward(self, input): + x = self.FFN_3x3(input) + if self.inplanes == self.planes: + y = self.FFN(x) + else: + y_1 = self.FFN_1(x) + y_2 = self.FFN_2(x) + y = torch.cat((y_1, y_2), dim=1) + + return y + + +class BNext(nn.Module): + + def __init__(self, + num_classes=1000, + size='small', + ELM_Attention=True, + Infor_Recoupling=True): + super(BNext, self).__init__() + drop_rate = 0.2 if num_classes == 100 else 0.0 + + if size == 'tiny': + stage_out_channel = stage_out_channel_tiny + elif size == 'small': + stage_out_channel = stage_out_channel_small + elif size == 'middle': + stage_out_channel = stage_out_channel_middle + elif size == 'large': + stage_out_channel = stage_out_channel_large + else: + raise ValueError('The size is not defined!') + + if ELM_Attention and Infor_Recoupling: + basicblock = BasicBlock + print('Model with ELM Attention and Infor-Recoupling') + elif (ELM_Attention and not Infor_Recoupling): + basicblock = BasicBlock_No_Infor_Recoupling + print('Model with ELM Attention, No Infor-Recoupling') + elif (not ELM_Attention and Infor_Recoupling): + basicblock = BasicBlock_No_ELM_Attention + print('Model with Infor-Recoupling, No ELM Attention') + else: + basicblock = BasicBlock_No_Extra_Design + print('Model with no Extra Design') + + self.feature = nn.ModuleList() + drop_rates = [ + x.item() + for x in torch.linspace(0, drop_rate, (len(stage_out_channel))) + ] + + for i in range(len(stage_out_channel)): + if i == 0: + self.feature.append( + firstconv3x3(3, stage_out_channel[i], + 1 if num_classes != 1000 else 2)) + elif i == 1: + self.feature.append((basicblock( + stage_out_channel[i - 1], + stage_out_channel[i], + 1, + drop_rate=drop_rates[i], + mode='bias'))) + elif stage_out_channel[i - 1] != stage_out_channel[ + i] and stage_out_channel[i] != stage_out_channel[1]: + self.feature.append( + basicblock( + stage_out_channel[i - 1], + stage_out_channel[i], + 2, + drop_rate=drop_rates[i], + mode='scale' if i % 2 == 0 else 'bias')) + else: + self.feature.append( + basicblock( + stage_out_channel[i - 1], + stage_out_channel[i], + 1, + drop_rate=drop_rates[i], + mode='scale' if i % 2 == 0 else 'bias')) + + self.prelu = nn.PReLU(stage_out_channel[-1]) + self.pool1 = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Linear(stage_out_channel[-1], num_classes) + + def forward(self, img, return_loss=False, img_metas=None): + x = img + for i, block in enumerate(self.feature): + x = block(x) + x = self.prelu(x) + x = self.pool1(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + x = list(x.detach().cpu().numpy()) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/__init__.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a04f110d41e6425ca0a7bfd2af180a5cf58d55fb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_body_reshaping import ImageBodyReshaping + +else: + _import_structure = {'image_body_reshaping': ['ImageBodyReshaping']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/image_body_reshaping.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/image_body_reshaping.py new file mode 100644 index 0000000000000000000000000000000000000000..4aed8d98ca41c168d344c2624d3702246ce350c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/image_body_reshaping.py @@ -0,0 +1,128 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .model import FlowGenerator +from .person_info import PersonInfo +from .pose_estimator.body import Body +from .slim_utils import image_warp_grid1, resize_on_long_side + +logger = get_logger() + +__all__ = ['ImageBodyReshaping'] + + +@MODELS.register_module( + Tasks.image_body_reshaping, module_name=Models.image_body_reshaping) +class ImageBodyReshaping(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image body reshaping model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + if torch.cuda.is_available(): + self.device = torch.device('cuda') + else: + self.device = torch.device('cpu') + + self.degree = 1.0 + self.reshape_model = FlowGenerator(n_channels=16).to(self.device) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoints = torch.load(model_path, map_location=torch.device('cpu')) + self.reshape_model.load_state_dict( + checkpoints['state_dict'], strict=True) + self.reshape_model.eval() + logger.info('load body reshaping model done') + + pose_model_ckpt = os.path.join(model_dir, 'body_pose_model.pth') + self.pose_esti = Body(pose_model_ckpt, self.device) + logger.info('load pose model done') + + def pred_joints(self, img): + if img is None: + return None + small_src, resize_scale = resize_on_long_side(img, 300) + body_joints = self.pose_esti(small_src) + + if body_joints.shape[0] >= 1: + body_joints[:, :, :2] = body_joints[:, :, :2] / resize_scale + + return body_joints + + def pred_flow(self, img): + + body_joints = self.pred_joints(img) + small_size = 1200 + + if img.shape[0] > small_size or img.shape[1] > small_size: + _img, _scale = resize_on_long_side(img, small_size) + body_joints[:, :, :2] = body_joints[:, :, :2] * _scale + else: + _img = img + + # We only reshape one person + if body_joints.shape[0] < 1 or body_joints.shape[0] > 1: + return None + + person = PersonInfo(body_joints[0]) + + with torch.no_grad(): + person_pred = person.pred_flow(_img, self.reshape_model, + self.device) + + flow = np.dstack((person_pred['rDx'], person_pred['rDy'])) + + scale = img.shape[0] * 1.0 / flow.shape[0] + + flow = cv2.resize(flow, (img.shape[1], img.shape[0])) + flow *= scale + + return flow + + def warp(self, src_img, flow): + + X_flow = flow[..., 0] + Y_flow = flow[..., 1] + + X_flow = np.ascontiguousarray(X_flow) + Y_flow = np.ascontiguousarray(Y_flow) + + pred = image_warp_grid1(X_flow, Y_flow, src_img, 1.0, 0, 0) + return pred + + def inference(self, img): + img = img.cpu().numpy() + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + flow = self.pred_flow(img) + + if flow is None: + return img + + assert flow.shape[:2] == img.shape[:2] + + mag, ang = cv2.cartToPolar(flow[..., 0] + 1e-8, flow[..., 1] + 1e-8) + mag -= 3 + mag[mag <= 0] = 0 + + x, y = cv2.polarToCart(mag, ang, angleInDegrees=False) + flow = np.dstack((x, y)) + + flow *= self.degree + pred = self.warp(img, flow) + out_img = np.clip(pred, 0, 255) + logger.info('model inference done') + + return out_img.astype(np.uint8) diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/model.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/model.py new file mode 100644 index 0000000000000000000000000000000000000000..174428a1dbccb27342985caa24bae6ca9b2043af --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/model.py @@ -0,0 +1,189 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ConvLayer(nn.Module): + + def __init__(self, in_ch, out_ch): + super(ConvLayer, self).__init__() + + self.conv = nn.Sequential( + nn.ReflectionPad2d(1), + nn.Conv2d(in_ch, out_ch, kernel_size=3, padding=0), + nn.BatchNorm2d(out_ch), nn.ReLU(inplace=True)) + + def forward(self, x): + x = self.conv(x) + return x + + +class SASA(nn.Module): + + def __init__(self, in_dim): + super(SASA, self).__init__() + self.chanel_in = in_dim + + self.query_conv = nn.Conv2d( + in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1) + self.key_conv = nn.Conv2d( + in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1) + self.value_conv = nn.Conv2d( + in_channels=in_dim, out_channels=in_dim, kernel_size=1) + self.mag_conv = nn.Conv2d( + in_channels=5, out_channels=in_dim // 32, kernel_size=1) + + self.gamma = nn.Parameter(torch.zeros(1)) + + self.softmax = nn.Softmax(dim=-1) # + self.sigmoid = nn.Sigmoid() + + def structure_encoder(self, paf_mag, target_height, target_width): + torso_mask = torch.sum(paf_mag[:, 1:3, :, :], dim=1, keepdim=True) + torso_mask = torch.clamp(torso_mask, 0, 1) + + arms_mask = torch.sum(paf_mag[:, 4:8, :, :], dim=1, keepdim=True) + arms_mask = torch.clamp(arms_mask, 0, 1) + + legs_mask = torch.sum(paf_mag[:, 8:12, :, :], dim=1, keepdim=True) + legs_mask = torch.clamp(legs_mask, 0, 1) + + fg_mask = paf_mag[:, 12, :, :].unsqueeze(1) + bg_mask = 1 - fg_mask + Y = torch.cat((arms_mask, torso_mask, legs_mask, fg_mask, bg_mask), + dim=1) + Y = F.interpolate(Y, size=(target_height, target_width), mode='area') + return Y + + def forward(self, X, PAF_mag): + """extract self-attention features. + Args: + X : input feature maps( B x C x H x W) + PAF_mag : ( B x C x H x W), 1 denotes connectivity, 0 denotes non-connectivity + + Returns: + out : self attention value + input feature + Y: B X N X N (N is Width*Height) + """ + + m_batchsize, C, height, width = X.size() + + Y = self.structure_encoder(PAF_mag, height, width) + + connectivity_mask_vec = self.mag_conv(Y).view(m_batchsize, -1, + width * height) + affinity = torch.bmm( + connectivity_mask_vec.permute(0, 2, 1), connectivity_mask_vec) + affinity_centered = affinity - torch.mean(affinity) + affinity_sigmoid = self.sigmoid(affinity_centered) + + proj_query = self.query_conv(X).view(m_batchsize, -1, + width * height).permute(0, 2, 1) + proj_key = self.key_conv(X).view(m_batchsize, -1, width * height) + selfatten_map = torch.bmm(proj_query, proj_key) + selfatten_centered = selfatten_map - torch.mean( + selfatten_map) # centering + selfatten_sigmoid = self.sigmoid(selfatten_centered) + + SASA_map = selfatten_sigmoid * affinity_sigmoid + + proj_value = self.value_conv(X).view(m_batchsize, -1, width * height) + + out = torch.bmm(proj_value, SASA_map.permute(0, 2, 1)) + out = out.view(m_batchsize, C, height, width) + + out = self.gamma * out + X + return out, Y + + +class FlowGenerator(nn.Module): + + def __init__(self, n_channels, deep_supervision=False): + super(FlowGenerator, self).__init__() + self.deep_supervision = deep_supervision + + self.Encoder = nn.Sequential( + ConvLayer(n_channels, 64), + ConvLayer(64, 64), + nn.MaxPool2d(2), + ConvLayer(64, 128), + ConvLayer(128, 128), + nn.MaxPool2d(2), + ConvLayer(128, 256), + ConvLayer(256, 256), + nn.MaxPool2d(2), + ConvLayer(256, 512), + ConvLayer(512, 512), + nn.MaxPool2d(2), + ConvLayer(512, 1024), + ConvLayer(1024, 1024), + ConvLayer(1024, 1024), + ConvLayer(1024, 1024), + ConvLayer(1024, 1024), + ) + + self.SASA = SASA(in_dim=1024) + + self.Decoder = nn.Sequential( + ConvLayer(1024, 1024), + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True), + ConvLayer(1024, 512), + ConvLayer(512, 512), + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True), + ConvLayer(512, 256), + ConvLayer(256, 256), + ConvLayer(256, 128), + ConvLayer(128, 64), + ConvLayer(64, 32), + nn.Conv2d(32, 2, kernel_size=1, padding=0), + nn.Tanh(), + nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True), + ) + + dilation_ksize = 17 + self.dilation = torch.nn.MaxPool2d( + kernel_size=dilation_ksize, + stride=1, + padding=int((dilation_ksize - 1) / 2)) + + def warp(self, x, flow, mode='bilinear', padding_mode='zeros', coff=0.2): + n, c, h, w = x.size() + yv, xv = torch.meshgrid([torch.arange(h), torch.arange(w)]) + xv = xv.float() / (w - 1) * 2.0 - 1 + yv = yv.float() / (h - 1) * 2.0 - 1 + grid = torch.cat((xv.unsqueeze(-1), yv.unsqueeze(-1)), -1).unsqueeze(0) + grid = grid.to(flow.device) + grid_x = grid + 2 * flow * coff + warp_x = F.grid_sample(x, grid_x, mode=mode, padding_mode=padding_mode) + return warp_x + + def forward(self, img, skeleton_map, coef=0.2): + """extract self-attention features. + Args: + img : input numpy image + skeleton_map : skeleton map of input image + coef: warp degree + + Returns: + warp_x : warped image + flow: predicted flow + """ + + img_concat = torch.cat((img, skeleton_map), dim=1) + X = self.Encoder(img_concat) + + _, _, height, width = X.size() + + # directly get PAF magnitude from skeleton maps via dilation + PAF_mag = self.dilation((skeleton_map + 1.0) * 0.5) + + out, Y = self.SASA(X, PAF_mag) + flow = self.Decoder(out) + + flow = flow.permute(0, 2, 3, 1) # [n, 2, h, w] ==> [n, h, w, 2] + + warp_x = self.warp(img, flow, coff=coef) + warp_x = torch.clamp(warp_x, min=-1.0, max=1.0) + + return warp_x, flow diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/person_info.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/person_info.py new file mode 100644 index 0000000000000000000000000000000000000000..509a2ce3025d1ceb4f2ab66e5706e806313b4189 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/person_info.py @@ -0,0 +1,339 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy + +import cv2 +import numpy as np +import torch + +from .slim_utils import (enlarge_box_tblr, gen_skeleton_map, + get_map_fusion_map_cuda, get_mask_bbox, + resize_on_long_side) + + +class PersonInfo(object): + + def __init__(self, joints): + self.joints = joints + self.flow = None + self.pad_boder = False + self.height_expand = 0 + self.width_expand = 0 + self.coeff = 0.2 + self.network_input_W = 256 + self.network_input_H = 256 + self.divider = 20 + self.flow_scales = ['upper_2'] + + def update_attribute(self, pad_boder, height_expand, width_expand): + self.pad_boder = pad_boder + self.height_expand = height_expand + self.width_expand = width_expand + if pad_boder: + self.joints[:, 0] += width_expand + self.joints[:, 1] += height_expand + + def pred_flow(self, img, flow_net, device): + with torch.no_grad(): + if img is None: + print('image is none') + self.flow = None + + if len(img.shape) == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + if self.pad_boder: + height_expand = self.height_expand + width_expand = self.width_expand + pad_img = cv2.copyMakeBorder( + img, + height_expand, + height_expand, + width_expand, + width_expand, + cv2.BORDER_CONSTANT, + value=(127, 127, 127)) + + else: + height_expand = 0 + width_expand = 0 + pad_img = img.copy() + + canvas = np.zeros( + shape=(pad_img.shape[0], pad_img.shape[1]), dtype=np.float32) + + self.human_joint_box = self.__joint_to_body_box() + + self.human_box = enlarge_box_tblr( + self.human_joint_box, pad_img, ratio=0.25) + human_box_height = self.human_box[1] - self.human_box[0] + human_box_width = self.human_box[3] - self.human_box[2] + + self.leg_joint_box = self.__joint_to_leg_box() + self.leg_box = enlarge_box_tblr( + self.leg_joint_box, pad_img, ratio=0.25) + + self.arm_joint_box = self.__joint_to_arm_box() + self.arm_box = enlarge_box_tblr( + self.arm_joint_box, pad_img, ratio=0.1) + + x_flows = [] + y_flows = [] + multi_bbox = [] + + for scale in self.flow_scales: # better for metric + scale_value = float(scale.split('_')[-1]) + + arm_box = copy.deepcopy(self.arm_box) + + if arm_box[0] is None: + arm_box = self.human_box + + arm_box_height = arm_box[1] - arm_box[0] + arm_box_width = arm_box[3] - arm_box[2] + + roi_bbox = None + + if arm_box_width < human_box_width * 0.1 or arm_box_height < human_box_height * 0.1: + roi_bbox = self.human_box + else: + arm_box = enlarge_box_tblr( + arm_box, pad_img, ratio=scale_value) + if scale == 'upper_0.2': + arm_box[0] = min(arm_box[0], int(self.joints[0][1])) + if scale.startswith('upper'): + roi_bbox = [ + max(self.human_box[0], arm_box[0]), + min(self.human_box[1], arm_box[1]), + max(self.human_box[2], arm_box[2]), + min(self.human_box[3], arm_box[3]) + ] + if roi_bbox[1] - roi_bbox[0] < 1 or roi_bbox[ + 3] - roi_bbox[2] < 1: + continue + + elif scale.startswith('lower'): + roi_bbox = [ + max(self.human_box[0], self.leg_box[0]), + min(self.human_box[1], self.leg_box[1]), + max(self.human_box[2], self.leg_box[2]), + min(self.human_box[3], self.leg_box[3]) + ] + + if roi_bbox[1] - roi_bbox[0] < 1 or roi_bbox[ + 3] - roi_bbox[2] < 1: + continue + + skel_map, roi_bbox = gen_skeleton_map( + self.joints, 'depth', input_roi_box=roi_bbox) + + if roi_bbox is None: + continue + + if skel_map.dtype != np.float32: + skel_map = skel_map.astype(np.float32) + + skel_map -= 1.0 # [0,2] ->[-1,1] + + multi_bbox.append(roi_bbox) + + roi_bbox_height = roi_bbox[1] - roi_bbox[0] + roi_bbox_width = roi_bbox[3] - roi_bbox[2] + + assert skel_map.shape[0] == roi_bbox_height + assert skel_map.shape[1] == roi_bbox_width + roi_height_pad = roi_bbox_height // self.divider + roi_width_pad = roi_bbox_width // self.divider + paded_roi_h = roi_bbox_height + 2 * roi_height_pad + paded_roi_w = roi_bbox_width + 2 * roi_width_pad + + roi_height_pad_joint = skel_map.shape[0] // self.divider + roi_width_pad_joint = skel_map.shape[1] // self.divider + skel_map = np.pad( + skel_map, + ((roi_height_pad_joint, roi_height_pad_joint), + (roi_width_pad_joint, roi_width_pad_joint), (0, 0)), + 'constant', + constant_values=-1) + + skel_map_resized = cv2.resize( + skel_map, (self.network_input_W, self.network_input_H)) + + skel_map_resized[skel_map_resized < 0] = -1.0 + skel_map_resized[skel_map_resized > -0.5] = 1.0 + skel_map_transformed = torch.from_numpy( + skel_map_resized.transpose((2, 0, 1))) + + roi_npy = pad_img[roi_bbox[0]:roi_bbox[1], + roi_bbox[2]:roi_bbox[3], :].copy() + if roi_npy.dtype != np.float32: + roi_npy = roi_npy.astype(np.float32) + + roi_npy = np.pad(roi_npy, + ((roi_height_pad, roi_height_pad), + (roi_width_pad, roi_width_pad), (0, 0)), + 'edge') + + roi_npy = roi_npy[:, :, ::-1] + + roi_npy = cv2.resize( + roi_npy, (self.network_input_W, self.network_input_H)) + + roi_npy *= 1.0 / 255 + roi_npy -= 0.5 + roi_npy *= 2 + + rgb_tensor = torch.from_numpy(roi_npy.transpose((2, 0, 1))) + + rgb_tensor = rgb_tensor.unsqueeze(0).to(device) + skel_map_tensor = skel_map_transformed.unsqueeze(0).to(device) + warped_img_val, flow_field_val = flow_net( + rgb_tensor, skel_map_tensor + ) # inference, connectivity_mask [1,12,16,16] + flow_field_val = flow_field_val.detach().squeeze().cpu().numpy( + ) + + flow_field_val = cv2.resize( + flow_field_val, (paded_roi_w, paded_roi_h), + interpolation=cv2.INTER_LINEAR) + flow_field_val[..., 0] = flow_field_val[ + ..., 0] * paded_roi_w * 0.5 * 2 * self.coeff + flow_field_val[..., 1] = flow_field_val[ + ..., 1] * paded_roi_h * 0.5 * 2 * self.coeff + + # remove pad areas + flow_field_val = flow_field_val[ + roi_height_pad:flow_field_val.shape[0] - roi_height_pad, + roi_width_pad:flow_field_val.shape[1] - roi_width_pad, :] + + diffuse_width = max(roi_bbox_width // 3, 1) + diffuse_height = max(roi_bbox_height // 3, 1) + assert roi_bbox_width == flow_field_val.shape[1] + assert roi_bbox_height == flow_field_val.shape[0] + + origin_flow = np.zeros( + (pad_img.shape[0] + 2 * diffuse_height, + pad_img.shape[1] + 2 * diffuse_width, 2), + dtype=np.float32) + + flow_field_val = np.pad(flow_field_val, + ((diffuse_height, diffuse_height), + (diffuse_width, diffuse_width), + (0, 0)), 'linear_ramp') + + origin_flow[roi_bbox[0]:roi_bbox[1] + 2 * diffuse_height, + roi_bbox[2]:roi_bbox[3] + + 2 * diffuse_width] = flow_field_val + + origin_flow = origin_flow[diffuse_height:-diffuse_height, + diffuse_width:-diffuse_width, :] + + x_flows.append(origin_flow[..., 0]) + y_flows.append(origin_flow[..., 1]) + + if len(x_flows) == 0: + return { + 'rDx': np.zeros(canvas.shape[:2], dtype=np.float32), + 'rDy': np.zeros(canvas.shape[:2], dtype=np.float32), + 'multi_bbox': multi_bbox, + 'x_fusion_map': + np.ones(canvas.shape[:2], dtype=np.float32), + 'y_fusion_map': + np.ones(canvas.shape[:2], dtype=np.float32) + } + else: + origin_rDx, origin_rDy, x_fusion_map, y_fusion_map = self.blend_multiscale_flow( + x_flows, y_flows, device=device) + + return { + 'rDx': origin_rDx, + 'rDy': origin_rDy, + 'multi_bbox': multi_bbox, + 'x_fusion_map': x_fusion_map, + 'y_fusion_map': y_fusion_map + } + + @staticmethod + def blend_multiscale_flow(x_flows, y_flows, device=None): + scale_num = len(x_flows) + if scale_num == 1: + return x_flows[0], y_flows[0], np.ones_like( + x_flows[0]), np.ones_like(x_flows[0]) + + origin_rDx = np.zeros((x_flows[0].shape[0], x_flows[0].shape[1]), + dtype=np.float32) + origin_rDy = np.zeros((y_flows[0].shape[0], y_flows[0].shape[1]), + dtype=np.float32) + + x_fusion_map, x_acc_map = get_map_fusion_map_cuda( + x_flows, 1, device=device) + y_fusion_map, y_acc_map = get_map_fusion_map_cuda( + y_flows, 1, device=device) + + x_flow_map = 1.0 / x_fusion_map + y_flow_map = 1.0 / y_fusion_map + + all_acc_map = x_acc_map + y_acc_map + all_acc_map = all_acc_map.astype(np.uint8) + roi_box = get_mask_bbox(all_acc_map, threshold=1) + + if roi_box[0] is None or roi_box[1] - roi_box[0] <= 0 or roi_box[ + 3] - roi_box[2] <= 0: + roi_box = [0, x_flow_map.shape[0], 0, x_flow_map.shape[1]] + + roi_x_flow_map = x_flow_map[roi_box[0]:roi_box[1], + roi_box[2]:roi_box[3]] + roi_y_flow_map = y_flow_map[roi_box[0]:roi_box[1], + roi_box[2]:roi_box[3]] + + roi_width = roi_x_flow_map.shape[1] + roi_height = roi_x_flow_map.shape[0] + + roi_x_flow_map, scale = resize_on_long_side(roi_x_flow_map, 320) + roi_y_flow_map, scale = resize_on_long_side(roi_y_flow_map, 320) + + roi_x_flow_map = cv2.blur(roi_x_flow_map, (55, 55)) + roi_y_flow_map = cv2.blur(roi_y_flow_map, (55, 55)) + + roi_x_flow_map = cv2.resize(roi_x_flow_map, (roi_width, roi_height)) + roi_y_flow_map = cv2.resize(roi_y_flow_map, (roi_width, roi_height)) + + x_flow_map[roi_box[0]:roi_box[1], + roi_box[2]:roi_box[3]] = roi_x_flow_map + y_flow_map[roi_box[0]:roi_box[1], + roi_box[2]:roi_box[3]] = roi_y_flow_map + + for i in range(scale_num): + origin_rDx += x_flows[i] + origin_rDy += y_flows[i] + + origin_rDx *= x_flow_map + origin_rDy *= y_flow_map + + return origin_rDx, origin_rDy, x_flow_map, y_flow_map + + def __joint_to_body_box(self): + joint_left = int(np.min(self.joints, axis=0)[0]) + joint_right = int(np.max(self.joints, axis=0)[0]) + joint_top = int(np.min(self.joints, axis=0)[1]) + joint_bottom = int(np.max(self.joints, axis=0)[1]) + return [joint_top, joint_bottom, joint_left, joint_right] + + def __joint_to_leg_box(self): + leg_joints = self.joints[8:, :] + if np.max(leg_joints, axis=0)[2] < 0.05: + return [0, 0, 0, 0] + joint_left = int(np.min(leg_joints, axis=0)[0]) + joint_right = int(np.max(leg_joints, axis=0)[0]) + joint_top = int(np.min(leg_joints, axis=0)[1]) + joint_bottom = int(np.max(leg_joints, axis=0)[1]) + return [joint_top, joint_bottom, joint_left, joint_right] + + def __joint_to_arm_box(self): + arm_joints = self.joints[2:8, :] + if np.max(arm_joints, axis=0)[2] < 0.05: + return [0, 0, 0, 0] + joint_left = int(np.min(arm_joints, axis=0)[0]) + joint_right = int(np.max(arm_joints, axis=0)[0]) + joint_top = int(np.min(arm_joints, axis=0)[1]) + joint_bottom = int(np.max(arm_joints, axis=0)[1]) + return [joint_top, joint_bottom, joint_left, joint_right] diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/__init__.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/body.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/body.py new file mode 100644 index 0000000000000000000000000000000000000000..45b0272448f289cf1bd0d83f916a7e8826a4c3a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/body.py @@ -0,0 +1,272 @@ +# The implementation is based on openpose, available at https://github.com/Hzzone/pytorch-openpose. + +import math + +import cv2 +import numpy as np +import torch +from scipy.ndimage.filters import gaussian_filter + +from .model import BodyposeModel +from .util import pad_rightdown_corner, transfer + + +class Body(object): + + def __init__(self, model_path, device): + self.model = BodyposeModel().to(device) + model_dict = transfer(self.model, torch.load(model_path)) + self.model.load_state_dict(model_dict) + self.model.eval() + + def __call__(self, oriImg): + scale_search = [0.5] + boxsize = 368 + stride = 8 + padValue = 128 + thre1 = 0.1 + thre2 = 0.05 + bodyparts = 18 + multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search] + heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) + paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) + + for m in range(len(multiplier)): + scale = multiplier[m] + imageToTest = cv2.resize( + oriImg, (0, 0), + fx=scale, + fy=scale, + interpolation=cv2.INTER_CUBIC) + imageToTest_padded, pad = pad_rightdown_corner( + imageToTest, stride, padValue) + im = np.transpose( + np.float32(imageToTest_padded[:, :, :, np.newaxis]), + (3, 2, 0, 1)) / 256 - 0.5 + im = np.ascontiguousarray(im) + + data = torch.from_numpy(im).float() + if torch.cuda.is_available(): + data = data.cuda() + with torch.no_grad(): + Mconv7_stage6_L1, Mconv7_stage6_L2 = self.model(data) + Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy() + Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy() + + # extract outputs, resize, and remove padding + heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), + (1, 2, 0)) # output 1 is heatmaps + heatmap = cv2.resize( + heatmap, (0, 0), + fx=stride, + fy=stride, + interpolation=cv2.INTER_CUBIC) + heatmap = heatmap[:imageToTest_padded.shape[0] + - pad[2], :imageToTest_padded.shape[1] + - pad[3], :] + heatmap = cv2.resize( + heatmap, (oriImg.shape[1], oriImg.shape[0]), + interpolation=cv2.INTER_CUBIC) + + paf = np.transpose(np.squeeze(Mconv7_stage6_L1), + (1, 2, 0)) # output 0 is PAFs + paf = cv2.resize( + paf, (0, 0), + fx=stride, + fy=stride, + interpolation=cv2.INTER_CUBIC) + paf = paf[:imageToTest_padded.shape[0] + - pad[2], :imageToTest_padded.shape[1] - pad[3], :] + paf = cv2.resize( + paf, (oriImg.shape[1], oriImg.shape[0]), + interpolation=cv2.INTER_CUBIC) + + heatmap_avg += heatmap_avg + heatmap / len(multiplier) + paf_avg += +paf / len(multiplier) + + all_peaks = [] + peak_counter = 0 + + for part in range(bodyparts): + map_ori = heatmap_avg[:, :, part] + one_heatmap = gaussian_filter(map_ori, sigma=3) + + map_left = np.zeros(one_heatmap.shape) + map_left[1:, :] = one_heatmap[:-1, :] + map_right = np.zeros(one_heatmap.shape) + map_right[:-1, :] = one_heatmap[1:, :] + map_up = np.zeros(one_heatmap.shape) + map_up[:, 1:] = one_heatmap[:, :-1] + map_down = np.zeros(one_heatmap.shape) + map_down[:, :-1] = one_heatmap[:, 1:] + + peaks_binary = np.logical_and.reduce( + (one_heatmap >= map_left, one_heatmap >= map_right, + one_heatmap >= map_up, one_heatmap >= map_down, + one_heatmap > thre1)) + peaks = list( + zip(np.nonzero(peaks_binary)[1], + np.nonzero(peaks_binary)[0])) # note reverse + peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks] + peak_id = range(peak_counter, peak_counter + len(peaks)) + peaks_with_score_and_id = [ + peaks_with_score[i] + (peak_id[i], ) + for i in range(len(peak_id)) + ] + + all_peaks.append(peaks_with_score_and_id) + peak_counter += len(peaks) + + # find connection in the specified sequence, center 29 is in the position 15 + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], + [9, 10], [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], + [1, 15], [15, 17], [1, 16], [16, 18], [3, 17], [6, 18]] + # the middle joints heatmap correpondence + mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], + [19, 20], [21, 22], [23, 24], [25, 26], [27, 28], [29, 30], + [47, 48], [49, 50], [53, 54], [51, 52], [55, 56], [37, 38], + [45, 46]] + + connection_all = [] + special_k = [] + mid_num = 10 + + for k in range(len(mapIdx)): + score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] + candA = all_peaks[limbSeq[k][0] - 1] + candB = all_peaks[limbSeq[k][1] - 1] + nA = len(candA) + nB = len(candB) + if (nA != 0 and nB != 0): + connection_candidate = [] + for i in range(nA): + for j in range(nB): + vec = np.subtract(candB[j][:2], candA[i][:2]) + norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) + norm = max(0.001, norm) + vec = np.divide(vec, norm) + + startend = list( + zip( + np.linspace( + candA[i][0], candB[j][0], num=mid_num), + np.linspace( + candA[i][1], candB[j][1], num=mid_num))) + + vec_x = np.array([ + score_mid[int(round(startend[item][1])), + int(round(startend[item][0])), 0] + for item in range(len(startend)) + ]) + vec_y = np.array([ + score_mid[int(round(startend[item][1])), + int(round(startend[item][0])), 1] + for item in range(len(startend)) + ]) + + score_midpts = np.multiply( + vec_x, vec[0]) + np.multiply(vec_y, vec[1]) + temp1 = sum(score_midpts) / len(score_midpts) + temp2 = min(0.5 * oriImg.shape[0] / norm - 1, 0) + score_with_dist_prior = temp1 + temp2 + criterion1 = len(np.nonzero( + score_midpts > thre2)[0]) > 0.8 * len(score_midpts) + criterion2 = score_with_dist_prior > 0 + if criterion1 and criterion2: + connection_candidate.append([ + i, j, score_with_dist_prior, + score_with_dist_prior + candA[i][2] + + candB[j][2] + ]) + + connection_candidate = sorted( + connection_candidate, key=lambda x: x[2], reverse=True) + connection = np.zeros((0, 5)) + for c in range(len(connection_candidate)): + i, j, s = connection_candidate[c][0:3] + if (i not in connection[:, 3] + and j not in connection[:, 4]): + connection = np.vstack( + [connection, [candA[i][3], candB[j][3], s, i, j]]) + if (len(connection) >= min(nA, nB)): + break + + connection_all.append(connection) + else: + special_k.append(k) + connection_all.append([]) + + # last number in each row is the total parts number of that person + # the second last number in each row is the score of the overall configuration + subset = -1 * np.ones((0, 20)) + candidate = np.array( + [item for sublist in all_peaks for item in sublist]) + + for k in range(len(mapIdx)): + if k not in special_k: + partAs = connection_all[k][:, 0] + partBs = connection_all[k][:, 1] + indexA, indexB = np.array(limbSeq[k]) - 1 + + for i in range(len(connection_all[k])): # = 1:size(temp,1) + found = 0 + subset_idx = [-1, -1] + for j in range(len(subset)): # 1:size(subset,1): + if subset[j][indexA] == partAs[i] or subset[j][ + indexB] == partBs[i]: + subset_idx[found] = j + found += 1 + + if found == 1: + j = subset_idx[0] + if subset[j][indexB] != partBs[i]: + subset[j][indexB] = partBs[i] + subset[j][-1] += 1 + subset[j][-2] += candidate[ + partBs[i].astype(int), + 2] + connection_all[k][i][2] + elif found == 2: # if found 2 and disjoint, merge them + j1, j2 = subset_idx + tmp1 = (subset[j1] >= 0).astype(int) + tmp2 = (subset[j2] >= 0).astype(int) + membership = (tmp1 + tmp2)[:-2] + if len(np.nonzero(membership == 2)[0]) == 0: # merge + subset[j1][:-2] += (subset[j2][:-2] + 1) + subset[j1][-2:] += subset[j2][-2:] + subset[j1][-2] += connection_all[k][i][2] + subset = np.delete(subset, j2, 0) + else: # as like found == 1 + subset[j1][indexB] = partBs[i] + subset[j1][-1] += 1 + subset[j1][-2] += candidate[ + partBs[i].astype(int), + 2] + connection_all[k][i][2] + + # if find no partA in the subset, create a new subset + elif not found and k < 17: + row = -1 * np.ones(20) + row[indexA] = partAs[i] + row[indexB] = partBs[i] + row[-1] = 2 + row[-2] = sum( + candidate[connection_all[k][i, :2].astype(int), + 2]) + connection_all[k][i][2] + subset = np.vstack([subset, row]) + # delete some rows of subset which has few parts occur + deleteIdx = [] + for i in range(len(subset)): + if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: + deleteIdx.append(i) + subset = np.delete(subset, deleteIdx, axis=0) + + # subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts + # candidate: x, y, score, id + count = subset.shape[0] + joints = np.zeros(shape=(count, bodyparts, 3)) + + for i in range(count): + for j in range(bodyparts): + joints[i, j, :3] = candidate[int(subset[i, j]), :3] + confidence = 1.0 if subset[i, j] >= 0 else 0.0 + joints[i, j, 2] *= confidence + return joints diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/model.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/model.py new file mode 100644 index 0000000000000000000000000000000000000000..12f6e84db27dbbf719123e4523ba57e4b9ed58d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/model.py @@ -0,0 +1,141 @@ +# The implementation is based on openpose, available at https://github.com/Hzzone/pytorch-openpose. + +from collections import OrderedDict + +import torch +import torch.nn as nn + + +def make_layers(block, no_relu_layers): + layers = [] + for layer_name, v in block.items(): + if 'pool' in layer_name: + layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2]) + layers.append((layer_name, layer)) + else: + conv2d = nn.Conv2d( + in_channels=v[0], + out_channels=v[1], + kernel_size=v[2], + stride=v[3], + padding=v[4]) + layers.append((layer_name, conv2d)) + if layer_name not in no_relu_layers: + layers.append(('relu_' + layer_name, nn.ReLU(inplace=True))) + + return nn.Sequential(OrderedDict(layers)) + + +class BodyposeModel(nn.Module): + + def __init__(self): + super(BodyposeModel, self).__init__() + + # these layers have no relu layer + no_relu_layers = [ + 'conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1', + 'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2', + 'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1', + 'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1' + ] + blocks = {} + block0 = OrderedDict([('conv1_1', [3, 64, 3, 1, 1]), + ('conv1_2', [64, 64, 3, 1, 1]), + ('pool1_stage1', [2, 2, 0]), + ('conv2_1', [64, 128, 3, 1, 1]), + ('conv2_2', [128, 128, 3, 1, 1]), + ('pool2_stage1', [2, 2, 0]), + ('conv3_1', [128, 256, 3, 1, 1]), + ('conv3_2', [256, 256, 3, 1, 1]), + ('conv3_3', [256, 256, 3, 1, 1]), + ('conv3_4', [256, 256, 3, 1, 1]), + ('pool3_stage1', [2, 2, 0]), + ('conv4_1', [256, 512, 3, 1, 1]), + ('conv4_2', [512, 512, 3, 1, 1]), + ('conv4_3_CPM', [512, 256, 3, 1, 1]), + ('conv4_4_CPM', [256, 128, 3, 1, 1])]) + + # Stage 1 + block1_1 = OrderedDict([('conv5_1_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L1', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L1', [512, 38, 1, 1, 0])]) + + block1_2 = OrderedDict([('conv5_1_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L2', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L2', [512, 19, 1, 1, 0])]) + blocks['block1_1'] = block1_1 + blocks['block1_2'] = block1_2 + + self.model0 = make_layers(block0, no_relu_layers) + + # Stages 2 - 6 + for i in range(2, 7): + blocks['block%d_1' % i] = OrderedDict([ + ('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0]) + ]) + + blocks['block%d_2' % i] = OrderedDict([ + ('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0]) + ]) + + for k in blocks.keys(): + blocks[k] = make_layers(blocks[k], no_relu_layers) + + self.model1_1 = blocks['block1_1'] + self.model2_1 = blocks['block2_1'] + self.model3_1 = blocks['block3_1'] + self.model4_1 = blocks['block4_1'] + self.model5_1 = blocks['block5_1'] + self.model6_1 = blocks['block6_1'] + + self.model1_2 = blocks['block1_2'] + self.model2_2 = blocks['block2_2'] + self.model3_2 = blocks['block3_2'] + self.model4_2 = blocks['block4_2'] + self.model5_2 = blocks['block5_2'] + self.model6_2 = blocks['block6_2'] + + def forward(self, x): + + out1 = self.model0(x) + + out1_1 = self.model1_1(out1) + out1_2 = self.model1_2(out1) + out2 = torch.cat([out1_1, out1_2, out1], 1) + + out2_1 = self.model2_1(out2) + out2_2 = self.model2_2(out2) + out3 = torch.cat([out2_1, out2_2, out1], 1) + + out3_1 = self.model3_1(out3) + out3_2 = self.model3_2(out3) + out4 = torch.cat([out3_1, out3_2, out1], 1) + + out4_1 = self.model4_1(out4) + out4_2 = self.model4_2(out4) + out5 = torch.cat([out4_1, out4_2, out1], 1) + + out5_1 = self.model5_1(out5) + out5_2 = self.model5_2(out5) + out6 = torch.cat([out5_1, out5_2, out1], 1) + + out6_1 = self.model6_1(out6) + out6_2 = self.model6_2(out6) + + return out6_1, out6_2 diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/util.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/util.py new file mode 100644 index 0000000000000000000000000000000000000000..13a42074db4334700beb107c689ff899fd7c8eda --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/pose_estimator/util.py @@ -0,0 +1,33 @@ +# The implementation is based on openpose, available at https://github.com/Hzzone/pytorch-openpose. +import numpy as np + + +def pad_rightdown_corner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :] * 0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :] * 0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join( + weights_name.split('.')[1:])] + return transfered_model_weights diff --git a/runtime/Lib/site_packages/models/cv/image_body_reshaping/slim_utils.py b/runtime/Lib/site_packages/models/cv/image_body_reshaping/slim_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..23d5a741fcd359902e84a9b1389d1df797d28fff --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_body_reshaping/slim_utils.py @@ -0,0 +1,507 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os +import random + +import cv2 +import numba +import numpy as np +import torch + + +def resize_on_long_side(img, long_side=800): + src_height = img.shape[0] + src_width = img.shape[1] + + if src_height > src_width: + scale = long_side * 1.0 / src_height + _img = cv2.resize( + img, (int(src_width * scale), long_side), + interpolation=cv2.INTER_LINEAR) + else: + scale = long_side * 1.0 / src_width + _img = cv2.resize( + img, (long_side, int(src_height * scale)), + interpolation=cv2.INTER_LINEAR) + + return _img, scale + + +def point_in_box(pt, box): + pt_x = pt[0] + pt_y = pt[1] + + if pt_x >= box[0] and pt_x <= box[0] + box[2] and pt_y >= box[ + 1] and pt_y <= box[1] + box[3]: + return True + else: + return False + + +def enlarge_box_tblr(roi_bbox, mask, ratio=0.4, use_long_side=True): + if roi_bbox is None or None in roi_bbox: + return [None, None, None, None] + + top = roi_bbox[0] + bottom = roi_bbox[1] + left = roi_bbox[2] + right = roi_bbox[3] + + roi_width = roi_bbox[3] - roi_bbox[2] + roi_height = roi_bbox[1] - roi_bbox[0] + right = left + roi_width + bottom = top + roi_height + + long_side = roi_width if roi_width > roi_height else roi_height + + if use_long_side: + new_left = left - int(long_side * ratio) + else: + new_left = left - int(roi_width * ratio) + new_left = 1 if new_left < 0 else new_left + + if use_long_side: + new_top = top - int(long_side * ratio) + else: + new_top = top - int(roi_height * ratio) + new_top = 1 if new_top < 0 else new_top + + if use_long_side: + new_right = right + int(long_side * ratio) + else: + new_right = right + int(roi_width * ratio) + new_right = mask.shape[1] - 2 if new_right > mask.shape[1] else new_right + + if use_long_side: + new_bottom = bottom + int(long_side * ratio) + else: + new_bottom = bottom + int(roi_height * ratio) + new_bottom = mask.shape[0] - 2 if new_bottom > mask.shape[0] else new_bottom + + bbox = [new_top, new_bottom, new_left, new_right] + return bbox + + +def gen_PAF(image, joints): + + assert joints.shape[0] == 18 + assert joints.shape[1] == 3 + + org_h = image.shape[0] + org_w = image.shape[1] + small_image, resize_scale = resize_on_long_side(image, 120) + + joints[:, :2] = joints[:, :2] * resize_scale + + joint_left = int(np.min(joints, axis=0)[0]) + joint_right = int(np.max(joints, axis=0)[0]) + joint_top = int(np.min(joints, axis=0)[1]) + joint_bottom = int(np.max(joints, axis=0)[1]) + + limb_width = min( + abs(joint_right - joint_left), abs(joint_bottom - joint_top)) // 6 + + if limb_width % 2 == 0: + limb_width += 1 + kernel_size = limb_width + + part_orders = [(5, 11), (2, 8), (5, 6), (6, 7), (2, 3), (3, 4), (11, 12), + (12, 13), (8, 9), (9, 10)] + + map_list = [] + mask_list = [] + PAF_all = np.zeros( + shape=(small_image.shape[0], small_image.shape[1], 2), + dtype=np.float32) + for c, pair in enumerate(part_orders): + idx_a_name = pair[0] + idx_b_name = pair[1] + + jointa = joints[idx_a_name] + jointb = joints[idx_b_name] + + confidence_threshold = 0.05 + if jointa[2] > confidence_threshold and jointb[ + 2] > confidence_threshold: + canvas = np.zeros( + shape=(small_image.shape[0], small_image.shape[1]), + dtype=np.uint8) + + canvas = cv2.line(canvas, (int(jointa[0]), int(jointa[1])), + (int(jointb[0]), int(jointb[1])), + (255, 255, 255), 5) + + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, + (kernel_size, kernel_size)) + + canvas = cv2.dilate(canvas, kernel, 1) + canvas = cv2.GaussianBlur(canvas, (kernel_size, kernel_size), 0) + canvas = canvas.astype(np.float32) / 255 + PAF = np.zeros( + shape=(small_image.shape[0], small_image.shape[1], 2), + dtype=np.float32) + PAF[..., 0] = jointb[0] - jointa[0] + PAF[..., 1] = jointb[1] - jointa[1] + mag, ang = cv2.cartToPolar(PAF[..., 0], PAF[..., 1]) + PAF /= (np.dstack((mag, mag)) + 1e-5) + + single_PAF = PAF * np.dstack((canvas, canvas)) + map_list.append( + cv2.GaussianBlur(single_PAF, + (kernel_size * 3, kernel_size * 3), 0)) + + mask_list.append( + cv2.GaussianBlur(canvas.copy(), + (kernel_size * 3, kernel_size * 3), 0)) + PAF_all = PAF_all * (1.0 - np.dstack( + (canvas, canvas))) + single_PAF + + PAF_all = cv2.GaussianBlur(PAF_all, (kernel_size * 3, kernel_size * 3), 0) + PAF_all = cv2.resize( + PAF_all, (org_w, org_h), interpolation=cv2.INTER_LINEAR) + map_list.append(PAF_all) + return PAF_all, map_list, mask_list + + +def gen_skeleton_map(joints, stack_mode='column', input_roi_box=None): + if type(joints) == list: + joints = np.array(joints) + assert stack_mode == 'column' or stack_mode == 'depth' + + part_orders = [(2, 5), (5, 11), (2, 8), (8, 11), (5, 6), (6, 7), (2, 3), + (3, 4), (11, 12), (12, 13), (8, 9), (9, 10)] + + def link(img, a, b, color, line_width, scale=1.0, x_offset=0, y_offset=0): + jointa = joints[a] + jointb = joints[b] + + temp1 = int((jointa[0] - x_offset) * scale) + temp2 = int((jointa[1] - y_offset) * scale) + temp3 = int((jointb[0] - x_offset) * scale) + temp4 = int((jointb[1] - y_offset) * scale) + + cv2.line(img, (temp1, temp2), (temp3, temp4), color, line_width) + + roi_box = input_roi_box + + roi_box_width = roi_box[3] - roi_box[2] + roi_box_height = roi_box[1] - roi_box[0] + short_side_length = min(roi_box_width, roi_box_height) + line_width = short_side_length // 30 + + line_width = max(line_width, 2) + + map_cube = np.zeros( + shape=(roi_box_height, roi_box_width, len(part_orders) + 1), + dtype=np.float32) + + use_line_width = min(5, line_width) + fx = use_line_width * 1.0 / line_width # fx 最大值为1 + + if fx < 0.99: + map_cube = cv2.resize(map_cube, (0, 0), fx=fx, fy=fx) + + for c, pair in enumerate(part_orders): + tmp = map_cube[..., c].copy() + link( + tmp, + pair[0], + pair[1], (2.0, 2.0, 2.0), + use_line_width, + scale=fx, + x_offset=roi_box[2], + y_offset=roi_box[0]) + map_cube[..., c] = tmp + + tmp = map_cube[..., -1].copy() + link( + tmp, + pair[0], + pair[1], (2.0, 2.0, 2.0), + use_line_width, + scale=fx, + x_offset=roi_box[2], + y_offset=roi_box[0]) + map_cube[..., -1] = tmp + + map_cube = cv2.resize(map_cube, (roi_box_width, roi_box_height)) + + if stack_mode == 'depth': + return map_cube, roi_box + elif stack_mode == 'column': + joint_maps = [] + for c in range(len(part_orders) + 1): + joint_maps.append(map_cube[..., c]) + joint_map = np.column_stack(joint_maps) + + return joint_map, roi_box + + +def plot_one_box(x, img, color=None, label=None, line_thickness=None): + tl = line_thickness or round( + 0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness + color = color or [random.randint(0, 255) for _ in range(3)] + c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3])) + cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA) + if label: + tf = max(tl - 1, 1) # font thickness + t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] + c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3 + cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled + cv2.putText( + img, + label, (c1[0], c1[1] - 2), + 0, + tl / 3, [225, 255, 255], + thickness=tf, + lineType=cv2.LINE_AA) + + +def draw_line(im, points, color, stroke_size=2, closed=False): + points = points.astype(np.int32) + for i in range(len(points) - 1): + cv2.line(im, tuple(points[i]), tuple(points[i + 1]), color, + stroke_size) + if closed: + cv2.line(im, tuple(points[0]), tuple(points[-1]), color, stroke_size) + + +def enlarged_bbox(bbox, img_width, img_height, enlarge_ratio=0.2): + left = bbox[0] + top = bbox[1] + + right = bbox[2] + bottom = bbox[3] + + roi_width = right - left + roi_height = bottom - top + + new_left = left - int(roi_width * enlarge_ratio) + new_left = 0 if new_left < 0 else new_left + + new_top = top - int(roi_height * enlarge_ratio) + new_top = 0 if new_top < 0 else new_top + + new_right = right + int(roi_width * enlarge_ratio) + new_right = img_width if new_right > img_width else new_right + + new_bottom = bottom + int(roi_height * enlarge_ratio) + new_bottom = img_height if new_bottom > img_height else new_bottom + + bbox = [new_left, new_top, new_right, new_bottom] + + bbox = [int(x) for x in bbox] + + return bbox + + +def get_map_fusion_map_cuda(map_list, threshold=1, device=torch.device('cpu')): + map_list_cuda = [torch.from_numpy(x).to(device) for x in map_list] + map_concat = torch.stack(tuple(map_list_cuda), dim=-1) + + map_concat = torch.abs(map_concat) + + map_concat[map_concat < threshold] = 0 + map_concat[map_concat > 1e-5] = 1.0 + + sum_map = torch.sum(map_concat, dim=2) + a = torch.ones_like(sum_map) + acc_map = torch.where(sum_map > 0, a * 2.0, torch.zeros_like(sum_map)) + + fusion_map = torch.where(sum_map < 0.5, a * 1.5, sum_map) + + fusion_map = fusion_map.float() + acc_map = acc_map.float() + + fusion_map = fusion_map.cpu().numpy().astype(np.float32) + acc_map = acc_map.cpu().numpy().astype(np.float32) + + return fusion_map, acc_map + + +def gen_border_shade(height, width, height_band, width_band): + height_ratio = height_band * 1.0 / height + width_ratio = width_band * 1.0 / width + + _height_band = int(256 * height_ratio) + _width_band = int(256 * width_ratio) + + canvas = np.zeros((256, 256), dtype=np.float32) + + canvas[_height_band // 2:-_height_band // 2, + _width_band // 2:-_width_band // 2] = 1.0 + + canvas = cv2.blur(canvas, (_height_band, _width_band)) + + canvas = cv2.resize(canvas, (width, height)) + + return canvas + + +def get_mask_bbox(mask, threshold=127): + ret, mask = cv2.threshold(mask, threshold, 1, 0) + + if cv2.countNonZero(mask) == 0: + return [None, None, None, None] + + col_acc = np.sum(mask, 0) + row_acc = np.sum(mask, 1) + + col_acc = col_acc.tolist() + row_acc = row_acc.tolist() + + for x in range(len(col_acc)): + if col_acc[x] > 0: + left = x + break + + for x in range(1, len(col_acc)): + if col_acc[-x] > 0: + right = len(col_acc) - x + break + + for x in range(len(row_acc)): + if row_acc[x] > 0: + top = x + break + + for x in range(1, len(row_acc)): + if row_acc[-x] > 0: + bottom = len(row_acc[::-1]) - x + break + return [top, bottom, left, right] + + +def visualize_flow(flow): + h, w = flow.shape[:2] + hsv = np.zeros((h, w, 3), np.uint8) + mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) + + hsv[..., 0] = ang * 180 / np.pi / 2 + hsv[..., 1] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) + hsv[..., 2] = 255 + bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) + bgr = bgr * 1.0 / 255 + return bgr.astype(np.float32) + + +def vis_joints(image, joints, color, show_text=True, confidence_threshold=0.1): + + part_orders = [(2, 5), (5, 11), (2, 8), (8, 11), (5, 6), (6, 7), (2, 3), + (3, 4), (11, 12), (12, 13), (8, 9), (9, 10)] + + abandon_idxs = [0, 1, 14, 15, 16, 17] + # draw joints + for i, joint in enumerate(joints): + if i in abandon_idxs: + continue + if joint[-1] > confidence_threshold: + + cv2.circle(image, (int(joint[0]), int(joint[1])), 1, color, 2) + if show_text: + cv2.putText(image, + str(i) + '[{:.2f}]'.format(joint[-1]), + (int(joint[0]), int(joint[1])), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + # draw link + for pair in part_orders: + if joints[pair[0]][-1] > confidence_threshold and joints[ + pair[1]][-1] > confidence_threshold: + cv2.line(image, (int(joints[pair[0]][0]), int(joints[pair[0]][1])), + (int(joints[pair[1]][0]), int(joints[pair[1]][1])), color, + 2) + return image + + +def get_heatmap_cv(img, magn, max_flow_mag): + min_flow_mag = .5 + cv_magn = np.clip( + 255 * (magn - min_flow_mag) / (max_flow_mag - min_flow_mag + 1e-7), + a_min=0, + a_max=255).astype(np.uint8) + if img.dtype != np.uint8: + img = (255 * img).astype(np.uint8) + + heatmap_img = cv2.applyColorMap(cv_magn, cv2.COLORMAP_JET) + heatmap_img = heatmap_img[..., ::-1] + + h, w = magn.shape + img_alpha = np.ones((h, w), dtype=np.double)[:, :, None] + heatmap_alpha = np.clip( + magn / (max_flow_mag + 1e-7), a_min=1e-7, a_max=1)[:, :, None]**.7 + heatmap_alpha[heatmap_alpha < .2]**.5 + pm_hm = heatmap_img * heatmap_alpha + pm_img = img * img_alpha + cv_out = pm_hm + pm_img * (1 - heatmap_alpha) + cv_out = np.clip(cv_out, a_min=0, a_max=255).astype(np.uint8) + + return cv_out + + +def save_heatmap_cv(img, flow, supression=2): + + flow_magn = np.sqrt(flow[:, :, 0]**2 + flow[:, :, 1]**2) + flow_magn -= supression + flow_magn[flow_magn <= 0] = 0 + cv_out = get_heatmap_cv(img, flow_magn, np.max(flow_magn) * 1.3) + return cv_out + + +@numba.jit(nopython=True, parallel=False) +def bilinear_interp(x, y, v11, v12, v21, v22): + temp1 = (v11 * (1 - y) + v12 * y) * (1 - x) + temp2 = (v21 * (1 - y) + v22 * y) * x + result = temp1 + temp2 + return result + + +@numba.jit(nopython=True, parallel=False) +def image_warp_grid1(rDx, rDy, oriImg, transRatio, width_expand, + height_expand): + srcW = oriImg.shape[1] + srcH = oriImg.shape[0] + + newImg = oriImg.copy() + + for i in range(srcH): + for j in range(srcW): + _i = i + _j = j + + deltaX = rDx[_i, _j] + deltaY = rDy[_i, _j] + + nx = _j + deltaX * transRatio + ny = _i + deltaY * transRatio + + if nx >= srcW - width_expand - 1: + if nx > srcW - 1: + nx = srcW - 1 + + if ny >= srcH - height_expand - 1: + if ny > srcH - 1: + ny = srcH - 1 + + if nx < width_expand: + if nx < 0: + nx = 0 + + if ny < height_expand: + if ny < 0: + ny = 0 + + nxi = int(math.floor(nx)) + nyi = int(math.floor(ny)) + nxi1 = int(math.ceil(nx)) + nyi1 = int(math.ceil(ny)) + + for ll in range(3): + newImg[_i, _j, + ll] = bilinear_interp(ny - nyi, nx - nxi, + oriImg[nyi, nxi, + ll], oriImg[nyi, nxi1, ll], + oriImg[nyi1, nxi, + ll], oriImg[nyi1, nxi1, + ll]) + return newImg diff --git a/runtime/Lib/site_packages/models/cv/image_classification/__init__.py b/runtime/Lib/site_packages/models/cv/image_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf14863f099298c443e822fd6ce8753ae03abb5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mmcls_model import ClassificationModel + from .resnet50_cc import ContentCheckBackbone + +else: + _import_structure = { + 'mmcls_model': ['ClassificationModel'], + 'resnet50_cc': ['ContentCheckBackbone'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_classification/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/image_classification/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b25ff4c9bbe3928f6d610ab9561471f38fc712e8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/backbones/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .beit_v2 import BEiTv2 +from .nextvit import NextViT diff --git a/runtime/Lib/site_packages/models/cv/image_classification/backbones/beit_v2.py b/runtime/Lib/site_packages/models/cv/image_classification/backbones/beit_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..eda117279c1caf3cbeecbd75301de3b68d65cb78 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/backbones/beit_v2.py @@ -0,0 +1,529 @@ +# Part of the implementation is borrowed and modified from beit2, +# publicly available at https://github.com/microsoft/unilm/tree/master/beit2 +import collections.abc +import itertools +import math +import os +import warnings +from functools import partial +from typing import Dict, Sequence + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from mmcls.models.backbones.base_backbone import BaseBackbone +from mmcls.models.builder import BACKBONES +from mmcv.cnn.bricks import DropPath, build_activation_layer, build_norm_layer +from mmcv.runner import BaseModule +from torch.nn.modules.batchnorm import _BatchNorm + +from ..utils import to_2tuple, trunc_normal_ + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + # x = self.drop(x) + # commit this for the orignal BERT implement + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + window_size=None, + attn_head_dim=None): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False) + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.v_bias = None + + if window_size: + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] + - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter( + torch.zeros(self.num_relative_distance, + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, + coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = \ + torch.zeros(size=(window_size[0] * window_size[1] + 1, ) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, 1:] = relative_coords.sum( + -1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + + self.register_buffer('relative_position_index', + relative_position_index) + else: + self.window_size = None + self.relative_position_bias_table = None + self.relative_position_index = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, + x, + rel_pos_bias=None, + return_attention=False, + return_qkv=False): + B, N, C = x.shape + qkv_bias = None + if self.q_bias is not None: + qkv_bias = torch.cat( + (self.q_bias, + torch.zeros_like(self.v_bias, + requires_grad=False), self.v_bias)) + # qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) (B, H, N, C) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + if self.relative_position_bias_table is not None: + relative_position_bias = \ + self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1] + 1, + self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if rel_pos_bias is not None: + attn = attn + rel_pos_bias + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + if return_attention: + return attn + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + if return_qkv: + return x, qkv + + return x + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + init_values=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + window_size=None, + attn_head_dim=None): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + window_size=window_size, + attn_head_dim=attn_head_dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + if init_values > 0: + self.gamma_1 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + self.gamma_2 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + else: + self.gamma_1, self.gamma_2 = None, None + + def forward(self, + x, + rel_pos_bias=None, + return_attention=False, + return_qkv=False): + if return_attention: + return self.attn( + self.norm1(x), + rel_pos_bias=rel_pos_bias, + return_attention=True) + if return_qkv: + y, qkv = self.attn( + self.norm1(x), + rel_pos_bias=rel_pos_bias, + return_qkv=return_qkv) + x = x + self.drop_path(self.gamma_1 * y) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x, qkv + + if self.gamma_1 is None: + x = x + self.drop_path( + self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + x = x + self.drop_path(self.gamma_1 * self.attn( + self.norm1(x), rel_pos_bias=rel_pos_bias)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * ( + img_size[0] // patch_size[0]) + self.patch_shape = (img_size[0] // patch_size[0], + img_size[1] // patch_size[1]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x, **kwargs): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + # assert H == self.img_size[0] and W == self.img_size[1], \ + # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x).flatten(2).transpose(1, 2) + return x + + +class RelativePositionBias(nn.Module): + + def __init__(self, window_size, num_heads): + super().__init__() + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] + - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter( + torch.zeros(self.num_relative_distance, + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = \ + torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, + 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + + self.register_buffer('relative_position_index', + relative_position_index) + + # trunc_normal_(self.relative_position_bias_table, std=.02) + + def forward(self): + relative_position_bias = \ + self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1] + 1, + self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + return relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + + +@BACKBONES.register_module() +class BEiTv2(BaseBackbone): + embed_dims = {'base': 768, 'large': 1024, 'huge': 1280, 'giant': 1408} + depths = {'base': 12, 'large': 24, 'huge': 32, 'giant': 40} + num_heads = {'base': 12, 'large': 16, 'huge': 16, 'giant': 16} + mlp_ratios = {'base': 4, 'large': 4, 'huge': 4, 'giant': 6144 / 1408} + """ Vision Transformer with support for patch or hybrid CNN input stage + """ + + def __init__(self, + arch='base', + patch_size=16, + img_size=224, + in_chans=3, + qkv_bias=False, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6), + init_values=None, + use_abs_pos_emb=True, + use_rel_pos_bias=False, + use_shared_rel_pos_bias=False, + use_mean_pooling=True, + init_scale=0.001, + out_indices=-1, + frozen_stages=-1, + init_cfg=None): + super().__init__(init_cfg=init_cfg) + embed_dim = self.embed_dims[arch] + depth = self.depths[arch] + num_heads = self.num_heads[arch] + mlp_ratio = self.mlp_ratios[arch] + + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + # self.mask_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + if use_abs_pos_emb: + self.pos_embed = nn.Parameter( + torch.zeros(1, num_patches + 1, embed_dim)) + else: + self.pos_embed = None + self.pos_drop = nn.Dropout(p=drop_rate) + + if use_shared_rel_pos_bias: + self.rel_pos_bias = RelativePositionBias( + window_size=self.patch_embed.patch_shape, num_heads=num_heads) + else: + self.rel_pos_bias = None + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.use_rel_pos_bias = use_rel_pos_bias + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + init_values=init_values, + window_size=self.patch_embed.patch_shape + if use_rel_pos_bias else None) for i in range(depth) + ]) + self.norm = nn.Identity() if use_mean_pooling else norm_layer( + embed_dim) + self.fc_norm = norm_layer(embed_dim) if use_mean_pooling else None + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=.02) + trunc_normal_(self.cls_token, std=.02) + + def init_weights(self): + super(BEiTv2, self).init_weights() + if (isinstance(self.init_cfg, dict) + and self.init_cfg['type'] == 'Pretrained'): + # Suppress default init if use pretrained model. + return + + self.apply(self._init_weights) + self.fix_init_weight() + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def fix_init_weight(self): + + def rescale(param, layer_id): + param.div_(math.sqrt(2.0 * layer_id)) + + for layer_id, layer in enumerate(self.blocks): + rescale(layer.attn.proj.weight.data, layer_id + 1) + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + + def interpolate_pos_encoding(self, x, w, h): + npatch = x.shape[1] - 1 + N = self.pos_embed.shape[1] - 1 + if npatch == N and w == h: + return self.pos_embed + class_pos_embed = self.pos_embed[:, 0] + patch_pos_embed = self.pos_embed[:, 1:] + dim = x.shape[-1] + w0 = w // self.patch_embed.patch_size[0] + h0 = h // self.patch_embed.patch_size[0] + # we add a small number to avoid floating point error in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + w0, h0 = w0 + 0.1, h0 + 0.1 + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), + dim).permute(0, 3, 1, 2), + scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)), + mode='bicubic', + ) + assert int(w0) == patch_pos_embed.shape[-2] and int( + h0) == patch_pos_embed.shape[-1] + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), + dim=1) + + def forward_features(self, + x, + return_patch_tokens=False, + return_all_tokens=False, + **kwargs): + B, nc, w, h = x.shape + x = self.patch_embed(x) + batch_size, seq_len, _ = x.size() + + cls_tokens = self.cls_token.expand( + batch_size, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + + x = torch.cat((cls_tokens, x), dim=1) + if self.pos_embed is not None: + if x.shape[1] != self.pos_embed.shape[1]: + x = x + self.interpolate_pos_encoding(x, w, h) + else: + x = x + self.pos_embed + + x = self.pos_drop(x) + + rel_pos_bias = self.rel_pos_bias( + ) if self.rel_pos_bias is not None else None + for blk in self.blocks: + x = blk(x, rel_pos_bias=rel_pos_bias) + + x = self.norm(x) + if self.fc_norm is not None: + if return_all_tokens: + return self.fc_norm(x) + t = x[:, 1:, :] + if return_patch_tokens: + return self.fc_norm(t) + else: + return self.fc_norm(t.mean(1)) + else: + if return_all_tokens: + return x + elif return_patch_tokens: + return x[:, 1:] + else: + return x[:, 0] + + def forward(self, + x, + return_patch_tokens=False, + return_all_tokens=False, + **kwargs): + x = self.forward_features( + x, + return_patch_tokens=return_patch_tokens, + return_all_tokens=return_all_tokens, + **kwargs) + return tuple([x]) + + def _freeze_stages(self): + if self.frozen_stages > 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + for idx, layer in enumerate(self.blocks): + if idx <= self.frozen_stages - 1: + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(BEiTv2, self).train(mode) + self._freeze_stages() diff --git a/runtime/Lib/site_packages/models/cv/image_classification/backbones/nextvit.py b/runtime/Lib/site_packages/models/cv/image_classification/backbones/nextvit.py new file mode 100644 index 0000000000000000000000000000000000000000..0b69f40e9590c4c96df955c748786f52c0816927 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/backbones/nextvit.py @@ -0,0 +1,502 @@ +# Part of the implementation is borrowed and modified from Next-ViT, +# publicly available at https://github.com/bytedance/Next-ViT +import collections.abc +import itertools +import math +import os +import warnings +from functools import partial +from typing import Dict, Sequence + +import torch +import torch.nn as nn +from einops import rearrange +from mmcls.models.backbones.base_backbone import BaseBackbone +from mmcls.models.builder import BACKBONES +from mmcv.cnn.bricks import DropPath, build_activation_layer, build_norm_layer +from mmcv.runner import BaseModule +from torch.nn.modules.batchnorm import _BatchNorm + +from ..utils import trunc_normal_ + +NORM_EPS = 1e-5 + + +class ConvBNReLU(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + groups=1): + super(ConvBNReLU, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=1, + groups=groups, + bias=False) + self.norm = nn.BatchNorm2d(out_channels, eps=NORM_EPS) + self.act = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.norm(x) + x = self.act(x) + return x + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class PatchEmbed(nn.Module): + + def __init__(self, in_channels, out_channels, stride=1): + super(PatchEmbed, self).__init__() + norm_layer = partial(nn.BatchNorm2d, eps=NORM_EPS) + if stride == 2: + self.avgpool = nn.AvgPool2d((2, 2), + stride=2, + ceil_mode=True, + count_include_pad=False) + self.conv = nn.Conv2d( + in_channels, out_channels, kernel_size=1, stride=1, bias=False) + self.norm = norm_layer(out_channels) + elif in_channels != out_channels: + self.avgpool = nn.Identity() + self.conv = nn.Conv2d( + in_channels, out_channels, kernel_size=1, stride=1, bias=False) + self.norm = norm_layer(out_channels) + else: + self.avgpool = nn.Identity() + self.conv = nn.Identity() + self.norm = nn.Identity() + + def forward(self, x): + return self.norm(self.conv(self.avgpool(x))) + + +class MHCA(nn.Module): + """ + Multi-Head Convolutional Attention + """ + + def __init__(self, out_channels, head_dim): + super(MHCA, self).__init__() + norm_layer = partial(nn.BatchNorm2d, eps=NORM_EPS) + self.group_conv3x3 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + groups=out_channels // head_dim, + bias=False) + self.norm = norm_layer(out_channels) + self.act = nn.ReLU(inplace=True) + self.projection = nn.Conv2d( + out_channels, out_channels, kernel_size=1, bias=False) + + def forward(self, x): + out = self.group_conv3x3(x) + out = self.norm(out) + out = self.act(out) + out = self.projection(out) + return out + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + out_features=None, + mlp_ratio=None, + drop=0., + bias=True): + super().__init__() + out_features = out_features or in_features + hidden_dim = _make_divisible(in_features * mlp_ratio, 32) + self.conv1 = nn.Conv2d( + in_features, hidden_dim, kernel_size=1, bias=bias) + self.act = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d( + hidden_dim, out_features, kernel_size=1, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.conv1(x) + x = self.act(x) + x = self.drop(x) + x = self.conv2(x) + x = self.drop(x) + return x + + +class NCB(nn.Module): + """ + Next Convolution Block + """ + + def __init__(self, + in_channels, + out_channels, + stride=1, + path_dropout=0, + drop=0, + head_dim=32, + mlp_ratio=3): + super(NCB, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + norm_layer = partial(nn.BatchNorm2d, eps=NORM_EPS) + assert out_channels % head_dim == 0 + + self.patch_embed = PatchEmbed(in_channels, out_channels, stride) + self.mhca = MHCA(out_channels, head_dim) + self.attention_path_dropout = DropPath(path_dropout) + + self.norm = norm_layer(out_channels) + self.mlp = Mlp(out_channels, mlp_ratio=mlp_ratio, drop=drop, bias=True) + self.mlp_path_dropout = DropPath(path_dropout) + self.is_bn_merged = False + + def forward(self, x): + x = self.patch_embed(x) + x = x + self.attention_path_dropout(self.mhca(x)) + if not torch.onnx.is_in_onnx_export() and not self.is_bn_merged: + out = self.norm(x) + else: + out = x + x = x + self.mlp_path_dropout(self.mlp(out)) + return x + + +class E_MHSA(nn.Module): + """ + Efficient Multi-Head Self Attention + """ + + def __init__(self, + dim, + out_dim=None, + head_dim=32, + qkv_bias=True, + qk_scale=None, + attn_drop=0, + proj_drop=0., + sr_ratio=1): + super().__init__() + self.dim = dim + self.out_dim = out_dim if out_dim is not None else dim + self.num_heads = self.dim // head_dim + self.scale = qk_scale or head_dim**-0.5 + self.q = nn.Linear(dim, self.dim, bias=qkv_bias) + self.k = nn.Linear(dim, self.dim, bias=qkv_bias) + self.v = nn.Linear(dim, self.dim, bias=qkv_bias) + self.proj = nn.Linear(self.dim, self.out_dim) + self.attn_drop = nn.Dropout(attn_drop) + self.proj_drop = nn.Dropout(proj_drop) + + self.sr_ratio = sr_ratio + self.N_ratio = sr_ratio**2 + if sr_ratio > 1: + self.sr = nn.AvgPool1d( + kernel_size=self.N_ratio, stride=self.N_ratio) + self.norm = nn.BatchNorm1d(dim, eps=NORM_EPS) + self.is_bn_merge = False + + def forward(self, x): + B, N, C = x.shape + q = self.q(x) + q = q.reshape(B, N, self.num_heads, + int(C // self.num_heads)).permute(0, 2, 1, 3) + + if self.sr_ratio > 1: + x_ = x.transpose(1, 2) + x_ = self.sr(x_) + if not torch.onnx.is_in_onnx_export() and not self.is_bn_merge: + x_ = self.norm(x_) + x_ = x_.transpose(1, 2) + k = self.k(x_) + k = k.reshape(B, -1, self.num_heads, + int(C // self.num_heads)).permute(0, 2, 3, 1) + v = self.v(x_) + v = v.reshape(B, -1, self.num_heads, + int(C // self.num_heads)).permute(0, 2, 1, 3) + else: + k = self.k(x) + k = k.reshape(B, -1, self.num_heads, + int(C // self.num_heads)).permute(0, 2, 3, 1) + v = self.v(x) + v = v.reshape(B, -1, self.num_heads, + int(C // self.num_heads)).permute(0, 2, 1, 3) + attn = (q @ k) * self.scale + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class NTB(nn.Module): + """ + Next Transformer Block + """ + + def __init__( + self, + in_channels, + out_channels, + path_dropout, + stride=1, + sr_ratio=1, + mlp_ratio=2, + head_dim=32, + mix_block_ratio=0.75, + attn_drop=0, + drop=0, + ): + super(NTB, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_block_ratio = mix_block_ratio + norm_func = partial(nn.BatchNorm2d, eps=NORM_EPS) + + self.mhsa_out_channels = _make_divisible( + int(out_channels * mix_block_ratio), 32) + self.mhca_out_channels = out_channels - self.mhsa_out_channels + + self.patch_embed = PatchEmbed(in_channels, self.mhsa_out_channels, + stride) + self.norm1 = norm_func(self.mhsa_out_channels) + self.e_mhsa = E_MHSA( + self.mhsa_out_channels, + head_dim=head_dim, + sr_ratio=sr_ratio, + attn_drop=attn_drop, + proj_drop=drop) + self.mhsa_path_dropout = DropPath(path_dropout * mix_block_ratio) + + self.projection = PatchEmbed( + self.mhsa_out_channels, self.mhca_out_channels, stride=1) + self.mhca = MHCA(self.mhca_out_channels, head_dim=head_dim) + self.mhca_path_dropout = DropPath(path_dropout * (1 - mix_block_ratio)) + + self.norm2 = norm_func(out_channels) + self.mlp = Mlp(out_channels, mlp_ratio=mlp_ratio, drop=drop) + self.mlp_path_dropout = DropPath(path_dropout) + + self.is_bn_merged = False + + def forward(self, x): + x = self.patch_embed(x) + B, C, H, W = x.shape + if not torch.onnx.is_in_onnx_export() and not self.is_bn_merged: + out = self.norm1(x) + else: + out = x + out = rearrange(out, 'b c h w -> b (h w) c') # b n c + out = self.mhsa_path_dropout(self.e_mhsa(out)) + x = x + rearrange(out, 'b (h w) c -> b c h w', h=H) + + out = self.projection(x) + out = out + self.mhca_path_dropout(self.mhca(out)) + x = torch.cat([x, out], dim=1) + + if not torch.onnx.is_in_onnx_export() and not self.is_bn_merged: + out = self.norm2(x) + else: + out = x + x = x + self.mlp_path_dropout(self.mlp(out)) + return x + + +@BACKBONES.register_module() +class NextViT(BaseBackbone): + stem_chs = { + 'x_small': [64, 32, 64], + 'small': [64, 32, 64], + 'base': [64, 32, 64], + 'large': [64, 32, 64], + } + depths = { + 'x_small': [1, 1, 5, 1], + 'small': [3, 4, 10, 3], + 'base': [3, 4, 20, 3], + 'large': [3, 4, 30, 3], + } + + def __init__(self, + arch='small', + path_dropout=0.2, + attn_drop=0, + drop=0, + strides=[1, 2, 2, 2], + sr_ratios=[8, 4, 2, 1], + head_dim=32, + mix_block_ratio=0.75, + resume='', + with_extra_norm=True, + norm_eval=False, + norm_cfg=None, + out_indices=-1, + frozen_stages=-1, + init_cfg=None): + super().__init__(init_cfg=init_cfg) + + stem_chs = self.stem_chs[arch] + depths = self.depths[arch] + + self.frozen_stages = frozen_stages + self.with_extra_norm = with_extra_norm + self.norm_eval = norm_eval + self.stage1_out_channels = [96] * (depths[0]) + self.stage2_out_channels = [192] * (depths[1] - 1) + [256] + self.stage3_out_channels = [384, 384, 384, 384, 512] * (depths[2] // 5) + self.stage4_out_channels = [768] * (depths[3] - 1) + [1024] + self.stage_out_channels = [ + self.stage1_out_channels, self.stage2_out_channels, + self.stage3_out_channels, self.stage4_out_channels + ] + + # Next Hybrid Strategy + self.stage1_block_types = [NCB] * depths[0] + self.stage2_block_types = [NCB] * (depths[1] - 1) + [NTB] + self.stage3_block_types = [NCB, NCB, NCB, NCB, NTB] * (depths[2] // 5) + self.stage4_block_types = [NCB] * (depths[3] - 1) + [NTB] + self.stage_block_types = [ + self.stage1_block_types, self.stage2_block_types, + self.stage3_block_types, self.stage4_block_types + ] + + self.stem = nn.Sequential( + ConvBNReLU(3, stem_chs[0], kernel_size=3, stride=2), + ConvBNReLU(stem_chs[0], stem_chs[1], kernel_size=3, stride=1), + ConvBNReLU(stem_chs[1], stem_chs[2], kernel_size=3, stride=1), + ConvBNReLU(stem_chs[2], stem_chs[2], kernel_size=3, stride=2), + ) + input_channel = stem_chs[-1] + features = [] + idx = 0 + dpr = [x.item() for x in torch.linspace(0, path_dropout, sum(depths)) + ] # stochastic depth decay rule + for stage_id in range(len(depths)): + numrepeat = depths[stage_id] + output_channels = self.stage_out_channels[stage_id] + block_types = self.stage_block_types[stage_id] + for block_id in range(numrepeat): + if strides[stage_id] == 2 and block_id == 0: + stride = 2 + else: + stride = 1 + output_channel = output_channels[block_id] + block_type = block_types[block_id] + if block_type is NCB: + layer = NCB( + input_channel, + output_channel, + stride=stride, + path_dropout=dpr[idx + block_id], + drop=drop, + head_dim=head_dim) + features.append(layer) + elif block_type is NTB: + layer = NTB( + input_channel, + output_channel, + path_dropout=dpr[idx + block_id], + stride=stride, + sr_ratio=sr_ratios[stage_id], + head_dim=head_dim, + mix_block_ratio=mix_block_ratio, + attn_drop=attn_drop, + drop=drop) + features.append(layer) + input_channel = output_channel + idx += numrepeat + self.features = nn.Sequential(*features) + self.norm = nn.BatchNorm2d(output_channel, eps=NORM_EPS) + + if isinstance(out_indices, int): + out_indices = [out_indices] + assert isinstance(out_indices, Sequence), \ + f'"out_indices" must by a sequence or int, ' \ + f'get {type(out_indices)} instead.' + for i, index in enumerate(out_indices): + if index < 0: + out_indices[i] = sum(depths) + index + assert out_indices[i] >= 0, f'Invalid out_indices {index}' + self.stage_out_idx = out_indices + + if norm_cfg is not None: + self = torch.nn.SyncBatchNorm.convert_sync_batchnorm(self) + + def init_weights(self): + super(NextViT, self).init_weights() + if (isinstance(self.init_cfg, dict) + and self.init_cfg['type'] == 'Pretrained'): + # Suppress default init if use pretrained model. + return + + self._initialize_weights() + + def _initialize_weights(self): + for n, m in self.named_modules(): + if isinstance(m, (nn.BatchNorm2d, + nn.BatchNorm1d)): # nn.GroupNorm, nn.LayerNorm, + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Conv2d): + trunc_normal_(m.weight, std=.02) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward(self, x): + outputs = list() + x = self.stem(x) + stage_id = 0 + for idx, layer in enumerate(self.features): + x = layer(x) + if idx == self.stage_out_idx[stage_id]: + if self.with_extra_norm: + x = self.norm(x) + outputs.append(x) + stage_id += 1 + return tuple(outputs) + + def _freeze_stages(self): + if self.frozen_stages > 0: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + for idx, layer in enumerate(self.features): + if idx <= self.stage_out_idx[self.frozen_stages - 1]: + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(NextViT, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/runtime/Lib/site_packages/models/cv/image_classification/mmcls_model.py b/runtime/Lib/site_packages/models/cv/image_classification/mmcls_model.py new file mode 100644 index 0000000000000000000000000000000000000000..bd37d3decb98030c609e30753cc27bb1332b5874 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/mmcls_model.py @@ -0,0 +1,58 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_classification, module_name=Models.classification_model) +class ClassificationModel(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + import mmcv + from mmcls.models import build_classifier + import modelscope.models.cv.image_classification.backbones + from modelscope.utils.hub import read_config + + super().__init__(model_dir) + + self.config_type = 'ms_config' + mm_config = os.path.join(model_dir, 'config.py') + if os.path.exists(mm_config): + cfg = mmcv.Config.fromfile(mm_config) + cfg.model.pretrained = None + self.cls_model = build_classifier(cfg.model) + self.config_type = 'mmcv_config' + else: + cfg = read_config(model_dir) + cfg.model.mm_model.pretrained = None + self.cls_model = build_classifier(cfg.model.mm_model) + self.config_type = 'ms_config' + self.cfg = cfg + + self.ms_model_dir = model_dir + + self.load_pretrained_checkpoint() + + def forward(self, inputs): + + return self.cls_model(**inputs) + + def load_pretrained_checkpoint(self): + import mmcv + if os.path.exists( + os.path.join(self.ms_model_dir, ModelFile.TORCH_MODEL_FILE)): + checkpoint_path = os.path.join(self.ms_model_dir, + ModelFile.TORCH_MODEL_FILE) + else: + checkpoint_path = os.path.join(self.ms_model_dir, + 'checkpoints.pth') + if os.path.exists(checkpoint_path): + checkpoint = mmcv.runner.load_checkpoint( + self.cls_model, checkpoint_path, map_location='cpu') + if 'CLASSES' in checkpoint.get('meta', {}): + self.cls_model.CLASSES = checkpoint['meta']['CLASSES'] + self.CLASSES = self.cls_model.CLASSES diff --git a/runtime/Lib/site_packages/models/cv/image_classification/resnet50_cc.py b/runtime/Lib/site_packages/models/cv/image_classification/resnet50_cc.py new file mode 100644 index 0000000000000000000000000000000000000000..607d4deb36218a066c21225c34b579a479b4b53d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/resnet50_cc.py @@ -0,0 +1,50 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import namedtuple +from math import lgamma + +import torch +import torch.nn as nn +from torchvision import models + +from modelscope.metainfo import Models +from modelscope.models import MODELS +from modelscope.models.base import TorchModel +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module(Tasks.image_classification, Models.content_check) +class ContentCheckBackbone(TorchModel): + + def __init__(self, *args, **kwargs): + super(ContentCheckBackbone, self).__init__() + cc_model = models.resnet50() + cc_model.fc = nn.Sequential( + nn.Linear(2048, 512), + nn.ReLU(), + nn.Dropout(0.2), + nn.Linear(512, 10), + ) + self.model = cc_model + + def forward(self, img): + x = self.model(img) + return x + + @classmethod + def _instantiate(cls, **kwargs): + model_file = kwargs.get('model_name', ModelFile.TORCH_MODEL_FILE) + ckpt_path = os.path.join(kwargs['model_dir'], model_file) + logger.info(f'loading model from {ckpt_path}') + model_dir = kwargs.pop('model_dir') + model = cls(**kwargs) + ckpt_path = os.path.join(model_dir, model_file) + load_dict = torch.load(ckpt_path, map_location='cpu') + new_dict = {} + for k, v in load_dict.items(): + new_dict['model.' + k] = v + model.load_state_dict(new_dict) + return model diff --git a/runtime/Lib/site_packages/models/cv/image_classification/utils.py b/runtime/Lib/site_packages/models/cv/image_classification/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5acb53aeace2fe6a5cae5841cbfecfd53a3c5c0b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_classification/utils.py @@ -0,0 +1,169 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import collections.abc +import math +import os.path as osp +from itertools import repeat + +import numpy as np +import torch +from mmcls.datasets.base_dataset import BaseDataset + + +def get_trained_checkpoints_name(work_path): + import os + file_list = os.listdir(work_path) + last = 0 + model_name = None + # find the best model + if model_name is None: + for f_name in file_list: + if 'best_' in f_name and f_name.endswith('.pth'): + best_epoch = f_name.replace('.pth', '').split('_')[-1] + if best_epoch.isdigit(): + last = int(best_epoch) + model_name = f_name + return model_name + # or find the latest model + if model_name is None: + for f_name in file_list: + if 'epoch_' in f_name and f_name.endswith('.pth'): + epoch_num = f_name.replace('epoch_', '').replace('.pth', '') + if not epoch_num.isdigit(): + continue + ind = int(epoch_num) + if ind > last: + last = ind + model_name = f_name + return model_name + + +def preprocess_transform(cfgs): + if cfgs is None: + return None + for i, cfg in enumerate(cfgs): + if cfg.type == 'Resize': + if isinstance(cfg.size, list): + cfgs[i].size = tuple(cfg.size) + return cfgs + + +def get_ms_dataset_root(ms_dataset): + if ms_dataset is None or len(ms_dataset) < 1: + return None + try: + data_root = ms_dataset[0]['image:FILE'].split('extracted')[0] + path_post = ms_dataset[0]['image:FILE'].split('extracted')[1].split( + '/') + extracted_data_root = osp.join(data_root, 'extracted', path_post[1], + path_post[2]) + return extracted_data_root + except Exception as e: + raise ValueError(f'Dataset Error: {e}') + return None + + +def get_classes(classes=None): + import mmcv + if isinstance(classes, str): + # take it as a file path + class_names = mmcv.list_from_file(classes) + elif isinstance(classes, (tuple, list)): + class_names = classes + else: + raise ValueError(f'Unsupported type {type(classes)} of classes.') + + return class_names + + +class MmDataset(BaseDataset): + + def __init__(self, + ms_dataset, + pipeline, + classes=None, + test_mode=False, + data_prefix=''): + self.ms_dataset = ms_dataset + if len(self.ms_dataset) < 1: + raise ValueError('Dataset Error: dataset is empty') + super(MmDataset, self).__init__( + data_prefix=data_prefix, + pipeline=pipeline, + classes=classes, + test_mode=test_mode) + + def load_annotations(self): + if self.CLASSES is None: + raise ValueError( + f'Dataset Error: Not found classesname.txt: {self.CLASSES}') + + data_infos = [] + for data_info in self.ms_dataset: + filename = data_info['image:FILE'] + gt_label = data_info['category'] + info = {'img_prefix': self.data_prefix} + info['img_info'] = {'filename': filename} + info['gt_label'] = np.array(gt_label, dtype=np.int64) + data_infos.append(info) + + return data_infos + + +def _trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + v = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [v, u], then translate to + # [2v-1, 2u-1]. + tensor.uniform_(2 * v - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + # type: (Tensor, float, float, float, float) -> Tensor + with torch.no_grad(): + return _trunc_normal_(tensor, mean, std, a, b) + + +# From PyTorch internals +def _ntuple(n): + + def parse(x): + if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/__init__.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67027d513c889da6793f60201e0392f7d73229d7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_color_enhance import ImageColorEnhance + from .adaint import AdaIntImageColorEnhance + from .deeplpf import DeepLPFImageColorEnhance + +else: + _import_structure = { + 'image_color_enhance': ['ImageColorEnhance'], + 'adaint': ['AdaIntImageColorEnhance'], + 'deeplpf': ['DeepLPFImageColorEnhance'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/__init__.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2ac677b752b50ba47c54b6beb6dbea96960b9771 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/__init__.py @@ -0,0 +1 @@ +from .adaint import AdaIntImageColorEnhance diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/adaint.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/adaint.py new file mode 100644 index 0000000000000000000000000000000000000000..8839f03a9c93519ba9252c34a5604d5c6dfb4ab8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/adaint/adaint.py @@ -0,0 +1,396 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numbers +import os.path as osp +from typing import Dict, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['AdaIntImageColorEnhance'] + +try: + from modelscope.ops.ailut import ailut_transform +except ImportError: + raise ImportError( + 'The model [AdaInt] requires cuda extension to be installed.') + + +class BasicBlock(nn.Sequential): + r"""The basic block module (Conv+LeakyReLU[+InstanceNorm]). + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + norm=False): + body = [ + nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=1), + nn.LeakyReLU(0.2) + ] + if norm: + body.append(nn.InstanceNorm2d(out_channels, affine=True)) + super(BasicBlock, self).__init__(*body) + + +class TPAMIBackbone(nn.Sequential): + r"""The 5-layer CNN backbone module in [TPAMI 3D-LUT] + (https://github.com/HuiZeng/Image-Adaptive-3DLUT). + + Args: + pretrained (bool, optional): [ignored]. + input_resolution (int, optional): Resolution for pre-downsampling. Default: 256. + extra_pooling (bool, optional): Whether to insert an extra pooling layer + at the very end of the module to reduce the number of parameters of + the subsequent module. Default: False. + """ + + def __init__(self, + pretrained=False, + input_resolution=256, + extra_pooling=False): + body = [ + BasicBlock(3, 16, stride=2, norm=True), + BasicBlock(16, 32, stride=2, norm=True), + BasicBlock(32, 64, stride=2, norm=True), + BasicBlock(64, 128, stride=2, norm=True), + BasicBlock(128, 128, stride=2), + nn.Dropout(p=0.5), + ] + if extra_pooling: + body.append(nn.AdaptiveAvgPool2d(2)) + super().__init__(*body) + self.input_resolution = input_resolution + self.out_channels = 128 * (4 if extra_pooling else 64) + + def forward(self, imgs): + imgs = F.interpolate( + imgs, + size=(self.input_resolution, ) * 2, + mode='bilinear', + align_corners=False) + return super().forward(imgs).view(imgs.shape[0], -1) + + +class Res18Backbone(nn.Module): + r"""The ResNet-18 backbone. + + Args: + pretrained (bool, optional): Whether to use the torchvison pretrained weights. + Default: True. + input_resolution (int, optional): Resolution for pre-downsampling. Default: 224. + extra_pooling (bool, optional): [ignore]. + """ + + def __init__(self, + pretrained=True, + input_resolution=224, + extra_pooling=False): + super().__init__() + net = torchvision.models.resnet18(pretrained=pretrained) + net.fc = nn.Identity() + self.net = net + self.input_resolution = input_resolution + self.out_channels = 512 + + def forward(self, imgs): + imgs = F.interpolate( + imgs, + size=(self.input_resolution, ) * 2, + mode='bilinear', + align_corners=False) + return self.net(imgs).view(imgs.shape[0], -1) + + +class LUTGenerator(nn.Module): + r"""The LUT generator module (mapping h). + + Args: + n_colors (int): Number of input color channels. + n_vertices (int): Number of sampling points along each lattice dimension. + n_feats (int): Dimension of the input image representation vector. + n_ranks (int): Number of ranks in the mapping h (or the number of basis LUTs). + """ + + def __init__(self, n_colors, n_vertices, n_feats, n_ranks) -> None: + super().__init__() + + # h0 + self.weights_generator = nn.Linear(n_feats, n_ranks) + # h1 + self.basis_luts_bank = nn.Linear( + n_ranks, n_colors * (n_vertices**n_colors), bias=False) + + self.n_colors = n_colors + self.n_vertices = n_vertices + self.n_feats = n_feats + self.n_ranks = n_ranks + + def init_weights(self): + r"""Init weights for models. + + For the mapping f (`backbone`) and h (`lut_generator`), we follow the initialization in + [TPAMI 3D-LUT](https://github.com/HuiZeng/Image-Adaptive-3DLUT). + + """ + nn.init.ones_(self.weights_generator.bias) + tmp1 = torch.meshgrid( + *[torch.arange(self.n_vertices) for _ in range(self.n_colors)]) + tmp2 = [ + torch.zeros(self.n_colors, *((self.n_vertices, ) * self.n_colors)) + for _ in range(self.n_ranks - 1) + ] + identity_lut = torch.stack( + [torch.stack(tmp1, dim=0).div(self.n_vertices - 1).flip(0), *tmp2], + dim=0).view(self.n_ranks, -1) + self.basis_luts_bank.weight.data.copy_(identity_lut.t()) + + def forward(self, x): + weights = self.weights_generator(x) + luts = self.basis_luts_bank(weights) + luts = luts.view(x.shape[0], -1, + *((self.n_vertices, ) * self.n_colors)) + return weights, luts + + def regularizations(self, smoothness, monotonicity): + basis_luts = self.basis_luts_bank.weight.t().view( + self.n_ranks, self.n_colors, + *((self.n_vertices, ) * self.n_colors)) + tv, mn = 0, 0 + diff = basis_luts[:, :, :-1, ...] - basis_luts[:, :, 1:, ...] + tv += torch.square(diff).sum(0).mean() + mn += F.relu(diff).sum(0).mean() + diff = basis_luts[:, :, :, :-1, :] - basis_luts[:, :, :, 1:, :] + tv += torch.square(diff).sum(0).mean() + mn += F.relu(diff).sum(0).mean() + diff = basis_luts[:, :, :, :, :-1] - basis_luts[:, :, :, :, 1:] + tv += torch.square(diff).sum(0).mean() + mn += F.relu(diff).sum(0).mean() + reg_smoothness = smoothness * tv + reg_monotonicity = monotonicity * mn + return reg_smoothness, reg_monotonicity + + +class AdaInt(nn.Module): + r"""The Adaptive Interval Learning (AdaInt) module (mapping g). + + It consists of a single fully-connected layer and some post-process operations. + + Args: + n_colors (int): Number of input color channels. + n_vertices (int): Number of sampling points along each lattice dimension. + n_feats (int): Dimension of the input image representation vector. + adaint_share (bool, optional): Whether to enable Share-AdaInt. Default: False. + """ + + def __init__(self, + n_colors, + n_vertices, + n_feats, + adaint_share=False) -> None: + super().__init__() + repeat_factor = n_colors if not adaint_share else 1 + self.intervals_generator = nn.Linear(n_feats, + (n_vertices - 1) * repeat_factor) + + self.n_colors = n_colors + self.n_vertices = n_vertices + self.adaint_share = adaint_share + + def init_weights(self): + r"""Init weights for models. + + We use all-zero and all-one initializations for its weights and bias, respectively. + """ + nn.init.zeros_(self.intervals_generator.weight) + nn.init.ones_(self.intervals_generator.bias) + + def forward(self, x): + r"""Forward function for AdaInt module. + + Args: + x (tensor): Input image representation, shape (b, f). + Returns: + Tensor: Sampling coordinates along each lattice dimension, shape (b, c, d). + """ + x = x.view(x.shape[0], -1) + intervals = self.intervals_generator(x).view(x.shape[0], -1, + self.n_vertices - 1) + if self.adaint_share: + intervals = intervals.repeat_interleave(self.n_colors, dim=1) + intervals = intervals.softmax(-1) + vertices = F.pad(intervals.cumsum(-1), (1, 0), 'constant', 0) + return vertices + + +@MODELS.register_module( + Tasks.image_color_enhancement, module_name=Models.adaint) +class AdaIntImageColorEnhance(TorchModel): + r"""Adaptive-Interval 3D Lookup Table for real-time image enhancement. + + Args: + n_ranks (int, optional): Number of ranks in the mapping h + (or the number of basis LUTs). Default: 3. + n_vertices (int, optional): Number of sampling points along + each lattice dimension. Default: 33. + en_adaint (bool, optional): Whether to enable AdaInt. Default: True. + en_adaint_share (bool, optional): Whether to enable Share-AdaInt. + Only used when `en_adaint` is True. Default: False. + backbone (str, optional): Backbone architecture to use. Can be either 'tpami' + or 'res18'. Default: 'tpami'. + pretrained (bool, optional): Whether to use ImageNet-pretrained weights. + Only used when `backbone` is 'res18'. Default: None. + n_colors (int, optional): Number of input color channels. Default: 3. + recons_loss (dict, optional): Config for pixel-wise reconstruction loss. + train_cfg (dict, optional): Config for training. Default: None. + test_cfg (dict, optional): Config for testing. Default: None. + """ + + def __init__(self, + n_ranks=3, + n_vertices=33, + en_adaint=True, + en_adaint_share=False, + backbone='tpami', + pretrained=False, + n_colors=3, + *args, + **kwargs): + super(AdaIntImageColorEnhance, self).__init__() + + assert backbone.lower() in ['tpami', 'res18'] + + # mapping f + self.backbone = dict( + tpami=TPAMIBackbone, res18=Res18Backbone)[backbone.lower()]( + pretrained, extra_pooling=en_adaint) + + # mapping h + self.lut_generator = LUTGenerator(n_colors, n_vertices, + self.backbone.out_channels, n_ranks) + + # mapping g + if en_adaint: + self.adaint = AdaInt(n_colors, n_vertices, + self.backbone.out_channels, en_adaint_share) + else: + uniform_vertices = torch.arange(n_vertices).div(n_vertices - 1) \ + .repeat(n_colors, 1) + self.register_buffer('uniform_vertices', + uniform_vertices.unsqueeze(0)) + + self.n_ranks = n_ranks + self.n_colors = n_colors + self.n_vertices = n_vertices + self.en_adaint = en_adaint + self.backbone_name = backbone.lower() + + self.init_weights() + + def init_weights(self): + r"""Init weights for models. + + For the mapping f (`backbone`) and h (`lut_generator`), we follow the initialization in + [TPAMI 3D-LUT](https://github.com/HuiZeng/Image-Adaptive-3DLUT). + For the mapping g (`adaint`), we use all-zero and all-one initializations for its weights + and bias, respectively. + """ + + def special_initilization(m): + classname = m.__class__.__name__ + if 'Conv' in classname: + nn.init.xavier_normal_(m.weight.data) + elif 'InstanceNorm' in classname: + nn.init.normal_(m.weight.data, 1.0, 0.02) + nn.init.constant_(m.bias.data, 0.0) + + if self.backbone_name not in ['res18']: + self.apply(special_initilization) + self.lut_generator.init_weights() + if self.en_adaint: + self.adaint.init_weights() + + def __forward(self, imgs): + r"""The real implementation of model forward. + + Args: + img (Tensor): Input image, shape (b, c, h, w). + Returns: + tuple(Tensor, Tensor, Tensor): + Output image, LUT weights, Sampling Coordinates. + """ + # E: (b, f) + codes = self.backbone(imgs) + # (b, m), T: (b, c, d, d, d) + weights, luts = self.lut_generator(codes) + # \hat{P}: (b, c, d) + if self.en_adaint: + vertices = self.adaint(codes) + else: + vertices = self.uniform_vertices + + outs = ailut_transform(imgs, luts, vertices) + + return outs, weights, vertices + + def _evaluate_postprocess(self, src: Tensor, + target: Tensor) -> Dict[str, list]: + preds, _, _ = self.__forward(src) + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target, 1, 0)) + + preds = [(pred.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for pred in preds] + targets = [(target.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for target in targets] + + return {'pred': preds, 'target': targets} + + def _inference_forward(self, src: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.__forward(src)[0].clamp(0, 1)} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + if 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self._inference_forward(**input) + + def regularizations(self, smoothness, monotonicity): + return self.lut_generator.regularizations(smoothness, monotonicity) + + @classmethod + def _instantiate(cls, **kwargs): + model_path = osp.join(kwargs['model_dir'], ModelFile.TORCH_MODEL_FILE) + model = cls(**kwargs) + model = model._load_pretrained( + model, model_path, strict=False, param_key='state_dict') + if model.training: + model.train() + else: + model.eval() + return model diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/csrnet.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/csrnet.py new file mode 100644 index 0000000000000000000000000000000000000000..502abf888c04d7a9ef05ade4c10fbee22b80f6fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/csrnet.py @@ -0,0 +1,113 @@ +# The implementation is adopted from Jingwen He, +# made publicly available at https://github.com/hejingwenhejingwen/CSRNet + +import functools +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Condition(nn.Module): + + def __init__(self, in_nc=3, nf=32): + super(Condition, self).__init__() + stride = 2 + pad = 0 + self.pad = nn.ZeroPad2d(1) + self.conv1 = nn.Conv2d(in_nc, nf, 7, stride, pad, bias=True) + self.conv2 = nn.Conv2d(nf, nf, 3, stride, pad, bias=True) + self.conv3 = nn.Conv2d(nf, nf, 3, stride, pad, bias=True) + self.act = nn.ReLU(inplace=True) + + def forward(self, x): + conv1_out = self.act(self.conv1(self.pad(x))) + conv2_out = self.act(self.conv2(self.pad(conv1_out))) + conv3_out = self.act(self.conv3(self.pad(conv2_out))) + out = torch.mean(conv3_out, dim=[2, 3], keepdim=False) + + return out + + +# 3layers with control +class CSRNet(nn.Module): + + def __init__(self, in_nc=3, out_nc=3, base_nf=64, cond_nf=32): + super(CSRNet, self).__init__() + + self.base_nf = base_nf + self.out_nc = out_nc + + self.cond_net = Condition(in_nc=in_nc, nf=cond_nf) + + self.cond_scale1 = nn.Linear(cond_nf, base_nf, bias=True) + self.cond_scale2 = nn.Linear(cond_nf, base_nf, bias=True) + self.cond_scale3 = nn.Linear(cond_nf, 3, bias=True) + + self.cond_shift1 = nn.Linear(cond_nf, base_nf, bias=True) + self.cond_shift2 = nn.Linear(cond_nf, base_nf, bias=True) + self.cond_shift3 = nn.Linear(cond_nf, 3, bias=True) + + self.conv1 = nn.Conv2d(in_nc, base_nf, 1, 1, bias=True) + self.conv2 = nn.Conv2d(base_nf, base_nf, 1, 1, bias=True) + self.conv3 = nn.Conv2d(base_nf, out_nc, 1, 1, bias=True) + + self.act = nn.ReLU(inplace=True) + + def forward(self, x): + cond = self.cond_net(x) + + scale1 = self.cond_scale1(cond) + shift1 = self.cond_shift1(cond) + + scale2 = self.cond_scale2(cond) + shift2 = self.cond_shift2(cond) + + scale3 = self.cond_scale3(cond) + shift3 = self.cond_shift3(cond) + + out = self.conv1(x) + out = out * scale1.view(-1, self.base_nf, 1, 1) + shift1.view( + -1, self.base_nf, 1, 1) + out + out = self.act(out) + + out = self.conv2(out) + out = out * scale2.view(-1, self.base_nf, 1, 1) + shift2.view( + -1, self.base_nf, 1, 1) + out + out = self.act(out) + + out = self.conv3(out) + out = out * scale3.view(-1, self.out_nc, 1, 1) + shift3.view( + -1, self.out_nc, 1, 1) + out + return out + + +class L1Loss(nn.Module): + """L1 (mean absolute error, MAE) loss. + + Args: + loss_weight (float): Loss weight for L1 loss. Default: 1.0. + reduction (str): Specifies the reduction to apply to the output. + Supported choices are 'none' | 'mean' | 'sum'. Default: 'mean'. + """ + + def __init__(self, loss_weight=1.0, reduction='mean'): + super(L1Loss, self).__init__() + if reduction not in ['none', 'mean', 'sum']: + raise ValueError( + f'Unsupported reduction mode: {reduction}. Supported ones are: {_reduction_modes}' + ) + + self.loss_weight = loss_weight + self.reduction = reduction + + def forward(self, pred, target, weight=None, **kwargs): + """ + Args: + pred (Tensor): of shape (N, C, H, W). Predicted tensor. + target (Tensor): of shape (N, C, H, W). Ground truth tensor. + weight (Tensor, optional): of shape (N, C, H, W). Element-wise weights. Default: None. + """ + return self.loss_weight * F.l1_loss( + pred, target, reduction=self.reduction) diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/__init__.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..916735e4d5b26ef0c518783aa96a3ab58a7a6425 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/__init__.py @@ -0,0 +1 @@ +from .deeplpf_image_color_enhance import DeepLPFImageColorEnhance diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpf_image_color_enhance.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpf_image_color_enhance.py new file mode 100644 index 0000000000000000000000000000000000000000..400e9b8093b4d7e6b6d0ec261213b1b85352e6e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpf_image_color_enhance.py @@ -0,0 +1,78 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Dict, Union + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .deeplpfnet import DeepLPFNet + +logger = get_logger() + +__all__ = ['DeepLPFImageColorEnhance'] + + +@MODELS.register_module( + Tasks.image_color_enhancement, module_name=Models.deeplpfnet) +class DeepLPFImageColorEnhance(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image color enhance model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.model = DeepLPFNet() + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.model = self.model.to(self._device) + + self.model = self._load_pretrained(self.model, model_path) + + if self.training: + self.model.train() + else: + self.model.eval() + + def _evaluate_postprocess(self, src: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(src) + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target, 1, 0)) + + preds = [(pred.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for pred in preds] + targets = [(target.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for target in targets] + + return {'pred': preds, 'target': targets} + + def _inference_forward(self, src: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.model(src).clamp(0, 1)} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + for key, value in input.items(): + input[key] = input[key].to(self._device) + if 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self._inference_forward(**input) diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpfnet.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpfnet.py new file mode 100644 index 0000000000000000000000000000000000000000..e7760de9f3852c35eaf14a89be9a9338b76c125f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/deeplpf/deeplpfnet.py @@ -0,0 +1,854 @@ +# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. + +# This program is free software; you can redistribute it and/or modify it under the terms of the BSD 0-Clause License. + +# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the BSD 0-Clause License for more details. +''' +This is a PyTorch implementation of the CVPR 2020 paper: +"Deep Local Parametric Filters for Image Enhancement": https://arxiv.org/abs/2003.13985 + +DeeLPF is a method for automatic estimation of parametric filters for +local image enhancement, which is instantiated using Elliptical, Graduated, Polynomial filters. + +Please cite the paper if you use this code + +Tested with Pytorch 1.7.1, Python 3.7.9 + +Authors: Sean Moran (sean.j.moran@gmail.com), + Pierre Marza (pierre.marza@gmail.com) + +''' +import math +from math import exp + +import matplotlib +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +matplotlib.use('agg') + + +class BinaryLayer(nn.Module): + + def forward(self, input): + return torch.sign(input) + + def backward(self, grad_output): + input = self.saved_tensors + grad_output[input > 1] = 0 + grad_output[input < -1] = 0 + return grad_output + + +class CubicFilter(nn.Module): + + def __init__(self, num_in_channels=64, num_out_channels=64, batch_size=1): + super(CubicFilter, self).__init__() + + self.cubic_layer1 = ConvBlock(num_in_channels, num_out_channels) + self.cubic_layer2 = MaxPoolBlock() + self.cubic_layer3 = ConvBlock(num_out_channels, num_out_channels) + self.cubic_layer4 = MaxPoolBlock() + self.cubic_layer5 = ConvBlock(num_out_channels, num_out_channels) + self.cubic_layer6 = MaxPoolBlock() + self.cubic_layer7 = ConvBlock(num_out_channels, num_out_channels) + self.cubic_layer8 = GlobalPoolingBlock(2) + self.fc_cubic = torch.nn.Linear(num_out_channels, 60) # cubic + self.upsample = torch.nn.Upsample( + size=(300, 300), mode='bilinear', align_corners=False) + self.dropout = nn.Dropout(0.5) + + def get_cubic_mask(self, feat, img): + feat_cubic = torch.cat((feat, img), 1) + feat_cubic = self.upsample(feat_cubic) + + x = self.cubic_layer1(feat_cubic) + x = self.cubic_layer2(x) + x = self.cubic_layer3(x) + x = self.cubic_layer4(x) + x = self.cubic_layer5(x) + x = self.cubic_layer6(x) + x = self.cubic_layer7(x) + x = self.cubic_layer8(x) + x = x.view(x.size()[0], -1) + x = self.dropout(x) + + R = self.fc_cubic(x) + + cubic_mask = torch.zeros_like(img) + + x_axis = Variable( + torch.arange(img.shape[2]).view(-1, 1).repeat( + 1, img.shape[3]).cuda()) / img.shape[2] + y_axis = Variable( + torch.arange(img.shape[3]).repeat(img.shape[2], + 1).cuda()) / img.shape[3] + ''' + Cubic for R channel + ''' + cubic_mask[0, 0, :, :] = R[0, 0] * (x_axis ** 3) + R[0, 1] * (x_axis ** 2) * y_axis + R[0, 2] * ( + x_axis ** 2) * img[0, 0, :, :] + R[0, 3] * (x_axis ** 2) + R[0, 4] * x_axis * (y_axis ** 2) + R[ + 0, 5] * x_axis * y_axis * img[0, 0, :, :] \ + + R[0, 6] * x_axis * y_axis + R[0, 7] * x_axis * (img[0, 0, :, :] ** 2) + R[ + 0, 8] * x_axis * img[0, 0, :, :] + R[0, 9] * x_axis + R[0, 10] * ( + y_axis ** 3) + R[0, 11] * (y_axis ** 2) * img[0, 0, :, :] \ + + R[0, 12] * (y_axis ** 2) + R[0, 13] * y_axis * (img[0, 0, :, :] ** 2) + R[ + 0, 14] * y_axis * img[0, 0, :, :] + R[0, 15] * y_axis + R[0, 16] * ( + img[0, 0, :, :] ** 3) + R[0, 17] * (img[0, 0, :, :] ** 2) \ + + R[0, 18] * \ + img[0, 0, :, :] + R[0, 19] + ''' + Cubic for G channel + ''' + cubic_mask[0, 1, :, :] = R[0, 20] * (x_axis ** 3) + R[0, 21] * (x_axis ** 2) * y_axis + R[0, 22] * ( + x_axis ** 2) * img[0, 1, :, :] + R[0, 23] * (x_axis ** 2) + R[0, 24] * x_axis * (y_axis ** 2) + R[ + 0, 25] * x_axis * y_axis * img[0, 1, :, :] \ + + R[0, 26] * x_axis * y_axis + R[0, 27] * x_axis * (img[0, 1, :, :] ** 2) + R[ + 0, 28] * x_axis * img[0, 1, :, :] + R[0, 29] * x_axis + R[0, 30] * ( + y_axis ** 3) + R[0, 31] * (y_axis ** 2) * img[0, 1, :, :] \ + + R[0, 32] * (y_axis ** 2) + R[0, 33] * y_axis * (img[0, 1, :, :] ** 2) + R[ + 0, 34] * y_axis * img[0, 1, :, :] + R[0, 35] * y_axis + R[0, 36] * ( + img[0, 1, :, :] ** 3) + R[0, 37] * (img[0, 1, :, :] ** 2) \ + + R[0, 38] * \ + img[0, 1, :, :] + R[0, 39] + ''' + Cubic for B channel + ''' + cubic_mask[0, 2, :, :] = R[0, 40] * (x_axis ** 3) + R[0, 41] * (x_axis ** 2) * y_axis + R[0, 42] * ( + x_axis ** 2) * img[0, 2, :, :] + R[0, 43] * (x_axis ** 2) + R[0, 44] * x_axis * (y_axis ** 2) + R[ + 0, 45] * x_axis * y_axis * img[0, 2, :, :] \ + + R[0, 46] * x_axis * y_axis + R[0, 47] * x_axis * (img[0, 2, :, :] ** 2) + R[ + 0, 48] * x_axis * img[0, 2, :, :] + R[0, 49] * x_axis + R[0, 50] * ( + y_axis ** 3) + R[0, 51] * (y_axis ** 2) * img[0, 2, :, :] \ + + R[0, 52] * (y_axis ** 2) + R[0, 53] * y_axis * (img[0, 2, :, :] ** 2) + R[ + 0, 54] * y_axis * img[0, 2, :, :] + R[0, 55] * y_axis + R[0, 56] * ( + img[0, 2, :, :] ** 3) + R[0, 57] * (img[0, 2, :, :] ** 2) \ + + R[0, 58] * \ + img[0, 2, :, :] + R[0, 59] + + img_cubic = torch.clamp(img + cubic_mask, 0, 1) + return img_cubic + + +class GraduatedFilter(nn.Module): + + def __init__(self, num_in_channels=64, num_out_channels=64): + super(GraduatedFilter, self).__init__() + + self.graduated_layer1 = ConvBlock(num_in_channels, num_out_channels) + self.graduated_layer2 = MaxPoolBlock() + self.graduated_layer3 = ConvBlock(num_out_channels, num_out_channels) + self.graduated_layer4 = MaxPoolBlock() + self.graduated_layer5 = ConvBlock(num_out_channels, num_out_channels) + self.graduated_layer6 = MaxPoolBlock() + self.graduated_layer7 = ConvBlock(num_out_channels, num_out_channels) + self.graduated_layer8 = GlobalPoolingBlock(2) + self.fc_graduated = torch.nn.Linear(num_out_channels, 24) + self.upsample = torch.nn.Upsample( + size=(300, 300), mode='bilinear', align_corners=False) + self.dropout = nn.Dropout(0.5) + self.bin_layer = BinaryLayer() + + def tanh01(self, x): + tanh = nn.Tanh() + return 0.5 * (tanh(x) + 1) + + def where(self, cond, x_1, x_2): + cond = cond.float() + return (cond * x_1) + ((1 - cond) * x_2) + + def get_inverted_mask(self, factor, invert, d1, d2, max_scale, top_line): + if (invert == 1).all(): + + if (factor >= 1).all(): + diff = ((factor - 1)) / 2 + 1 + grad1 = (diff - factor) / d1 + grad2 = (1 - diff) / d2 + mask_scale = torch.clamp( + factor + grad1 * top_line + grad2 * top_line, + min=1, + max=max_scale) + else: + diff = ((1 - factor)) / 2 + factor + grad1 = (diff - factor) / d1 + grad2 = (1 - diff) / d2 + mask_scale = torch.clamp( + factor + grad1 * top_line + grad2 * top_line, min=0, max=1) + else: + + if (factor >= 1).all(): + diff = ((factor - 1)) / 2 + 1 + grad1 = (diff - factor) / d1 + grad2 = (factor - diff) / d2 + mask_scale = torch.clamp( + 1 + grad1 * top_line + grad2 * top_line, + min=1, + max=max_scale) + else: + diff = ((1 - factor)) / 2 + factor + grad1 = (diff - 1) / d1 + grad2 = (factor - diff) / d2 + mask_scale = torch.clamp( + 1 + grad1 * top_line + grad2 * top_line, min=0, max=1) + + mask_scale = torch.clamp(mask_scale.unsqueeze(0), 0, max_scale) + return mask_scale + + def get_graduated_mask(self, feat, img): + eps = 1e-10 + + x_axis = Variable( + torch.arange(img.shape[2]).view(-1, 1).repeat( + 1, img.shape[3]).cuda()) / img.shape[2] + y_axis = Variable( + torch.arange(img.shape[3]).repeat(img.shape[2], + 1).cuda()) / img.shape[3] + + feat_graduated = torch.cat((feat, img), 1) + feat_graduated = self.upsample(feat_graduated) + + # The following layers calculate the parameters of the graduated filters that we use for image enhancement + x = self.graduated_layer1(feat_graduated) + x = self.graduated_layer2(x) + x = self.graduated_layer3(x) + x = self.graduated_layer4(x) + x = self.graduated_layer5(x) + x = self.graduated_layer6(x) + x = self.graduated_layer7(x) + x = self.graduated_layer8(x) + x = x.view(x.size()[0], -1) + x = self.dropout(x) + G = self.fc_graduated(x) + + # Classification values (above or below the line) + above_or_below_line1 = ((self.bin_layer(G[0, 0])) + 1) / 2 + above_or_below_line2 = ((self.bin_layer(G[0, 1])) + 1) / 2 + above_or_below_line3 = ((self.bin_layer(G[0, 2])) + 1) / 2 + + slope1 = G[0, 3].clone() + slope2 = G[0, 4].clone() + slope3 = G[0, 5].clone() + + y_axis_dist1 = self.tanh01(G[0, 6]) + eps + y_axis_dist2 = self.tanh01(G[0, 7]) + eps + y_axis_dist3 = self.tanh01(G[0, 8]) + eps + + y_axis_dist1 = torch.clamp( + self.tanh01(G[0, 9]), y_axis_dist1.data, 1.0) + y_axis_dist2 = torch.clamp( + self.tanh01(G[0, 10]), y_axis_dist2.data, 1.0) + y_axis_dist3 = torch.clamp( + self.tanh01(G[0, 11]), y_axis_dist3.data, 1.0) + + y_axis_dist4 = torch.clamp(self.tanh01(G[0, 12]), 0, y_axis_dist1.data) + y_axis_dist5 = torch.clamp(self.tanh01(G[0, 13]), 0, y_axis_dist2.data) + y_axis_dist6 = torch.clamp(self.tanh01(G[0, 14]), 0, y_axis_dist3.data) + + # Scales + max_scale = 2 + + scale_factor1 = self.tanh01(G[0, 15]) * max_scale + scale_factor2 = self.tanh01(G[0, 16]) * max_scale + scale_factor3 = self.tanh01(G[0, 17]) * max_scale + + scale_factor4 = self.tanh01(G[0, 18]) * max_scale + scale_factor5 = self.tanh01(G[0, 19]) * max_scale + scale_factor6 = self.tanh01(G[0, 20]) * max_scale + + scale_factor7 = self.tanh01(G[0, 21]) * max_scale + scale_factor8 = self.tanh01(G[0, 22]) * max_scale + scale_factor9 = self.tanh01(G[0, 23]) * max_scale + + slope1_angle = torch.atan(slope1) + slope2_angle = torch.atan(slope2) + slope3_angle = torch.atan(slope3) + + # Distances between central line and two outer lines + d1 = self.tanh01(y_axis_dist1 * torch.cos(slope1_angle)) + d2 = self.tanh01(y_axis_dist4 * torch.cos(slope1_angle)) + d3 = self.tanh01(y_axis_dist2 * torch.cos(slope2_angle)) + d4 = self.tanh01(y_axis_dist5 * torch.cos(slope2_angle)) + d5 = self.tanh01(y_axis_dist3 * torch.cos(slope3_angle)) + d6 = self.tanh01(y_axis_dist6 * torch.cos(slope3_angle)) + + top_line1 = self.tanh01(y_axis - (slope1 * x_axis + y_axis_dist1 + d1)) + top_line2 = self.tanh01(y_axis - (slope2 * x_axis + y_axis_dist2 + d3)) + top_line3 = self.tanh01(y_axis - (slope3 * x_axis + y_axis_dist3 + d5)) + ''' + The following are the scale factors for each of the 9 graduated filters + ''' + mask_scale1 = self.get_inverted_mask(scale_factor1, + above_or_below_line1, d1, d2, + max_scale, top_line1) + mask_scale2 = self.get_inverted_mask(scale_factor2, + above_or_below_line1, d1, d2, + max_scale, top_line1) + mask_scale3 = self.get_inverted_mask(scale_factor3, + above_or_below_line1, d1, d2, + max_scale, top_line1) + + mask_scale_1 = torch.cat((mask_scale1, mask_scale2, mask_scale3), + dim=0) + mask_scale_1 = torch.clamp(mask_scale_1.unsqueeze(0), 0, max_scale) + + mask_scale4 = self.get_inverted_mask(scale_factor4, + above_or_below_line2, d3, d4, + max_scale, top_line2) + mask_scale5 = self.get_inverted_mask(scale_factor5, + above_or_below_line2, d3, d4, + max_scale, top_line2) + mask_scale6 = self.get_inverted_mask(scale_factor6, + above_or_below_line2, d3, d4, + max_scale, top_line2) + + mask_scale_4 = torch.cat((mask_scale4, mask_scale5, mask_scale6), + dim=0) + mask_scale_4 = torch.clamp(mask_scale_4.unsqueeze(0), 0, max_scale) + + mask_scale7 = self.get_inverted_mask(scale_factor7, + above_or_below_line3, d5, d6, + max_scale, top_line3) + mask_scale8 = self.get_inverted_mask(scale_factor8, + above_or_below_line3, d5, d6, + max_scale, top_line3) + mask_scale9 = self.get_inverted_mask(scale_factor9, + above_or_below_line3, d5, d6, + max_scale, top_line3) + + mask_scale_7 = torch.cat((mask_scale7, mask_scale8, mask_scale9), + dim=0) + mask_scale_7 = torch.clamp(mask_scale_7.unsqueeze(0), 0, max_scale) + + mask_scale = torch.clamp(mask_scale_1 * mask_scale_4 * mask_scale_7, 0, + max_scale) + + return mask_scale + + +class EllipticalFilter(nn.Module): + + def __init__(self, num_in_channels=64, num_out_channels=64): + super(EllipticalFilter, self).__init__() + + self.elliptical_layer1 = ConvBlock(num_in_channels, num_out_channels) + self.elliptical_layer2 = MaxPoolBlock() + self.elliptical_layer3 = ConvBlock(num_out_channels, num_out_channels) + self.elliptical_layer4 = MaxPoolBlock() + self.elliptical_layer5 = ConvBlock(num_out_channels, num_out_channels) + self.elliptical_layer6 = MaxPoolBlock() + self.elliptical_layer7 = ConvBlock(num_out_channels, num_out_channels) + self.elliptical_layer8 = GlobalPoolingBlock(2) + self.fc_elliptical = torch.nn.Linear(num_out_channels, + 24) # elliptical + self.upsample = torch.nn.Upsample( + size=(300, 300), mode='bilinear', align_corners=False) + self.dropout = nn.Dropout(0.5) + + def tanh01(self, x): + tanh = nn.Tanh() + return 0.5 * (tanh(x) + 1) + + def where(self, cond, x_1, x_2): + cond = cond.float() + return (cond * x_1) + ((1 - cond) * x_2) + + def get_mask(self, + x_axis, + y_axis, + shift_x=0, + shift_y=0, + semi_axis_x=1, + semi_axis_y=1, + alpha=0, + scale_factor=2, + max_scale=2, + eps=1e-7, + radius=1): + # Check whether a point is inside our outside of the ellipse and set the scaling factor accordingly + ellipse_equation_part1 = \ + (((x_axis - shift_x) * torch.cos(alpha) + (y_axis - shift_y) * torch.sin(alpha))**2) + ellipse_equation_part1 /= ((semi_axis_x)**2) + ellipse_equation_part2 = \ + (((x_axis - shift_x) * torch.sin(alpha) - (y_axis - shift_y) * torch.cos(alpha))**2) + ellipse_equation_part2 /= ((semi_axis_y)**2) + + # Set the scaling factors to decay with radius inside the ellipse + tmp = torch.sqrt((x_axis - shift_x)**2 + (y_axis - shift_y)**2 + eps) + tmp *= (1 - scale_factor) + tmp = tmp / radius + scale_factor + mask_scale = self.where( + ellipse_equation_part1 + ellipse_equation_part2 < 1, tmp, 1) + + mask_scale = torch.clamp(mask_scale.unsqueeze(0), 0, max_scale) + + return mask_scale + + def get_elliptical_mask(self, feat, img): + # The two eps parameters are used to avoid numerical issues in the learning + eps2 = 1e-7 + eps1 = 1e-10 + + # max_scale is the maximum an ellipse can scale the image R,G,B values by + max_scale = 2 + + feat_elliptical = torch.cat((feat, img), 1) + feat_elliptical = self.upsample(feat_elliptical) + + # The following layers calculate the parameters of the ellipses that we use for image enhancement + x = self.elliptical_layer1(feat_elliptical) + x = self.elliptical_layer2(x) + x = self.elliptical_layer3(x) + x = self.elliptical_layer4(x) + x = self.elliptical_layer5(x) + x = self.elliptical_layer6(x) + x = self.elliptical_layer7(x) + x = self.elliptical_layer8(x) + x = x.view(x.size()[0], -1) + x = self.dropout(x) + G = self.fc_elliptical(x) + + # The next code implements a rotated ellipse according to: + # https://math.stackexchange.com/questions/426150/what-is-the-general-equation-of-the-ellipse-that-is-not-in-the-origin-and-rotate + + # Normalised coordinates for x and y-axes, we instantiate the ellipses in these coordinates + x_axis = Variable( + torch.arange(img.shape[2]).view(-1, 1).repeat( + 1, img.shape[3]).cuda()) / img.shape[2] + y_axis = Variable( + torch.arange(img.shape[3]).repeat(img.shape[2], + 1).cuda()) / img.shape[3] + + # Centre of ellipse, x-coordinate + x_coord1 = self.tanh01(G[0, 0]) + eps1 + x_coord2 = self.tanh01(G[0, 1]) + eps1 + x_coord3 = self.tanh01(G[0, 2]) + eps1 + + # Centre of ellipse, y-coordinate + y_coord1 = self.tanh01(G[0, 3]) + eps1 + y_coord2 = self.tanh01(G[0, 4]) + eps1 + y_coord3 = self.tanh01(G[0, 5]) + eps1 + + # a value of ellipse + a1 = self.tanh01(G[0, 6]) + eps1 + a2 = self.tanh01(G[0, 7]) + eps1 + a3 = self.tanh01(G[0, 8]) + eps1 + + # b value + b1 = self.tanh01(G[0, 9]) + eps1 + b2 = self.tanh01(G[0, 10]) + eps1 + b3 = self.tanh01(G[0, 11]) + eps1 + + # A value is angle to the x-axis + A1 = self.tanh01(G[0, 12]) * math.pi + eps1 + A2 = self.tanh01(G[0, 13]) * math.pi + eps1 + A3 = self.tanh01(G[0, 14]) * math.pi + eps1 + ''' + The following are the scale factors for each of the 9 ellipses + ''' + scale1 = self.tanh01(G[0, 15]) * max_scale + eps1 + scale2 = self.tanh01(G[0, 16]) * max_scale + eps1 + scale3 = self.tanh01(G[0, 17]) * max_scale + eps1 + + scale4 = self.tanh01(G[0, 18]) * max_scale + eps1 + scale5 = self.tanh01(G[0, 19]) * max_scale + eps1 + scale6 = self.tanh01(G[0, 20]) * max_scale + eps1 + + scale7 = self.tanh01(G[0, 21]) * max_scale + eps1 + scale8 = self.tanh01(G[0, 22]) * max_scale + eps1 + scale9 = self.tanh01(G[0, 23]) * max_scale + eps1 + + # Angle of orientation of the ellipses with respect to the y semi-axis + tmp = torch.sqrt((x_axis - x_coord1)**2 + (y_axis - y_coord1)**2 + + eps1) + angle_1 = torch.acos( + torch.clamp((y_axis - y_coord1) / tmp, -1 + eps2, 1 - eps2)) - A1 + + tmp = torch.sqrt((x_axis - x_coord2)**2 + (y_axis - y_coord2)**2 + + eps1) + angle_2 = torch.acos( + torch.clamp((y_axis - y_coord2) / tmp, -1 + eps2, 1 - eps2)) - A2 + + tmp = torch.sqrt((x_axis - x_coord3)**2 + (y_axis - y_coord3)**2 + + eps1) + angle_3 = torch.acos( + torch.clamp((y_axis - y_coord3) / tmp, -1 + eps2, 1 - eps2)) - A3 + + # Radius of the ellipses + # https://math.stackexchange.com/questions/432902/how-to-get-the-radius-of-an-ellipse-at-a-specific-angle-by-knowing-its-semi-majo + radius_1 = (a1 * b1) / torch.sqrt((a1**2) * (torch.sin(angle_1)**2) + + (b1**2) * (torch.cos(angle_1)**2) + + eps1) + + radius_2 = (a2 * b2) / torch.sqrt((a2**2) * (torch.sin(angle_2)**2) + + (b2**2) * (torch.cos(angle_2)**2) + + eps1) + + radius_3 = (a3 * b3) / torch.sqrt((a3**2) * (torch.sin(angle_3)**2) + + (b3**2) * (torch.cos(angle_3)**2) + + eps1) + + # Scaling factors for the R,G,B channels, here we learn three ellipses + mask_scale1 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord1, + shift_y=y_coord1, + semi_axis_x=a1, + semi_axis_y=b1, + alpha=angle_1, + scale_factor=scale1, + radius=radius_1) + + mask_scale2 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord1, + shift_y=y_coord1, + semi_axis_x=a1, + semi_axis_y=b1, + alpha=angle_1, + scale_factor=scale2, + radius=radius_1) + + mask_scale3 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord1, + shift_y=y_coord1, + semi_axis_x=a1, + semi_axis_y=b1, + alpha=angle_1, + scale_factor=scale3, + radius=radius_1) + + mask_scale_1 = torch.cat((mask_scale1, mask_scale2, mask_scale3), + dim=0) + mask_scale_1_rad = torch.clamp(mask_scale_1.unsqueeze(0), 0, max_scale) + + # Scaling factors for the R,G,B channels, here we learn three ellipses + mask_scale4 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord2, + shift_y=y_coord2, + semi_axis_x=a2, + semi_axis_y=b2, + alpha=angle_2, + scale_factor=scale4, + radius=radius_2) + + mask_scale5 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord2, + shift_y=y_coord2, + semi_axis_x=a2, + semi_axis_y=b2, + alpha=angle_2, + scale_factor=scale5, + radius=radius_2) + + mask_scale6 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord2, + shift_y=y_coord2, + semi_axis_x=a2, + semi_axis_y=b3, + alpha=angle_2, + scale_factor=scale6, + radius=radius_2) + + mask_scale_4 = torch.cat((mask_scale4, mask_scale5, mask_scale6), + dim=0) + mask_scale_4_rad = torch.clamp(mask_scale_4.unsqueeze(0), 0, max_scale) + + # Scaling factors for the R,G,B channels, here we learn three ellipses + mask_scale7 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord3, + shift_y=y_coord3, + semi_axis_x=a3, + semi_axis_y=b3, + alpha=angle_3, + scale_factor=scale7, + radius=radius_3) + + mask_scale8 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord3, + shift_y=y_coord3, + semi_axis_x=a3, + semi_axis_y=b3, + alpha=angle_3, + scale_factor=scale8, + radius=radius_3) + + mask_scale9 = self.get_mask( + x_axis, + y_axis, + shift_x=x_coord3, + shift_y=y_coord3, + semi_axis_x=a3, + semi_axis_y=b3, + alpha=angle_3, + scale_factor=scale9, + radius=radius_3) + + mask_scale_7 = torch.cat((mask_scale7, mask_scale8, mask_scale9), + dim=0) + mask_scale_7_rad = torch.clamp(mask_scale_7.unsqueeze(0), 0, max_scale) + + # Mix the ellipses together by multiplication + mask_scale_elliptical = torch.clamp( + mask_scale_1_rad * mask_scale_4_rad * mask_scale_7_rad, 0, + max_scale) + + return mask_scale_elliptical + + +class Block(nn.Module): + + def __init__(self): + super(Block, self).__init__() + + def conv3x3(self, in_channels, out_channels, stride=1): + return nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=True) + + +class ConvBlock(Block, nn.Module): + + def __init__(self, num_in_channels, num_out_channels, stride=1): + super(Block, self).__init__() + self.conv = self.conv3x3(num_in_channels, num_out_channels, stride=2) + self.lrelu = nn.LeakyReLU() + + def forward(self, x): + img_out = self.lrelu(self.conv(x)) + return img_out + + +class MaxPoolBlock(Block, nn.Module): + + def __init__(self): + super(Block, self).__init__() + + self.max_pool = nn.MaxPool2d(kernel_size=2, stride=2) + + def forward(self, x): + img_out = self.max_pool(x) + return img_out + + +class GlobalPoolingBlock(Block, nn.Module): + + def __init__(self, receptive_field): + super(Block, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + + def forward(self, x): + out = self.avg_pool(x) + return out + + +class DeepLPFParameterPrediction(nn.Module): + + def __init__(self, num_in_channels=64, num_out_channels=64, batch_size=1): + super(DeepLPFParameterPrediction, self).__init__() + self.num_in_channels = num_in_channels + self.num_out_channels = num_out_channels + self.cubic_filter = CubicFilter() + self.graduated_filter = GraduatedFilter() + self.elliptical_filter = EllipticalFilter() + + def forward(self, x): + x.contiguous() # remove memory holes + x.cuda() + + feat = x[:, 3:64, :, :] + img = x[:, 0:3, :, :] + + torch.cuda.empty_cache() + + img_cubic = self.cubic_filter.get_cubic_mask(feat, img) + + mask_scale_graduated = self.graduated_filter.get_graduated_mask( + feat, img_cubic) + mask_scale_elliptical = self.elliptical_filter.get_elliptical_mask( + feat, img_cubic) + + mask_scale_fuse = torch.clamp( + mask_scale_graduated + mask_scale_elliptical, 0, 2) + + img_fuse = torch.clamp(img_cubic * mask_scale_fuse, 0, 1) + + img = torch.clamp(img_fuse + img, 0, 1) + + return img + + +class UNet(nn.Module): + + def __init__(self): + super().__init__() + + self.conv1 = nn.Conv2d(16, 64, 1) + self.conv2 = nn.Conv2d(32, 64, 1) + self.conv3 = nn.Conv2d(64, 64, 1) + + self.local_net = LocalNet(16) + + self.dconv_down1 = LocalNet(3, 16) + self.dconv_down2 = LocalNet(16, 32) + self.dconv_down3 = LocalNet(32, 64) + self.dconv_down4 = LocalNet(64, 128) + self.dconv_down5 = LocalNet(128, 128) + + self.maxpool = nn.MaxPool2d(2, padding=0) + + self.upsample = nn.UpsamplingNearest2d(scale_factor=2) + self.up_conv1x1_1 = nn.Conv2d(128, 128, 1) + self.up_conv1x1_2 = nn.Conv2d(128, 128, 1) + self.up_conv1x1_3 = nn.Conv2d(64, 64, 1) + self.up_conv1x1_4 = nn.Conv2d(32, 32, 1) + + self.dconv_up4 = LocalNet(256, 128) + self.dconv_up3 = LocalNet(192, 64) + self.dconv_up2 = LocalNet(96, 32) + self.dconv_up1 = LocalNet(48, 16) + + self.conv_last = LocalNet(16, 3) + + def forward(self, x): + x_in_tile = x.clone() + + conv1 = self.dconv_down1(x) + x = self.maxpool(conv1) + + conv2 = self.dconv_down2(x) + x = self.maxpool(conv2) + + conv3 = self.dconv_down3(x) + x = self.maxpool(conv3) + + conv4 = self.dconv_down4(x) + x = self.maxpool(conv4) + + x = self.dconv_down5(x) + + x = self.up_conv1x1_1(self.upsample(x)) + + if x.shape[3] != conv4.shape[3] and x.shape[2] != conv4.shape[2]: + x = torch.nn.functional.pad(x, (1, 0, 0, 1)) + elif x.shape[2] != conv4.shape[2]: + x = torch.nn.functional.pad(x, (0, 0, 0, 1)) + elif x.shape[3] != conv4.shape[3]: + x = torch.nn.functional.pad(x, (1, 0, 0, 0)) + + x = torch.cat([x, conv4], dim=1) + + x = self.dconv_up4(x) + x = self.up_conv1x1_2(self.upsample(x)) + + if x.shape[3] != conv3.shape[3] and x.shape[2] != conv3.shape[2]: + x = torch.nn.functional.pad(x, (1, 0, 0, 1)) + elif x.shape[2] != conv3.shape[2]: + x = torch.nn.functional.pad(x, (0, 0, 0, 1)) + elif x.shape[3] != conv3.shape[3]: + x = torch.nn.functional.pad(x, (1, 0, 0, 0)) + + x = torch.cat([x, conv3], dim=1) + + x = self.dconv_up3(x) + x = self.up_conv1x1_3(self.upsample(x)) + + del conv3 + + if x.shape[3] != conv2.shape[3] and x.shape[2] != conv2.shape[2]: + x = torch.nn.functional.pad(x, (1, 0, 0, 1)) + elif x.shape[2] != conv2.shape[2]: + x = torch.nn.functional.pad(x, (0, 0, 0, 1)) + elif x.shape[3] != conv2.shape[3]: + x = torch.nn.functional.pad(x, (1, 0, 0, 0)) + + x = torch.cat([x, conv2], dim=1) + + x = self.dconv_up2(x) + x = self.up_conv1x1_4(self.upsample(x)) + + del conv2 + + if x.shape[3] != conv1.shape[3] and x.shape[2] != conv1.shape[2]: + x = torch.nn.functional.pad(x, (1, 0, 0, 1)) + elif x.shape[2] != conv1.shape[2]: + x = torch.nn.functional.pad(x, (0, 0, 0, 1)) + elif x.shape[3] != conv1.shape[3]: + x = torch.nn.functional.pad(x, (1, 0, 0, 0)) + + x = torch.cat([x, conv1], dim=1) + del conv1 + + x = self.dconv_up1(x) + + out = self.conv_last(x) + out = out + x_in_tile + + return out + + +class LocalNet(nn.Module): + + def forward(self, x_in): + x = self.lrelu(self.conv1(self.refpad(x_in))) + x = self.lrelu(self.conv2(self.refpad(x))) + + return x + + def __init__(self, in_channels=16, out_channels=64): + super(LocalNet, self).__init__() + self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, 0, 1) + self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, 0, 1) + self.lrelu = nn.LeakyReLU() + self.refpad = nn.ReflectionPad2d(1) + + +# Model definition +class UNetModel(nn.Module): + + def __init__(self): + super(UNetModel, self).__init__() + + self.unet = UNet() + self.final_conv = nn.Conv2d(3, 64, 3, 1, 0, 1) + self.refpad = nn.ReflectionPad2d(1) + + def forward(self, img): + output_img = self.unet(img) + return self.final_conv(self.refpad(output_img)) + + +class DeepLPFNet(nn.Module): + + def __init__(self): + super(DeepLPFNet, self).__init__() + self.backbonenet = UNetModel() + self.deeplpfnet = DeepLPFParameterPrediction() + + def forward(self, img): + feat = self.backbonenet(img) + img = self.deeplpfnet(feat) + img = torch.clamp(img, 0.0, 1.0) + + return img diff --git a/runtime/Lib/site_packages/models/cv/image_color_enhance/image_color_enhance.py b/runtime/Lib/site_packages/models/cv/image_color_enhance/image_color_enhance.py new file mode 100644 index 0000000000000000000000000000000000000000..5c17aa582b5fd0198b31d587f6e5f6c27823f4a9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_color_enhance/image_color_enhance.py @@ -0,0 +1,85 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Dict, Union + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .csrnet import CSRNet, L1Loss + +logger = get_logger() + +__all__ = ['ImageColorEnhance'] + + +@MODELS.register_module( + Tasks.image_color_enhancement, module_name=Models.csrnet) +class ImageColorEnhance(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image color enhance model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.loss = L1Loss() + self.model = CSRNet() + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.model = self.model.to(self._device) + + self.model = self._load_pretrained(self.model, model_path) + + if self.training: + self.model.train() + else: + self.model.eval() + + def _evaluate_postprocess(self, src: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(src) + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target, 1, 0)) + + preds = [(pred.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for pred in preds] + targets = [(target.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for target in targets] + + return {'pred': preds, 'target': targets} + + def _train_forward(self, src: Tensor, target: Tensor) -> Dict[str, Tensor]: + preds = self.model(src) + return {'loss': self.loss(preds, target)} + + def _inference_forward(self, src: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.model(src).clamp(0, 1)} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + for key, value in input.items(): + input[key] = input[key].to(self._device) + if self.training: + return self._train_forward(**input) + elif 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self._inference_forward(**input) diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/__init__.py b/runtime/Lib/site_packages/models/cv/image_colorization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9facc8a63c6520d7dca67e135561f948df67087 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .unet import DynamicUnetWide, DynamicUnetDeep, NormType + from .ddcolor import DDColorForImageColorization + +else: + _import_structure = { + 'unet': ['DynamicUnetWide', 'DynamicUnetDeep', 'NormType'], + 'ddcolor': ['DDColorForImageColorization'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/__init__.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a185e294d945cb3bb2002b617392048c7110fe0e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .ddcolor_for_image_colorization import DDColorForImageColorization diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor.py new file mode 100644 index 0000000000000000000000000000000000000000..6faa008780d223c2d5e2ebd553da82dd356a4112 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor.py @@ -0,0 +1,283 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + +from .utils.convnext import ConvNeXt +from .utils.position_encoding import PositionEmbeddingSine +from .utils.transformer_utils import (MLP, CrossAttentionLayer, FFNLayer, + SelfAttentionLayer) +from .utils.unet import (CustomPixelShuffle_ICNR, Hook, NormType, + UnetBlockWide, custom_conv_layer) + + +class DDColor(nn.Module): + + def __init__(self, + encoder_name='convnext-l', + input_size=(256, 256), + num_queries=100): + + super().__init__() + + self.encoder = Encoder(encoder_name, + ['norm0', 'norm1', 'norm2', 'norm3']) + self.encoder.eval() + test_input = torch.randn(1, 3, *input_size) + self.encoder(test_input) + + self.decoder = Decoder( + self.encoder.hooks, + nf=512, + last_norm='Spectral', + num_queries=num_queries, + num_scales=3, + dec_layers=9, + ) + self.refine_net = nn.Sequential( + custom_conv_layer( + num_queries + 3, + 2, + ks=1, + use_activ=False, + norm_type=NormType.Spectral)) + + self.register_buffer( + 'mean', + torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) + self.register_buffer( + 'std', + torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) + + def normalize(self, img): + return (img - self.mean) / self.std + + def forward(self, img): + if img.shape[1] == 3: + img = self.normalize(img) + + self.encoder(img) + out_feat = self.decoder() + coarse_input = torch.cat([out_feat, img], dim=1) + out = self.refine_net(coarse_input) + + return out + + +class Decoder(nn.Module): + + def __init__(self, + hooks, + nf=512, + blur=True, + last_norm='Spectral', + num_queries=100, + num_scales=3, + dec_layers=9): + super().__init__() + self.hooks = hooks + self.nf = nf + self.blur = blur + self.last_norm = getattr(NormType, last_norm) + + self.layers = self.make_layers() + embed_dim = nf // 2 + + self.last_shuf = CustomPixelShuffle_ICNR( + embed_dim, + embed_dim, + blur=self.blur, + norm_type=self.last_norm, + scale=4) + + self.color_decoder = MultiScaleColorDecoder( + in_channels=[512, 512, 256], + num_queries=num_queries, + num_scales=num_scales, + dec_layers=dec_layers, + ) + + def forward(self): + encode_feat = self.hooks[-1].feature + out0 = self.layers[0](encode_feat) + out1 = self.layers[1](out0) + out2 = self.layers[2](out1) + out3 = self.last_shuf(out2) + out = self.color_decoder([out0, out1, out2], out3) + + return out + + def make_layers(self): + decoder_layers = [] + + e_in_c = self.hooks[-1].feature.shape[1] + in_c = e_in_c + + out_c = self.nf + setup_hooks = self.hooks[-2::-1] + for layer_index, hook in enumerate(setup_hooks): + feature_c = hook.feature.shape[1] + if layer_index == len(setup_hooks) - 1: + out_c = out_c // 2 + decoder_layers.append( + UnetBlockWide( + in_c, + feature_c, + out_c, + hook, + blur=self.blur, + self_attention=False, + norm_type=NormType.Spectral)) + in_c = out_c + return nn.Sequential(*decoder_layers) + + +class Encoder(nn.Module): + + def __init__(self, encoder_name, hook_names, **kwargs): + super().__init__() + if encoder_name == 'convnext-t' or encoder_name == 'convnext': + self.arch = ConvNeXt() + elif encoder_name == 'convnext-s': + self.arch = ConvNeXt( + depths=[3, 3, 27, 3], dims=[96, 192, 384, 768]) + elif encoder_name == 'convnext-b': + self.arch = ConvNeXt( + depths=[3, 3, 27, 3], dims=[128, 256, 512, 1024]) + elif encoder_name == 'convnext-l': + self.arch = ConvNeXt( + depths=[3, 3, 27, 3], dims=[192, 384, 768, 1536]) + else: + raise NotImplementedError + + self.hook_names = hook_names + self.hooks = self.setup_hooks() + + def setup_hooks(self): + hooks = [Hook(self.arch._modules[name]) for name in self.hook_names] + return hooks + + def forward(self, img): + return self.arch.forward_features(img) + + +class MultiScaleColorDecoder(nn.Module): + + def __init__(self, + in_channels, + hidden_dim=256, + num_queries=100, + nheads=8, + dim_feedforward=2048, + dec_layers=9, + pre_norm=False, + color_embed_dim=256, + enforce_input_project=True, + num_scales=3): + super().__init__() + + # positional encoding + N_steps = hidden_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + # define Transformer decoder + self.num_heads = nheads + self.num_layers = dec_layers + self.transformer_self_attention_layers = nn.ModuleList() + self.transformer_cross_attention_layers = nn.ModuleList() + self.transformer_ffn_layers = nn.ModuleList() + + for _ in range(self.num_layers): + self.transformer_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + )) + self.transformer_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + )) + self.transformer_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm, + )) + + self.decoder_norm = nn.LayerNorm(hidden_dim) + + self.num_queries = num_queries + # learnable color query features + self.query_feat = nn.Embedding(num_queries, hidden_dim) + # learnable color query p.e. + self.query_embed = nn.Embedding(num_queries, hidden_dim) + + # level embedding + self.num_feature_levels = num_scales + self.level_embed = nn.Embedding(self.num_feature_levels, hidden_dim) + + # input projections + self.input_proj = nn.ModuleList() + for i in range(self.num_feature_levels): + if in_channels[i] != hidden_dim or enforce_input_project: + self.input_proj.append( + nn.Conv2d(in_channels[i], hidden_dim, kernel_size=1)) + nn.init.kaiming_uniform_(self.input_proj[-1].weight, a=1) + if self.input_proj[-1].bias is not None: + nn.init.constant_(self.input_proj[-1].bias, 0) + else: + self.input_proj.append(nn.Sequential()) + + # output FFNs + self.color_embed = MLP(hidden_dim, hidden_dim, color_embed_dim, 3) + + def forward(self, feature_pyramid, last_img_feature): + assert len(feature_pyramid) == self.num_feature_levels + src, pos = [], [] + + for i in range(self.num_feature_levels): + pos.append(self.pe_layer(feature_pyramid[i], None).flatten(2)) + src.append(self.input_proj[i](feature_pyramid[i]).flatten(2) + + self.level_embed.weight[i][None, :, None]) + + # flatten NxCxHxW to HWxNxC + pos[-1] = pos[-1].permute(2, 0, 1) + src[-1] = src[-1].permute(2, 0, 1) + + _, bs, _ = src[0].shape + + # QxNxC + query_embed = self.query_embed.weight.unsqueeze(1).repeat(1, bs, 1) + output = self.query_feat.weight.unsqueeze(1).repeat(1, bs, 1) + + for i in range(self.num_layers): + level_index = i % self.num_feature_levels + # attention: cross-attention first + output = self.transformer_cross_attention_layers[i]( + output, + src[level_index], + memory_mask=None, + memory_key_padding_mask=None, + pos=pos[level_index], + query_pos=query_embed) + output = self.transformer_self_attention_layers[i]( + output, + tgt_mask=None, + tgt_key_padding_mask=None, + query_pos=query_embed) + # FFN + output = self.transformer_ffn_layers[i](output) + + decoder_output = self.decoder_norm(output) + decoder_output = decoder_output.transpose( + 0, 1) # [N, bs, C] -> [bs, N, C] + color_embed = self.color_embed(decoder_output) + out = torch.einsum('bqc,bchw->bqhw', color_embed, last_img_feature) + + return out diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor_for_image_colorization.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor_for_image_colorization.py new file mode 100644 index 0000000000000000000000000000000000000000..e4479f13eb643b2f28165e510fdd4b2bf035cb96 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/ddcolor_for_image_colorization.py @@ -0,0 +1,173 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from copy import deepcopy +from typing import Dict, Union + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .ddcolor import DDColor +from .loss import L1Loss + +logger = get_logger() + +__all__ = ['DDColorForImageColorization'] + + +def tensor_lab2rgb(labs, illuminant='D65', observer='2'): + """ + Args: + lab : (B, C, H, W) + Returns: + tuple : (C, H, W) + """ + illuminants = \ + {'A': {'2': (1.098466069456375, 1, 0.3558228003436005), + '10': (1.111420406956693, 1, 0.3519978321919493)}, + 'D50': {'2': (0.9642119944211994, 1, 0.8251882845188288), + '10': (0.9672062750333777, 1, 0.8142801513128616)}, + 'D55': {'2': (0.956797052643698, 1, 0.9214805860173273), + '10': (0.9579665682254781, 1, 0.9092525159847462)}, + 'D65': {'2': (0.95047, 1., 1.08883), # This was: `lab_ref_white` + '10': (0.94809667673716, 1, 1.0730513595166162)}, + 'D75': {'2': (0.9497220898840717, 1, 1.226393520724154), + '10': (0.9441713925645873, 1, 1.2064272211720228)}, + 'E': {'2': (1.0, 1.0, 1.0), + '10': (1.0, 1.0, 1.0)}} + rgb_from_xyz = np.array([[3.240481340, -0.96925495, 0.055646640], + [-1.53715152, 1.875990000, -0.20404134], + [-0.49853633, 0.041555930, 1.057311070]]) + B, C, H, W = labs.shape + arrs = labs.permute( + (0, 2, 3, 1)).contiguous() # (B, 3, H, W) -> (B, H, W, 3) + L, a, b = arrs[:, :, :, 0:1], arrs[:, :, :, 1:2], arrs[:, :, :, 2:] + y = (L + 16.) / 116. + x = (a / 500.) + y + z = y - (b / 200.) + invalid = z.data < 0 + z[invalid] = 0 + xyz = torch.cat([x, y, z], dim=3) + mask = xyz.data > 0.2068966 + mask_xyz = xyz.clone() + mask_xyz[mask] = torch.pow(xyz[mask], 3.0) + mask_xyz[~mask] = (xyz[~mask] - 16.0 / 116.) / 7.787 + xyz_ref_white = illuminants[illuminant][observer] + for i in range(C): + mask_xyz[:, :, :, i] = mask_xyz[:, :, :, i] * xyz_ref_white[i] + + rgb_trans = torch.mm( + mask_xyz.view(-1, 3), + torch.from_numpy(rgb_from_xyz).type_as(xyz)).view(B, H, W, C) + rgb = rgb_trans.permute((0, 3, 1, 2)).contiguous() + mask = rgb.data > 0.0031308 + mask_rgb = rgb.clone() + mask_rgb[mask] = 1.055 * torch.pow(rgb[mask], 1 / 2.4) - 0.055 + mask_rgb[~mask] = rgb[~mask] * 12.92 + neg_mask = mask_rgb.data < 0 + large_mask = mask_rgb.data > 1 + mask_rgb[neg_mask] = 0 + mask_rgb[large_mask] = 1 + return mask_rgb + + +@MODELS.register_module(Tasks.image_colorization, module_name=Models.ddcolor) +class DDColorForImageColorization(TorchModel): + """DDColor model for Image Colorization: + Colorize an image using unet with dual decoders, + while the image decoder restores the spatial resolution, + and the color decoder learn adaptive color queries. + """ + + def __init__(self, + model_dir, + encoder_name='convnext-l', + input_size=(512, 512), + num_queries=100, + *args, + **kwargs): + """initialize the image colorization model from the `model_dir` path. + + Args: + model_dir (str): the model path. + encoder_name (str): the encoder name. + input_size (tuple): size of the model input image. + num_queries (int): number of decoder queries + """ + super().__init__(model_dir, *args, **kwargs) + + self.model = DDColor(encoder_name, input_size, num_queries) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + self.model = self._load_pretrained(self.model, model_path) + self.loss = L1Loss(loss_weight=0.1) + + def _load_pretrained(self, + net, + load_path, + strict=True, + param_key='params'): + load_net = torch.load( + load_path, map_location=lambda storage, loc: storage) + if param_key is not None: + if param_key not in load_net and 'params' in load_net: + param_key = 'params' + logger.info( + f'Loading: {param_key} does not exist, use params.') + if param_key in load_net: + load_net = load_net[param_key] + logger.info( + f'Loading {net.__class__.__name__} model from {load_path}, with param key: [{param_key}].' + ) + # remove unnecessary 'module.' or 'model.' + for k, v in deepcopy(load_net).items(): + if k.startswith('module.'): + load_net[k[7:]] = v + load_net.pop(k) + elif k.startswith('model.'): + load_net[k[6:]] = v + load_net.pop(k) + net.load_state_dict(load_net, strict=strict) + logger.info('load model done.') + return net + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + preds = self.model(input) + return {'loss': self.loss(preds, target)} + + def _evaluate_postprocess(self, input: Tensor, target: Tensor, + img_l: Tensor, + gt_rgb: Tensor) -> Dict[str, list]: + preds = self.model(input) # (n, 2, h, w) + + preds_lab = torch.cat((img_l, preds), 1) # (n, 3, h, w) + preds_rgb = tensor_lab2rgb(preds_lab) + + # preds = list(torch.split(preds_rgb, 1, 0)) + # targets = list(torch.split(gt_rgb, 1, 0)) + preds = preds_rgb + targets = gt_rgb + + return {'preds': preds, 'targets': targets} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result of the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**input) + elif 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self.model(**input) diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/loss.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..db2be54e7cc1a352b05ed590cd116ab7358dab79 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/loss.py @@ -0,0 +1,270 @@ +# The implementation here is modified based on BasicSR, originally Apache 2.0 license and publicly available at +# https://github.com/XPixelGroup/BasicSR/blob/master/basicsr/losses/basic_loss.py + +import torch +from torch import nn as nn +from torch.nn import functional as F + +from .utils.vgg import VGGFeatureExtractor + + +def l1_loss(pred, target, reduction): + return F.l1_loss(pred, target, reduction=reduction) + + +class L1Loss(nn.Module): + """L1 (mean absolute error, MAE) loss. + + Args: + loss_weight (float): Loss weight for L1 loss. Default: 1.0. + reduction (str): Specifies the reduction to apply to the output. + Supported choices are 'none' | 'mean' | 'sum'. Default: 'mean'. + """ + + def __init__(self, loss_weight=1.0, reduction='mean'): + super(L1Loss, self).__init__() + if reduction not in ['none', 'mean', 'sum']: + raise ValueError( + f'Unsupported reduction mode: {reduction}. Supported ones are: {_reduction_modes}' + ) + + self.loss_weight = loss_weight + self.reduction = reduction + + def forward(self, pred, target, weight=None, **kwargs): + """ + Args: + pred (Tensor): of shape (N, C, H, W). Predicted tensor. + target (Tensor): of shape (N, C, H, W). Ground truth tensor. + weight (Tensor, optional): of shape (N, C, H, W). Element-wise + weights. Default: None. + """ + return self.loss_weight * l1_loss( + pred, target, reduction=self.reduction) + + +class PerceptualLoss(nn.Module): + """Perceptual loss with commonly used style loss. + + Args: + layer_weights (dict): The weight for each layer of vgg feature. + Here is an example: {'conv5_4': 1.}, which means the conv5_4 + feature layer (before relu5_4) will be extracted with weight + 1.0 in calculating losses. + vgg_type (str): The type of vgg network used as feature extractor. + Default: 'vgg19'. + use_input_norm (bool): If True, normalize the input image in vgg. + Default: True. + range_norm (bool): If True, norm images with range [-1, 1] to [0, 1]. + Default: False. + perceptual_weight (float): If `perceptual_weight > 0`, the perceptual + loss will be calculated and the loss will multiplied by the + weight. Default: 1.0. + style_weight (float): If `style_weight > 0`, the style loss will be + calculated and the loss will multiplied by the weight. + Default: 0. + criterion (str): Criterion used for perceptual loss. Default: 'l1'. + """ + + def __init__(self, + layer_weights, + vgg_type='vgg19', + use_input_norm=True, + range_norm=False, + perceptual_weight=1.0, + style_weight=0., + criterion='l1'): + super(PerceptualLoss, self).__init__() + self.perceptual_weight = perceptual_weight + self.style_weight = style_weight + self.layer_weights = layer_weights + self.vgg = VGGFeatureExtractor( + layer_name_list=list(layer_weights.keys()), + vgg_type=vgg_type, + use_input_norm=use_input_norm, + range_norm=range_norm) + + self.criterion_type = criterion + if self.criterion_type == 'l1': + self.criterion = torch.nn.L1Loss() + elif self.criterion_type == 'l2': + self.criterion = torch.nn.L2loss() + elif self.criterion_type == 'fro': + self.criterion = None + else: + raise NotImplementedError( + f'{criterion} criterion has not been supported.') + + def forward(self, x, gt): + """Forward function. + + Args: + x (Tensor): Input tensor with shape (n, c, h, w). + gt (Tensor): Ground-truth tensor with shape (n, c, h, w). + + Returns: + Tensor: Forward results. + """ + # extract vgg features + x_features = self.vgg(x) + gt_features = self.vgg(gt.detach()) + + # calculate perceptual loss + if self.perceptual_weight > 0: + percep_loss = 0 + for k in x_features.keys(): + if self.criterion_type == 'fro': + percep_loss += torch.norm( + x_features[k] - gt_features[k], + p='fro') * self.layer_weights[k] + else: + percep_loss += self.criterion( + x_features[k], gt_features[k]) * self.layer_weights[k] + percep_loss *= self.perceptual_weight + else: + percep_loss = None + + # calculate style loss + if self.style_weight > 0: + style_loss = 0 + for k in x_features.keys(): + if self.criterion_type == 'fro': + style_loss += torch.norm( + self._gram_mat(x_features[k]) + - self._gram_mat(gt_features[k]), + p='fro') * self.layer_weights[k] + else: + style_loss += self.criterion( + self._gram_mat(x_features[k]), + self._gram_mat(gt_features[k])) * self.layer_weights[k] + style_loss *= self.style_weight + else: + style_loss = None + + return percep_loss, style_loss + + def _gram_mat(self, x): + """Calculate Gram matrix. + + Args: + x (torch.Tensor): Tensor with shape of (n, c, h, w). + + Returns: + torch.Tensor: Gram matrix. + """ + n, c, h, w = x.size() + features = x.view(n, c, w * h) + features_t = features.transpose(1, 2) + gram = features.bmm(features_t) / (c * h * w) + return gram + + +class GANLoss(nn.Module): + """Define GAN loss. + + Args: + gan_type (str): Support 'vanilla', 'lsgan', 'wgan', 'hinge'. + real_label_val (float): The value for real label. Default: 1.0. + fake_label_val (float): The value for fake label. Default: 0.0. + loss_weight (float): Loss weight. Default: 1.0. + Note that loss_weight is only for generators; and it is always 1.0 + for discriminators. + """ + + def __init__(self, + gan_type, + real_label_val=1.0, + fake_label_val=0.0, + loss_weight=1.0): + super(GANLoss, self).__init__() + self.gan_type = gan_type + self.loss_weight = loss_weight + self.real_label_val = real_label_val + self.fake_label_val = fake_label_val + + if self.gan_type == 'vanilla': + self.loss = nn.BCEWithLogitsLoss() + elif self.gan_type == 'lsgan': + self.loss = nn.MSELoss() + elif self.gan_type == 'wgan': + self.loss = self._wgan_loss + elif self.gan_type == 'wgan_softplus': + self.loss = self._wgan_softplus_loss + elif self.gan_type == 'hinge': + self.loss = nn.ReLU() + else: + raise NotImplementedError( + f'GAN type {self.gan_type} is not implemented.') + + def _wgan_loss(self, input, target): + """wgan loss. + + Args: + input (Tensor): Input tensor. + target (bool): Target label. + + Returns: + Tensor: wgan loss. + """ + return -input.mean() if target else input.mean() + + def _wgan_softplus_loss(self, input, target): + """wgan loss with soft plus. softplus is a smooth approximation to the + ReLU function. + + In StyleGAN2, it is called: + Logistic loss for discriminator; + Non-saturating loss for generator. + + Args: + input (Tensor): Input tensor. + target (bool): Target label. + + Returns: + Tensor: wgan loss. + """ + return F.softplus(-input).mean() if target else F.softplus( + input).mean() + + def get_target_label(self, input, target_is_real): + """Get target label. + + Args: + input (Tensor): Input tensor. + target_is_real (bool): Whether the target is real or fake. + + Returns: + (bool | Tensor): Target tensor. Return bool for wgan, otherwise, + return Tensor. + """ + + if self.gan_type in ['wgan', 'wgan_softplus']: + return target_is_real + target_val = ( + self.real_label_val if target_is_real else self.fake_label_val) + return input.new_ones(input.size()) * target_val + + def forward(self, input, target_is_real, is_disc=False): + """ + Args: + input (Tensor): The input for the loss module, i.e., the network + prediction. + target_is_real (bool): Whether the targe is real or fake. + is_disc (bool): Whether the loss for discriminators or not. + Default: False. + + Returns: + Tensor: GAN loss value. + """ + target_label = self.get_target_label(input, target_is_real) + if self.gan_type == 'hinge': + if is_disc: # for discriminators in hinge-gan + input = -input if target_is_real else input + loss = self.loss(1 + input).mean() + else: # for generators in hinge-gan + loss = -input.mean() + else: # other gan types + loss = self.loss(input, target_label) + + # loss_weight is always 1.0 for discriminators + return loss if is_disc else loss * self.loss_weight diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/__init__.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/convnext.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/convnext.py new file mode 100644 index 0000000000000000000000000000000000000000..d0be934feaaf5f066c0aa7b835286a747575d465 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/convnext.py @@ -0,0 +1,177 @@ +# The implementation here is modified based on ConvNeXt, originally MIT license +# and publicly available at https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath, trunc_normal_ + + +class Block(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + self.dwconv = nn.Conv2d( + dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv + self.norm = LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, + 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) + + x = input + self.drop_path(x) + return x + + +class ConvNeXt(nn.Module): + r""" ConvNeXt + A PyTorch impl of : `A ConvNet for the 2020s` - + https://arxiv.org/pdf/2201.03545.pdf + Args: + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3] + dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768] + drop_path_rate (float): Stochastic depth rate. Default: 0. + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1. + """ + + def __init__( + self, + in_chans=3, + num_classes=1000, + depths=[3, 3, 9, 3], + dims=[96, 192, 384, 768], + drop_path_rate=0., + layer_scale_init_value=1e-6, + head_init_scale=1., + ): + super().__init__() + + self.downsample_layers = nn.ModuleList( + ) # stem and 3 intermediate downsampling conv layers + stem = nn.Sequential( + nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4), + LayerNorm(dims[0], eps=1e-6, data_format='channels_first')) + self.downsample_layers.append(stem) + for i in range(3): + downsample_layer = nn.Sequential( + LayerNorm(dims[i], eps=1e-6, data_format='channels_first'), + nn.Conv2d(dims[i], dims[i + 1], kernel_size=2, stride=2), + ) + self.downsample_layers.append(downsample_layer) + + self.stages = nn.ModuleList( + ) # 4 feature resolution stages, each consisting of multiple residual blocks + dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + cur = 0 + for i in range(4): + stage = nn.Sequential(*[ + Block( + dim=dims[i], + drop_path=dp_rates[cur + j], + layer_scale_init_value=layer_scale_init_value) + for j in range(depths[i]) + ]) + self.stages.append(stage) + cur += depths[i] + + # add norm layers for each output + out_indices = (0, 1, 2, 3) + for i in out_indices: + layer = LayerNorm(dims[i], eps=1e-6, data_format='channels_first') + # layer = nn.Identity() + layer_name = f'norm{i}' + self.add_module(layer_name, layer) + + self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer + self.head_cls = nn.Linear(dims[-1], 4) + + self.apply(self._init_weights) + # self.head_cls.weight.data.mul_(head_init_scale) + # self.head_cls.bias.data.mul_(head_init_scale) + + def _init_weights(self, m): + if isinstance(m, (nn.Conv2d, nn.Linear)): + trunc_normal_(m.weight, std=.02) + nn.init.constant_(m.bias, 0) + + def forward_features(self, x): + for i in range(4): + x = self.downsample_layers[i](x) + x = self.stages[i](x) + + # add extra norm + norm_layer = getattr(self, f'norm{i}') + norm_layer(x) + + return self.norm(x.mean( + [-2, -1])) # global average pooling, (N, C, H, W) -> (N, C) + + def forward(self, x): + x = self.forward_features(x) + x = self.head_cls(x) + return x + + +class LayerNorm(nn.Module): + r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with + shape (batch_size, height, width, channels) while channels_first corresponds to inputs + with shape (batch_size, channels, height, width). + """ + + def __init__(self, + normalized_shape, + eps=1e-6, + data_format='channels_last'): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ['channels_last', 'channels_first']: + raise NotImplementedError + self.normalized_shape = (normalized_shape, ) + + def forward(self, x): + if self.data_format == 'channels_last': # B H W C + return F.layer_norm(x, self.normalized_shape, self.weight, + self.bias, self.eps) + elif self.data_format == 'channels_first': # B C H W + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/position_encoding.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..e613d15b6ffe3a325f382da4cc841fab50ef17d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/position_encoding.py @@ -0,0 +1,57 @@ +# The implementation here is modified based on Mask2Former, originally MIT license and publicly available at +# https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/transformer_decoder/position_encoding.py + +import math + +import torch +from torch import nn + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, + num_pos_feats=64, + temperature=10000, + normalize=False, + scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError('normalize should be True if scale is passed') + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, x, mask=None): + if mask is None: + mask = torch.zeros((x.size(0), x.size(2), x.size(3)), + device=x.device, + dtype=torch.bool) + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange( + self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature**(2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/transformer_utils.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/transformer_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..66f2d82150200e732755fe244fe1e30fb18c60fd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/transformer_utils.py @@ -0,0 +1,232 @@ +# The implementation here is modified based on Mask2Former, originally MIT license and publicly available at +# https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/transformer_decoder/mask2former_transformer_decoder.py + +from typing import Optional + +from torch import Tensor, nn +from torch.nn import functional as F + + +class SelfAttentionLayer(nn.Module): + + def __init__(self, + d_model, + nhead, + dropout=0.0, + activation='relu', + normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + + self.norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn( + q, + k, + value=tgt, + attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + + return tgt + + def forward_pre(self, + tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn( + q, + k, + value=tgt2, + attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + + return tgt + + def forward(self, + tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, tgt_mask, tgt_key_padding_mask, + query_pos) + return self.forward_post(tgt, tgt_mask, tgt_key_padding_mask, + query_pos) + + +class CrossAttentionLayer(nn.Module): + + def __init__(self, + d_model, + nhead, + dropout=0.0, + activation='relu', + normalize_before=False): + super().__init__() + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + + self.norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + tgt, + memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + + return tgt + + def forward_pre(self, + tgt, + memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm(tgt) + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + + return tgt + + def forward(self, + tgt, + memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, memory, memory_mask, + memory_key_padding_mask, pos, query_pos) + return self.forward_post(tgt, memory, memory_mask, + memory_key_padding_mask, pos, query_pos) + + +class FFNLayer(nn.Module): + + def __init__(self, + d_model, + dim_feedforward=2048, + dropout=0.0, + activation='relu', + normalize_before=False): + super().__init__() + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm = nn.LayerNorm(d_model) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, tgt): + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + return tgt + + def forward_pre(self, tgt): + tgt2 = self.norm(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout(tgt2) + return tgt + + def forward(self, tgt): + if self.normalize_before: + return self.forward_pre(tgt) + return self.forward_post(tgt) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == 'relu': + return F.relu + if activation == 'gelu': + return F.gelu + if activation == 'glu': + return F.glu + raise RuntimeError(F'activation should be relu/gelu, not {activation}.') + + +class MLP(nn.Module): + """ Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList( + nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/unet.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..2b8b9c146181a2a5c482a2dc258fff25e6e6ba41 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/unet.py @@ -0,0 +1,203 @@ +# The implementation here is modified based on DeOldify, originally MIT License +# and publicly available at https://github.com/jantic/DeOldify/blob/master/deoldify/unet.py + +import collections +from enum import Enum + +import torch +import torch.nn as nn +from torch.nn import functional as F + +NormType = Enum('NormType', 'Batch BatchZero Weight Spectral') + + +class Hook: + feature = None + + def __init__(self, module): + self.hook = module.register_forward_hook(self.hook_fn) + + def hook_fn(self, module, input, output): + if isinstance(output, torch.Tensor): + self.feature = output + elif isinstance(output, collections.OrderedDict): + self.feature = output['out'] + + def remove(self): + self.hook.remove() + + +class SelfAttention(nn.Module): + 'Self attention layer for nd.' + + def __init__(self, n_channels: int): + super().__init__() + self.query = conv1d(n_channels, n_channels // 8) + self.key = conv1d(n_channels, n_channels // 8) + self.value = conv1d(n_channels, n_channels) + self.gamma = nn.Parameter(torch.tensor([0.])) + + def forward(self, x): + # Notation from https://arxiv.org/pdf/1805.08318.pdf + size = x.size() + x = x.view(*size[:2], -1) + f, g, h = self.query(x), self.key(x), self.value(x) + beta = F.softmax(torch.bmm(f.permute(0, 2, 1).contiguous(), g), dim=1) + o = self.gamma * torch.bmm(h, beta) + x + return o.view(*size).contiguous() + + +def batchnorm_2d(nf: int, norm_type: NormType = NormType.Batch): + 'A batchnorm2d layer with `nf` features initialized depending on `norm_type`.' + bn = nn.BatchNorm2d(nf) + with torch.no_grad(): + bn.bias.fill_(1e-3) + bn.weight.fill_(0. if norm_type == NormType.BatchZero else 1.) + return bn + + +def init_default(m: nn.Module, func=nn.init.kaiming_normal_) -> None: + 'Initialize `m` weights with `func` and set `bias` to 0.' + if func: + if hasattr(m, 'weight'): + func(m.weight) + if hasattr(m, 'bias') and hasattr(m.bias, 'data'): + m.bias.data.fill_(0.) + return m + + +def icnr(x, scale=2, init=nn.init.kaiming_normal_): + 'ICNR init of `x`, with `scale` and `init` function.' + ni, nf, h, w = x.shape + ni2 = int(ni / (scale**2)) + k = init(torch.zeros([ni2, nf, h, w])).transpose(0, 1) + k = k.contiguous().view(ni2, nf, -1) + k = k.repeat(1, 1, scale**2) + k = k.contiguous().view([nf, ni, h, w]).transpose(0, 1) + x.data.copy_(k) + + +def conv1d(ni: int, + no: int, + ks: int = 1, + stride: int = 1, + padding: int = 0, + bias: bool = False): + 'Create and initialize a `nn.Conv1d` layer with spectral normalization.' + conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias) + nn.init.kaiming_normal_(conv.weight) + if bias: + conv.bias.data.zero_() + return nn.utils.spectral_norm(conv) + + +def custom_conv_layer( + ni: int, + nf: int, + ks: int = 3, + stride: int = 1, + padding: int = None, + bias: bool = None, + is_1d: bool = False, + norm_type=NormType.Batch, + use_activ: bool = True, + transpose: bool = False, + init=nn.init.kaiming_normal_, + self_attention: bool = False, + extra_bn: bool = False, +): + 'Create a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and batchnorm (if `bn`) layers.' + if padding is None: + padding = (ks - 1) // 2 if not transpose else 0 + bn = norm_type in (NormType.Batch, NormType.BatchZero) or extra_bn + if bias is None: + bias = not bn + conv_func = nn.ConvTranspose2d if transpose else nn.Conv1d if is_1d else nn.Conv2d + conv = init_default( + conv_func( + ni, nf, kernel_size=ks, bias=bias, stride=stride, padding=padding), + init, + ) + + if norm_type == NormType.Weight: + conv = nn.utils.weight_norm(conv) + elif norm_type == NormType.Spectral: + conv = nn.utils.spectral_norm(conv) + layers = [conv] + if use_activ: + layers.append(nn.ReLU(True)) + if bn: + layers.append((nn.BatchNorm1d if is_1d else nn.BatchNorm2d)(nf)) + if self_attention: + layers.append(SelfAttention(nf)) + return nn.Sequential(*layers) + + +class CustomPixelShuffle_ICNR(nn.Module): + """ + Upsample by `scale` from `ni` filters to `nf` (default `ni`), + using `nn.PixelShuffle`, `icnr` init, and `weight_norm`. + """ + + def __init__(self, + ni: int, + nf: int = None, + scale: int = 2, + blur: bool = True, + norm_type=NormType.Spectral, + extra_bn=False): + super().__init__() + self.conv = custom_conv_layer( + ni, + nf * (scale**2), + ks=1, + use_activ=False, + norm_type=norm_type, + extra_bn=extra_bn) + icnr(self.conv[0].weight) + self.shuf = nn.PixelShuffle(scale) + self.do_blur = blur + # Blurring over (h*w) kernel + # "Super-Resolution using Convolutional Neural Networks without Any Checkerboard Artifacts" + # - https://arxiv.org/abs/1806.02658 + self.pad = nn.ReplicationPad2d((1, 0, 1, 0)) + self.blur = nn.AvgPool2d(2, stride=1) + self.relu = nn.ReLU(True) + + def forward(self, x): + x = self.shuf(self.relu(self.conv(x))) + return self.blur(self.pad(x)) if self.do_blur else x + + +class UnetBlockWide(nn.Module): + 'A quasi-UNet block, using `PixelShuffle_ICNR upsampling`.' + + def __init__(self, + up_in_c: int, + x_in_c: int, + n_out: int, + hook, + blur: bool = False, + self_attention: bool = False, + norm_type=NormType.Spectral): + super().__init__() + + self.hook = hook + up_out = n_out + self.shuf = CustomPixelShuffle_ICNR( + up_in_c, up_out, blur=blur, norm_type=norm_type, extra_bn=True) + self.bn = batchnorm_2d(x_in_c) + ni = up_out + x_in_c + self.conv = custom_conv_layer( + ni, + n_out, + norm_type=norm_type, + self_attention=self_attention, + extra_bn=True) + self.relu = nn.ReLU() + + def forward(self, up_in): + s = self.hook.feature + up_out = self.shuf(up_in) + cat_x = self.relu(torch.cat([up_out, self.bn(s)], dim=1)) + return self.conv(cat_x) diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/vgg.py b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/vgg.py new file mode 100644 index 0000000000000000000000000000000000000000..dc0125c260a60cbb9e01d7c2b29b83e869b4013a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/ddcolor/utils/vgg.py @@ -0,0 +1,180 @@ +# The implementation here is modified based on BasicSR, originally Apache 2.0 license and publicly available at +# https://github.com/XPixelGroup/BasicSR/blob/master/basicsr/archs/vgg_arch.py + +import os +from collections import OrderedDict + +import torch +from torch import nn as nn +from torchvision.models import vgg as vgg + +VGG_PRETRAIN_PATH = { + 'vgg19': './pretrain/vgg19-dcbb9e9d.pth', + 'vgg16_bn': './pretrain/vgg16_bn-6c64b313.pth' +} + +NAMES = { + 'vgg11': [ + 'conv1_1', 'relu1_1', 'pool1', 'conv2_1', 'relu2_1', 'pool2', + 'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'pool3', 'conv4_1', + 'relu4_1', 'conv4_2', 'relu4_2', 'pool4', 'conv5_1', 'relu5_1', + 'conv5_2', 'relu5_2', 'pool5' + ], + 'vgg13': [ + 'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1', + 'relu2_1', 'conv2_2', 'relu2_2', 'pool2', 'conv3_1', 'relu3_1', + 'conv3_2', 'relu3_2', 'pool3', 'conv4_1', 'relu4_1', 'conv4_2', + 'relu4_2', 'pool4', 'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'pool5' + ], + 'vgg16': [ + 'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1', + 'relu2_1', 'conv2_2', 'relu2_2', 'pool2', 'conv3_1', 'relu3_1', + 'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'pool3', 'conv4_1', + 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3', 'relu4_3', 'pool4', + 'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', + 'pool5' + ], + 'vgg19': [ + 'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', 'conv2_1', + 'relu2_1', 'conv2_2', 'relu2_2', 'pool2', 'conv3_1', 'relu3_1', + 'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4', + 'pool3', 'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3', + 'relu4_3', 'conv4_4', 'relu4_4', 'pool4', 'conv5_1', 'relu5_1', + 'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4', + 'pool5' + ] +} + + +def insert_bn(names): + """Insert bn layer after each conv. + + Args: + names (list): The list of layer names. + + Returns: + list: The list of layer names with bn layers. + """ + names_bn = [] + for name in names: + names_bn.append(name) + if 'conv' in name: + position = name.replace('conv', '') + names_bn.append('bn' + position) + return names_bn + + +class VGGFeatureExtractor(nn.Module): + """VGG network for feature extraction. + + In this implementation, we allow users to choose whether use normalization + in the input feature and the type of vgg network. Note that the pretrained + path must fit the vgg type. + + Args: + layer_name_list (list[str]): Forward function returns the corresponding + features according to the layer_name_list. + Example: {'relu1_1', 'relu2_1', 'relu3_1'}. + vgg_type (str): Set the type of vgg network. Default: 'vgg19'. + use_input_norm (bool): If True, normalize the input image. Importantly, + the input feature must in the range [0, 1]. Default: True. + range_norm (bool): If True, norm images with range [-1, 1] to [0, 1]. + Default: False. + requires_grad (bool): If true, the parameters of VGG network will be + optimized. Default: False. + remove_pooling (bool): If true, the max pooling operations in VGG net + will be removed. Default: False. + pooling_stride (int): The stride of max pooling operation. Default: 2. + """ + + def __init__(self, + layer_name_list, + vgg_type='vgg19', + use_input_norm=True, + range_norm=False, + requires_grad=False, + remove_pooling=False, + pooling_stride=2): + super(VGGFeatureExtractor, self).__init__() + + self.layer_name_list = layer_name_list + self.use_input_norm = use_input_norm + self.range_norm = range_norm + + self.names = NAMES[vgg_type.replace('_bn', '')] + if 'bn' in vgg_type: + self.names = insert_bn(self.names) + + # only borrow layers that will be used to avoid unused params + max_idx = 0 + for v in layer_name_list: + idx = self.names.index(v) + if idx > max_idx: + max_idx = idx + + if os.path.exists(VGG_PRETRAIN_PATH[vgg_type]): + vgg_net = getattr(vgg, vgg_type)(pretrained=False) + state_dict = torch.load( + VGG_PRETRAIN_PATH[vgg_type], + map_location=lambda storage, loc: storage) + vgg_net.load_state_dict(state_dict) + else: + vgg_net = getattr(vgg, vgg_type)(pretrained=True) + + features = vgg_net.features[:max_idx + 1] + + modified_net = OrderedDict() + for k, v in zip(self.names, features): + if 'pool' in k: + # if remove_pooling is true, pooling operation will be removed + if remove_pooling: + continue + else: + # in some cases, we may want to change the default stride + modified_net[k] = nn.MaxPool2d( + kernel_size=2, stride=pooling_stride) + else: + modified_net[k] = v + + self.vgg_net = nn.Sequential(modified_net) + + if not requires_grad: + self.vgg_net.eval() + for param in self.parameters(): + param.requires_grad = False + else: + self.vgg_net.train() + for param in self.parameters(): + param.requires_grad = True + + if self.use_input_norm: + # the mean is for image with range [0, 1] + self.register_buffer( + 'mean', + torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) + # the std is for image with range [0, 1] + self.register_buffer( + 'std', + torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) + + def forward(self, x): + """Forward function. + + Args: + x (Tensor): Input tensor with shape (n, c, h, w). + + Returns: + Tensor: Forward results. + """ + if self.range_norm: + x = (x + 1) / 2 + if self.use_input_norm: + x = (x - self.mean) / self.std + + output = {} + for key, layer in self.vgg_net._modules.items(): + x = layer(x) + if key in self.layer_name_list: + output[key] = x.clone() + + return output diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/unet/__init__.py b/runtime/Lib/site_packages/models/cv/image_colorization/unet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..66879422f74f8c5eaad9af7e5b963632b41ada7e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/unet/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .unet import DynamicUnetDeep, DynamicUnetWide +from .utils import NormType diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/unet/unet.py b/runtime/Lib/site_packages/models/cv/image_colorization/unet/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..19f6ab6279cf5eb2841ec556c53f07a7ba4bb018 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/unet/unet.py @@ -0,0 +1,302 @@ +# The implementation here is modified based on DeOldify, originally MIT License +# and publicly available at https://github.com/jantic/DeOldify/blob/master/deoldify/unet.py +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils import spectral_norm, weight_norm + +from .utils import (MergeLayer, NormType, PixelShuffle_ICNR, SelfAttention, + SequentialEx, SigmoidRange, dummy_eval, hook_outputs, + in_channels, model_sizes, relu, res_block) + +__all__ = ['DynamicUnetDeep', 'DynamicUnetWide'] + + +def custom_conv_layer( + ni, + nf, + ks=3, + stride=1, + padding=None, + bias=None, + is_1d=False, + norm_type=NormType.Batch, + use_activ=True, + leaky=None, + transpose=False, + init=nn.init.kaiming_normal_, + self_attention=False, + extra_bn=False, +): + 'Create a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and batchnorm (if `bn`) layers.' + if padding is None: + padding = (ks - 1) // 2 if not transpose else 0 + bn = norm_type in (NormType.Batch, NormType.BatchZero) or extra_bn is True + if bias is None: + bias = not bn + conv_func = nn.ConvTranspose2d if transpose is True else nn.Conv1d + conv_func = conv_func if is_1d else nn.Conv2d + conv = conv_func( + ni, nf, kernel_size=ks, bias=bias, stride=stride, padding=padding) + if norm_type == NormType.Weight: + conv = weight_norm(conv) + elif norm_type == NormType.Spectral: + conv = spectral_norm(conv) + + layers = [conv] + if use_activ: + layers.append(relu(True, leaky=leaky)) + if bn: + layers.append((nn.BatchNorm1d if is_1d else nn.BatchNorm2d)(nf)) + if self_attention: + layers.append(SelfAttention(nf)) + return nn.Sequential(*layers) + + +def _get_sfs_idxs(sizes): + 'Get the indexes of the layers where the size of the activation changes.' + feature_szs = [size[-1] for size in sizes] + sfs_idxs = list( + np.where(np.array(feature_szs[:-1]) != np.array(feature_szs[1:]))[0]) + if feature_szs[0] != feature_szs[1]: + sfs_idxs = [0] + sfs_idxs + return sfs_idxs + + +class CustomPixelShuffle_ICNR(nn.Module): + 'Upsample by `scale` from `ni` filters to `nf` (default `ni`), using `nn.PixelShuffle`, and `weight_norm`.' + + def __init__(self, ni, nf=None, scale=2, blur=False, leaky=None, **kwargs): + super().__init__() + nf = ni if nf is None else nf + self.conv = custom_conv_layer( + ni, nf * (scale**2), ks=1, use_activ=False, **kwargs) + self.shuf = nn.PixelShuffle(scale) + # Blurring over (h*w) kernel + # "Super-Resolution using Convolutional Neural Networks without Any Checkerboard Artifacts" + # - https://arxiv.org/abs/1806.02658 + self.pad = nn.ReplicationPad2d((1, 0, 1, 0)) + self.blur = nn.AvgPool2d(2, stride=1) + self.relu = relu(True, leaky=leaky) + + def forward(self, x): + x = self.shuf(self.relu(self.conv(x))) + return self.blur(self.pad(x)) if self.blur else x + + +class UnetBlockDeep(nn.Module): + 'A quasi-UNet block, using `PixelShuffle_ICNR upsampling`.' + + def __init__(self, + up_in_c, + x_in_c, + hook, + final_div=True, + blur=False, + leaky=None, + self_attention=False, + nf_factor=1.0, + **kwargs): + super().__init__() + self.hook = hook + self.shuf = CustomPixelShuffle_ICNR( + up_in_c, up_in_c // 2, blur=blur, leaky=leaky, **kwargs) + self.bn = nn.BatchNorm2d(x_in_c) + ni = up_in_c // 2 + x_in_c + nf = int((ni if final_div else ni // 2) * nf_factor) + self.conv1 = custom_conv_layer(ni, nf, leaky=leaky, **kwargs) + self.conv2 = custom_conv_layer( + nf, nf, leaky=leaky, self_attention=self_attention, **kwargs) + self.relu = relu(leaky=leaky) + + def forward(self, up_in): + s = self.hook.stored + up_out = self.shuf(up_in) + ssh = s.shape[-2:] + if ssh != up_out.shape[-2:]: + up_out = F.interpolate(up_out, s.shape[-2:], mode='nearest') + cat_x = self.relu(torch.cat([up_out, self.bn(s)], dim=1)) + return self.conv2(self.conv1(cat_x)) + + +class DynamicUnetDeep(SequentialEx): + 'Create a U-Net from a given architecture.' + + def __init__(self, + encoder, + n_classes, + blur=False, + blur_final=True, + self_attention=False, + y_range=None, + last_cross=True, + bottle=False, + norm_type=NormType.Batch, + nf_factor=1.0, + **kwargs): + extra_bn = norm_type == NormType.Spectral + imsize = (256, 256) + sfs_szs = model_sizes(encoder, size=imsize) + sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs))) + self.sfs = hook_outputs([encoder[i] for i in sfs_idxs], detach=False) + x = dummy_eval(encoder, imsize).detach() + + ni = sfs_szs[-1][1] + middle_conv = nn.Sequential( + custom_conv_layer( + ni, ni * 2, norm_type=norm_type, extra_bn=extra_bn, **kwargs), + custom_conv_layer( + ni * 2, ni, norm_type=norm_type, extra_bn=extra_bn, **kwargs), + ).eval() + x = middle_conv(x) + layers = [encoder, nn.BatchNorm2d(ni), nn.ReLU(), middle_conv] + + for i, idx in enumerate(sfs_idxs): + not_final = i != len(sfs_idxs) - 1 + up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1]) + sa = self_attention and (i == len(sfs_idxs) - 3) + unet_block = UnetBlockDeep( + up_in_c, + x_in_c, + self.sfs[i], + final_div=not_final, + blur=blur, + self_attention=sa, + norm_type=norm_type, + extra_bn=extra_bn, + nf_factor=nf_factor, + **kwargs).eval() + layers.append(unet_block) + x = unet_block(x) + + ni = x.shape[1] + if imsize != sfs_szs[0][-2:]: + layers.append(PixelShuffle_ICNR(ni, **kwargs)) + if last_cross: + layers.append(MergeLayer(dense=True)) + ni += in_channels(encoder) + layers.append( + res_block(ni, bottle=bottle, norm_type=norm_type, **kwargs)) + layers += [ + custom_conv_layer( + ni, n_classes, ks=1, use_activ=False, norm_type=norm_type) + ] + if y_range is not None: + layers.append(SigmoidRange(*y_range)) + super().__init__(*layers) + + def __del__(self): + if hasattr(self, 'sfs'): + self.sfs.remove() + + +# ------------------------------------------------------ +class UnetBlockWide(nn.Module): + 'A quasi-UNet block, using `PixelShuffle_ICNR upsampling`.' + + def __init__(self, + up_in_c, + x_in_c, + n_out, + hook, + final_div=True, + blur=False, + leaky=None, + self_attention=False, + **kwargs): + super().__init__() + self.hook = hook + up_out = x_out = n_out // 2 + self.shuf = CustomPixelShuffle_ICNR( + up_in_c, up_out, blur=blur, leaky=leaky, **kwargs) + self.bn = nn.BatchNorm2d(x_in_c) + ni = up_out + x_in_c + self.conv = custom_conv_layer( + ni, x_out, leaky=leaky, self_attention=self_attention, **kwargs) + self.relu = relu(leaky=leaky) + + def forward(self, up_in): + s = self.hook.stored + up_out = self.shuf(up_in) + ssh = s.shape[-2:] + if ssh != up_out.shape[-2:]: + up_out = F.interpolate(up_out, s.shape[-2:], mode='nearest') + cat_x = self.relu(torch.cat([up_out, self.bn(s)], dim=1)) + return self.conv(cat_x) + + +class DynamicUnetWide(SequentialEx): + 'Create a U-Net from a given architecture.' + + def __init__(self, + encoder, + n_classes, + blur=False, + blur_final=True, + self_attention=False, + y_range=None, + last_cross=True, + bottle=False, + norm_type=NormType.Batch, + nf_factor=1, + **kwargs): + + nf = 512 * nf_factor + extra_bn = norm_type == NormType.Spectral + imsize = (256, 256) + sfs_szs = model_sizes(encoder, size=imsize) + sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs))) + self.sfs = hook_outputs([encoder[i] for i in sfs_idxs], detach=False) + x = dummy_eval(encoder, imsize).detach() + + ni = sfs_szs[-1][1] + middle_conv = nn.Sequential( + custom_conv_layer( + ni, ni * 2, norm_type=norm_type, extra_bn=extra_bn, **kwargs), + custom_conv_layer( + ni * 2, ni, norm_type=norm_type, extra_bn=extra_bn, **kwargs), + ).eval() + x = middle_conv(x) + layers = [encoder, nn.BatchNorm2d(ni), nn.ReLU(), middle_conv] + + for i, idx in enumerate(sfs_idxs): + not_final = i != len(sfs_idxs) - 1 + up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1]) + sa = self_attention and (i == len(sfs_idxs) - 3) + + n_out = nf if not_final else nf // 2 + + unet_block = UnetBlockWide( + up_in_c, + x_in_c, + n_out, + self.sfs[i], + final_div=not_final, + blur=blur, + self_attention=sa, + norm_type=norm_type, + extra_bn=extra_bn, + **kwargs).eval() + layers.append(unet_block) + x = unet_block(x) + + ni = x.shape[1] + if imsize != sfs_szs[0][-2:]: + layers.append(PixelShuffle_ICNR(ni, **kwargs)) + if last_cross: + layers.append(MergeLayer(dense=True)) + ni += in_channels(encoder) + layers.append( + res_block(ni, bottle=bottle, norm_type=norm_type, **kwargs)) + layers += [ + custom_conv_layer( + ni, n_classes, ks=1, use_activ=False, norm_type=norm_type) + ] + if y_range is not None: + layers.append(SigmoidRange(*y_range)) + super().__init__(*layers) + + def __del__(self): + if hasattr(self, 'sfs'): + self.sfs.remove() diff --git a/runtime/Lib/site_packages/models/cv/image_colorization/unet/utils.py b/runtime/Lib/site_packages/models/cv/image_colorization/unet/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b8968aa023d2fd46b24a576545ac581084621cdb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_colorization/unet/utils.py @@ -0,0 +1,350 @@ +# The implementation here is modified based on DeOldify, originally MIT License and +# publicly available at https://github.com/jantic/DeOldify/blob/master/fastai/callbacks/hooks.py +import functools +from enum import Enum + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils import spectral_norm, weight_norm + +NormType = Enum('NormType', + 'Batch BatchZero Weight Spectral Group Instance SpectralGN') + + +def is_listy(x): + return isinstance(x, (tuple, list)) + + +class Hook(): + 'Create a hook on `m` with `hook_func`.' + + def __init__(self, m, hook_func, is_forward=True, detach=True): + self.hook_func, self.detach, self.stored = hook_func, detach, None + f = m.register_forward_hook if is_forward else m.register_backward_hook + self.hook = f(self.hook_fn) + self.removed = False + + def hook_fn(self, module, input, output): + 'Applies `hook_func` to `module`, `input`, `output`.' + if self.detach: + input = (o.detach() + for o in input) if is_listy(input) else input.detach() + output = ( + o.detach() + for o in output) if is_listy(output) else output.detach() + self.stored = self.hook_func(module, input, output) + + def remove(self): + 'Remove the hook from the model.' + if not self.removed: + self.hook.remove() + self.removed = True + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + self.remove() + + +class Hooks(): + 'Create several hooks on the modules in `ms` with `hook_func`.' + + def __init__(self, ms, hook_func, is_forward=True, detach=True): + self.hooks = [Hook(m, hook_func, is_forward, detach) for m in ms] + + def __getitem__(self, i): + return self.hooks[i] + + def __len__(self): + return len(self.hooks) + + def __iter__(self): + return iter(self.hooks) + + @property + def stored(self): + return [o.stored for o in self] + + def remove(self): + 'Remove the hooks from the model.' + for h in self.hooks: + h.remove() + + def __enter__(self, *args): + return self + + def __exit__(self, *args): + self.remove() + + +def _hook_inner(m, i, o): + return o if isinstance(o, torch.Tensor) else o if is_listy(o) else list(o) + + +def hook_outputs(modules, detach=True, grad=False): + 'Return `Hooks` that store activations of all `modules` in `self.stored`' + return Hooks(modules, _hook_inner, detach=detach, is_forward=not grad) + + +def one_param(m): + 'Return the first parameter of `m`.' + return next(m.parameters()) + + +def dummy_batch(m, size=(64, 64)): + 'Create a dummy batch to go through `m` with `size`.' + ch_in = in_channels(m) + return one_param(m).new(1, ch_in, + *size).requires_grad_(False).uniform_(-1., 1.) + + +def dummy_eval(m, size=(64, 64)): + 'Pass a `dummy_batch` in evaluation mode in `m` with `size`.' + return m.eval()(dummy_batch(m, size)) + + +def model_sizes(m, size=(64, 64)): + 'Pass a dummy input through the model `m` to get the various sizes of activations.' + with hook_outputs(m) as hooks: + dummy_eval(m, size) + return [o.stored.shape for o in hooks] + + +class PrePostInitMeta(type): + 'A metaclass that calls optional `__pre_init__` and `__post_init__` methods' + + def __new__(cls, name, bases, dct): + x = super().__new__(cls, name, bases, dct) + old_init = x.__init__ + + def _pass(self): + pass + + @functools.wraps(old_init) + def _init(self, *args, **kwargs): + self.__pre_init__() + old_init(self, *args, **kwargs) + self.__post_init__() + + x.__init__ = _init + if not hasattr(x, '__pre_init__'): + x.__pre_init__ = _pass + if not hasattr(x, '__post_init__'): + x.__post_init__ = _pass + return x + + +class Module(nn.Module, metaclass=PrePostInitMeta): + 'Same as `nn.Module`, but no need for subclasses to call `super().__init__`' + + def __pre_init__(self): + super().__init__() + + def __init__(self): + pass + + +def children(m): + 'Get children of `m`.' + return list(m.children()) + + +def num_children(m): + 'Get number of children modules in `m`.' + return len(children(m)) + + +def children_and_parameters(m: nn.Module): + 'Return the children of `m` and its direct parameters not registered in modules.' + children = list(m.children()) + children_p = sum([[id(p) for p in c.parameters()] for c in m.children()], + []) + for p in m.parameters(): + if id(p) not in children_p: + children.append(ParameterModule(p)) + return children + + +def flatten_model(m): + if num_children(m): + mapped = map(flatten_model, children_and_parameters(m)) + return sum(mapped, []) + else: + return [m] + + +def in_channels(m): + 'Return the shape of the first weight layer in `m`.' + for layer in flatten_model(m): + if hasattr(layer, 'weight'): + return layer.weight.shape[1] + raise Exception('No weight layer') + + +def relu(inplace: bool = False, leaky: float = None): + 'Return a relu activation, maybe `leaky` and `inplace`.' + return nn.LeakyReLU( + inplace=inplace, + negative_slope=leaky) if leaky is not None else nn.ReLU( + inplace=inplace) + + +def conv_layer(ni, + nf, + ks=3, + stride=1, + padding=None, + bias=None, + is_1d=False, + norm_type=NormType.Batch, + use_activ=True, + leaky=None, + transpose=False, + init=nn.init.kaiming_normal_, + self_attention=False): + 'Create a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and batchnorm (if `bn`) layers.' + if padding is None: + padding = (ks - 1) // 2 if not transpose else 0 + bn = norm_type in (NormType.Batch, NormType.BatchZero) + if bias is None: + bias = not bn + conv_func = nn.ConvTranspose2d if transpose else nn.Conv1d if is_1d else nn.Conv2d + conv = conv_func( + ni, nf, kernel_size=ks, bias=bias, stride=stride, padding=padding) + if norm_type == NormType.Weight: + conv = weight_norm(conv) + elif norm_type == NormType.Spectral: + conv = spectral_norm(conv) + layers = [conv] + if use_activ: + layers.append(relu(True, leaky=leaky)) + if bn: + layers.append((nn.BatchNorm1d if is_1d else nn.BatchNorm2d)(nf)) + if self_attention: + layers.append(SelfAttention(nf)) + return nn.Sequential(*layers) + + +def res_block(nf, + dense=False, + norm_type=NormType.Batch, + bottle=False, + **conv_kwargs): + 'Resnet block of `nf` features. `conv_kwargs` are passed to `conv_layer`.' + norm2 = norm_type + if not dense and (norm_type == NormType.Batch): + norm2 = NormType.BatchZero + nf_inner = nf // 2 if bottle else nf + return SequentialEx( + conv_layer(nf, nf_inner, norm_type=norm_type, **conv_kwargs), + conv_layer(nf_inner, nf, norm_type=norm2, **conv_kwargs), + MergeLayer(dense)) + + +def conv1d(ni, no, ks=1, stride=1, padding=0, bias=False): + 'Create and initialize a `nn.Conv1d` layer with spectral normalization.' + conv = nn.Conv1d(ni, no, ks, stride=stride, padding=padding, bias=bias) + nn.init.kaiming_normal_(conv.weight) + if bias: + conv.bias.data.zero_() + return spectral_norm(conv) + + +class SelfAttention(Module): + 'Self attention layer for nd.' + + def __init__(self, n_channels): + self.query = conv1d(n_channels, n_channels // 8) + self.key = conv1d(n_channels, n_channels // 8) + self.value = conv1d(n_channels, n_channels) + self.gamma = nn.Parameter(torch.tensor([0.])) + + def forward(self, x): + 'Notation from https://arxiv.org/pdf/1805.08318.pdf' + size = x.size() + x = x.view(*size[:2], -1) + f, g, h = self.query(x), self.key(x), self.value(x) + beta = F.softmax(torch.bmm(f.permute(0, 2, 1).contiguous(), g), dim=1) + o = self.gamma * torch.bmm(h, beta) + x + return o.view(*size).contiguous() + + +def sigmoid_range(x, low, high): + 'Sigmoid function with range `(low, high)`' + return torch.sigmoid(x) * (high - low) + low + + +class SigmoidRange(Module): + 'Sigmoid module with range `(low,x_max)`' + + def __init__(self, low, high): + self.low, self.high = low, high + + def forward(self, x): + return sigmoid_range(x, self.low, self.high) + + +class SequentialEx(Module): + 'Like `nn.Sequential`, but with ModuleList semantics, and can access module input' + + def __init__(self, *layers): + self.layers = nn.ModuleList(layers) + + def forward(self, x): + res = x + for layer in self.layers: + res.orig = x + nres = layer(res) + res.orig = None + res = nres + return res + + def __getitem__(self, i): + return self.layers[i] + + def append(self, layer): + return self.layers.append(layer) + + def extend(self, layer): + return self.layers.extend(layer) + + def insert(self, i, layer): + return self.layers.insert(i, layer) + + +class MergeLayer(Module): + 'Merge a shortcut with the result of the module by adding them or concatenating thme if `dense=True`.' + + def __init__(self, dense: bool = False): + self.dense = dense + + def forward(self, x): + return torch.cat([x, x.orig], dim=1) if self.dense else (x + x.orig) + + +class PixelShuffle_ICNR(Module): + 'Upsample by `scale` from `ni` filters to `nf` (default `ni`), using `nn.PixelShuffle`, and `weight_norm`.' + + def __init__(self, + ni: int, + nf: int = None, + scale: int = 2, + blur: bool = False, + norm_type=NormType.Weight, + leaky: float = None): + nf = ni if nf is None else nf + self.conv = conv_layer( + ni, nf * (scale**2), ks=1, norm_type=norm_type, use_activ=False) + self.shuf = nn.PixelShuffle(scale) + # Blurring over (h*w) kernel + # "Super-Resolution using Convolutional Neural Networks without Any Checkerboard Artifacts" + # - https://arxiv.org/abs/1806.02658 + self.pad = nn.ReplicationPad2d((1, 0, 1, 0)) + self.blur = nn.AvgPool2d(2, stride=1) + self.relu = relu(True, leaky=leaky) + + def forward(self, x): + x = self.shuf(self.relu(self.conv(x))) + return self.blur(self.pad(x)) if self.blur else x diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/__init__.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ac92484c71920c0b0795f8aea2d376031d84b9f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_control_3d_portrait import ImageControl3dPortrait + +else: + _import_structure = { + 'image_control_3d_portrait': ['ImageControl3dPortrait'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/image_control_3d_portrait.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/image_control_3d_portrait.py new file mode 100644 index 0000000000000000000000000000000000000000..91fbd4a2199d3a18016bb38c990881a059170bae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/image_control_3d_portrait.py @@ -0,0 +1,468 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os +from collections import OrderedDict +from typing import Any, Dict + +import cv2 +import json +import numpy as np +import PIL.Image as Image +import torch +import torchvision.transforms as transforms +from scipy.io import loadmat + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.face_detection.peppa_pig_face.facer import FaceAna +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger +from .network.camera_utils import FOV_to_intrinsics, LookAtPoseSampler +from .network.shape_utils import convert_sdf_samples_to_ply +from .network.triplane import TriPlaneGenerator +from .network.triplane_encoder import TriplaneEncoder + +logger = get_logger() + +__all__ = ['ImageControl3dPortrait'] + + +@MODELS.register_module( + Tasks.image_control_3d_portrait, + module_name=Models.image_control_3d_portrait) +class ImageControl3dPortrait(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image face fusion model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + logger.info('model params:{}'.format(kwargs)) + self.neural_rendering_resolution = kwargs[ + 'neural_rendering_resolution'] + self.cam_radius = kwargs['cam_radius'] + self.fov_deg = kwargs['fov_deg'] + self.truncation_psi = kwargs['truncation_psi'] + self.truncation_cutoff = kwargs['truncation_cutoff'] + self.z_dim = kwargs['z_dim'] + self.image_size = kwargs['image_size'] + self.shape_res = kwargs['shape_res'] + self.pitch_range = kwargs['pitch_range'] + self.yaw_range = kwargs['yaw_range'] + self.max_batch = kwargs['max_batch'] + self.num_frames = kwargs['num_frames'] + self.box_warp = kwargs['box_warp'] + self.save_shape = kwargs['save_shape'] + self.save_images = kwargs['save_images'] + + device = kwargs['device'] + self.device = create_device(device) + + self.facer = FaceAna(model_dir) + + similarity_mat_path = os.path.join(model_dir, 'BFM', + 'similarity_Lm3D_all.mat') + self.lm3d_std = self.load_lm3d(similarity_mat_path) + + init_model_json = os.path.join(model_dir, 'configs', + 'init_encoder.json') + with open(init_model_json, 'r') as fr: + init_kwargs_encoder = json.load(fr) + encoder_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + self.model = TriplaneEncoder(**init_kwargs_encoder) + ckpt_encoder = torch.load(encoder_path, map_location='cpu') + model_state = self.convert_state_dict(ckpt_encoder['state_dict']) + self.model.load_state_dict(model_state) + self.model = self.model.to(self.device) + self.model.eval() + + init_args_G = () + init_netG_json = os.path.join(model_dir, 'configs', 'init_G.json') + with open(init_netG_json, 'r') as fr: + init_kwargs_G = json.load(fr) + self.netG = TriPlaneGenerator(*init_args_G, **init_kwargs_G) + netG_path = os.path.join(model_dir, 'ffhqrebalanced512-128.pth') + ckpt_G = torch.load(netG_path) + self.netG.load_state_dict(ckpt_G['G_ema'], strict=False) + self.netG.neural_rendering_resolution = self.neural_rendering_resolution + self.netG = self.netG.to(self.device) + self.netG.eval() + + self.intrinsics = FOV_to_intrinsics(self.fov_deg, device=self.device) + col, row = np.meshgrid( + np.arange(self.image_size), np.arange(self.image_size)) + np_coord = np.stack((col, row), axis=2) / self.image_size # [0,1] + self.coord = torch.from_numpy(np_coord.astype( + np.float32)).unsqueeze(0).permute(0, 3, 1, 2).to(self.device) + + self.image_transform = transforms.Compose([ + transforms.Resize((self.image_size, self.image_size)), + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) + ]) + + logger.info('init done') + + def convert_state_dict(self, state_dict): + if not next(iter(state_dict)).startswith('module.'): + return state_dict + new_state_dict = OrderedDict() + + split_index = 0 + for cur_key, cur_value in state_dict.items(): + if cur_key.startswith('module.model'): + split_index = 13 + elif cur_key.startswith('module'): + split_index = 7 + + break + + for k, v in state_dict.items(): + name = k[split_index:] + new_state_dict[name] = v + return new_state_dict + + def detect_face(self, img): + src_h, src_w, _ = img.shape + boxes, landmarks, _ = self.facer.run(img) + if boxes.shape[0] == 0: + return None + elif boxes.shape[0] > 1: + max_area = 0 + max_index = 0 + for i in range(boxes.shape[0]): + bbox_width = boxes[i][2] - boxes[i][0] + bbox_height = boxes[i][3] - boxes[i][1] + area = int(bbox_width) * int(bbox_height) + if area > max_area: + max_index = i + max_area = area + + return landmarks[max_index] + else: + return landmarks[0] + + def get_f5p(self, landmarks, np_img): + eye_left = self.find_pupil(landmarks[36:41], np_img) + eye_right = self.find_pupil(landmarks[42:47], np_img) + if eye_left is None or eye_right is None: + logger.warning( + 'cannot find 5 points with find_pupil, used mean instead.!') + eye_left = landmarks[36:41].mean(axis=0) + eye_right = landmarks[42:47].mean(axis=0) + nose = landmarks[30] + mouth_left = landmarks[48] + mouth_right = landmarks[54] + f5p = [[eye_left[0], eye_left[1]], [eye_right[0], eye_right[1]], + [nose[0], nose[1]], [mouth_left[0], mouth_left[1]], + [mouth_right[0], mouth_right[1]]] + return np.array(f5p) + + def find_pupil(self, landmarks, np_img): + h, w, _ = np_img.shape + xmax = int(landmarks[:, 0].max()) + xmin = int(landmarks[:, 0].min()) + ymax = int(landmarks[:, 1].max()) + ymin = int(landmarks[:, 1].min()) + + if ymin >= ymax or xmin >= xmax or ymin < 0 or xmin < 0 or ymax > h or xmax > w: + return None + eye_img_bgr = np_img[ymin:ymax, xmin:xmax, :] + eye_img = cv2.cvtColor(eye_img_bgr, cv2.COLOR_BGR2GRAY) + eye_img = cv2.equalizeHist(eye_img) + n_marks = landmarks - np.array([xmin, ymin]).reshape([1, 2]) + eye_mask = cv2.fillConvexPoly( + np.zeros_like(eye_img), n_marks.astype(np.int32), 1) + ret, thresh = cv2.threshold(eye_img, 100, 255, + cv2.THRESH_BINARY | cv2.THRESH_OTSU) + thresh = (1 - thresh / 255.) * eye_mask + cnt = 0 + xm = [] + ym = [] + for i in range(thresh.shape[0]): + for j in range(thresh.shape[1]): + if thresh[i, j] > 0.5: + xm.append(j) + ym.append(i) + cnt += 1 + if cnt != 0: + xm.sort() + ym.sort() + xm = xm[cnt // 2] + ym = ym[cnt // 2] + else: + xm = thresh.shape[1] / 2 + ym = thresh.shape[0] / 2 + + return xm + xmin, ym + ymin + + def load_lm3d(self, similarity_mat_path): + + Lm3D = loadmat(similarity_mat_path) + Lm3D = Lm3D['lm'] + + lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 + lm_data1 = Lm3D[lm_idx[0], :] + lm_data2 = np.mean(Lm3D[lm_idx[[1, 2]], :], 0) + lm_data3 = np.mean(Lm3D[lm_idx[[3, 4]], :], 0) + lm_data4 = Lm3D[lm_idx[5], :] + lm_data5 = Lm3D[lm_idx[6], :] + + Lm3D = np.stack([lm_data1, lm_data2, lm_data3, lm_data4, lm_data5], + axis=0) + + Lm3D = Lm3D[[1, 2, 0, 3, 4], :] + + return Lm3D + + def POS(self, xp, x): + npts = xp.shape[1] + + A = np.zeros([2 * npts, 8]) + + A[0:2 * npts - 1:2, 0:3] = x.transpose() + A[0:2 * npts - 1:2, 3] = 1 + + A[1:2 * npts:2, 4:7] = x.transpose() + A[1:2 * npts:2, 7] = 1 + + b = np.reshape(xp.transpose(), [2 * npts, 1]) + + k, _, _, _ = np.linalg.lstsq(A, b) + + R1 = k[0:3] + R2 = k[4:7] + sTx = k[3] + sTy = k[7] + s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2 + t = np.stack([sTx, sTy], axis=0) + + return t, s + + def resize_n_crop_img(self, img, lm, t, s, target_size=224., mask=None): + w0, h0 = img.size + w = (w0 * s).astype(np.int32) + h = (h0 * s).astype(np.int32) + left = (w / 2 - target_size / 2 + float( + (t[0] - w0 / 2) * s)).astype(np.int32) + right = left + target_size + up = (h / 2 - target_size / 2 + float( + (h0 / 2 - t[1]) * s)).astype(np.int32) + below = up + target_size + + img = img.resize((w, h), resample=Image.BICUBIC) + img = img.crop((left, up, right, below)) + + if mask is not None: + mask = mask.resize((w, h), resample=Image.BICUBIC) + mask = mask.crop((left, up, right, below)) + + lm = np.stack([lm[:, 0] - t[0] + w0 / 2, lm[:, 1] - t[1] + h0 / 2], + axis=1) * s + lm = lm - np.reshape( + np.array([(w / 2 - target_size / 2), + (h / 2 - target_size / 2)]), [1, 2]) + + return img, lm, mask + + def align_img(self, + img, + lm, + lm3D, + mask=None, + target_size=224., + rescale_factor=102.): + w0, h0 = img.size + lm5p = lm + t, s = self.POS(lm5p.transpose(), lm3D.transpose()) + s = rescale_factor / s + + img_new, lm_new, mask_new = self.resize_n_crop_img( + img, lm, t, s, target_size=target_size, mask=mask) + trans_params = np.array([w0, h0, s, t[0], t[1]], dtype=object) + + return trans_params, img_new, lm_new, mask_new + + def crop_image(self, img, lm): + _, H = img.size + lm[:, -1] = H - 1 - lm[:, -1] + + target_size = 1024. + rescale_factor = 300 + center_crop_size = 700 + output_size = 512 + + _, im_high, _, _, = self.align_img( + img, + lm, + self.lm3d_std, + target_size=target_size, + rescale_factor=rescale_factor) + + left = int(im_high.size[0] / 2 - center_crop_size / 2) + upper = int(im_high.size[1] / 2 - center_crop_size / 2) + right = left + center_crop_size + lower = upper + center_crop_size + im_cropped = im_high.crop((left, upper, right, lower)) + im_cropped = im_cropped.resize((output_size, output_size), + resample=Image.LANCZOS) + logger.info('crop image done!') + return im_cropped + + def create_samples(self, N=256, voxel_origin=[0, 0, 0], cube_length=2.0): + voxel_origin = np.array(voxel_origin) - cube_length / 2 + voxel_size = cube_length / (N - 1) + + overall_index = torch.arange(0, N**3, 1, out=torch.LongTensor()) + samples = torch.zeros(N**3, 3) + + samples[:, 2] = overall_index % N + samples[:, 1] = (overall_index.float() / N) % N + samples[:, 0] = ((overall_index.float() / N) / N) % N + + samples[:, 0] = (samples[:, 0] * voxel_size) + voxel_origin[2] + samples[:, 1] = (samples[:, 1] * voxel_size) + voxel_origin[1] + samples[:, 2] = (samples[:, 2] * voxel_size) + voxel_origin[0] + + return samples.unsqueeze(0), voxel_origin, voxel_size + + def numpy_array_to_video(self, numpy_list, video_out_path): + assert len(numpy_list) > 0 + video_height = numpy_list[0].shape[0] + video_width = numpy_list[0].shape[1] + + out_video_size = (video_width, video_height) + output_video_fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') + video_write_capture = cv2.VideoWriter(video_out_path, + output_video_fourcc, 30, + out_video_size) + + for frame in numpy_list: + frame_bgr = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + video_write_capture.write(frame_bgr) + + video_write_capture.release() + + def inference(self, image_path, save_dir): + basename = os.path.basename(image_path).split('.')[0] + img = Image.open(image_path).convert('RGB') + img_array = np.array(img) + img_bgr = img_array[:, :, ::-1] + landmark = self.detect_face(img_array) + if landmark is None: + logger.warning('No face detected in the image!') + f5p = self.get_f5p(landmark, img_bgr) + + logger.info('f5p is:{}'.format(f5p)) + img_cropped = self.crop_image(img, f5p) + img_cropped.save(os.path.join(save_dir, 'crop.jpg')) + + in_image = self.image_transform(img_cropped).unsqueeze(0).to( + self.device) + input = torch.cat((in_image, self.coord), 1) + + save_video_path = os.path.join(save_dir, f'{basename}.mp4') + pred_imgs = [] + + for frame_idx in range(self.num_frames): + cam_pivot = torch.tensor([0, 0, 0.2], device=self.device) + + cam2world_pose = LookAtPoseSampler.sample( + 3.14 / 2 + self.yaw_range + * np.sin(2 * 3.14 * frame_idx / self.num_frames), + 3.14 / 2 - 0.05 + self.pitch_range + * np.cos(2 * 3.14 * frame_idx / self.num_frames), + cam_pivot, + radius=self.cam_radius, + device=self.device) + + camera_params = torch.cat([ + cam2world_pose.reshape(-1, 16), + self.intrinsics.reshape(-1, 9) + ], 1) + + conditioning_cam2world_pose = LookAtPoseSampler.sample( + np.pi / 2, + np.pi / 2, + cam_pivot, + radius=self.cam_radius, + device=self.device) + conditioning_params = torch.cat([ + conditioning_cam2world_pose.reshape(-1, 16), + self.intrinsics.reshape(-1, 9) + ], 1) + + z = torch.from_numpy(np.random.randn(1, + self.z_dim)).to(self.device) + + with torch.no_grad(): + ws = self.netG.mapping( + z, + conditioning_params, + truncation_psi=self.truncation_psi, + truncation_cutoff=self.truncation_cutoff) + + planes, pred_depth, pred_feature, pred_rgb, pred_sr, _, _, _, _ = self.model( + ws, input, camera_params, None) + + pred_img = (pred_sr.permute(0, 2, 3, 1) * 127.5 + 128).clamp( + 0, 255).to(torch.uint8) + pred_img = pred_img.squeeze().cpu().numpy() + if self.save_images: + cv2.imwrite( + os.path.join(save_dir, '{}.jpg'.format(frame_idx)), + pred_img[:, :, ::-1]) + pred_imgs.append(pred_img) + + self.numpy_array_to_video(pred_imgs, save_video_path) + + if self.save_shape: + max_batch = 1000000 + + samples, voxel_origin, voxel_size = self.create_samples( + N=self.shape_res, + voxel_origin=[0, 0, 0], + cube_length=self.box_warp) + samples = samples.to(z.device) + sigmas = torch.zeros((samples.shape[0], samples.shape[1], 1), + device=z.device) + transformed_ray_directions_expanded = torch.zeros( + (samples.shape[0], max_batch, 3), device=z.device) + transformed_ray_directions_expanded[..., -1] = -1 + + head = 0 + with torch.no_grad(): + while head < samples.shape[1]: + torch.manual_seed(0) + sigma = self.model.sample( + samples[:, head:head + max_batch], + transformed_ray_directions_expanded[:, :samples. + shape[1] - head], + planes)['sigma'] + sigmas[:, head:head + max_batch] = sigma + head += max_batch + + sigmas = sigmas.reshape((self.shape_res, self.shape_res, + self.shape_res)).cpu().numpy() + sigmas = np.flip(sigmas, 0) + + pad = int(30 * self.shape_res / 256) + pad_value = -1000 + sigmas[:pad] = pad_value + sigmas[-pad:] = pad_value + sigmas[:, :pad] = pad_value + sigmas[:, -pad:] = pad_value + sigmas[:, :, :pad] = pad_value + sigmas[:, :, -pad:] = pad_value + convert_sdf_samples_to_ply( + np.transpose(sigmas, (2, 1, 0)), [0, 0, 0], + 1, + os.path.join(save_dir, f'{basename}.ply'), + level=10) + + logger.info('model inference done') diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/__init__.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/camera_utils.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/camera_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9961a8e006380b8bdf87bb1207a9172ff942ebdf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/camera_utils.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +""" +Helper functions for constructing camera parameter matrices. Primarily used in visualization and inference scripts. +""" + +import math + +import torch +import torch.nn as nn + +from .volumetric_rendering import math_utils + + +class GaussianCameraPoseSampler: + """ + Samples pitch and yaw from a Gaussian distribution and returns a camera pose. + Camera is specified as looking at the origin. + If horizontal and vertical stddev (specified in radians) are zero, gives a + deterministic camera pose with yaw=horizontal_mean, pitch=vertical_mean. + The coordinate system is specified with y-up, z-forward, x-left. + Horizontal mean is the azimuthal angle (rotation around y axis) in radians, + vertical mean is the polar angle (angle from the y axis) in radians. + A point along the z-axis has azimuthal_angle=0, polar_angle=pi/2. + + Example: + For a camera pose looking at the origin with the camera at position [0, 0, 1]: + cam2world = GaussianCameraPoseSampler.sample(math.pi/2, math.pi/2, radius=1) + """ + + @staticmethod + def sample(horizontal_mean, + vertical_mean, + horizontal_stddev=0, + vertical_stddev=0, + radius=1, + batch_size=1, + device='cpu'): + h = torch.randn((batch_size, 1), + device=device) * horizontal_stddev + horizontal_mean + v = torch.randn( + (batch_size, 1), device=device) * vertical_stddev + vertical_mean + v = torch.clamp(v, 1e-5, math.pi - 1e-5) + + theta = h + v = v / math.pi + phi = torch.arccos(1 - 2 * v) + + camera_origins = torch.zeros((batch_size, 3), device=device) + + camera_origins[:, 0:1] = radius * torch.sin(phi) * torch.cos(math.pi + - theta) + camera_origins[:, 2:3] = radius * torch.sin(phi) * torch.sin(math.pi + - theta) + camera_origins[:, 1:2] = radius * torch.cos(phi) + + forward_vectors = math_utils.normalize_vecs(-camera_origins) + return create_cam2world_matrix(forward_vectors, camera_origins) + + +class LookAtPoseSampler: + """ + Same as GaussianCameraPoseSampler, except the + camera is specified as looking at 'lookat_position', a 3-vector. + + Example: + For a camera pose looking at the origin with the camera at position [0, 0, 1]: + cam2world = LookAtPoseSampler.sample(math.pi/2, math.pi/2, torch.tensor([0, 0, 0]), radius=1) + """ + + @staticmethod + def sample(horizontal_mean, + vertical_mean, + lookat_position, + horizontal_stddev=0, + vertical_stddev=0, + radius=1, + batch_size=1, + device='cpu'): + h = torch.randn((batch_size, 1), + device=device) * horizontal_stddev + horizontal_mean + v = torch.randn( + (batch_size, 1), device=device) * vertical_stddev + vertical_mean + v = torch.clamp(v, 1e-5, math.pi - 1e-5) + + theta = h + v = v / math.pi + phi = torch.arccos(1 - 2 * v) + + camera_origins = torch.zeros((batch_size, 3), device=device) + + camera_origins[:, 0:1] = radius * torch.sin(phi) * torch.cos(math.pi + - theta) + camera_origins[:, 2:3] = radius * torch.sin(phi) * torch.sin(math.pi + - theta) + camera_origins[:, 1:2] = radius * torch.cos(phi) + + # forward_vectors = math_utils.normalize_vecs(-camera_origins) + forward_vectors = math_utils.normalize_vecs(lookat_position + - camera_origins) + return create_cam2world_matrix(forward_vectors, camera_origins) + + +class UniformCameraPoseSampler: + """ + Same as GaussianCameraPoseSampler, except the + pose is sampled from a uniform distribution with range +-[horizontal/vertical]_stddev. + + Example: + For a batch of random camera poses looking at the origin with yaw sampled from [-pi/2, +pi/2] radians: + + cam2worlds = UniformCameraPoseSampler.sample + (math.pi/2, math.pi/2, horizontal_stddev=math.pi/2, radius=1, batch_size=16) + """ + + @staticmethod + def sample(horizontal_mean, + vertical_mean, + horizontal_stddev=0, + vertical_stddev=0, + radius=1, + batch_size=1, + device='cpu'): + h = (torch.rand((batch_size, 1), device=device) * 2 + - 1) * horizontal_stddev + horizontal_mean + v = (torch.rand((batch_size, 1), device=device) * 2 + - 1) * vertical_stddev + vertical_mean + v = torch.clamp(v, 1e-5, math.pi - 1e-5) + + theta = h + v = v / math.pi + phi = torch.arccos(1 - 2 * v) + + camera_origins = torch.zeros((batch_size, 3), device=device) + + camera_origins[:, 0:1] = radius * torch.sin(phi) * torch.cos(math.pi + - theta) + camera_origins[:, 2:3] = radius * torch.sin(phi) * torch.sin(math.pi + - theta) + camera_origins[:, 1:2] = radius * torch.cos(phi) + + forward_vectors = math_utils.normalize_vecs(-camera_origins) + return create_cam2world_matrix(forward_vectors, camera_origins) + + +def create_cam2world_matrix(forward_vector, origin): + """ + Takes in the direction the camera is pointing and the camera origin and returns a cam2world matrix. + Works on batches of forward_vectors, origins. Assumes y-axis is up and that there is no camera roll. + """ + + forward_vector = math_utils.normalize_vecs(forward_vector) + up_vector = torch.tensor([0, 1, 0], + dtype=torch.float, + device=origin.device).expand_as(forward_vector) + + right_vector = -math_utils.normalize_vecs( + torch.cross(up_vector, forward_vector, dim=-1)) + up_vector = math_utils.normalize_vecs( + torch.cross(forward_vector, right_vector, dim=-1)) + + rotation_matrix = torch.eye( + 4, device=origin.device).unsqueeze(0).repeat(forward_vector.shape[0], + 1, 1) + rotation_matrix[:, :3, :3] = torch.stack( + (right_vector, up_vector, forward_vector), axis=-1) + + translation_matrix = torch.eye( + 4, device=origin.device).unsqueeze(0).repeat(forward_vector.shape[0], + 1, 1) + translation_matrix[:, :3, 3] = origin + cam2world = (translation_matrix @ rotation_matrix)[:, :, :] + assert (cam2world.shape[1:] == (4, 4)) + return cam2world + + +def FOV_to_intrinsics(fov_degrees, device='cpu'): + """ + Creates a 3x3 camera intrinsics matrix from the camera field of view, specified in degrees. + Note the intrinsics are returned as normalized by image size, rather than in pixel units. + Assumes principal point is at image center. + """ + + focal_length = float(1 / (math.tan(fov_degrees * 3.14159 / 360) * 1.414)) + intrinsics = torch.tensor( + [[focal_length, 0, 0.5], [0, focal_length, 0.5], [0, 0, 1]], + device=device) + return intrinsics diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/networks_stylegan2.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/networks_stylegan2.py new file mode 100644 index 0000000000000000000000000000000000000000..7d3daee3a3a55a45503fd2d49dc2a6563d04f361 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/networks_stylegan2.py @@ -0,0 +1,1062 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +"""Network architectures from the paper +"Analyzing and Improving the Image Quality of StyleGAN". +Matches the original implementation of configs E-F by Karras et al. at +https://github.com/NVlabs/stylegan2/blob/master/training/networks_stylegan2.py""" + +import numpy as np +import torch + +from modelscope.ops.image_control_3d_portrait.torch_utils import (misc, + persistence) +from modelscope.ops.image_control_3d_portrait.torch_utils.ops import ( + bias_act, conv2d_resample, fma, upfirdn2d) + + +@misc.profiled_function +def normalize_2nd_moment(x, dim=1, eps=1e-8): + return x * (x.square().mean(dim=dim, keepdim=True) + eps).rsqrt() + + +@misc.profiled_function +def modulated_conv2d( + x, # Input tensor of shape [batch_size, in_channels, in_height, in_width]. + weight, # Weight tensor of shape [out_channels, in_channels, kernel_height, kernel_width]. + styles, # Modulation coefficients of shape [batch_size, in_channels]. + noise=None, # Optional noise tensor to add to the output activations. + up=1, # Integer upsampling factor. + down=1, # Integer downsampling factor. + padding=0, # Padding with respect to the upsampled image. + resample_filter=None, # Low-pass filter to apply when resampling activations. + demodulate=True, # Apply weight demodulation? + flip_weight=True, # False = convolution, True = correlation (matches torch.nn.functional.conv2d). + fused_modconv=True, # Perform modulation, convolution, and demodulation as a single fused operation? +): + batch_size = x.shape[0] + out_channels, in_channels, kh, kw = weight.shape + misc.assert_shape(weight, [out_channels, in_channels, kh, kw]) # [OIkk] + misc.assert_shape(x, [batch_size, in_channels, None, None]) # [NIHW] + misc.assert_shape(styles, [batch_size, in_channels]) # [NI] + + # Pre-normalize inputs to avoid FP16 overflow. + if x.dtype == torch.float16 and demodulate: + weight = weight * (1 / np.sqrt(in_channels * kh * kw) / weight.norm( + float('inf'), dim=[1, 2, 3], keepdim=True)) # max_Ikk + styles = styles / styles.norm( + float('inf'), dim=1, keepdim=True) # max_I + + # Calculate per-sample weights and demodulation coefficients. + w = None + dcoefs = None + if demodulate or fused_modconv: + w = weight.unsqueeze(0) # [NOIkk] + w = w * styles.reshape(batch_size, 1, -1, 1, 1) # [NOIkk] + if demodulate: + dcoefs = (w.square().sum(dim=[2, 3, 4]) + 1e-8).rsqrt() # [NO] + if demodulate and fused_modconv: + w = w * dcoefs.reshape(batch_size, -1, 1, 1, 1) # [NOIkk] + + # Execute by scaling the activations before and after the convolution. + if not fused_modconv: + x = x * styles.to(x.dtype).reshape(batch_size, -1, 1, 1) + x = conv2d_resample.conv2d_resample( + x=x, + w=weight.to(x.dtype), + f=resample_filter, + up=up, + down=down, + padding=padding, + flip_weight=flip_weight) + if demodulate and noise is not None: + x = fma.fma(x, + dcoefs.to(x.dtype).reshape(batch_size, -1, 1, 1), + noise.to(x.dtype)) + elif demodulate: + x = x * dcoefs.to(x.dtype).reshape(batch_size, -1, 1, 1) + elif noise is not None: + x = x.add_(noise.to(x.dtype)) + return x + + # Execute as one fused op using grouped convolution. + with misc.suppress_tracer_warnings( + ): # this value will be treated as a constant + batch_size = int(batch_size) + misc.assert_shape(x, [batch_size, in_channels, None, None]) + x = x.reshape(1, -1, *x.shape[2:]) + w = w.reshape(-1, in_channels, kh, kw) + x = conv2d_resample.conv2d_resample( + x=x, + w=w.to(x.dtype), + f=resample_filter, + up=up, + down=down, + padding=padding, + groups=batch_size, + flip_weight=flip_weight) + x = x.reshape(batch_size, -1, *x.shape[2:]) + if noise is not None: + x = x.add_(noise) + return x + + +@persistence.persistent_class +class FullyConnectedLayer(torch.nn.Module): + + def __init__( + self, + in_features, # Number of input features. + out_features, # Number of output features. + bias=True, # Apply additive bias before the activation function? + activation='linear', # Activation function: 'relu', 'lrelu', etc. + lr_multiplier=1, # Learning rate multiplier. + bias_init=0, # Initial value for the additive bias. + ): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.activation = activation + self.weight = torch.nn.Parameter( + torch.randn([out_features, in_features]) / lr_multiplier) + self.bias = torch.nn.Parameter( + torch.full([out_features], + np.float32(bias_init))) if bias else None + self.weight_gain = lr_multiplier / np.sqrt(in_features) + self.bias_gain = lr_multiplier + + def forward(self, x): + w = self.weight.to(x.dtype) * self.weight_gain + b = self.bias + if b is not None: + b = b.to(x.dtype) + if self.bias_gain != 1: + b = b * self.bias_gain + + if self.activation == 'linear' and b is not None: + x = torch.addmm(b.unsqueeze(0), x, w.t()) + else: + x = x.matmul(w.t()) + x = bias_act.bias_act(x, b, act=self.activation) + return x + + def extra_repr(self): + return f'in_features={self.in_features:d}, out_features={self.out_features:d}, activation={self.activation:s}' + + +@persistence.persistent_class +class Conv2dLayer(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels. + out_channels, # Number of output channels. + kernel_size, # Width and height of the convolution kernel. + bias=True, # Apply additive bias before the activation function? + activation='linear', # Activation function: 'relu', 'lrelu', etc. + up=1, # Integer upsampling factor. + down=1, # Integer downsampling factor. + resample_filter=[ + 1, 3, 3, 1 + ], # Low-pass filter to apply when resampling activations. + conv_clamp=None, # Clamp the output to +-X, None = disable clamping. + channels_last=False, # Expect the input to have memory_format=channels_last? + trainable=True, # Update the weights of this layer during training? + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.activation = activation + self.up = up + self.down = down + self.conv_clamp = conv_clamp + self.register_buffer('resample_filter', + upfirdn2d.setup_filter(resample_filter)) + self.padding = kernel_size // 2 + self.weight_gain = 1 / np.sqrt(in_channels * (kernel_size**2)) + self.act_gain = bias_act.activation_funcs[activation].def_gain + + memory_format = torch.channels_last if channels_last else torch.contiguous_format + weight = torch.randn( + [out_channels, in_channels, kernel_size, + kernel_size]).to(memory_format=memory_format) + bias = torch.zeros([out_channels]) if bias else None + if trainable: + self.weight = torch.nn.Parameter(weight) + self.bias = torch.nn.Parameter(bias) if bias is not None else None + else: + self.register_buffer('weight', weight) + if bias is not None: + self.register_buffer('bias', bias) + else: + self.bias = None + + def forward(self, x, gain=1): + w = self.weight * self.weight_gain + b = self.bias.to(x.dtype) if self.bias is not None else None + flip_weight = (self.up == 1) # slightly faster + x = conv2d_resample.conv2d_resample( + x=x, + w=w.to(x.dtype), + f=self.resample_filter, + up=self.up, + down=self.down, + padding=self.padding, + flip_weight=flip_weight) + + act_gain = self.act_gain * gain + act_clamp = self.conv_clamp * gain if self.conv_clamp is not None else None + x = bias_act.bias_act( + x, b, act=self.activation, gain=act_gain, clamp=act_clamp) + return x + + def extra_repr(self): + return ' '.join([ + f'in_channels={self.in_channels:d}, out_channels={self.out_channels:d}, activation={self.activation:s},', + f'up={self.up}, down={self.down}' + ]) + + +@persistence.persistent_class +class MappingNetwork(torch.nn.Module): + + def __init__( + self, + z_dim, # Input latent (Z) dimensionality, 0 = no latent. + c_dim, # Conditioning label (C) dimensionality, 0 = no label. + w_dim, # Intermediate latent (W) dimensionality. + num_ws, # Number of intermediate latents to output, None = do not broadcast. + num_layers=8, # Number of mapping layers. + embed_features=None, # Label embedding dimensionality, None = same as w_dim. + layer_features=None, # Number of intermediate features in the mapping layers, None = same as w_dim. + activation='lrelu', # Activation function: 'relu', 'lrelu', etc. + lr_multiplier=0.01, # Learning rate multiplier for the mapping layers. + w_avg_beta=0.998, # Decay for tracking the moving average of W during training, None = do not track. + ): + super().__init__() + self.z_dim = z_dim + self.c_dim = c_dim + self.w_dim = w_dim + self.num_ws = num_ws + self.num_layers = num_layers + self.w_avg_beta = w_avg_beta + + if embed_features is None: + embed_features = w_dim + if c_dim == 0: + embed_features = 0 + if layer_features is None: + layer_features = w_dim + features_list = [z_dim + embed_features + ] + [layer_features] * (num_layers - 1) + [w_dim] + + if c_dim > 0: + self.embed = FullyConnectedLayer(c_dim, embed_features) + for idx in range(num_layers): + in_features = features_list[idx] + out_features = features_list[idx + 1] + layer = FullyConnectedLayer( + in_features, + out_features, + activation=activation, + lr_multiplier=lr_multiplier) + setattr(self, f'fc{idx}', layer) + + if num_ws is not None and w_avg_beta is not None: + self.register_buffer('w_avg', torch.zeros([w_dim])) + + def forward(self, + z, + c, + truncation_psi=1, + truncation_cutoff=None, + update_emas=False): + # Embed, normalize, and concat inputs. + x = None + with torch.autograd.profiler.record_function('input'): + if self.z_dim > 0: + misc.assert_shape(z, [None, self.z_dim]) + x = normalize_2nd_moment(z.to(torch.float32)) + if self.c_dim > 0: + misc.assert_shape(c, [None, self.c_dim]) + y = normalize_2nd_moment(self.embed(c.to(torch.float32))) + x = torch.cat([x, y], dim=1) if x is not None else y + + # Main layers. + for idx in range(self.num_layers): + layer = getattr(self, f'fc{idx}') + x = layer(x) + + # Update moving average of W. + if update_emas and self.w_avg_beta is not None: + with torch.autograd.profiler.record_function('update_w_avg'): + self.w_avg.copy_(x.detach().mean(dim=0).lerp( + self.w_avg, self.w_avg_beta)) + + # Broadcast. + if self.num_ws is not None: + with torch.autograd.profiler.record_function('broadcast'): + x = x.unsqueeze(1).repeat([1, self.num_ws, 1]) + + # Apply truncation. + if truncation_psi != 1: + with torch.autograd.profiler.record_function('truncate'): + assert self.w_avg_beta is not None + if self.num_ws is None or truncation_cutoff is None: + x = self.w_avg.lerp(x, truncation_psi) + else: + x[:, :truncation_cutoff] = self.w_avg.lerp( + x[:, :truncation_cutoff], truncation_psi) + return x + + def extra_repr(self): + return f'z_dim={self.z_dim:d}, c_dim={self.c_dim:d}, w_dim={self.w_dim:d}, num_ws={self.num_ws:d}' + + +@persistence.persistent_class +class SynthesisLayer(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels. + out_channels, # Number of output channels. + w_dim, # Intermediate latent (W) dimensionality. + resolution, # Resolution of this layer. + kernel_size=3, # Convolution kernel size. + up=1, # Integer upsampling factor. + use_noise=True, # Enable noise input? + activation='lrelu', # Activation function: 'relu', 'lrelu', etc. + resample_filter=[ + 1, 3, 3, 1 + ], # Low-pass filter to apply when resampling activations. + conv_clamp=None, # Clamp the output of convolution layers to +-X, None = disable clamping. + channels_last=False, # Use channels_last format for the weights? + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.w_dim = w_dim + self.resolution = resolution + self.up = up + self.use_noise = use_noise + self.activation = activation + self.conv_clamp = conv_clamp + self.register_buffer('resample_filter', + upfirdn2d.setup_filter(resample_filter)) + self.padding = kernel_size // 2 + self.act_gain = bias_act.activation_funcs[activation].def_gain + + self.affine = FullyConnectedLayer(w_dim, in_channels, bias_init=1) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + self.weight = torch.nn.Parameter( + torch.randn([out_channels, in_channels, kernel_size, + kernel_size]).to(memory_format=memory_format)) + if use_noise: + self.register_buffer('noise_const', + torch.randn([resolution, resolution])) + self.noise_strength = torch.nn.Parameter(torch.zeros([])) + self.bias = torch.nn.Parameter(torch.zeros([out_channels])) + + def forward(self, x, w, noise_mode='random', fused_modconv=True, gain=1): + assert noise_mode in ['random', 'const', 'none'] + in_resolution = self.resolution // self.up + misc.assert_shape( + x, [None, self.in_channels, in_resolution, in_resolution]) + styles = self.affine(w) + + noise = None + if self.use_noise and noise_mode == 'random': + noise = torch.randn( + [x.shape[0], 1, self.resolution, self.resolution], + device=x.device) * self.noise_strength + if self.use_noise and noise_mode == 'const': + noise = self.noise_const * self.noise_strength + + flip_weight = (self.up == 1) # slightly faster + x = modulated_conv2d( + x=x, + weight=self.weight, + styles=styles, + noise=noise, + up=self.up, + padding=self.padding, + resample_filter=self.resample_filter, + flip_weight=flip_weight, + fused_modconv=fused_modconv) + + act_gain = self.act_gain * gain + act_clamp = self.conv_clamp * gain if self.conv_clamp is not None else None + x = bias_act.bias_act( + x, + self.bias.to(x.dtype), + act=self.activation, + gain=act_gain, + clamp=act_clamp) + return x + + def extra_repr(self): + return ' '.join([ + f'in_channels={self.in_channels:d}, out_channels={self.out_channels:d}, w_dim={self.w_dim:d},', + f'resolution={self.resolution:d}, up={self.up}, activation={self.activation:s}' + ]) + + +@persistence.persistent_class +class ToRGBLayer(torch.nn.Module): + + def __init__(self, + in_channels, + out_channels, + w_dim, + kernel_size=1, + conv_clamp=None, + channels_last=False): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.w_dim = w_dim + self.conv_clamp = conv_clamp + self.affine = FullyConnectedLayer(w_dim, in_channels, bias_init=1) + memory_format = torch.channels_last if channels_last else torch.contiguous_format + self.weight = torch.nn.Parameter( + torch.randn([out_channels, in_channels, kernel_size, + kernel_size]).to(memory_format=memory_format)) + self.bias = torch.nn.Parameter(torch.zeros([out_channels])) + self.weight_gain = 1 / np.sqrt(in_channels * (kernel_size**2)) + + def forward(self, x, w, fused_modconv=True): + styles = self.affine(w) * self.weight_gain + x = modulated_conv2d( + x=x, + weight=self.weight, + styles=styles, + demodulate=False, + fused_modconv=fused_modconv) + x = bias_act.bias_act(x, self.bias.to(x.dtype), clamp=self.conv_clamp) + return x + + def extra_repr(self): + return f'in_channels={self.in_channels:d}, out_channels={self.out_channels:d}, w_dim={self.w_dim:d}' + + +@persistence.persistent_class +class SynthesisBlock(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels, 0 = first block. + out_channels, # Number of output channels. + w_dim, # Intermediate latent (W) dimensionality. + resolution, # Resolution of this block. + img_channels, # Number of output color channels. + is_last, # Is this the last block? + architecture='skip', # Architecture: 'orig', 'skip', 'resnet'. + resample_filter=[ + 1, 3, 3, 1 + ], # Low-pass filter to apply when resampling activations. + conv_clamp=256, # Clamp the output of convolution layers to +-X, None = disable clamping. + use_fp16=False, # Use FP16 for this block? + fp16_channels_last=False, # Use channels-last memory format with FP16? + fused_modconv_default=True, # Default value of fused_modconv. + **layer_kwargs, # Arguments for SynthesisLayer. + ): + assert architecture in ['orig', 'skip', 'resnet'] + super().__init__() + self.in_channels = in_channels + self.w_dim = w_dim + self.resolution = resolution + self.img_channels = img_channels + self.is_last = is_last + self.architecture = architecture + self.use_fp16 = use_fp16 + self.channels_last = (use_fp16 and fp16_channels_last) + self.fused_modconv_default = fused_modconv_default + self.register_buffer('resample_filter', + upfirdn2d.setup_filter(resample_filter)) + self.num_conv = 0 + self.num_torgb = 0 + + if in_channels == 0: + self.const = torch.nn.Parameter( + torch.randn([out_channels, resolution, resolution])) + + if in_channels != 0: + self.conv0 = SynthesisLayer( + in_channels, + out_channels, + w_dim=w_dim, + resolution=resolution, + up=2, + resample_filter=resample_filter, + conv_clamp=conv_clamp, + channels_last=self.channels_last, + **layer_kwargs) + self.num_conv += 1 + + self.conv1 = SynthesisLayer( + out_channels, + out_channels, + w_dim=w_dim, + resolution=resolution, + conv_clamp=conv_clamp, + channels_last=self.channels_last, + **layer_kwargs) + self.num_conv += 1 + + if is_last or architecture == 'skip': + self.torgb = ToRGBLayer( + out_channels, + img_channels, + w_dim=w_dim, + conv_clamp=conv_clamp, + channels_last=self.channels_last) + self.num_torgb += 1 + + if in_channels != 0 and architecture == 'resnet': + self.skip = Conv2dLayer( + in_channels, + out_channels, + kernel_size=1, + bias=False, + up=2, + resample_filter=resample_filter, + channels_last=self.channels_last) + + def forward(self, + x, + img, + ws, + force_fp32=False, + fused_modconv=None, + update_emas=False, + **layer_kwargs): + _ = update_emas # unused + misc.assert_shape(ws, + [None, self.num_conv + self.num_torgb, self.w_dim]) + w_iter = iter(ws.unbind(dim=1)) + if ws.device.type != 'cuda': + force_fp32 = True + dtype = torch.float16 if self.use_fp16 and not force_fp32 else torch.float32 + memory_format = torch.channels_last if self.channels_last and not force_fp32 else torch.contiguous_format + if fused_modconv is None: + fused_modconv = self.fused_modconv_default + if fused_modconv == 'inference_only': + fused_modconv = (not self.training) + + # Input. + if self.in_channels == 0: + x = self.const.to(dtype=dtype, memory_format=memory_format) + x = x.unsqueeze(0).repeat([ws.shape[0], 1, 1, 1]) + else: + misc.assert_shape(x, [ + None, self.in_channels, self.resolution // 2, + self.resolution // 2 + ]) + x = x.to(dtype=dtype, memory_format=memory_format) + + # Main layers. + if self.in_channels == 0: + x = self.conv1( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + elif self.architecture == 'resnet': + y = self.skip(x, gain=np.sqrt(0.5)) + x = self.conv0( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + x = self.conv1( + x, + next(w_iter), + fused_modconv=fused_modconv, + gain=np.sqrt(0.5), + **layer_kwargs) + x = y.add_(x) + else: + x = self.conv0( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + x = self.conv1( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + + # ToRGB. + if img is not None: + misc.assert_shape(img, [ + None, self.img_channels, self.resolution // 2, + self.resolution // 2 + ]) + img = upfirdn2d.upsample2d(img, self.resample_filter) + if self.is_last or self.architecture == 'skip': + y = self.torgb(x, next(w_iter), fused_modconv=fused_modconv) + y = y.to( + dtype=torch.float32, memory_format=torch.contiguous_format) + img = img.add_(y) if img is not None else y + + assert x.dtype == dtype + assert img is None or img.dtype == torch.float32 + return x, img + + def extra_repr(self): + return f'resolution={self.resolution:d}, architecture={self.architecture:s}' + + +@persistence.persistent_class +class SynthesisNetwork(torch.nn.Module): + + def __init__( + self, + w_dim, # Intermediate latent (W) dimensionality. + img_resolution, # Output image resolution. + img_channels, # Number of color channels. + channel_base=32768, # Overall multiplier for the number of channels. + channel_max=512, # Maximum number of channels in any layer. + num_fp16_res=4, # Use FP16 for the N highest resolutions. + **block_kwargs, # Arguments for SynthesisBlock. + ): + assert img_resolution >= 4 and img_resolution & (img_resolution + - 1) == 0 + super().__init__() + self.w_dim = w_dim + self.img_resolution = img_resolution + self.img_resolution_log2 = int(np.log2(img_resolution)) + self.img_channels = img_channels + self.num_fp16_res = num_fp16_res + self.block_resolutions = [ + 2**i for i in range(2, self.img_resolution_log2 + 1) + ] + channels_dict = { + res: min(channel_base // res, channel_max) + for res in self.block_resolutions + } + fp16_resolution = max(2**(self.img_resolution_log2 + 1 - num_fp16_res), + 8) + + self.num_ws = 0 + for res in self.block_resolutions: + in_channels = channels_dict[res // 2] if res > 4 else 0 + out_channels = channels_dict[res] + use_fp16 = (res >= fp16_resolution) + is_last = (res == self.img_resolution) + block = SynthesisBlock( + in_channels, + out_channels, + w_dim=w_dim, + resolution=res, + img_channels=img_channels, + is_last=is_last, + use_fp16=use_fp16, + **block_kwargs) + self.num_ws += block.num_conv + if is_last: + self.num_ws += block.num_torgb + setattr(self, f'b{res}', block) + + def forward(self, ws, **block_kwargs): + block_ws = [] + with torch.autograd.profiler.record_function('split_ws'): + misc.assert_shape(ws, [None, self.num_ws, self.w_dim]) + ws = ws.to(torch.float32) + w_idx = 0 + for res in self.block_resolutions: + block = getattr(self, f'b{res}') + block_ws.append( + ws.narrow(1, w_idx, block.num_conv + block.num_torgb)) + w_idx += block.num_conv + + x = img = None + for res, cur_ws in zip(self.block_resolutions, block_ws): + block = getattr(self, f'b{res}') + x, img = block(x, img, cur_ws, **block_kwargs) + return img + + def extra_repr(self): + return ' '.join([ + f'w_dim={self.w_dim:d}, num_ws={self.num_ws:d},', + f'img_resolution={self.img_resolution:d}, img_channels={self.img_channels:d},', + f'num_fp16_res={self.num_fp16_res:d}' + ]) + + +@persistence.persistent_class +class Generator(torch.nn.Module): + + def __init__( + self, + z_dim, # Input latent (Z) dimensionality. + c_dim, # Conditioning label (C) dimensionality. + w_dim, # Intermediate latent (W) dimensionality. + img_resolution, # Output resolution. + img_channels, # Number of output color channels. + mapping_kwargs={}, # Arguments for MappingNetwork. + **synthesis_kwargs, # Arguments for SynthesisNetwork. + ): + super().__init__() + self.z_dim = z_dim + self.c_dim = c_dim + self.w_dim = w_dim + self.img_resolution = img_resolution + self.img_channels = img_channels + self.synthesis = SynthesisNetwork( + w_dim=w_dim, + img_resolution=img_resolution, + img_channels=img_channels, + **synthesis_kwargs) + self.num_ws = self.synthesis.num_ws + self.mapping = MappingNetwork( + z_dim=z_dim, + c_dim=c_dim, + w_dim=w_dim, + num_ws=self.num_ws, + **mapping_kwargs) + + def forward(self, + z, + c, + truncation_psi=1, + truncation_cutoff=None, + update_emas=False, + **synthesis_kwargs): + ws = self.mapping( + z, + c, + truncation_psi=truncation_psi, + truncation_cutoff=truncation_cutoff, + update_emas=update_emas) + img = self.synthesis(ws, update_emas=update_emas, **synthesis_kwargs) + return img + + +@persistence.persistent_class +class DiscriminatorBlock(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels, 0 = first block. + tmp_channels, # Number of intermediate channels. + out_channels, # Number of output channels. + resolution, # Resolution of this block. + img_channels, # Number of input color channels. + first_layer_idx, # Index of the first layer. + architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'. + activation='lrelu', # Activation function: 'relu', 'lrelu', etc. + resample_filter=[ + 1, 3, 3, 1 + ], # Low-pass filter to apply when resampling activations. + conv_clamp=None, # Clamp the output of convolution layers to +-X, None = disable clamping. + use_fp16=False, # Use FP16 for this block? + fp16_channels_last=False, # Use channels-last memory format with FP16? + freeze_layers=0, # Freeze-D: Number of layers to freeze. + ): + assert in_channels in [0, tmp_channels] + assert architecture in ['orig', 'skip', 'resnet'] + super().__init__() + self.in_channels = in_channels + self.resolution = resolution + self.img_channels = img_channels + self.first_layer_idx = first_layer_idx + self.architecture = architecture + self.use_fp16 = use_fp16 + self.channels_last = (use_fp16 and fp16_channels_last) + self.register_buffer('resample_filter', + upfirdn2d.setup_filter(resample_filter)) + + self.num_layers = 0 + + def trainable_gen(): + while True: + layer_idx = self.first_layer_idx + self.num_layers + trainable = (layer_idx >= freeze_layers) + self.num_layers += 1 + yield trainable + + trainable_iter = trainable_gen() + + if in_channels == 0 or architecture == 'skip': + self.fromrgb = Conv2dLayer( + img_channels, + tmp_channels, + kernel_size=1, + activation=activation, + trainable=next(trainable_iter), + conv_clamp=conv_clamp, + channels_last=self.channels_last) + + self.conv0 = Conv2dLayer( + tmp_channels, + tmp_channels, + kernel_size=3, + activation=activation, + trainable=next(trainable_iter), + conv_clamp=conv_clamp, + channels_last=self.channels_last) + + self.conv1 = Conv2dLayer( + tmp_channels, + out_channels, + kernel_size=3, + activation=activation, + down=2, + trainable=next(trainable_iter), + resample_filter=resample_filter, + conv_clamp=conv_clamp, + channels_last=self.channels_last) + + if architecture == 'resnet': + self.skip = Conv2dLayer( + tmp_channels, + out_channels, + kernel_size=1, + bias=False, + down=2, + trainable=next(trainable_iter), + resample_filter=resample_filter, + channels_last=self.channels_last) + + def forward(self, x, img, force_fp32=False): + if (x if x is not None else img).device.type != 'cuda': + force_fp32 = True + dtype = torch.float16 if self.use_fp16 and not force_fp32 else torch.float32 + memory_format = torch.channels_last if self.channels_last and not force_fp32 else torch.contiguous_format + + # Input. + if x is not None: + misc.assert_shape( + x, [None, self.in_channels, self.resolution, self.resolution]) + x = x.to(dtype=dtype, memory_format=memory_format) + + # FromRGB. + if self.in_channels == 0 or self.architecture == 'skip': + misc.assert_shape( + img, + [None, self.img_channels, self.resolution, self.resolution]) + img = img.to(dtype=dtype, memory_format=memory_format) + y = self.fromrgb(img) + x = x + y if x is not None else y + img = upfirdn2d.downsample2d( + img, + self.resample_filter) if self.architecture == 'skip' else None + + # Main layers. + if self.architecture == 'resnet': + y = self.skip(x, gain=np.sqrt(0.5)) + x = self.conv0(x) + x = self.conv1(x, gain=np.sqrt(0.5)) + x = y.add_(x) + else: + x = self.conv0(x) + x = self.conv1(x) + + assert x.dtype == dtype + return x, img + + def extra_repr(self): + return f'resolution={self.resolution:d}, architecture={self.architecture:s}' + + +@persistence.persistent_class +class MinibatchStdLayer(torch.nn.Module): + + def __init__(self, group_size, num_channels=1): + super().__init__() + self.group_size = group_size + self.num_channels = num_channels + + def forward(self, x): + N, C, H, W = x.shape + with misc.suppress_tracer_warnings( + ): # as_tensor results are registered as constants + G = torch.min( + torch.as_tensor(self.group_size), + torch.as_tensor(N)) if self.group_size is not None else N + F = self.num_channels + c = C // F + + y = x.reshape( + G, -1, F, c, H, W + ) # [GnFcHW] Split minibatch N into n groups of size G, and channels C into F groups of size c. + y = y - y.mean(dim=0) # [GnFcHW] Subtract mean over group. + y = y.square().mean(dim=0) # [nFcHW] Calc variance over group. + y = (y + 1e-8).sqrt() # [nFcHW] Calc stddev over group. + y = y.mean(dim=[2, 3, + 4]) # [nF] Take average over channels and pixels. + y = y.reshape(-1, F, 1, 1) # [nF11] Add missing dimensions. + y = y.repeat(G, 1, H, W) # [NFHW] Replicate over group and pixels. + x = torch.cat([x, y], + dim=1) # [NCHW] Append to input as new channels. + return x + + def extra_repr(self): + return f'group_size={self.group_size}, num_channels={self.num_channels:d}' + + +@persistence.persistent_class +class DiscriminatorEpilogue(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels. + cmap_dim, # Dimensionality of mapped conditioning label, 0 = no label. + resolution, # Resolution of this block. + img_channels, # Number of input color channels. + architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'. + mbstd_group_size=4, # Group size for the minibatch standard deviation layer, None = entire minibatch. + mbstd_num_channels=1, # Number of features for the minibatch standard deviation layer, 0 = disable. + activation='lrelu', # Activation function: 'relu', 'lrelu', etc. + conv_clamp=None, # Clamp the output of convolution layers to +-X, None = disable clamping. + ): + assert architecture in ['orig', 'skip', 'resnet'] + super().__init__() + self.in_channels = in_channels + self.cmap_dim = cmap_dim + self.resolution = resolution + self.img_channels = img_channels + self.architecture = architecture + + if architecture == 'skip': + self.fromrgb = Conv2dLayer( + img_channels, + in_channels, + kernel_size=1, + activation=activation) + self.mbstd = MinibatchStdLayer( + group_size=mbstd_group_size, num_channels=mbstd_num_channels + ) if mbstd_num_channels > 0 else None + self.conv = Conv2dLayer( + in_channels + mbstd_num_channels, + in_channels, + kernel_size=3, + activation=activation, + conv_clamp=conv_clamp) + self.fc = FullyConnectedLayer( + in_channels * (resolution**2), in_channels, activation=activation) + self.out = FullyConnectedLayer(in_channels, + 1 if cmap_dim == 0 else cmap_dim) + + def forward(self, x, img, cmap, force_fp32=False): + misc.assert_shape( + x, [None, self.in_channels, self.resolution, self.resolution + ]) # [NCHW] + _ = force_fp32 # unused + dtype = torch.float32 + memory_format = torch.contiguous_format + + # FromRGB. + x = x.to(dtype=dtype, memory_format=memory_format) + if self.architecture == 'skip': + misc.assert_shape( + img, + [None, self.img_channels, self.resolution, self.resolution]) + img = img.to(dtype=dtype, memory_format=memory_format) + x = x + self.fromrgb(img) + + # Main layers. + if self.mbstd is not None: + x = self.mbstd(x) + x = self.conv(x) + x = self.fc(x.flatten(1)) + x = self.out(x) + + # Conditioning. + if self.cmap_dim > 0: + misc.assert_shape(cmap, [None, self.cmap_dim]) + x = (x * cmap).sum( + dim=1, keepdim=True) * (1 / np.sqrt(self.cmap_dim)) + + assert x.dtype == dtype + return x + + def extra_repr(self): + return f'resolution={self.resolution:d}, architecture={self.architecture:s}' + + +@persistence.persistent_class +class Discriminator(torch.nn.Module): + + def __init__( + self, + c_dim, # Conditioning label (C) dimensionality. + img_resolution, # Input resolution. + img_channels, # Number of input color channels. + architecture='resnet', # Architecture: 'orig', 'skip', 'resnet'. + channel_base=32768, # Overall multiplier for the number of channels. + channel_max=512, # Maximum number of channels in any layer. + num_fp16_res=4, # Use FP16 for the N highest resolutions. + conv_clamp=256, # Clamp the output of convolution layers to +-X, None = disable clamping. + cmap_dim=None, # Dimensionality of mapped conditioning label, None = default. + block_kwargs={}, # Arguments for DiscriminatorBlock. + mapping_kwargs={}, # Arguments for MappingNetwork. + epilogue_kwargs={}, # Arguments for DiscriminatorEpilogue. + ): + super().__init__() + self.c_dim = c_dim + self.img_resolution = img_resolution + self.img_resolution_log2 = int(np.log2(img_resolution)) + self.img_channels = img_channels + self.block_resolutions = [ + 2**i for i in range(self.img_resolution_log2, 2, -1) + ] + channels_dict = { + res: min(channel_base // res, channel_max) + for res in self.block_resolutions + [4] + } + fp16_resolution = max(2**(self.img_resolution_log2 + 1 - num_fp16_res), + 8) + + if cmap_dim is None: + cmap_dim = channels_dict[4] + if c_dim == 0: + cmap_dim = 0 + + common_kwargs = dict( + img_channels=img_channels, + architecture=architecture, + conv_clamp=conv_clamp) + cur_layer_idx = 0 + for res in self.block_resolutions: + in_channels = channels_dict[res] if res < img_resolution else 0 + tmp_channels = channels_dict[res] + out_channels = channels_dict[res // 2] + use_fp16 = (res >= fp16_resolution) + block = DiscriminatorBlock( + in_channels, + tmp_channels, + out_channels, + resolution=res, + first_layer_idx=cur_layer_idx, + use_fp16=use_fp16, + **block_kwargs, + **common_kwargs) + setattr(self, f'b{res}', block) + cur_layer_idx += block.num_layers + if c_dim > 0: + self.mapping = MappingNetwork( + z_dim=0, + c_dim=c_dim, + w_dim=cmap_dim, + num_ws=None, + w_avg_beta=None, + **mapping_kwargs) + self.b4 = DiscriminatorEpilogue( + channels_dict[4], + cmap_dim=cmap_dim, + resolution=4, + **epilogue_kwargs, + **common_kwargs) + + def forward(self, img, c, update_emas=False, **block_kwargs): + _ = update_emas # unused + x = None + for res in self.block_resolutions: + block = getattr(self, f'b{res}') + x, img = block(x, img, **block_kwargs) + + cmap = None + if self.c_dim > 0: + cmap = self.mapping(None, c) + x = self.b4(x, img, cmap) + return x + + def extra_repr(self): + return f'c_dim={self.c_dim:d}, img_resolution={self.img_resolution:d}, img_channels={self.img_channels:d}' diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/shape_utils.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/shape_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..14c8963ff667ce222608fe70b9e55333791771f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/shape_utils.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +""" +Utils for extracting 3D shapes using marching cubes. Based on code from DeepSDF (Park et al.) + +Takes as input an .mrc file and extracts a mesh. + +Ex. + python shape_utils.py my_shape.mrc +Ex. + python shape_utils.py myshapes_directory --level=12 +""" + +import numpy as np +import plyfile +import skimage.measure + + +def convert_sdf_samples_to_ply(numpy_3d_sdf_tensor, + voxel_grid_origin, + voxel_size, + ply_filename_out, + offset=None, + scale=None, + level=0.0): + + verts, faces, normals, values = skimage.measure.marching_cubes( + numpy_3d_sdf_tensor, level=level, spacing=[voxel_size] * 3) + mesh_points = np.zeros_like(verts) + mesh_points[:, 0] = voxel_grid_origin[0] + verts[:, 0] + mesh_points[:, 1] = voxel_grid_origin[1] + verts[:, 1] + mesh_points[:, 2] = voxel_grid_origin[2] + verts[:, 2] + + if scale is not None: + mesh_points = mesh_points / scale + if offset is not None: + mesh_points = mesh_points - offset + + num_verts = verts.shape[0] + num_faces = faces.shape[0] + + verts_tuple = np.zeros((num_verts, ), + dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) + + for i in range(0, num_verts): + verts_tuple[i] = tuple(mesh_points[i, :]) + + faces_building = [] + for i in range(0, num_faces): + faces_building.append(((faces[i, :].tolist(), ))) + faces_tuple = np.array( + faces_building, dtype=[('vertex_indices', 'i4', (3, ))]) + + el_verts = plyfile.PlyElement.describe(verts_tuple, 'vertex') + el_faces = plyfile.PlyElement.describe(faces_tuple, 'face') + + ply_data = plyfile.PlyData([el_verts, el_faces]) + ply_data.write(ply_filename_out) diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/superresolution.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/superresolution.py new file mode 100644 index 0000000000000000000000000000000000000000..5e8257f47e48b393bd67f072a40b751ee1bde8d3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/superresolution.py @@ -0,0 +1,493 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +"""Superresolution network architectures from the paper +"Efficient Geometry-aware 3D Generative Adversarial Networks".""" + +import numpy as np +import torch + +from modelscope.ops.image_control_3d_portrait.torch_utils import (misc, + persistence) +from modelscope.ops.image_control_3d_portrait.torch_utils.ops import upfirdn2d +from .networks_stylegan2 import (Conv2dLayer, SynthesisBlock, SynthesisLayer, + ToRGBLayer) + + +# for 512x512 generation +@persistence.persistent_class +class SuperresolutionHybrid8X(torch.nn.Module): + + def __init__( + self, + channels, + img_resolution, + sr_num_fp16_res, + sr_antialias, + num_fp16_res=4, + conv_clamp=None, + channel_base=None, + channel_max=None, # IGNORE + **block_kwargs): + super().__init__() + assert img_resolution == 512 + + use_fp16 = sr_num_fp16_res > 0 + self.input_resolution = 128 + self.sr_antialias = sr_antialias + self.block0 = SynthesisBlock( + channels, + 128, + w_dim=512, + resolution=256, + img_channels=3, + is_last=False, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.block1 = SynthesisBlock( + 128, + 64, + w_dim=512, + resolution=512, + img_channels=3, + is_last=True, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.register_buffer('resample_filter', + upfirdn2d.setup_filter([1, 3, 3, 1])) + + def forward(self, rgb, x, ws, **block_kwargs): + ws = ws[:, -1:, :].repeat(1, 3, 1) + + if x.shape[-1] != self.input_resolution: + x = torch.nn.functional.interpolate( + x, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + rgb = torch.nn.functional.interpolate( + rgb, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + + x, rgb = self.block0(x, rgb, ws, **block_kwargs) + x, rgb = self.block1(x, rgb, ws, **block_kwargs) + return rgb + + +# for 256x256 generation +@persistence.persistent_class +class SuperresolutionHybrid4X(torch.nn.Module): + + def __init__( + self, + channels, + img_resolution, + sr_num_fp16_res, + sr_antialias, + num_fp16_res=4, + conv_clamp=None, + channel_base=None, + channel_max=None, # IGNORE + **block_kwargs): + super().__init__() + assert img_resolution == 256 + use_fp16 = sr_num_fp16_res > 0 + self.sr_antialias = sr_antialias + self.input_resolution = 128 + self.block0 = SynthesisBlockNoUp( + channels, + 128, + w_dim=512, + resolution=128, + img_channels=3, + is_last=False, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.block1 = SynthesisBlock( + 128, + 64, + w_dim=512, + resolution=256, + img_channels=3, + is_last=True, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.register_buffer('resample_filter', + upfirdn2d.setup_filter([1, 3, 3, 1])) + + def forward(self, rgb, x, ws, **block_kwargs): + ws = ws[:, -1:, :].repeat(1, 3, 1) + + if x.shape[-1] < self.input_resolution: + x = torch.nn.functional.interpolate( + x, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + rgb = torch.nn.functional.interpolate( + rgb, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + + x, rgb = self.block0(x, rgb, ws, **block_kwargs) + x, rgb = self.block1(x, rgb, ws, **block_kwargs) + return rgb + + +# for 128 x 128 generation +@persistence.persistent_class +class SuperresolutionHybrid2X(torch.nn.Module): + + def __init__( + self, + channels, + img_resolution, + sr_num_fp16_res, + sr_antialias, + num_fp16_res=4, + conv_clamp=None, + channel_base=None, + channel_max=None, # IGNORE + **block_kwargs): + super().__init__() + assert img_resolution == 128 + + use_fp16 = sr_num_fp16_res > 0 + self.input_resolution = 64 + self.sr_antialias = sr_antialias + self.block0 = SynthesisBlockNoUp( + channels, + 128, + w_dim=512, + resolution=64, + img_channels=3, + is_last=False, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.block1 = SynthesisBlock( + 128, + 64, + w_dim=512, + resolution=128, + img_channels=3, + is_last=True, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.register_buffer('resample_filter', + upfirdn2d.setup_filter([1, 3, 3, 1])) + + def forward(self, rgb, x, ws, **block_kwargs): + ws = ws[:, -1:, :].repeat(1, 3, 1) + + if x.shape[-1] != self.input_resolution: + x = torch.nn.functional.interpolate( + x, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + rgb = torch.nn.functional.interpolate( + rgb, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + + x, rgb = self.block0(x, rgb, ws, **block_kwargs) + x, rgb = self.block1(x, rgb, ws, **block_kwargs) + return rgb + + +# TODO: Delete (here for backwards compatibility with old 256x256 models) +@persistence.persistent_class +class SuperresolutionHybridDeepfp32(torch.nn.Module): + + def __init__( + self, + channels, + img_resolution, + sr_num_fp16_res, + num_fp16_res=4, + conv_clamp=None, + channel_base=None, + channel_max=None, # IGNORE + **block_kwargs): + super().__init__() + assert img_resolution == 256 + use_fp16 = sr_num_fp16_res > 0 + + self.input_resolution = 128 + self.block0 = SynthesisBlockNoUp( + channels, + 128, + w_dim=512, + resolution=128, + img_channels=3, + is_last=False, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.block1 = SynthesisBlock( + 128, + 64, + w_dim=512, + resolution=256, + img_channels=3, + is_last=True, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.register_buffer('resample_filter', + upfirdn2d.setup_filter([1, 3, 3, 1])) + + def forward(self, rgb, x, ws, **block_kwargs): + ws = ws[:, -1:, :].repeat(1, 3, 1) + + if x.shape[-1] < self.input_resolution: + x = torch.nn.functional.interpolate( + x, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False) + rgb = torch.nn.functional.interpolate( + rgb, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False) + + x, rgb = self.block0(x, rgb, ws, **block_kwargs) + x, rgb = self.block1(x, rgb, ws, **block_kwargs) + return rgb + + +@persistence.persistent_class +class SynthesisBlockNoUp(torch.nn.Module): + + def __init__( + self, + in_channels, # Number of input channels, 0 = first block. + out_channels, # Number of output channels. + w_dim, # Intermediate latent (W) dimensionality. + resolution, # Resolution of this block. + img_channels, # Number of output color channels. + is_last, # Is this the last block? + architecture='skip', # Architecture: 'orig', 'skip', 'resnet'. + resample_filter=[ + 1, 3, 3, 1 + ], # Low-pass filter to apply when resampling activations. + conv_clamp=256, # Clamp the output of convolution layers to +-X, None = disable clamping. + use_fp16=False, # Use FP16 for this block? + fp16_channels_last=False, # Use channels-last memory format with FP16? + fused_modconv_default=True, # Default value of fused_modconv. + **layer_kwargs, # Arguments for SynthesisLayer. + ): + assert architecture in ['orig', 'skip', 'resnet'] + super().__init__() + self.in_channels = in_channels + self.w_dim = w_dim + self.resolution = resolution + self.img_channels = img_channels + self.is_last = is_last + self.architecture = architecture + self.use_fp16 = use_fp16 + self.channels_last = (use_fp16 and fp16_channels_last) + self.fused_modconv_default = fused_modconv_default + self.register_buffer('resample_filter', + upfirdn2d.setup_filter(resample_filter)) + self.num_conv = 0 + self.num_torgb = 0 + + if in_channels == 0: + self.const = torch.nn.Parameter( + torch.randn([out_channels, resolution, resolution])) + + if in_channels != 0: + self.conv0 = SynthesisLayer( + in_channels, + out_channels, + w_dim=w_dim, + resolution=resolution, + conv_clamp=conv_clamp, + channels_last=self.channels_last, + **layer_kwargs) + self.num_conv += 1 + + self.conv1 = SynthesisLayer( + out_channels, + out_channels, + w_dim=w_dim, + resolution=resolution, + conv_clamp=conv_clamp, + channels_last=self.channels_last, + **layer_kwargs) + self.num_conv += 1 + + if is_last or architecture == 'skip': + self.torgb = ToRGBLayer( + out_channels, + img_channels, + w_dim=w_dim, + conv_clamp=conv_clamp, + channels_last=self.channels_last) + self.num_torgb += 1 + + if in_channels != 0 and architecture == 'resnet': + self.skip = Conv2dLayer( + in_channels, + out_channels, + kernel_size=1, + bias=False, + up=2, + resample_filter=resample_filter, + channels_last=self.channels_last) + + def forward(self, + x, + img, + ws, + force_fp32=False, + fused_modconv=None, + update_emas=False, + **layer_kwargs): + _ = update_emas # unused + misc.assert_shape(ws, + [None, self.num_conv + self.num_torgb, self.w_dim]) + w_iter = iter(ws.unbind(dim=1)) + if ws.device.type != 'cuda': + force_fp32 = True + dtype = torch.float16 if self.use_fp16 and not force_fp32 else torch.float32 + memory_format = torch.channels_last if self.channels_last and not force_fp32 else torch.contiguous_format + if fused_modconv is None: + fused_modconv = self.fused_modconv_default + if fused_modconv == 'inference_only': + fused_modconv = (not self.training) + + # Input. + if self.in_channels == 0: + x = self.const.to(dtype=dtype, memory_format=memory_format) + x = x.unsqueeze(0).repeat([ws.shape[0], 1, 1, 1]) + else: + misc.assert_shape( + x, [None, self.in_channels, self.resolution, self.resolution]) + x = x.to(dtype=dtype, memory_format=memory_format) + + # Main layers. + if self.in_channels == 0: + x = self.conv1( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + elif self.architecture == 'resnet': + y = self.skip(x, gain=np.sqrt(0.5)) + x = self.conv0( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + x = self.conv1( + x, + next(w_iter), + fused_modconv=fused_modconv, + gain=np.sqrt(0.5), + **layer_kwargs) + x = y.add_(x) + else: + x = self.conv0( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + x = self.conv1( + x, next(w_iter), fused_modconv=fused_modconv, **layer_kwargs) + + # ToRGB. + # if img is not None: + # misc.assert_shape(img, [None, self.img_channels, self.resolution // 2, self.resolution // 2]) + # img = upfirdn2d.upsample2d(img, self.resample_filter) + if self.is_last or self.architecture == 'skip': + y = self.torgb(x, next(w_iter), fused_modconv=fused_modconv) + y = y.to( + dtype=torch.float32, memory_format=torch.contiguous_format) + img = img.add_(y) if img is not None else y + + assert x.dtype == dtype + assert img is None or img.dtype == torch.float32 + return x, img + + def extra_repr(self): + return f'resolution={self.resolution:d}, architecture={self.architecture:s}' + + +# for 512x512 generation +@persistence.persistent_class +class SuperresolutionHybrid8XDC(torch.nn.Module): + + def __init__( + self, + channels, + img_resolution, + sr_num_fp16_res, + sr_antialias, + num_fp16_res=4, + conv_clamp=None, + channel_base=None, + channel_max=None, # IGNORE + **block_kwargs): + super().__init__() + assert img_resolution == 512 + + use_fp16 = sr_num_fp16_res > 0 + self.input_resolution = 128 + self.sr_antialias = sr_antialias + self.block0 = SynthesisBlock( + channels, + 256, + w_dim=512, + resolution=256, + img_channels=3, + is_last=False, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + self.block1 = SynthesisBlock( + 256, + 128, + w_dim=512, + resolution=512, + img_channels=3, + is_last=True, + use_fp16=use_fp16, + conv_clamp=(256 if use_fp16 else None), + **block_kwargs) + + def forward(self, rgb, x, ws, **block_kwargs): + ws = ws[:, -1:, :].repeat(1, 3, 1) + + if x.shape[-1] != self.input_resolution: + x = torch.nn.functional.interpolate( + x, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + rgb = torch.nn.functional.interpolate( + rgb, + size=(self.input_resolution, self.input_resolution), + mode='bilinear', + align_corners=False, + antialias=self.sr_antialias) + + x, rgb = self.block0(x, rgb, ws, **block_kwargs) + x, rgb = self.block1(x, rgb, ws, **block_kwargs) + return rgb diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane.py new file mode 100644 index 0000000000000000000000000000000000000000..79a7f449d50f178dde84df3111ebe332a82cfd23 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import torch + +from modelscope.ops.image_control_3d_portrait.torch_utils import persistence +from .networks_stylegan2 import FullyConnectedLayer +from .networks_stylegan2 import Generator as StyleGAN2Backbone +from .superresolution import SuperresolutionHybrid8XDC +from .volumetric_rendering.ray_sampler import RaySampler +from .volumetric_rendering.renderer import ImportanceRenderer + + +@persistence.persistent_class +class TriPlaneGenerator(torch.nn.Module): + + def __init__( + self, + z_dim, # Input latent (Z) dimensionality. + c_dim, # Conditioning label (C) dimensionality. + w_dim, # Intermediate latent (W) dimensionality. + img_resolution, # Output resolution. + img_channels, # Number of output color channels. + sr_num_fp16_res=0, + mapping_kwargs={}, # Arguments for MappingNetwork. + rendering_kwargs={}, + sr_kwargs={}, + **synthesis_kwargs, # Arguments for SynthesisNetwork. + ): + super().__init__() + self.z_dim = z_dim + self.c_dim = c_dim + self.w_dim = w_dim + self.img_resolution = img_resolution + self.img_channels = img_channels + self.renderer = ImportanceRenderer() + self.ray_sampler = RaySampler() + self.backbone = StyleGAN2Backbone( + z_dim, + c_dim, + w_dim, + img_resolution=256, + img_channels=32 * 3, + mapping_kwargs=mapping_kwargs, + **synthesis_kwargs) + self.superresolution = SuperresolutionHybrid8XDC( + channels=32, + img_resolution=img_resolution, + sr_num_fp16_res=sr_num_fp16_res, + sr_antialias=rendering_kwargs['sr_antialias'], + **sr_kwargs) + self.decoder = OSGDecoder( + 32, { + 'decoder_lr_mul': rendering_kwargs.get('decoder_lr_mul', 1), + 'decoder_output_dim': 32 + }) + self.neural_rendering_resolution = 64 + self.rendering_kwargs = rendering_kwargs + + self._last_planes = None + + def mapping(self, + z, + c, + truncation_psi=1, + truncation_cutoff=None, + update_emas=False): + if self.rendering_kwargs['c_gen_conditioning_zero']: + c = torch.zeros_like(c) + return self.backbone.mapping( + z, + c * self.rendering_kwargs.get('c_scale', 0), + truncation_psi=truncation_psi, + truncation_cutoff=truncation_cutoff, + update_emas=update_emas) + + def synthesis(self, + ws, + c, + neural_rendering_resolution=None, + update_emas=False, + cache_backbone=False, + use_cached_backbone=False, + **synthesis_kwargs): + cam2world_matrix = c[:, :16].view(-1, 4, 4) + intrinsics = c[:, 16:25].view(-1, 3, 3) + + if neural_rendering_resolution is None: + neural_rendering_resolution = self.neural_rendering_resolution + else: + self.neural_rendering_resolution = neural_rendering_resolution + + # Create a batch of rays for volume rendering + ray_origins, ray_directions = self.ray_sampler( + cam2world_matrix, intrinsics, neural_rendering_resolution) + + # Create triplanes by running StyleGAN backbone + N, M, _ = ray_origins.shape + if use_cached_backbone and self._last_planes is not None: + planes = self._last_planes + else: + planes = self.backbone.synthesis( + ws, update_emas=update_emas, **synthesis_kwargs) + if cache_backbone: + self._last_planes = planes + + # Reshape output into three 32-channel planes + planes = planes.view( + len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) + + # Perform volume rendering + feature_samples, depth_samples, weights_samples = self.renderer( + planes, self.decoder, ray_origins, ray_directions, + self.rendering_kwargs) # channels last + + # Reshape into 'raw' neural-rendered image + H = W = self.neural_rendering_resolution + feature_image = feature_samples.permute(0, 2, 1).reshape( + N, feature_samples.shape[-1], H, W).contiguous() + depth_image = depth_samples.permute(0, 2, 1).reshape(N, 1, H, W) + + # Run superresolution to get final image + rgb_image = feature_image[:, :3] + sr_image = self.superresolution( + rgb_image, + feature_image, + ws, + noise_mode=self.rendering_kwargs['superresolution_noise_mode'], + **{ + k: synthesis_kwargs[k] + for k in synthesis_kwargs.keys() if k != 'noise_mode' + }) + + return { + 'image': sr_image, + 'image_raw': rgb_image, + 'image_depth': depth_image + } + + def sample(self, + coordinates, + directions, + z, + c, + truncation_psi=1, + truncation_cutoff=None, + update_emas=False, + **synthesis_kwargs): + # Compute RGB features, density for arbitrary 3D coordinates. Mostly used for extracting shapes. + ws = self.mapping( + z, + c, + truncation_psi=truncation_psi, + truncation_cutoff=truncation_cutoff, + update_emas=update_emas) + planes = self.backbone.synthesis( + ws, update_emas=update_emas, **synthesis_kwargs) + planes = planes.view( + len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) + return self.renderer.run_model(planes, self.decoder, coordinates, + directions, self.rendering_kwargs) + + def sample_mixed(self, + coordinates, + directions, + ws, + truncation_psi=1, + truncation_cutoff=None, + update_emas=False, + **synthesis_kwargs): + # Same as sample, but expects latent vectors 'ws' instead of Gaussian noise 'z' + planes = self.backbone.synthesis( + ws, update_emas=update_emas, **synthesis_kwargs) + planes = planes.view( + len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) + return self.renderer.run_model(planes, self.decoder, coordinates, + directions, self.rendering_kwargs) + + def forward(self, + z, + c, + truncation_psi=1, + truncation_cutoff=None, + neural_rendering_resolution=None, + update_emas=False, + cache_backbone=False, + use_cached_backbone=False, + **synthesis_kwargs): + # Render a batch of generated images. + ws = self.mapping( + z, + c, + truncation_psi=truncation_psi, + truncation_cutoff=truncation_cutoff, + update_emas=update_emas) + return self.synthesis( + ws, + c, + update_emas=update_emas, + neural_rendering_resolution=neural_rendering_resolution, + cache_backbone=cache_backbone, + use_cached_backbone=use_cached_backbone, + **synthesis_kwargs) + + +class OSGDecoder(torch.nn.Module): + + def __init__(self, n_features, options): + super().__init__() + self.hidden_dim = 64 + + self.net = torch.nn.Sequential( + FullyConnectedLayer( + n_features, + self.hidden_dim, + lr_multiplier=options['decoder_lr_mul']), torch.nn.Softplus(), + FullyConnectedLayer( + self.hidden_dim, + 1 + options['decoder_output_dim'], + lr_multiplier=options['decoder_lr_mul'])) + + def forward(self, sampled_features, ray_directions): + # Aggregate features + sampled_features = sampled_features.mean(1) + x = sampled_features + + N, M, C = x.shape + x = x.view(N * M, C) + + x = self.net(x) + x = x.view(N, M, -1) + rgb = torch.sigmoid(x[..., 1:]) * ( + 1 + 2 * 0.001) - 0.001 # Uses sigmoid clamping from MipNeRF + sigma = x[..., 0:1] + return {'rgb': rgb, 'sigma': sigma} diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane_encoder.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..29ec1e4055f13461e9b3bdb01b827c55af68a0ee --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/triplane_encoder.py @@ -0,0 +1,697 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +from functools import partial + +import segmentation_models_pytorch as smp +import torch +import torch.nn as nn +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from .networks_stylegan2 import FullyConnectedLayer +from .superresolution import SuperresolutionHybrid8XDC +from .volumetric_rendering.ray_sampler import RaySampler +from .volumetric_rendering.renderer import ImportanceRenderer + + +class DWConv(nn.Module): + + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x, H, W): + B, N, C = x.shape + x = x.transpose(1, 2).view(B, C, H, W) + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + + return x + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = self.fc1(x) + x = self.dwconv(x, H, W) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + sr_ratio=1): + super().__init__() + assert dim % num_heads == 0, f'dim {dim} should be divided by num_heads {num_heads}.' + + self.dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.sr_ratio = sr_ratio + if sr_ratio > 1: + self.sr = nn.Conv2d( + dim, dim, kernel_size=sr_ratio, stride=sr_ratio) + self.norm = nn.LayerNorm(dim) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + B, N, C = x.shape + q = self.q(x).reshape(B, N, self.num_heads, + C // self.num_heads).permute(0, 2, 1, 3) + + if self.sr_ratio > 1: + x_ = x.permute(0, 2, 1).reshape(B, C, H, W) + x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1) + x_ = self.norm(x_) + kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, + C // self.num_heads).permute( + 2, 0, 3, 1, 4) + else: + kv = self.kv(x).reshape(B, -1, 2, self.num_heads, + C // self.num_heads).permute( + 2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + sr_ratio=1): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + sr_ratio=sr_ratio) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = x + self.drop_path(self.attn(self.norm1(x), H, W)) + x = x + self.drop_path(self.mlp(self.norm2(x), H, W)) + + return x + + +class OverlapPatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + + def __init__(self, + img_size=224, + patch_size=7, + stride=4, + in_chans=3, + embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + + self.img_size = img_size + self.patch_size = patch_size + self.H, self.W = img_size[0] // patch_size[0], img_size[ + 1] // patch_size[1] + self.num_patches = self.H * self.W + self.proj = nn.Conv2d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=stride, + padding=(patch_size[0] // 2, patch_size[1] // 2)) + self.norm = nn.LayerNorm(embed_dim) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x): + x = self.proj(x) + _, _, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + + return x, H, W + + +class Encoder_low(nn.Module): + + def __init__(self, + img_size=64, + depth=5, + in_chans=256, + embed_dims=1024, + num_head=4, + mlp_ratio=2, + sr_ratio=1, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6)): + super().__init__() + self.depth = depth + + self.deeplabnet = smp.DeepLabV3( + encoder_name='resnet34', + encoder_depth=5, + encoder_weights=None, + decoder_channels=256, + in_channels=5, + classes=1) + + self.deeplabnet.encoder.conv1 = nn.Conv2d( + 5, + 64, + kernel_size=(7, 7), + stride=(2, 2), + padding=(3, 3), + bias=False) + self.deeplabnet.segmentation_head = nn.Sequential() + self.deeplabnet.encoder.bn1 = nn.Sequential() + self.deeplabnet.encoder.layer1[0].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer1[0].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer1[1].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer1[1].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer1[2].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer1[2].bn2 = nn.Sequential() + + self.deeplabnet.encoder.layer2[0].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer2[0].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer2[0].downsample[1] = nn.Sequential() + self.deeplabnet.encoder.layer2[1].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer2[1].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer2[2].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer2[2].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer2[3].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer2[3].bn2 = nn.Sequential() + + self.deeplabnet.encoder.layer3[0].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[0].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer3[0].downsample[1] = nn.Sequential() + self.deeplabnet.encoder.layer3[1].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[1].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer3[2].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[2].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer3[3].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[3].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer3[4].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[4].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer3[5].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer3[5].bn2 = nn.Sequential() + + self.deeplabnet.encoder.layer4[0].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer4[0].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer4[0].downsample[1] = nn.Sequential() + self.deeplabnet.encoder.layer4[1].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer4[1].bn2 = nn.Sequential() + self.deeplabnet.encoder.layer4[2].bn1 = nn.Sequential() + self.deeplabnet.encoder.layer4[2].bn2 = nn.Sequential() + + self.deeplabnet.decoder[0].convs[0][1] = nn.Sequential() + self.deeplabnet.decoder[0].convs[1][1] = nn.Sequential() + self.deeplabnet.decoder[0].convs[2][1] = nn.Sequential() + self.deeplabnet.decoder[0].convs[3][1] = nn.Sequential() + self.deeplabnet.decoder[0].convs[4][2] = nn.Sequential() + self.deeplabnet.decoder[0].project[1] = nn.Sequential() + self.deeplabnet.decoder[2] = nn.Sequential() + + self.patch_embed = OverlapPatchEmbed( + img_size=img_size, + patch_size=3, + stride=2, + in_chans=in_chans, + embed_dim=embed_dims) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + cur = 0 + self.vit_block = nn.ModuleList([ + Block( + dim=embed_dims, + num_heads=num_head, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[cur + i], + norm_layer=norm_layer, + sr_ratio=sr_ratio) for i in range(depth) + ]) + self.norm1 = norm_layer(embed_dims) + self.ps = nn.PixelShuffle(2) + + self.upsample1 = nn.UpsamplingBilinear2d(scale_factor=2) + self.conv1 = nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1) + self.relu1 = nn.ReLU() + self.upsample2 = nn.UpsamplingBilinear2d(scale_factor=2) + self.conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1) + self.relu2 = nn.ReLU() + self.conv3 = nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, input): + B = input.shape[0] + + f_low = self.deeplabnet(input) + x, H, W = self.patch_embed(f_low) + + for i, blk in enumerate(self.vit_block): + x = blk(x, H, W) + x = self.norm1(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + x = self.ps(x) + + x = self.relu1(self.conv1(self.upsample1(x))) + x = self.relu2(self.conv2(self.upsample2(x))) + x = self.conv3(x) + + return x + + +class Encoder_high(nn.Module): + + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(5, 64, kernel_size=7, stride=2, padding=3) + self.relu1 = nn.LeakyReLU(0.01) + self.conv2 = nn.Conv2d(64, 96, kernel_size=3, stride=1, padding=1) + self.relu2 = nn.LeakyReLU(0.01) + self.conv3 = nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1) + self.relu3 = nn.LeakyReLU(0.01) + self.conv4 = nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1) + self.relu4 = nn.LeakyReLU(0.01) + self.conv5 = nn.Conv2d(96, 96, kernel_size=3, stride=1, padding=1) + self.relu5 = nn.LeakyReLU(0.01) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x): + x = self.relu1(self.conv1(x)) + x = self.relu2(self.conv2(x)) + x = self.relu3(self.conv3(x)) + x = self.relu4(self.conv4(x)) + x = self.relu5(self.conv5(x)) + + return x + + +class MixFeature(nn.Module): + + def __init__(self, + img_size=256, + depth=1, + in_chans=128, + embed_dims=1024, + num_head=2, + mlp_ratio=2, + sr_ratio=2, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6)): + super().__init__() + self.conv1 = nn.Conv2d(192, 256, kernel_size=3, stride=1, padding=1) + self.relu1 = nn.LeakyReLU(0.01) + self.conv2 = nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1) + self.relu2 = nn.LeakyReLU(0.01) + + self.patch_embed = OverlapPatchEmbed( + img_size=img_size, + patch_size=3, + stride=2, + in_chans=in_chans, + embed_dim=embed_dims) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + cur = 0 + self.vit_block = nn.ModuleList([ + Block( + dim=embed_dims, + num_heads=num_head, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[cur + i], + norm_layer=norm_layer, + sr_ratio=sr_ratio) for i in range(depth) + ]) + self.norm1 = norm_layer(embed_dims) + self.ps = nn.PixelShuffle(2) + + self.conv3 = nn.Conv2d(352, 256, kernel_size=3, stride=1, padding=1) + self.relu3 = nn.LeakyReLU(0.01) + self.conv4 = nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1) + self.relu4 = nn.LeakyReLU(0.01) + self.conv5 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1) + self.relu5 = nn.LeakyReLU(0.01) + self.conv6 = nn.Conv2d(128, 96, kernel_size=3, stride=1, padding=1) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x_low, x_high): + x = torch.cat((x_low, x_high), 1) + B = x.shape[0] + + x = self.relu1(self.conv1(x)) + x = self.relu2(self.conv2(x)) + + x, H, W = self.patch_embed(x) + + for i, blk in enumerate(self.vit_block): + x = blk(x, H, W) + x = self.norm1(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + x = self.ps(x) + + x = torch.cat((x, x_low), 1) + x = self.relu3(self.conv3(x)) + x = self.relu4(self.conv4(x)) + x = self.relu5(self.conv5(x)) + x = self.conv6(x) + + return x + + +class OSGDecoder(torch.nn.Module): + + def __init__(self, n_features, options): + super().__init__() + self.hidden_dim = 64 + + self.net = torch.nn.Sequential( + FullyConnectedLayer( + n_features, + self.hidden_dim, + lr_multiplier=options['decoder_lr_mul']), torch.nn.Softplus(), + FullyConnectedLayer( + self.hidden_dim, + 1 + options['decoder_output_dim'], + lr_multiplier=options['decoder_lr_mul'])) + + def forward(self, sampled_features, ray_directions): + sampled_features = sampled_features.mean(1) + x = sampled_features + + N, M, C = x.shape + x = x.view(N * M, C) + + x = self.net(x) + x = x.view(N, M, -1) + rgb = torch.sigmoid(x[..., 1:]) * (1 + 2 * 0.001) - 0.001 + sigma = x[..., 0:1] + return {'rgb': rgb, 'sigma': sigma} + + +class TriplaneEncoder(nn.Module): + + def __init__(self, + img_resolution, + sr_num_fp16_res=0, + rendering_kwargs={}, + sr_kwargs={}): + super().__init__() + self.encoder_low = Encoder_low( + img_size=64, + depth=5, + in_chans=256, + embed_dims=1024, + num_head=4, + mlp_ratio=2, + sr_ratio=1, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6)) + self.encoder_high = Encoder_high() + self.mix = MixFeature( + img_size=256, + depth=1, + in_chans=128, + embed_dims=1024, + num_head=2, + mlp_ratio=2, + sr_ratio=2, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6)) + + self.renderer = ImportanceRenderer() + self.ray_sampler = RaySampler() + self.superresolution = SuperresolutionHybrid8XDC( + channels=32, + img_resolution=img_resolution, + sr_num_fp16_res=sr_num_fp16_res, + sr_antialias=rendering_kwargs['sr_antialias'], + **sr_kwargs) + self.decoder = OSGDecoder( + 32, { + 'decoder_lr_mul': rendering_kwargs.get('decoder_lr_mul', 1), + 'decoder_output_dim': 32 + }) + self.neural_rendering_resolution = 128 + self.rendering_kwargs = rendering_kwargs + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def gen_interfeats(self, ws, planes, camera_params): + planes = planes.view( + len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) + + cam2world_matrix = camera_params[:, :16].view(-1, 4, 4) + intrinsics = camera_params[:, 16:25].view(-1, 3, 3) + H = W = self.neural_rendering_resolution + ray_origins, ray_directions = self.ray_sampler( + cam2world_matrix, intrinsics, self.neural_rendering_resolution) + N, M, _ = ray_origins.shape + feature_samples, depth_samples, weights_samples = self.renderer( + planes, self.decoder, ray_origins, ray_directions, + self.rendering_kwargs) + feature_image = feature_samples.permute(0, 2, 1).reshape( + N, feature_samples.shape[-1], H, W).contiguous() + depth_image = depth_samples.permute(0, 2, 1).reshape(N, 1, H, W) + + rgb_image = feature_image[:, :3] + sr_image = self.superresolution( + rgb_image, feature_image, ws, noise_mode='const') + + return depth_image, feature_image, rgb_image, sr_image + + def sample(self, coordinates, directions, planes): + planes = planes.view( + len(planes), 3, 32, planes.shape[-2], planes.shape[-1]) + return self.renderer.run_model(planes, self.decoder, coordinates, + directions, self.rendering_kwargs) + + def forward(self, ws, x, camera_ref, camera_mv): + f = self.encoder_low(x) + f_high = self.encoder_high(x) + planes = self.mix(f, f_high) + + depth_ref, feature_ref, rgb_ref, sr_ref = self.gen_interfeats( + ws, planes, camera_ref) + if camera_mv is not None: + depth_mv, feature_mv, rgb_mv, sr_mv = self.gen_interfeats( + ws, planes, camera_mv) + else: + depth_mv = feature_mv = rgb_mv = sr_mv = None + + return planes, depth_ref, feature_ref, rgb_ref, sr_ref, depth_mv, feature_mv, rgb_mv, sr_mv + + +def get_parameter_number(net): + total_num = sum(p.numel() for p in net.parameters()) + trainable_num = sum(p.numel() for p in net.parameters() if p.requires_grad) + return {'Total': total_num, 'Trainable': trainable_num} diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/__init__.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dfebd04f47e6f6b1b44984c14c23b57d56f72240 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +# empty diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/math_utils.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/math_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc71f6306336f145340444349703c0236e423df0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/math_utils.py @@ -0,0 +1,137 @@ +# MIT License + +# Copyright (c) 2022 Petr Kellnhofer + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import torch + + +def transform_vectors(matrix: torch.Tensor, + vectors4: torch.Tensor) -> torch.Tensor: + """ + Left-multiplies MxM @ NxM. Returns NxM. + """ + res = torch.matmul(vectors4, matrix.T) + return res + + +def normalize_vecs(vectors: torch.Tensor) -> torch.Tensor: + """ + Normalize vector lengths. + """ + return vectors / (torch.norm(vectors, dim=-1, keepdim=True)) + + +def torch_dot(x: torch.Tensor, y: torch.Tensor): + """ + Dot product of two tensors. + """ + return (x * y).sum(-1) + + +def get_ray_limits_box(rays_o: torch.Tensor, rays_d: torch.Tensor, + box_side_length): + """ + Author: Petr Kellnhofer + Intersects rays with the [-1, 1] NDC volume. + Returns min and max distance of entry. + Returns -1 for no intersection. + https://www.scratchapixel.com/lessons/3d-basic-rendering/minimal-ray-tracer-rendering-simple-shapes/ray-box-intersection + """ + o_shape = rays_o.shape + rays_o = rays_o.detach().reshape(-1, 3) + rays_d = rays_d.detach().reshape(-1, 3) + + temp_min_1 = -1 * (box_side_length / 2) + temp_min_2 = -1 * (box_side_length / 2) + temp_min_3 = -1 * (box_side_length / 2) + bb_min = [temp_min_1, temp_min_2, temp_min_3] + temp_max_1 = 1 * (box_side_length / 2) + temp_max_2 = 1 * (box_side_length / 2) + temp_max_3 = 1 * (box_side_length / 2) + bb_max = [temp_max_1, temp_max_2, temp_max_3] + bounds = torch.tensor([bb_min, bb_max], + dtype=rays_o.dtype, + device=rays_o.device) + is_valid = torch.ones(rays_o.shape[:-1], dtype=bool, device=rays_o.device) + + # Precompute inverse for stability. + invdir = 1 / rays_d + sign = (invdir < 0).long() + + # Intersect with YZ plane. + tmin = (bounds.index_select(0, sign[..., 0])[..., 0] + - rays_o[..., 0]) * invdir[..., 0] + tmax = (bounds.index_select(0, 1 - sign[..., 0])[..., 0] + - rays_o[..., 0]) * invdir[..., 0] + + # Intersect with XZ plane. + tymin = (bounds.index_select(0, sign[..., 1])[..., 1] + - rays_o[..., 1]) * invdir[..., 1] + tymax = (bounds.index_select(0, 1 - sign[..., 1])[..., 1] + - rays_o[..., 1]) * invdir[..., 1] + + # Resolve parallel rays. + is_valid[torch.logical_or(tmin > tymax, tymin > tmax)] = False + + # Use the shortest intersection. + tmin = torch.max(tmin, tymin) + tmax = torch.min(tmax, tymax) + + # Intersect with XY plane. + tzmin = (bounds.index_select(0, sign[..., 2])[..., 2] + - rays_o[..., 2]) * invdir[..., 2] + tzmax = (bounds.index_select(0, 1 - sign[..., 2])[..., 2] + - rays_o[..., 2]) * invdir[..., 2] + + # Resolve parallel rays. + is_valid[torch.logical_or(tmin > tzmax, tzmin > tmax)] = False + + # Use the shortest intersection. + tmin = torch.max(tmin, tzmin) + tmax = torch.min(tmax, tzmax) + + # Mark invalid. + tmin[torch.logical_not(is_valid)] = -1 + tmax[torch.logical_not(is_valid)] = -2 + + return tmin.reshape(*o_shape[:-1], 1), tmax.reshape(*o_shape[:-1], 1) + + +def linspace(start: torch.Tensor, stop: torch.Tensor, num: int): + """ + Creates a tensor of shape [num, *start.shape] whose values are evenly spaced from start to end, inclusive. + Replicates but the multi-dimensional bahaviour of numpy.linspace in PyTorch. + """ + # create a tensor of 'num' steps from 0 to 1 + steps = torch.arange( + num, dtype=torch.float32, device=start.device) / ( + num - 1) + + # reshape the 'steps' tensor to [-1, *([1]*start.ndim)] to allow for broadcastings + # - using 'steps.reshape([-1, *([1]*start.ndim)])' would be nice here but torchscript + # "cannot statically infer the expected size of a list in this contex", hence the code below + for i in range(start.ndim): + steps = steps.unsqueeze(-1) + + # the output starts at 'start' and increments until 'stop' in each dimension + out = start[None] + steps * (stop - start)[None] + + return out diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_marcher.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_marcher.py new file mode 100644 index 0000000000000000000000000000000000000000..2e039f5c27714fbf25ff62143310304079911250 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_marcher.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +""" +The ray marcher takes the raw output of the implicit representation and +uses the volume rendering equation to produce composited colors and depths. +Based off of the implementation in MipNeRF (this one doesn't do any cone tracing though!) +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class MipRayMarcher2(nn.Module): + + def __init__(self): + super().__init__() + + def run_forward(self, colors, densities, depths, rendering_options): + deltas = depths[:, :, 1:] - depths[:, :, :-1] + colors_mid = (colors[:, :, :-1] + colors[:, :, 1:]) / 2 + densities_mid = (densities[:, :, :-1] + densities[:, :, 1:]) / 2 + depths_mid = (depths[:, :, :-1] + depths[:, :, 1:]) / 2 + + if rendering_options['clamp_mode'] == 'softplus': + densities_mid = F.softplus( + densities_mid + - 1) # activation bias of -1 makes things initialize better + else: + assert False, 'MipRayMarcher only supports `clamp_mode`=`softplus`!' + + density_delta = densities_mid * deltas + + alpha = 1 - torch.exp(-density_delta) + + alpha_shifted = torch.cat( + [torch.ones_like(alpha[:, :, :1]), 1 - alpha + 1e-10], -2) + weights = alpha * torch.cumprod(alpha_shifted, -2)[:, :, :-1] + + composite_rgb = torch.sum(weights * colors_mid, -2) + weight_total = weights.sum(2) + composite_depth = torch.sum(weights * depths_mid, -2) / weight_total + + # clip the composite to min/max range of depths + composite_depth = torch.nan_to_num(composite_depth, float('inf')) + composite_depth = torch.clamp(composite_depth, torch.min(depths), + torch.max(depths)) + + if rendering_options.get('white_back', False): + composite_rgb = composite_rgb + 1 - weight_total + + composite_rgb = composite_rgb * 2 - 1 # Scale to (-1, 1) + + return composite_rgb, composite_depth, weights + + def forward(self, colors, densities, depths, rendering_options): + composite_rgb, composite_depth, weights = self.run_forward( + colors, densities, depths, rendering_options) + + return composite_rgb, composite_depth, weights diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_sampler.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..40529087b9e11b06324a740b236ec3bef01338a8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/ray_sampler.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +""" +The ray sampler is a module that takes in camera matrices and resolution and batches of rays. +Expects cam2world matrices that use the OpenCV camera coordinate system conventions. +""" + +import torch + + +class RaySampler(torch.nn.Module): + + def __init__(self): + super().__init__() + self.ray_origins_h, self.ray_directions, self.depths, self.image_coords, self.rendering_options = \ + None, None, None, None, None + + def forward(self, cam2world_matrix, intrinsics, resolution): + """ + Create batches of rays and return origins and directions. + + cam2world_matrix: (N, 4, 4) + intrinsics: (N, 3, 3) + resolution: int + + ray_origins: (N, M, 3) + ray_dirs: (N, M, 2) + """ + N, M = cam2world_matrix.shape[0], resolution**2 + cam_locs_world = cam2world_matrix[:, :3, 3] + fx = intrinsics[:, 0, 0] + fy = intrinsics[:, 1, 1] + cx = intrinsics[:, 0, 2] + cy = intrinsics[:, 1, 2] + sk = intrinsics[:, 0, 1] + + uv = torch.stack( + torch.meshgrid( + torch.arange( + resolution, + dtype=torch.float32, + device=cam2world_matrix.device), + torch.arange( + resolution, + dtype=torch.float32, + device=cam2world_matrix.device))) * (1. / resolution) + ( + 0.5 / resolution) + uv = uv.flip(0).reshape(2, -1).transpose(1, 0) + uv = uv.unsqueeze(0).repeat(cam2world_matrix.shape[0], 1, 1) + + x_cam = uv[:, :, 0].view(N, -1) + y_cam = uv[:, :, 1].view(N, -1) + z_cam = torch.ones((N, M), device=cam2world_matrix.device) + + x_lift = (x_cam - cx.unsqueeze(-1) + cy.unsqueeze(-1) + * sk.unsqueeze(-1) / fy.unsqueeze(-1) - sk.unsqueeze(-1) + * y_cam / fy.unsqueeze(-1)) / fx.unsqueeze(-1) * z_cam + y_lift = (y_cam - cy.unsqueeze(-1)) / fy.unsqueeze(-1) * z_cam + + cam_rel_points = torch.stack( + (x_lift, y_lift, z_cam, torch.ones_like(z_cam)), dim=-1) + + world_rel_points = torch.bmm(cam2world_matrix, + cam_rel_points.permute(0, 2, 1)).permute( + 0, 2, 1)[:, :, :3] + + ray_dirs = world_rel_points - cam_locs_world[:, None, :] + ray_dirs = torch.nn.functional.normalize(ray_dirs, dim=2) + + ray_origins = cam_locs_world.unsqueeze(1).repeat( + 1, ray_dirs.shape[1], 1) + + return ray_origins, ray_dirs diff --git a/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/renderer.py b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..fce288bf998ee2f091c11d0690c0daf3be4c53d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_control_3d_portrait/network/volumetric_rendering/renderer.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +""" +The renderer is a module that takes in rays, decides where to sample along each +ray, and computes pixel colors using the volume rendering equation. +""" + +import math + +import torch +import torch.nn as nn + +from . import math_utils +from .ray_marcher import MipRayMarcher2 + + +def generate_planes(): + """ + Defines planes by the three vectors that form the "axes" of the + plane. Should work with arbitrary number of planes and planes of + arbitrary orientation. + """ + return torch.tensor( + [[[1, 0, 0], [0, 1, 0], [0, 0, 1]], [[1, 0, 0], [0, 0, 1], [0, 1, 0]], + [[0, 0, 1], [1, 0, 0], [0, 1, 0]]], + dtype=torch.float32) + + +def project_onto_planes(planes, coordinates): + """ + Does a projection of a 3D point onto a batch of 2D planes, + returning 2D plane coordinates. + + Takes plane axes of shape n_planes, 3, 3 + # Takes coordinates of shape N, M, 3 + # returns projections of shape N*n_planes, M, 2 + """ + N, M, C = coordinates.shape + n_planes, _, _ = planes.shape + coordinates = coordinates.unsqueeze(1).expand(-1, n_planes, -1, + -1).reshape( + N * n_planes, M, 3) + inv_planes = torch.linalg.inv(planes).unsqueeze(0).expand( + N, -1, -1, -1).reshape(N * n_planes, 3, 3).to(coordinates.device) + projections = torch.bmm(coordinates, inv_planes) + return projections[..., :2] + + +def sample_from_planes(plane_axes, + plane_features, + coordinates, + mode='bilinear', + padding_mode='zeros', + box_warp=None): + assert padding_mode == 'zeros' + N, n_planes, C, H, W = plane_features.shape + _, M, _ = coordinates.shape + plane_features = plane_features.view(N * n_planes, C, H, W) + + coordinates = (2 / box_warp) * coordinates # TODO: add specific box bounds + + projected_coordinates = project_onto_planes(plane_axes, + coordinates).unsqueeze(1) + output_features = torch.nn.functional.grid_sample( + plane_features, + projected_coordinates.float(), + mode=mode, + padding_mode=padding_mode, + align_corners=False).permute(0, 3, 2, 1).reshape(N, n_planes, M, C) + return output_features + + +def sample_from_3dgrid(grid, coordinates): + """ + Expects coordinates in shape (batch_size, num_points_per_batch, 3) + Expects grid in shape (1, channels, H, W, D) + (Also works if grid has batch size) + Returns sampled features of shape (batch_size, num_points_per_batch, feature_channels) + """ + batch_size, n_coords, n_dims = coordinates.shape + sampled_features = torch.nn.functional.grid_sample( + grid.expand(batch_size, -1, -1, -1, -1), + coordinates.reshape(batch_size, 1, 1, -1, n_dims), + mode='bilinear', + padding_mode='zeros', + align_corners=False) + N, C, H, W, D = sampled_features.shape + sampled_features = sampled_features.permute(0, 4, 3, 2, + 1).reshape(N, H * W * D, C) + return sampled_features + + +class ImportanceRenderer(torch.nn.Module): + + def __init__(self): + super().__init__() + self.ray_marcher = MipRayMarcher2() + self.plane_axes = generate_planes() + + def forward(self, planes, decoder, ray_origins, ray_directions, + rendering_options): + self.plane_axes = self.plane_axes.to(ray_origins.device) + + if rendering_options['ray_start'] == rendering_options[ + 'ray_end'] == 'auto': + ray_start, ray_end = math_utils.get_ray_limits_box( + ray_origins, + ray_directions, + box_side_length=rendering_options['box_warp']) + is_ray_valid = ray_end > ray_start + if torch.any(is_ray_valid).item(): + ray_start[~is_ray_valid] = ray_start[is_ray_valid].min() + ray_end[~is_ray_valid] = ray_start[is_ray_valid].max() + depths_coarse = self.sample_stratified( + ray_origins, ray_start, ray_end, + rendering_options['depth_resolution'], + rendering_options['disparity_space_sampling']) + else: + # Create stratified depth samples + depths_coarse = self.sample_stratified( + ray_origins, rendering_options['ray_start'], + rendering_options['ray_end'], + rendering_options['depth_resolution'], + rendering_options['disparity_space_sampling']) + + batch_size, num_rays, samples_per_ray, _ = depths_coarse.shape + + # Coarse Pass + sample_coordinates = ( + ray_origins.unsqueeze(-2) + + depths_coarse * ray_directions.unsqueeze(-2)).reshape( + batch_size, -1, 3) + sample_directions = ray_directions.unsqueeze(-2).expand( + -1, -1, samples_per_ray, -1).reshape(batch_size, -1, 3) + + out = self.run_model(planes, decoder, sample_coordinates, + sample_directions, rendering_options) + colors_coarse = out['rgb'] + densities_coarse = out['sigma'] + colors_coarse = colors_coarse.reshape(batch_size, num_rays, + samples_per_ray, + colors_coarse.shape[-1]) + densities_coarse = densities_coarse.reshape(batch_size, num_rays, + samples_per_ray, 1) + + # Fine Pass + N_importance = rendering_options['depth_resolution_importance'] + if N_importance > 0: + _, _, weights = self.ray_marcher(colors_coarse, densities_coarse, + depths_coarse, rendering_options) + + depths_fine = self.sample_importance(depths_coarse, weights, + N_importance) + + sample_directions = ray_directions.unsqueeze(-2).expand( + -1, -1, N_importance, -1).reshape(batch_size, -1, 3) + sample_coordinates = ( + ray_origins.unsqueeze(-2) + + depths_fine * ray_directions.unsqueeze(-2)).reshape( + batch_size, -1, 3) + + out = self.run_model(planes, decoder, sample_coordinates, + sample_directions, rendering_options) + colors_fine = out['rgb'] + densities_fine = out['sigma'] + colors_fine = colors_fine.reshape(batch_size, num_rays, + N_importance, + colors_fine.shape[-1]) + densities_fine = densities_fine.reshape(batch_size, num_rays, + N_importance, 1) + + all_depths, all_colors, all_densities = self.unify_samples( + depths_coarse, colors_coarse, densities_coarse, depths_fine, + colors_fine, densities_fine) + + # Aggregate + rgb_final, depth_final, weights = self.ray_marcher( + all_colors, all_densities, all_depths, rendering_options) + else: + rgb_final, depth_final, weights = self.ray_marcher( + colors_coarse, densities_coarse, depths_coarse, + rendering_options) + + return rgb_final, depth_final, weights.sum(2) + + def run_model(self, planes, decoder, sample_coordinates, sample_directions, + options): + sampled_features = sample_from_planes( + self.plane_axes, + planes, + sample_coordinates, + padding_mode='zeros', + box_warp=options['box_warp']) + + out = decoder(sampled_features, sample_directions) + if options.get('density_noise', 0) > 0: + out['sigma'] += torch.randn_like( + out['sigma']) * options['density_noise'] + return out + + def sort_samples(self, all_depths, all_colors, all_densities): + _, indices = torch.sort(all_depths, dim=-2) + all_depths = torch.gather(all_depths, -2, indices) + all_colors = torch.gather( + all_colors, -2, indices.expand(-1, -1, -1, all_colors.shape[-1])) + all_densities = torch.gather(all_densities, -2, + indices.expand(-1, -1, -1, 1)) + return all_depths, all_colors, all_densities + + def unify_samples(self, depths1, colors1, densities1, depths2, colors2, + densities2): + all_depths = torch.cat([depths1, depths2], dim=-2) + all_colors = torch.cat([colors1, colors2], dim=-2) + all_densities = torch.cat([densities1, densities2], dim=-2) + + _, indices = torch.sort(all_depths, dim=-2) + all_depths = torch.gather(all_depths, -2, indices) + all_colors = torch.gather( + all_colors, -2, indices.expand(-1, -1, -1, all_colors.shape[-1])) + all_densities = torch.gather(all_densities, -2, + indices.expand(-1, -1, -1, 1)) + + return all_depths, all_colors, all_densities + + def sample_stratified(self, + ray_origins, + ray_start, + ray_end, + depth_resolution, + disparity_space_sampling=False): + """ + Return depths of approximately uniformly spaced samples along rays. + """ + N, M, _ = ray_origins.shape + if disparity_space_sampling: + depths_coarse = torch.linspace( + 0, 1, depth_resolution, + device=ray_origins.device).reshape(1, 1, depth_resolution, + 1).repeat(N, M, 1, 1) + depth_delta = 1 / (depth_resolution - 1) + depths_coarse += torch.rand_like(depths_coarse) * depth_delta + depths_coarse = 1. / (1. / ray_start * (1. - depths_coarse) + + 1. / ray_end * depths_coarse) + else: + if type(ray_start) == torch.Tensor: + depths_coarse = math_utils.linspace(ray_start, ray_end, + depth_resolution).permute( + 1, 2, 0, 3) + depth_delta = (ray_end - ray_start) / (depth_resolution - 1) + depths_coarse += torch.rand_like(depths_coarse) * depth_delta[ + ..., None] + else: + depths_coarse = torch.linspace( + ray_start, + ray_end, + depth_resolution, + device=ray_origins.device).reshape(1, 1, depth_resolution, + 1).repeat(N, M, 1, 1) + depth_delta = (ray_end - ray_start) / (depth_resolution - 1) + depths_coarse += torch.rand_like(depths_coarse) * depth_delta + + return depths_coarse + + def sample_importance(self, z_vals, weights, N_importance): + """ + Return depths of importance sampled points along rays. See NeRF importance sampling for more. + """ + with torch.no_grad(): + batch_size, num_rays, samples_per_ray, _ = z_vals.shape + + z_vals = z_vals.reshape(batch_size * num_rays, samples_per_ray) + weights = weights.reshape( + batch_size * num_rays, + -1) # -1 to account for loss of 1 sample in MipRayMarcher + + # smooth weights + weights = torch.nn.functional.max_pool1d( + weights.unsqueeze(1).float(), 2, 1, padding=1) + weights = torch.nn.functional.avg_pool1d(weights, 2, 1).squeeze() + weights = weights + 0.01 + + z_vals_mid = 0.5 * (z_vals[:, :-1] + z_vals[:, 1:]) + importance_z_vals = self.sample_pdf(z_vals_mid, weights[:, 1:-1], + N_importance).detach().reshape( + batch_size, num_rays, + N_importance, 1) + return importance_z_vals + + def sample_pdf(self, bins, weights, N_importance, det=False, eps=1e-5): + """ + Sample @N_importance samples from @bins with distribution defined by @weights. + Inputs: + bins: (N_rays, N_samples_+1) where N_samples_ is "the number of coarse samples per ray - 2" + weights: (N_rays, N_samples_) + N_importance: the number of samples to draw from the distribution + det: deterministic or not + eps: a small number to prevent division by zero + Outputs: + samples: the sampled samples + """ + N_rays, N_samples_ = weights.shape + weights = weights + eps # prevent division by zero (don't do inplace op!) + pdf = weights / torch.sum( + weights, -1, keepdim=True) # (N_rays, N_samples_) + cdf = torch.cumsum( + pdf, -1) # (N_rays, N_samples), cumulative distribution function + cdf = torch.cat([torch.zeros_like(cdf[:, :1]), cdf], + -1) # (N_rays, N_samples_+1) + # padded to 0~1 inclusive + + if det: + u = torch.linspace(0, 1, N_importance, device=bins.device) + u = u.expand(N_rays, N_importance) + else: + u = torch.rand(N_rays, N_importance, device=bins.device) + u = u.contiguous() + + inds = torch.searchsorted(cdf, u, right=True) + below = torch.clamp_min(inds - 1, 0) + above = torch.clamp_max(inds, N_samples_) + + inds_sampled = torch.stack([below, above], + -1).view(N_rays, 2 * N_importance) + cdf_g = torch.gather(cdf, 1, + inds_sampled).view(N_rays, N_importance, 2) + bins_g = torch.gather(bins, 1, + inds_sampled).view(N_rays, N_importance, 2) + + denom = cdf_g[..., 1] - cdf_g[..., 0] + denom[denom < eps] = 1 + + samples = bins_g[..., 0] + (u - cdf_g[..., 0]) / denom * ( + bins_g[..., 1] - bins_g[..., 0]) + return samples diff --git a/runtime/Lib/site_packages/models/cv/image_debanding/__init__.py b/runtime/Lib/site_packages/models/cv/image_debanding/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..850a03ec4efb06c9fa3e94ed43558bad329a3f58 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_debanding/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .rrdb import RRDBImageDebanding + +else: + _import_structure = { + 'rrdb': ['RRDBImageDebanding'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/__init__.py b/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a063db5a319541b0bf0d6caad197ca7165861d69 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/__init__.py @@ -0,0 +1 @@ +from .rrdb_image_debanding import RRDBImageDebanding diff --git a/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/rrdb_image_debanding.py b/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/rrdb_image_debanding.py new file mode 100644 index 0000000000000000000000000000000000000000..c21965f450faa15ed24f2982ac7530e23a02d9e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_debanding/rrdb/rrdb_image_debanding.py @@ -0,0 +1,91 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +'''RRDB debanding network +This model use rrdbnet to achieve image debanding task. +Training data is obtained from: +https://github.com/akshay-kap/Meng-699-Image-Banding-detection +''' +import os.path as osp +from typing import Dict, Union + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.super_resolution import RRDBNet +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['RRDBImageDebanding'] + + +@MODELS.register_module(Tasks.image_debanding, module_name=Models.rrdb) +class RRDBImageDebanding(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image color enhance model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.num_feat = 64 + self.num_block = 23 + self.scale = 1 + self.model = RRDBNet( + num_in_ch=3, + num_out_ch=3, + num_feat=self.num_feat, + num_block=self.num_block, + num_grow_ch=32, + scale=self.scale) + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.model = self.model.to(self._device) + + self.model = self._load_pretrained(self.model, model_path) + + if self.training: + self.model.train() + else: + self.model.eval() + + def _evaluate_postprocess(self, src: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(src) + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target, 1, 0)) + + preds = [(pred.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for pred in preds] + targets = [(target.data * 255.).squeeze(0).type(torch.uint8).permute( + 1, 2, 0).cpu().numpy() for target in targets] + + return {'pred': preds, 'target': targets} + + def _inference_forward(self, src: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.model(src).clamp(0, 1)} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + for key, value in input.items(): + input[key] = input[key].to(self._device) + if 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self._inference_forward(**input) diff --git a/runtime/Lib/site_packages/models/cv/image_deblur/__init__.py b/runtime/Lib/site_packages/models/cv/image_deblur/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..36edb9123c123393e4a0726fa8d22b202c98f8e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_deblur/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .nafnet_for_image_deblur import NAFNetForImageDeblur + +else: + _import_structure = {'nafnet_for_image_deblur': ['NAFNetForImageDeblur']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_deblur/nafnet_for_image_deblur.py b/runtime/Lib/site_packages/models/cv/image_deblur/nafnet_for_image_deblur.py new file mode 100644 index 0000000000000000000000000000000000000000..129e806162cd035e07fff9e93cf85cf48784cf00 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_deblur/nafnet_for_image_deblur.py @@ -0,0 +1,106 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch.cuda + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_denoise.nafnet.NAFNet_arch import (NAFNet, + PSNRLoss) +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() +__all__ = ['NAFNetForImageDeblur'] + + +@MODELS.register_module(Tasks.image_deblurring, module_name=Models.nafnet) +class NAFNetForImageDeblur(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image deblur model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + self.model = NAFNet(**self.config.model.network_g) + self.loss = PSNRLoss() + self.model = self._load_pretrained(self.model, model_path) + + def crop_process(self, input): + output = torch.zeros_like(input) # [1, C, H, W] + # determine crop_h and crop_w + ih, iw = input.shape[-2:] + crop_rows, crop_cols = max(ih // 512, 1), max(iw // 512, 1) + overlap = 16 + + step_h, step_w = ih // crop_rows, iw // crop_cols + for y in range(crop_rows): + for x in range(crop_cols): + crop_y = step_h * y + crop_x = step_w * x + + crop_h = step_h if y < crop_rows - 1 else ih - crop_y + crop_w = step_w if x < crop_cols - 1 else iw - crop_x + + crop_frames = input[:, :, + max(0, crop_y - overlap + ):min(crop_y + crop_h + overlap, ih), + max(0, crop_x - overlap + ):min(crop_x + crop_w + + overlap, iw)].contiguous() + h_start = overlap if max(0, crop_y - overlap) > 0 else 0 + w_start = overlap if max(0, crop_x - overlap) > 0 else 0 + h_end = h_start + crop_h if min(crop_y + crop_h + + overlap, ih) < ih else ih + w_end = w_start + crop_w if min(crop_x + crop_w + + overlap, iw) < iw else iw + + output[:, :, crop_y:crop_y + crop_h, + crop_x:crop_x + crop_w] = self.model( + crop_frames)[:, :, h_start:h_end, + w_start:w_end].clamp(0, 1) + return output + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + preds = self.model(input) + return {'loss': self.loss(preds, target)} + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.crop_process(input).cpu()} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.crop_process(input).cpu() + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target.cpu(), 1, 0)) + + return {'pred': preds, 'target': targets} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/__init__.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ef73351a3d403c8275b9f9a552da2fa0372c22fa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .defrcn_for_fewshot import DeFRCNForFewShot + +else: + _import_structure = {'defrcn_for_fewshot': ['DeFRCNForFewShot']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/defrcn_for_fewshot.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/defrcn_for_fewshot.py new file mode 100644 index 0000000000000000000000000000000000000000..77b71888ad4bc492d2c483f0bbff898530ac2825 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/defrcn_for_fewshot.py @@ -0,0 +1,112 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.msdatasets import MsDataset +from modelscope.utils.config import Config +from modelscope.utils.constant import DownloadMode, ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .models.calibration_layer import PrototypicalCalibrationBlock +from .models.defrcn import DeFRCN +from .utils.configuration_mapper import CfgMapper +from .utils.register_data import register_data +from .utils.requirements_check import requires_version + +logger = get_logger() +__all__ = ['DeFRCNForFewShot'] + + +@MODELS.register_module( + Tasks.image_fewshot_detection, module_name=Models.defrcn) +class DeFRCNForFewShot(TorchModel): + """ Few-shot object detection model DeFRCN. The model requires detectron2-0.3 and pytorch-1.11. + Model config params mainly from detectron2, you can use detectron2 config file to initialize model. + Detail configs can be visited on detectron2.config.defaults and .models.defaults_config. + """ + + def __init__(self, + model_dir: str, + _cfg_dict: Config = None, + *args, + **kwargs): + """initialize the few-shot defrcn model from the `model_dir` path. + + Args: + model_dir (str): the model path. + _cfg_dict (Config): An optional model config. If provided, it will replace + the config read out of the `model_name_or_path` + """ + requires_version() + + super().__init__(model_dir, *args, **kwargs) + + if _cfg_dict is None: + self.config = Config.from_file( + os.path.join(model_dir, ModelFile.CONFIGURATION)) + else: + self.config = _cfg_dict + + self.model_cfg = CfgMapper(self.config).__call__() + + data_dir = self.config.safe_get('datasets.root', None) + data_type = self.config.safe_get('datasets.type', 'pascal_voc') + + if self.training or self.model_cfg.TEST.PCB_ENABLE: + if data_dir is None: # use default datasets + dataset_name = 'VOC_fewshot' if data_type == 'pascal_voc' else 'coco2014_fewshot' + logger.warning('data_dir is none, use default {} data.'.format( + dataset_name)) + data_voc = MsDataset.load( + dataset_name=dataset_name, + namespace='shimin2023', + split='train', + download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) + data_dir = os.path.join( + data_voc.config_kwargs['split_config']['train'], 'data') + logger.info('{} datasets download dir is {}'.format( + dataset_name, data_dir)) + register_data(data_type, data_dir) + + self.model = DeFRCN(self.model_cfg) + + if self.model_cfg.TEST.PCB_ENABLE: + if not os.path.exists(self.model_cfg.TEST.PCB_MODELPATH): + logger.warning('{} no model.'.format( + self.model_cfg.TEST.PCB_MODELPATH)) + self.model_cfg.TEST.PCB_MODELPATH = os.path.join( + model_dir, + 'ImageNetPretrained/torchvision/resnet101-5d3b4d8f.pth') + logger.info('PCB use default model {}'.format( + self.model_cfg.TEST.PCB_MODELPATH)) + self.pcb = PrototypicalCalibrationBlock(self.model_cfg) + + self.model_cfg.freeze() + + def forward(self, inputs) -> Any: + """return the result by the model + + Args: + inputs (list): the preprocessed data + + Returns: + Any: results + """ + if self.training: + return self.model.forward(inputs) + else: + return self.model.inference(inputs) + + def inference(self, input: Dict[str, Any]) -> Any: + with torch.no_grad(): + results = self.model([input]) + if self.model_cfg.TEST.PCB_ENABLE: + results = self.pcb.execute_calibration([input], results) + return results[0] if len(results) > 0 else None + + def get_model_cfg(self): + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/__init__.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/coco_evaluation.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..9dd08cc2d0e993b08cc709f0c3354a72aa62908b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/coco_evaluation.py @@ -0,0 +1,368 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/evaluation/coco_evaluation.py + +import contextlib +import copy +import io +import itertools +import logging +import os +from collections import OrderedDict + +import json +import numpy as np +import torch +from detectron2.data import MetadataCatalog +from detectron2.data.datasets.coco import convert_to_coco_json +from detectron2.evaluation import DatasetEvaluator +from detectron2.structures import BoxMode +from detectron2.utils import comm as comm +from detectron2.utils.logger import create_small_table +from fvcore.common.file_io import PathManager +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + + +class COCOEvaluator(DatasetEvaluator): + + def __init__(self, dataset_name, distributed, output_dir=None): + + self._distributed = distributed + self._output_dir = output_dir + self._dataset_name = dataset_name + self._cpu_device = torch.device('cpu') + self._logger = logging.getLogger(__name__) + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, 'json_file'): + self._logger.warning( + f"json_file was not found in MetaDataCatalog for '{dataset_name}'" + ) + cache_path = convert_to_coco_json(dataset_name, output_dir) + self._metadata.json_file = cache_path + self._is_splits = 'all' in dataset_name or 'base' in dataset_name \ + or 'novel' in dataset_name + self._base_classes = [ + 8, + 10, + 11, + 13, + 14, + 15, + 22, + 23, + 24, + 25, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 65, + 70, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + ] + self._novel_classes = [ + 1, 2, 3, 4, 5, 6, 7, 9, 16, 17, 18, 19, 20, 21, 44, 62, 63, 64, 67, + 72 + ] + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + self._do_evaluation = 'annotations' in self._coco_api.dataset + + def reset(self): + self._predictions = [] + self._coco_results = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {'image_id': input['image_id']} + # TODO this is ugly + if 'instances' in output: + instances = output['instances'].to(self._cpu_device) + prediction['instances'] = instances_to_coco_json( + instances, input['image_id']) + self._predictions.append(prediction) + + def evaluate(self): + if self._distributed: + comm.synchronize() + self._predictions = comm.gather(self._predictions, dst=0) + self._predictions = list(itertools.chain(*self._predictions)) + if not comm.is_main_process(): + return {} + + if len(self._predictions) == 0: + self._logger.warning( + '[COCOEvaluator] Did not receive valid predictions.') + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, + 'instances_predictions.pth') + with PathManager.open(file_path, 'wb') as f: + torch.save(self._predictions, f) + + self._results = OrderedDict() + if 'instances' in self._predictions[0]: + self._eval_predictions() + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _eval_predictions(self): + """ + Evaluate self._predictions on the instance detection task. + Fill self._results with the metrics of the instance detection task. + """ + self._logger.info('Preparing results for COCO format ...') + self._coco_results = list( + itertools.chain(*[x['instances'] for x in self._predictions])) + + # unmap the category ids for COCO + if hasattr(self._metadata, 'thing_dataset_id_to_contiguous_id'): + reverse_id_mapping = { + v: k + for k, v in + self._metadata.thing_dataset_id_to_contiguous_id.items() + } + for result in self._coco_results: + result['category_id'] = reverse_id_mapping[ + result['category_id']] + + if self._output_dir: + file_path = os.path.join(self._output_dir, + 'coco_instances_results.json') + self._logger.info('Saving results to {}'.format(file_path)) + with PathManager.open(file_path, 'w') as f: + f.write(json.dumps(self._coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info('Annotations are not available for evaluation.') + return + + self._logger.info('Evaluating predictions ...') + if self._is_splits: + self._results['bbox'] = {} + for split, classes, names in [ + ('all', None, self._metadata.get('thing_classes')), + ('base', self._base_classes, + self._metadata.get('base_classes')), + ('novel', self._novel_classes, + self._metadata.get('novel_classes')) + ]: + if 'all' not in self._dataset_name and \ + split not in self._dataset_name: + continue + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + self._coco_results, + 'bbox', + classes, + ) if len(self._coco_results) > 0 else + None # cocoapi does not handle empty results very well + ) + res_ = self._derive_coco_results( + coco_eval, 'bbox', class_names=names) + res = {} + for metric in res_.keys(): + if len(metric) <= 4: + if split == 'all': + res[metric] = res_[metric] + elif split == 'base': + res['b' + metric] = res_[metric] + elif split == 'novel': + res['n' + metric] = res_[metric] + self._results['bbox'].update(res) + + # add "AP" if not already in + if 'AP' not in self._results['bbox']: + if 'nAP' in self._results['bbox']: + self._results['bbox']['AP'] = self._results['bbox']['nAP'] + else: + self._results['bbox']['AP'] = self._results['bbox']['bAP'] + else: + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + self._coco_results, + 'bbox', + ) if len(self._coco_results) > 0 else + None # cocoapi does not handle empty results very well + ) + res = self._derive_coco_results( + coco_eval, + 'bbox', + class_names=self._metadata.get('thing_classes')) + self._results['bbox'] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = ['AP', 'AP50', 'AP75', 'APs', 'APm', 'APl'] + + if coco_eval is None: + self._logger.warn( + 'No predictions from the model! Set scores to -1') + return {metric: -1 for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100) + for idx, metric in enumerate(metrics) + } + self._logger.info('Evaluation results for {}: \n'.format(iou_type) + + create_small_table(results)) + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + precisions = coco_eval.eval['precision'] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float('nan') + results_per_category.append(('{}'.format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest( + *[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt='pipe', + floatfmt='.3f', + headers=['category', 'AP'] * (N_COLS // 2), + numalign='left', + ) + self._logger.info('Per-category {} AP: \n'.format(iou_type) + table) + + results.update({'AP-' + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + results = [] + for k in range(num_instance): + result = { + 'image_id': img_id, + 'category_id': classes[k], + 'bbox': boxes[k], + 'score': scores[k], + } + results.append(result) + return results + + +def _evaluate_predictions_on_coco(coco_gt, + coco_results, + iou_type, + catIds=None): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = COCOeval(coco_gt, coco_dt, iou_type) + if catIds is not None: + coco_eval.params.catIds = catIds + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/evaluator.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..13f73acfae938e4a0ade0317836cb14433f1467d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/evaluator.py @@ -0,0 +1,77 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/facebookresearch/detectron2/blob/v0.3/detectron2/evaluation/evaluator.py + +import datetime +import logging +import time + +import torch +from detectron2.evaluation.evaluator import inference_context + +from ..models.calibration_layer import PrototypicalCalibrationBlock + + +def inference_on_dataset(model, data_loader, evaluator, cfg=None): + + num_devices = torch.distributed.get_world_size( + ) if torch.distributed.is_initialized() else 1 + logger = logging.getLogger(__name__) + + pcb = None + if cfg.TEST.PCB_ENABLE: + logger.info('Start initializing PCB module, please wait a seconds...') + pcb = PrototypicalCalibrationBlock(cfg) + + logger.info('Start inference on {} images'.format(len(data_loader))) + total = len(data_loader) # inference data loader must have a fixed length + evaluator.reset() + + logging_interval = 50 + num_warmup = min(5, logging_interval - 1, total - 1) + start_time = time.time() + total_compute_time = 0 + with inference_context(model), torch.no_grad(): + for idx, inputs in enumerate(data_loader): + if idx == num_warmup: + start_time = time.time() + total_compute_time = 0 + + start_compute_time = time.time() + outputs = model(inputs) + if cfg.TEST.PCB_ENABLE: + outputs = pcb.execute_calibration(inputs, outputs) + torch.cuda.synchronize() + total_compute_time += time.time() - start_compute_time + evaluator.process(inputs, outputs) + + if (idx + 1) % logging_interval == 0: + duration = time.time() - start_time + seconds_per_img = duration / (idx + 1 - num_warmup) + eta = datetime.timedelta( + seconds=int(seconds_per_img * (total - num_warmup) + - duration)) + logger.info( + 'Inference done {}/{}. {:.4f} s / img. ETA={}'.format( + idx + 1, total, seconds_per_img, str(eta))) + + # Measure the time only for this worker (before the synchronization barrier) + total_time = int(time.time() - start_time) + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info( + 'Total inference time: {} ({:.6f} s / img per device, on {} devices)'. + format(total_time_str, total_time / (total - num_warmup), num_devices)) + total_compute_time_str = str( + datetime.timedelta(seconds=int(total_compute_time))) + time_per_device = total_compute_time / (total - num_warmup) + logger.info( + 'Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)' + .format(total_compute_time_str, time_per_device, num_devices)) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/pascal_voc_evaluation.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/pascal_voc_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..eff7fecf74e929ab7652ca0bf49070008e28f6a3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/evaluation/pascal_voc_evaluation.py @@ -0,0 +1,118 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/evaluation/pascal_voc_evaluation.py + +import os +import tempfile +from collections import OrderedDict, defaultdict + +import numpy as np +from detectron2.data import MetadataCatalog +from detectron2.evaluation.pascal_voc_evaluation import ( + PascalVOCDetectionEvaluator, voc_eval) +from detectron2.utils import comm +from detectron2.utils.logger import create_small_table + + +class PascalVOCEvaluator(PascalVOCDetectionEvaluator): + """ + Evaluate Pascal VOC AP. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): name of the dataset, e.g., "voc_2007_test" + """ + super().__init__(dataset_name) + meta = MetadataCatalog.get(dataset_name) + self._base_classes = meta.base_classes + self._novel_classes = meta.novel_classes + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP", "AP50", and "AP75". + """ + all_predictions = comm.gather(self._predictions, dst=0) + if not comm.is_main_process(): + return + predictions = defaultdict(list) + for predictions_per_rank in all_predictions: + for clsid, lines in predictions_per_rank.items(): + predictions[clsid].extend(lines) + del all_predictions + + self._logger.info( + 'Evaluating {} using {} metric. ' + 'Note that results do not use the official Matlab API.'.format( + self._dataset_name, 2007 if self._is_2007 else 2012)) + + with tempfile.TemporaryDirectory(prefix='pascal_voc_eval_') as dirname: + res_file_template = os.path.join(dirname, '{}.txt') + + aps = defaultdict(list) # iou -> ap per class + aps_base = defaultdict(list) + aps_novel = defaultdict(list) + exist_base, exist_novel = False, False + for cls_id, cls_name in enumerate(self._class_names): + lines = predictions.get(cls_id, ['']) + + with open(res_file_template.format(cls_name), 'w') as f: + f.write('\n'.join(lines)) + + for thresh in range(50, 100, 5): + rec, prec, ap = voc_eval( + res_file_template, + self._anno_file_template, + self._image_set_path, + cls_name, + ovthresh=thresh / 100.0, + use_07_metric=self._is_2007, + ) + aps[thresh].append(ap * 100) + + if self._base_classes is not None and cls_name in self._base_classes: + aps_base[thresh].append(ap * 100) + exist_base = True + + if self._novel_classes is not None and cls_name in self._novel_classes: + aps_novel[thresh].append(ap * 100) + exist_novel = True + + ret = OrderedDict() + mAP = {iou: np.mean(x) for iou, x in aps.items()} + ret['bbox'] = { + 'AP': np.mean(list(mAP.values())), + 'AP50': mAP[50], + 'AP75': mAP[75] + } + + # adding evaluation of the base and novel classes + if exist_base: + mAP_base = {iou: np.mean(x) for iou, x in aps_base.items()} + ret['bbox'].update({ + 'bAP': np.mean(list(mAP_base.values())), + 'bAP50': mAP_base[50], + 'bAP75': mAP_base[75] + }) + + if exist_novel: + mAP_novel = {iou: np.mean(x) for iou, x in aps_novel.items()} + ret['bbox'].update({ + 'nAP': np.mean(list(mAP_novel.values())), + 'nAP50': mAP_novel[50], + 'nAP75': mAP_novel[75] + }) + + # write per class AP to logger + per_class_res = { + self._class_names[idx]: ap + for idx, ap in enumerate(aps[50]) + } + + self._logger.info('Evaluate per-class mAP50:\n' + + create_small_table(per_class_res)) + self._logger.info('Evaluate overall bbox:\n' + + create_small_table(ret['bbox'])) + return ret diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/__init__.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d463e460f3369b443a651a6667c5626ebf953c0e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .defrcn import DeFRCN + +else: + _import_structure = {'defrcn': ['DeFRCN']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/calibration_layer.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/calibration_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..c6d7674de0be00ca2639b211e51549878282a3c3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/calibration_layer.py @@ -0,0 +1,188 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/evaluation/calibration_layer.py + +import cv2 +import torch +from detectron2.data import DatasetMapper, build_detection_test_loader +from detectron2.modeling.poolers import ROIPooler +from detectron2.structures import ImageList +from sklearn.metrics.pairwise import cosine_similarity + +from modelscope.utils.logger import get_logger +from .resnet import resnet101 + + +class DatasetMapperIns(DatasetMapper): + + def __init__(self, cfg, is_train: bool): + super(DatasetMapperIns, self).__init__(cfg, is_train) + + def __call__(self, dataset_dict): + is_train = self.is_train + self.is_train = True + dataset_dict = super(DatasetMapperIns, self).__call__(dataset_dict) + self.is_train = is_train + return dataset_dict + + +class PrototypicalCalibrationBlock: + + def __init__(self, cfg): + super().__init__() + + self.logger = get_logger() + + self.cfg = cfg + self.device = torch.device(cfg.MODEL.DEVICE) + self.alpha = self.cfg.TEST.PCB_ALPHA + + self.imagenet_model = self.build_model() + self.dataloader = build_detection_test_loader( + self.cfg, + self.cfg.DATASETS.TRAIN[0], + mapper=DatasetMapperIns(cfg, False)) + self.roi_pooler = ROIPooler( + output_size=(1, 1), + scales=(1 / 32, ), + sampling_ratio=(0), + pooler_type='ROIAlignV2') + self.prototypes = self.build_prototypes() + + self.exclude_cls = self.clsid_filter() + + def build_model(self): + self.logger.info('Loading ImageNet Pre-train Model from {}'.format( + self.cfg.TEST.PCB_MODELPATH)) + if self.cfg.TEST.PCB_MODELTYPE == 'resnet': + imagenet_model = resnet101() + else: + raise NotImplementedError + state_dict = torch.load(self.cfg.TEST.PCB_MODELPATH) + imagenet_model.load_state_dict(state_dict) + imagenet_model = imagenet_model.to(self.device) + imagenet_model.eval() + return imagenet_model + + def build_prototypes(self): + + all_features, all_labels = [], [] + for index in range(len(self.dataloader.dataset)): + inputs = [self.dataloader.dataset[index]] + assert len(inputs) == 1 + # load support images and gt-boxes + img = cv2.imread(inputs[0]['file_name']) # BGR + img_h, _ = img.shape[0], img.shape[1] + ratio = img_h / inputs[0]['instances'].image_size[0] + inputs[0]['instances'].gt_boxes.tensor = inputs[0][ + 'instances'].gt_boxes.tensor * ratio + boxes = [x['instances'].gt_boxes.to(self.device) for x in inputs] + + # extract roi features + features = self.extract_roi_features(img, boxes) + all_features.append(features.cpu().data) + + gt_classes = [x['instances'].gt_classes for x in inputs] + all_labels.append(gt_classes[0].cpu().data) + + # concat + all_features = torch.cat(all_features, dim=0) + all_labels = torch.cat(all_labels, dim=0) + assert all_features.shape[0] == all_labels.shape[0] + + # calculate prototype + features_dict = {} + for i, label in enumerate(all_labels): + label = int(label) + if label not in features_dict: + features_dict[label] = [] + features_dict[label].append(all_features[i].unsqueeze(0)) + + prototypes_dict = {} + for label in features_dict: + features = torch.cat(features_dict[label], dim=0) + prototypes_dict[label] = torch.mean(features, dim=0, keepdim=True) + + return prototypes_dict + + def extract_roi_features(self, img, boxes): + """ + :param img: + :param boxes: + :return: + """ + + mean = torch.tensor([0.406, 0.456, 0.485]).reshape( + (3, 1, 1)).to(self.device) + std = torch.tensor([[0.225, 0.224, 0.229]]).reshape( + (3, 1, 1)).to(self.device) + + img = img.transpose((2, 0, 1)) + img = torch.from_numpy(img).to(self.device) + images = [(img / 255. - mean) / std] + images = ImageList.from_tensors(images, 0) + conv_feature = self.imagenet_model( + images.tensor[:, [2, 1, 0]])[1] # size: BxCxHxW + + box_features = self.roi_pooler([conv_feature], + boxes).squeeze(2).squeeze(2) + + activation_vectors = self.imagenet_model.fc(box_features) + + return activation_vectors + + def execute_calibration(self, inputs, dts): + if 'file_name' in inputs[0]: + img = cv2.imread(inputs[0]['file_name']) + elif 'image_numpy' in inputs[0]: + img = inputs[0]['image_numpy'] + + ileft = (dts[0]['instances'].scores > self.cfg.TEST.PCB_UPPER).sum() + iright = (dts[0]['instances'].scores > self.cfg.TEST.PCB_LOWER).sum() + assert ileft <= iright + boxes = [dts[0]['instances'].pred_boxes[ileft:iright]] + + features = self.extract_roi_features(img, boxes) + + for i in range(ileft, iright): + tmp_class = int(dts[0]['instances'].pred_classes[i]) + if tmp_class in self.exclude_cls: + continue + tmp_cos = cosine_similarity( + features[i - ileft].cpu().data.numpy().reshape((1, -1)), + self.prototypes[tmp_class].cpu().data.numpy())[0][0] + dts[0]['instances'].scores[i] = dts[0]['instances'].scores[ + i] * self.alpha + tmp_cos * (1 - self.alpha) + return dts + + def clsid_filter(self): + dsname = self.cfg.DATASETS.TEST[0] + exclude_ids = [] + if 'test_all' in dsname: + if 'coco' in dsname: + exclude_ids = [ + 7, 9, 10, 11, 12, 13, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 59, 61, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + ] + elif 'voc' in dsname: + exclude_ids = list(range(0, 15)) + else: + raise NotImplementedError + return exclude_ids + + +@torch.no_grad() +def concat_all_gather(tensor): + """ + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + tensors_gather = [ + torch.ones_like(tensor) + for _ in range(torch.distributed.get_world_size()) + ] + torch.distributed.all_gather(tensors_gather, tensor, async_op=False) + output = torch.cat(tensors_gather, dim=0) + return output diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/defrcn.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/defrcn.py new file mode 100644 index 0000000000000000000000000000000000000000..a5258017a4d09dd3ac6f7c3e688a311c117d6068 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/defrcn.py @@ -0,0 +1,179 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/modeling/meta_arch/rcnn.py + +import os +from typing import Dict + +import torch +from detectron2.layers import ShapeSpec +from detectron2.modeling.anchor_generator import DefaultAnchorGenerator +from detectron2.modeling.backbone.resnet import build_resnet_backbone +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.modeling.matcher import Matcher +from detectron2.modeling.postprocessing import detector_postprocess +from detectron2.modeling.proposal_generator.rpn import RPN, StandardRPNHead +from detectron2.structures import ImageList +from torch import nn + +from .gdl import AffineLayer, decouple_layer +from .roi_heads import Res5ROIHeads + + +class DeFRCN(nn.Module): + + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + + self.device = torch.device(cfg.MODEL.DEVICE) + + self.backbone = build_resnet_backbone( + cfg, ShapeSpec(channels=len(cfg.MODEL.PIXEL_MEAN))) + self._SHAPE_ = self.backbone.output_shape() + + rpn_config = DeFRCN.from_rpn_config(cfg, self._SHAPE_) + self.proposal_generator = RPN(**rpn_config) + + self.roi_heads = Res5ROIHeads(cfg, self._SHAPE_) + self.normalizer = self.normalize_fn() + self.affine_rpn = AffineLayer( + num_channels=self._SHAPE_['res4'].channels, bias=True) + self.affine_rcnn = AffineLayer( + num_channels=self._SHAPE_['res4'].channels, bias=True) + self.to(self.device) + + if cfg.MODEL.BACKBONE.FREEZE: + for p in self.backbone.parameters(): + p.requires_grad = False + + if cfg.MODEL.RPN.FREEZE: + for p in self.proposal_generator.parameters(): + p.requires_grad = False + + if cfg.MODEL.ROI_HEADS.FREEZE_FEAT: + for p in self.roi_heads.res5.parameters(): + p.requires_grad = False + + def forward(self, batched_inputs): + if not self.training: + return self.inference(batched_inputs) + assert 'instances' in batched_inputs[0] + gt_instances = [x['instances'].to(self.device) for x in batched_inputs] + proposal_losses, detector_losses, _, _ = self._forward_once_( + batched_inputs, gt_instances) + losses = {} + losses.update(detector_losses) + losses.update(proposal_losses) + return losses + + def inference(self, batched_inputs): + assert not self.training + _, _, results, image_sizes = self._forward_once_(batched_inputs, None) + processed_results = [] + for r, input, image_size in zip(results, batched_inputs, image_sizes): + height = input.get('height', image_size[0]) + width = input.get('width', image_size[1]) + r = detector_postprocess(r, height, width) + processed_results.append({'instances': r}) + return processed_results + + def _forward_once_(self, batched_inputs, gt_instances=None): + + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + + features_de_rpn = features + if self.cfg.MODEL.RPN.ENABLE_DECOUPLE: + scale = self.cfg.MODEL.RPN.BACKWARD_SCALE + features_de_rpn = { + k: self.affine_rpn(decouple_layer(features[k], scale)) + for k in features + } + proposals, proposal_losses = self.proposal_generator( + images, features_de_rpn, gt_instances) + + features_de_rcnn = features + if self.cfg.MODEL.ROI_HEADS.ENABLE_DECOUPLE: + scale = self.cfg.MODEL.ROI_HEADS.BACKWARD_SCALE + features_de_rcnn = { + k: self.affine_rcnn(decouple_layer(features[k], scale)) + for k in features + } + results, detector_losses = self.roi_heads(images, features_de_rcnn, + proposals, gt_instances) + + return proposal_losses, detector_losses, results, images.image_sizes + + def preprocess_image(self, batched_inputs): + images = [x['image'].to(self.device) for x in batched_inputs] + images = [self.normalizer(x) for x in images] + images = ImageList.from_tensors(images, + self.backbone.size_divisibility) + return images + + def normalize_fn(self): + assert len(self.cfg.MODEL.PIXEL_MEAN) == len(self.cfg.MODEL.PIXEL_STD) + num_channels = len(self.cfg.MODEL.PIXEL_MEAN) + pixel_mean = ( + torch.Tensor(self.cfg.MODEL.PIXEL_MEAN).to(self.device).view( + num_channels, 1, 1)) + pixel_std = ( + torch.Tensor(self.cfg.MODEL.PIXEL_STD).to(self.device).view( + num_channels, 1, 1)) + return lambda x: (x - pixel_mean) / pixel_std + + @classmethod + def from_rpn_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + 'in_features': + in_features, + 'min_box_size': + cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + 'nms_thresh': + cfg.MODEL.RPN.NMS_THRESH, + 'batch_size_per_image': + cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + 'positive_fraction': + cfg.MODEL.RPN.POSITIVE_FRACTION, + 'loss_weight': { + 'loss_rpn_cls': + cfg.MODEL.RPN.LOSS_WEIGHT, + 'loss_rpn_loc': + cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + 'anchor_boundary_thresh': + cfg.MODEL.RPN.BOUNDARY_THRESH, + 'box2box_transform': + Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + 'box_reg_loss_type': + cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + 'smooth_l1_beta': + cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret['pre_nms_topk'] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, + cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret['post_nms_topk'] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, + cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + # ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + anchor_cfg = DefaultAnchorGenerator.from_config( + cfg, [input_shape[f] for f in in_features]) + ret['anchor_generator'] = DefaultAnchorGenerator(**anchor_cfg) + ret['anchor_matcher'] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, + cfg.MODEL.RPN.IOU_LABELS, + allow_low_quality_matches=True) + rpn_head_cfg = { + 'in_channels': + [s.channels for s in [input_shape[f] for f in in_features]][0], + 'num_anchors': + ret['anchor_generator'].num_anchors[0], + 'box_dim': + ret['anchor_generator'].box_dim + } + + ret['head'] = StandardRPNHead(**rpn_head_cfg) + return ret diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/fast_rcnn.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..88311774d155a365aa328b4ad32a8ed4038d64be --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/fast_rcnn.py @@ -0,0 +1,274 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/modeling/meta_arch/rcnn.py + +import numpy as np +import torch +from detectron2.layers import cat +from detectron2.modeling.roi_heads.fast_rcnn import \ + fast_rcnn_inference_single_image +from detectron2.utils.events import get_event_storage +from fvcore.nn import smooth_l1_loss +from torch import nn +from torch.nn import functional as F + + +def fast_rcnn_inference(boxes, scores, image_shapes, score_thresh, nms_thresh, + topk_per_image): + + result_per_image = [ + fast_rcnn_inference_single_image( + boxes_per_image, + scores_per_image, + image_shape, + score_thresh, + nms_thresh, + topk_per_image, + ) for scores_per_image, boxes_per_image, image_shape in zip( + scores, boxes, image_shapes) + ] + return tuple(list(x) for x in zip(*result_per_image)) + + +class FastRCNNOutputs(object): + """ + A class that stores information about outputs of a Fast R-CNN head. + """ + + def __init__( + self, + box2box_transform, + pred_class_logits, + pred_proposal_deltas, + proposals, + smooth_l1_beta, + ): + """ + Args: + box2box_transform (Box2BoxTransform/Box2BoxTransformRotated): + box2box transform instance for proposal-to-detection transformations. + pred_class_logits (Tensor): A tensor of shape (R, K + 1) storing the predicted class + logits for all R predicted object instances. + Each row corresponds to a predicted object instance. + pred_proposal_deltas (Tensor): A tensor of shape (R, K * B) or (R, B) for + class-specific or class-agnostic regression. It stores the predicted deltas that + transform proposals into final box detections. + B is the box dimension (4 or 5). + When B is 4, each row is [dx, dy, dw, dh (, ....)]. + When B is 5, each row is [dx, dy, dw, dh, da (, ....)]. + proposals (list[Instances]): A list of N Instances, where Instances i stores the + proposals for image i, in the field "proposal_boxes". + When training, each Instances must have ground-truth labels + stored in the field "gt_classes" and "gt_boxes". + smooth_l1_beta (float): The transition point between L1 and L2 loss in + the smooth L1 loss function. When set to 0, the loss becomes L1. When + set to +inf, the loss becomes constant 0. + """ + self.box2box_transform = box2box_transform + self.num_preds_per_image = [len(p) for p in proposals] + self.pred_class_logits = pred_class_logits + self.pred_proposal_deltas = pred_proposal_deltas + self.smooth_l1_beta = smooth_l1_beta + + box_type = type(proposals[0].proposal_boxes) + # cat(..., dim=0) concatenates over all images in the batch + self.proposals = box_type.cat([p.proposal_boxes for p in proposals]) + assert (not self.proposals.tensor.requires_grad + ), 'Proposals should not require gradients!' + self.image_shapes = [x.image_size for x in proposals] + + # The following fields should exist only when training. + if proposals[0].has('gt_boxes'): + self.gt_boxes = box_type.cat([p.gt_boxes for p in proposals]) + assert proposals[0].has('gt_classes') + self.gt_classes = cat([p.gt_classes for p in proposals], dim=0) + + def _log_accuracy(self): + """ + Log the accuracy metrics to EventStorage. + """ + num_instances = self.gt_classes.numel() + pred_classes = self.pred_class_logits.argmax(dim=1) + bg_class_ind = self.pred_class_logits.shape[1] - 1 + + fg_inds = (self.gt_classes >= 0) & (self.gt_classes < bg_class_ind) + num_fg = fg_inds.nonzero().numel() + fg_gt_classes = self.gt_classes[fg_inds] + fg_pred_classes = pred_classes[fg_inds] + + num_false_negative = (( + fg_pred_classes == bg_class_ind).nonzero().numel()) + num_accurate = (pred_classes == self.gt_classes).nonzero().numel() + fg_num_accurate = (fg_pred_classes == fg_gt_classes).nonzero().numel() + + storage = get_event_storage() + storage.put_scalar('fast_rcnn/cls_accuracy', + num_accurate / num_instances) + if num_fg > 0: + storage.put_scalar('fast_rcnn/fg_cls_accuracy', + fg_num_accurate / num_fg) + storage.put_scalar('fast_rcnn/false_negative', + num_false_negative / num_fg) + + def softmax_cross_entropy_loss(self): + """ + Compute the softmax cross entropy loss for box classification. + + Returns: + scalar Tensor + """ + self._log_accuracy() + return F.cross_entropy( + self.pred_class_logits, self.gt_classes, reduction='mean') + + def smooth_l1_loss(self): + """ + Compute the smooth L1 loss for box regression. + + Returns: + scalar Tensor + """ + gt_proposal_deltas = self.box2box_transform.get_deltas( + self.proposals.tensor, self.gt_boxes.tensor) + box_dim = gt_proposal_deltas.size(1) # 4 or 5 + cls_agnostic_bbox_reg = self.pred_proposal_deltas.size(1) == box_dim + device = self.pred_proposal_deltas.device + + bg_class_ind = self.pred_class_logits.shape[1] - 1 + + fg_inds = torch.nonzero((self.gt_classes >= 0) + & (self.gt_classes < bg_class_ind)).squeeze(1) + if cls_agnostic_bbox_reg: + # pred_proposal_deltas only corresponds to foreground class for agnostic + gt_class_cols = torch.arange(box_dim, device=device) + else: + fg_gt_classes = self.gt_classes[fg_inds] + gt_class_cols = box_dim * fg_gt_classes[:, None] + torch.arange( + box_dim, device=device) + + loss_box_reg = smooth_l1_loss( + self.pred_proposal_deltas[fg_inds[:, None], gt_class_cols], + gt_proposal_deltas[fg_inds], + self.smooth_l1_beta, + reduction='sum', + ) + + loss_box_reg = loss_box_reg / self.gt_classes.numel() + return loss_box_reg + + def losses(self): + """ + Compute the default losses for box head in Fast(er) R-CNN, + with softmax cross entropy loss and smooth L1 loss. + + Returns: + A dict of losses (scalar tensors) containing keys "loss_cls" and "loss_box_reg". + """ + return { + 'loss_cls': self.softmax_cross_entropy_loss(), + 'loss_box_reg': self.smooth_l1_loss(), + } + + def predict_boxes(self): + """ + Returns: + list[Tensor]: A list of Tensors of predicted class-specific or class-agnostic boxes + for each image. Element i has shape (Ri, K * B) or (Ri, B), where Ri is + the number of predicted objects for image i and B is the box dimension (4 or 5) + """ + num_pred = len(self.proposals) + B = self.proposals.tensor.shape[1] + K = self.pred_proposal_deltas.shape[1] // B + boxes = self.box2box_transform.apply_deltas( + self.pred_proposal_deltas.view(num_pred * K, B), + self.proposals.tensor.unsqueeze(1).expand(num_pred, K, + B).reshape(-1, B), + ) + return boxes.view(num_pred, K * B).split( + self.num_preds_per_image, dim=0) + + def predict_probs(self): + """ + Returns: + list[Tensor]: A list of Tensors of predicted class probabilities for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. + """ + probs = F.softmax(self.pred_class_logits, dim=-1) + return probs.split(self.num_preds_per_image, dim=0) + + def inference(self, score_thresh, nms_thresh, topk_per_image): + """ + Args: + score_thresh (float): same as fast_rcnn_inference. + nms_thresh (float): same as fast_rcnn_inference. + topk_per_image (int): same as fast_rcnn_inference. + Returns: + list[Instances]: same as fast_rcnn_inference. + list[Tensor]: same as fast_rcnn_inference. + """ + boxes = self.predict_boxes() + scores = self.predict_probs() + image_shapes = self.image_shapes + + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + score_thresh, + nms_thresh, + topk_per_image, + ) + + +class FastRCNNOutputLayers(nn.Module): + """ + Two linear layers for predicting Fast R-CNN outputs: + (1) proposal-to-detection box regression deltas + (2) classification scores + """ + + def __init__(self, + cfg, + input_size, + num_classes, + cls_agnostic_bbox_reg, + box_dim=4): + """ + Args: + cfg: config + input_size (int): channels, or (channels, height, width) + num_classes (int): number of foreground classes + cls_agnostic_bbox_reg (bool): whether to use class agnostic for bbox regression + box_dim (int): the dimension of bounding boxes. + Example box dimensions: 4 for regular XYXY boxes and 5 for rotated XYWHA boxes + """ + super(FastRCNNOutputLayers, self).__init__() + + if not isinstance(input_size, int): + input_size = np.prod(input_size) + + # The prediction layer for num_classes foreground classes and one + # background class + self.cls_score = nn.Linear(input_size, num_classes + 1) + num_bbox_reg_classes = 1 if cls_agnostic_bbox_reg else num_classes + self.bbox_pred = nn.Linear(input_size, num_bbox_reg_classes * box_dim) + + nn.init.normal_(self.cls_score.weight, std=0.01) + nn.init.normal_(self.bbox_pred.weight, std=0.001) + for b in [self.cls_score, self.bbox_pred]: + nn.init.constant_(b.bias, 0) + + self._do_cls_dropout = cfg.MODEL.ROI_HEADS.CLS_DROPOUT + self._dropout_ratio = cfg.MODEL.ROI_HEADS.DROPOUT_RATIO + + def forward(self, x): + if x.dim() > 2: + x = torch.flatten(x, start_dim=1) + proposal_deltas = self.bbox_pred(x) + + if self._do_cls_dropout: + x = F.dropout(x, self._dropout_ratio, training=self.training) + scores = self.cls_score(x) + + return scores, proposal_deltas diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/gdl.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/gdl.py new file mode 100644 index 0000000000000000000000000000000000000000..0d228fa7a39013d8596199aae96cfd4e4c6bdc85 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/gdl.py @@ -0,0 +1,43 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/modeling/meta_arch/gdl.py + +import torch +import torch.nn as nn +from torch.autograd import Function + + +class GradientDecoupleLayer(Function): + + @staticmethod + def forward(ctx, x, _lambda): + ctx._lambda = _lambda + return x + + @staticmethod + def backward(ctx, grad_output): + grad_output = grad_output * ctx._lambda + return grad_output, None + + +class AffineLayer(nn.Module): + + def __init__(self, num_channels, bias=False): + super(AffineLayer, self).__init__() + weight = torch.FloatTensor(1, num_channels, 1, 1).fill_(1) + self.weight = nn.Parameter(weight, requires_grad=True) + + self.bias = None + if bias: + bias = torch.FloatTensor(1, num_channels, 1, 1).fill_(0) + self.bias = nn.Parameter(bias, requires_grad=True) + + def forward(self, X): + out = X * self.weight.expand_as(X) + if self.bias is not None: + out = out + self.bias.expand_as(X) + return out + + +def decouple_layer(x, _lambda): + return GradientDecoupleLayer.apply(x, _lambda) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/resnet.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..0b0abcfe5835bcac216c6ca879f4e5f85d51da70 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/resnet.py @@ -0,0 +1,66 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +from torchvision.models.resnet import BasicBlock, Bottleneck, ResNet + +__all__ = [ + 'ResNetFeatures', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152' +] + + +class ResNetFeatures(ResNet): + """ + Modified from torchvision.models.resnet, the only one difference is outputing layer4 feature in forward. + """ + + def __init__(self, + block, + layers, + num_classes=1000, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None): + super(ResNetFeatures, + self).__init__(block, layers, num_classes, zero_init_residual, + groups, width_per_group, + replace_stride_with_dilation, norm_layer) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + feature = self.layer4(x) + + x = self.avgpool(feature) + x = torch.flatten(x, 1) + x = self.fc(x) + + return x, feature + + +def resnet18(**kwargs): + return ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs) + + +def resnet34(**kwargs): + return ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs) + + +def resnet50(**kwargs): + return ResNetFeatures(Bottleneck, [3, 4, 6, 3], **kwargs) + + +def resnet101(**kwargs): + return ResNetFeatures(Bottleneck, [3, 4, 23, 3], **kwargs) + + +def resnet152(**kwargs): + return ResNetFeatures(Bottleneck, [3, 8, 36, 3], **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/roi_heads.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/roi_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..9de00ad55c9f3a4b8d28583bff062cf75414b2e4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/models/roi_heads.py @@ -0,0 +1,126 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/modeling/roi_heads/roi_heads.py + +from detectron2.modeling.backbone.resnet import BottleneckBlock, make_stage +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.modeling.poolers import ROIPooler +from detectron2.modeling.roi_heads import ROIHeads +from torch import nn + +from .fast_rcnn import FastRCNNOutputLayers, FastRCNNOutputs + + +class Res5ROIHeads(ROIHeads): + """ + The ROIHeads in a typical "C4" R-CNN model, where the heads share the + cropping and the per-region feature computation by a Res5 block. + """ + + def __init__(self, cfg, input_shape): + + cfg_dict = ROIHeads.from_config(cfg) + super().__init__(**cfg_dict) + + self.test_score_thresh = cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST + self.test_nms_thresh = cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST + self.test_detections_per_img = cfg.TEST.DETECTIONS_PER_IMAGE + self.in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + + self.feature_strides = {k: v.stride for k, v in input_shape.items()} + self.feature_channels = {k: v.channels for k, v in input_shape.items()} + self.cls_agnostic_bbox_reg = cfg.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG + self.smooth_l1_beta = cfg.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA + + # Box2BoxTransform for bounding box regression + self.box2box_transform = Box2BoxTransform( + weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS) + + assert len(self.in_features) == 1 + + # fmt: off + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + pooler_scales = (1.0 / self.feature_strides[self.in_features[0]], ) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + + self.pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + + self.res5, out_channels = self._build_res5_block(cfg) + self.box_predictor = FastRCNNOutputLayers(cfg, out_channels, + self.num_classes, + self.cls_agnostic_bbox_reg) + + def _build_res5_block(self, cfg): + # fmt: off + stage_channel_factor = 2**3 # res5 is 8x res2 + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group * stage_channel_factor + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS * stage_channel_factor + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + norm = cfg.MODEL.RESNETS.NORM + assert not cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE[-1], \ + 'Deformable conv is not yet supported in res5 head.' + # fmt: on + + blocks = make_stage( + BottleneckBlock, + 3, + first_stride=2, + in_channels=out_channels // 2, + bottleneck_channels=bottleneck_channels, + out_channels=out_channels, + num_groups=num_groups, + norm=norm, + stride_in_1x1=stride_in_1x1, + ) + return nn.Sequential(*blocks), out_channels + + def _shared_roi_transform(self, features, boxes): + x = self.pooler(features, boxes) + x = self.res5(x) + return x + + def forward(self, images, features, proposals, targets=None): + """ + See :class:`ROIHeads.forward`. + """ + del images + + if self.training: + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + proposal_boxes = [x.proposal_boxes for x in proposals] + box_features = self._shared_roi_transform( + [features[f] for f in self.in_features], proposal_boxes) + feature_pooled = box_features.mean(dim=[2, 3]) # pooled to 1x1 + pred_class_logits, pred_proposal_deltas = self.box_predictor( + feature_pooled) + del feature_pooled + + outputs = FastRCNNOutputs( + self.box2box_transform, + pred_class_logits, + pred_proposal_deltas, + proposals, + self.smooth_l1_beta, + ) + + if self.training: + del features + losses = outputs.losses() + return [], losses + else: + pred_instances, _ = outputs.inference( + self.test_score_thresh, + self.test_nms_thresh, + self.test_detections_per_img, + ) + return pred_instances, {} diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/__init__.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/coco_register.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/coco_register.py new file mode 100644 index 0000000000000000000000000000000000000000..c71be0ebbc97c039045b3faf45ab3e67a19640b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/coco_register.py @@ -0,0 +1,1086 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/data/meta_coco.py +# https://github.com/cocodataset/panopticapi/blob/master/panoptic_coco_categories.json + +import contextlib +import io +import os + +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.structures import BoxMode +from fvcore.common.file_io import PathManager +from pycocotools.coco import COCO + +COCO_CATEGORIES = [ + { + 'color': [220, 20, 60], + 'isthing': 1, + 'id': 1, + 'name': 'person' + }, + { + 'color': [119, 11, 32], + 'isthing': 1, + 'id': 2, + 'name': 'bicycle' + }, + { + 'color': [0, 0, 142], + 'isthing': 1, + 'id': 3, + 'name': 'car' + }, + { + 'color': [0, 0, 230], + 'isthing': 1, + 'id': 4, + 'name': 'motorcycle' + }, + { + 'color': [106, 0, 228], + 'isthing': 1, + 'id': 5, + 'name': 'airplane' + }, + { + 'color': [0, 60, 100], + 'isthing': 1, + 'id': 6, + 'name': 'bus' + }, + { + 'color': [0, 80, 100], + 'isthing': 1, + 'id': 7, + 'name': 'train' + }, + { + 'color': [0, 0, 70], + 'isthing': 1, + 'id': 8, + 'name': 'truck' + }, + { + 'color': [0, 0, 192], + 'isthing': 1, + 'id': 9, + 'name': 'boat' + }, + { + 'color': [250, 170, 30], + 'isthing': 1, + 'id': 10, + 'name': 'traffic light' + }, + { + 'color': [100, 170, 30], + 'isthing': 1, + 'id': 11, + 'name': 'fire hydrant' + }, + { + 'color': [220, 220, 0], + 'isthing': 1, + 'id': 13, + 'name': 'stop sign' + }, + { + 'color': [175, 116, 175], + 'isthing': 1, + 'id': 14, + 'name': 'parking meter', + }, + { + 'color': [250, 0, 30], + 'isthing': 1, + 'id': 15, + 'name': 'bench' + }, + { + 'color': [165, 42, 42], + 'isthing': 1, + 'id': 16, + 'name': 'bird' + }, + { + 'color': [255, 77, 255], + 'isthing': 1, + 'id': 17, + 'name': 'cat' + }, + { + 'color': [0, 226, 252], + 'isthing': 1, + 'id': 18, + 'name': 'dog' + }, + { + 'color': [182, 182, 255], + 'isthing': 1, + 'id': 19, + 'name': 'horse' + }, + { + 'color': [0, 82, 0], + 'isthing': 1, + 'id': 20, + 'name': 'sheep' + }, + { + 'color': [120, 166, 157], + 'isthing': 1, + 'id': 21, + 'name': 'cow' + }, + { + 'color': [110, 76, 0], + 'isthing': 1, + 'id': 22, + 'name': 'elephant' + }, + { + 'color': [174, 57, 255], + 'isthing': 1, + 'id': 23, + 'name': 'bear' + }, + { + 'color': [199, 100, 0], + 'isthing': 1, + 'id': 24, + 'name': 'zebra' + }, + { + 'color': [72, 0, 118], + 'isthing': 1, + 'id': 25, + 'name': 'giraffe' + }, + { + 'color': [255, 179, 240], + 'isthing': 1, + 'id': 27, + 'name': 'backpack' + }, + { + 'color': [0, 125, 92], + 'isthing': 1, + 'id': 28, + 'name': 'umbrella' + }, + { + 'color': [209, 0, 151], + 'isthing': 1, + 'id': 31, + 'name': 'handbag' + }, + { + 'color': [188, 208, 182], + 'isthing': 1, + 'id': 32, + 'name': 'tie' + }, + { + 'color': [0, 220, 176], + 'isthing': 1, + 'id': 33, + 'name': 'suitcase' + }, + { + 'color': [255, 99, 164], + 'isthing': 1, + 'id': 34, + 'name': 'frisbee' + }, + { + 'color': [92, 0, 73], + 'isthing': 1, + 'id': 35, + 'name': 'skis' + }, + { + 'color': [133, 129, 255], + 'isthing': 1, + 'id': 36, + 'name': 'snowboard' + }, + { + 'color': [78, 180, 255], + 'isthing': 1, + 'id': 37, + 'name': 'sports ball' + }, + { + 'color': [0, 228, 0], + 'isthing': 1, + 'id': 38, + 'name': 'kite' + }, + { + 'color': [174, 255, 243], + 'isthing': 1, + 'id': 39, + 'name': 'baseball bat' + }, + { + 'color': [45, 89, 255], + 'isthing': 1, + 'id': 40, + 'name': 'baseball glove' + }, + { + 'color': [134, 134, 103], + 'isthing': 1, + 'id': 41, + 'name': 'skateboard' + }, + { + 'color': [145, 148, 174], + 'isthing': 1, + 'id': 42, + 'name': 'surfboard' + }, + { + 'color': [255, 208, 186], + 'isthing': 1, + 'id': 43, + 'name': 'tennis racket', + }, + { + 'color': [197, 226, 255], + 'isthing': 1, + 'id': 44, + 'name': 'bottle' + }, + { + 'color': [171, 134, 1], + 'isthing': 1, + 'id': 46, + 'name': 'wine glass' + }, + { + 'color': [109, 63, 54], + 'isthing': 1, + 'id': 47, + 'name': 'cup' + }, + { + 'color': [207, 138, 255], + 'isthing': 1, + 'id': 48, + 'name': 'fork' + }, + { + 'color': [151, 0, 95], + 'isthing': 1, + 'id': 49, + 'name': 'knife' + }, + { + 'color': [9, 80, 61], + 'isthing': 1, + 'id': 50, + 'name': 'spoon' + }, + { + 'color': [84, 105, 51], + 'isthing': 1, + 'id': 51, + 'name': 'bowl' + }, + { + 'color': [74, 65, 105], + 'isthing': 1, + 'id': 52, + 'name': 'banana' + }, + { + 'color': [166, 196, 102], + 'isthing': 1, + 'id': 53, + 'name': 'apple' + }, + { + 'color': [208, 195, 210], + 'isthing': 1, + 'id': 54, + 'name': 'sandwich' + }, + { + 'color': [255, 109, 65], + 'isthing': 1, + 'id': 55, + 'name': 'orange' + }, + { + 'color': [0, 143, 149], + 'isthing': 1, + 'id': 56, + 'name': 'broccoli' + }, + { + 'color': [179, 0, 194], + 'isthing': 1, + 'id': 57, + 'name': 'carrot' + }, + { + 'color': [209, 99, 106], + 'isthing': 1, + 'id': 58, + 'name': 'hot dog' + }, + { + 'color': [5, 121, 0], + 'isthing': 1, + 'id': 59, + 'name': 'pizza' + }, + { + 'color': [227, 255, 205], + 'isthing': 1, + 'id': 60, + 'name': 'donut' + }, + { + 'color': [147, 186, 208], + 'isthing': 1, + 'id': 61, + 'name': 'cake' + }, + { + 'color': [153, 69, 1], + 'isthing': 1, + 'id': 62, + 'name': 'chair' + }, + { + 'color': [3, 95, 161], + 'isthing': 1, + 'id': 63, + 'name': 'couch' + }, + { + 'color': [163, 255, 0], + 'isthing': 1, + 'id': 64, + 'name': 'potted plant' + }, + { + 'color': [119, 0, 170], + 'isthing': 1, + 'id': 65, + 'name': 'bed' + }, + { + 'color': [0, 182, 199], + 'isthing': 1, + 'id': 67, + 'name': 'dining table' + }, + { + 'color': [0, 165, 120], + 'isthing': 1, + 'id': 70, + 'name': 'toilet' + }, + { + 'color': [183, 130, 88], + 'isthing': 1, + 'id': 72, + 'name': 'tv' + }, + { + 'color': [95, 32, 0], + 'isthing': 1, + 'id': 73, + 'name': 'laptop' + }, + { + 'color': [130, 114, 135], + 'isthing': 1, + 'id': 74, + 'name': 'mouse' + }, + { + 'color': [110, 129, 133], + 'isthing': 1, + 'id': 75, + 'name': 'remote' + }, + { + 'color': [166, 74, 118], + 'isthing': 1, + 'id': 76, + 'name': 'keyboard' + }, + { + 'color': [219, 142, 185], + 'isthing': 1, + 'id': 77, + 'name': 'cell phone' + }, + { + 'color': [79, 210, 114], + 'isthing': 1, + 'id': 78, + 'name': 'microwave' + }, + { + 'color': [178, 90, 62], + 'isthing': 1, + 'id': 79, + 'name': 'oven' + }, + { + 'color': [65, 70, 15], + 'isthing': 1, + 'id': 80, + 'name': 'toaster' + }, + { + 'color': [127, 167, 115], + 'isthing': 1, + 'id': 81, + 'name': 'sink' + }, + { + 'color': [59, 105, 106], + 'isthing': 1, + 'id': 82, + 'name': 'refrigerator' + }, + { + 'color': [142, 108, 45], + 'isthing': 1, + 'id': 84, + 'name': 'book' + }, + { + 'color': [196, 172, 0], + 'isthing': 1, + 'id': 85, + 'name': 'clock' + }, + { + 'color': [95, 54, 80], + 'isthing': 1, + 'id': 86, + 'name': 'vase' + }, + { + 'color': [128, 76, 255], + 'isthing': 1, + 'id': 87, + 'name': 'scissors' + }, + { + 'color': [201, 57, 1], + 'isthing': 1, + 'id': 88, + 'name': 'teddy bear' + }, + { + 'color': [246, 0, 122], + 'isthing': 1, + 'id': 89, + 'name': 'hair drier' + }, + { + 'color': [191, 162, 208], + 'isthing': 1, + 'id': 90, + 'name': 'toothbrush' + }, + { + 'color': [255, 255, 128], + 'isthing': 0, + 'id': 92, + 'name': 'banner' + }, + { + 'color': [147, 211, 203], + 'isthing': 0, + 'id': 93, + 'name': 'blanket' + }, + { + 'color': [150, 100, 100], + 'isthing': 0, + 'id': 95, + 'name': 'bridge' + }, + { + 'color': [168, 171, 172], + 'isthing': 0, + 'id': 100, + 'name': 'cardboard' + }, + { + 'color': [146, 112, 198], + 'isthing': 0, + 'id': 107, + 'name': 'counter' + }, + { + 'color': [210, 170, 100], + 'isthing': 0, + 'id': 109, + 'name': 'curtain' + }, + { + 'color': [92, 136, 89], + 'isthing': 0, + 'id': 112, + 'name': 'door-stuff' + }, + { + 'color': [218, 88, 184], + 'isthing': 0, + 'id': 118, + 'name': 'floor-wood' + }, + { + 'color': [241, 129, 0], + 'isthing': 0, + 'id': 119, + 'name': 'flower' + }, + { + 'color': [217, 17, 255], + 'isthing': 0, + 'id': 122, + 'name': 'fruit' + }, + { + 'color': [124, 74, 181], + 'isthing': 0, + 'id': 125, + 'name': 'gravel' + }, + { + 'color': [70, 70, 70], + 'isthing': 0, + 'id': 128, + 'name': 'house' + }, + { + 'color': [255, 228, 255], + 'isthing': 0, + 'id': 130, + 'name': 'light' + }, + { + 'color': [154, 208, 0], + 'isthing': 0, + 'id': 133, + 'name': 'mirror-stuff' + }, + { + 'color': [193, 0, 92], + 'isthing': 0, + 'id': 138, + 'name': 'net' + }, + { + 'color': [76, 91, 113], + 'isthing': 0, + 'id': 141, + 'name': 'pillow' + }, + { + 'color': [255, 180, 195], + 'isthing': 0, + 'id': 144, + 'name': 'platform' + }, + { + 'color': [106, 154, 176], + 'isthing': 0, + 'id': 145, + 'name': 'playingfield' + }, + { + 'color': [230, 150, 140], + 'isthing': 0, + 'id': 147, + 'name': 'railroad' + }, + { + 'color': [60, 143, 255], + 'isthing': 0, + 'id': 148, + 'name': 'river' + }, + { + 'color': [128, 64, 128], + 'isthing': 0, + 'id': 149, + 'name': 'road' + }, + { + 'color': [92, 82, 55], + 'isthing': 0, + 'id': 151, + 'name': 'roof' + }, + { + 'color': [254, 212, 124], + 'isthing': 0, + 'id': 154, + 'name': 'sand' + }, + { + 'color': [73, 77, 174], + 'isthing': 0, + 'id': 155, + 'name': 'sea' + }, + { + 'color': [255, 160, 98], + 'isthing': 0, + 'id': 156, + 'name': 'shelf' + }, + { + 'color': [255, 255, 255], + 'isthing': 0, + 'id': 159, + 'name': 'snow' + }, + { + 'color': [104, 84, 109], + 'isthing': 0, + 'id': 161, + 'name': 'stairs' + }, + { + 'color': [169, 164, 131], + 'isthing': 0, + 'id': 166, + 'name': 'tent' + }, + { + 'color': [225, 199, 255], + 'isthing': 0, + 'id': 168, + 'name': 'towel' + }, + { + 'color': [137, 54, 74], + 'isthing': 0, + 'id': 171, + 'name': 'wall-brick' + }, + { + 'color': [135, 158, 223], + 'isthing': 0, + 'id': 175, + 'name': 'wall-stone' + }, + { + 'color': [7, 246, 231], + 'isthing': 0, + 'id': 176, + 'name': 'wall-tile' + }, + { + 'color': [107, 255, 200], + 'isthing': 0, + 'id': 177, + 'name': 'wall-wood' + }, + { + 'color': [58, 41, 149], + 'isthing': 0, + 'id': 178, + 'name': 'water-other' + }, + { + 'color': [183, 121, 142], + 'isthing': 0, + 'id': 180, + 'name': 'window-blind' + }, + { + 'color': [255, 73, 97], + 'isthing': 0, + 'id': 181, + 'name': 'window-other' + }, + { + 'color': [107, 142, 35], + 'isthing': 0, + 'id': 184, + 'name': 'tree-merged' + }, + { + 'color': [190, 153, 153], + 'isthing': 0, + 'id': 185, + 'name': 'fence-merged' + }, + { + 'color': [146, 139, 141], + 'isthing': 0, + 'id': 186, + 'name': 'ceiling-merged' + }, + { + 'color': [70, 130, 180], + 'isthing': 0, + 'id': 187, + 'name': 'sky-other-merged' + }, + { + 'color': [134, 199, 156], + 'isthing': 0, + 'id': 188, + 'name': 'cabinet-merged' + }, + { + 'color': [209, 226, 140], + 'isthing': 0, + 'id': 189, + 'name': 'table-merged' + }, + { + 'color': [96, 36, 108], + 'isthing': 0, + 'id': 190, + 'name': 'floor-other-merged' + }, + { + 'color': [96, 96, 96], + 'isthing': 0, + 'id': 191, + 'name': 'pavement-merged' + }, + { + 'color': [64, 170, 64], + 'isthing': 0, + 'id': 192, + 'name': 'mountain-merged' + }, + { + 'color': [152, 251, 152], + 'isthing': 0, + 'id': 193, + 'name': 'grass-merged' + }, + { + 'color': [208, 229, 228], + 'isthing': 0, + 'id': 194, + 'name': 'dirt-merged' + }, + { + 'color': [206, 186, 171], + 'isthing': 0, + 'id': 195, + 'name': 'paper-merged' + }, + { + 'color': [152, 161, 64], + 'isthing': 0, + 'id': 196, + 'name': 'food-other-merged' + }, + { + 'color': [116, 112, 0], + 'isthing': 0, + 'id': 197, + 'name': 'building-other-merged' + }, + { + 'color': [0, 114, 143], + 'isthing': 0, + 'id': 198, + 'name': 'rock-merged' + }, + { + 'color': [102, 102, 156], + 'isthing': 0, + 'id': 199, + 'name': 'wall-other-merged' + }, + { + 'color': [250, 141, 255], + 'isthing': 0, + 'id': 200, + 'name': 'rug-merged' + }, +] + +# Novel COCO categories +COCO_NOVEL_CATEGORIES = [ + { + 'color': [220, 20, 60], + 'isthing': 1, + 'id': 1, + 'name': 'person' + }, + { + 'color': [119, 11, 32], + 'isthing': 1, + 'id': 2, + 'name': 'bicycle' + }, + { + 'color': [0, 0, 142], + 'isthing': 1, + 'id': 3, + 'name': 'car' + }, + { + 'color': [0, 0, 230], + 'isthing': 1, + 'id': 4, + 'name': 'motorcycle' + }, + { + 'color': [106, 0, 228], + 'isthing': 1, + 'id': 5, + 'name': 'airplane' + }, + { + 'color': [0, 60, 100], + 'isthing': 1, + 'id': 6, + 'name': 'bus' + }, + { + 'color': [0, 80, 100], + 'isthing': 1, + 'id': 7, + 'name': 'train' + }, + { + 'color': [0, 0, 192], + 'isthing': 1, + 'id': 9, + 'name': 'boat' + }, + { + 'color': [165, 42, 42], + 'isthing': 1, + 'id': 16, + 'name': 'bird' + }, + { + 'color': [255, 77, 255], + 'isthing': 1, + 'id': 17, + 'name': 'cat' + }, + { + 'color': [0, 226, 252], + 'isthing': 1, + 'id': 18, + 'name': 'dog' + }, + { + 'color': [182, 182, 255], + 'isthing': 1, + 'id': 19, + 'name': 'horse' + }, + { + 'color': [0, 82, 0], + 'isthing': 1, + 'id': 20, + 'name': 'sheep' + }, + { + 'color': [120, 166, 157], + 'isthing': 1, + 'id': 21, + 'name': 'cow' + }, + { + 'color': [197, 226, 255], + 'isthing': 1, + 'id': 44, + 'name': 'bottle' + }, + { + 'color': [153, 69, 1], + 'isthing': 1, + 'id': 62, + 'name': 'chair' + }, + { + 'color': [3, 95, 161], + 'isthing': 1, + 'id': 63, + 'name': 'couch' + }, + { + 'color': [163, 255, 0], + 'isthing': 1, + 'id': 64, + 'name': 'potted plant' + }, + { + 'color': [0, 182, 199], + 'isthing': 1, + 'id': 67, + 'name': 'dining table' + }, + { + 'color': [183, 130, 88], + 'isthing': 1, + 'id': 72, + 'name': 'tv' + }, +] + + +def _get_coco_fewshot_instances_meta(): + thing_ids = [k['id'] for k in COCO_CATEGORIES if k['isthing'] == 1] + thing_colors = [k['color'] for k in COCO_CATEGORIES if k['isthing'] == 1] + assert len(thing_ids) == 80, len(thing_ids) + # Mapping from the incontiguous COCO category id to an id in [0, 79] + thing_dataset_id_to_contiguous_id = {k: i for i, k in enumerate(thing_ids)} + thing_classes = [k['name'] for k in COCO_CATEGORIES if k['isthing'] == 1] + ret = { + 'thing_dataset_id_to_contiguous_id': thing_dataset_id_to_contiguous_id, + 'thing_classes': thing_classes, + 'thing_colors': thing_colors, + } + + novel_ids = [k['id'] for k in COCO_NOVEL_CATEGORIES if k['isthing'] == 1] + novel_dataset_id_to_contiguous_id = {k: i for i, k in enumerate(novel_ids)} + novel_classes = [ + k['name'] for k in COCO_NOVEL_CATEGORIES if k['isthing'] == 1 + ] + base_categories = [ + k for k in COCO_CATEGORIES + if k['isthing'] == 1 and k['name'] not in novel_classes + ] + base_ids = [k['id'] for k in base_categories] + base_dataset_id_to_contiguous_id = {k: i for i, k in enumerate(base_ids)} + base_classes = [k['name'] for k in base_categories] + ret['novel_dataset_id_to_contiguous_id'] = novel_dataset_id_to_contiguous_id + ret['novel_classes'] = novel_classes + ret['base_dataset_id_to_contiguous_id'] = base_dataset_id_to_contiguous_id + ret['base_classes'] = base_classes + return ret + + +def load_coco_json(root, json_file, image_root, metadata, dataset_name): + is_shots = 'shot' in dataset_name + if is_shots: + imgid2info = {} + shot = dataset_name.split('_')[-2].split('shot')[0] + seed = int(dataset_name.split('_seed')[-1]) + split_dir = os.path.join(root, 'cocosplit', 'seed{}'.format(seed)) + for idx, cls in enumerate(metadata['thing_classes']): + json_file = os.path.join( + split_dir, + 'full_box_{}shot_{}_trainval.json'.format(shot, cls)) + json_file = PathManager.get_local_path(json_file) + with contextlib.redirect_stdout(io.StringIO()): + coco_api = COCO(json_file) + img_ids = sorted(list(coco_api.imgs.keys())) + for img_id in img_ids: + if img_id not in imgid2info: + imgid2info[img_id] = [ + coco_api.loadImgs([img_id])[0], + coco_api.imgToAnns[img_id] + ] + else: + for item in coco_api.imgToAnns[img_id]: + imgid2info[img_id][1].append(item) + imgs, anns = [], [] + for img_id in imgid2info: + imgs.append(imgid2info[img_id][0]) + anns.append(imgid2info[img_id][1]) + else: + json_file = PathManager.get_local_path(json_file) + with contextlib.redirect_stdout(io.StringIO()): + coco_api = COCO(json_file) + # sort indices for reproducible results + img_ids = sorted(list(coco_api.imgs.keys())) + imgs = coco_api.loadImgs(img_ids) + anns = [coco_api.imgToAnns[img_id] for img_id in img_ids] + + imgs_anns = list(zip(imgs, anns)) + id_map = metadata['thing_dataset_id_to_contiguous_id'] + + dataset_dicts = [] + ann_keys = ['iscrowd', 'bbox', 'category_id'] + + for (img_dict, anno_dict_list) in imgs_anns: + record = {} + record['file_name'] = os.path.join(image_root, img_dict['file_name']) + record['height'] = img_dict['height'] + record['width'] = img_dict['width'] + image_id = record['image_id'] = img_dict['id'] + + objs = [] + for anno in anno_dict_list: + assert anno['image_id'] == image_id + assert anno.get('ignore', 0) == 0 + + obj = {key: anno[key] for key in ann_keys if key in anno} + + obj['bbox_mode'] = BoxMode.XYWH_ABS + if obj['category_id'] in id_map: + obj['category_id'] = id_map[obj['category_id']] + objs.append(obj) + record['annotations'] = objs + dataset_dicts.append(record) + + return dataset_dicts + + +def register_meta_coco(name, root, metadata, imgdir, annofile): + DatasetCatalog.register( + name, + lambda: load_coco_json(root, annofile, imgdir, metadata, name), + ) + + if '_base' in name or '_novel' in name: + split = 'base' if '_base' in name else 'novel' + metadata['thing_dataset_id_to_contiguous_id'] = metadata[ + '{}_dataset_id_to_contiguous_id'.format(split)] + metadata['thing_classes'] = metadata['{}_classes'.format(split)] + + MetadataCatalog.get(name).set( + json_file=annofile, + image_root=imgdir, + evaluator_type='coco', + dirname='datasets/coco', # os.path.join(root, 'coco') + **metadata, + ) + + +def register_all_coco(root='datasets'): + + METASPLITS = [ + ('coco14_trainval_all', 'coco/trainval2014', + 'cocosplit/datasplit/trainvalno5k.json'), + ('coco14_trainval_base', 'coco/trainval2014', + 'cocosplit/datasplit/trainvalno5k.json'), + ('coco14_test_all', 'coco/val2014', 'cocosplit/datasplit/5k.json'), + ('coco14_test_base', 'coco/val2014', 'cocosplit/datasplit/5k.json'), + ('coco14_test_novel', 'coco/val2014', 'cocosplit/datasplit/5k.json'), + ] + for prefix in ['all', 'novel']: + for shot in [1, 2, 3, 5, 10, 30]: + for seed in range(10): + name = 'coco14_trainval_{}_{}shot_seed{}'.format( + prefix, shot, seed) + METASPLITS.append((name, 'coco/trainval2014', '')) + + for name, imgdir, annofile in METASPLITS: + register_meta_coco( + name, + root, + _get_coco_fewshot_instances_meta(), + os.path.join(root, imgdir), + os.path.join(root, annofile), + ) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/configuration_mapper.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/configuration_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..f1b1ce2385ad3732a18078ee9ecb715ab14b03aa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/configuration_mapper.py @@ -0,0 +1,113 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/config/defaults.py + +from detectron2.config.defaults import _C + +from modelscope.utils.config import Config + + +def detectron2_default_cfg(): + + _CC = _C + + # ----------- Backbone ----------- # + _CC.MODEL.BACKBONE.FREEZE = False + _CC.MODEL.BACKBONE.FREEZE_AT = 3 + + # ------------- RPN -------------- # + _CC.MODEL.RPN.FREEZE = False + _CC.MODEL.RPN.ENABLE_DECOUPLE = False + _CC.MODEL.RPN.BACKWARD_SCALE = 1.0 + + # ------------- ROI -------------- # + _CC.MODEL.ROI_HEADS.NAME = 'Res5ROIHeads' + _CC.MODEL.ROI_HEADS.FREEZE_FEAT = False + _CC.MODEL.ROI_HEADS.ENABLE_DECOUPLE = False + _CC.MODEL.ROI_HEADS.BACKWARD_SCALE = 1.0 + _CC.MODEL.ROI_HEADS.OUTPUT_LAYER = 'FastRCNNOutputLayers' + _CC.MODEL.ROI_HEADS.CLS_DROPOUT = False + _CC.MODEL.ROI_HEADS.DROPOUT_RATIO = 0.8 + _CC.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION = 7 # for faster + + # ------------- TEST ------------- # + _CC.TEST.PCB_ENABLE = False + _CC.TEST.PCB_MODELTYPE = 'resnet' # res-like + _CC.TEST.PCB_MODELPATH = '' + _CC.TEST.PCB_ALPHA = 0.50 + _CC.TEST.PCB_UPPER = 1.0 + _CC.TEST.PCB_LOWER = 0.05 + + # ------------ Other ------------- # + _CC.SOLVER.WEIGHT_DECAY = 5e-5 + _CC.MUTE_HEADER = True + + return _CC + + +class CfgMapper(): + + def __init__(self, cfg: Config): + + self.cfg = cfg + self.model_cfg = detectron2_default_cfg().clone() + + def __call__(self, *args, **kwargs): + cfg_list = [ + 'MODEL.WEIGHTS', + self.cfg.safe_get('model.weights', ''), 'MODEL.MASK_ON', + self.cfg.safe_get('model.mask_on', False), 'MODEL.BACKBONE.FREEZE', + self.cfg.safe_get('model.backbone.freezed', + False), 'MODEL.RESNETS.DEPTH', + self.cfg.safe_get('model.resnets.depth', + 101), 'MODEL.ROI_HEADS.ENABLE_DECOUPLE', + self.cfg.safe_get('model.roi_heads.enable_decouple', + False), 'MODEL.ROI_HEADS.BACKWARD_SCALE', + self.cfg.safe_get('model.roi_heads.backward_scale', + 1.0), 'MODEL.ROI_HEADS.NUM_CLASSES', + self.cfg.safe_get('model.roi_heads.num_classes', + 80), 'MODEL.ROI_HEADS.FREEZE_FEAT', + self.cfg.safe_get('model.roi_heads.freeze_feat', + False), 'MODEL.ROI_HEADS.CLS_DROPOUT', + self.cfg.safe_get('model.roi_heads.cls_dropout', + False), 'MODEL.RPN.ENABLE_DECOUPLE', + self.cfg.safe_get('model.rpn.enable_decouple', + False), 'MODEL.RPN.BACKWARD_SCALE', + self.cfg.safe_get('model.rpn.backward_scale', + 1.0), 'MODEL.RPN.FREEZE', + self.cfg.safe_get('model.rpn.freezed', + False), 'MODEL.RPN.PRE_NMS_TOPK_TEST', + self.cfg.safe_get('model.rpn.pre_nms_topk_test', + 6000), 'MODEL.RPN.POST_NMS_TOPK_TEST', + self.cfg.safe_get('model.rpn.post_nms_topk_test', + 1000), 'DATASETS.TRAIN', + tuple(self.cfg.safe_get('datasets.train', + ('coco_2017_train', ))), 'DATASETS.TEST', + tuple(self.cfg.safe_get('datasets.test', ('coco_2017_val', ))), + 'SOLVER.IMS_PER_BATCH', + self.cfg.safe_get('train.dataloader.ims_per_batch', + 16), 'SOLVER.BASE_LR', + self.cfg.safe_get('train.optimizer.lr', 0.02), 'SOLVER.STEPS', + tuple( + self.cfg.safe_get('train.lr_scheduler.steps', + (60000, 80000))), 'SOLVER.MAX_ITER', + self.cfg.safe_get('train.max_iter', + 90000), 'SOLVER.CHECKPOINT_PERIOD', + self.cfg.safe_get('train.checkpoint_period', + 5000), 'SOLVER.WARMUP_ITERS', + self.cfg.safe_get('train.lr_scheduler.warmup_iters', + 1000), 'OUTPUT_DIR', + self.cfg.safe_get('train.work_dir', + './output/'), 'INPUT.MIN_SIZE_TRAIN', + tuple( + self.cfg.safe_get('input.min_size_train', + (640, 672, 704, 736, 768, 800))), + 'INPUT.MIN_SIZE_TEST', + self.cfg.safe_get('input.min_size_test', 800), 'TEST.PCB_ENABLE', + self.cfg.safe_get('test.pcb_enable', False), 'TEST.PCB_MODELPATH', + self.cfg.safe_get('test.pcb_modelpath', '') + ] + + self.model_cfg.merge_from_list(cfg_list) + + return self.model_cfg diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/model_surgery_op.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/model_surgery_op.py new file mode 100644 index 0000000000000000000000000000000000000000..b40cc077b9de4d819bfe2481ded99b15f760d1e6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/model_surgery_op.py @@ -0,0 +1,99 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/tools/model_surgery.py + +import argparse +import os + +import torch + +COCO_NOVEL_CLASSES = [ + 1, 2, 3, 4, 5, 6, 7, 9, 16, 17, 18, 19, 20, 21, 44, 62, 63, 64, 67, 72 +] +COCO_BASE_CLASSES = [ + 8, 10, 11, 13, 14, 15, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, + 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 65, 70, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, + 88, 89, 90 +] +COCO_ALL_CLASSES = sorted(COCO_BASE_CLASSES + COCO_NOVEL_CLASSES) +COCO_IDMAP = {v: i for i, v in enumerate(COCO_ALL_CLASSES)} + + +def surgery(data_type, param_name, is_weight, tar_size, ckpt): + weight_name = param_name + ('.weight' if is_weight else '.bias') + pretrained_weight = ckpt['model'][weight_name] + prev_cls = pretrained_weight.size(0) + if 'cls_score' in param_name: + prev_cls -= 1 + if is_weight: + feat_size = pretrained_weight.size(1) + new_weight = torch.rand((tar_size, feat_size)) + torch.nn.init.normal_(new_weight, 0, 0.01) + else: + new_weight = torch.zeros(tar_size) + if data_type == 'coco': + for idx, c in enumerate(COCO_BASE_CLASSES): + if 'cls_score' in param_name: + new_weight[COCO_IDMAP[c]] = pretrained_weight[idx] + else: + new_weight[COCO_IDMAP[c] * 4:(COCO_IDMAP[c] + 1) * 4] = \ + pretrained_weight[idx * 4:(idx + 1) * 4] + else: + new_weight[:prev_cls] = pretrained_weight[:prev_cls] + if 'cls_score' in param_name: + new_weight[-1] = pretrained_weight[-1] # bg class + ckpt['model'][weight_name] = new_weight + + +def model_surgery(src_path, + save_dir, + data_type='pascal_voc', + method='remove', + params_name=[ + 'model.roi_heads.box_predictor.cls_score', + 'model.roi_heads.box_predictor.bbox_pred' + ]): + """ + Either remove the final layer weights for fine-tuning on novel dataset or + append randomly initialized weights for the novel classes. + """ + + assert method in ['remove', + 'randinit'], '{} not implemented'.format(method) + + if data_type == 'coco': + TAR_SIZE = 80 + elif data_type == 'pascal_voc': + TAR_SIZE = 20 + else: + NotImplementedError('{} dataset does not supported'.format(data_type)) + + save_name = 'model_reset_' + ('remove' if method == 'remove' else + 'surgery') + '.pth' + save_path = os.path.join(save_dir, save_name) + os.makedirs(save_dir, exist_ok=True) + + ckpt = torch.load(src_path) + if 'scheduler' in ckpt: + del ckpt['scheduler'] + if 'optimizer' in ckpt: + del ckpt['optimizer'] + if 'iteration' in ckpt: + ckpt['iteration'] = 0 + + if method == 'remove': + for param_name in params_name: + del ckpt['model'][param_name + '.weight'] + if param_name + '.bias' in ckpt['model']: + del ckpt['model'][param_name + '.bias'] + elif method == 'randinit': + tar_sizes = [TAR_SIZE + 1, TAR_SIZE * 4] + for idx, (param_name, + tar_size) in enumerate(zip(params_name, tar_sizes)): + surgery(data_type, param_name, True, tar_size, ckpt) + surgery(data_type, param_name, False, tar_size, ckpt) + else: + raise NotImplementedError + + torch.save(ckpt, save_path) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/register_data.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/register_data.py new file mode 100644 index 0000000000000000000000000000000000000000..ad392c5ce5777cef70f4b50f4fdd0c50770d71a3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/register_data.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .coco_register import register_all_coco +from .voc_register import register_all_voc + + +def register_data(data_type='pascal_voc', data_dir=None): + + if data_type == 'pascal_voc': + if data_dir: + register_all_voc(data_dir) + else: + register_all_voc() + elif data_type == 'coco': + if data_dir: + register_all_coco(data_dir) + else: + register_all_coco() + else: + raise NotImplementedError( + 'no {} dataset was registered'.format(data_type)) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/requirements_check.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/requirements_check.py new file mode 100644 index 0000000000000000000000000000000000000000..bc118ff21c89eebe196dfd1546b3a8bd135cc20d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/requirements_check.py @@ -0,0 +1,81 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import importlib +import sys +from collections import OrderedDict + +from packaging import version + +from modelscope.utils.import_utils import _torch_available + +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + +DETECTRON2_REQUIRED_VERSION = version.parse('0.3') + + +def is_detectron2_version_available(): + _detectron2_available = importlib.util.find_spec('detectron2') is not None + _detectron2_version_available = False + if _detectron2_available: + _detectron2_version = version.parse( + importlib_metadata.version('detectron2')) + _detectron2_version_available = (_detectron2_version.major, + _detectron2_version.minor) == ( + DETECTRON2_REQUIRED_VERSION.major, + DETECTRON2_REQUIRED_VERSION.minor) + + return _detectron2_version_available + + +TORCH_REQUIRED_VERSION = version.parse('1.11') + + +def is_torch_version_available(): + _torch_version_available = False + if _torch_available: + torch_version = version.parse(importlib_metadata.version('torch')) + _torch_version_available = (torch_version.major, + torch_version.minor) == ( + TORCH_REQUIRED_VERSION.major, + TORCH_REQUIRED_VERSION.minor) + + return _torch_version_available + + +DETECTRON2_IMPORT_ERROR = """ +{0} requires the detectron2-0.3 but it was not found in your environment. +You can install it from modelscope lib with pip: +`pip install detectron2==0.3` +""" + +TORCH_VERSION_IMPORT_ERROR = """ +{0} requires the torch-1.11 but it was not found in your environment. You can install it with pip: +`pip install torch==1.11` +""" + +REQUIREMENTS_MAAPING_VERSION = OrderedDict([ + ('detectron2-0.3', (is_detectron2_version_available, + DETECTRON2_IMPORT_ERROR)), + ('torch-1.11', (is_torch_version_available, TORCH_VERSION_IMPORT_ERROR)), +]) + +REQUIREMENTS = ['detectron2-0.3', 'torch-1.11'] + + +def requires_version(): + checks = [] + for req in REQUIREMENTS: + if req in REQUIREMENTS_MAAPING_VERSION: + check = REQUIREMENTS_MAAPING_VERSION[req] + else: + raise NotImplementedError('{} do not supported check'.format(req)) + checks.append(check) + + failed = [ + msg.format('DeFRCN') for available, msg in checks if not available() + ] + if failed: + raise ImportError(''.join(failed)) diff --git a/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/voc_register.py b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/voc_register.py new file mode 100644 index 0000000000000000000000000000000000000000..0b043493ff5b1fe238b9b5d59b59530dbf99dfa1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_defrcn_fewshot/utils/voc_register.py @@ -0,0 +1,342 @@ +# The implementation is adopted from er-muyue/DeFRCN +# made publicly available under the MIT License at +# https://github.com/er-muyue/DeFRCN/blob/main/defrcn/data/meta_voc.py + +import os +import xml.etree.ElementTree as ET + +import numpy as np +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.structures import BoxMode +from fvcore.common.file_io import PathManager + +# PASCAL VOC categories +PASCAL_VOC_ALL_CATEGORIES = { + 1: [ + 'aeroplane', + 'bicycle', + 'boat', + 'bottle', + 'car', + 'cat', + 'chair', + 'diningtable', + 'dog', + 'horse', + 'person', + 'pottedplant', + 'sheep', + 'train', + 'tvmonitor', + 'bird', + 'bus', + 'cow', + 'motorbike', + 'sofa', + ], + 2: [ + 'bicycle', + 'bird', + 'boat', + 'bus', + 'car', + 'cat', + 'chair', + 'diningtable', + 'dog', + 'motorbike', + 'person', + 'pottedplant', + 'sheep', + 'train', + 'tvmonitor', + 'aeroplane', + 'bottle', + 'cow', + 'horse', + 'sofa', + ], + 3: [ + 'aeroplane', + 'bicycle', + 'bird', + 'bottle', + 'bus', + 'car', + 'chair', + 'cow', + 'diningtable', + 'dog', + 'horse', + 'person', + 'pottedplant', + 'train', + 'tvmonitor', + 'boat', + 'cat', + 'motorbike', + 'sheep', + 'sofa', + ] +} + +PASCAL_VOC_NOVEL_CATEGORIES = { + 1: ['bird', 'bus', 'cow', 'motorbike', 'sofa'], + 2: ['aeroplane', 'bottle', 'cow', 'horse', 'sofa'], + 3: ['boat', 'cat', 'motorbike', 'sheep', 'sofa'] +} + +PASCAL_VOC_BASE_CATEGORIES = { + 1: [ + 'aeroplane', + 'bicycle', + 'boat', + 'bottle', + 'car', + 'cat', + 'chair', + 'diningtable', + 'dog', + 'horse', + 'person', + 'pottedplant', + 'sheep', + 'train', + 'tvmonitor', + ], + 2: [ + 'bicycle', + 'bird', + 'boat', + 'bus', + 'car', + 'cat', + 'chair', + 'diningtable', + 'dog', + 'motorbike', + 'person', + 'pottedplant', + 'sheep', + 'train', + 'tvmonitor', + ], + 3: [ + 'aeroplane', + 'bicycle', + 'bird', + 'bottle', + 'bus', + 'car', + 'chair', + 'cow', + 'diningtable', + 'dog', + 'horse', + 'person', + 'pottedplant', + 'train', + 'tvmonitor', + ] +} + + +def load_filtered_voc_instances(name: str, root: str, dirname: str, split: str, + classnames: str): + """ + Load Pascal VOC detection annotations to Detectron2 format. + Args: + dirname: Contain "Annotations", "ImageSets", "JPEGImages" + split (str): one of "train", "test", "val", "trainval" + """ + is_shots = 'shot' in name + dicts = [] + if is_shots: + fileids = {} + # split_dir = os.path.join("datasets", "vocsplit") + split_dir = os.path.join(root, 'vocsplit') + shot = name.split('_')[-2].split('shot')[0] + seed = int(name.split('_seed')[-1]) + split_dir = os.path.join(split_dir, 'seed{}'.format(seed)) + for cls in classnames: + with PathManager.open( + os.path.join(split_dir, + 'box_{}shot_{}_train.txt'.format(shot, + cls))) as f: + fileids_ = np.loadtxt(f, dtype=np.str_).tolist() + if isinstance(fileids_, str): + fileids_ = [fileids_] + fileids_ = [ + fid.split('/')[-1].split('.jpg')[0] for fid in fileids_ + ] + fileids[cls] = fileids_ + + for cls, fileids_ in fileids.items(): + dicts_ = [] + for fileid in fileids_: + year = '2012' if '_' in fileid else '2007' + # dirname = os.path.join("datasets", "VOC{}".format(year)) + # anno_file = os.path.join(dirname, "Annotations", fileid + ".xml") + # jpeg_file = os.path.join(dirname, "JPEGImages", fileid + ".jpg") + + dir_voc = os.path.join(root, 'VOC{}'.format(year)) + anno_file = os.path.join(dir_voc, 'Annotations', + fileid + '.xml') + jpeg_file = os.path.join(dir_voc, 'JPEGImages', + fileid + '.jpg') + + tree = ET.parse(anno_file) + + for obj in tree.findall('object'): + r = { + 'file_name': jpeg_file, + 'image_id': fileid, + 'height': int(tree.findall('./size/height')[0].text), + 'width': int(tree.findall('./size/width')[0].text), + } + cls_ = obj.find('name').text + if cls != cls_: + continue + bbox = obj.find('bndbox') + bbox = [ + float(bbox.find(x).text) + for x in ['xmin', 'ymin', 'xmax', 'ymax'] + ] + bbox[0] -= 1.0 + bbox[1] -= 1.0 + + instances = [{ + 'category_id': classnames.index(cls), + 'bbox': bbox, + 'bbox_mode': BoxMode.XYXY_ABS, + }] + r['annotations'] = instances + dicts_.append(r) + if len(dicts_) > int(shot): + dicts_ = np.random.choice(dicts_, int(shot), replace=False) + dicts.extend(dicts_) + else: + with PathManager.open( + os.path.join(root, dirname, 'ImageSets', 'Main', + split + '.txt')) as f: + fileids = np.loadtxt(f, dtype=np.str_) + + for fileid in fileids: + anno_file = os.path.join(root, dirname, 'Annotations', + fileid + '.xml') + jpeg_file = os.path.join(root, dirname, 'JPEGImages', + fileid + '.jpg') + + tree = ET.parse(anno_file) + + r = { + 'file_name': jpeg_file, + 'image_id': fileid, + 'height': int(tree.findall('./size/height')[0].text), + 'width': int(tree.findall('./size/width')[0].text), + } + instances = [] + + for obj in tree.findall('object'): + cls = obj.find('name').text + if not (cls in classnames): + continue + bbox = obj.find('bndbox') + bbox = [ + float(bbox.find(x).text) + for x in ['xmin', 'ymin', 'xmax', 'ymax'] + ] + bbox[0] -= 1.0 + bbox[1] -= 1.0 + + instances.append({ + 'category_id': classnames.index(cls), + 'bbox': bbox, + 'bbox_mode': BoxMode.XYXY_ABS, + }) + r['annotations'] = instances + dicts.append(r) + + return dicts + + +def register_meta_voc(name, root, dirname, split, year, keepclasses, sid): + if keepclasses.startswith('base_novel'): + thing_classes = PASCAL_VOC_ALL_CATEGORIES[sid] + elif keepclasses.startswith('base'): + thing_classes = PASCAL_VOC_BASE_CATEGORIES[sid] + elif keepclasses.startswith('novel'): + thing_classes = PASCAL_VOC_NOVEL_CATEGORIES[sid] + + DatasetCatalog.register( + name, + lambda: load_filtered_voc_instances(name, root, dirname, split, + thing_classes), + ) + + MetadataCatalog.get(name).set( + thing_classes=thing_classes, + dirname=os.path.join(root, dirname), + year=year, + split=split, + base_classes=PASCAL_VOC_BASE_CATEGORIES[sid], + novel_classes=PASCAL_VOC_NOVEL_CATEGORIES[sid], + ) + + +def register_all_voc(root='datasets'): + + METASPLITS = [ + ('voc_2007_trainval_base1', 'VOC2007', 'trainval', 'base1', 1), + ('voc_2007_trainval_base2', 'VOC2007', 'trainval', 'base2', 2), + ('voc_2007_trainval_base3', 'VOC2007', 'trainval', 'base3', 3), + ('voc_2012_trainval_base1', 'VOC2012', 'trainval', 'base1', 1), + ('voc_2012_trainval_base2', 'VOC2012', 'trainval', 'base2', 2), + ('voc_2012_trainval_base3', 'VOC2012', 'trainval', 'base3', 3), + ('voc_2007_trainval_all1', 'VOC2007', 'trainval', 'base_novel_1', 1), + ('voc_2007_trainval_all2', 'VOC2007', 'trainval', 'base_novel_2', 2), + ('voc_2007_trainval_all3', 'VOC2007', 'trainval', 'base_novel_3', 3), + ('voc_2012_trainval_all1', 'VOC2012', 'trainval', 'base_novel_1', 1), + ('voc_2012_trainval_all2', 'VOC2012', 'trainval', 'base_novel_2', 2), + ('voc_2012_trainval_all3', 'VOC2012', 'trainval', 'base_novel_3', 3), + ('voc_2007_test_base1', 'VOC2007', 'test', 'base1', 1), + ('voc_2007_test_base2', 'VOC2007', 'test', 'base2', 2), + ('voc_2007_test_base3', 'VOC2007', 'test', 'base3', 3), + ('voc_2007_test_novel1', 'VOC2007', 'test', 'novel1', 1), + ('voc_2007_test_novel2', 'VOC2007', 'test', 'novel2', 2), + ('voc_2007_test_novel3', 'VOC2007', 'test', 'novel3', 3), + ('voc_2007_test_all1', 'VOC2007', 'test', 'base_novel_1', 1), + ('voc_2007_test_all2', 'VOC2007', 'test', 'base_novel_2', 2), + ('voc_2007_test_all3', 'VOC2007', 'test', 'base_novel_3', 3), + ] + for prefix in ['all', 'novel']: + for sid in range(1, 4): + for shot in [1, 2, 3, 5, 10]: + for year in [2007, 2012]: + for seed in range(30): + seed = '_seed{}'.format(seed) + name = 'voc_{}_trainval_{}{}_{}shot{}'.format( + year, prefix, sid, shot, seed) + dirname = 'VOC{}'.format(year) + img_file = '{}_{}shot_split_{}_trainval'.format( + prefix, shot, sid) + keepclasses = ('base_novel_{}'.format(sid) if prefix + == 'all' else 'novel{}'.format(sid)) + METASPLITS.append( + (name, dirname, img_file, keepclasses, sid)) + + for name, dirname, split, keepclasses, sid in METASPLITS: + if name in DatasetCatalog: + continue + + year = 2007 if '2007' in name else 2012 + register_meta_voc( + name, + root, + dirname, + split, + year, + keepclasses, + sid, + ) + MetadataCatalog.get(name).evaluator_type = 'pascal_voc' diff --git a/runtime/Lib/site_packages/models/cv/image_denoise/__init__.py b/runtime/Lib/site_packages/models/cv/image_denoise/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa925dafc3bfc4b24bfc632dc0c74fee3f2f66d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_denoise/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .nafnet_for_image_denoise import NAFNetForImageDenoise + +else: + _import_structure = {'nafnet_for_image_denoise': ['NAFNetForImageDenoise']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/NAFNet_arch.py b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/NAFNet_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..c4de072904f0ea325e6dabf18937b90d8f002d30 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/NAFNet_arch.py @@ -0,0 +1,238 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/megvii-research/NAFNet/blob/main/basicsr/models/archs/NAFNet_arch.py +# Copyright (c) 2022 megvii-model. All Rights Reserved. +# ------------------------------------------------------------------------ + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .arch_util import LayerNorm2d + + +class SimpleGate(nn.Module): + + def forward(self, x): + x1, x2 = x.chunk(2, dim=1) + return x1 * x2 + + +class NAFBlock(nn.Module): + + def __init__(self, c, DW_Expand=2, FFN_Expand=2, drop_out_rate=0.): + super().__init__() + dw_channel = c * DW_Expand + self.conv1 = nn.Conv2d( + in_channels=c, + out_channels=dw_channel, + kernel_size=1, + padding=0, + stride=1, + groups=1, + bias=True) + self.conv2 = nn.Conv2d( + in_channels=dw_channel, + out_channels=dw_channel, + kernel_size=3, + padding=1, + stride=1, + groups=dw_channel, + bias=True) + self.conv3 = nn.Conv2d( + in_channels=dw_channel // 2, + out_channels=c, + kernel_size=1, + padding=0, + stride=1, + groups=1, + bias=True) + + # Simplified Channel Attention + self.sca = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d( + in_channels=dw_channel // 2, + out_channels=dw_channel // 2, + kernel_size=1, + padding=0, + stride=1, + groups=1, + bias=True), + ) + + # SimpleGate + self.sg = SimpleGate() + + ffn_channel = FFN_Expand * c + self.conv4 = nn.Conv2d( + in_channels=c, + out_channels=ffn_channel, + kernel_size=1, + padding=0, + stride=1, + groups=1, + bias=True) + self.conv5 = nn.Conv2d( + in_channels=ffn_channel // 2, + out_channels=c, + kernel_size=1, + padding=0, + stride=1, + groups=1, + bias=True) + + self.norm1 = LayerNorm2d(c) + self.norm2 = LayerNorm2d(c) + + self.dropout1 = nn.Dropout( + drop_out_rate) if drop_out_rate > 0. else nn.Identity() + self.dropout2 = nn.Dropout( + drop_out_rate) if drop_out_rate > 0. else nn.Identity() + + self.beta = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True) + self.gamma = nn.Parameter( + torch.zeros((1, c, 1, 1)), requires_grad=True) + + def forward(self, inp): + x = inp + + x = self.norm1(x) + + x = self.conv1(x) + x = self.conv2(x) + x = self.sg(x) + x = x * self.sca(x) + x = self.conv3(x) + + x = self.dropout1(x) + + y = inp + x * self.beta + + x = self.conv4(self.norm2(y)) + x = self.sg(x) + x = self.conv5(x) + + x = self.dropout2(x) + + return y + x * self.gamma + + +class NAFNet(nn.Module): + + def __init__(self, + img_channel=3, + width=16, + middle_blk_num=1, + enc_blk_nums=[], + dec_blk_nums=[]): + super().__init__() + + self.intro = nn.Conv2d( + in_channels=img_channel, + out_channels=width, + kernel_size=3, + padding=1, + stride=1, + groups=1, + bias=True) + self.ending = nn.Conv2d( + in_channels=width, + out_channels=img_channel, + kernel_size=3, + padding=1, + stride=1, + groups=1, + bias=True) + + self.encoders = nn.ModuleList() + self.decoders = nn.ModuleList() + self.middle_blks = nn.ModuleList() + self.ups = nn.ModuleList() + self.downs = nn.ModuleList() + + chan = width + for num in enc_blk_nums: + self.encoders.append( + nn.Sequential(*[NAFBlock(chan) for _ in range(num)])) + self.downs.append(nn.Conv2d(chan, 2 * chan, 2, 2)) + chan = chan * 2 + + self.middle_blks = \ + nn.Sequential( + *[NAFBlock(chan) for _ in range(middle_blk_num)] + ) + + for num in dec_blk_nums: + self.ups.append( + nn.Sequential( + nn.Conv2d(chan, chan * 2, 1, bias=False), + nn.PixelShuffle(2))) + chan = chan // 2 + self.decoders.append( + nn.Sequential(*[NAFBlock(chan) for _ in range(num)])) + + self.padder_size = 2**len(self.encoders) + + def forward(self, inp): + B, C, H, W = inp.shape + inp = self.check_image_size(inp) + + x = self.intro(inp) + + encs = [] + + for encoder, down in zip(self.encoders, self.downs): + x = encoder(x) + encs.append(x) + x = down(x) + + x = self.middle_blks(x) + + for decoder, up, enc_skip in zip(self.decoders, self.ups, encs[::-1]): + x = up(x) + x = x + enc_skip + x = decoder(x) + + x = self.ending(x) + x = x + inp + + return x[:, :, :H, :W] + + def check_image_size(self, x): + _, _, h, w = x.size() + mod_pad_h = (self.padder_size + - h % self.padder_size) % self.padder_size + mod_pad_w = (self.padder_size + - w % self.padder_size) % self.padder_size + x = F.pad(x, (0, mod_pad_w, 0, mod_pad_h)) + return x + + +class PSNRLoss(nn.Module): + + def __init__(self, loss_weight=1.0, reduction='mean', toY=False): + super(PSNRLoss, self).__init__() + assert reduction == 'mean' + self.loss_weight = loss_weight + self.scale = 10 / np.log(10) + self.toY = toY + self.coef = torch.tensor([65.481, 128.553, 24.966]).reshape(1, 3, 1, 1) + self.first = True + + def forward(self, pred, target): + assert len(pred.size()) == 4 + if self.toY: + if self.first: + self.coef = self.coef.to(pred.device) + self.first = False + + pred = (pred * self.coef).sum(dim=1).unsqueeze(dim=1) + 16. + target = (target * self.coef).sum(dim=1).unsqueeze(dim=1) + 16. + + pred, target = pred / 255., target / 255. + pass + assert len(pred.size()) == 4 + + return self.loss_weight * self.scale * torch.log(( + (pred - target)**2).mean(dim=(1, 2, 3)) + 1e-8).mean() diff --git a/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/__init__.py b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/arch_util.py b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/arch_util.py new file mode 100644 index 0000000000000000000000000000000000000000..2d406141d7fa70ab682022a28a1a3827e6fac79d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet/arch_util.py @@ -0,0 +1,47 @@ +# ------------------------------------------------------------------------ +# Modified from BasicSR (https://github.com/xinntao/BasicSR) +# Copyright 2018-2020 BasicSR Authors +# ------------------------------------------------------------------------ + +import torch +import torch.nn as nn + + +class LayerNormFunction(torch.autograd.Function): + + @staticmethod + def forward(ctx, x, weight, bias, eps): + ctx.eps = eps + N, C, H, W = x.size() + mu = x.mean(1, keepdim=True) + var = (x - mu).pow(2).mean(1, keepdim=True) + y = (x - mu) / (var + eps).sqrt() + ctx.save_for_backward(y, var, weight) + y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1) + return y + + @staticmethod + def backward(ctx, grad_output): + eps = ctx.eps + + N, C, H, W = grad_output.size() + y, var, weight = ctx.saved_variables + g = grad_output * weight.view(1, C, 1, 1) + mean_g = g.mean(dim=1, keepdim=True) + + mean_gy = (g * y).mean(dim=1, keepdim=True) + gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g) + return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum( + dim=0), grad_output.sum(dim=3).sum(dim=2).sum(dim=0), None + + +class LayerNorm2d(nn.Module): + + def __init__(self, channels, eps=1e-6): + super(LayerNorm2d, self).__init__() + self.register_parameter('weight', nn.Parameter(torch.ones(channels))) + self.register_parameter('bias', nn.Parameter(torch.zeros(channels))) + self.eps = eps + + def forward(self, x): + return LayerNormFunction.apply(x, self.weight, self.bias, self.eps) diff --git a/runtime/Lib/site_packages/models/cv/image_denoise/nafnet_for_image_denoise.py b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet_for_image_denoise.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c43c4aff95270ffddc560832981e1b0fffd6ad --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_denoise/nafnet_for_image_denoise.py @@ -0,0 +1,70 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch.cuda + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .nafnet.NAFNet_arch import NAFNet, PSNRLoss + +logger = get_logger() +__all__ = ['NAFNetForImageDenoise'] + + +@MODELS.register_module(Tasks.image_denoising, module_name=Models.nafnet) +class NAFNetForImageDenoise(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image denoise model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + self.model = NAFNet(**self.config.model.network_g) + self.loss = PSNRLoss() + self.model = self._load_pretrained(self.model, model_path) + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + preds = self.model(input) + return {'loss': self.loss(preds, target)} + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'outputs': self.model(input).clamp(0, 1)} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(input) + preds = list(torch.split(preds.clamp(0, 1), 1, 0)) + targets = list(torch.split(target.clamp(0, 1), 1, 0)) + + return {'pred': preds, 'target': targets} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/__init__.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_depth.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..1e5444e237b5c69e69ae1ec57da8e162b3b83aef --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_depth.py @@ -0,0 +1,215 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .newcrf_layers import NewCRF +from .swin_transformer import SwinTransformer +from .uper_crf_head import PSP + + +class NewCRFDepth(nn.Module): + """ + Depth network based on neural window FC-CRFs architecture. + """ + + def __init__(self, + version=None, + inv_depth=False, + pretrained=None, + frozen_stages=-1, + min_depth=0.1, + max_depth=100.0, + **kwargs): + super().__init__() + + self.inv_depth = inv_depth + self.with_auxiliary_head = False + self.with_neck = False + + norm_cfg = dict(type='BN', requires_grad=True) + # norm_cfg = dict(type='GN', requires_grad=True, num_groups=8) + + window_size = int(version[-2:]) + + if version[:-2] == 'base': + embed_dim = 128 + depths = [2, 2, 18, 2] + num_heads = [4, 8, 16, 32] + in_channels = [128, 256, 512, 1024] + elif version[:-2] == 'large': + embed_dim = 192 + depths = [2, 2, 18, 2] + num_heads = [6, 12, 24, 48] + in_channels = [192, 384, 768, 1536] + elif version[:-2] == 'tiny': + embed_dim = 96 + depths = [2, 2, 6, 2] + num_heads = [3, 6, 12, 24] + in_channels = [96, 192, 384, 768] + + backbone_cfg = dict( + embed_dim=embed_dim, + depths=depths, + num_heads=num_heads, + window_size=window_size, + ape=False, + drop_path_rate=0.3, + patch_norm=True, + use_checkpoint=False, + frozen_stages=frozen_stages) + + embed_dim = 512 + decoder_cfg = dict( + in_channels=in_channels, + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=embed_dim, + dropout_ratio=0.0, + num_classes=32, + norm_cfg=norm_cfg, + align_corners=False) + + self.backbone = SwinTransformer(**backbone_cfg) + # v_dim = decoder_cfg['num_classes'] * 4 + win = 7 + crf_dims = [128, 256, 512, 1024] + v_dims = [64, 128, 256, embed_dim] + self.crf3 = NewCRF( + input_dim=in_channels[3], + embed_dim=crf_dims[3], + window_size=win, + v_dim=v_dims[3], + num_heads=32) + self.crf2 = NewCRF( + input_dim=in_channels[2], + embed_dim=crf_dims[2], + window_size=win, + v_dim=v_dims[2], + num_heads=16) + self.crf1 = NewCRF( + input_dim=in_channels[1], + embed_dim=crf_dims[1], + window_size=win, + v_dim=v_dims[1], + num_heads=8) + self.crf0 = NewCRF( + input_dim=in_channels[0], + embed_dim=crf_dims[0], + window_size=win, + v_dim=v_dims[0], + num_heads=4) + + self.decoder = PSP(**decoder_cfg) + self.disp_head1 = DispHead(input_dim=crf_dims[0]) + + self.up_mode = 'bilinear' + if self.up_mode == 'mask': + self.mask_head = nn.Sequential( + nn.Conv2d(crf_dims[0], 64, 3, padding=1), + nn.ReLU(inplace=True), nn.Conv2d(64, 16 * 9, 1, padding=0)) + + self.min_depth = min_depth + self.max_depth = max_depth + + self.init_weights(pretrained=pretrained) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + # print(f'== Load encoder backbone from: {pretrained}') + self.backbone.init_weights(pretrained=pretrained) + self.decoder.init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def upsample_mask(self, disp, mask): + """ Upsample disp [H/4, W/4, 1] -> [H, W, 1] using convex combination """ + N, _, H, W = disp.shape + mask = mask.view(N, 1, 9, 4, 4, H, W) + mask = torch.softmax(mask, dim=2) + + up_disp = F.unfold(disp, kernel_size=3, padding=1) + up_disp = up_disp.view(N, 1, 9, 1, 1, H, W) + + up_disp = torch.sum(mask * up_disp, dim=2) + up_disp = up_disp.permute(0, 1, 4, 2, 5, 3) + return up_disp.reshape(N, 1, 4 * H, 4 * W) + + def forward(self, imgs): + + feats = self.backbone(imgs) + if self.with_neck: + feats = self.neck(feats) + + ppm_out = self.decoder(feats) + + e3 = self.crf3(feats[3], ppm_out) + e3 = nn.PixelShuffle(2)(e3) + e2 = self.crf2(feats[2], e3) + e2 = nn.PixelShuffle(2)(e2) + e1 = self.crf1(feats[1], e2) + e1 = nn.PixelShuffle(2)(e1) + e0 = self.crf0(feats[0], e1) + + if self.up_mode == 'mask': + mask = self.mask_head(e0) + d1 = self.disp_head1(e0, 1) + d1 = self.upsample_mask(d1, mask) + else: + d1 = self.disp_head1(e0, 4) + + depth = d1 * self.max_depth + + return depth + + +class DispHead(nn.Module): + + def __init__(self, input_dim=100): + super(DispHead, self).__init__() + # self.norm1 = nn.BatchNorm2d(input_dim) + self.conv1 = nn.Conv2d(input_dim, 1, 3, padding=1) + # self.relu = nn.ReLU(inplace=True) + self.sigmoid = nn.Sigmoid() + + def forward(self, x, scale): + # x = self.relu(self.norm1(x)) + x = self.sigmoid(self.conv1(x)) + if scale > 1: + x = upsample(x, scale_factor=scale) + return x + + +class DispUnpack(nn.Module): + + def __init__(self, input_dim=100, hidden_dim=128): + super(DispUnpack, self).__init__() + self.conv1 = nn.Conv2d(input_dim, hidden_dim, 3, padding=1) + self.conv2 = nn.Conv2d(hidden_dim, 16, 3, padding=1) + self.relu = nn.ReLU(inplace=True) + self.sigmoid = nn.Sigmoid() + self.pixel_shuffle = nn.PixelShuffle(4) + + def forward(self, x, output_size): + x = self.relu(self.conv1(x)) + x = self.sigmoid(self.conv2(x)) # [b, 16, h/4, w/4] + # x = torch.reshape(x, [x.shape[0], 1, x.shape[2]*4, x.shape[3]*4]) + x = self.pixel_shuffle(x) + + return x + + +def upsample(x, scale_factor=2, mode='bilinear', align_corners=False): + """Upsample input tensor by a factor of 2 + """ + return F.interpolate( + x, scale_factor=scale_factor, mode=mode, align_corners=align_corners) diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_layers.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a57081e3e3403f64d90e28cdaa69cd61d80a854e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_layers.py @@ -0,0 +1,504 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + v_dim, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qk = nn.Linear(dim, dim * 2, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(v_dim, v_dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, v, mask=None): + """ Forward function. + + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qk = self.qk(x).reshape(B_, N, 2, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k = qk[0], qk[ + 1] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + # assert self.dim % v.shape[-1] == 0, "self.dim % v.shape[-1] != 0" + # repeat_num = self.dim // v.shape[-1] + # v = v.view(B_, N, self.num_heads // repeat_num, -1).transpose(1, 2).repeat(1, repeat_num, 1, 1) + + assert self.dim == v.shape[-1], 'self.dim != v.shape[-1]' + v = v.view(B_, N, self.num_heads, -1).transpose(1, 2) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class CRFBlock(nn.Module): + """ CRF Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + v_dim, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.v_dim = v_dim + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + v_dim=v_dim, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(v_dim) + mlp_hidden_dim = int(v_dim * mlp_ratio) + self.mlp = Mlp( + in_features=v_dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + self.H = None + self.W = None + + def forward(self, x, v, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, 'input feature has wrong size' + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + v = F.pad(v, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + shifted_v = torch.roll( + v, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + shifted_v = v + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + v_windows = window_partition( + shifted_v, self.window_size) # nW*B, window_size, window_size, C + v_windows = v_windows.view( + -1, self.window_size * self.window_size, + v_windows.shape[-1]) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, v_windows, + mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, self.v_dim) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, + Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, self.v_dim) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class BasicCRFLayer(nn.Module): + """ A basic NeWCRFs layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + v_dim, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + CRFBlock( + dim=dim, + num_heads=num_heads, + v_dim=v_dim, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, v, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, v, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class NewCRF(nn.Module): + + def __init__(self, + input_dim=96, + embed_dim=96, + v_dim=64, + window_size=7, + num_heads=4, + depth=2, + patch_size=4, + in_chans=3, + norm_layer=nn.LayerNorm, + patch_norm=True): + super().__init__() + + self.embed_dim = embed_dim + self.patch_norm = patch_norm + + if input_dim != embed_dim: + self.proj_x = nn.Conv2d(input_dim, embed_dim, 3, padding=1) + else: + self.proj_x = None + + if v_dim != embed_dim: + self.proj_v = nn.Conv2d(v_dim, embed_dim, 3, padding=1) + elif embed_dim % v_dim == 0: + self.proj_v = None + + v_dim = embed_dim + assert v_dim == embed_dim + + self.crf_layer = BasicCRFLayer( + dim=embed_dim, + depth=depth, + num_heads=num_heads, + v_dim=v_dim, + window_size=window_size, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=norm_layer, + downsample=None, + use_checkpoint=False) + + layer = norm_layer(embed_dim) + layer_name = 'norm_crf' + self.add_module(layer_name, layer) + + def forward(self, x, v): + if self.proj_x is not None: + x = self.proj_x(x) + if self.proj_v is not None: + v = self.proj_v(v) + + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + v = v.transpose(1, 2).transpose(2, 3) + + x_out, H, W, x, Wh, Ww = self.crf_layer(x, v, Wh, Ww) + norm_layer = getattr(self, 'norm_crf') + x_out = norm_layer(x_out) + out = x_out.view(-1, H, W, self.embed_dim).permute(0, 3, 1, + 2).contiguous() + + return out diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_utils.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aa4076023095fb8137399f0178168e71216c66d0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/newcrf_utils.py @@ -0,0 +1,272 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import os.path as osp +import pkgutil +import warnings +from collections import OrderedDict +from importlib import import_module + +import torch +import torch.nn as nn +import torchvision +from torch import distributed as dist +from torch.nn import functional as F +from torch.nn.parallel import DataParallel, DistributedDataParallel +from torch.utils import model_zoo + +TORCH_VERSION = torch.__version__ + + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > output_h: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + if isinstance(size, torch.Size): + size = tuple(int(x) for x in size) + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +def normal_init(module, mean=0, std=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.normal_(module.weight, mean, std) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def is_module_wrapper(module): + module_wrappers = (DataParallel, DistributedDataParallel) + return isinstance(module, module_wrappers) + + +def get_dist_info(): + if TORCH_VERSION < '1.0': + initialized = dist._initialized + else: + if dist.is_available(): + initialized = dist.is_initialized() + else: + initialized = False + if initialized: + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + return rank, world_size + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + else: + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, + filename, + map_location='cpu', + strict=False, + logger=None): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = { + k.replace('encoder.', ''): v + for k, v in state_dict.items() if k.startswith('encoder.') + } + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H * W: + logger.warning('Error in loading absolute_pos_embed, pass') + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view( + N2, H, W, C2).permute(0, 3, 1, 2) + + # interpolate position bias table if needed + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + logger.warning(f'Error in loading {table_key}, pass') + else: + if L1 != L2: + S1 = int(L1**0.5) + S2 = int(L2**0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), + mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view( + nH2, L2).permute(1, 0) + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/swin_transformer.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..ba219b4ad541a10c52ecdc15e93584d0a86898a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/swin_transformer.py @@ -0,0 +1,706 @@ +# The implementation is adopted from Swin Transformer +# made publicly available under the MIT License at https://github.com/microsoft/Swin-Transformer + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from .newcrf_utils import load_checkpoint + + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """ Swin Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, 'input feature has wrong size' + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, + Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, 'input feature has wrong size' + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=4, + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, + (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """ Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1] + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], + patches_resolution[1])) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if + (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if isinstance(pretrained, str): + self.apply(_init_weights) + # logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False) + elif pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic') + x = (x + absolute_pos_embed).flatten(2).transpose(1, + 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, + self.num_features[i]).permute(0, 3, 1, + 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/uper_crf_head.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/uper_crf_head.py new file mode 100644 index 0000000000000000000000000000000000000000..93e1edf6ea9f7b7ca04f30753f085ebd081e13c7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/networks/uper_crf_head.py @@ -0,0 +1,365 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule + +from .newcrf_utils import normal_init, resize + + +class PPM(nn.ModuleList): + """Pooling Pyramid Module used in PSPNet. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + align_corners (bool): align_corners argument of F.interpolate. + """ + + def __init__(self, pool_scales, in_channels, channels, conv_cfg, norm_cfg, + act_cfg, align_corners): + super(PPM, self).__init__() + self.pool_scales = pool_scales + self.align_corners = align_corners + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for pool_scale in pool_scales: + # == if batch size = 1, BN is not supported, change to GN + if pool_scale == 1: + norm_cfg = dict(type='GN', requires_grad=True, num_groups=256) + self.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(pool_scale), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=norm_cfg, + act_cfg=self.act_cfg))) + + def forward(self, x): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(x) + upsampled_ppm_out = resize( + ppm_out, + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ppm_outs.append(upsampled_ppm_out) + return ppm_outs + + +class BaseDecodeHead(nn.Module): + """Base class for BaseDecodeHead. + + Args: + in_channels (int|Sequence[int]): Input channels. + channels (int): Channels after modules, before conv_seg. + num_classes (int): Number of classes. + dropout_ratio (float): Ratio of dropout layer. Default: 0.1. + conv_cfg (dict|None): Config of conv layers. Default: None. + norm_cfg (dict|None): Config of norm layers. Default: None. + act_cfg (dict): Config of activation layers. + Default: dict(type='ReLU') + in_index (int|Sequence[int]): Input feature index. Default: -1 + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + Default: None. + loss_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss'). + ignore_index (int | None): The label index to be ignored. When using + masked BCE loss, ignore_index should be set to None. Default: 255 + sampler (dict|None): The config of segmentation map sampler. + Default: None. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + """ + + def __init__(self, + in_channels, + channels, + *, + num_classes, + dropout_ratio=0.1, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + in_index=-1, + input_transform=None, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + ignore_index=255, + sampler=None, + align_corners=False): + super(BaseDecodeHead, self).__init__() + self._init_inputs(in_channels, in_index, input_transform) + self.channels = channels + self.num_classes = num_classes + self.dropout_ratio = dropout_ratio + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.in_index = in_index + # self.loss_decode = build_loss(loss_decode) + self.ignore_index = ignore_index + self.align_corners = align_corners + # if sampler is not None: + # self.sampler = build_pixel_sampler(sampler, context=self) + # else: + # self.sampler = None + + # self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + # self.conv1 = nn.Conv2d(channels, num_classes, 3, padding=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.fp16_enabled = False + + def extra_repr(self): + """Extra repr.""" + s = f'input_transform={self.input_transform}, ' \ + f'ignore_index={self.ignore_index}, ' \ + f'align_corners={self.align_corners}' + return s + + def _init_inputs(self, in_channels, in_index, input_transform): + """Check and initialize input transforms. + + The in_channels, in_index and input_transform must match. + Specifically, when input_transform is None, only single feature map + will be selected. So in_channels and in_index must be of type int. + When input_transform + + Args: + in_channels (int|Sequence[int]): Input channels. + in_index (int|Sequence[int]): Input feature index. + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + """ + + if input_transform is not None: + assert input_transform in ['resize_concat', 'multiple_select'] + self.input_transform = input_transform + self.in_index = in_index + if input_transform is not None: + assert isinstance(in_channels, (list, tuple)) + assert isinstance(in_index, (list, tuple)) + assert len(in_channels) == len(in_index) + if input_transform == 'resize_concat': + self.in_channels = sum(in_channels) + else: + self.in_channels = in_channels + else: + assert isinstance(in_channels, int) + assert isinstance(in_index, int) + self.in_channels = in_channels + + def init_weights(self): + """Initialize weights of classification layer.""" + # normal_init(self.conv_seg, mean=0, std=0.01) + # normal_init(self.conv1, mean=0, std=0.01) + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + + if self.input_transform == 'resize_concat': + inputs = [inputs[i] for i in self.in_index] + upsampled_inputs = [ + resize( + input=x, + size=inputs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) for x in inputs + ] + inputs = torch.cat(upsampled_inputs, dim=1) + elif self.input_transform == 'multiple_select': + inputs = [inputs[i] for i in self.in_index] + else: + inputs = inputs[self.in_index] + + return inputs + + def forward(self, inputs): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, img_metas, gt_semantic_seg, train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs) + losses = self.losses(seg_logits, gt_semantic_seg) + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs) + + +class UPerHead(BaseDecodeHead): + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(UPerHead, self).__init__( + input_transform='multiple_select', **kwargs) + # FPN Module + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + for in_channels in self.in_channels: # skip the top layer + l_conv = ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=True) + fpn_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=True) + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + def forward(self, inputs): + """Forward function.""" + + inputs = self._transform_inputs(inputs) + + # build laterals + laterals = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # laterals.append(self.psp_forward(inputs)) + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += resize( + laterals[i], + size=prev_shape, + mode='bilinear', + align_corners=self.align_corners) + + # build outputs + fpn_outs = [ + self.fpn_convs[i](laterals[i]) + for i in range(used_backbone_levels - 1) + ] + # append psp feature + fpn_outs.append(laterals[-1]) + + return fpn_outs[0] + + +class PSP(BaseDecodeHead): + """Unified Perceptual Parsing for Scene Understanding. + + This head is the implementation of `UPerNet + `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module applied on the last feature. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(PSP, self).__init__(input_transform='multiple_select', **kwargs) + # PSP Module + self.psp_modules = PPM( + pool_scales, + self.in_channels[-1], + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels[-1] + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def psp_forward(self, inputs): + """Forward function of PSP module.""" + x = inputs[-1] + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + + return output + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + + return self.psp_forward(inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation/newcrfs_model.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation/newcrfs_model.py new file mode 100644 index 0000000000000000000000000000000000000000..4087cb67db137680977b24f7c6a947ca3b9ba0cf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation/newcrfs_model.py @@ -0,0 +1,53 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_depth_estimation.networks.newcrf_depth import \ + NewCRFDepth +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_depth_estimation, module_name=Models.newcrfs_depth_estimation) +class DepthEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + # build model + self.model = NewCRFDepth( + version='large07', inv_depth=False, max_depth=10) + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoint = torch.load(model_path) + + state_dict = {} + for k in checkpoint['model'].keys(): + if k.startswith('module.'): + state_dict[k[7:]] = checkpoint['model'][k] + else: + state_dict[k] = checkpoint['model'][k] + self.model.load_state_dict(state_dict) + self.model.eval() + + def forward(self, Inputs): + return self.model(Inputs['imgs']) + + def postprocess(self, Inputs): + depth_result = Inputs + + results = {OutputKeys.DEPTHS: depth_result} + return results + + def inference(self, data): + results = self.forward(data) + + return results diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/__init__.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..29b182618e1596ba49b65583e22d85fe54502599 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .depth_estimation_bts_model import DepthEstimationBtsModel + +else: + _import_structure = { + 'depth_estimation_bts_model': ['DepthEstimationBtsModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/depth_estimation_bts_model.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/depth_estimation_bts_model.py new file mode 100644 index 0000000000000000000000000000000000000000..08e04220fd29184231c4079088f80e7f3ce3621b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/depth_estimation_bts_model.py @@ -0,0 +1,69 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .networks.bts_model import BtsModel + +logger = get_logger() +__all__ = ['DepthEstimationBtsModel'] + + +@MODELS.register_module( + Tasks.image_depth_estimation, module_name=Models.bts_depth_estimation) +class DepthEstimationBtsModel(TorchModel): + """ Depth estimation model bts, implemented from paper https://arxiv.org/pdf/1907.10326.pdf. + The network utilizes novel local planar guidance layers located at multiple stage in the decoding phase. + The bts model is composed with encoder and decoder, an encoder for dense feature extraction and a decoder + for predicting the desired depth. + """ + + def __init__(self, model_dir: str, **kwargs): + """initialize the bts model from the `model_dir` path. + + Args: + model_dir (str): the model path. + focal: focal length, pictures that do not work are input according to + the camera setting value at the time of shooting + dataset: used to set focal value according dataset type, only support 'kitti' + """ + super().__init__(model_dir, **kwargs) + self.focal = 715.0873 # focal length, different dataset has different value + if 'focal' in kwargs: + self.focal = kwargs['focal'] + elif 'dataset' in kwargs: + if kwargs['dataset'] == 'nyu': + self.focal = 518.8579 + elif kwargs['dataset'] == 'kitti': + self.focal = 715.0873 + + self.model = BtsModel(focal=self.focal) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoint = torch.load(model_path) + + state_dict = {} + for k in checkpoint['model_state_dict'].keys(): + if k.startswith('module.'): + state_dict[k[7:]] = checkpoint['model_state_dict'][k] + else: + state_dict[k] = checkpoint['model_state_dict'][k] + self.model.load_state_dict(state_dict) + self.model.eval() + + def forward(self, inputs): + return self.model(inputs['imgs']) + + def postprocess(self, inputs): + results = {OutputKeys.DEPTHS: inputs} + return results + + def inference(self, data): + results = self.forward(data) + return results diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/__init__.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/bts_model.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/bts_model.py new file mode 100644 index 0000000000000000000000000000000000000000..776f3074407b03fffd48dc084fd7ec455a32fbf0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/bts_model.py @@ -0,0 +1,42 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from .decoder import Decoder +from .encoder import Encoder + + +class BtsModel(nn.Module): + """Depth estimation model bts, implemented from paper https://arxiv.org/pdf/1907.10326.pdf. + The network utilizes novel local planar guidance layers located at multiple stage in the decoding phase. + The bts model is composed with encoder and decoder, an encoder for dense feature extraction and a decoder + for predicting the desired depth. + """ + + def __init__(self, focal=715.0873): + """initial bts model + + Args: + focal (float): focal length, pictures that do not work are input according to + the camera setting value at the time of shooting + """ + super(BtsModel, self).__init__() + self.focal = focal + self.encoder = Encoder() + self.decoder = Decoder() + + def forward(self, x, focal=None): + """forward to estimation depth + + Args: + x (Tensor): input image data + focal (float): The focal length when the picture is taken. By default, the focal length + of the data set when the model is created is used + + Returns: + Tensor: Depth estimation image + """ + focal_run = focal if focal else self.focal + skip_feat = self.encoder(x) + depth = self.decoder(skip_feat, torch.tensor(focal_run).cuda()) + return depth diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/decoder.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..e9cf9fa7f8bd904e31e4f040dd8801da07d6684b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/decoder.py @@ -0,0 +1,72 @@ +# The implementation is modified from ErenBalatkan/Bts-PyTorch +# made publicly available under the MIT license +# https://github.com/ErenBalatkan/Bts-PyTorch/blob/master/BTS.py + +import torch +import torch.nn as nn + +from .utils import (MAX_DEPTH, ASSPBlock, LPGBlock, Reduction, UpscaleBlock, + UpscaleLayer, UpscaleNetwork, activation_fn) + + +class Decoder(nn.Module): + + def __init__(self, dataset='kitti'): + super(Decoder, self).__init__() + self.UpscaleNet = UpscaleNetwork() + self.DenseASSPNet = ASSPBlock() + + self.upscale_block3 = UpscaleBlock(64, 96, 128) # H4 + self.upscale_block4 = UpscaleBlock(128, 96, 128) # H2 + + self.LPGBlock8 = LPGBlock(8, 128) + self.LPGBlock4 = LPGBlock(4, 64) # 64 Filter + self.LPGBlock2 = LPGBlock(2, 64) # 64 Filter + + self.upconv_h4 = UpscaleLayer(128, 64) + self.upconv_h2 = UpscaleLayer(64, 32) # 64 Filter + self.upconv_h = UpscaleLayer(64, 32) # 32 filter + + self.conv_h4 = nn.Conv2d(161, 64, 3, 1, 1, bias=True) # 64 Filter + self.conv_h2 = nn.Conv2d(129, 64, 3, 1, 1, bias=True) # 64 Filter + self.conv_h1 = nn.Conv2d(36, 32, 3, 1, 1, bias=True) + + self.reduction1x1 = Reduction(1, 32, True) + + self.final_conv = nn.Conv2d(32, 1, 3, 1, 1, bias=True) + + self.dataset = dataset + + def forward(self, joint_input, focal): + (dense_features, dense_op_h2, dense_op_h4, dense_op_h8, + dense_op_h16) = joint_input + upscaled_out = self.UpscaleNet(joint_input) + + dense_assp_out = self.DenseASSPNet(upscaled_out) + + upconv_h4 = self.upconv_h4(dense_assp_out) + depth_8x8 = self.LPGBlock8(dense_assp_out) / MAX_DEPTH + depth_8x8_ds = nn.functional.interpolate( + depth_8x8, scale_factor=1 / 4, mode='nearest') + depth_concat_4x4 = torch.cat((depth_8x8_ds, dense_op_h4, upconv_h4), 1) + + conv_h4 = activation_fn(self.conv_h4(depth_concat_4x4)) + upconv_h2 = self.upconv_h2(conv_h4) + depth_4x4 = self.LPGBlock4(conv_h4) / MAX_DEPTH + + depth_4x4_ds = nn.functional.interpolate( + depth_4x4, scale_factor=1 / 2, mode='nearest') + depth_concat_2x2 = torch.cat((depth_4x4_ds, dense_op_h2, upconv_h2), 1) + + conv_h2 = activation_fn(self.conv_h2(depth_concat_2x2)) + upconv_h = self.upconv_h(conv_h2) + depth_1x1 = self.reduction1x1(upconv_h) + depth_2x2 = self.LPGBlock2(conv_h2) / MAX_DEPTH + + depth_concat = torch.cat( + (upconv_h, depth_1x1, depth_2x2, depth_4x4, depth_8x8), 1) + depth = activation_fn(self.conv_h1(depth_concat)) + depth = self.final_conv(depth).sigmoid() * MAX_DEPTH + 0.1 + if self.dataset == 'kitti': + depth *= focal.view(-1, 1, 1, 1) / 715.0873 + return depth diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/encoder.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..784064c2be4e9ca8247e2334b64ea5581249e837 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/encoder.py @@ -0,0 +1,70 @@ +# The implementation is modified from ErenBalatkan/Bts-PyTorch +# made publicly available under the MIT license +# https://github.com/ErenBalatkan/Bts-PyTorch/blob/master/BTS.py + +import torch.nn as nn +import torchvision.models as models + + +class Encoder(nn.Module): + + def __init__(self, pretrained=False): + super(Encoder, self).__init__() + self.dense_op_h2 = None + self.dense_op_h4 = None + self.dense_op_h8 = None + self.dense_op_h16 = None + self.dense_features = None + + self.dense_feature_extractor = self.initial_feature_extractor( + pretrained) + self.freeze_batch_norm() + self.initialize_hooks() + + def freeze_batch_norm(self): + for module in self.dense_feature_extractor.modules(): + if isinstance(module, nn.modules.BatchNorm2d): + module.track_running_stats = True + module.eval() + module.affine = True + module.requires_grad = True + + def initial_feature_extractor(self, pretrained=False): + dfe = models.densenet161(pretrained=pretrained) + dfe.features.denseblock1.requires_grad = False + dfe.features.denseblock2.requires_grad = False + dfe.features.conv0.requires_grad = False + return dfe + + def set_h2(self, module, input_, output): + self.dense_op_h2 = output + + def set_h4(self, module, input_, output): + self.dense_op_h4 = output + + def set_h8(self, module, input_, output): + self.dense_op_h8 = output + + def set_h16(self, module, input_, output): + self.dense_op_h16 = output + + def set_dense_features(self, module, input_, output): + self.dense_features = output + + def initialize_hooks(self): + self.dense_feature_extractor.features.relu0.register_forward_hook( + self.set_h2) + self.dense_feature_extractor.features.pool0.register_forward_hook( + self.set_h4) + self.dense_feature_extractor.features.transition1.register_forward_hook( + self.set_h8) + self.dense_feature_extractor.features.transition2.register_forward_hook( + self.set_h16) + self.dense_feature_extractor.features.norm5.register_forward_hook( + self.set_dense_features) + + def forward(self, x): + _ = self.dense_feature_extractor(x) + joint_input = (self.dense_features.relu(), self.dense_op_h2, + self.dense_op_h4, self.dense_op_h8, self.dense_op_h16) + return joint_input diff --git a/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/utils.py b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6566a511fdeda06eedfe5d5fe5f9503f376a84a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_depth_estimation_bts/networks/utils.py @@ -0,0 +1,246 @@ +# The implementation is modified from ErenBalatkan/Bts-PyTorch +# made publicly available under the MIT license +# https://github.com/ErenBalatkan/Bts-PyTorch/blob/master/BTS.py + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +activation_fn = nn.ELU() +MAX_DEPTH = 81 + + +class UpscaleLayer(nn.Module): + + def __init__(self, in_channels, out_channels): + super(UpscaleLayer, self).__init__() + self.conv = nn.Conv2d( + in_channels, out_channels, 3, padding=1, bias=True) + self.bn = nn.BatchNorm2d(out_channels, momentum=0.005) + + def forward(self, input): + input = nn.functional.interpolate( + input, scale_factor=2, mode='nearest') + input = activation_fn(self.conv(input)) + input = self.bn(input) + return input + + +class UpscaleBlock(nn.Module): + + def __init__(self, in_channels, skip_channels, out_channels): + super(UpscaleBlock, self).__init__() + self.uplayer = UpscaleLayer(in_channels, out_channels) + self.conv = nn.Conv2d( + out_channels + skip_channels, + out_channels, + 3, + padding=1, + bias=True) + self.bn2 = nn.BatchNorm2d(out_channels, 0.005) + + def forward(self, input_j): + input, skip = input_j + input = self.uplayer(input) + cat = torch.cat((input, skip), 1) + input = activation_fn(self.conv(cat)) + input = self.bn2(input) + return input, cat + + +class UpscaleNetwork(nn.Module): + + def __init__(self, filters=[512, 256]): + super( + UpscaleNetwork, + self, + ).__init__() + self.upscale_block1 = UpscaleBlock(2208, 384, filters[0]) # H16 + self.upscale_block2 = UpscaleBlock(filters[0], 192, filters[1]) # H8 + + def forward(self, raw_input): + input, h2, h4, h8, h16 = raw_input + input, _ = self.upscale_block1((input, h16)) + input, cat = self.upscale_block2((input, h8)) + return input, cat + + +class AtrousBlock(nn.Module): + + def __init__(self, + input_filters, + filters, + dilation, + apply_initial_bn=True): + super(AtrousBlock, self).__init__() + + self.initial_bn = nn.BatchNorm2d(input_filters, 0.005) + self.apply_initial_bn = apply_initial_bn + + self.conv1 = nn.Conv2d(input_filters, filters * 2, 1, 1, 0, bias=False) + self.norm1 = nn.BatchNorm2d(filters * 2, 0.005) + + self.atrous_conv = nn.Conv2d( + filters * 2, filters, 3, 1, dilation, dilation, bias=False) + self.norm2 = nn.BatchNorm2d(filters, 0.005) + + def forward(self, input): + if self.apply_initial_bn: + input = self.initial_bn(input) + + input = self.conv1(input.relu()) + input = self.norm1(input) + input = self.atrous_conv(input.relu()) + input = self.norm2(input) + return input + + +class ASSPBlock(nn.Module): + + def __init__(self, input_filters=256, cat_filters=448, atrous_filters=128): + super(ASSPBlock, self).__init__() + + self.atrous_conv_r3 = AtrousBlock( + input_filters, atrous_filters, 3, apply_initial_bn=False) + self.atrous_conv_r6 = AtrousBlock(cat_filters + atrous_filters, + atrous_filters, 6) + self.atrous_conv_r12 = AtrousBlock(cat_filters + atrous_filters * 2, + atrous_filters, 12) + self.atrous_conv_r18 = AtrousBlock(cat_filters + atrous_filters * 3, + atrous_filters, 18) + self.atrous_conv_r24 = AtrousBlock(cat_filters + atrous_filters * 4, + atrous_filters, 24) + + self.conv = nn.Conv2d( + 5 * atrous_filters + cat_filters, + atrous_filters, + 3, + 1, + 1, + bias=True) + + def forward(self, input): + input, cat = input + layer1_out = self.atrous_conv_r3(input) + concat1 = torch.cat((cat, layer1_out), 1) + + layer2_out = self.atrous_conv_r6(concat1) + concat2 = torch.cat((concat1, layer2_out), 1) + + layer3_out = self.atrous_conv_r12(concat2) + concat3 = torch.cat((concat2, layer3_out), 1) + + layer4_out = self.atrous_conv_r18(concat3) + concat4 = torch.cat((concat3, layer4_out), 1) + + layer5_out = self.atrous_conv_r24(concat4) + concat5 = torch.cat((concat4, layer5_out), 1) + + features = activation_fn(self.conv(concat5)) + return features + + +class Reduction(nn.Module): + + def __init__(self, scale, input_filters, is_final=False): + super(Reduction, self).__init__() + reduction_count = int(math.log(input_filters, 2)) - 2 + self.reductions = torch.nn.Sequential() + for i in range(reduction_count): + if i != reduction_count - 1: + self.reductions.add_module( + '1x1_reduc_%d_%d' % (scale, i), + nn.Sequential( + nn.Conv2d( + int(input_filters / math.pow(2, i)), + int(input_filters / math.pow(2, i + 1)), + 1, + 1, + 0, + bias=True), activation_fn)) + else: + if not is_final: + self.reductions.add_module( + '1x1_reduc_%d_%d' % (scale, i), + nn.Sequential( + nn.Conv2d( + int(input_filters / math.pow(2, i)), + int(input_filters / math.pow(2, i + 1)), + 1, + 1, + 0, + bias=True))) + else: + self.reductions.add_module( + '1x1_reduc_%d_%d' % (scale, i), + nn.Sequential( + nn.Conv2d( + int(input_filters / math.pow(2, i)), + 1, + 1, + 1, + 0, + bias=True), nn.Sigmoid())) + + def forward(self, ip): + return self.reductions(ip) + + +class LPGBlock(nn.Module): + + def __init__(self, scale, input_filters=128): + super(LPGBlock, self).__init__() + self.scale = scale + + self.reduction = Reduction(scale, input_filters) + self.conv = nn.Conv2d(4, 3, 1, 1, 0) + + self.u = torch.arange(self.scale).reshape([1, 1, self.scale]).float() + self.v = torch.arange(int(self.scale)).reshape([1, self.scale, + 1]).float() + + def forward(self, input): + input = self.reduction(input) + + plane_parameters = torch.zeros_like(input) + input = self.conv(input) + + theta = input[:, 0, :, :].sigmoid() * 3.1415926535 / 6 + phi = input[:, 1, :, :].sigmoid() * 3.1415926535 * 2 + dist = input[:, 2, :, :].sigmoid() * MAX_DEPTH + + plane_parameters[:, 0, :, :] = torch.sin(theta) * torch.cos(phi) + plane_parameters[:, 1, :, :] = torch.sin(theta) * torch.sin(phi) + plane_parameters[:, 2, :, :] = torch.cos(theta) + plane_parameters[:, 3, :, :] = dist + + plane_parameters[:, 0:3, :, :] = F.normalize( + plane_parameters.clone()[:, 0:3, :, :], 2, 1) + + plane_eq = plane_parameters.float() + + plane_eq_expanded = torch.repeat_interleave(plane_eq, int(self.scale), + 2) + plane_eq_expanded = torch.repeat_interleave(plane_eq_expanded, + int(self.scale), 3) + + n1 = plane_eq_expanded[:, 0, :, :] + n2 = plane_eq_expanded[:, 1, :, :] + n3 = plane_eq_expanded[:, 2, :, :] + n4 = plane_eq_expanded[:, 3, :, :] + + u = self.u.repeat( + plane_eq.size(0), + plane_eq.size(2) * int(self.scale), plane_eq.size(3)).cuda() + u = (u - (self.scale - 1) * 0.5) / self.scale + + v = self.v.repeat( + plane_eq.size(0), plane_eq.size(2), + plane_eq.size(3) * int(self.scale)).cuda() + v = (v - (self.scale - 1) * 0.5) / self.scale + + depth = n4 / (n1 * u + n2 * v + n3) + depth = depth.unsqueeze(1) + return depth diff --git a/runtime/Lib/site_packages/models/cv/image_driving_perception/__init__.py b/runtime/Lib/site_packages/models/cv/image_driving_perception/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f7aef488409f29831d62cd79913cddf768aad556 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_driving_perception/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_driving_percetion_model import YOLOPv2 + from .preprocessor import ImageDrivingPerceptionPreprocessor + from .utils import (scale_coords, non_max_suppression, + split_for_trace_model, driving_area_mask, + lane_line_mask) + +else: + _import_structure = { + 'image_driving_percetion_model': ['YOLOPv2'], + 'preprocessor': ['ImageDrivingPerceptionPreprocessor'], + 'utils': [ + 'scale_coords', 'non_max_suppression', 'split_for_trace_model', + 'driving_area_mask', 'lane_line_mask' + ], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_driving_perception/image_driving_percetion_model.py b/runtime/Lib/site_packages/models/cv/image_driving_perception/image_driving_percetion_model.py new file mode 100644 index 0000000000000000000000000000000000000000..e29ad2b9e1f558810597b6ba0ce8d12a048247ae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_driving_perception/image_driving_percetion_model.py @@ -0,0 +1,61 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['YOLOPv2'] + + +@MODELS.register_module( + Tasks.image_driving_perception, module_name=Models.yolopv2) +class YOLOPv2(TorchModel): + """ YOLOPv2 use E-ELAN which first adopted in Yolov7 as backbone, SPP+FPN+PAN as neck and head. + For more infomation, please refer to https://arxiv.org/pdf/2208.11434.pdf + """ + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + + self.model_dir = model_dir + self._load_pretrained_checkpoint() + + def forward(self, data): + img = data['img'] + with torch.no_grad(): + [pred, anchor_grid], seg, ll = self.model(img) + return { + 'img_hw': data['img'].shape[2:], + 'ori_img_shape': data['ori_img_shape'], + 'pred': pred, + 'anchor_grid': anchor_grid, + 'driving_area_mask': seg, + 'lane_line_mask': ll, + } + + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: + return super().postprocess(inputs, **kwargs) + + def _load_pretrained_checkpoint(self): + model_path = os.path.join(self.model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(model_path) + if os.path.exists(model_path): + self.model = torch.jit.load(model_path, 'cpu') + self.model = self.model.eval() + + else: + logger.error( + '[checkModelPath]:model path dose not exits!!! model Path:' + + model_path) + raise Exception('[checkModelPath]:model path dose not exits!') diff --git a/runtime/Lib/site_packages/models/cv/image_driving_perception/preprocessor.py b/runtime/Lib/site_packages/models/cv/image_driving_perception/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..3e0e476fd8a1e5ebcda5f4c5caae04cee7c55f17 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_driving_perception/preprocessor.py @@ -0,0 +1,121 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict, Union + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.preprocessors.image import LoadImage +from modelscope.utils.constant import Fields, ModeKeys +from modelscope.utils.type_assert import type_assert + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.image_driving_perception_preprocessor) +class ImageDrivingPerceptionPreprocessor(Preprocessor): + + def __init__(self, mode: str = ModeKeys.INFERENCE, *args, **kwargs): + """ + Args: + model_dir (str): model directory to initialize some resource. + mode: The mode for the preprocessor. + """ + super().__init__(mode, *args, **kwargs) + + def _check_image(self, input_img): + whole_temp_shape = input_img.shape + if len(whole_temp_shape) == 2: + input_img = np.stack([input_img, input_img, input_img], axis=2) + elif whole_temp_shape[2] == 1: + input_img = np.concatenate([input_img, input_img, input_img], + axis=2) + elif whole_temp_shape[2] == 4: + input_img = input_img[:, :, + 0:3] * 1.0 * input_img[:, :, + 3:4] * 1.0 / 255.0 + return input_img + + def _letterbox(self, + img, + new_shape=(640, 640), + color=(114, 114, 114), + auto=True, + scaleFill=False, + scaleup=True, + stride=32): + # Resize and pad image while meeting stride-multiple constraints + shape = img.shape[:2] # current shape [height, width] + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + + if not scaleup: # only scale down, do not scale up (for better test mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[ + 1] # wh padding + if auto: # minimum rectangle + dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding + elif scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[ + 0] # width, height ratios + + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) + + top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) + + img = cv2.copyMakeBorder( + img, top, bottom, left, right, cv2.BORDER_CONSTANT, + value=color) # add border + + return img, ratio, (dw, dh) + + @type_assert(object, object) + def __call__( + self, data: str, output_shape=(1280, 720), new_shape=(640, 640) + ) -> Dict[str, Any]: + """process the raw input data + Args: + data (str): image path + Returns: + Dict[ndarry, Any]: the preprocessed data + { + "img": the preprocessed resized image (640x640) + } + """ + img = LoadImage.convert_to_ndarray(data) + if img is not None: + img = self._check_image(img) + else: + raise Exception('img is None') + ori_h, ori_w, _ = img.shape + img = self._letterbox(img, new_shape)[0] + img = img.transpose(2, 0, 1) # to 3x640x640 + + img = np.ascontiguousarray(img) + img = torch.from_numpy(img) + img = img.float() # uint8 to fp16/32 + # Convert + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + + if img.ndimension() == 3: + img = img.unsqueeze(0) + + return { + 'img': img, + 'ori_img_shape': (ori_h, ori_w), + } diff --git a/runtime/Lib/site_packages/models/cv/image_driving_perception/utils.py b/runtime/Lib/site_packages/models/cv/image_driving_perception/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15b228a23c4749cc1e26b321c197cfd03c00da75 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_driving_perception/utils.py @@ -0,0 +1,203 @@ +# Part of the implementation is borrowed and modified from internet, +# publicly available at https://github.com/CAIC-AD/YOLOPv2 +import time + +import numpy as np +import torch +from torchvision.ops import nms + + +def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid( + [torch.arange(ny), torch.arange(nx)], indexing='ij') + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +def split_for_trace_model(pred=None, anchor_grid=None): + z = [] + st = [8, 16, 32] + for i in range(3): + bs, _, ny, nx = pred[i].shape + pred[i] = pred[i].view(bs, 3, 85, ny, nx).permute(0, 1, 3, 4, + 2).contiguous() + y = pred[i].sigmoid() + gr = _make_grid(nx, ny).to(pred[i].device) + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + gr) * st[i] # xy + y[..., 2:4] = (y[..., 2:4] * 2)**2 * anchor_grid[i] # wh + z.append(y.view(bs, -1, 85)) + pred = torch.cat(z, 1) + return pred + + +def scale_coords(img1_shape, coords, img0_shape=(720, 1280), ratio_pad=None): + # Rescale coords (xyxy) from img1_shape to img0_shape + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], + img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, ( + img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + coords[:, [0, 2]] -= pad[0] # x padding + coords[:, [1, 3]] -= pad[1] # y padding + coords[:, :4] /= gain + clip_coords(coords, img0_shape) + return coords + + +def clip_coords(boxes, img_shape): + # Clip bounding xyxy bounding boxes to image shape (height, width) + boxes[:, 0].clamp_(0, img_shape[1]) # x1 + boxes[:, 1].clamp_(0, img_shape[0]) # y1 + boxes[:, 2].clamp_(0, img_shape[1]) # x2 + boxes[:, 3].clamp_(0, img_shape[0]) # y2 + + +def xywh2xyxy(x): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x + y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y + y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x + y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y + return y + + +def non_max_suppression(prediction, + conf_thres=0.3, + iou_thres=0.45, + classes=None, + agnostic=False, + multi_label=False, + labels=()): + """Runs Non-Maximum Suppression (NMS) on inference results + + Returns: + list of detections, on (n,6) tensor per image [xyxy, conf, cls] + """ + + nc = prediction.shape[2] - 5 # number of classes + xc = prediction[..., 4] > conf_thres # candidates + + # Settings + max_wh = 4096 # (pixels) minimum and maximum box width and height + max_det = 300 # maximum number of detections per image + max_nms = 30000 # maximum number of boxes into torchvision.ops.nms() + time_limit = 10.0 # seconds to quit after + redundant = True # require redundant detections + multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img) + merge = False # use merge-NMS + + t = time.time() + output = [torch.zeros( + (0, 6), device=prediction.device)] * prediction.shape[0] + for xi, x in enumerate(prediction): # image index, image inference + # Apply constraints + x = x[xc[xi]] # confidence + + # Cat apriori labels if autolabelling + if labels and len(labels[xi]): + lbs = labels[xi] + v = torch.zeros((len(lbs), nc + 5), device=x.device) + v[:, :4] = lbs[:, 1:5] # box + v[:, 4] = 1.0 # conf + v[range(len(lbs)), lbs[:, 0].long() + 5] = 1.0 # cls + x = torch.cat((x, v), 0) + + # If none remain process next image + if not x.shape[0]: + continue + + # Compute conf + x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf + + # Box (center x, center y, width, height) to (x1, y1, x2, y2) + box = xywh2xyxy(x[:, :4]) + + # Detections matrix nx6 (xyxy, conf, cls) + if multi_label: + i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T + x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1) + else: # best class only + conf, j = x[:, 5:].max(1, keepdim=True) + x = torch.cat((box, conf, j.float()), + 1)[conf.view(-1) > conf_thres] + + # Filter by class + if classes is not None: + x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] + + # Check shape + n = x.shape[0] # number of boxes + if not n: # no boxes + continue + elif n > max_nms: # excess boxes + x = x[x[:, 4].argsort( + descending=True)[:max_nms]] # sort by confidence + + # Batched NMS + c = x[:, 5:6] * (0 if agnostic else max_wh) # classes + boxes, scores = x[:, :4] + c, x[:, + 4] # boxes (offset by class), scores + i = nms(boxes, scores, iou_thres) # NMS + if i.shape[0] > max_det: # limit detections + i = i[:max_det] + if merge and (1 < n and n < 3E3): + # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) + iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix + weights = iou * scores[None] # box weights + x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum( + 1, keepdim=True) # merged boxes + if redundant: + i = i[iou.sum(1) > 1] # require redundancy + + output[xi] = x[i] + if (time.time() - t) > time_limit: + print(f'WARNING: NMS time limit {time_limit}s exceeded') + break # time limit exceeded + + return output + + +def box_iou(box1, box2): + # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Args: + box1 (Tensor[N, 4]) + box2 (Tensor[M, 4]) + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) + - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + return inter / (area1[:, None] + area2 - inter + ) # iou = inter / (area1 + area2 - inter) + + +def driving_area_mask(da_predict=None, out_shape=(720, 1280)): + da_seg_mask = torch.nn.functional.interpolate( + da_predict, size=out_shape, mode='bilinear') + _, da_seg_mask = torch.max(da_seg_mask, 1) + da_seg_mask = da_seg_mask.int().squeeze().cpu().numpy() + return da_seg_mask + + +def lane_line_mask(ll_predict=None, out_shape=(720, 1280)): + ll_seg_mask = torch.nn.functional.interpolate( + ll_predict, size=out_shape, mode='bilinear') + ll_seg_mask = torch.round(ll_seg_mask).squeeze(1) + ll_seg_mask = ll_seg_mask.int().squeeze().cpu().numpy() + return ll_seg_mask diff --git a/runtime/Lib/site_packages/models/cv/image_editing/__init__.py b/runtime/Lib/site_packages/models/cv/image_editing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35341a189eb15848ffd41e1de2136b9888223db7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_editing/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .masactrl import MutualSelfAttentionControl + from .masactrl_utils import regiter_attention_editor_diffusers +else: + _import_structure = { + 'masactrl': ['MutualSelfAttentionControl'], + 'masactrl_utils': ['regiter_attention_editor_diffusers'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_editing/masactrl.py b/runtime/Lib/site_packages/models/cv/image_editing/masactrl.py new file mode 100644 index 0000000000000000000000000000000000000000..e1e505d1b6a76db719d95f1f487af97e25bc3ce9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_editing/masactrl.py @@ -0,0 +1,77 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/TencentARC/MasaCtrl/blob/main/masactrl/masactrl.py +# Copyright (c) 2023 TencentARC. All Rights Reserved. +# ------------------------------------------------------------------------ + +import torch +from einops import rearrange + +from .masactrl_utils import AttentionBase + + +class MutualSelfAttentionControl(AttentionBase): + + def __init__(self, + start_step=4, + start_layer=10, + layer_idx=None, + step_idx=None, + total_steps=50): + """ + Mutual self-attention control for Stable-Diffusion model + Args: + start_step: the step to start mutual self-attention control + start_layer: the layer to start mutual self-attention control + layer_idx: list of the layers to apply mutual self-attention control + step_idx: list the steps to apply mutual self-attention control + total_steps: the total number of steps + """ + super().__init__() + self.total_steps = total_steps + self.start_step = start_step + self.start_layer = start_layer + self.layer_idx = layer_idx if layer_idx is not None else list( + range(start_layer, 16)) + self.step_idx = step_idx if step_idx is not None else list( + range(start_step, total_steps)) # denoise index + print('step_idx: ', self.step_idx) + print('layer_idx: ', self.layer_idx) + + def attn_batch(self, q, k, v, sim, attn, is_cross, place_in_unet, + num_heads, **kwargs): + b = q.shape[0] // num_heads + q = rearrange(q, '(b h) n d -> h (b n) d', h=num_heads) + k = rearrange(k, '(b h) n d -> h (b n) d', h=num_heads) + v = rearrange(v, '(b h) n d -> h (b n) d', h=num_heads) + + sim = torch.einsum('h i d, h j d -> h i j', q, k) * kwargs.get('scale') + attn = sim.softmax(-1) + out = torch.einsum('h i j, h j d -> h i d', attn, v) + out = rearrange(out, 'h (b n) d -> b n (h d)', b=b) + return out + + def forward(self, q, k, v, sim, attn, is_cross, place_in_unet, num_heads, + **kwargs): + """ + Attention forward function + """ + if is_cross or self.cur_step not in self.step_idx or self.cur_att_layer // 2 not in self.layer_idx: + return super().forward(q, k, v, sim, attn, is_cross, place_in_unet, + num_heads, **kwargs) + + qu, qc = q.chunk(2) # uncond, cond + ku, kc = k.chunk(2) + vu, vc = v.chunk(2) + attnu, attnc = attn.chunk(2) + # uncond + # ku[:num_heads], vu[:num_heads] -> source + # qu -> [source, target] + out_u = self.attn_batch(qu, ku[:num_heads], vu[:num_heads], + sim[:num_heads], attnu, is_cross, + place_in_unet, num_heads, **kwargs) + out_c = self.attn_batch(qc, kc[:num_heads], vc[:num_heads], + sim[:num_heads], attnc, is_cross, + place_in_unet, num_heads, **kwargs) + out = torch.cat([out_u, out_c], dim=0) + + return out diff --git a/runtime/Lib/site_packages/models/cv/image_editing/masactrl_utils.py b/runtime/Lib/site_packages/models/cv/image_editing/masactrl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a59e987f6a354ee131f4dd4f1035337edae990dd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_editing/masactrl_utils.py @@ -0,0 +1,124 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/TencentARC/MasaCtrl/blob/main/masactrl/masactrl_utils.py +# Copyright (c) 2023 TencentARC. All Rights Reserved. +# ------------------------------------------------------------------------ + +import torch +import torch.nn as nn +from einops import rearrange, repeat + + +class AttentionBase: + + def __init__(self): + self.cur_step = 0 + self.num_att_layers = -1 + self.cur_att_layer = 0 + + def after_step(self): + pass + + def __call__(self, q, k, v, sim, attn, is_cross, place_in_unet, num_heads, + **kwargs): + out = self.forward(q, k, v, sim, attn, is_cross, place_in_unet, + num_heads, **kwargs) + self.cur_att_layer += 1 + if self.cur_att_layer == self.num_att_layers: + self.cur_att_layer = 0 + self.cur_step += 1 + # after step + self.after_step() + return out + + def forward(self, q, k, v, sim, attn, is_cross, place_in_unet, num_heads, + **kwargs): + out = torch.einsum('b i j, b j d -> b i d', attn, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=num_heads) + return out + + def reset(self): + self.cur_step = 0 + self.cur_att_layer = 0 + + +def regiter_attention_editor_diffusers(model, editor: AttentionBase): + """ + Register a attention editor to Diffuser Pipeline, refer from [Prompt-to-Prompt] + """ + + def ca_forward(self, place_in_unet): + + def forward(x, + encoder_hidden_states=None, + attention_mask=None, + context=None, + mask=None): + """ + The attention is similar to the original implementation of LDM CrossAttention class + except adding some modifications on the attention + """ + if encoder_hidden_states is not None: + context = encoder_hidden_states + if attention_mask is not None: + mask = attention_mask + + to_out = self.to_out + if isinstance(to_out, nn.modules.container.ModuleList): + to_out = self.to_out[0] + else: + to_out = self.to_out + + h = self.heads + q = self.to_q(x) + is_cross = context is not None + context = context if is_cross else x + k = self.to_k(context) + v = self.to_v(context) + q, k, v = map( + lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + + if mask is not None: + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + mask = mask[:, None, :].repeat(h, 1, 1) + sim.masked_fill_(~mask, max_neg_value) + + attn = sim.softmax(dim=-1) + # the only difference + out = editor( + q, + k, + v, + sim, + attn, + is_cross, + place_in_unet, + self.heads, + scale=self.scale) + + return to_out(out) + + return forward + + def register_editor(net, count, place_in_unet): + for name, subnet in net.named_children(): + if net.__class__.__name__ == 'Attention': # spatial Transformer layer + net.forward = ca_forward(net, place_in_unet) + return count + 1 + elif hasattr(net, 'children'): + count = register_editor(subnet, count, place_in_unet) + return count + + cross_att_count = 0 + for net_name, net in model.unet.named_children(): + if 'down' in net_name: + cross_att_count += register_editor(net, 0, 'down') + elif 'mid' in net_name: + cross_att_count += register_editor(net, 0, 'mid') + elif 'up' in net_name: + cross_att_count += register_editor(net, 0, 'up') + editor.num_att_layers = cross_att_count diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/__init__.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c77e2314b8c3d5564c4916370065d3ff835244e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_face_fusion import ImageFaceFusion + +else: + _import_structure = {'image_face_fusion': ['ImageFaceFusion']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/__init__.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/face_gan.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/face_gan.py new file mode 100644 index 0000000000000000000000000000000000000000..80d62997e5aee900151c200d1c56e87df002c9b0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/face_gan.py @@ -0,0 +1,55 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import numpy as np +import torch +import torch.nn.functional as F + +from .gpen_model import FullGenerator + + +class GPEN(object): + + def __init__(self, + model_path, + size=512, + channel_multiplier=2, + device=torch.device('cpu')): + self.mfile = model_path + self.n_mlp = 8 + self.resolution = size + self.device = device + self.load_model(channel_multiplier) + + def load_model(self, channel_multiplier=2): + self.model = FullGenerator(self.resolution, 512, self.n_mlp, + channel_multiplier).to(self.device) + pretrained_dict = torch.load(self.mfile) + self.model.load_state_dict(pretrained_dict) + self.model.eval() + + def process(self, im): + preds = [] + imt = self.img2tensor(im) + imt = F.interpolate(imt, (self.resolution, self.resolution)) + + with torch.no_grad(): + img_out, __ = self.model(imt) + + face = self.tensor2img(img_out) + + return face, preds + + def img2tensor(self, img): + img_t = torch.from_numpy(img).to(self.device) + img_t = (img_t / 255. - 0.5) / 0.5 + img_t = img_t.permute(2, 0, 1).unsqueeze(0).flip(1) # BGR->RGB + return img_t + + def tensor2img(self, image_tensor, pmax=255.0, imtype=np.uint8): + image_tensor = image_tensor * 0.5 + 0.5 + image_tensor = image_tensor.squeeze(0).permute(1, 2, + 0).flip(2) # RGB->BGR + image_numpy = np.clip(image_tensor.float().cpu().numpy(), 0, 1) * pmax + + return image_numpy.astype(imtype) diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/gpen_model.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/gpen_model.py new file mode 100644 index 0000000000000000000000000000000000000000..22dfb1e151bd33f17eec8567774d404e36c91aaa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/gpen_model.py @@ -0,0 +1,891 @@ +# The implementation is adopted from InsightFace_Pytorch, made publicly available under the MIT License +# at https://github.com/yangxy/GPEN +import functools +import itertools +import math +import operator +import random + +import torch +from torch import nn +from torch.autograd import Function +from torch.nn import functional as F + +from .op import FusedLeakyReLU, fused_leaky_relu, upfirdn2d + +isconcat = True + + +class PixelNorm(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, input): + return input * torch.rsqrt( + torch.mean(input**2, dim=1, keepdim=True) + 1e-8) + + +def make_kernel(k): + k = torch.tensor(k, dtype=torch.float32) + + if k.ndim == 1: + k = k[None, :] * k[:, None] + + k /= k.sum() + + return k + + +class Upsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) * (factor**2) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=self.factor, down=1, pad=self.pad) + + return out + + +class Downsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=1, down=self.factor, pad=self.pad) + + return out + + +class Blur(nn.Module): + + def __init__(self, kernel, pad, upsample_factor=1): + super().__init__() + + kernel = make_kernel(kernel) + + if upsample_factor > 1: + kernel = kernel * (upsample_factor**2) + + self.register_buffer('kernel', kernel) + + self.pad = pad + + def forward(self, input): + out = upfirdn2d(input, self.kernel, pad=self.pad) + + return out + + +class EqualConv2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + bias=True): + super().__init__() + + self.weight = nn.Parameter( + torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size**2) + + self.stride = stride + self.padding = padding + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + + else: + self.bias = None + + def forward(self, input): + out = F.conv2d( + input, + self.weight * self.scale, + bias=self.bias, + stride=self.stride, + padding=self.padding, + ) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class EqualLinear(nn.Module): + + def __init__(self, + in_dim, + out_dim, + bias=True, + bias_init=0, + lr_mul=1, + activation=None): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) + + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) + + else: + self.bias = None + + self.activation = activation + + self.scale = (1 / math.sqrt(in_dim)) * lr_mul + self.lr_mul = lr_mul + + def forward(self, input): + if self.activation: + out = F.linear(input, self.weight * self.scale) + out = fused_leaky_relu(out, self.bias * self.lr_mul) + + else: + out = F.linear( + input, self.weight * self.scale, bias=self.bias * self.lr_mul) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})' + ) + + +class ScaledLeakyReLU(nn.Module): + + def __init__(self, negative_slope=0.2): + super().__init__() + + self.negative_slope = negative_slope + + def forward(self, input): + out = F.leaky_relu(input, negative_slope=self.negative_slope) + + return out * math.sqrt(2) + + +class ModulatedConv2d(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + demodulate=True, + upsample=False, + downsample=False, + blur_kernel=[1, 3, 3, 1], + ): + super().__init__() + + self.eps = 1e-8 + self.kernel_size = kernel_size + self.in_channel = in_channel + self.out_channel = out_channel + self.upsample = upsample + self.downsample = downsample + + if upsample: + factor = 2 + p = (len(blur_kernel) - factor) - (kernel_size - 1) + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + 1 + + self.blur = Blur( + blur_kernel, pad=(pad0, pad1), upsample_factor=factor) + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.blur = Blur(blur_kernel, pad=(pad0, pad1)) + + fan_in = in_channel * kernel_size**2 + self.scale = 1 / math.sqrt(fan_in) + self.padding = kernel_size // 2 + + self.weight = nn.Parameter( + torch.randn(1, out_channel, in_channel, kernel_size, kernel_size)) + + self.modulation = EqualLinear(style_dim, in_channel, bias_init=1) + + self.demodulate = demodulate + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, ' + f'upsample={self.upsample}, downsample={self.downsample})') + + def forward(self, input, style): + batch, in_channel, height, width = input.shape + + style = self.modulation(style).view(batch, 1, in_channel, 1, 1) + weight = self.scale * self.weight * style + + if self.demodulate: + demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8) + weight = weight * demod.view(batch, self.out_channel, 1, 1, 1) + + weight = weight.view(batch * self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + + if self.upsample: + input = input.view(1, batch * in_channel, height, width) + weight = weight.view(batch, self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + weight = weight.transpose(1, 2).reshape(batch * in_channel, + self.out_channel, + self.kernel_size, + self.kernel_size) + out = F.conv_transpose2d( + input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + out = self.blur(out) + + elif self.downsample: + input = self.blur(input) + _, _, height, width = input.shape + input = input.view(1, batch * in_channel, height, width) + out = F.conv2d(input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + else: + input = input.view(1, batch * in_channel, height, width) + out = F.conv2d(input, weight, padding=self.padding, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + return out + + +class NoiseInjection(nn.Module): + + def __init__(self): + super().__init__() + + self.weight = nn.Parameter(torch.zeros(1)) + + def forward(self, image, noise=None): + + if noise is not None: + # print(image.shape, noise.shape) + if isconcat: + return torch.cat((image, self.weight * noise), dim=1) # concat + return image + self.weight * noise + + if noise is None: + batch, _, height, width = image.shape + noise = image.new_empty(batch, 1, height, width).normal_() + + return image + self.weight * noise + + +class ConstantInput(nn.Module): + + def __init__(self, channel, size=4): + super().__init__() + + self.input = nn.Parameter(torch.randn(1, channel, size, size)) + + def forward(self, input): + batch = input.shape[0] + out = self.input.repeat(batch, 1, 1, 1) + + return out + + +class StyledConv(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=False, + blur_kernel=[1, 3, 3, 1], + demodulate=True, + ): + super().__init__() + + self.conv = ModulatedConv2d( + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=upsample, + blur_kernel=blur_kernel, + demodulate=demodulate, + ) + + self.noise = NoiseInjection() + feat_multiplier = 2 + self.activate = FusedLeakyReLU(out_channel * feat_multiplier) + + def forward(self, input, style, noise=None): + out = self.conv(input, style) + out = self.noise(out, noise=noise) + # out = out + self.bias + out = self.activate(out) + + return out + + +class ToRGB(nn.Module): + + def __init__(self, + in_channel, + style_dim, + upsample=True, + blur_kernel=[1, 3, 3, 1]): + super().__init__() + + if upsample: + self.upsample = Upsample(blur_kernel) + + self.conv = ModulatedConv2d( + in_channel, 3, 1, style_dim, demodulate=False) + self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1)) + + def forward(self, input, style, skip=None): + out = self.conv(input, style) + out = out + self.bias + + if skip is not None: + skip = self.upsample(skip) + + out = out + skip + + return out + + +class Generator(nn.Module): + + def __init__( + self, + size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + narrow=1, + ): + super().__init__() + + self.size = size + self.n_mlp = n_mlp + self.style_dim = style_dim + self.feat_multiplier = 2 + + layers = [PixelNorm()] + + for i in range(n_mlp): + layers.append( + EqualLinear( + style_dim, + style_dim, + lr_mul=lr_mlp, + activation='fused_lrelu')) + + self.style = nn.Sequential(*layers) + + self.channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + self.input = ConstantInput(self.channels[4]) + self.conv1 = StyledConv( + self.channels[4], + self.channels[4], + 3, + style_dim, + blur_kernel=blur_kernel) + self.to_rgb1 = ToRGB( + self.channels[4] * self.feat_multiplier, style_dim, upsample=False) + + self.log_size = int(math.log(size, 2)) + + self.convs = nn.ModuleList() + self.upsamples = nn.ModuleList() + self.to_rgbs = nn.ModuleList() + + in_channel = self.channels[4] + + for i in range(3, self.log_size + 1): + out_channel = self.channels[2**i] + + self.convs.append( + StyledConv( + in_channel * self.feat_multiplier, + out_channel, + 3, + style_dim, + upsample=True, + blur_kernel=blur_kernel)) + + self.convs.append( + StyledConv( + out_channel * self.feat_multiplier, + out_channel, + 3, + style_dim, + blur_kernel=blur_kernel)) + + self.to_rgbs.append( + ToRGB(out_channel * self.feat_multiplier, style_dim)) + + in_channel = out_channel + + self.n_latent = self.log_size * 2 - 2 + + def make_noise(self): + device = self.input.input.device + + noises = [torch.randn(1, 1, 2**2, 2**2, device=device)] + + for i in range(3, self.log_size + 1): + for _ in range(2): + noises.append(torch.randn(1, 1, 2**i, 2**i, device=device)) + + return noises + + def mean_latent(self, n_latent): + latent_in = torch.randn( + n_latent, self.style_dim, device=self.input.input.device) + latent = self.style(latent_in).mean(0, keepdim=True) + + return latent + + def get_latent(self, input): + return self.style(input) + + def forward( + self, + styles, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + noise=None, + ): + if not input_is_latent: + styles = [self.style(s) for s in styles] + + if noise is None: + ''' + noise = [None] * (2 * (self.log_size - 2) + 1) + ''' + noise = [] + batch = styles[0].shape[0] + for i in range(self.n_mlp + 1): + size = 2**(i + 2) + noise.append( + torch.randn( + batch, + self.channels[size], + size, + size, + device=styles[0].device)) + + if truncation < 1: + style_t = [] + + for style in styles: + style_t.append(truncation_latent + + truncation * (style - truncation_latent)) + + styles = style_t + + if len(styles) < 2: + inject_index = self.n_latent + + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + + else: + if inject_index is None: + inject_index = random.randint(1, self.n_latent - 1) + + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + latent2 = styles[1].unsqueeze(1).repeat( + 1, self.n_latent - inject_index, 1) + + latent = torch.cat([latent, latent2], 1) + + out = self.input(latent) + out = self.conv1(out, latent[:, 0], noise=noise[0]) + + skip = self.to_rgb1(out, latent[:, 1]) + + i = 1 + for conv1, conv2, noise1, noise2, to_rgb in zip( + self.convs[::2], self.convs[1::2], noise[1::2], noise[2::2], + self.to_rgbs): + out = conv1(out, latent[:, i], noise=noise1) + out = conv2(out, latent[:, i + 1], noise=noise2) + skip = to_rgb(out, latent[:, i + 2], skip) + + i += 2 + + image = skip + + if return_latents: + return image, latent + + else: + return image, None + + +class ConvLayer(nn.Sequential): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + ): + layers = [] + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + layers.append(Blur(blur_kernel, pad=(pad0, pad1))) + + stride = 2 + self.padding = 0 + + else: + stride = 1 + self.padding = kernel_size // 2 + + layers.append( + EqualConv2d( + in_channel, + out_channel, + kernel_size, + padding=self.padding, + stride=stride, + bias=bias and not activate, + )) + + if activate: + if bias: + layers.append(FusedLeakyReLU(out_channel)) + + else: + layers.append(ScaledLeakyReLU(0.2)) + + super().__init__(*layers) + + +class ResBlock(nn.Module): + + def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + self.conv1 = ConvLayer(in_channel, in_channel, 3) + self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True) + + self.skip = ConvLayer( + in_channel, + out_channel, + 1, + downsample=True, + activate=False, + bias=False) + + def forward(self, input): + out = self.conv1(input) + out = self.conv2(out) + + skip = self.skip(input) + out = (out + skip) / math.sqrt(2) + + return out + + +class FullGenerator(nn.Module): + + def __init__( + self, + size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + narrow=1, + ): + super().__init__() + channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + self.log_size = int(math.log(size, 2)) + self.generator = Generator( + size, + style_dim, + n_mlp, + channel_multiplier=channel_multiplier, + blur_kernel=blur_kernel, + lr_mlp=lr_mlp, + narrow=narrow) + + conv = [ConvLayer(3, channels[size], 1)] + self.ecd0 = nn.Sequential(*conv) + in_channel = channels[size] + + self.names = ['ecd%d' % i for i in range(self.log_size - 1)] + for i in range(self.log_size, 2, -1): + out_channel = channels[2**(i - 1)] + conv = [ConvLayer(in_channel, out_channel, 3, downsample=True)] + setattr(self, self.names[self.log_size - i + 1], + nn.Sequential(*conv)) + in_channel = out_channel + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, style_dim, activation='fused_lrelu')) + + def forward( + self, + inputs, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + ): + noise = [] + for i in range(self.log_size - 1): + ecd = getattr(self, self.names[i]) + inputs = ecd(inputs) + noise.append(inputs) + inputs = inputs.view(inputs.shape[0], -1) + outs = self.final_linear(inputs) + noise = list( + itertools.chain.from_iterable( + itertools.repeat(x, 2) for x in noise))[::-1] + outs = self.generator([outs], + return_latents, + inject_index, + truncation, + truncation_latent, + input_is_latent, + noise=noise[1:]) + return outs + + +class Discriminator(nn.Module): + + def __init__(self, + size, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + narrow=1): + super().__init__() + + channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + convs = [ConvLayer(3, channels[size], 1)] + + log_size = int(math.log(size, 2)) + + in_channel = channels[size] + + for i in range(log_size, 2, -1): + out_channel = channels[2**(i - 1)] + + convs.append(ResBlock(in_channel, out_channel, blur_kernel)) + + in_channel = out_channel + + self.convs = nn.Sequential(*convs) + + self.stddev_group = 4 + self.stddev_feat = 1 + + self.final_conv = ConvLayer(in_channel + 1, channels[4], 3) + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, channels[4], activation='fused_lrelu'), + EqualLinear(channels[4], 1), + ) + + def forward(self, input): + out = self.convs(input) + + batch, channel, height, width = out.shape + group = min(batch, self.stddev_group) + stddev = out.view(group, -1, self.stddev_feat, + channel // self.stddev_feat, height, width) + stddev = torch.sqrt(stddev.var(0, unbiased=False) + 1e-8) + stddev = stddev.mean([2, 3, 4], keepdims=True).squeeze(2) + stddev = stddev.repeat(group, 1, height, width) + out = torch.cat([out, stddev], 1) + + out = self.final_conv(out) + + out = out.view(batch, -1) + out = self.final_linear(out) + return out + + +class FullGenerator_SR(nn.Module): + + def __init__( + self, + in_size, + out_size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + narrow=1, + ): + super().__init__() + channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow), + } + + self.log_insize = int(math.log(in_size, 2)) + self.log_outsize = int(math.log(out_size, 2)) + self.generator = Generator( + out_size, + style_dim, + n_mlp, + channel_multiplier=channel_multiplier, + blur_kernel=blur_kernel, + lr_mlp=lr_mlp, + narrow=narrow) + + conv = [ConvLayer(3, channels[in_size], 1)] + self.ecd0 = nn.Sequential(*conv) + in_channel = channels[in_size] + + self.names = ['ecd%d' % i for i in range(self.log_insize - 1)] + for i in range(self.log_insize, 2, -1): + out_channel = channels[2**(i - 1)] + conv = [ConvLayer(in_channel, out_channel, 3, downsample=True)] + setattr(self, self.names[self.log_insize - i + 1], + nn.Sequential(*conv)) + in_channel = out_channel + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, style_dim, activation='fused_lrelu')) + + def forward( + self, + inputs, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + ): + noise = [] + for i in range(self.log_outsize - self.log_insize): + noise.append(None) + for i in range(self.log_insize - 1): + ecd = getattr(self, self.names[i]) + inputs = ecd(inputs) + noise.append(inputs) + inputs = inputs.view(inputs.shape[0], -1) + outs = self.final_linear(inputs) + noise = list( + itertools.chain.from_iterable( + itertools.repeat(x, 2) for x in noise))[::-1] + image, latent = self.generator([outs], + return_latents, + inject_index, + truncation, + truncation_latent, + input_is_latent, + noise=noise[1:]) + return image, latent diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/__init__.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0918d92285955855be89f00096b888ee5597ce3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/__init__.py @@ -0,0 +1,2 @@ +from .fused_act import FusedLeakyReLU, fused_leaky_relu +from .upfirdn2d import upfirdn2d diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/conv2d_gradfix.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/conv2d_gradfix.py new file mode 100644 index 0000000000000000000000000000000000000000..a3aba91f36f33469672151dcb196565d65bf184d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/conv2d_gradfix.py @@ -0,0 +1,228 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# at https://github.com/rosinality/stylegan2-pytorch/blob/master/op/conv2d_gradfix.py +import contextlib +import warnings + +import torch +from torch import autograd +from torch.nn import functional as F + +enabled = True +weight_gradients_disabled = False + + +@contextlib.contextmanager +def no_weight_gradients(): + global weight_gradients_disabled + + old = weight_gradients_disabled + weight_gradients_disabled = True + yield + weight_gradients_disabled = old + + +def conv2d(input, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + if could_use_op(input): + return conv2d_gradfix( + transpose=False, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=0, + dilation=dilation, + groups=groups, + ).apply(input, weight, bias) + + return F.conv2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + +def conv_transpose2d( + input, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, +): + if could_use_op(input): + return conv2d_gradfix( + transpose=True, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + dilation=dilation, + ).apply(input, weight, bias) + + return F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + output_padding=output_padding, + dilation=dilation, + groups=groups, + ) + + +def could_use_op(input): + if (not enabled) or (not torch.backends.cudnn.enabled): + return False + + if input.device.type != 'cuda': + return False + + warnings.warn( + f'conv2d_gradfix not supported on PyTorch {torch.__version__}. Falling back to torch.nn.functional.conv2d().' + ) + + return False + + +def ensure_tuple(xs, ndim): + xs = tuple(xs) if isinstance(xs, (tuple, list)) else (xs, ) * ndim + + return xs + + +conv2d_gradfix_cache = dict() + + +def conv2d_gradfix(transpose, weight_shape, stride, padding, output_padding, + dilation, groups): + ndim = 2 + weight_shape = tuple(weight_shape) + stride = ensure_tuple(stride, ndim) + padding = ensure_tuple(padding, ndim) + output_padding = ensure_tuple(output_padding, ndim) + dilation = ensure_tuple(dilation, ndim) + + key = (transpose, weight_shape, stride, padding, output_padding, dilation, + groups) + if key in conv2d_gradfix_cache: + return conv2d_gradfix_cache[key] + + common_kwargs = dict( + stride=stride, padding=padding, dilation=dilation, groups=groups) + + def calc_output_padding(input_shape, output_shape): + if transpose: + return [0, 0] + + a = input_shape[i + 2] - (output_shape[i + 2] - 1) * stride[i] + return [ + a - (1 - 2 * padding[i]) - dilation[i] * (weight_shape[i + 2] - 1) + for i in range(ndim) + ] + + class Conv2d(autograd.Function): + + @staticmethod + def forward(ctx, input, weight, bias): + if not transpose: + out = F.conv2d( + input=input, weight=weight, bias=bias, **common_kwargs) + + else: + out = F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + output_padding=output_padding, + **common_kwargs, + ) + + ctx.save_for_backward(input, weight) + + return out + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input, grad_weight, grad_bias = None, None, None + + if ctx.needs_input_grad[0]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, weight, None) + + if ctx.needs_input_grad[1] and not weight_gradients_disabled: + grad_weight = Conv2dGradWeight.apply(grad_output, input) + + if ctx.needs_input_grad[2]: + grad_bias = grad_output.sum((0, 2, 3)) + + return grad_input, grad_weight, grad_bias + + class Conv2dGradWeight(autograd.Function): + + @staticmethod + def forward(ctx, grad_output, input): + op = torch._C._jit_get_operation( + 'aten::cudnn_convolution_backward_weight' if not transpose else + 'aten::cudnn_convolution_transpose_backward_weight') + flags = [ + torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, + torch.backends.cudnn.allow_tf32, + ] + grad_weight = op( + weight_shape, + grad_output, + input, + padding, + stride, + dilation, + groups, + *flags, + ) + ctx.save_for_backward(grad_output, input) + + return grad_weight + + @staticmethod + def backward(ctx, grad_grad_weight): + grad_output, input = ctx.saved_tensors + grad_grad_output, grad_grad_input = None, None + + if ctx.needs_input_grad[0]: + grad_grad_output = Conv2d.apply(input, grad_grad_weight, None) + + if ctx.needs_input_grad[1]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape) + grad_grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, grad_grad_weight, None) + + return grad_grad_output, grad_grad_input + + conv2d_gradfix_cache[key] = Conv2d + + return Conv2d diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/fused_act.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/fused_act.py new file mode 100644 index 0000000000000000000000000000000000000000..7db45d3c3c11db1cf70e7e3caa51d3e93d5978f5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/fused_act.py @@ -0,0 +1,113 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# at https://github.com/rosinality/stylegan2-pytorch/blob/master/op/fused_act.py +import torch +from torch import nn +from torch.autograd import Function +from torch.nn import functional as F + +def_lib = False + + +class FusedLeakyReLUFunctionBackward(Function): + + @staticmethod + def forward(ctx, grad_output, out, bias, negative_slope, scale): + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + empty = grad_output.new_empty(0) + + grad_input = fused.fused_bias_act(grad_output.contiguous(), empty, out, + 3, 1, negative_slope, scale) + + dim = [0] + + if grad_input.ndim > 2: + dim += list(range(2, grad_input.ndim)) + + if bias: + grad_bias = grad_input.sum(dim).detach() + + else: + grad_bias = empty + + return grad_input, grad_bias + + @staticmethod + def backward(ctx, gradgrad_input, gradgrad_bias): + out, = ctx.saved_tensors + gradgrad_out = fused.fused_bias_act( + gradgrad_input.contiguous(), + gradgrad_bias, + out, + 3, + 1, + ctx.negative_slope, + ctx.scale, + ) + + return gradgrad_out, None, None, None, None + + +class FusedLeakyReLUFunction(Function): + + @staticmethod + def forward(ctx, input, bias, negative_slope, scale): + empty = input.new_empty(0) + + ctx.bias = bias is not None + + if bias is None: + bias = empty + + out = fused.fused_bias_act(input, bias, empty, 3, 0, negative_slope, + scale) + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + return out + + @staticmethod + def backward(ctx, grad_output): + out, = ctx.saved_tensors + + grad_input, grad_bias = FusedLeakyReLUFunctionBackward.apply( + grad_output, out, ctx.bias, ctx.negative_slope, ctx.scale) + + if not ctx.bias: + grad_bias = None + + return grad_input, grad_bias, None, None + + +class FusedLeakyReLU(nn.Module): + + def __init__(self, channel, bias=True, negative_slope=0.2, scale=2**0.5): + super().__init__() + + if bias: + self.bias = nn.Parameter(torch.zeros(channel)) + + else: + self.bias = None + + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + return fused_leaky_relu(input, self.bias, self.negative_slope, + self.scale) + + +def fused_leaky_relu(input, bias=None, negative_slope=0.2, scale=2**0.5): + if not def_lib: + if bias is not None: + rest_dim = [1] * (input.ndim - bias.ndim - 1) + return (F.leaky_relu( + input + bias.view(1, bias.shape[0], *rest_dim), + negative_slope=0.2) * scale) + + else: + return F.leaky_relu(input, negative_slope=0.2) * scale diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/upfirdn2d.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/upfirdn2d.py new file mode 100644 index 0000000000000000000000000000000000000000..3b93c08248e16c3c0cc3ea156863b04bd510967c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facegan/op/upfirdn2d.py @@ -0,0 +1,198 @@ +# The implementation is adopted from stylegan2-pytorch, made public available under the MIT License +# at https://github.com/rosinality/stylegan2-pytorch/blob/master/op/upfirdn2d.py +from collections import abc + +import torch +from torch.autograd import Function +from torch.nn import functional as F + +def_lib = False + + +class UpFirDn2dBackward(Function): + + @staticmethod + def forward(ctx, grad_output, kernel, grad_kernel, up, down, pad, g_pad, + in_size, out_size): + + up_x, up_y = up + down_x, down_y = down + g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1 = g_pad + + grad_output = grad_output.reshape(-1, out_size[0], out_size[1], 1) + + grad_input = upfirdn2d_op.upfirdn2d( + grad_output, + grad_kernel, + down_x, + down_y, + up_x, + up_y, + g_pad_x0, + g_pad_x1, + g_pad_y0, + g_pad_y1, + ) + grad_input = grad_input.view(in_size[0], in_size[1], in_size[2], + in_size[3]) + + ctx.save_for_backward(kernel) + + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + ctx.up_x = up_x + ctx.up_y = up_y + ctx.down_x = down_x + ctx.down_y = down_y + ctx.pad_x0 = pad_x0 + ctx.pad_x1 = pad_x1 + ctx.pad_y0 = pad_y0 + ctx.pad_y1 = pad_y1 + ctx.in_size = in_size + ctx.out_size = out_size + + return grad_input + + @staticmethod + def backward(ctx, gradgrad_input): + kernel, = ctx.saved_tensors + + gradgrad_input = gradgrad_input.reshape(-1, ctx.in_size[2], + ctx.in_size[3], 1) + + gradgrad_out = upfirdn2d_op.upfirdn2d( + gradgrad_input, + kernel, + ctx.up_x, + ctx.up_y, + ctx.down_x, + ctx.down_y, + ctx.pad_x0, + ctx.pad_x1, + ctx.pad_y0, + ctx.pad_y1, + ) + # gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.out_size[0], ctx.out_size[1], ctx.in_size[3]) + gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.in_size[1], + ctx.out_size[0], ctx.out_size[1]) + + return gradgrad_out, None, None, None, None, None, None, None, None + + +class UpFirDn2d(Function): + + @staticmethod + def forward(ctx, input, kernel, up, down, pad): + up_x, up_y = up + down_x, down_y = down + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + kernel_h, kernel_w = kernel.shape + batch, channel, in_h, in_w = input.shape + ctx.in_size = input.shape + + input = input.reshape(-1, in_h, in_w, 1) + + ctx.save_for_backward(kernel, torch.flip(kernel, [0, 1])) + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + ctx.out_size = (out_h, out_w) + + ctx.up = (up_x, up_y) + ctx.down = (down_x, down_y) + ctx.pad = (pad_x0, pad_x1, pad_y0, pad_y1) + + g_pad_x0 = kernel_w - pad_x0 - 1 + g_pad_y0 = kernel_h - pad_y0 - 1 + g_pad_x1 = in_w * up_x - out_w * down_x + pad_x0 - up_x + 1 + g_pad_y1 = in_h * up_y - out_h * down_y + pad_y0 - up_y + 1 + + ctx.g_pad = (g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1) + + out = upfirdn2d_op.upfirdn2d(input, kernel, up_x, up_y, down_x, down_y, + pad_x0, pad_x1, pad_y0, pad_y1) + # out = out.view(major, out_h, out_w, minor) + out = out.view(-1, channel, out_h, out_w) + + return out + + @staticmethod + def backward(ctx, grad_output): + kernel, grad_kernel = ctx.saved_tensors + + grad_input = None + + if ctx.needs_input_grad[0]: + grad_input = UpFirDn2dBackward.apply( + grad_output, + kernel, + grad_kernel, + ctx.up, + ctx.down, + ctx.pad, + ctx.g_pad, + ctx.in_size, + ctx.out_size, + ) + + return grad_input, None, None, None, None + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + if not isinstance(up, abc.Iterable): + up = (up, up) + + if not isinstance(down, abc.Iterable): + down = (down, down) + + if len(pad) == 2: + pad = (pad[0], pad[1], pad[0], pad[1]) + + if not def_lib: + out = upfirdn2d_native(input, kernel, *up, *down, *pad) + + return out + + +def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, + pad_y0, pad_y1): + _, channel, in_h, in_w = input.shape + input = input.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad( + out, + [0, 0, + max(pad_x0, 0), + max(pad_x1, 0), + max(pad_y0, 0), + max(pad_y1, 0)]) + out = out[:, + max(-pad_y0, 0):out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0):out.shape[2] - max(-pad_x1, 0)] + + out = out.permute(0, 3, 1, 2) + out = out.reshape( + [-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + + return out.view(-1, channel, out_h, out_w) diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/__init__.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/align_trans.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/align_trans.py new file mode 100644 index 0000000000000000000000000000000000000000..0d7ebbb60b4fbae293b8f4585d5839cf777bf332 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/align_trans.py @@ -0,0 +1,431 @@ +# The implementation here is modified based on InsightFace_Pytorch, originally MIT License and publicly available +# at https://github.com/TreB1eN/InsightFace_Pytorch/blob/master/mtcnn_pytorch/src/align_trans.py +import cv2 +import numpy as np + +from .matlab_cp2tform import get_similarity_transform_for_cv2 + +# reference facial points, a list of coordinates (x,y) +REFERENCE_FACIAL_POINTS = [[30.29459953, 51.69630051], + [65.53179932, 51.50139999], + [48.02519989, + 71.73660278], [33.54930115, 92.3655014], + [62.72990036, 92.20410156]] + +DEFAULT_CROP_SIZE = (96, 112) + + +def _umeyama(src, dst, estimate_scale=True, scale=1.0): + """Estimate N-D similarity transformation with or without scaling. + Parameters + ---------- + src : (M, N) array + Source coordinates. + dst : (M, N) array + Destination coordinates. + estimate_scale : bool + Whether to estimate scaling factor. + Returns + ------- + T : (N + 1, N + 1) + The homogeneous similarity transformation matrix. The matrix contains + NaN values only if the problem is not well-conditioned. + References + ---------- + .. [1] "Least-squares estimation of transformation parameters between two + point patterns", Shinji Umeyama, PAMI 1991, :DOI:`10.1109/34.88573` + """ + + num = src.shape[0] + dim = src.shape[1] + + # Compute mean of src and dst. + src_mean = src.mean(axis=0) + dst_mean = dst.mean(axis=0) + + # Subtract mean from src and dst. + src_demean = src - src_mean + dst_demean = dst - dst_mean + + # Eq. (38). + A = dst_demean.T @ src_demean / num + + # Eq. (39). + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + + T = np.eye(dim + 1, dtype=np.double) + + U, S, V = np.linalg.svd(A) + + # Eq. (40) and (43). + rank = np.linalg.matrix_rank(A) + if rank == 0: + return np.nan * T + elif rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = U @ V + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = U @ np.diag(d) @ V + d[dim - 1] = s + else: + T[:dim, :dim] = U @ np.diag(d) @ V + + if estimate_scale: + # Eq. (41) and (42). + scale = 1.0 / src_demean.var(axis=0).sum() * (S @ d) + else: + scale = scale + + T[:dim, dim] = dst_mean - scale * (T[:dim, :dim] @ src_mean.T) + T[:dim, :dim] *= scale + + return T, scale + + +class FaceWarpException(Exception): + + def __str__(self): + return 'In File {}:{}'.format(__file__, super.__str__(self)) + + +def get_reference_facial_points(output_size=None, + inner_padding_factor=0.0, + outer_padding=(0, 0), + default_square=False): + """ + Function: + ---------- + get reference 5 key points according to crop settings: + 0. Set default crop_size: + if default_square: + crop_size = (112, 112) + else: + crop_size = (96, 112) + 1. Pad the crop_size by inner_padding_factor in each side; + 2. Resize crop_size into (output_size - outer_padding*2), + pad into output_size with outer_padding; + 3. Output reference_5point; + Parameters: + ---------- + @output_size: (w, h) or None + size of aligned face image + @inner_padding_factor: (w_factor, h_factor) + padding factor for inner (w, h) + @outer_padding: (w_pad, h_pad) + each row is a pair of coordinates (x, y) + @default_square: True or False + if True: + default crop_size = (112, 112) + else: + default crop_size = (96, 112); + !!! make sure, if output_size is not None: + (output_size - outer_padding) + = some_scale * (default crop_size * (1.0 + inner_padding_factor)) + Returns: + ---------- + @reference_5point: 5x2 np.array + each row is a pair of transformed coordinates (x, y) + """ + + tmp_5pts = np.array(REFERENCE_FACIAL_POINTS) + tmp_crop_size = np.array(DEFAULT_CROP_SIZE) + + # 0) make the inner region a square + if default_square: + size_diff = max(tmp_crop_size) - tmp_crop_size + tmp_5pts += size_diff / 2 + tmp_crop_size += size_diff + + if (output_size and output_size[0] == tmp_crop_size[0] + and output_size[1] == tmp_crop_size[1]): + return tmp_5pts + + if (inner_padding_factor == 0 and outer_padding == (0, 0)): + if output_size is None: + return tmp_5pts + else: + raise FaceWarpException( + 'No paddings to do, output_size must be None or {}'.format( + tmp_crop_size)) + + if not (0 <= inner_padding_factor <= 1.0): + raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)') + + if ((inner_padding_factor > 0 or outer_padding[0] > 0 + or outer_padding[1] > 0) and output_size is None): + output_size = tmp_crop_size * \ + (1 + inner_padding_factor * 2).astype(np.int32) + output_size += np.array(outer_padding) + + if not (outer_padding[0] < output_size[0] + and outer_padding[1] < output_size[1]): + raise FaceWarpException('Not (outer_padding[0] < output_size[0]' + 'and outer_padding[1] < output_size[1])') + + # 1) pad the inner region according inner_padding_factor + if inner_padding_factor > 0: + size_diff = tmp_crop_size * inner_padding_factor * 2 + tmp_5pts += size_diff / 2 + tmp_crop_size += np.round(size_diff).astype(np.int32) + + # 2) resize the padded inner region + size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2 + + if size_bf_outer_pad[0] * tmp_crop_size[1] != size_bf_outer_pad[ + 1] * tmp_crop_size[0]: + raise FaceWarpException( + 'Must have (output_size - outer_padding)' + '= some_scale * (crop_size * (1.0 + inner_padding_factor)') + + scale_factor = size_bf_outer_pad[0].astype(np.float32) / tmp_crop_size[0] + tmp_5pts = tmp_5pts * scale_factor + tmp_crop_size = size_bf_outer_pad + + # 3) add outer_padding to make output_size + reference_5point = tmp_5pts + np.array(outer_padding) + tmp_crop_size = output_size + + return reference_5point + + +def get_affine_transform_matrix(src_pts, dst_pts): + """ + Function: + ---------- + get affine transform matrix 'tfm' from src_pts to dst_pts + Parameters: + ---------- + @src_pts: Kx2 np.array + source points matrix, each row is a pair of coordinates (x, y) + @dst_pts: Kx2 np.array + destination points matrix, each row is a pair of coordinates (x, y) + Returns: + ---------- + @tfm: 2x3 np.array + transform matrix from src_pts to dst_pts + """ + + tfm = np.float32([[1, 0, 0], [0, 1, 0]]) + n_pts = src_pts.shape[0] + ones = np.ones((n_pts, 1), src_pts.dtype) + src_pts_ = np.hstack([src_pts, ones]) + dst_pts_ = np.hstack([dst_pts, ones]) + + A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_) + + if rank == 3: + tfm = np.float32([[A[0, 0], A[1, 0], A[2, 0]], + [A[0, 1], A[1, 1], A[2, 1]]]) + elif rank == 2: + tfm = np.float32([[A[0, 0], A[1, 0], 0], [A[0, 1], A[1, 1], 0]]) + + return tfm + + +def warp_and_crop_face(src_img, + facial_pts, + reference_pts=None, + crop_size=(96, 112), + align_type='smilarity', + return_trans_inv=False): + """ + Function: + ---------- + apply affine transform 'trans' to uv + Parameters: + ---------- + @src_img: 3x3 np.array + input image + @facial_pts: could be + 1)a list of K coordinates (x,y) + or + 2) Kx2 or 2xK np.array + each row or col is a pair of coordinates (x, y) + @reference_pts: could be + 1) a list of K coordinates (x,y) + or + 2) Kx2 or 2xK np.array + each row or col is a pair of coordinates (x, y) + or + 3) None + if None, use default reference facial points + @crop_size: (w, h) + output face image size + @align_type: transform type, could be one of + 1) 'similarity': use similarity transform + 2) 'cv2_affine': use the first 3 points to do affine transform, + by calling cv2.getAffineTransform() + 3) 'affine': use all points to do affine transform + Returns: + ---------- + @face_img: output face image with size (w, h) = @crop_size + """ + + if reference_pts is None: + if crop_size[0] == 96 and crop_size[1] == 112: + reference_pts = REFERENCE_FACIAL_POINTS + else: + default_square = False + inner_padding_factor = 0 + outer_padding = (0, 0) + output_size = crop_size + + reference_pts = get_reference_facial_points( + output_size, inner_padding_factor, outer_padding, + default_square) + + ref_pts = np.float32(reference_pts) + ref_pts = (ref_pts - 112 / 2) * 0.85 + 112 / 2 + ref_pts *= crop_size[0] / 112. + ref_pts_shp = ref_pts.shape + if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: + raise FaceWarpException( + 'reference_pts.shape must be (K,2) or (2,K) and K>2') + + if ref_pts_shp[0] == 2: + ref_pts = ref_pts.T + + src_pts = np.float32(facial_pts) + src_pts_shp = src_pts.shape + if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: + raise FaceWarpException( + 'facial_pts.shape must be (K,2) or (2,K) and K>2') + + if src_pts_shp[0] == 2: + src_pts = src_pts.T + + if src_pts.shape != ref_pts.shape: + raise FaceWarpException( + 'facial_pts and reference_pts must have the same shape') + + if align_type == 'cv2_affine': + tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) + elif align_type == 'affine': + tfm = get_affine_transform_matrix(src_pts, ref_pts) + else: + tfm, tfm_inv = get_similarity_transform_for_cv2(src_pts, ref_pts) + + face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1])) + + if return_trans_inv: + return face_img, tfm_inv + else: + return face_img + + +def warp_and_crop_face_enhance(src_img, + facial_pts, + reference_pts=None, + crop_size=(96, 112), + align_type='smilarity'): + if reference_pts is None: + if crop_size[0] == 96 and crop_size[1] == 112: + reference_pts = REFERENCE_FACIAL_POINTS + else: + default_square = False + inner_padding_factor = 0 + outer_padding = (0, 0) + output_size = crop_size + + reference_pts = get_reference_facial_points( + output_size, inner_padding_factor, outer_padding, + default_square) + + ref_pts = np.float32(reference_pts) + ref_pts_shp = ref_pts.shape + if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: + raise FaceWarpException( + 'reference_pts.shape must be (K,2) or (2,K) and K>2') + + if ref_pts_shp[0] == 2: + ref_pts = ref_pts.T + + src_pts = np.float32(facial_pts) + src_pts_shp = src_pts.shape + if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: + raise FaceWarpException( + 'facial_pts.shape must be (K,2) or (2,K) and K>2') + + if src_pts_shp[0] == 2: + src_pts = src_pts.T + + if src_pts.shape != ref_pts.shape: + raise FaceWarpException( + 'facial_pts and reference_pts must have the same shape') + + if align_type == 'cv2_affine': + tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) + tfm_inv = cv2.getAffineTransform(ref_pts[0:3], src_pts[0:3]) + elif align_type == 'affine': + tfm = get_affine_transform_matrix(src_pts, ref_pts) + tfm_inv = get_affine_transform_matrix(ref_pts, src_pts) + else: + params, scale = _umeyama(src_pts, ref_pts) + tfm = params[:2, :] + + params, _ = _umeyama(ref_pts, src_pts, False, scale=1.0 / scale) + tfm_inv = params[:2, :] + + face_img = cv2.warpAffine( + src_img, tfm, (crop_size[0], crop_size[1]), flags=3) + + return face_img, tfm_inv + + +def get_f5p(landmarks, np_img): + eye_left = find_pupil(landmarks[36:41], np_img) + eye_right = find_pupil(landmarks[42:47], np_img) + if eye_left is None or eye_right is None: + print('cannot find 5 points with find_pupil, used mean instead.!') + eye_left = landmarks[36:41].mean(axis=0) + eye_right = landmarks[42:47].mean(axis=0) + nose = landmarks[30] + mouth_left = landmarks[48] + mouth_right = landmarks[54] + f5p = [[eye_left[0], eye_left[1]], [eye_right[0], eye_right[1]], + [nose[0], nose[1]], [mouth_left[0], mouth_left[1]], + [mouth_right[0], mouth_right[1]]] + return f5p + + +def find_pupil(landmarks, np_img): + h, w, _ = np_img.shape + xmax = int(landmarks[:, 0].max()) + xmin = int(landmarks[:, 0].min()) + ymax = int(landmarks[:, 1].max()) + ymin = int(landmarks[:, 1].min()) + + if ymin >= ymax or xmin >= xmax or ymin < 0 or xmin < 0 or ymax > h or xmax > w: + return None + eye_img_bgr = np_img[ymin:ymax, xmin:xmax, :] + eye_img = cv2.cvtColor(eye_img_bgr, cv2.COLOR_BGR2GRAY) + eye_img = cv2.equalizeHist(eye_img) + n_marks = landmarks - np.array([xmin, ymin]).reshape([1, 2]) + eye_mask = cv2.fillConvexPoly( + np.zeros_like(eye_img), n_marks.astype(np.int32), 1) + ret, thresh = cv2.threshold(eye_img, 100, 255, + cv2.THRESH_BINARY | cv2.THRESH_OTSU) + thresh = (1 - thresh / 255.) * eye_mask + cnt = 0 + xm = [] + ym = [] + for i in range(thresh.shape[0]): + for j in range(thresh.shape[1]): + if thresh[i, j] > 0.5: + xm.append(j) + ym.append(i) + cnt += 1 + if cnt != 0: + xm.sort() + ym.sort() + xm = xm[cnt // 2] + ym = ym[cnt // 2] + else: + xm = thresh.shape[1] / 2 + ym = thresh.shape[0] / 2 + + return xm + xmin, ym + ymin diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/matlab_cp2tform.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/matlab_cp2tform.py new file mode 100644 index 0000000000000000000000000000000000000000..87b9fd1e5335f72be90eb6e465187d27980b9a35 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/facelib/matlab_cp2tform.py @@ -0,0 +1,230 @@ +# The implementation is adopted from InsightFace_Pytorch, made publicly available under the MIT License +# at https://github.com/TreB1eN/InsightFace_Pytorch/blob/master/mtcnn_pytorch/src/matlab_cp2tform.py + +import numpy as np +from numpy.linalg import inv, lstsq +from numpy.linalg import matrix_rank as rank +from numpy.linalg import norm + + +class MatlabCp2tormException(Exception): + + def __str__(self): + return 'In File {}:{}'.format(__file__, super.__str__(self)) + + +def tformfwd(trans, uv): + """ + Function: + ---------- + apply affine transform 'trans' to uv + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix + @uv: Kx2 np.array + each row is a pair of coordinates (x, y) + + Returns: + ---------- + @xy: Kx2 np.array + each row is a pair of transformed coordinates (x, y) + """ + uv = np.hstack((uv, np.ones((uv.shape[0], 1)))) + xy = np.dot(uv, trans) + xy = xy[:, 0:-1] + return xy + + +def tforminv(trans, uv): + """ + Function: + ---------- + apply the inverse of affine transform 'trans' to uv + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix + @uv: Kx2 np.array + each row is a pair of coordinates (x, y) + + Returns: + ---------- + @xy: Kx2 np.array + each row is a pair of inverse-transformed coordinates (x, y) + """ + Tinv = inv(trans) + xy = tformfwd(Tinv, uv) + return xy + + +def findNonreflectiveSimilarity(uv, xy, options=None): + + options = {'K': 2} + + K = options['K'] + M = xy.shape[0] + x = xy[:, 0].reshape((-1, 1)) + y = xy[:, 1].reshape((-1, 1)) + + tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1)))) + tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1)))) + X = np.vstack((tmp1, tmp2)) + + u = uv[:, 0].reshape((-1, 1)) + v = uv[:, 1].reshape((-1, 1)) + U = np.vstack((u, v)) + + if rank(X) >= 2 * K: + r, _, _, _ = lstsq(X, U) + r = np.squeeze(r) + else: + raise Exception('cp2tform:twoUniquePointsReq') + + sc = r[0] + ss = r[1] + tx = r[2] + ty = r[3] + + Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]]) + + T = inv(Tinv) + T[:, 2] = np.array([0, 0, 1]) + + return T, Tinv + + +def findSimilarity(uv, xy, options=None): + + options = {'K': 2} + + trans1, trans1_inv = findNonreflectiveSimilarity(uv, xy, options) + + xyR = xy + xyR[:, 0] = -1 * xyR[:, 0] + + trans2r, trans2r_inv = findNonreflectiveSimilarity(uv, xyR, options) + + TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]]) + + trans2 = np.dot(trans2r, TreflectY) + + xy1 = tformfwd(trans1, uv) + norm1 = norm(xy1 - xy) + + xy2 = tformfwd(trans2, uv) + norm2 = norm(xy2 - xy) + + if norm1 <= norm2: + return trans1, trans1_inv + else: + trans2_inv = inv(trans2) + return trans2, trans2_inv + + +def get_similarity_transform(src_pts, dst_pts, reflective=True): + """ + Function: + ---------- + Find Similarity Transform Matrix 'trans': + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y, 1] = [u, v, 1] * trans + + Parameters: + ---------- + @src_pts: Kx2 np.array + source points, each row is a pair of coordinates (x, y) + @dst_pts: Kx2 np.array + destination points, each row is a pair of transformed + coordinates (x, y) + @reflective: True or False + if True: + use reflective similarity transform + else: + use non-reflective similarity transform + + Returns: + ---------- + @trans: 3x3 np.array + transform matrix from uv to xy + trans_inv: 3x3 np.array + inverse of trans, transform matrix from xy to uv + """ + + if reflective: + trans, trans_inv = findSimilarity(src_pts, dst_pts) + else: + trans, trans_inv = findNonreflectiveSimilarity(src_pts, dst_pts) + + return trans, trans_inv + + +def cvt_tform_mat_for_cv2(trans): + """ + Function: + ---------- + Convert Transform Matrix 'trans' into 'cv2_trans' which could be + directly used by cv2.warpAffine(): + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y].T = cv_trans * [u, v, 1].T + + Parameters: + ---------- + @trans: 3x3 np.array + transform matrix from uv to xy + + Returns: + ---------- + @cv2_trans: 2x3 np.array + transform matrix from src_pts to dst_pts, could be directly used + for cv2.warpAffine() + """ + cv2_trans = trans[:, 0:2].T + + return cv2_trans + + +def get_similarity_transform_for_cv2(src_pts, dst_pts, reflective=True): + """ + Function: + ---------- + Find Similarity Transform Matrix 'cv2_trans' which could be + directly used by cv2.warpAffine(): + u = src_pts[:, 0] + v = src_pts[:, 1] + x = dst_pts[:, 0] + y = dst_pts[:, 1] + [x, y].T = cv_trans * [u, v, 1].T + + Parameters: + ---------- + @src_pts: Kx2 np.array + source points, each row is a pair of coordinates (x, y) + @dst_pts: Kx2 np.array + destination points, each row is a pair of transformed + coordinates (x, y) + reflective: True or False + if True: + use reflective similarity transform + else: + use non-reflective similarity transform + + Returns: + ---------- + @cv2_trans: 2x3 np.array + transform matrix from src_pts to dst_pts, could be directly used + for cv2.warpAffine() + """ + trans, trans_inv = get_similarity_transform(src_pts, dst_pts, reflective) + cv2_trans = cvt_tform_mat_for_cv2(trans) + cv2_trans_inv = cvt_tform_mat_for_cv2(trans_inv) + + return cv2_trans, cv2_trans_inv diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/image_face_fusion.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/image_face_fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..b7a876b0ebd1867025a4ab04cc806c5a738a0661 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/image_face_fusion.py @@ -0,0 +1,317 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import OrderedDict +from typing import Any, Dict + +import cv2 +import numpy as np +import PIL.Image as Image +import torch +import torch.nn.functional as F +import torchvision.transforms as transforms + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.face_detection.peppa_pig_face.facer import FaceAna +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .facegan.face_gan import GPEN +from .facelib.align_trans import (get_f5p, get_reference_facial_points, + warp_and_crop_face, + warp_and_crop_face_enhance) +from .network.aei_flow_net import AEI_Net +from .network.bfm import ParametricFaceModel +from .network.facerecon_model import ReconNetWrapper +from .network.model_irse import Backbone +from .network.ops import warp_affine_torch + +logger = get_logger() + +__all__ = ['ImageFaceFusion'] + + +@MODELS.register_module( + Tasks.image_face_fusion, module_name=Models.image_face_fusion) +class ImageFaceFusion(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image face fusion model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + if torch.cuda.is_available(): + self.device = torch.device('cuda') + else: + self.device = torch.device('cpu') + + self.num_kp = 17 + self.id_dim = 512 + + self.netG = AEI_Net( + c_id=self.id_dim, num_kp=self.num_kp, device=self.device) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoints = torch.load(model_path, map_location='cpu') + model_state = self.convert_state_dict(checkpoints['state_dict']) + self.netG.load_state_dict(model_state) + self.netG = self.netG.to(self.device) + self.netG.eval() + + self.arcface = Backbone([112, 112], 100, 'ir') + arcface_path = os.path.join(model_dir, 'faceRecog', + 'CurricularFace_Backbone.pth') + self.arcface.load_state_dict( + torch.load(arcface_path, map_location='cpu'), strict=False) + self.arcface = self.arcface.to(self.device) + self.arcface.eval() + + self.f_3d = ReconNetWrapper(net_recon='resnet50', use_last_fc=False) + f_3d_path = os.path.join(model_dir, '3dRecon', 'face_3d.pth') + self.f_3d.load_state_dict( + torch.load(f_3d_path, map_location='cpu')['net_recon']) + self.f_3d = self.f_3d.to(self.device) + self.f_3d.eval() + + bfm_dir = os.path.join(model_dir, 'BFM') + self.face_model = ParametricFaceModel(bfm_folder=bfm_dir) + self.face_model.to(self.device) + + self.facer = FaceAna(model_dir) + + logger.info('load facefusion models done') + + self.mask_init = cv2.imread(os.path.join(model_dir, 'alpha.jpg')) + self.mask_init = cv2.resize(self.mask_init, (256, 256)) + self.mask = self.image_transform(self.mask_init, is_norm=False) + + face_enhance_path = os.path.join(model_dir, 'faceEnhance', + 'GPEN-BFR-1024.pth') + + if not os.path.exists(face_enhance_path): + logger.warning( + 'model path not found, please update the latest model!') + + self.ganwrap_1024 = GPEN(face_enhance_path, 1024, 2, self.device) + + self.mask_enhance = np.zeros((512, 512), np.float32) + cv2.rectangle(self.mask_enhance, (26, 26), (486, 486), (1, 1, 1), -1, + cv2.LINE_AA) + self.mask_enhance = cv2.GaussianBlur(self.mask_enhance, (101, 101), 11) + self.mask_enhance = cv2.GaussianBlur(self.mask_enhance, (101, 101), 11) + + default_square = True + inner_padding_factor = 0.25 + outer_padding = (0, 0) + self.reference_5pts_1024 = get_reference_facial_points( + (1024, 1024), inner_padding_factor, outer_padding, default_square) + + self.test_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + + logger.info('init done') + + def convert_state_dict(self, state_dict): + if not next(iter(state_dict)).startswith('module.'): + return state_dict + new_state_dict = OrderedDict() + + split_index = 0 + for cur_key, cur_value in state_dict.items(): + if cur_key.startswith('module.model'): + split_index = 13 + elif cur_key.startswith('module'): + split_index = 7 + + break + + for k, v in state_dict.items(): + name = k[split_index:] + new_state_dict[name] = v + return new_state_dict + + def image_transform(self, + image, + is_norm=True, + mean=(0.5, 0.5, 0.5), + std=(0.5, 0.5, 0.5)): + image = image.astype(np.float32) + image = image / 255.0 + if is_norm: + image -= mean + image /= std + + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + image = torch.from_numpy(image) + image = image.to(self.device) + return image + + def extract_id(self, np_source, f5p): + Xs = warp_and_crop_face( + np_source, + f5p, + reference_pts=get_reference_facial_points(default_square=True), + crop_size=(256, 256)) + + Xs = Image.fromarray(Xs) + Xs = self.test_transform(Xs) + Xs = Xs.unsqueeze(0).to(self.device) + with torch.no_grad(): + embeds, Xs_feats = self.arcface( + F.interpolate( + Xs, (112, 112), mode='bilinear', align_corners=True)) + return embeds, Xs + + def detect_face(self, img): + src_h, src_w, _ = img.shape + boxes, landmarks, _ = self.facer.run(img) + if boxes.shape[0] == 0: + return None, None, None + elif boxes.shape[0] > 1: + max_area = 0 + max_index = 0 + for i in range(boxes.shape[0]): + bbox_width = boxes[i][2] - boxes[i][0] + bbox_height = boxes[i][3] - boxes[i][1] + area = int(bbox_width) * int(bbox_height) + if area > max_area: + max_index = i + max_area = area + + fw = boxes[max_index][2] - boxes[max_index][0] + fh = boxes[max_index][3] - boxes[max_index][1] + return landmarks[max_index], fw, fh + else: + fw = boxes[0][2] - boxes[0][0] + fh = boxes[0][3] - boxes[0][1] + return landmarks[0], fw, fh + + def compute_3d_params(self, Xs, Xt): + kp_fuse = {} + kp_t = {} + + c_s = self.f_3d( + F.interpolate(Xs * 0.5 + 0.5, size=224, mode='bilinear')) + c_t = self.f_3d( + F.interpolate(Xt * 0.5 + 0.5, size=224, mode='bilinear')) + c_fuse = torch.cat(((c_s[:, :80] + c_t[:, :80]) / 2, c_t[:, 80:]), + dim=1) + _, _, _, q_fuse = self.face_model.compute_for_render(c_fuse) + q_fuse = q_fuse / 224 + q_fuse[..., 1] = 1 - q_fuse[..., 1] + q_fuse = q_fuse * 2 - 1 + delta = int((17 - self.num_kp) / 2) + + _, _, _, q_t = self.face_model.compute_for_render(c_t) + q_t = q_t / 224 + q_t[..., 1] = 1 - q_t[..., 1] + q_t = q_t * 2 - 1 + + kp_fuse['value'] = q_fuse[:, delta:17 - delta, :] + kp_t['value'] = q_t[:, delta:17 - delta, :] + + return kp_fuse, kp_t + + def process_enhance(self, im, f5p, fh, fw): + height, width, _ = im.shape + + of, tfm_inv = warp_and_crop_face_enhance( + im, + f5p, + reference_pts=self.reference_5pts_1024, + crop_size=(1024, 1024)) + ef, pred = self.ganwrap_1024.process(of) + + tmp_mask = self.mask_enhance + tmp_mask = cv2.resize(tmp_mask, ef.shape[:2]) + tmp_mask = cv2.warpAffine(tmp_mask, tfm_inv, (width, height), flags=3) + + full_mask = np.zeros((height, width), dtype=np.float32) + full_img = np.zeros(im.shape, dtype=np.uint8) + + if min(fh, fw) < 40: + ef = cv2.pyrDown(ef) + ef = cv2.pyrDown(ef) + ef = cv2.pyrUp(ef) + ef = cv2.pyrUp(ef) + elif min(fh, fw) < 60: + ef = cv2.pyrDown(ef) + ef = cv2.resize(ef, (0, 0), fx=2, fy=2) + ef = cv2.resize(ef, (0, 0), fx=0.5, fy=0.5) + ef = cv2.pyrUp(ef) + elif min(fh, fw) < 80: + ef = cv2.pyrDown(ef) + ef = cv2.pyrUp(ef) + elif min(fh, fw) < 100: + ef = cv2.pyrDown(ef) + ef = cv2.resize(ef, (0, 0), fx=2, fy=2) + + tmp_img = cv2.warpAffine(ef, tfm_inv, (width, height), flags=3) + + mask = tmp_mask - full_mask + full_mask[np.where(mask > 0)] = tmp_mask[np.where(mask > 0)] + full_img[np.where(mask > 0)] = tmp_img[np.where(mask > 0)] + + full_mask = full_mask[:, :, np.newaxis] + im = cv2.convertScaleAbs(im * (1 - full_mask) + full_img * full_mask) + im = cv2.resize(im, (width, height)) + return im + + def inference(self, template_img, user_img): + ori_h, ori_w, _ = template_img.shape + + template_img = template_img.cpu().numpy() + user_img = user_img.cpu().numpy() + + user_img_bgr = user_img[:, :, ::-1] + landmark_source, _, _ = self.detect_face(user_img) + if landmark_source is None: + logger.warning('No face detected in user image!') + return template_img + f5p_user = get_f5p(landmark_source, user_img_bgr) + + template_img_bgr = template_img[:, :, ::-1] + landmark_template, fw, fh = self.detect_face(template_img) + if landmark_template is None: + logger.warning('No face detected in template image!') + return template_img + f5p_template = get_f5p(landmark_template, template_img_bgr) + + Xs_embeds, Xs = self.extract_id(user_img, f5p_user) + Xt, trans_inv = warp_and_crop_face( + template_img, + f5p_template, + reference_pts=get_reference_facial_points(default_square=True), + crop_size=(256, 256), + return_trans_inv=True) + + trans_inv = trans_inv.astype(np.float32) + trans_inv = torch.from_numpy(trans_inv) + trans_inv = trans_inv.to(self.device) + Xt_raw = self.image_transform(template_img, is_norm=False) + Xt = self.image_transform(Xt) + + with torch.no_grad(): + kp_fuse, kp_t = self.compute_3d_params(Xs, Xt) + Yt, _, _ = self.netG(Xt, Xs_embeds, kp_fuse, kp_t) + Yt = Yt * 0.5 + 0.5 + Yt = torch.clamp(Yt, 0, 1) + + Yt_trans_inv = warp_affine_torch(Yt, trans_inv, (ori_h, ori_w)) + mask_ = warp_affine_torch(self.mask, trans_inv, (ori_h, ori_w)) + + Yt_trans_inv = mask_ * Yt_trans_inv + (1 - mask_) * Xt_raw + Yt_trans_inv = Yt_trans_inv.squeeze().permute(1, 2, + 0).cpu().numpy() + Yt_trans_inv = Yt_trans_inv.astype(np.float32) + out_img = Yt_trans_inv[:, :, ::-1] * 255. + out_img = self.process_enhance(out_img, f5p_template, fh, fw) + + logger.info('model inference done') + + return out_img.astype(np.uint8) diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/__init__.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aad_layer.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aad_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..adb6851810d215598d3b63ddb143bdba3e61cd7a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aad_layer.py @@ -0,0 +1,99 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from .ops import SpectralNorm + + +class AADLayer(nn.Module): + + def __init__(self, c_x, attr_c, c_id=256): + super(AADLayer, self).__init__() + self.attr_c = attr_c + self.c_id = c_id + self.c_x = c_x + + ks = 3 + pw = ks // 2 + + nhidden = 128 + + self.mlp_shared = nn.Sequential( + nn.ReflectionPad2d(pw), + nn.Conv2d(attr_c, nhidden, kernel_size=ks, padding=0), nn.ReLU()) + self.pad = nn.ReflectionPad2d(pw) + self.conv1 = nn.Conv2d( + nhidden, c_x, kernel_size=ks, stride=1, padding=0) + self.conv2 = nn.Conv2d( + nhidden, c_x, kernel_size=ks, stride=1, padding=0) + self.fc1 = nn.Linear(c_id, c_x) + self.fc2 = nn.Linear(c_id, c_x) + + self.norm = PositionalNorm2d + + self.pad_h = nn.ReflectionPad2d(pw) + self.conv_h = nn.Conv2d(c_x, 1, kernel_size=ks, stride=1, padding=0) + + def forward(self, h_in, z_attr, z_id): + + h = self.norm(h_in) + actv = self.mlp_shared(z_attr) + gamma_attr = self.conv1(self.pad(actv)) + beta_attr = self.conv2(self.pad(actv)) + + gamma_id = self.fc1(z_id) + beta_id = self.fc2(z_id) + A = gamma_attr * h + beta_attr + gamma_id = gamma_id.reshape(h.shape[0], self.c_x, 1, 1).expand_as(h) + beta_id = beta_id.reshape(h.shape[0], self.c_x, 1, 1).expand_as(h) + B = gamma_id * h + beta_id + + M = torch.sigmoid(self.conv_h(self.pad_h(h))) + + out = (torch.ones_like(M).to(M.device) - M) * A + M * B + + return out + + +def PositionalNorm2d(x, epsilon=1e-5): + mean = x.mean(dim=1, keepdim=True) + std = x.var(dim=1, keepdim=True).add(epsilon).sqrt() + output = (x - mean) / std + return output + + +class AAD_ResBlk(nn.Module): + + def __init__(self, cin, cout, c_attr, c_id=256): + super(AAD_ResBlk, self).__init__() + self.cin = cin + self.cout = cout + self.learned_shortcut = (self.cin != self.cout) + fmiddle = min(self.cin, self.cout) + + self.AAD1 = AADLayer(cin, c_attr, c_id) + self.AAD2 = AADLayer(fmiddle, c_attr, c_id) + self.pad = nn.ReflectionPad2d(1) + self.conv1 = SpectralNorm( + nn.Conv2d(cin, fmiddle, kernel_size=3, stride=1, padding=0)) + self.conv2 = SpectralNorm( + nn.Conv2d(fmiddle, cout, kernel_size=3, stride=1, padding=0)) + + self.relu1 = nn.LeakyReLU(2e-1) + self.relu2 = nn.LeakyReLU(2e-1) + + if self.learned_shortcut: + self.AAD3 = AADLayer(cin, c_attr, c_id) + self.conv3 = SpectralNorm( + nn.Conv2d(cin, cout, kernel_size=1, bias=False)) + + def forward(self, h, z_attr, z_id): + x = self.conv1(self.pad(self.relu1(self.AAD1(h, z_attr, z_id)))) + x = self.conv2(self.pad(self.relu2(self.AAD2(x, z_attr, z_id)))) + + if self.learned_shortcut: + h = self.conv3(self.AAD3(h, z_attr, z_id)) + + x = x + h + + return x diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aei_flow_net.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aei_flow_net.py new file mode 100644 index 0000000000000000000000000000000000000000..a2047671af99cb3dc53edb077f5efacd670ccd78 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/aei_flow_net.py @@ -0,0 +1,251 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .aad_layer import AAD_ResBlk +from .dense_motion import DenseMotionNetwork +from .ops import SpectralNorm, init_func + + +class Conv4x4(nn.Module): + + def __init__(self, in_c, out_c): + super(Conv4x4, self).__init__() + self.conv = nn.Conv2d( + in_channels=in_c, + out_channels=out_c, + kernel_size=4, + stride=2, + padding=1, + bias=False) + self.norm = nn.BatchNorm2d(out_c) + self.lrelu = nn.LeakyReLU(0.1) + + def forward(self, feat): + x = self.conv(feat) + x = self.norm(x) + x = self.lrelu(x) + return x + + +class DeConv4x4(nn.Module): + + def __init__(self, in_c, out_c): + super(DeConv4x4, self).__init__() + self.deconv = nn.ConvTranspose2d( + in_channels=in_c, + out_channels=out_c, + kernel_size=4, + stride=2, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(out_c) + self.lrelu = nn.LeakyReLU(0.1) + + def forward(self, input, skip): + x = self.deconv(input) + x = self.bn(x) + x = self.lrelu(x) + return torch.cat((x, skip), dim=1) + + +class Attention(nn.Module): + + def __init__(self, ch, use_sn=True): + super(Attention, self).__init__() + self.ch = ch + self.theta = nn.Conv2d( + self.ch, self.ch // 8, kernel_size=1, padding=0, bias=False) + self.phi = nn.Conv2d( + self.ch, self.ch // 8, kernel_size=1, padding=0, bias=False) + self.g = nn.Conv2d( + self.ch, self.ch // 2, kernel_size=1, padding=0, bias=False) + self.o = nn.Conv2d( + self.ch // 2, self.ch, kernel_size=1, padding=0, bias=False) + if use_sn: + self.theta = SpectralNorm(self.theta) + self.phi = SpectralNorm(self.phi) + self.g = SpectralNorm(self.g) + self.o = SpectralNorm(self.o) + self.gamma = nn.Parameter(torch.tensor(0.), requires_grad=True) + + def forward(self, x, y=None): + theta = self.theta(x) + phi = F.max_pool2d(self.phi(x), [2, 2]) + g = F.max_pool2d(self.g(x), [2, 2]) + theta = theta.view(-1, self.ch // 8, x.shape[2] * x.shape[3]) + phi = phi.view(-1, self.ch // 8, x.shape[2] * x.shape[3] // 4) + g = g.view(-1, self.ch // 2, x.shape[2] * x.shape[3] // 4) + beta = F.softmax(torch.bmm(theta.transpose(1, 2), phi), -1) + o = self.o( + torch.bmm(g, beta.transpose(1, 2)).view(-1, self.ch // 2, + x.shape[2], x.shape[3])) + return self.gamma * o + x + + +class MLAttrEncoder(nn.Module): + + def __init__(self): + super(MLAttrEncoder, self).__init__() + self.conv1 = Conv4x4(3, 32) + self.conv2 = Conv4x4(32, 64) + self.conv3 = Conv4x4(64, 128) + self.conv4 = Conv4x4(128, 256) + self.conv5 = Conv4x4(256, 512) + self.conv6 = Conv4x4(512, 1024) + self.conv7 = Conv4x4(1024, 1024) + + self.deconv1 = DeConv4x4(1024, 1024) + self.deconv2 = DeConv4x4(2048, 512) + self.deconv3 = DeConv4x4(1024, 256) + self.deconv4 = DeConv4x4(512, 128) + self.deconv5 = DeConv4x4(256, 64) + self.deconv6 = DeConv4x4(128, 32) + + self.apply(init_func) + + def forward(self, Xt): + feat1 = self.conv1(Xt) + feat2 = self.conv2(feat1) + feat3 = self.conv3(feat2) + feat4 = self.conv4(feat3) + feat5 = self.conv5(feat4) + feat6 = self.conv6(feat5) + z_attr1 = self.conv7(feat6) + + z_attr2 = self.deconv1(z_attr1, feat6) + z_attr3 = self.deconv2(z_attr2, feat5) + z_attr4 = self.deconv3(z_attr3, feat4) + z_attr5 = self.deconv4(z_attr4, feat3) + z_attr6 = self.deconv5(z_attr5, feat2) + z_attr7 = self.deconv6(z_attr6, feat1) + z_attr8 = F.interpolate( + z_attr7, scale_factor=2, mode='bilinear', align_corners=True) + + return z_attr1, z_attr2, z_attr3, z_attr4, z_attr5, z_attr6, z_attr7, z_attr8 + + +class AADGenerator(nn.Module): + + def __init__(self, c_id=256): + super(AADGenerator, self).__init__() + self.up1 = nn.ConvTranspose2d( + c_id, 1024, kernel_size=2, stride=1, padding=0) + self.AADBlk1 = AAD_ResBlk(1024, 1024, 1024, c_id) + self.AADBlk2 = AAD_ResBlk(1024, 1024, 2048, c_id) + self.AADBlk3 = AAD_ResBlk(1024, 1024, 1024, c_id) + self.AADBlk4 = AAD_ResBlk(1024, 512, 512, c_id) + self.AADBlk5 = AAD_ResBlk(512, 256, 256, c_id) + self.AADBlk6 = AAD_ResBlk(256, 128, 128, c_id) + self.AADBlk7 = AAD_ResBlk(128, 64, 64, c_id) + self.AADBlk8 = AAD_ResBlk(64, 3, 64, c_id) + + self.sa = Attention(512, use_sn=True) + + self.apply(init_func) + + def forward(self, z_attr, z_id, deformation): + + m = self.up1(z_id.reshape(z_id.shape[0], -1, 1, 1)) + m2 = F.interpolate( + self.AADBlk1(m, z_attr[0], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m3 = F.interpolate( + self.AADBlk2(m2, z_attr[1], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m4 = F.interpolate( + self.AADBlk3(m3, z_attr[2], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m5 = F.interpolate( + self.AADBlk4(m4, z_attr[3], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m5 = self.sa(m5) + m6 = F.interpolate( + self.AADBlk5(m5, z_attr[4], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m7 = F.interpolate( + self.AADBlk6(m6, z_attr[5], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + m8 = F.interpolate( + self.AADBlk7(m7, z_attr[6], z_id), + scale_factor=2, + mode='bilinear', + align_corners=True) + + y = self.AADBlk8(m8, z_attr[7], z_id) + + return torch.tanh(y) + + def deform_input(self, inp, deformation): + _, h_old, w_old, _ = deformation.shape + _, _, h, w = inp.shape + if h_old != h or w_old != w: + deformation = deformation.permute(0, 3, 1, 2) + deformation = F.interpolate( + deformation, size=(h, w), mode='bilinear') + deformation = deformation.permute(0, 2, 3, 1) + return F.grid_sample(inp, deformation), deformation + + +class AEI_Net(nn.Module): + + def __init__(self, c_id=256, num_kp=17, device=torch.device('cuda')): + super(AEI_Net, self).__init__() + self.device = device + self.encoder = MLAttrEncoder() + self.generator = AADGenerator(c_id) + self.dense_motion_network = DenseMotionNetwork( + num_kp=num_kp, num_channels=3, estimate_occlusion_map=False) + + def deform_input(self, inp, deformation): + _, h_old, w_old, _ = deformation.shape + _, _, h, w = inp.shape + if h_old != h or w_old != w: + deformation = deformation.permute(0, 3, 1, 2) + deformation = F.interpolate( + deformation, size=(h, w), mode='bilinear') + deformation = deformation.permute(0, 2, 3, 1) + return F.grid_sample(inp, deformation), deformation + + def flow_change(self, x, flow): + n, c, h, w = x.size() + yv, xv = torch.meshgrid([torch.arange(h), torch.arange(w)]) + xv = xv.float() / (w - 1) * 2.0 - 1 + yv = yv.float() / (h - 1) * 2.0 - 1 + grid = torch.cat((xv.unsqueeze(-1), yv.unsqueeze(-1)), + -1).unsqueeze(0).to(self.device) + flow_delta = flow - grid + return flow_delta + + def forward(self, Xt, z_id, kp_fuse, kp_t): + output_flow = {} + dense_motion = self.dense_motion_network( + source_image=Xt, kp_driving=kp_fuse, kp_source=kp_t) + deformation = dense_motion['deformation'] + + with torch.no_grad(): + Xt_warp, _ = self.deform_input(Xt, deformation) + attr = self.encoder(Xt_warp) + + Y = self.generator(attr, z_id, deformation) + + output_flow['deformed'], flow = self.deform_input(Xt, deformation) + output_flow['flow'] = self.flow_change(Xt, flow) + + return Y, attr, output_flow + + def get_attr(self, X): + return self.encoder(X) diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/bfm.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/bfm.py new file mode 100644 index 0000000000000000000000000000000000000000..fe6a42e50279f2f01cd7b17d7605d4834dd5ce54 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/bfm.py @@ -0,0 +1,249 @@ +# The implementation is adopted from Deep3DFaceRecon_pytorch, made publicly available under the MIT License +# at https://github.com/sicxu/Deep3DFaceRecon_pytorch/blob/master/models/bfm.py +import os + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.io import loadmat + + +def perspective_projection(focal, center): + return np.array([focal, 0, center, 0, focal, center, 0, 0, + 1]).reshape([3, 3]).astype(np.float32).transpose() + + +class SH: + + def __init__(self): + self.a = [np.pi, 2 * np.pi / np.sqrt(3.), 2 * np.pi / np.sqrt(8.)] + self.c = [ + 1 / np.sqrt(4 * np.pi), + np.sqrt(3.) / np.sqrt(4 * np.pi), + 3 * np.sqrt(5.) / np.sqrt(12 * np.pi) + ] + + +class ParametricFaceModel(): + + def __init__(self, + bfm_folder='./BFM', + recenter=True, + camera_distance=10., + init_lit=np.array([0.8, 0, 0, 0, 0, 0, 0, 0, 0]), + focal=1015., + center=112., + is_train=True, + default_name='BFM_model_front.mat'): + + model = loadmat(os.path.join(bfm_folder, default_name)) + # mean face shape. [3*N,1] + self.mean_shape = model['meanshape'].astype(np.float32) + # identity basis. [3*N,80] + self.id_base = model['idBase'].astype(np.float32) + # expression basis. [3*N,64] + self.exp_base = model['exBase'].astype(np.float32) + # mean face texture. [3*N,1] (0-255) + self.mean_tex = model['meantex'].astype(np.float32) + # texture basis. [3*N,80] + self.tex_base = model['texBase'].astype(np.float32) + # face indices for each vertex that lies in. starts from 0. [N,8] + self.point_buf = model['point_buf'].astype(np.int64) - 1 + # vertex indices for each face. starts from 0. [F,3] + self.face_buf = model['tri'].astype(np.int64) - 1 + # vertex indices for 68 landmarks. starts from 0. [68,1] + self.keypoints = np.squeeze(model['keypoints']).astype(np.int64) - 1 + + if is_train: + # vertex indices for small face region to compute photometric error. starts from 0. + self.front_mask = np.squeeze(model['frontmask2_idx']).astype( + np.int64) - 1 + # vertex indices for each face from small face region. starts from 0. [f,3] + self.front_face_buf = model['tri_mask2'].astype(np.int64) - 1 + # vertex indices for pre-defined skin region to compute reflectance loss + self.skin_mask = np.squeeze(model['skinmask']) + + if recenter: + mean_shape = self.mean_shape.reshape([-1, 3]) + mean_shape = mean_shape - np.mean( + mean_shape, axis=0, keepdims=True) + self.mean_shape = mean_shape.reshape([-1, 1]) + + self.persc_proj = perspective_projection(focal, center) + self.device = 'cpu' + self.camera_distance = camera_distance + self.SH = SH() + self.init_lit = init_lit.reshape([1, 1, -1]).astype(np.float32) + + def to(self, device): + self.device = device + for key, value in self.__dict__.items(): + if type(value).__module__ == np.__name__: + setattr(self, key, torch.tensor(value).to(device)) + + def compute_shape(self, id_coeff, exp_coeff): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) + + Parameters: + id_coeff -- torch.tensor, size (B, 80), identity coeffs + exp_coeff -- torch.tensor, size (B, 64), expression coeffs + """ + batch_size = id_coeff.shape[0] + id_part = torch.einsum('ij,aj->ai', self.id_base, id_coeff) + exp_part = torch.einsum('ij,aj->ai', self.exp_base, exp_coeff) + face_shape = id_part + exp_part + self.mean_shape.reshape([1, -1]) + return face_shape.reshape([batch_size, -1, 3]) + + def compute_texture(self, tex_coeff, normalize=True): + """ + Return: + face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.) + + Parameters: + tex_coeff -- torch.tensor, size (B, 80) + """ + batch_size = tex_coeff.shape[0] + face_texture = torch.einsum('ij,aj->ai', self.tex_base, + tex_coeff) + self.mean_tex + if normalize: + face_texture = face_texture / 255. + return face_texture.reshape([batch_size, -1, 3]) + + def compute_norm(self, face_shape): + """ + Return: + vertex_norm -- torch.tensor, size (B, N, 3) + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + v1 = face_shape[:, self.face_buf[:, 0]] + v2 = face_shape[:, self.face_buf[:, 1]] + v3 = face_shape[:, self.face_buf[:, 2]] + e1 = v1 - v2 + e2 = v2 - v3 + face_norm = torch.cross(e1, e2, dim=-1) + face_norm = F.normalize(face_norm, dim=-1, p=2) + face_norm = torch.cat( + [face_norm, + torch.zeros(face_norm.shape[0], 1, 3).to(self.device)], + dim=1) + + vertex_norm = torch.sum(face_norm[:, self.point_buf], dim=2) + vertex_norm = F.normalize(vertex_norm, dim=-1, p=2) + return vertex_norm + + def compute_color(self, face_texture, face_norm, gamma): + batch_size = gamma.shape[0] + a, c = self.SH.a, self.SH.c + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + self.init_lit + gamma = gamma.permute(0, 2, 1) + face_norm_p1 = face_norm[..., :1] + face_norm_p2 = face_norm[..., 1:2] + face_norm_p3 = face_norm[..., 2:] + face_norm_diff = face_norm_p1**2 - face_norm_p2**2 + temp = [ + a[0] * c[0] * torch.ones_like(face_norm_p1).to(self.device), + -a[1] * c[1] * face_norm_p2, a[1] * c[1] * face_norm_p3, + -a[1] * c[1] * face_norm_p1, + a[2] * c[2] * face_norm_p1 * face_norm_p2, + -a[2] * c[2] * face_norm_p2 * face_norm_p3, + 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm_p3**2 - 1), + -a[2] * c[2] * face_norm_p1 * face_norm_p3, + 0.5 * a[2] * c[2] * face_norm_diff + ] + + Y = torch.cat(temp, dim=-1) + r = Y @ gamma[..., :1] + g = Y @ gamma[..., 1:2] + b = Y @ gamma[..., 2:] + face_color = torch.cat([r, g, b], dim=-1) * face_texture + return face_color + + def compute_rotation(self, angles): + batch_size = angles.shape[0] + ones = torch.ones([batch_size, 1]).to(self.device) + zeros = torch.zeros([batch_size, 1]).to(self.device) + x, y, z = angles[:, :1], angles[:, 1:2], angles[:, 2:], + + temp_x = [ + ones, zeros, zeros, zeros, + torch.cos(x), -torch.sin(x), zeros, + torch.sin(x), + torch.cos(x) + ] + rot_x = torch.cat(temp_x, dim=1).reshape([batch_size, 3, 3]) + + temp_y = [ + torch.cos(y), zeros, + torch.sin(y), zeros, ones, zeros, -torch.sin(y), zeros, + torch.cos(y) + ] + rot_y = torch.cat(temp_y, dim=1).reshape([batch_size, 3, 3]) + + temp_z = [ + torch.cos(z), -torch.sin(z), zeros, + torch.sin(z), + torch.cos(z), zeros, zeros, zeros, ones + ] + rot_z = torch.cat(temp_z, dim=1).reshape([batch_size, 3, 3]) + + rot = rot_z @ rot_y @ rot_x + return rot.permute(0, 2, 1) + + def to_camera(self, face_shape): + face_shape[..., -1] = self.camera_distance - face_shape[..., -1] + return face_shape + + def to_image(self, face_shape): + # to image_plane + face_proj = face_shape @ self.persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + def transform(self, face_shape, rot, trans): + return face_shape @ rot + trans.unsqueeze(1) + + def get_landmarks(self, face_proj): + return face_proj[:, self.keypoints] + + def split_coeff(self, coeffs): + id_coeffs = coeffs[:, :80] + exp_coeffs = coeffs[:, 80:144] + tex_coeffs = coeffs[:, 144:224] + angles = coeffs[:, 224:227] + gammas = coeffs[:, 227:254] + translations = coeffs[:, 254:] + return { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + + def compute_for_render(self, coeffs): + coef_dict = self.split_coeff(coeffs) + face_shape = self.compute_shape(coef_dict['id'], coef_dict['exp']) + rotation = self.compute_rotation(coef_dict['angle']) + + face_shape_transformed = self.transform(face_shape, rotation, + coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_texture = self.compute_texture(coef_dict['tex']) + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + face_color = self.compute_color(face_texture, face_norm_roted, + coef_dict['gamma']) + + return face_vertex, face_texture, face_color, landmark diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/dense_motion.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/dense_motion.py new file mode 100644 index 0000000000000000000000000000000000000000..ba5cc2db24b39e1414313de9e27d261034246cf0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/dense_motion.py @@ -0,0 +1,376 @@ +# The implementation is adopted from first-order-model, made publicly available under the MIT License +# at https://github.com/AliaksandrSiarohin/first-order-model/blob/master/modules/dense_motion.py + +import torch +import torch.nn.functional as F +from torch import nn + + +def kp2gaussian(kp, spatial_size, kp_variance): + """ + Transform a keypoint into gaussian like representation + """ + mean = kp['value'] + + coordinate_grid = make_coordinate_grid(spatial_size, mean.type()) + number_of_leading_dimensions = len(mean.shape) - 1 + shape = (1, ) * number_of_leading_dimensions + coordinate_grid.shape + coordinate_grid = coordinate_grid.view(*shape) + repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1) + coordinate_grid = coordinate_grid.repeat(*repeats) + + # Preprocess kp shape + shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 2) + mean = mean.view(*shape) + + mean_sub = (coordinate_grid - mean) + + out = torch.exp(-0.5 * (mean_sub**2).sum(-1) / kp_variance) + + return out + + +def make_coordinate_grid(spatial_size, type): + """ + Create a meshgrid [-1,1] x [-1,1] of given spatial_size. + """ + h, w = spatial_size + x = torch.arange(w).type(type) + y = torch.arange(h).type(type) + + x = (2 * (x / (w - 1)) - 1) + y = (2 * (y / (h - 1)) - 1) + + yy = y.view(-1, 1).repeat(1, w) + xx = x.view(1, -1).repeat(h, 1) + + meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2) + + return meshed + + +class UpBlock2d(nn.Module): + """ + Upsampling block for use in decoder. + """ + + def __init__(self, + in_features, + out_features, + kernel_size=3, + padding=1, + groups=1): + super(UpBlock2d, self).__init__() + + self.conv = nn.Conv2d( + in_channels=in_features, + out_channels=out_features, + kernel_size=kernel_size, + padding=padding, + groups=groups) + self.norm = nn.BatchNorm2d(out_features, affine=True) + + def forward(self, x): + out = F.interpolate(x, scale_factor=2) + out = self.conv(out) + out = self.norm(out) + out = F.relu(out) + return out + + +class DownBlock2d(nn.Module): + """ + Downsampling block for use in encoder. + """ + + def __init__(self, + in_features, + out_features, + kernel_size=3, + padding=1, + groups=1): + super(DownBlock2d, self).__init__() + self.conv = nn.Conv2d( + in_channels=in_features, + out_channels=out_features, + kernel_size=kernel_size, + padding=padding, + groups=groups) + self.norm = nn.BatchNorm2d(out_features, affine=True) + self.pool = nn.AvgPool2d(kernel_size=(2, 2)) + + def forward(self, x): + out = self.conv(x) + out = self.norm(out) + out = F.relu(out) + out = self.pool(out) + return out + + +class Encoder(nn.Module): + """ + Hourglass Encoder + """ + + def __init__(self, + block_expansion, + in_features, + num_blocks=3, + max_features=256): + super(Encoder, self).__init__() + + down_blocks = [] + for i in range(num_blocks): + down_blocks.append( + DownBlock2d( + in_features if i == 0 else min(max_features, + block_expansion * (2**i)), + min(max_features, block_expansion * (2**(i + 1))), + kernel_size=3, + padding=1)) + self.down_blocks = nn.ModuleList(down_blocks) + + def forward(self, x): + outs = [x] + for down_block in self.down_blocks: + outs.append(down_block(outs[-1])) + return outs + + +class Decoder(nn.Module): + """ + Hourglass Decoder + """ + + def __init__(self, + block_expansion, + in_features, + num_blocks=3, + max_features=256): + super(Decoder, self).__init__() + + up_blocks = [] + + for i in range(num_blocks)[::-1]: + in_filters = (1 if i == num_blocks - 1 else 2) * min( + max_features, block_expansion * (2**(i + 1))) + out_filters = min(max_features, block_expansion * (2**i)) + up_blocks.append( + UpBlock2d(in_filters, out_filters, kernel_size=3, padding=1)) + + self.up_blocks = nn.ModuleList(up_blocks) + self.out_filters = block_expansion + in_features + + def forward(self, x): + out = x.pop() + for up_block in self.up_blocks: + out = up_block(out) + skip = x.pop() + out = torch.cat([out, skip], dim=1) + return out + + +class Hourglass(nn.Module): + """ + Hourglass architecture. + """ + + def __init__(self, + block_expansion, + in_features, + num_blocks=3, + max_features=256): + super(Hourglass, self).__init__() + self.encoder = Encoder(block_expansion, in_features, num_blocks, + max_features) + self.decoder = Decoder(block_expansion, in_features, num_blocks, + max_features) + self.out_filters = self.decoder.out_filters + + def forward(self, x): + return self.decoder(self.encoder(x)) + + +class AntiAliasInterpolation2d(nn.Module): + """ + Band-limited downsampling, for better preservation of the input signal. + """ + + def __init__(self, channels, scale): + super(AntiAliasInterpolation2d, self).__init__() + sigma = (1 / scale - 1) / 2 + kernel_size = 2 * round(sigma * 4) + 1 + self.ka = kernel_size // 2 + self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka + + kernel_size = [kernel_size, kernel_size] + sigma = [sigma, sigma] + # The gaussian kernel is the product of the + # gaussian function of each dimension. + kernel = 1 + meshgrids = torch.meshgrid( + [torch.arange(size, dtype=torch.float32) for size in kernel_size]) + for size, std, mgrid in zip(kernel_size, sigma, meshgrids): + mean = (size - 1) / 2 + kernel *= torch.exp(-(mgrid - mean)**2 / (2 * std**2)) + + # Make sure sum of values in gaussian kernel equals 1. + kernel = kernel / torch.sum(kernel) + # Reshape to depthwise convolutional weight + kernel = kernel.view(1, 1, *kernel.size()) + kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) + + self.register_buffer('weight', kernel) + self.groups = channels + self.scale = scale + inv_scale = 1 / scale + self.int_inv_scale = int(inv_scale) + + def forward(self, input): + if self.scale == 1.0: + return input + + out = F.pad(input, (self.ka, self.kb, self.ka, self.kb)) + out = F.conv2d(out, weight=self.weight, groups=self.groups) + out = out[:, :, ::self.int_inv_scale, ::self.int_inv_scale] + + return out + + +class DenseMotionNetwork(nn.Module): + """ + Module that predicting a dense motion from sparse motion representation given by kp_source and kp_driving + """ + + def __init__(self, + num_kp, + num_channels, + estimate_occlusion_map=False, + kp_variance=0.01): + super(DenseMotionNetwork, self).__init__() + block_expansion = 64 + num_blocks = 5 + max_features = 1024 + scale_factor = 0.25 + + self.hourglass = Hourglass( + block_expansion=block_expansion, + in_features=(num_kp + 1) * (num_channels + 1), + max_features=max_features, + num_blocks=num_blocks) + + self.mask = nn.Conv2d( + self.hourglass.out_filters, + num_kp + 1, + kernel_size=(7, 7), + padding=(3, 3)) + + if estimate_occlusion_map: + self.occlusion = nn.Conv2d( + self.hourglass.out_filters, + 1, + kernel_size=(7, 7), + padding=(3, 3)) + else: + self.occlusion = None + + self.num_kp = num_kp + self.scale_factor = scale_factor + self.kp_variance = kp_variance + + if self.scale_factor != 1: + self.down = AntiAliasInterpolation2d(num_channels, + self.scale_factor) + + def create_heatmap_representations(self, source_image, kp_driving, + kp_source): + """ + Eq 6. in the paper H_k(z) + """ + spatial_size = source_image.shape[2:] + gaussian_driving = kp2gaussian( + kp_driving, + spatial_size=spatial_size, + kp_variance=self.kp_variance) + gaussian_source = kp2gaussian( + kp_source, spatial_size=spatial_size, kp_variance=self.kp_variance) + heatmap = gaussian_driving - gaussian_source + + zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], + spatial_size[1]).type(heatmap.type()) + heatmap = torch.cat([zeros, heatmap], dim=1) + heatmap = heatmap.unsqueeze(2) + return heatmap + + def create_sparse_motions(self, source_image, kp_driving, kp_source): + """ + Eq 4. in the paper T_{s<-d}(z) + """ + bs, _, h, w = source_image.shape + identity_grid = make_coordinate_grid((h, w), + type=kp_source['value'].type()) + identity_grid = identity_grid.view(1, 1, h, w, 2) + coordinate_grid = identity_grid - kp_driving['value'].view( + bs, self.num_kp, 1, 1, 2) + if 'jacobian' in kp_driving: + jacobian = torch.matmul(kp_source['jacobian'], + torch.inverse(kp_driving['jacobian'])) + jacobian = jacobian.unsqueeze(-3).unsqueeze(-3) + jacobian = jacobian.repeat(1, 1, h, w, 1, 1) + coordinate_grid = torch.matmul(jacobian, + coordinate_grid.unsqueeze(-1)) + coordinate_grid = coordinate_grid.squeeze(-1) + + driving_to_source = coordinate_grid + kp_source['value'].view( + bs, self.num_kp, 1, 1, 2) + + identity_grid = identity_grid.repeat(bs, 1, 1, 1, 1) + sparse_motions = torch.cat([identity_grid, driving_to_source], dim=1) + return sparse_motions + + def create_deformed_source_image(self, source_image, sparse_motions): + bs, _, h, w = source_image.shape + source_repeat = source_image.unsqueeze(1).unsqueeze(1).repeat( + 1, self.num_kp + 1, 1, 1, 1, 1) + temp_dim = bs * (self.num_kp + 1) + source_repeat = source_repeat.view(temp_dim, -1, h, w) + sparse_motions = sparse_motions.view((temp_dim, h, w, -1)) + sparse_deformed = F.grid_sample(source_repeat, sparse_motions) + sparse_deformed = sparse_deformed.view((bs, self.num_kp + 1, -1, h, w)) + return sparse_deformed + + def forward(self, source_image, kp_driving, kp_source): + if self.scale_factor != 1: + source_image = self.down(source_image) + + bs, _, h, w = source_image.shape + + out_dict = dict() + heatmap_representation = self.create_heatmap_representations( + source_image, kp_driving, kp_source) + sparse_motion = self.create_sparse_motions(source_image, kp_driving, + kp_source) + deformed_source = self.create_deformed_source_image( + source_image, sparse_motion) + out_dict['sparse_deformed'] = deformed_source + + input = torch.cat([heatmap_representation, deformed_source], dim=2) + input = input.view(bs, -1, h, w) + + prediction = self.hourglass(input) + + mask = self.mask(prediction) + mask = F.softmax(mask, dim=1) + out_dict['mask'] = mask + mask = mask.unsqueeze(2) + sparse_motion = sparse_motion.permute(0, 1, 4, 2, 3) + deformation = (sparse_motion * mask).sum(dim=1) + deformation = deformation.permute(0, 2, 3, 1) + + out_dict['deformation'] = deformation + + if self.occlusion: + occlusion_map = torch.sigmoid(self.occlusion(prediction)) + out_dict['occlusion_map'] = occlusion_map + + return out_dict diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/facerecon_model.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/facerecon_model.py new file mode 100644 index 0000000000000000000000000000000000000000..53b34d3c2c9a3acc5d170601154ef283f08f9360 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/facerecon_model.py @@ -0,0 +1,559 @@ +# The implementation is adopted from Deep3DFaceRecon_pytorch, made publicly available under the MIT License +# at https://github.com/sicxu/Deep3DFaceRecon_pytorch/blob/master/models/networks.py + +import os +from typing import Any, Callable, List, Optional, Type, Union + +import torch +import torch.nn as nn +from torch import Tensor +from torch.optim import lr_scheduler + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + + +def filter_state_dict(state_dict, remove_name='fc'): + new_state_dict = {} + for key in state_dict: + if remove_name in key: + continue + new_state_dict[key] = state_dict[key] + return new_state_dict + + +def get_scheduler(optimizer, opt): + """Return a learning rate scheduler + + Parameters: + optimizer -- the optimizer of the network + opt (option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions.  + opt.lr_policy is the name of learning rate policy: linear | step | plateau | cosine + + For other schedulers (step, plateau, and cosine), we use the default PyTorch schedulers. + See https://pytorch.org/docs/stable/optim.html for more details. + """ + if opt.lr_policy == 'linear': + + def lambda_rule(epoch): + lr_l = 1.0 - max(0, epoch + opt.epoch_count + - opt.n_epochs) / float(opt.n_epochs + 1) + return lr_l + + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) + elif opt.lr_policy == 'step': + scheduler = lr_scheduler.StepLR( + optimizer, step_size=opt.lr_decay_epochs, gamma=0.2) + elif opt.lr_policy == 'plateau': + scheduler = lr_scheduler.ReduceLROnPlateau( + optimizer, mode='min', factor=0.2, threshold=0.01, patience=5) + elif opt.lr_policy == 'cosine': + scheduler = lr_scheduler.CosineAnnealingLR( + optimizer, T_max=opt.n_epochs, eta_min=0) + else: + return NotImplementedError( + 'learning rate policy [%s] is not implemented', opt.lr_policy) + return scheduler + + +def define_net_recon(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper( + net_recon, use_last_fc=use_last_fc, init_path=init_path) + + +class ReconNetWrapper(nn.Module): + fc_dim = 257 + + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', + net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict( + torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print('loading init net_recon %s from %s' % (net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 64, bias=True), # exp layer + conv1x1(last_dim, 80, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +# adapted from https://github.com/pytorch/vision/edit/master/torchvision/models/resnet.py +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + +model_urls = { + 'resnet18': + 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': + 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': + 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': + 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': + 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': + 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': + 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': + 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': + 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes: int, + out_planes: int, + stride: int = 1, + groups: int = 1, + dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes: int, + out_planes: int, + stride: int = 1, + bias: bool = False) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=bias) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + use_last_fc: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.use_last_fc = use_last_fc + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + if self.use_last_fc: + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, + 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, + 0) # type: ignore[arg-type] + + def _make_layer(self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + if self.use_last_fc: + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet(arch: str, block: Type[Union[BasicBlock, + Bottleneck]], layers: List[int], + pretrained: bool, progress: bool, **kwargs: Any) -> ResNet: + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url( + model_urls[arch], progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained: bool = False, + progress: bool = True, + **kwargs: Any) -> ResNet: + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +func_dict = {'resnet18': (resnet18, 512), 'resnet50': (resnet50, 2048)} diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/model_irse.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/model_irse.py new file mode 100644 index 0000000000000000000000000000000000000000..4e212bd099fdd2d02449e044e1b50a02880846f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/model_irse.py @@ -0,0 +1,249 @@ +# The implementation is adopted from face.evoLVe, made publicly available under the MIT License +# at https://github.com/ZhaoJ9014/face.evoLVe/blob/master/backbone/model_irse.py +from collections import namedtuple + +import torch +import torch.nn as nn +from torch.nn import (AdaptiveAvgPool2d, BatchNorm1d, BatchNorm2d, Conv2d, + Dropout, Linear, MaxPool2d, Module, PReLU, ReLU, + Sequential, Sigmoid) + + +class Flatten(Module): + + def forward(self, input): + return input.view(input.size(0), -1) + + +def l2_norm(input, axis=1): + norm = torch.norm(input, 2, axis, True) + output = torch.div(input, norm) + + return output + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + + nn.init.xavier_uniform_(self.fc1.weight.data) + + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + + return module_input * x + + +class bottleneck_IR(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + + return res + shortcut + + +class bottleneck_IR_SE(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR_SE, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth), SEModule(depth, 16)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + + return res + shortcut + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + '''A named tuple describing a ResNet block.''' + + +def get_block(in_channel, depth, num_units, stride=2): + + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + + return blocks + + +class Backbone(Module): + + def __init__(self, input_size, num_layers, mode='ir'): + super(Backbone, self).__init__() + assert input_size[0] in [ + 112, 224 + ], 'input_size should be [112, 112] or [224, 224]' + assert num_layers in [50, 100, + 152], 'num_layers should be 50, 100 or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = bottleneck_IR + elif mode == 'ir_se': + unit_module = bottleneck_IR_SE + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + if input_size[0] == 112: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(0.4), Flatten(), + Linear(512 * 7 * 7, 512), BatchNorm1d(512, affine=False)) + else: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(0.4), Flatten(), + Linear(512 * 14 * 14, 512), BatchNorm1d(512, affine=False)) + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + self._initialize_weights() + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + conv_out = x.view(x.shape[0], -1) + x = self.output_layer(x) + + return l2_norm(x), conv_out + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + + +def IR_50(input_size): + """Constructs a ir-50 model. + """ + model = Backbone(input_size, 50, 'ir') + + return model + + +def IR_101(input_size): + """Constructs a ir-101 model. + """ + model = Backbone(input_size, 100, 'ir') + + return model + + +def IR_152(input_size): + """Constructs a ir-152 model. + """ + model = Backbone(input_size, 152, 'ir') + + return model + + +def IR_SE_50(input_size): + """Constructs a ir_se-50 model. + """ + model = Backbone(input_size, 50, 'ir_se') + + return model + + +def IR_SE_101(input_size): + """Constructs a ir_se-101 model. + """ + model = Backbone(input_size, 100, 'ir_se') + + return model + + +def IR_SE_152(input_size): + """Constructs a ir_se-152 model. + """ + model = Backbone(input_size, 152, 'ir_se') + + return model diff --git a/runtime/Lib/site_packages/models/cv/image_face_fusion/network/ops.py b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..d168e69124199636e74d8cd8ba563901ffa0ea92 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_face_fusion/network/ops.py @@ -0,0 +1,211 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import Parameter + + +def init_func(m, init_type='xavier', gain=0.02): + classname = m.__class__.__name__ + if classname.find('BatchNorm2d') != -1: + if hasattr(m, 'weight') and m.weight is not None: + nn.init.normal_(m.weight, 1.0, gain) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias, 0.0) + elif hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + if init_type == 'normal': + nn.init.normal_(m.weight, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight, gain=gain) + elif init_type == 'xavier_uniform': + nn.init.xavier_uniform_(m.weight, gain=1.0) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight, gain=gain) + elif init_type == 'none': # uses pytorch's default init method + m.reset_parameters() + else: + raise NotImplementedError( + 'initialization method [%s] is not implemented' % init_type) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias, 0.0) + elif hasattr(m, 'weight_bar') and (classname.find('Conv') != -1): + if init_type == 'normal': + nn.init.normal_(m.weight_bar, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight_bar, gain=gain) + elif init_type == 'xavier_uniform': + nn.init.xavier_uniform_(m.weight_bar, gain=1.0) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight_bar, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight_bar, gain=gain) + elif init_type == 'none': # uses pytorch's default init method + m.reset_parameters() + else: + raise NotImplementedError( + 'initialization method [%s] is not implemented' % init_type) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias, 0.0) + + +def l2normalize(v, eps=1e-12): + return v / (v.norm() + eps) + + +class SpectralNorm(nn.Module): + + def __init__(self, module, name='weight', power_iterations=1): + super(SpectralNorm, self).__init__() + self.module = module + self.name = name + self.power_iterations = power_iterations + self._make_params() + + def _update_u_v(self): + u = getattr(self.module, self.name + '_u') + v = getattr(self.module, self.name + '_v') + w = getattr(self.module, self.name + '_bar') + + height = w.data.shape[0] + for _ in range(self.power_iterations): + v.data = l2normalize( + torch.mv(torch.t(w.view(height, -1).data), u.data)) + u.data = l2normalize(torch.mv(w.view(height, -1).data, v.data)) + + sigma = u.dot(w.view(height, -1).mv(v)) + setattr(self.module, self.name, w / sigma.expand_as(w)) + + def _noupdate_u_v(self): + u = getattr(self.module, self.name + '_u') + v = getattr(self.module, self.name + '_v') + w = getattr(self.module, self.name + '_bar') + + height = w.data.shape[0] + sigma = u.dot(w.view(height, -1).mv(v)) + setattr(self.module, self.name, w / sigma.expand_as(w)) + + def _make_params(self): + w = getattr(self.module, self.name) + + height = w.data.shape[0] + width = w.view(height, -1).data.shape[1] + + u = Parameter(w.data.new(height).normal_(0, 1), requires_grad=False) + v = Parameter(w.data.new(width).normal_(0, 1), requires_grad=False) + u.data = l2normalize(u.data) + v.data = l2normalize(v.data) + w_bar = Parameter(w.data) + + del self.module._parameters[self.name] + + self.module.register_parameter(self.name + '_u', u) + self.module.register_parameter(self.name + '_v', v) + self.module.register_parameter(self.name + '_bar', w_bar) + + def forward(self, *args): + if self.module.training: + self._update_u_v() + else: + self._noupdate_u_v() + return self.module.forward(*args) + + +def convert_affinematrix_to_homography(A): + H = torch.nn.functional.pad(A, [0, 0, 0, 1], 'constant', value=0.0) + H[..., -1, -1] += 1.0 + return H + + +def normal_transform_pixel(height, width, eps=1e-14): + tr_mat = torch.tensor([[1.0, 0.0, -1.0], [0.0, 1.0, -1.0], [0.0, 0.0, + 1.0]]) # 3x3 + + # prevent divide by zero bugs + width_denom = eps if width == 1 else width - 1.0 + height_denom = eps if height == 1 else height - 1.0 + + tr_mat[0, 0] = tr_mat[0, 0] * 2.0 / width_denom + tr_mat[1, 1] = tr_mat[1, 1] * 2.0 / height_denom + + return tr_mat.unsqueeze(0) # 1x3x3 + + +def _torch_inverse_cast(input): + if not isinstance(input, torch.Tensor): + raise AssertionError( + f'Input must be torch.Tensor. Got: {type(input)}.') + dtype = input.dtype + if dtype not in (torch.float32, torch.float64): + dtype = torch.float32 + return torch.inverse(input.to(dtype)).to(input.dtype) + + +def normalize_homography(dst_pix_trans_src_pix, dsize_src, dsize_dst): + if not isinstance(dst_pix_trans_src_pix, torch.Tensor): + raise TypeError( + f'Input type is not a torch.Tensor. Got {type(dst_pix_trans_src_pix)}' + ) + + if not (len(dst_pix_trans_src_pix.shape) == 3 + or dst_pix_trans_src_pix.shape[-2:] == (3, 3)): + raise ValueError( + f'Input dst_pix_trans_src_pix must be a Bx3x3 tensor. Got {dst_pix_trans_src_pix.shape}' + ) + + # source and destination sizes + src_h, src_w = dsize_src + dst_h, dst_w = dsize_dst + + # compute the transformation pixel/norm for src/dst + src_norm_trans_src_pix: torch.Tensor = normal_transform_pixel( + src_h, src_w).to(dst_pix_trans_src_pix) + + src_pix_trans_src_norm = _torch_inverse_cast(src_norm_trans_src_pix) + dst_norm_trans_dst_pix = normal_transform_pixel( + dst_h, dst_w).to(dst_pix_trans_src_pix) + + # compute chain transformations + dst_norm_trans_src_norm = dst_norm_trans_dst_pix @ ( + dst_pix_trans_src_pix @ src_pix_trans_src_norm) + return dst_norm_trans_src_norm + + +def warp_affine_torch(src, + M, + dsize, + mode='bilinear', + padding_mode='zeros', + align_corners=True): + + if not isinstance(src, torch.Tensor): + raise TypeError( + f'Input src type is not a torch.Tensor. Got {type(src)}') + + if not isinstance(M, torch.Tensor): + raise TypeError(f'Input M type is not a torch.Tensor. Got {type(M)}') + + if not len(src.shape) == 4: + raise ValueError( + f'Input src must be a BxCxHxW tensor. Got {src.shape}') + + if not (len(M.shape) == 3 or M.shape[-2:] == (2, 3)): + raise ValueError(f'Input M must be a Bx2x3 tensor. Got {M.shape}') + + B, C, H, W = src.size() + + # we generate a 3x3 transformation matrix from 2x3 affine + M_3x3 = convert_affinematrix_to_homography(M) + dst_norm_trans_src_norm = normalize_homography(M_3x3, (H, W), dsize) + src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm) + grid = F.affine_grid( + src_norm_trans_dst_norm[:, :2, :], [B, C, dsize[0], dsize[1]], + align_corners=align_corners) + return F.grid_sample( + src, + grid, + align_corners=align_corners, + mode=mode, + padding_mode=padding_mode) diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/__init__.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..787c0353d57c4e12005e0e7393b73a4bdd177059 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .m2fp_net import M2FP + from parsing_utils import center_to_target_size_test +else: + _import_structure = { + 'm2fp_net': ['M2FP'], + 'parsing_utils': ['center_to_target_size_test'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..47bf7f728de077e30fe1d39942e4a105022cdab1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .deeplab_resnet import build_resnet_deeplab_backbone + +else: + _import_structure = { + 'deeplab_resnet': ['build_resnet_deeplab_backbone'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/deeplab_resnet.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/deeplab_resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..d8f890b8c0e07da905550654b9807c4f0f8502ae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/backbone/deeplab_resnet.py @@ -0,0 +1,377 @@ +# Part of the implementation is borrowed and modified from Detectron2, publicly available at +# https://github.com/facebookresearch/detectron2/blob/main/projects/DeepLab/deeplab/resnet.py + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +def get_norm(norm, out_channels): + if norm is None: + return None + if isinstance(norm, str): + if len(norm) == 0: + return None + norm = { + 'BN': torch.nn.BatchNorm2d, + 'GN': lambda channels: nn.GroupNorm(32, channels), + 'nnSyncBN': nn.SyncBatchNorm, + }[norm] + return norm(out_channels) + + +class BasicBlock(nn.Module): + + def __init__(self, in_channels, out_channels, *, stride=1, norm='BN'): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels)) + else: + self.shortcut = None + + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + norm=get_norm(norm, out_channels)) + + self.conv2 = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels)) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + out = self.conv2(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BottleneckBlock(nn.Module): + + def __init__(self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm='BN', + stride_in_1x1=False, + dilation=1): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels)) + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + norm=get_norm(norm, bottleneck_channels)) + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels)) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + out = self.conv2(out) + out = F.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class DeepLabStem(nn.Module): + + def __init__(self, in_channels=3, out_channels=128, norm='BN'): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = 4 + self.conv1 = Conv2d( + in_channels, + out_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False, + norm=get_norm(norm, out_channels // 2)) + self.conv2 = Conv2d( + out_channels // 2, + out_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels // 2)) + self.conv3 = Conv2d( + out_channels // 2, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels)) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = self.conv2(x) + x = F.relu_(x) + x = self.conv3(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +class DeeplabResNet(nn.Module): + + def __init__(self, stem, stages, num_classes=None, out_features=None): + super().__init__() + self.stem = stem + self.num_classes = num_classes + + current_stride = self.stem.stride + self._out_feature_strides = {'stem': current_stride} + self._out_feature_channels = {'stem': self.stem.out_channels} + + self.stage_names, self.stages = [], [] + + if out_features is not None: + num_stages = max([{ + 'res2': 1, + 'res3': 2, + 'res4': 3, + 'res5': 4 + }.get(f, 0) for f in out_features]) + stages = stages[:num_stages] + for i, blocks in enumerate(stages): + assert len(blocks) > 0, len(blocks) + for block in blocks: + assert isinstance(block, nn.Module), block + + name = 'res' + str(i + 2) + stage = nn.Sequential(*blocks) + + self.add_module(name, stage) + self.stage_names.append(name) + self.stages.append(stage) + + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in blocks])) + self._out_feature_channels[name] = curr_channels = blocks[ + -1].out_channels + self.stage_names = tuple( + self.stage_names) # Make it static for scripting + + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.linear = nn.Linear(curr_channels, num_classes) + nn.init.normal_(self.linear.weight, std=0.01) + name = 'linear' + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, 'Available children: {}'.format( + ', '.join(children)) + + def forward(self, x): + assert x.dim( + ) == 4, f'ResNet takes an input of shape (N, C, H, W). Got {x.shape} instead!' + outputs = {} + x = self.stem(x) + if 'stem' in self._out_features: + outputs['stem'] = x + for name, stage in zip(self.stage_names, self.stages): + x = stage(x) + if name in self._out_features: + outputs[name] = x + if self.num_classes is not None: + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.linear(x) + if 'linear' in self._out_features: + outputs['linear'] = x + return outputs + + def output_shape(self): + return { + name: dict( + channels=self._out_feature_channels[name], + stride=self._out_feature_strides[name]) + for name in self._out_features + } + + @property + def size_divisibility(self) -> int: + return 0 + + @staticmethod + def make_stage(block_class, num_blocks, *, in_channels, out_channels, + **kwargs): + blocks = [] + for i in range(num_blocks): + curr_kwargs = {} + for k, v in kwargs.items(): + if k.endswith('_per_block'): + assert len(v) == num_blocks, ( + f"Argument '{k}' of make_stage should have the " + f'same length as num_blocks={num_blocks}.') + newk = k[:-len('_per_block')] + assert newk not in kwargs, f'Cannot call make_stage with both {k} and {newk}!' + curr_kwargs[newk] = v[i] + else: + curr_kwargs[k] = v + + blocks.append( + block_class( + in_channels=in_channels, + out_channels=out_channels, + **curr_kwargs)) + in_channels = out_channels + return blocks + + +def build_resnet_deeplab_backbone(out_features, depth, num_groups, + width_per_group, norm, stem_out_channels, + res2_out_channels, stride_in_1x1, + res4_dilation, res5_dilation, + res5_multi_grid, input_shape): + stem = DeepLabStem( + in_channels=input_shape['channels'], + out_channels=stem_out_channels, + norm=norm) + bottleneck_channels = num_groups * width_per_group + in_channels = stem_out_channels + out_channels = res2_out_channels + + assert res4_dilation in { + 1, 2 + }, 'res4_dilation cannot be {}.'.format(res4_dilation) + assert res5_dilation in { + 1, 2, 4 + }, 'res5_dilation cannot be {}.'.format(res5_dilation) + if res4_dilation == 2: + # Always dilate res5 if res4 is dilated. + assert res5_dilation == 4 + + num_blocks_per_stage = { + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3] + }[depth] + + stages = [] + out_stage_idx = [{ + 'res2': 2, + 'res3': 3, + 'res4': 4, + 'res5': 5 + }[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + if stage_idx == 4: + dilation = res4_dilation + elif stage_idx == 5: + dilation = res5_dilation + else: + dilation = 1 + first_stride = 1 if idx == 0 or dilation > 1 else 2 + stride_per_block = [first_stride] + stride_per_block += [1] * (num_blocks_per_stage[idx] - 1) + stage_kargs = { + 'num_blocks': num_blocks_per_stage[idx], + 'stride_per_block': stride_per_block, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'norm': norm, + 'bottleneck_channels': bottleneck_channels, + 'stride_in_1x1': stride_in_1x1, + 'dilation': dilation, + 'num_groups': num_groups, + 'block_class': BottleneckBlock + } + if stage_idx == 5: + stage_kargs.pop('dilation') + stage_kargs['dilation_per_block'] = [ + dilation * mg for mg in res5_multi_grid + ] + blocks = DeeplabResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return DeeplabResNet(stem, stages, out_features=out_features) diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/__init__.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d90618d3f84a4f7ba64eae75ee17af4e1fd4051a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .m2fp_encoder import MSDeformAttnPixelDecoder + from .m2fp_decoder import MultiScaleMaskedTransformerDecoder + +else: + _import_structure = { + 'm2fp_encoder': ['MSDeformAttnPixelDecoder'], + 'm2fp_decoder': ['MultiScaleMaskedTransformerDecoder'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_decoder.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..f21eae46fd75199348f03c255d52e3b01e698a1a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_decoder.py @@ -0,0 +1,221 @@ +# The implementation is adopted from Mask2Former, made publicly available under the MIT License at +# https://github.com/facebookresearch/Mask2Former + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_colorization.ddcolor.utils.transformer_utils import ( + MLP, CrossAttentionLayer, FFNLayer, SelfAttentionLayer) +from modelscope.models.cv.image_instance_segmentation.maskdino.position_encoding import \ + PositionEmbeddingSine +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +class MultiScaleMaskedTransformerDecoder(nn.Module): + + def __init__( + self, + in_channels, + mask_classification=True, + *, + num_classes: int, + hidden_dim: int, + num_queries: int, + nheads: int, + dim_feedforward: int, + dec_layers: int, + pre_norm: bool, + mask_dim: int, + enforce_input_project: bool, + ): + """ + NOTE: this interface is experimental. + Args: + in_channels: channels of the input features + mask_classification: whether to add mask classifier or not + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + dec_layers: number of Transformer decoder layers + pre_norm: whether to use pre-LayerNorm or not + mask_dim: mask feature dimension + enforce_input_project: add input project 1x1 conv even if input + channels and hidden dim is identical + """ + super().__init__() + + assert mask_classification, 'Only support mask classification model' + self.mask_classification = mask_classification + + # positional encoding + N_steps = hidden_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + # define Transformer decoder here + self.num_heads = nheads + self.num_layers = dec_layers + self.num_classes = num_classes + self.transformer_self_attention_layers = nn.ModuleList() + self.transformer_cross_attention_layers = nn.ModuleList() + self.transformer_ffn_layers = nn.ModuleList() + + for _ in range(self.num_layers): + self.transformer_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + )) + + self.transformer_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + )) + + self.transformer_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm, + )) + + self.decoder_norm = nn.LayerNorm(hidden_dim) + + self.num_queries = num_queries + # learnable query features + self.query_feat = nn.Embedding(num_queries, hidden_dim) + # learnable query p.e. + self.query_embed = nn.Embedding(num_queries, hidden_dim) + + # level embedding (we always use 3 scales) + self.num_feature_levels = 3 + self.level_embed = nn.Embedding(self.num_feature_levels, hidden_dim) + self.input_proj = nn.ModuleList() + for _ in range(self.num_feature_levels): + if in_channels != hidden_dim or enforce_input_project: + self.input_proj.append( + Conv2d(in_channels, hidden_dim, kernel_size=1)) + else: + self.input_proj.append(nn.Sequential()) + + # output FFNs + if self.mask_classification: + self.class_embed = nn.Linear(hidden_dim, num_classes + 1) + self.mask_embed = MLP(hidden_dim, hidden_dim, mask_dim, 3) + + def forward(self, x, mask_features, mask=None): + # x is a list of multi-scale feature + assert len(x) == self.num_feature_levels + src = [] + pos = [] + size_list = [] + + # disable mask, it does not affect performance + del mask + + for i in range(self.num_feature_levels): + size_list.append(x[i].shape[-2:]) + pos.append(self.pe_layer(x[i], None).flatten(2)) + src.append(self.input_proj[i](x[i]).flatten(2) + + self.level_embed.weight[i][None, :, None]) + + # flatten NxCxHxW to HWxNxC + pos[-1] = pos[-1].permute(2, 0, 1) + src[-1] = src[-1].permute(2, 0, 1) + + _, bs, _ = src[0].shape + + # QxNxC + query_embed = self.query_embed.weight.unsqueeze(1).repeat(1, bs, 1) + output = self.query_feat.weight.unsqueeze(1).repeat(1, bs, 1) + + predictions_class = [] + predictions_mask = [] + + # prediction heads on learnable query features + outputs_class, outputs_mask, attn_mask = self.forward_prediction_heads( + output, mask_features, attn_mask_target_size=size_list[0]) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + for i in range(self.num_layers): + level_index = i % self.num_feature_levels + attn_mask[torch.where( + attn_mask.sum(-1) == attn_mask.shape[-1])] = False + # attention: cross-attention first + output = self.transformer_cross_attention_layers[i]( + output, + src[level_index], + memory_mask=attn_mask, + memory_key_padding_mask= + None, # here we do not apply masking on padded region + pos=pos[level_index], + query_pos=query_embed) + + output = self.transformer_self_attention_layers[i]( + output, + tgt_mask=None, + tgt_key_padding_mask=None, + query_pos=query_embed) + + # FFN + output = self.transformer_ffn_layers[i](output) + + outputs_class, outputs_mask, attn_mask = \ + self.forward_prediction_heads( + output, mask_features, attn_mask_target_size=size_list[(i + 1) % self.num_feature_levels]) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + assert len(predictions_class) == self.num_layers + 1 + + out = { + 'pred_logits': + predictions_class[-1], + 'pred_masks': + predictions_mask[-1], + 'aux_outputs': + self._set_aux_loss( + predictions_class if self.mask_classification else None, + predictions_mask) + } + return out + + def forward_prediction_heads(self, output, mask_features, + attn_mask_target_size): + decoder_output = self.decoder_norm(output) + decoder_output = decoder_output.transpose(0, 1) + outputs_class = self.class_embed(decoder_output) + mask_embed = self.mask_embed(decoder_output) + outputs_mask = torch.einsum('bqc,bchw->bqhw', mask_embed, + mask_features) + + attn_mask = F.interpolate( + outputs_mask, + size=attn_mask_target_size, + mode='bilinear', + align_corners=False) + attn_mask = (attn_mask.sigmoid().flatten(2).unsqueeze(1).repeat( + 1, self.num_heads, 1, 1).flatten(0, 1) < 0.5).bool() + attn_mask = attn_mask.detach() + + return outputs_class, outputs_mask, attn_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks): + if self.mask_classification: + return [{ + 'pred_logits': a, + 'pred_masks': b + } for a, b in zip(outputs_class[:-1], outputs_seg_masks[:-1])] + else: + return [{'pred_masks': b} for b in outputs_seg_masks[:-1]] diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_encoder.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9cf78d015446a049d82ce9d5940e09f269e428 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp/m2fp_encoder.py @@ -0,0 +1,215 @@ +# The implementation is adopted from Mask2Former, made publicly available under the MIT License at +# https://github.com/facebookresearch/Mask2Former + +from typing import Any, Dict, List + +import numpy as np +import torch +from torch import nn +from torch.cuda.amp import autocast +from torch.nn import functional as F + +from modelscope.models.cv.image_instance_segmentation.maskdino.maskdino_encoder import \ + MSDeformAttnTransformerEncoderOnly +from modelscope.models.cv.image_instance_segmentation.maskdino.position_encoding import \ + PositionEmbeddingSine +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +class MSDeformAttnPixelDecoder(nn.Module): + + def __init__( + self, + input_shape: Dict[str, Any], + *, + transformer_dropout: float, + transformer_nheads: int, + transformer_dim_feedforward: int, + transformer_enc_layers: int, + conv_dim: int, + mask_dim: int, + # deformable transformer encoder args + transformer_in_features: List[str], + common_stride: int, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + transformer_dropout: dropout probability in transformer + transformer_nheads: number of heads in transformer + transformer_dim_feedforward: dimension of feedforward network + transformer_enc_layers: number of transformer encoder layers + conv_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + """ + super().__init__() + self.conv_dim = conv_dim + + transformer_input_shape = { + k: v + for k, v in input_shape.items() if k in transformer_in_features + } + + # this is the input shape of pixel decoder + input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride']) + self.in_features = [k for k, v in input_shape + ] # starting from "res2" to "res5" + self.feature_strides = [v['stride'] for k, v in input_shape] + self.feature_channels = [v['channels'] for k, v in input_shape] + + # this is the input shape of transformer encoder (could use less features than pixel decoder + transformer_input_shape = sorted( + transformer_input_shape.items(), key=lambda x: x[1]['stride']) + self.transformer_in_features = [k for k, v in transformer_input_shape + ] # starting from "res2" to "res5" + transformer_in_channels = [ + v['channels'] for k, v in transformer_input_shape + ] + self.transformer_feature_strides = [ + v['stride'] for k, v in transformer_input_shape + ] # to decide extra FPN layers + + self.transformer_num_feature_levels = len(self.transformer_in_features) + if self.transformer_num_feature_levels > 1: + input_proj_list = [] + # from low resolution to high resolution (res5 -> res2) + for in_channels in transformer_in_channels[::-1]: + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + )) + self.input_proj = nn.ModuleList(input_proj_list) + else: + self.input_proj = nn.ModuleList([ + nn.Sequential( + nn.Conv2d( + transformer_in_channels[-1], conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + ) + ]) + + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + self.transformer = MSDeformAttnTransformerEncoderOnly( + d_model=conv_dim, + dropout=transformer_dropout, + nhead=transformer_nheads, + dim_feedforward=transformer_dim_feedforward, + num_encoder_layers=transformer_enc_layers, + num_feature_levels=self.transformer_num_feature_levels, + ) + N_steps = conv_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + self.mask_dim = mask_dim + # use 1x1 conv instead + self.mask_features = Conv2d( + conv_dim, + mask_dim, + kernel_size=1, + stride=1, + padding=0, + ) + + self.maskformer_num_feature_levels = 3 # always use 3 scales + self.common_stride = common_stride + + # extra fpn levels + stride = min(self.transformer_feature_strides) + self.num_fpn_levels = int( + np.log2(stride) - np.log2(self.common_stride)) + + lateral_convs = [] + output_convs = [] + + use_bias = False + for idx, in_channels in enumerate( + self.feature_channels[:self.num_fpn_levels]): + lateral_norm = nn.GroupNorm(32, conv_dim) + output_norm = nn.GroupNorm(32, conv_dim) + + lateral_conv = Conv2d( + in_channels, + conv_dim, + kernel_size=1, + bias=use_bias, + norm=lateral_norm) + output_conv = Conv2d( + conv_dim, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + self.add_module('adapter_{}'.format(idx + 1), lateral_conv) + self.add_module('layer_{}'.format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + @autocast(enabled=False) + def forward_features(self, features): + srcs = [] + pos = [] + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.transformer_in_features[::-1]): + x = features[f].float( + ) # deformable detr does not support half precision + srcs.append(self.input_proj[idx](x)) + pos.append(self.pe_layer(x)) + + y, spatial_shapes, level_start_index = self.transformer( + srcs, None, pos) + bs = y.shape[0] + + split_size_or_sections = [None] * self.transformer_num_feature_levels + for i in range(self.transformer_num_feature_levels): + if i < self.transformer_num_feature_levels - 1: + split_size_or_sections[i] = level_start_index[ + i + 1] - level_start_index[i] + else: + split_size_or_sections[i] = y.shape[1] - level_start_index[i] + y = torch.split(y, split_size_or_sections, dim=1) + + out = [] + multi_scale_features = [] + num_cur_levels = 0 + for i, z in enumerate(y): + out.append( + z.transpose(1, 2).view(bs, -1, spatial_shapes[i][0], + spatial_shapes[i][1])) + + # append `out` with extra FPN levels + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[:self.num_fpn_levels][::-1]): + x = features[f].float() + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + cur_fpn = lateral_conv(x) + # Following FPN implementation, we use nearest upsampling here + y = cur_fpn + F.interpolate( + out[-1], + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + out.append(y) + + for o in out: + if num_cur_levels < self.maskformer_num_feature_levels: + multi_scale_features.append(o) + num_cur_levels += 1 + + return self.mask_features(out[-1]), out[0], multi_scale_features diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp_net.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp_net.py new file mode 100644 index 0000000000000000000000000000000000000000..3f771663b8ca7eb6cf48f67b447bbc7144eb82c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/m2fp_net.py @@ -0,0 +1,363 @@ +# Part of the implementation is borrowed and modified from M2FP, made publicly available +# under the CC BY-NC 4.0 License at https://github.com/soeaver/M2FP +import os +from typing import Any, Dict + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation.maskdino_swin import \ + ImageList +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .backbone import build_resnet_deeplab_backbone +from .m2fp.m2fp_decoder import MultiScaleMaskedTransformerDecoder +from .m2fp.m2fp_encoder import MSDeformAttnPixelDecoder + +logger = get_logger() + + +@MODELS.register_module(Tasks.image_segmentation, module_name=Models.m2fp) +class M2FP(TorchModel): + + def __init__(self, + model_dir, + backbone=None, + encoder=None, + decoder=None, + pretrained=None, + input_single_human=None, + classes=None, + num_parsing=None, + single_human=True, + parsing_ins_score_thr=0.5, + parsing_on=False, + semantic_on=True, + sem_seg_postprocess_before_inference=True, + **kwargs): + """ + Deep Learning Technique for Human Parsing: A Survey and Outlook. See https://arxiv.org/abs/2301.00394 + Args: + backbone (dict): backbone config. + encoder (dict): encoder config. + decoder (dict): decoder config. + pretrained (bool): whether to use pretrained model + input_single_human (dict): input size config for single human parsing + classes (list): class names + num_parsing (int): total number of parsing instances, only for multiple human parsing + single_human (bool): whether the task is single human parsing + parsing_ins_score_thr: instance score threshold for multiple human parsing + parsing_on (bool): whether to parse results, only for multiple human parsing + semantic_on (bool): whether to output semantic map + sem_seg_postprocess_before_inference: whether to resize the prediction back + to original input size before semantic segmentation inference or after. + """ + super(M2FP, self).__init__(model_dir, **kwargs) + + self.register_buffer( + 'pixel_mean', + torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False) + self.register_buffer( + 'pixel_std', + torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False) + self.size_divisibility = 32 + + self.backbone = build_resnet_deeplab_backbone( + **backbone, input_shape={'channels': 3}) + in_features = encoder.pop('in_features') + input_shape = { + k: v + for k, v in self.backbone.output_shape().items() + if k in in_features + } + encoder = MSDeformAttnPixelDecoder(input_shape=input_shape, **encoder) + decoder = MultiScaleMaskedTransformerDecoder( + in_channels=encoder.conv_dim, **decoder) + self.sem_seg_head = M2FPHead( + pixel_decoder=encoder, transformer_predictor=decoder) + self.num_classes = decoder.num_classes + self.num_queries = decoder.num_queries + self.test_topk_per_image = 100 + + self.input_single_human = input_single_human + self.classes = classes + self.num_parsing = num_parsing + self.single_human = single_human + self.parsing_ins_score_thr = parsing_ins_score_thr + self.parsing_on = parsing_on + self.semantic_on = semantic_on + self.sem_seg_postprocess_before_inference = sem_seg_postprocess_before_inference or parsing_on + + if not self.semantic_on: + assert self.sem_seg_postprocess_before_inference + + if pretrained: + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['model'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info( + f'size mismatch for {name} ' + f'({load_size} -> {tgt_size}), skip loading.') + del weight[name] + else: + logger.info( + f'{name} doesn\'t exist in current model, skip loading.' + ) + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + batched_inputs = input['batched_inputs'] + images = [x['image'].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + features = self.backbone(images.tensor) + outputs = self.sem_seg_head(features) + + return dict( + outputs=outputs, batched_inputs=batched_inputs, images=images) + + def postprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + outputs = input['outputs'] + batched_inputs = input['batched_inputs'] + images = input['images'] + if self.training: + raise NotImplementedError + else: + mask_cls_results = outputs['pred_logits'] # (B, Q, C+1) + mask_pred_results = outputs['pred_masks'] # (B, Q, H, W) + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode='bilinear', + align_corners=False, + ) + + del outputs + + processed_results = [] + for mask_cls_result, mask_pred_result, input_per_image, image_size in zip( + mask_cls_results, mask_pred_results, batched_inputs, + images.image_sizes): + height = input_per_image.get('height', image_size[0]) + width = input_per_image.get('width', image_size[1]) + processed_results.append({}) # for each image + + if self.sem_seg_postprocess_before_inference: + if not self.single_human: + mask_pred_result = self.sem_seg_postprocess( + mask_pred_result, image_size, height, width) + else: + mask_pred_result = self.single_human_sem_seg_postprocess( + mask_pred_result, image_size, + input_per_image['crop_box'], height, width) + mask_cls_result = mask_cls_result.to(mask_pred_result) + + # semantic segmentation inference + if self.semantic_on: + r = self.semantic_inference(mask_cls_result, + mask_pred_result) + if not self.sem_seg_postprocess_before_inference: + if not self.single_human: + r = self.sem_seg_postprocess( + r, image_size, height, width) + else: + r = self.single_human_sem_seg_postprocess( + r, image_size, input_per_image['crop_box'], + height, width) + processed_results[-1]['sem_seg'] = r + + # parsing inference + if self.parsing_on: + parsing_r = self.instance_parsing_inference( + mask_cls_result, mask_pred_result) + processed_results[-1]['parsing'] = parsing_r + + return dict(eval_result=processed_results) + + @property + def device(self): + return self.pixel_mean.device + + def single_human_sem_seg_postprocess(self, result, img_size, crop_box, + output_height, output_width): + result = result[:, :img_size[0], :img_size[1]] + result = result[:, crop_box[1]:crop_box[3], + crop_box[0]:crop_box[2]].expand(1, -1, -1, -1) + result = F.interpolate( + result, + size=(output_height, output_width), + mode='bilinear', + align_corners=False)[0] + return result + + def sem_seg_postprocess(self, result, img_size, output_height, + output_width): + result = result[:, :img_size[0], :img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate( + result, + size=(output_height, output_width), + mode='bilinear', + align_corners=False)[0] + return result + + def semantic_inference(self, mask_cls, mask_pred): + mask_cls = F.softmax( + mask_cls, dim=-1)[..., :-1] # discard non-sense category + mask_pred = mask_pred.sigmoid() + semseg = torch.einsum('qc,qhw->chw', mask_cls, mask_pred) + return semseg + + def instance_parsing_inference(self, mask_cls, mask_pred): + scores = F.softmax(mask_cls, dim=-1)[:, :-1] + labels = torch.arange( + self.num_classes, + device=self.device).unsqueeze(0).repeat(self.num_queries, + 1).flatten(0, 1) + + scores_per_image, topk_indices = scores.flatten(0, 1).topk( + self.test_topk_per_image, sorted=False) + labels_per_image = labels[topk_indices] + + topk_indices = topk_indices // self.num_classes + mask_pred = mask_pred[topk_indices] + + binary_pred_masks = (mask_pred > 0).float() + mask_scores_per_image = (mask_pred.sigmoid().flatten(1) * binary_pred_masks.flatten(1)).sum(1) / \ + (binary_pred_masks.flatten(1).sum(1) + 1e-6) + + pred_scores = scores_per_image * mask_scores_per_image + pred_labels = labels_per_image + pred_masks = mask_pred + + # prepare outputs + part_instance_res = [] + human_instance_res = [] + + # bkg and part instances + bkg_part_index = torch.where(pred_labels != self.num_parsing)[0] + bkg_part_labels = pred_labels[bkg_part_index] + bkg_part_scores = pred_scores[bkg_part_index] + bkg_part_masks = pred_masks[bkg_part_index, :, :] + + # human instances + human_index = torch.where(pred_labels == self.num_parsing)[0] + human_labels = pred_labels[human_index] + human_scores = pred_scores[human_index] + human_masks = pred_masks[human_index, :, :] + + semantic_res = self.paste_instance_to_semseg_probs( + bkg_part_labels, bkg_part_scores, bkg_part_masks) + + # part instances + part_index = torch.where(bkg_part_labels != 0)[0] + part_labels = bkg_part_labels[part_index] + part_scores = bkg_part_scores[part_index] + part_masks = bkg_part_masks[part_index, :, :] + + # part instance results + for idx in range(part_labels.shape[0]): + if part_scores[idx] < 0.1: + continue + part_instance_res.append({ + 'category_id': + part_labels[idx].cpu().tolist(), + 'score': + part_scores[idx].cpu().tolist(), + 'mask': + part_masks[idx], + }) + + # human instance results + for human_idx in range(human_scores.shape[0]): + if human_scores[human_idx] > 0.1: + human_instance_res.append({ + 'category_id': + human_labels[human_idx].cpu().tolist(), + 'score': + human_scores[human_idx].cpu().tolist(), + 'mask': + human_masks[human_idx], + }) + + return { + 'semantic_outputs': semantic_res, + 'part_outputs': part_instance_res, + 'human_outputs': human_instance_res, + } + + def paste_instance_to_semseg_probs(self, labels, scores, mask_probs): + im_h, im_w = mask_probs.shape[-2:] + semseg_im = [] + for cls_ind in range(self.num_parsing): + cate_inds = torch.where(labels == cls_ind)[0] + cate_scores = scores[cate_inds] + cate_mask_probs = mask_probs[cate_inds, :, :].sigmoid() + semseg_im.append( + self.paste_category_probs(cate_scores, cate_mask_probs, im_h, + im_w)) + + return torch.stack(semseg_im, dim=0) + + def paste_category_probs(self, scores, mask_probs, h, w): + category_probs = torch.zeros((h, w), + dtype=torch.float32, + device=mask_probs.device) + paste_times = torch.zeros((h, w), + dtype=torch.float32, + device=mask_probs.device) + + index = scores.argsort() + for k in range(len(index)): + if scores[index[k]] < self.parsing_ins_score_thr: + continue + ins_mask_probs = mask_probs[index[k], :, :] * scores[index[k]] + category_probs = torch.where(ins_mask_probs > 0.5, + ins_mask_probs + category_probs, + category_probs) + paste_times += torch.where(ins_mask_probs > 0.5, 1, 0) + + paste_times = torch.where(paste_times == 0, paste_times + 1, + paste_times) + category_probs /= paste_times + + return category_probs + + +class M2FPHead(nn.Module): + + def __init__(self, pixel_decoder: nn.Module, + transformer_predictor: nn.Module): + super().__init__() + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + + def forward(self, features, mask=None): + return self.layers(features, mask) + + def layers(self, features, mask=None): + mask_features, transformer_encoder_features, multi_scale_features = self.pixel_decoder.forward_features( + features) + predictions = self.predictor(multi_scale_features, mask_features, mask) + return predictions diff --git a/runtime/Lib/site_packages/models/cv/image_human_parsing/parsing_utils.py b/runtime/Lib/site_packages/models/cv/image_human_parsing/parsing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4d1652b49c07e3b23016dcb87f5165d1114944c9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_human_parsing/parsing_utils.py @@ -0,0 +1,156 @@ +# Part of the implementation is borrowed and modified from M2FP, made publicly available +# under the CC BY-NC 4.0 License at https://github.com/soeaver/M2FP +# Part of the implementation is borrowed and modified from Detectron2, made publicly available +# under the Apache-2.0 License at https://github.com/facebookresearch/detectron2 + +import copy + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + + +def center_to_target_size_test(img, target_size): + src_h, src_w = img.shape[0], img.shape[1] + trg_h, trg_w = target_size[1], target_size[0] + + new_h, new_w = 0, 0 + tfm_list = [] + if src_h > trg_h and src_w > trg_w: + if src_h >= src_w: + new_h = trg_h + new_w = int(new_h * src_w / src_h) + if new_w > trg_w: + new_w = trg_w + new_h = int(new_w * src_h / src_w) + elif src_w > src_h: + new_w = trg_w + new_h = int(new_w * src_h / src_w) + if new_h > trg_h: + new_h = trg_h + new_w = int(new_h * src_w / src_h) + tfm_list.append(ResizeTransform(src_h, src_w, new_h, new_w)) + tfm_list.append(PadTransform(new_h, new_w, trg_h, trg_w)) + + elif src_h > trg_h and src_w <= trg_w: + new_h = trg_h + new_w = int(new_h * src_w / src_h) + tfm_list.append(ResizeTransform(src_h, src_w, new_h, new_w)) + tfm_list.append(PadTransform(new_h, new_w, trg_h, trg_w)) + + elif src_h <= trg_h and src_w > trg_w: + new_w = trg_w + new_h = int(new_w * src_h / src_w) + tfm_list.append(ResizeTransform(src_h, src_w, new_h, new_w)) + tfm_list.append(PadTransform(new_h, new_w, trg_h, trg_w)) + + else: + new_h, new_w = src_h, src_w + tfm_list.append(PadTransform(new_h, new_w, trg_h, trg_w)) + + box = get_box(new_h, new_w, trg_h, trg_w) + + new_img = copy.deepcopy(img) + for tfm in tfm_list: + new_img = tfm.apply_image(new_img) + + return new_img, box + + +def get_box(src_h, src_w, trg_h, trg_w): + assert src_h <= trg_h, 'expect src_h <= trg_h' + assert src_w <= trg_w, 'expect src_w <= trg_w' + + x0 = int((trg_w - src_w) / 2) + x1 = src_w + x0 + y0 = int((trg_h - src_h) / 2) + y1 = src_h + y0 + + box = [x0, y0, x1, y1] + return box + + +class PadTransform: + + def __init__(self, src_h, src_w, trg_h, trg_w): + super().__init__() + assert src_h <= trg_h, 'expect src_h <= trg_h' + assert src_w <= trg_w, 'expect src_w <= trg_w' + + self.src_h, self.src_w = src_h, src_w + self.trg_h, self.trg_w = trg_h, trg_w + self.pad_left = int((trg_w - src_w) / 2) + self.pad_right = trg_w - src_w - self.pad_left + self.pad_top = int((trg_h - src_h) / 2) + self.pad_bottom = trg_h - src_h - self.pad_top + + def apply_image(self, img, pad_value=128): + if self.pad_left == 0 and self.pad_top == 0: + return img + + if len(img.shape) == 2: + return np.pad( + img, ((self.pad_top, self.pad_bottom), + (self.pad_left, self.pad_right)), + 'constant', + constant_values=((pad_value, pad_value), (pad_value, + pad_value))) + elif len(img.shape) == 3: + return np.pad( + img, ((self.pad_top, self.pad_bottom), + (self.pad_left, self.pad_right), (0, 0)), + 'constant', + constant_values=((pad_value, pad_value), + (pad_value, pad_value), (pad_value, + pad_value))) + + +class ResizeTransform: + + def __init__(self, h, w, new_h, new_w, interp=None): + super().__init__() + if interp is None: + interp = Image.BILINEAR + self.h, self.w = h, w + self.new_h, self.new_w = new_h, new_w + self.interp = interp + + def apply_image(self, img, interp=None): + assert img.shape[:2] == (self.h, self.w) + assert len(img.shape) <= 4 + interp_method = interp if interp is not None else self.interp + + if img.dtype == np.uint8: + if len(img.shape) > 2 and img.shape[2] == 1: + pil_image = Image.fromarray(img[:, :, 0], mode='L') + else: + pil_image = Image.fromarray(img) + pil_image = pil_image.resize((self.new_w, self.new_h), + interp_method) + ret = np.asarray(pil_image) + if len(img.shape) > 2 and img.shape[2] == 1: + ret = np.expand_dims(ret, -1) + else: + # PIL only supports uint8 + if any(x < 0 for x in img.strides): + img = np.ascontiguousarray(img) + img = torch.from_numpy(img) + shape = list(img.shape) + shape_4d = shape[:2] + [1] * (4 - len(shape)) + shape[2:] + img = img.view(shape_4d).permute(2, 3, 0, 1) # hw(c) -> nchw + _PIL_RESIZE_TO_INTERPOLATE_MODE = { + Image.NEAREST: 'nearest', + Image.BILINEAR: 'bilinear', + Image.BICUBIC: 'bicubic', + } + mode = _PIL_RESIZE_TO_INTERPOLATE_MODE[interp_method] + align_corners = None if mode == 'nearest' else False + img = F.interpolate( + img, (self.new_h, self.new_w), + mode=mode, + align_corners=align_corners) + shape[:2] = (self.new_h, self.new_w) + ret = img.permute(2, 3, 0, 1).view(shape).numpy() # nchw -> hw(c) + + return ret diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/__init__.py b/runtime/Lib/site_packages/models/cv/image_inpainting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7c63cd419520af5985e29e8eb66ec21c0c80672 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import FFTInpainting + +else: + _import_structure = { + 'model': ['FFTInpainting'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/base.py b/runtime/Lib/site_packages/models/cv/image_inpainting/base.py new file mode 100644 index 0000000000000000000000000000000000000000..04e7363098d2e934734ab92529ed7944e790b5d7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/base.py @@ -0,0 +1,75 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +from typing import Dict, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.utils.logger import get_logger +from .modules.adversarial import NonSaturatingWithR1 +from .modules.ffc import FFCResNetGenerator +from .modules.perceptual import ResNetPL +from .modules.pix2pixhd import NLayerDiscriminator + +LOGGER = get_logger() + + +class BaseInpaintingTrainingModule(nn.Module): + + def __init__(self, + model_dir='', + use_ddp=True, + predict_only=False, + visualize_each_iters=100, + average_generator=False, + generator_avg_beta=0.999, + average_generator_start_step=30000, + average_generator_period=10, + store_discr_outputs_for_vis=False, + **kwargs): + super().__init__() + LOGGER.info( + f'BaseInpaintingTrainingModule init called, predict_only is {predict_only}' + ) + + self.generator = FFCResNetGenerator() + self.use_ddp = use_ddp + + if not predict_only: + self.discriminator = NLayerDiscriminator() + self.adversarial_loss = NonSaturatingWithR1( + weight=10, + gp_coef=0.001, + mask_as_fake_target=True, + allow_scale_mask=True) + + self.average_generator = average_generator + self.generator_avg_beta = generator_avg_beta + self.average_generator_start_step = average_generator_start_step + self.average_generator_period = average_generator_period + self.generator_average = None + self.last_generator_averaging_step = -1 + self.store_discr_outputs_for_vis = store_discr_outputs_for_vis + + self.loss_l1 = nn.L1Loss(reduction='none') + + self.loss_resnet_pl = ResNetPL(weight=30, weights_path=model_dir) + + self.visualize_each_iters = visualize_each_iters + LOGGER.info('BaseInpaintingTrainingModule init done') + + def forward(self, batch: Dict[str, + torch.Tensor]) -> Dict[str, torch.Tensor]: + """Pass data through generator and obtain at leas 'predicted_image' and 'inpainted' keys""" + raise NotImplementedError() + + def generator_loss(self, + batch) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + raise NotImplementedError() + + def discriminator_loss( + self, batch) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + raise NotImplementedError() diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/default.py b/runtime/Lib/site_packages/models/cv/image_inpainting/default.py new file mode 100644 index 0000000000000000000000000000000000000000..5f57d63fa545ddbd0535bc88a3002b0f56567527 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/default.py @@ -0,0 +1,210 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +import bisect + +import torch +import torch.nn.functional as F + +from modelscope.utils.logger import get_logger +from .base import BaseInpaintingTrainingModule +from .modules.feature_matching import feature_matching_loss, masked_l1_loss + +LOGGER = get_logger() + + +def set_requires_grad(module, value): + for param in module.parameters(): + param.requires_grad = value + + +def add_prefix_to_keys(dct, prefix): + return {prefix + k: v for k, v in dct.items()} + + +class LinearRamp: + + def __init__(self, start_value=0, end_value=1, start_iter=-1, end_iter=0): + self.start_value = start_value + self.end_value = end_value + self.start_iter = start_iter + self.end_iter = end_iter + + def __call__(self, i): + if i < self.start_iter: + return self.start_value + if i >= self.end_iter: + return self.end_value + part = (i - self.start_iter) / (self.end_iter - self.start_iter) + return self.start_value * (1 - part) + self.end_value * part + + +class LadderRamp: + + def __init__(self, start_iters, values): + self.start_iters = start_iters + self.values = values + assert len(values) == len(start_iters) + 1, (len(values), + len(start_iters)) + + def __call__(self, i): + segment_i = bisect.bisect_right(self.start_iters, i) + return self.values[segment_i] + + +def get_ramp(kind='ladder', **kwargs): + if kind == 'linear': + return LinearRamp(**kwargs) + if kind == 'ladder': + return LadderRamp(**kwargs) + raise ValueError(f'Unexpected ramp kind: {kind}') + + +class DefaultInpaintingTrainingModule(BaseInpaintingTrainingModule): + + def __init__(self, + model_dir='', + predict_only=False, + concat_mask=True, + rescale_scheduler_kwargs=None, + image_to_discriminator='predicted_image', + add_noise_kwargs=None, + noise_fill_hole=False, + const_area_crop_kwargs=None, + distance_weighter_kwargs=None, + distance_weighted_mask_for_discr=False, + fake_fakes_proba=0, + fake_fakes_generator_kwargs=None, + **kwargs): + super().__init__(model_dir=model_dir, predict_only=predict_only) + self.concat_mask = concat_mask + self.rescale_size_getter = get_ramp( + **rescale_scheduler_kwargs + ) if rescale_scheduler_kwargs is not None else None + self.image_to_discriminator = image_to_discriminator + self.add_noise_kwargs = add_noise_kwargs + self.noise_fill_hole = noise_fill_hole + self.const_area_crop_kwargs = const_area_crop_kwargs + self.refine_mask_for_losses = None + self.distance_weighted_mask_for_discr = distance_weighted_mask_for_discr + + self.feature_matching_weight = 100 + self.losses_l1_weight_known = 10 + self.losses_l1_weight_missing = 0 + self.fake_fakes_proba = fake_fakes_proba + + def forward(self, batch): + img = batch['image'] + mask = batch['mask'] + + masked_img = img * (1 - mask) + + if self.concat_mask: + masked_img = torch.cat([masked_img, mask], dim=1) + + batch['predicted_image'] = self.generator(masked_img) + batch['inpainted'] = mask * batch['predicted_image'] + ( + 1 - mask) * batch['image'] + + batch['mask_for_losses'] = mask + + return batch + + def generator_loss(self, batch): + img = batch['image'] + predicted_img = batch[self.image_to_discriminator] + original_mask = batch['mask'] + supervised_mask = batch['mask_for_losses'] + + # L1 + l1_value = masked_l1_loss(predicted_img, img, supervised_mask, + self.losses_l1_weight_known, + self.losses_l1_weight_missing) + + total_loss = l1_value + metrics = dict(gen_l1=l1_value) + + # discriminator + # adversarial_loss calls backward by itself + mask_for_discr = supervised_mask if self.distance_weighted_mask_for_discr else original_mask + self.adversarial_loss.pre_generator_step( + real_batch=img, + fake_batch=predicted_img, + generator=self.generator, + discriminator=self.discriminator) + discr_real_pred, discr_real_features = self.discriminator(img) + discr_fake_pred, discr_fake_features = self.discriminator( + predicted_img) + adv_gen_loss, adv_metrics = self.adversarial_loss.generator_loss( + real_batch=img, + fake_batch=predicted_img, + discr_real_pred=discr_real_pred, + discr_fake_pred=discr_fake_pred, + mask=mask_for_discr) + total_loss = total_loss + adv_gen_loss + metrics['gen_adv'] = adv_gen_loss + metrics.update(add_prefix_to_keys(adv_metrics, 'adv_')) + + # feature matching + if self.feature_matching_weight > 0: + need_mask_in_fm = False + mask_for_fm = supervised_mask if need_mask_in_fm else None + fm_value = feature_matching_loss( + discr_fake_features, discr_real_features, + mask=mask_for_fm) * self.feature_matching_weight + total_loss = total_loss + fm_value + metrics['gen_fm'] = fm_value + + if self.loss_resnet_pl is not None: + resnet_pl_value = self.loss_resnet_pl(predicted_img, img) + total_loss = total_loss + resnet_pl_value + metrics['gen_resnet_pl'] = resnet_pl_value + + return total_loss, metrics + + def discriminator_loss(self, batch): + total_loss = 0 + metrics = {} + + predicted_img = batch[self.image_to_discriminator].detach() + self.adversarial_loss.pre_discriminator_step( + real_batch=batch['image'], + fake_batch=predicted_img, + generator=self.generator, + discriminator=self.discriminator) + discr_real_pred, discr_real_features = self.discriminator( + batch['image']) + discr_fake_pred, discr_fake_features = self.discriminator( + predicted_img) + adv_discr_loss, adv_metrics = self.adversarial_loss.discriminator_loss( + real_batch=batch['image'], + fake_batch=predicted_img, + discr_real_pred=discr_real_pred, + discr_fake_pred=discr_fake_pred, + mask=batch['mask']) + + total_loss = (total_loss + adv_discr_loss) * 0.1 + metrics['discr_adv'] = adv_discr_loss + metrics.update(add_prefix_to_keys(adv_metrics, 'adv_')) + + return total_loss, metrics + + def _do_step(self, batch, optimizer_idx=None): + if optimizer_idx == 0: # step for generator + set_requires_grad(self.generator, True) + set_requires_grad(self.discriminator, False) + elif optimizer_idx == 1: # step for discriminator + set_requires_grad(self.generator, False) + set_requires_grad(self.discriminator, True) + + batch = self(batch) + total_loss = 0 + if optimizer_idx is None or optimizer_idx == 0: # step for generator + total_loss, metrics = self.generator_loss(batch) + + elif optimizer_idx is None or optimizer_idx == 1: # step for discriminator + total_loss, metrics = self.discriminator_loss(batch) + + result = dict(loss=total_loss) + return result diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/model.py b/runtime/Lib/site_packages/models/cv/image_inpainting/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b12f6eddb2824b04e68f2d417bbe68903d272933 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/model.py @@ -0,0 +1,36 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Optional, Union + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +LOGGER = get_logger() + + +@MODELS.register_module( + Tasks.image_inpainting, module_name=Models.image_inpainting) +class FFTInpainting(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + super().__init__(model_dir, **kwargs) + + from .default import DefaultInpaintingTrainingModule + pretrained = kwargs.get('pretrained', True) + predict_only = kwargs.get('predict_only', False) + net = DefaultInpaintingTrainingModule( + model_dir=model_dir, predict_only=predict_only) + if pretrained: + path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + LOGGER.info(f'loading pretrained model from {path}') + state = torch.load(path, map_location='cpu') + net.load_state_dict(state, strict=False) + self.model = net + + def forward(self, inputs): + return self.model(inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/__init__.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/__init__.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89c3e2930eabc8eef0177a00dd960edb564274ae --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .base import ModelBuilder diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/base.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/base.py new file mode 100644 index 0000000000000000000000000000000000000000..02bd3cc4e83fdbd3959321219d7f5239b9bdd0a8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/base.py @@ -0,0 +1,380 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" + +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules import BatchNorm2d + +from . import resnet + +NUM_CLASS = 150 + + +# Model Builder +class ModelBuilder: + # custom weights initialization + @staticmethod + def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + nn.init.kaiming_normal_(m.weight.data) + elif classname.find('BatchNorm') != -1: + m.weight.data.fill_(1.) + m.bias.data.fill_(1e-4) + + @staticmethod + def build_encoder(arch='resnet50dilated', + fc_dim=512, + weights='', + model_dir=''): + pretrained = True if len(weights) == 0 else False + arch = arch.lower() + if arch == 'resnet50dilated': + orig_resnet = resnet.__dict__['resnet50']( + pretrained=pretrained, model_dir=model_dir) + net_encoder = ResnetDilated(orig_resnet, dilate_scale=8) + elif arch == 'resnet50': + orig_resnet = resnet.__dict__['resnet50']( + pretrained=pretrained, model_dir=model_dir) + net_encoder = Resnet(orig_resnet) + else: + raise Exception('Architecture undefined!') + + # encoders are usually pretrained + # net_encoder.apply(ModelBuilder.weights_init) + if len(weights) > 0: + print('Loading weights for net_encoder') + net_encoder.load_state_dict( + torch.load(weights, map_location=lambda storage, loc: storage), + strict=False) + return net_encoder + + @staticmethod + def build_decoder(arch='ppm_deepsup', + fc_dim=512, + num_class=NUM_CLASS, + weights='', + use_softmax=False, + drop_last_conv=False): + arch = arch.lower() + if arch == 'ppm_deepsup': + net_decoder = PPMDeepsup( + num_class=num_class, + fc_dim=fc_dim, + use_softmax=use_softmax, + drop_last_conv=drop_last_conv) + elif arch == 'c1_deepsup': + net_decoder = C1DeepSup( + num_class=num_class, + fc_dim=fc_dim, + use_softmax=use_softmax, + drop_last_conv=drop_last_conv) + else: + raise Exception('Architecture undefined!') + + net_decoder.apply(ModelBuilder.weights_init) + if len(weights) > 0: + print('Loading weights for net_decoder') + net_decoder.load_state_dict( + torch.load(weights, map_location=lambda storage, loc: storage), + strict=False) + return net_decoder + + @staticmethod + def get_decoder(weights_path, arch_encoder, arch_decoder, fc_dim, + drop_last_conv, *arts, **kwargs): + path = os.path.join( + weights_path, 'ade20k', + f'ade20k-{arch_encoder}-{arch_decoder}/decoder_epoch_20.pth') + return ModelBuilder.build_decoder( + arch=arch_decoder, + fc_dim=fc_dim, + weights=path, + use_softmax=True, + drop_last_conv=drop_last_conv) + + @staticmethod + def get_encoder(weights_path, arch_encoder, arch_decoder, fc_dim, + segmentation, *arts, **kwargs): + if segmentation: + path = os.path.join( + weights_path, 'ade20k', + f'ade20k-{arch_encoder}-{arch_decoder}/encoder_epoch_20.pth') + else: + path = '' + return ModelBuilder.build_encoder( + arch=arch_encoder, + fc_dim=fc_dim, + weights=path, + model_dir=weights_path) + + +def conv3x3_bn_relu(in_planes, out_planes, stride=1): + return nn.Sequential( + nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False), + BatchNorm2d(out_planes), + nn.ReLU(inplace=True), + ) + + +# pyramid pooling, deep supervision +class PPMDeepsup(nn.Module): + + def __init__(self, + num_class=NUM_CLASS, + fc_dim=4096, + use_softmax=False, + pool_scales=(1, 2, 3, 6), + drop_last_conv=False): + super().__init__() + self.use_softmax = use_softmax + self.drop_last_conv = drop_last_conv + + self.ppm = [] + for scale in pool_scales: + self.ppm.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(scale), + nn.Conv2d(fc_dim, 512, kernel_size=1, bias=False), + BatchNorm2d(512), nn.ReLU(inplace=True))) + self.ppm = nn.ModuleList(self.ppm) + self.cbr_deepsup = conv3x3_bn_relu(fc_dim // 2, fc_dim // 4, 1) + + self.conv_last = nn.Sequential( + nn.Conv2d( + fc_dim + len(pool_scales) * 512, + 512, + kernel_size=3, + padding=1, + bias=False), BatchNorm2d(512), nn.ReLU(inplace=True), + nn.Dropout2d(0.1), nn.Conv2d(512, num_class, kernel_size=1)) + self.conv_last_deepsup = nn.Conv2d(fc_dim // 4, num_class, 1, 1, 0) + self.dropout_deepsup = nn.Dropout2d(0.1) + + def forward(self, conv_out, segSize=None): + conv5 = conv_out[-1] + + input_size = conv5.size() + ppm_out = [conv5] + for pool_scale in self.ppm: + ppm_out.append( + nn.functional.interpolate( + pool_scale(conv5), (input_size[2], input_size[3]), + mode='bilinear', + align_corners=False)) + ppm_out = torch.cat(ppm_out, 1) + + if self.drop_last_conv: + return ppm_out + else: + x = self.conv_last(ppm_out) + + if self.use_softmax: # is True during inference + x = nn.functional.interpolate( + x, size=segSize, mode='bilinear', align_corners=False) + x = nn.functional.softmax(x, dim=1) + return x + + # deep sup + conv4 = conv_out[-2] + _ = self.cbr_deepsup(conv4) + _ = self.dropout_deepsup(_) + _ = self.conv_last_deepsup(_) + + x = nn.functional.log_softmax(x, dim=1) + _ = nn.functional.log_softmax(_, dim=1) + + return (x, _) + + +class Resnet(nn.Module): + + def __init__(self, orig_resnet): + super(Resnet, self).__init__() + + # take pretrained resnet, except AvgPool and FC + self.conv1 = orig_resnet.conv1 + self.bn1 = orig_resnet.bn1 + self.relu1 = orig_resnet.relu1 + self.conv2 = orig_resnet.conv2 + self.bn2 = orig_resnet.bn2 + self.relu2 = orig_resnet.relu2 + self.conv3 = orig_resnet.conv3 + self.bn3 = orig_resnet.bn3 + self.relu3 = orig_resnet.relu3 + self.maxpool = orig_resnet.maxpool + self.layer1 = orig_resnet.layer1 + self.layer2 = orig_resnet.layer2 + self.layer3 = orig_resnet.layer3 + self.layer4 = orig_resnet.layer4 + + def forward(self, x, return_feature_maps=False): + conv_out = [] + + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.maxpool(x) + + x = self.layer1(x) + conv_out.append(x) + x = self.layer2(x) + conv_out.append(x) + x = self.layer3(x) + conv_out.append(x) + x = self.layer4(x) + conv_out.append(x) + + if return_feature_maps: + return conv_out + return [x] + + +# Resnet Dilated +class ResnetDilated(nn.Module): + + def __init__(self, orig_resnet, dilate_scale=8): + super().__init__() + from functools import partial + + if dilate_scale == 8: + orig_resnet.layer3.apply(partial(self._nostride_dilate, dilate=2)) + orig_resnet.layer4.apply(partial(self._nostride_dilate, dilate=4)) + elif dilate_scale == 16: + orig_resnet.layer4.apply(partial(self._nostride_dilate, dilate=2)) + + # take pretrained resnet, except AvgPool and FC + self.conv1 = orig_resnet.conv1 + self.bn1 = orig_resnet.bn1 + self.relu1 = orig_resnet.relu1 + self.conv2 = orig_resnet.conv2 + self.bn2 = orig_resnet.bn2 + self.relu2 = orig_resnet.relu2 + self.conv3 = orig_resnet.conv3 + self.bn3 = orig_resnet.bn3 + self.relu3 = orig_resnet.relu3 + self.maxpool = orig_resnet.maxpool + self.layer1 = orig_resnet.layer1 + self.layer2 = orig_resnet.layer2 + self.layer3 = orig_resnet.layer3 + self.layer4 = orig_resnet.layer4 + + def _nostride_dilate(self, m, dilate): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + # the convolution with stride + if m.stride == (2, 2): + m.stride = (1, 1) + if m.kernel_size == (3, 3): + m.dilation = (dilate // 2, dilate // 2) + m.padding = (dilate // 2, dilate // 2) + # other convoluions + else: + if m.kernel_size == (3, 3): + m.dilation = (dilate, dilate) + m.padding = (dilate, dilate) + + def forward(self, x, return_feature_maps=False): + conv_out = [] + + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.maxpool(x) + + x = self.layer1(x) + conv_out.append(x) + x = self.layer2(x) + conv_out.append(x) + x = self.layer3(x) + conv_out.append(x) + x = self.layer4(x) + conv_out.append(x) + + if return_feature_maps: + return conv_out + return [x] + + +# last conv, deep supervision +class C1DeepSup(nn.Module): + + def __init__(self, + num_class=150, + fc_dim=2048, + use_softmax=False, + drop_last_conv=False): + super(C1DeepSup, self).__init__() + self.use_softmax = use_softmax + self.drop_last_conv = drop_last_conv + + self.cbr = conv3x3_bn_relu(fc_dim, fc_dim // 4, 1) + self.cbr_deepsup = conv3x3_bn_relu(fc_dim // 2, fc_dim // 4, 1) + + # last conv + self.conv_last = nn.Conv2d(fc_dim // 4, num_class, 1, 1, 0) + self.conv_last_deepsup = nn.Conv2d(fc_dim // 4, num_class, 1, 1, 0) + + def forward(self, conv_out, segSize=None): + conv5 = conv_out[-1] + + x = self.cbr(conv5) + + if self.drop_last_conv: + return x + else: + x = self.conv_last(x) + + if self.use_softmax: # is True during inference + x = nn.functional.interpolate( + x, size=segSize, mode='bilinear', align_corners=False) + x = nn.functional.softmax(x, dim=1) + return x + + # deep sup + conv4 = conv_out[-2] + _ = self.cbr_deepsup(conv4) + _ = self.conv_last_deepsup(_) + + x = nn.functional.log_softmax(x, dim=1) + _ = nn.functional.log_softmax(_, dim=1) + + return (x, _) + + +# last conv +class C1(nn.Module): + + def __init__(self, num_class=150, fc_dim=2048, use_softmax=False): + super(C1, self).__init__() + self.use_softmax = use_softmax + + self.cbr = conv3x3_bn_relu(fc_dim, fc_dim // 4, 1) + + # last conv + self.conv_last = nn.Conv2d(fc_dim // 4, num_class, 1, 1, 0) + + def forward(self, conv_out, segSize=None): + conv5 = conv_out[-1] + x = self.cbr(conv5) + x = self.conv_last(x) + + if self.use_softmax: # is True during inference + x = nn.functional.interpolate( + x, size=segSize, mode='bilinear', align_corners=False) + x = nn.functional.softmax(x, dim=1) + else: + x = nn.functional.log_softmax(x, dim=1) + + return x diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/resnet.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..7da9ff073fdc1b55d46e24f33495a2370c1cb653 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ade20k/resnet.py @@ -0,0 +1,183 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +import math +import os + +import torch +import torch.nn as nn +from torch.nn import BatchNorm2d + +__all__ = ['ResNet', 'resnet50'] + + +def conv3x3(in_planes, out_planes, stride=1): + '3x3 convolution with padding' + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = BatchNorm2d(planes) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000): + self.inplanes = 128 + super(ResNet, self).__init__() + self.conv1 = conv3x3(3, 64, stride=2) + self.bn1 = BatchNorm2d(64) + self.relu1 = nn.ReLU(inplace=True) + self.conv2 = conv3x3(64, 64) + self.bn2 = BatchNorm2d(64) + self.relu2 = nn.ReLU(inplace=True) + self.conv3 = conv3x3(64, 128) + self.bn3 = BatchNorm2d(128) + self.relu3 = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.avgpool = nn.AvgPool2d(7, stride=1) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + + +def resnet50(pretrained=False, model_dir='', **kwargs): + """Constructs a ResNet-50 model. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) + if pretrained: + cached_file = os.path.join(model_dir, 'resnet50-imagenet.pth') + model.load_state_dict( + torch.load(cached_file, map_location='cpu'), strict=False) + return model diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/adversarial.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/adversarial.py new file mode 100644 index 0000000000000000000000000000000000000000..b183876bbe4c71b9644805b7f283159c121603aa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/adversarial.py @@ -0,0 +1,167 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BaseAdversarialLoss: + + def pre_generator_step(self, real_batch: torch.Tensor, + fake_batch: torch.Tensor, generator: nn.Module, + discriminator: nn.Module): + """ + Prepare for generator step + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param generator: + :param discriminator: + :return: None + """ + + def pre_discriminator_step(self, real_batch: torch.Tensor, + fake_batch: torch.Tensor, generator: nn.Module, + discriminator: nn.Module): + """ + Prepare for discriminator step + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param generator: + :param discriminator: + :return: None + """ + + def generator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask: Optional[torch.Tensor] = None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Calculate generator loss + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param discr_real_pred: Tensor, discriminator output for real_batch + :param discr_fake_pred: Tensor, discriminator output for fake_batch + :param mask: Tensor, actual mask, which was at input of generator when making fake_batch + :return: total generator loss along with some values that might be interesting to log + """ + raise NotImplementedError + + def discriminator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask: Optional[torch.Tensor] = None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Calculate discriminator loss and call .backward() on it + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param discr_real_pred: Tensor, discriminator output for real_batch + :param discr_fake_pred: Tensor, discriminator output for fake_batch + :param mask: Tensor, actual mask, which was at input of generator when making fake_batch + :return: total discriminator loss along with some values that might be interesting to log + """ + raise NotImplementedError + + def interpolate_mask(self, mask, shape): + assert mask is not None + assert self.allow_scale_mask or shape == mask.shape[-2:] + if shape != mask.shape[-2:] and self.allow_scale_mask: + if self.mask_scale_mode == 'maxpool': + mask = F.adaptive_max_pool2d(mask, shape) + else: + mask = F.interpolate( + mask, size=shape, mode=self.mask_scale_mode) + return mask + + +def make_r1_gp(discr_real_pred, real_batch): + if torch.is_grad_enabled(): + grad_real = torch.autograd.grad( + outputs=discr_real_pred.sum(), + inputs=real_batch, + create_graph=True)[0] + grad_penalty = (grad_real.view(grad_real.shape[0], + -1).norm(2, dim=1)**2).mean() + else: + grad_penalty = 0 + real_batch.requires_grad = False + + return grad_penalty + + +class NonSaturatingWithR1(BaseAdversarialLoss): + + def __init__(self, + gp_coef=5, + weight=1, + mask_as_fake_target=False, + allow_scale_mask=False, + mask_scale_mode='nearest', + extra_mask_weight_for_gen=0, + use_unmasked_for_gen=True, + use_unmasked_for_discr=True): + self.gp_coef = gp_coef + self.weight = weight + # use for discr => use for gen; + # otherwise we teach only the discr to pay attention to very small difference + assert use_unmasked_for_gen or (not use_unmasked_for_discr) + # mask as target => use unmasked for discr: + # if we don't care about unmasked regions at all + # then it doesn't matter if the value of mask_as_fake_target is true or false + assert use_unmasked_for_discr or (not mask_as_fake_target) + self.use_unmasked_for_gen = use_unmasked_for_gen + self.use_unmasked_for_discr = use_unmasked_for_discr + self.mask_as_fake_target = mask_as_fake_target + self.allow_scale_mask = allow_scale_mask + self.mask_scale_mode = mask_scale_mode + self.extra_mask_weight_for_gen = extra_mask_weight_for_gen + + def generator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask=None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + fake_loss = F.softplus(-discr_fake_pred) + if (self.mask_as_fake_target and self.extra_mask_weight_for_gen > 0) or \ + not self.use_unmasked_for_gen: # == if masked region should be treated differently + mask = self.interpolate_mask(mask, discr_fake_pred.shape[-2:]) + if not self.use_unmasked_for_gen: + fake_loss = fake_loss * mask + else: + pixel_weights = 1 + mask * self.extra_mask_weight_for_gen + fake_loss = fake_loss * pixel_weights + + return fake_loss.mean() * self.weight, dict() + + def pre_discriminator_step(self, real_batch: torch.Tensor, + fake_batch: torch.Tensor, generator: nn.Module, + discriminator: nn.Module): + real_batch.requires_grad = True + + def discriminator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask=None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + + real_loss = F.softplus(-discr_real_pred) + grad_penalty = make_r1_gp(discr_real_pred, real_batch) * self.gp_coef + fake_loss = F.softplus(discr_fake_pred) + + if not self.use_unmasked_for_discr or self.mask_as_fake_target: + # == if masked region should be treated differently + mask = self.interpolate_mask(mask, discr_fake_pred.shape[-2:]) + # use_unmasked_for_discr=False only makes sense for fakes; + # for reals there is no difference beetween two regions + fake_loss = fake_loss * mask + if self.mask_as_fake_target: + fake_loss = fake_loss + (1 + - mask) * F.softplus(-discr_fake_pred) + + sum_discr_loss = real_loss + grad_penalty + fake_loss + metrics = dict( + discr_real_out=discr_real_pred.mean(), + discr_fake_out=discr_fake_pred.mean(), + discr_real_gp=grad_penalty) + return sum_discr_loss.mean(), metrics diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/feature_matching.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/feature_matching.py new file mode 100644 index 0000000000000000000000000000000000000000..c2effb20ffbc5d6871f1cc2df1f1f8e85c6a828d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/feature_matching.py @@ -0,0 +1,45 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +from typing import List + +import torch +import torch.nn.functional as F + + +def masked_l2_loss(pred, target, mask, weight_known, weight_missing): + per_pixel_l2 = F.mse_loss(pred, target, reduction='none') + pixel_weights = mask * weight_missing + (1 - mask) * weight_known + return (pixel_weights * per_pixel_l2).mean() + + +def masked_l1_loss(pred, target, mask, weight_known, weight_missing): + per_pixel_l1 = F.l1_loss(pred, target, reduction='none') + pixel_weights = mask * weight_missing + (1 - mask) * weight_known + return (pixel_weights * per_pixel_l1).mean() + + +def feature_matching_loss(fake_features: List[torch.Tensor], + target_features: List[torch.Tensor], + mask=None): + if mask is None: + res = torch.stack([ + F.mse_loss(fake_feat, target_feat) + for fake_feat, target_feat in zip(fake_features, target_features) + ]).mean() + else: + res = 0 + norm = 0 + for fake_feat, target_feat in zip(fake_features, target_features): + cur_mask = F.interpolate( + mask, + size=fake_feat.shape[-2:], + mode='bilinear', + align_corners=False) + error_weights = 1 - cur_mask + cur_val = ((fake_feat - target_feat).pow(2) * error_weights).mean() + res = res + cur_val + norm += 1 + res = res / norm + return res diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ffc.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ffc.py new file mode 100644 index 0000000000000000000000000000000000000000..c74425e38341809d75997c624d9bc18146897585 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/ffc.py @@ -0,0 +1,588 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from kornia.geometry.transform import rotate + + +def get_activation(kind='tanh'): + if kind == 'tanh': + return nn.Tanh() + if kind == 'sigmoid': + return nn.Sigmoid() + if kind is False: + return nn.Identity() + raise ValueError(f'Unknown activation kind {kind}') + + +class SELayer(nn.Module): + + def __init__(self, channel, reduction=16): + super(SELayer, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), nn.Sigmoid()) + + def forward(self, x): + b, c, _, _ = x.size() + y = self.avg_pool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + res = x * y.expand_as(x) + return res + + +class FourierUnit(nn.Module): + + def __init__(self, + in_channels, + out_channels, + groups=1, + spatial_scale_factor=None, + spatial_scale_mode='bilinear', + spectral_pos_encoding=False, + use_se=False, + se_kwargs=None, + ffc3d=False, + fft_norm='ortho'): + # bn_layer not used + super(FourierUnit, self).__init__() + self.groups = groups + + self.conv_layer = torch.nn.Conv2d( + in_channels=in_channels * 2 + (2 if spectral_pos_encoding else 0), + out_channels=out_channels * 2, + kernel_size=1, + stride=1, + padding=0, + groups=self.groups, + bias=False) + self.bn = torch.nn.BatchNorm2d(out_channels * 2) + self.relu = torch.nn.ReLU(inplace=True) + + # squeeze and excitation block + self.use_se = use_se + if use_se: + if se_kwargs is None: + se_kwargs = {} + self.se = SELayer(self.conv_layer.in_channels, **se_kwargs) + + self.spatial_scale_factor = spatial_scale_factor + self.spatial_scale_mode = spatial_scale_mode + self.spectral_pos_encoding = spectral_pos_encoding + self.ffc3d = ffc3d + self.fft_norm = fft_norm + + def forward(self, x): + batch = x.shape[0] + + if self.spatial_scale_factor is not None: + orig_size = x.shape[-2:] + x = F.interpolate( + x, + scale_factor=self.spatial_scale_factor, + mode=self.spatial_scale_mode, + align_corners=False) + + # (batch, c, h, w/2+1, 2) + fft_dim = (-3, -2, -1) if self.ffc3d else (-2, -1) + ffted = torch.fft.rfftn(x, dim=fft_dim, norm=self.fft_norm) + ffted = torch.stack((ffted.real, ffted.imag), dim=-1) + ffted = ffted.permute(0, 1, 4, 2, + 3).contiguous() # (batch, c, 2, h, w/2+1) + ffted = ffted.view(( + batch, + -1, + ) + ffted.size()[3:]) + + if self.spectral_pos_encoding: + height, width = ffted.shape[-2:] + coords_vert = torch.linspace(0, 1, + height)[None, None, :, None].expand( + batch, 1, height, width).to(ffted) + coords_hor = torch.linspace(0, 1, + width)[None, None, None, :].expand( + batch, 1, height, width).to(ffted) + ffted = torch.cat((coords_vert, coords_hor, ffted), dim=1) + + if self.use_se: + ffted = self.se(ffted) + + ffted = self.conv_layer(ffted) # (batch, c*2, h, w/2+1) + ffted = self.relu(self.bn(ffted)) + + ffted = ffted.view(( + batch, + -1, + 2, + ) + ffted.size()[2:]).permute( + 0, 1, 3, 4, 2).contiguous() # (batch,c, t, h, w/2+1, 2) + ffted = torch.complex(ffted[..., 0], ffted[..., 1]) + + ifft_shape_slice = x.shape[-3:] if self.ffc3d else x.shape[-2:] + output = torch.fft.irfftn( + ffted, s=ifft_shape_slice, dim=fft_dim, norm=self.fft_norm) + + if self.spatial_scale_factor is not None: + output = F.interpolate( + output, + size=orig_size, + mode=self.spatial_scale_mode, + align_corners=False) + + return output + + +class SpectralTransform(nn.Module): + + def __init__(self, + in_channels, + out_channels, + stride=1, + groups=1, + enable_lfu=True, + **fu_kwargs): + # bn_layer not used + super(SpectralTransform, self).__init__() + self.enable_lfu = enable_lfu + if stride == 2: + self.downsample = nn.AvgPool2d(kernel_size=(2, 2), stride=2) + else: + self.downsample = nn.Identity() + + self.stride = stride + self.conv1 = nn.Sequential( + nn.Conv2d( + in_channels, + out_channels // 2, + kernel_size=1, + groups=groups, + bias=False), nn.BatchNorm2d(out_channels // 2), + nn.ReLU(inplace=True)) + self.fu = FourierUnit(out_channels // 2, out_channels // 2, groups, + **fu_kwargs) + if self.enable_lfu: + self.lfu = FourierUnit(out_channels // 2, out_channels // 2, + groups) + self.conv2 = torch.nn.Conv2d( + out_channels // 2, + out_channels, + kernel_size=1, + groups=groups, + bias=False) + + def forward(self, x): + + x = self.downsample(x) + x = self.conv1(x) + output = self.fu(x) + + if self.enable_lfu: + n, c, h, w = x.shape + split_no = 2 + split_s = h // split_no + xs = torch.cat( + torch.split(x[:, :c // 4], split_s, dim=-2), + dim=1).contiguous() + xs = torch.cat( + torch.split(xs, split_s, dim=-1), dim=1).contiguous() + xs = self.lfu(xs) + xs = xs.repeat(1, 1, split_no, split_no).contiguous() + else: + xs = 0 + + output = self.conv2(x + output + xs) + + return output + + +class LearnableSpatialTransformWrapper(nn.Module): + + def __init__(self, + impl, + pad_coef=0.5, + angle_init_range=80, + train_angle=True): + super().__init__() + self.impl = impl + self.angle = torch.rand(1) * angle_init_range + if train_angle: + self.angle = nn.Parameter(self.angle, requires_grad=True) + self.pad_coef = pad_coef + + def forward(self, x): + if torch.is_tensor(x): + return self.inverse_transform(self.impl(self.transform(x)), x) + elif isinstance(x, tuple): + x_trans = tuple(self.transform(elem) for elem in x) + y_trans = self.impl(x_trans) + return tuple( + self.inverse_transform(elem, orig_x) + for elem, orig_x in zip(y_trans, x)) + else: + raise ValueError(f'Unexpected input type {type(x)}') + + def transform(self, x): + height, width = x.shape[2:] + pad_h, pad_w = int(height * self.pad_coef), int(width * self.pad_coef) + x_padded = F.pad(x, [pad_w, pad_w, pad_h, pad_h], mode='reflect') + x_padded_rotated = rotate(x_padded, angle=self.angle.to(x_padded)) + return x_padded_rotated + + def inverse_transform(self, y_padded_rotated, orig_x): + height, width = orig_x.shape[2:] + pad_h, pad_w = int(height * self.pad_coef), int(width * self.pad_coef) + + y_padded = rotate( + y_padded_rotated, angle=-self.angle.to(y_padded_rotated)) + y_height, y_width = y_padded.shape[2:] + y = y_padded[:, :, pad_h:y_height - pad_h, pad_w:y_width - pad_w] + return y + + +class FFC(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + ratio_gin, + ratio_gout, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=False, + enable_lfu=True, + padding_type='reflect', + gated=False, + **spectral_kwargs): + super(FFC, self).__init__() + + assert stride == 1 or stride == 2, 'Stride should be 1 or 2.' + self.stride = stride + + in_cg = int(in_channels * ratio_gin) + in_cl = in_channels - in_cg + out_cg = int(out_channels * ratio_gout) + out_cl = out_channels - out_cg + + self.ratio_gin = ratio_gin + self.ratio_gout = ratio_gout + self.global_in_num = in_cg + + module = nn.Identity if in_cl == 0 or out_cl == 0 else nn.Conv2d + self.convl2l = module( + in_cl, + out_cl, + kernel_size, + stride, + padding, + dilation, + groups, + bias, + padding_mode=padding_type) + module = nn.Identity if in_cl == 0 or out_cg == 0 else nn.Conv2d + self.convl2g = module( + in_cl, + out_cg, + kernel_size, + stride, + padding, + dilation, + groups, + bias, + padding_mode=padding_type) + module = nn.Identity if in_cg == 0 or out_cl == 0 else nn.Conv2d + self.convg2l = module( + in_cg, + out_cl, + kernel_size, + stride, + padding, + dilation, + groups, + bias, + padding_mode=padding_type) + module = nn.Identity if in_cg == 0 or out_cg == 0 else SpectralTransform + self.convg2g = module(in_cg, out_cg, stride, + 1 if groups == 1 else groups // 2, enable_lfu, + **spectral_kwargs) + + self.gated = gated + module = nn.Identity if in_cg == 0 or out_cl == 0 or not self.gated else nn.Conv2d + self.gate = module(in_channels, 2, 1) + + def forward(self, x): + x_l, x_g = x if type(x) is tuple else (x, 0) + out_xl, out_xg = 0, 0 + + if self.gated: + total_input_parts = [x_l] + if torch.is_tensor(x_g): + total_input_parts.append(x_g) + total_input = torch.cat(total_input_parts, dim=1) + + gates = torch.sigmoid(self.gate(total_input)) + g2l_gate, l2g_gate = gates.chunk(2, dim=1) + else: + g2l_gate, l2g_gate = 1, 1 + + if self.ratio_gout != 1: + out_xl = self.convl2l(x_l) + self.convg2l(x_g) * g2l_gate + if self.ratio_gout != 0: + out_xg = self.convl2g(x_l) * l2g_gate + self.convg2g(x_g) + + return out_xl, out_xg + + +class FFC_BN_ACT(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + ratio_gin, + ratio_gout, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=False, + norm_layer=nn.BatchNorm2d, + activation_layer=nn.Identity, + padding_type='reflect', + enable_lfu=True, + **kwargs): + super(FFC_BN_ACT, self).__init__() + self.ffc = FFC( + in_channels, + out_channels, + kernel_size, + ratio_gin, + ratio_gout, + stride, + padding, + dilation, + groups, + bias, + enable_lfu, + padding_type=padding_type, + **kwargs) + lnorm = nn.Identity if ratio_gout == 1 else norm_layer + gnorm = nn.Identity if ratio_gout == 0 else norm_layer + global_channels = int(out_channels * ratio_gout) + self.bn_l = lnorm(out_channels - global_channels) + self.bn_g = gnorm(global_channels) + + lact = nn.Identity if ratio_gout == 1 else activation_layer + gact = nn.Identity if ratio_gout == 0 else activation_layer + self.act_l = lact(inplace=True) + self.act_g = gact(inplace=True) + + def forward(self, x): + x_l, x_g = self.ffc(x) + x_l = self.act_l(self.bn_l(x_l)) + x_g = self.act_g(self.bn_g(x_g)) + return x_l, x_g + + +class FFCResnetBlock(nn.Module): + + def __init__(self, + dim, + padding_type, + norm_layer, + activation_layer=nn.ReLU, + dilation=1, + spatial_transform_kwargs=None, + inline=False, + **conv_kwargs): + super().__init__() + self.conv1 = FFC_BN_ACT( + dim, + dim, + kernel_size=3, + padding=dilation, + dilation=dilation, + norm_layer=norm_layer, + activation_layer=activation_layer, + padding_type=padding_type, + **conv_kwargs) + self.conv2 = FFC_BN_ACT( + dim, + dim, + kernel_size=3, + padding=dilation, + dilation=dilation, + norm_layer=norm_layer, + activation_layer=activation_layer, + padding_type=padding_type, + **conv_kwargs) + if spatial_transform_kwargs is not None: + self.conv1 = LearnableSpatialTransformWrapper( + self.conv1, **spatial_transform_kwargs) + self.conv2 = LearnableSpatialTransformWrapper( + self.conv2, **spatial_transform_kwargs) + self.inline = inline + + def forward(self, x): + if self.inline: + x_l, x_g = x[:, :-self.conv1.ffc. + global_in_num], x[:, -self.conv1.ffc.global_in_num:] + else: + x_l, x_g = x if type(x) is tuple else (x, 0) + + id_l, id_g = x_l, x_g + + x_l, x_g = self.conv1((x_l, x_g)) + x_l, x_g = self.conv2((x_l, x_g)) + + x_l, x_g = id_l + x_l, id_g + x_g + out = x_l, x_g + if self.inline: + out = torch.cat(out, dim=1) + return out + + +class ConcatTupleLayer(nn.Module): + + def forward(self, x): + assert isinstance(x, tuple) + x_l, x_g = x + assert torch.is_tensor(x_l) or torch.is_tensor(x_g) + if not torch.is_tensor(x_g): + return x_l + return torch.cat(x, dim=1) + + +class FFCResNetGenerator(nn.Module): + + def __init__(self, + input_nc=4, + output_nc=3, + ngf=64, + n_downsampling=3, + n_blocks=18, + norm_layer=nn.BatchNorm2d, + padding_type='reflect', + activation_layer=nn.ReLU, + up_norm_layer=nn.BatchNorm2d, + up_activation=nn.ReLU(True), + init_conv_kwargs={ + 'ratio_gin': 0, + 'ratio_gout': 0, + 'enable_lfu': False + }, + downsample_conv_kwargs={ + 'ratio_gin': 0, + 'ratio_gout': 0, + 'enable_lfu': False + }, + resnet_conv_kwargs={ + 'ratio_gin': 0.75, + 'ratio_gout': 0.75, + 'enable_lfu': False + }, + spatial_transform_layers=None, + spatial_transform_kwargs={}, + add_out_act='sigmoid', + max_features=1024, + out_ffc=False, + out_ffc_kwargs={}): + assert (n_blocks >= 0) + super().__init__() + + model = [ + nn.ReflectionPad2d(3), + FFC_BN_ACT( + input_nc, + ngf, + kernel_size=7, + padding=0, + norm_layer=norm_layer, + activation_layer=activation_layer, + **init_conv_kwargs) + ] + + # downsample + for i in range(n_downsampling): + mult = 2**i + if i == n_downsampling - 1: + cur_conv_kwargs = dict(downsample_conv_kwargs) + cur_conv_kwargs['ratio_gout'] = resnet_conv_kwargs.get( + 'ratio_gin', 0) + else: + cur_conv_kwargs = downsample_conv_kwargs + model += [ + FFC_BN_ACT( + min(max_features, ngf * mult), + min(max_features, ngf * mult * 2), + kernel_size=3, + stride=2, + padding=1, + norm_layer=norm_layer, + activation_layer=activation_layer, + **cur_conv_kwargs) + ] + + mult = 2**n_downsampling + feats_num_bottleneck = min(max_features, ngf * mult) + + # resnet blocks + for i in range(n_blocks): + cur_resblock = FFCResnetBlock( + feats_num_bottleneck, + padding_type=padding_type, + activation_layer=activation_layer, + norm_layer=norm_layer, + **resnet_conv_kwargs) + if spatial_transform_layers is not None and i in spatial_transform_layers: + cur_resblock = LearnableSpatialTransformWrapper( + cur_resblock, **spatial_transform_kwargs) + model += [cur_resblock] + + model += [ConcatTupleLayer()] + + # upsample + for i in range(n_downsampling): + mult = 2**(n_downsampling - i) + model += [ + nn.ConvTranspose2d( + min(max_features, ngf * mult), + min(max_features, int(ngf * mult / 2)), + kernel_size=3, + stride=2, + padding=1, + output_padding=1), + up_norm_layer(min(max_features, int(ngf * mult / 2))), + up_activation + ] + + if out_ffc: + model += [ + FFCResnetBlock( + ngf, + padding_type=padding_type, + activation_layer=activation_layer, + norm_layer=norm_layer, + inline=True, + **out_ffc_kwargs) + ] + + model += [ + nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0) + ] + if add_out_act: + model.append( + get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/inception.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/inception.py new file mode 100644 index 0000000000000000000000000000000000000000..5070533d8c27d7567bdd3ab230571c04412c921a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/inception.py @@ -0,0 +1,324 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import models + +from modelscope.utils.logger import get_logger + +try: + from torchvision.models.utils import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + +# Inception weights ported to Pytorch from +# http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz +FID_WEIGHTS_URL = 'https://github.com/mseitzer/pytorch-fid/releases/download/' \ + 'fid_weights/pt_inception-2015-12-05-6726825d.pth' + +LOGGER = get_logger() + + +class InceptionV3(nn.Module): + """Pretrained InceptionV3 network returning feature maps""" + + # Index of default block of inception to return, + # corresponds to output of final average pooling + DEFAULT_BLOCK_INDEX = 3 + + # Maps feature dimensionality to their output blocks indices + BLOCK_INDEX_BY_DIM = { + 64: 0, # First max pooling features + 192: 1, # Second max pooling featurs + 768: 2, # Pre-aux classifier features + 2048: 3 # Final average pooling features + } + + def __init__(self, + output_blocks=[DEFAULT_BLOCK_INDEX], + resize_input=True, + normalize_input=True, + requires_grad=False, + use_fid_inception=True): + """Build pretrained InceptionV3 + + Parameters + ---------- + output_blocks : list of int + Indices of blocks to return features of. Possible values are: + - 0: corresponds to output of first max pooling + - 1: corresponds to output of second max pooling + - 2: corresponds to output which is fed to aux classifier + - 3: corresponds to output of final average pooling + resize_input : bool + If true, bilinearly resizes input to width and height 299 before + feeding input to model. As the network without fully connected + layers is fully convolutional, it should be able to handle inputs + of arbitrary size, so resizing might not be strictly needed + normalize_input : bool + If true, scales the input from range (0, 1) to the range the + pretrained Inception network expects, namely (-1, 1) + requires_grad : bool + If true, parameters of the model require gradients. Possibly useful + for finetuning the network + use_fid_inception : bool + If true, uses the pretrained Inception model used in Tensorflow's + FID implementation. If false, uses the pretrained Inception model + available in torchvision. The FID Inception model has different + weights and a slightly different structure from torchvision's + Inception model. If you want to compute FID scores, you are + strongly advised to set this parameter to true to get comparable + results. + """ + super(InceptionV3, self).__init__() + + self.resize_input = resize_input + self.normalize_input = normalize_input + self.output_blocks = sorted(output_blocks) + self.last_needed_block = max(output_blocks) + + assert self.last_needed_block <= 3, \ + 'Last possible output block index is 3' + + self.blocks = nn.ModuleList() + + if use_fid_inception: + inception = fid_inception_v3() + else: + inception = models.inception_v3(pretrained=True) + + # Block 0: input to maxpool1 + block0 = [ + inception.Conv2d_1a_3x3, inception.Conv2d_2a_3x3, + inception.Conv2d_2b_3x3, + nn.MaxPool2d(kernel_size=3, stride=2) + ] + self.blocks.append(nn.Sequential(*block0)) + + # Block 1: maxpool1 to maxpool2 + if self.last_needed_block >= 1: + block1 = [ + inception.Conv2d_3b_1x1, inception.Conv2d_4a_3x3, + nn.MaxPool2d(kernel_size=3, stride=2) + ] + self.blocks.append(nn.Sequential(*block1)) + + # Block 2: maxpool2 to aux classifier + if self.last_needed_block >= 2: + block2 = [ + inception.Mixed_5b, + inception.Mixed_5c, + inception.Mixed_5d, + inception.Mixed_6a, + inception.Mixed_6b, + inception.Mixed_6c, + inception.Mixed_6d, + inception.Mixed_6e, + ] + self.blocks.append(nn.Sequential(*block2)) + + # Block 3: aux classifier to final avgpool + if self.last_needed_block >= 3: + block3 = [ + inception.Mixed_7a, inception.Mixed_7b, inception.Mixed_7c, + nn.AdaptiveAvgPool2d(output_size=(1, 1)) + ] + self.blocks.append(nn.Sequential(*block3)) + + for param in self.parameters(): + param.requires_grad = requires_grad + + def forward(self, inp): + """Get Inception feature maps + + Parameters + ---------- + inp : torch.autograd.Variable + Input tensor of shape Bx3xHxW. Values are expected to be in + range (0, 1) + + Returns + ------- + List of torch.autograd.Variable, corresponding to the selected output + block, sorted ascending by index + """ + outp = [] + x = inp + + if self.resize_input: + x = F.interpolate( + x, size=(299, 299), mode='bilinear', align_corners=False) + + if self.normalize_input: + x = 2 * x - 1 # Scale from range (0, 1) to range (-1, 1) + + for idx, block in enumerate(self.blocks): + x = block(x) + if idx in self.output_blocks: + outp.append(x) + + if idx == self.last_needed_block: + break + + return outp + + +def fid_inception_v3(): + """Build pretrained Inception model for FID computation + + The Inception model for FID computation uses a different set of weights + and has a slightly different structure than torchvision's Inception. + + This method first constructs torchvision's Inception and then patches the + necessary parts that are different in the FID Inception model. + """ + LOGGER.info('fid_inception_v3 called') + inception = models.inception_v3( + num_classes=1008, aux_logits=False, pretrained=False) + LOGGER.info('models.inception_v3 done') + inception.Mixed_5b = FIDInceptionA(192, pool_features=32) + inception.Mixed_5c = FIDInceptionA(256, pool_features=64) + inception.Mixed_5d = FIDInceptionA(288, pool_features=64) + inception.Mixed_6b = FIDInceptionC(768, channels_7x7=128) + inception.Mixed_6c = FIDInceptionC(768, channels_7x7=160) + inception.Mixed_6d = FIDInceptionC(768, channels_7x7=160) + inception.Mixed_6e = FIDInceptionC(768, channels_7x7=192) + inception.Mixed_7b = FIDInceptionE_1(1280) + inception.Mixed_7c = FIDInceptionE_2(2048) + + LOGGER.info('fid_inception_v3 patching done') + + state_dict = load_state_dict_from_url(FID_WEIGHTS_URL, progress=True) + LOGGER.info('fid_inception_v3 weights downloaded') + + inception.load_state_dict(state_dict) + LOGGER.info('fid_inception_v3 weights loaded into model') + + return inception + + +class FIDInceptionA(models.inception.InceptionA): + """InceptionA block patched for FID computation""" + + def __init__(self, in_channels, pool_features): + super(FIDInceptionA, self).__init__(in_channels, pool_features) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch5x5 = self.branch5x5_1(x) + branch5x5 = self.branch5x5_2(branch5x5) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl) + + # Patch: Tensorflow's average pool does not use the padded zero's in + # its average calculation + branch_pool = F.avg_pool2d( + x, kernel_size=3, stride=1, padding=1, count_include_pad=False) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) + + +class FIDInceptionC(models.inception.InceptionC): + """InceptionC block patched for FID computation""" + + def __init__(self, in_channels, channels_7x7): + super(FIDInceptionC, self).__init__(in_channels, channels_7x7) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch7x7 = self.branch7x7_1(x) + branch7x7 = self.branch7x7_2(branch7x7) + branch7x7 = self.branch7x7_3(branch7x7) + + branch7x7dbl = self.branch7x7dbl_1(x) + branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl) + + # Patch: Tensorflow's average pool does not use the padded zero's in + # its average calculation + branch_pool = F.avg_pool2d( + x, kernel_size=3, stride=1, padding=1, count_include_pad=False) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool] + return torch.cat(outputs, 1) + + +class FIDInceptionE_1(models.inception.InceptionE): + """First InceptionE block patched for FID computation""" + + def __init__(self, in_channels): + super(FIDInceptionE_1, self).__init__(in_channels) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch3x3 = self.branch3x3_1(x) + branch3x3 = [ + self.branch3x3_2a(branch3x3), + self.branch3x3_2b(branch3x3), + ] + branch3x3 = torch.cat(branch3x3, 1) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = [ + self.branch3x3dbl_3a(branch3x3dbl), + self.branch3x3dbl_3b(branch3x3dbl), + ] + branch3x3dbl = torch.cat(branch3x3dbl, 1) + + # Patch: Tensorflow's average pool does not use the padded zero's in + # its average calculation + branch_pool = F.avg_pool2d( + x, kernel_size=3, stride=1, padding=1, count_include_pad=False) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) + + +class FIDInceptionE_2(models.inception.InceptionE): + """Second InceptionE block patched for FID computation""" + + def __init__(self, in_channels): + super(FIDInceptionE_2, self).__init__(in_channels) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch3x3 = self.branch3x3_1(x) + branch3x3 = [ + self.branch3x3_2a(branch3x3), + self.branch3x3_2b(branch3x3), + ] + branch3x3 = torch.cat(branch3x3, 1) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = [ + self.branch3x3dbl_3a(branch3x3dbl), + self.branch3x3dbl_3b(branch3x3dbl), + ] + branch3x3dbl = torch.cat(branch3x3dbl, 1) + + # Patch: The FID Inception model uses max pooling instead of average + # pooling. This is likely an error in this specific Inception + # implementation, as other Inception models use average pooling here + # (which matches the description in the paper). + branch_pool = F.max_pool2d(x, kernel_size=3, stride=1, padding=1) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/perceptual.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/perceptual.py new file mode 100644 index 0000000000000000000000000000000000000000..80fe2b968acaf72d61bf167410a5e057f9ce4e71 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/perceptual.py @@ -0,0 +1,47 @@ +""" +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +""" +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +from .ade20k import ModelBuilder + +IMAGENET_MEAN = torch.FloatTensor([0.485, 0.456, 0.406])[None, :, None, None] +IMAGENET_STD = torch.FloatTensor([0.229, 0.224, 0.225])[None, :, None, None] + + +class ResNetPL(nn.Module): + + def __init__(self, + weight=1, + weights_path=None, + arch_encoder='resnet50dilated', + segmentation=True): + super().__init__() + self.impl = ModelBuilder.get_encoder( + weights_path=weights_path, + arch_encoder=arch_encoder, + arch_decoder='ppm_deepsup', + fc_dim=2048, + segmentation=segmentation) + self.impl.eval() + for w in self.impl.parameters(): + w.requires_grad_(False) + + self.weight = weight + + def forward(self, pred, target): + pred = (pred - IMAGENET_MEAN.to(pred)) / IMAGENET_STD.to(pred) + target = (target - IMAGENET_MEAN.to(target)) / IMAGENET_STD.to(target) + + pred_feats = self.impl(pred, return_feature_maps=True) + target_feats = self.impl(target, return_feature_maps=True) + + result = torch.stack([ + F.mse_loss(cur_pred, cur_target) + for cur_pred, cur_target in zip(pred_feats, target_feats) + ]).sum() * self.weight + return result diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/modules/pix2pixhd.py b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/pix2pixhd.py new file mode 100644 index 0000000000000000000000000000000000000000..32e18f3ecd5bbe41debeca59f311687221c00c58 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/modules/pix2pixhd.py @@ -0,0 +1,75 @@ +""" +The implementation is adopted from +https://github.com/NVIDIA/pix2pixHD/blob/master/models/networks.py +""" +import collections +import functools +import logging +from collections import defaultdict +from functools import partial + +import numpy as np +import torch.nn as nn + + +# Defines the PatchGAN discriminator with the specified arguments. +class NLayerDiscriminator(nn.Module): + + def __init__( + self, + input_nc=3, + ndf=64, + n_layers=4, + norm_layer=nn.BatchNorm2d, + ): + super().__init__() + self.n_layers = n_layers + + kw = 4 + padw = int(np.ceil((kw - 1.0) / 2)) + sequence = [[ + nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), + nn.LeakyReLU(0.2, True) + ]] + + nf = ndf + for n in range(1, n_layers): + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=2, padding=padw), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=1, padding=padw), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + sequence += [[ + nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw) + ]] + + for n in range(len(sequence)): + setattr(self, 'model' + str(n), nn.Sequential(*sequence[n])) + + def get_all_activations(self, x): + res = [x] + for n in range(self.n_layers + 2): + model = getattr(self, 'model' + str(n)) + res.append(model(res[-1])) + return res[1:] + + def forward(self, x): + act = self.get_all_activations(x) + return act[-1], act[:-1] diff --git a/runtime/Lib/site_packages/models/cv/image_inpainting/refinement.py b/runtime/Lib/site_packages/models/cv/image_inpainting/refinement.py new file mode 100644 index 0000000000000000000000000000000000000000..662d8a05c2a653189ed23d3004fd629f90a263d8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_inpainting/refinement.py @@ -0,0 +1,393 @@ +''' +Part of the implementation is borrowed and modified from LaMa, publicly available at +https://github.com/saic-mdal/lama +''' +import cv2 +import numpy as np +import torch +import torch.nn as nn +from kornia.filters import gaussian_blur2d +from kornia.geometry.transform import resize +from kornia.morphology import erosion +from torch.nn import functional as F +from torch.optim import SGD, Adam +from tqdm import tqdm + +from .modules.ffc import FFCResnetBlock + + +def move_to_device(obj, device): + if isinstance(obj, nn.Module): + return obj.to(device) + if torch.is_tensor(obj): + return obj.to(device) + if isinstance(obj, (tuple, list)): + return [move_to_device(el, device) for el in obj] + if isinstance(obj, dict): + return {name: move_to_device(val, device) for name, val in obj.items()} + raise ValueError(f'Unexpected type {type(obj)}') + + +def ceil_modulo(x, mod): + if x % mod == 0: + return x + return (x // mod + 1) * mod + + +def pad_tensor_to_modulo(img, mod): + batch_size, channels, height, width = img.shape + out_height = ceil_modulo(height, mod) + out_width = ceil_modulo(width, mod) + return F.pad( + img, + pad=(0, out_width - width, 0, out_height - height), + mode='reflect') + + +def _pyrdown(im: torch.Tensor, downsize: tuple = None): + """downscale the image""" + if downsize is None: + downsize = (im.shape[2] // 2, im.shape[3] // 2) + assert im.shape[ + 1] == 3, 'Expected shape for the input to be (n,3,height,width)' + im = gaussian_blur2d(im, kernel_size=(5, 5), sigma=(1.0, 1.0)) + im = F.interpolate(im, size=downsize, mode='bilinear', align_corners=False) + return im + + +def _pyrdown_mask(mask: torch.Tensor, + downsize: tuple = None, + eps: float = 1e-8, + blur_mask: bool = True, + round_up: bool = True): + """downscale the mask tensor + + Parameters + ---------- + mask : torch.Tensor + mask of size (B, 1, H, W) + downsize : tuple, optional + size to downscale to. If None, image is downscaled to half, by default None + eps : float, optional + threshold value for binarizing the mask, by default 1e-8 + blur_mask : bool, optional + if True, apply gaussian filter before downscaling, by default True + round_up : bool, optional + if True, values above eps are marked 1, else, values below 1-eps are marked 0, by default True + + Returns + ------- + torch.Tensor + downscaled mask + """ + + if downsize is None: + downsize = (mask.shape[2] // 2, mask.shape[3] // 2) + assert mask.shape[ + 1] == 1, 'Expected shape for the input to be (n,1,height,width)' + if blur_mask is True: + mask = gaussian_blur2d(mask, kernel_size=(5, 5), sigma=(1.0, 1.0)) + mask = F.interpolate( + mask, size=downsize, mode='bilinear', align_corners=False) + else: + mask = F.interpolate( + mask, size=downsize, mode='bilinear', align_corners=False) + if round_up: + mask[mask >= eps] = 1 + mask[mask < eps] = 0 + else: + mask[mask >= 1.0 - eps] = 1 + mask[mask < 1.0 - eps] = 0 + return mask + + +def _erode_mask(mask: torch.Tensor, + ekernel: torch.Tensor = None, + eps: float = 1e-8): + """erode the mask, and set gray pixels to 0""" + if ekernel is not None: + mask = erosion(mask, ekernel) + mask[mask >= 1.0 - eps] = 1 + mask[mask < 1.0 - eps] = 0 + return mask + + +def _l1_loss(pred: torch.Tensor, + pred_downscaled: torch.Tensor, + ref: torch.Tensor, + mask: torch.Tensor, + mask_downscaled: torch.Tensor, + image: torch.Tensor, + on_pred: bool = True): + """l1 loss on src pixels, and downscaled predictions if on_pred=True""" + loss = torch.mean(torch.abs(pred[mask < 1e-8] - image[mask < 1e-8])) + if on_pred: + loss += torch.mean( + torch.abs(pred_downscaled[mask_downscaled >= 1e-8] + - ref[mask_downscaled >= 1e-8])) + return loss + + +def _infer(image: torch.Tensor, + mask: torch.Tensor, + forward_front: nn.Module, + forward_rears: nn.Module, + ref_lower_res: torch.Tensor, + orig_shape: tuple, + devices: list, + scale_ind: int, + n_iters: int = 15, + lr: float = 0.002): + """Performs inference with refinement at a given scale. + + Parameters + ---------- + image : torch.Tensor + input image to be inpainted, of size (1,3,H,W) + mask : torch.Tensor + input inpainting mask, of size (1,1,H,W) + forward_front : nn.Module + the front part of the inpainting network + forward_rears : nn.Module + the rear part of the inpainting network + ref_lower_res : torch.Tensor + the inpainting at previous scale, used as reference image + orig_shape : tuple + shape of the original input image before padding + devices : list + list of available devices + scale_ind : int + the scale index + n_iters : int, optional + number of iterations of refinement, by default 15 + lr : float, optional + learning rate, by default 0.002 + + Returns + ------- + torch.Tensor + inpainted image + """ + masked_image = image * (1 - mask) + masked_image = torch.cat([masked_image, mask], dim=1) + + mask = mask.repeat(1, 3, 1, 1) + if ref_lower_res is not None: + ref_lower_res = ref_lower_res.detach() + with torch.no_grad(): + z1, z2 = forward_front(masked_image) + # Inference + mask = mask.to(devices[-1]) + ekernel = torch.from_numpy( + cv2.getStructuringElement(cv2.MORPH_ELLIPSE, + (15, 15)).astype(bool)).float() + ekernel = ekernel.to(devices[-1]) + image = image.to(devices[-1]) + z1, z2 = z1.detach().to(devices[0]), z2.detach().to(devices[0]) + z1.requires_grad, z2.requires_grad = True, True + + optimizer = Adam([z1, z2], lr=lr) + + pbar = tqdm(range(n_iters), leave=False) + for idi in pbar: + optimizer.zero_grad() + input_feat = (z1, z2) + for idd, forward_rear in enumerate(forward_rears): + output_feat = forward_rear(input_feat) + if idd < len(devices) - 1: + midz1, midz2 = output_feat + midz1, midz2 = midz1.to(devices[idd + 1]), midz2.to( + devices[idd + 1]) + input_feat = (midz1, midz2) + else: + pred = output_feat + + if ref_lower_res is None: + break + losses = {} + # scaled loss with downsampler + pred_downscaled = _pyrdown(pred[:, :, :orig_shape[0], :orig_shape[1]]) + mask_downscaled = _pyrdown_mask( + mask[:, :1, :orig_shape[0], :orig_shape[1]], + blur_mask=False, + round_up=False) + mask_downscaled = _erode_mask(mask_downscaled, ekernel=ekernel) + mask_downscaled = mask_downscaled.repeat(1, 3, 1, 1) + losses['ms_l1'] = _l1_loss( + pred, + pred_downscaled, + ref_lower_res, + mask, + mask_downscaled, + image, + on_pred=True) + + loss = sum(losses.values()) + pbar.set_description( + 'Refining scale {} using scale {} ...current loss: {:.4f}'.format( + scale_ind + 1, scale_ind, loss.item())) + if idi < n_iters - 1: + loss.backward() + optimizer.step() + del pred_downscaled + del loss + del pred + # "pred" is the prediction after Plug-n-Play module + inpainted = mask * pred + (1 - mask) * image + inpainted = inpainted.detach().cpu() + return inpainted + + +def _get_image_mask_pyramid(batch: dict, min_side: int, max_scales: int, + px_budget: int): + """Build the image mask pyramid + + Parameters + ---------- + batch : dict + batch containing image, mask, etc + min_side : int + minimum side length to limit the number of scales of the pyramid + max_scales : int + maximum number of scales allowed + px_budget : int + the product H*W cannot exceed this budget, because of resource constraints + + Returns + ------- + tuple + image-mask pyramid in the form of list of images and list of masks + """ + + assert batch['image'].shape[ + 0] == 1, 'refiner works on only batches of size 1!' + + h, w = batch['unpad_to_size'] + h, w = h[0].item(), w[0].item() + + image = batch['image'][..., :h, :w] + mask = batch['mask'][..., :h, :w] + if h * w > px_budget: + # resize + ratio = np.sqrt(px_budget / float(h * w)) + h_orig, w_orig = h, w + h, w = int(h * ratio), int(w * ratio) + print( + f'Original image too large for refinement! Resizing {(h_orig,w_orig)} to {(h,w)}...' + ) + image = resize( + image, (h, w), interpolation='bilinear', align_corners=False) + mask = resize( + mask, (h, w), interpolation='bilinear', align_corners=False) + mask[mask > 1e-8] = 1 + breadth = min(h, w) + n_scales = min(1 + int(round(max(0, np.log2(breadth / min_side)))), + max_scales) + ls_images = [] + ls_masks = [] + + ls_images.append(image) + ls_masks.append(mask) + + for _ in range(n_scales - 1): + image_p = _pyrdown(ls_images[-1]) + mask_p = _pyrdown_mask(ls_masks[-1]) + ls_images.append(image_p) + ls_masks.append(mask_p) + # reverse the lists because we want the lowest resolution image as index 0 + return ls_images[::-1], ls_masks[::-1] + + +def refine_predict(batch: dict, inpainter: nn.Module, gpu_ids: str, + modulo: int, n_iters: int, lr: float, min_side: int, + max_scales: int, px_budget: int): + """Refines the inpainting of the network + + Parameters + ---------- + batch : dict + image-mask batch, currently we assume the batchsize to be 1 + inpainter : nn.Module + the inpainting neural network + gpu_ids : str + the GPU ids of the machine to use. If only single GPU, use: "0," + modulo : int + pad the image to ensure dimension % modulo == 0 + n_iters : int + number of iterations of refinement for each scale + lr : float + learning rate + min_side : int + all sides of image on all scales should be >= min_side / sqrt(2) + max_scales : int + max number of downscaling scales for the image-mask pyramid + px_budget : int + pixels budget. Any image will be resized to satisfy height*width <= px_budget + + Returns + ------- + torch.Tensor + inpainted image of size (1,3,H,W) + """ + inpainter = inpainter.model + assert not inpainter.training + assert not inpainter.add_noise_kwargs + assert inpainter.concat_mask + + gpu_ids = [ + f'cuda:{gpuid}' for gpuid in gpu_ids.replace(' ', '').split(',') + if gpuid.isdigit() + ] + n_resnet_blocks = 0 + first_resblock_ind = 0 + found_first_resblock = False + for idl in range(len(inpainter.generator.model)): + if isinstance(inpainter.generator.model[idl], FFCResnetBlock): + n_resnet_blocks += 1 + found_first_resblock = True + elif not found_first_resblock: + first_resblock_ind += 1 + resblocks_per_gpu = n_resnet_blocks // len(gpu_ids) + + devices = [torch.device(gpu_id) for gpu_id in gpu_ids] + + # split the model into front, and rear parts + forward_front = inpainter.generator.model[0:first_resblock_ind] + forward_front.to(devices[0]) + forward_rears = [] + for idd in range(len(gpu_ids)): + if idd < len(gpu_ids) - 1: + forward_rears.append( + inpainter.generator.model[first_resblock_ind + + resblocks_per_gpu + * (idd):first_resblock_ind + + resblocks_per_gpu * (idd + 1)]) + else: + forward_rears.append( + inpainter.generator.model[first_resblock_ind + + resblocks_per_gpu * (idd):]) + forward_rears[idd].to(devices[idd]) + + ls_images, ls_masks = _get_image_mask_pyramid(batch, min_side, max_scales, + px_budget) + image_inpainted = None + + for ids, (image, mask) in enumerate(zip(ls_images, ls_masks)): + orig_shape = image.shape[2:] + image = pad_tensor_to_modulo(image, modulo) + mask = pad_tensor_to_modulo(mask, modulo) + mask[mask >= 1e-8] = 1.0 + mask[mask < 1e-8] = 0.0 + image, mask = move_to_device(image, devices[0]), move_to_device( + mask, devices[0]) + if image_inpainted is not None: + image_inpainted = move_to_device(image_inpainted, devices[-1]) + image_inpainted = _infer(image, mask, forward_front, forward_rears, + image_inpainted, orig_shape, devices, ids, + n_iters, lr) + image_inpainted = image_inpainted[:, :, :orig_shape[0], :orig_shape[1]] + # detach everything to save resources + image = image.detach().cpu() + mask = mask.detach().cpu() + + return image_inpainted diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8041a7e7aa4470df524de62fc71770bfea734750 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .cascade_mask_rcnn_swin import CascadeMaskRCNNSwin + from .maskdino_swin import MaskDINOSwin + from .model import CascadeMaskRCNNSwinModel + from .maskdino_model import MaskDINOSwinModel + from .fastinst_model import FastInst + from .postprocess_utils import get_img_ins_seg_result, get_maskdino_ins_seg_result +else: + _import_structure = { + 'cascade_mask_rcnn_swin': ['CascadeMaskRCNNSwin'], + 'fastinst_model': ['FastInst'], + 'maskdino_swin': ['MaskDINOSwin'], + 'model': ['CascadeMaskRCNNSwinModel'], + 'maskdino_model': ['MaskDINOSwinModel'], + 'postprocess_utils': + ['get_img_ins_seg_result', 'get_maskdino_ins_seg_result'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1e7325f32e24b03aeb9fa2336edaf3de70c3035f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .swin_transformer import SwinTransformer + from .swin_transformer import D2SwinTransformer + from .resnet import build_resnet_backbone + +else: + _import_structure = { + 'swin_transformer': ['SwinTransformer', 'D2SwinTransformer'], + 'resnet': ['build_resnet_backbone'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/resnet.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..4e2a5ec14c991b609e5742407c9b00a180512c83 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/resnet.py @@ -0,0 +1,114 @@ +# Part of the implementation is borrowed and modified from Detectron2, publicly available at +# https://github.com/facebookresearch/detectron2/blob/main/projects/DeepLab/deeplab/resnet.py + +import torch.nn.functional as F +from torch import nn + +from modelscope.models.cv.image_human_parsing.backbone.deeplab_resnet import ( + BottleneckBlock, DeeplabResNet, get_norm) +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +class BasicStem(nn.Module): + """ + The standard ResNet stem (layers before the first residual block), + with a conv, relu and max_pool. + """ + + def __init__(self, in_channels=3, out_channels=64, norm='BN'): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = 4 + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +def build_resnet_backbone(out_features, depth, num_groups, width_per_group, + norm, stem_out_channels, res2_out_channels, + stride_in_1x1, res4_dilation, res5_dilation, + res5_multi_grid, input_shape): + stem = BasicStem( + in_channels=input_shape['channels'], + out_channels=stem_out_channels, + norm=norm) + bottleneck_channels = num_groups * width_per_group + in_channels = stem_out_channels + out_channels = res2_out_channels + + assert res4_dilation in { + 1, 2 + }, 'res4_dilation cannot be {}.'.format(res4_dilation) + assert res5_dilation in { + 1, 2, 4 + }, 'res5_dilation cannot be {}.'.format(res5_dilation) + if res4_dilation == 2: + # Always dilate res5 if res4 is dilated. + assert res5_dilation == 4 + + num_blocks_per_stage = { + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3] + }[depth] + + stages = [] + out_stage_idx = [{ + 'res2': 2, + 'res3': 3, + 'res4': 4, + 'res5': 5 + }[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + if stage_idx == 4: + dilation = res4_dilation + elif stage_idx == 5: + dilation = res5_dilation + else: + dilation = 1 + first_stride = 1 if idx == 0 or dilation > 1 else 2 + stride_per_block = [first_stride] + stride_per_block += [1] * (num_blocks_per_stage[idx] - 1) + stage_kargs = { + 'num_blocks': num_blocks_per_stage[idx], + 'stride_per_block': stride_per_block, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'norm': norm, + 'bottleneck_channels': bottleneck_channels, + 'stride_in_1x1': stride_in_1x1, + 'dilation': dilation, + 'num_groups': num_groups, + 'block_class': BottleneckBlock + } + if stage_idx == 5: + stage_kargs.pop('dilation') + stage_kargs['dilation_per_block'] = [ + dilation * mg for mg in res5_multi_grid + ] + blocks = DeeplabResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return DeeplabResNet(stem, stages, out_features=out_features) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/swin_transformer.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..09ab7c20cfa7aee5d4d7e0808df141b77e53311d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/backbones/swin_transformer.py @@ -0,0 +1,747 @@ +# The implementation is adopted from Swin Transformer, made publicly available under the MIT License at +# https://github.com/microsoft/Swin-Transformer/blob/main/models/swin_transformer.py +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """ Swin Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, 'input feature has wrong size' + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, + Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, 'input feature has wrong size' + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=4, + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, + (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """ Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + + Inspiration from + https://github.com/SwinTransformer/Swin-Transformer-Object-Detection + + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1] + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], + patches_resolution[1])) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if + (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self): + """Initialize the weights in backbone.""" + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + self.apply(_init_weights) + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic') + x = (x + absolute_pos_embed).flatten(2).transpose(1, + 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, + self.num_features[i]).permute(0, 3, 1, + 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() + + +class D2SwinTransformer(SwinTransformer): + + def __init__(self, *args, **kwargs): + self._out_features = kwargs.pop('out_features') + super().__init__(*args, **kwargs) + + self._out_feature_strides = { + 'res2': 4, + 'res3': 8, + 'res4': 16, + 'res5': 32, + } + self._out_feature_channels = { + 'res2': self.num_features[0], + 'res3': self.num_features[1], + 'res4': self.num_features[2], + 'res5': self.num_features[3], + } + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert ( + x.dim() == 4 + ), f'SwinTransformer takes an input of shape (N, C, H, W). Got {x.shape} instead!' + outputs = {} + outs = super().forward(x) + y = {} + for out, i in zip(outs, self.out_indices): + y['res{}'.format(i + 2)] = out + + for k in y.keys(): + if k in self._out_features: + outputs[k] = y[k] + return outputs + + def output_shape(self): + return { + name: dict( + channels=self._out_feature_channels[name], + stride=self._out_feature_strides[name]) + for name in self._out_features + } + + @property + def size_divisibility(self): + return 32 diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/cascade_mask_rcnn_swin.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/cascade_mask_rcnn_swin.py new file mode 100644 index 0000000000000000000000000000000000000000..375a5e45ce348870b9fa40bc171ee9a9e2a4f8f3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/cascade_mask_rcnn_swin.py @@ -0,0 +1,268 @@ +# Part of the implementation is borrowed and modified from MMDetection, publicly available at +# https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/two_stage.py +import os +from collections import OrderedDict + +import torch +import torch.distributed as dist +import torch.nn as nn + +from modelscope.models.cv.image_instance_segmentation.backbones import \ + SwinTransformer +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def build_backbone(cfg): + assert isinstance(cfg, dict) + cfg = cfg.copy() + type = cfg.pop('type') + if type == 'SwinTransformer': + return SwinTransformer(**cfg) + else: + raise ValueError(f'backbone \'{type}\' is not supported.') + + +def build_neck(cfg): + assert isinstance(cfg, dict) + cfg = cfg.copy() + type = cfg.pop('type') + if type == 'FPN': + from mmdet.models import FPN + return FPN(**cfg) + else: + raise ValueError(f'neck \'{type}\' is not supported.') + + +def build_rpn_head(cfg): + assert isinstance(cfg, dict) + cfg = cfg.copy() + type = cfg.pop('type') + if type == 'RPNHead': + from mmdet.models import RPNHead + return RPNHead(**cfg) + else: + raise ValueError(f'rpn head \'{type}\' is not supported.') + + +def build_roi_head(cfg): + assert isinstance(cfg, dict) + cfg = cfg.copy() + type = cfg.pop('type') + if type == 'CascadeRoIHead': + from mmdet.models import CascadeRoIHead + return CascadeRoIHead(**cfg) + else: + raise ValueError(f'roi head \'{type}\' is not supported.') + + +class CascadeMaskRCNNSwin(nn.Module): + + def __init__(self, + backbone, + neck, + rpn_head, + roi_head, + pretrained=None, + **kwargs): + """ + Args: + backbone (dict): backbone config. + neck (dict): neck config. + rpn_head (dict): rpn_head config. + roi_head (dict): roi_head config. + pretrained (bool): whether to use pretrained model + """ + super(CascadeMaskRCNNSwin, self).__init__() + + self.backbone = build_backbone(backbone) + self.neck = build_neck(neck) + self.rpn_head = build_rpn_head(rpn_head) + self.roi_head = build_roi_head(roi_head) + + self.classes = kwargs.pop('classes', None) + + if pretrained: + assert 'model_dir' in kwargs, 'pretrained model dir is missing.' + model_path = os.path.join(kwargs['model_dir'], + ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['state_dict'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info(f'size mismatch for {name}, skip loading.') + del weight[name] + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + from mmcv.parallel import DataContainer, scatter + + self.data_container = DataContainer + self.scatter = scatter + + def extract_feat(self, img): + x = self.backbone(img) + x = self.neck(x) + return x + + def forward_train(self, + img, + img_metas, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + gt_masks=None, + proposals=None, + **kwargs): + """ + Args: + img (Tensor): of shape (N, C, H, W) encoding input images. + Typically these should be mean centered and std scaled. + + img_metas (list[dict]): list of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmdet/datasets/pipelines/formatting.py:Collect`. + + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + + gt_labels (list[Tensor]): class indices corresponding to each box + + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + gt_masks (None | Tensor) : true segmentation masks for each box + used if the architecture supports a segmentation task. + + proposals : override rpn proposals with custom proposals. Use when + `with_rpn` is False. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self.extract_feat(img) + + losses = dict() + + # RPN forward and loss + proposal_cfg = self.rpn_head.train_cfg.get('rpn_proposal', + self.rpn_head.test_cfg) + rpn_losses, proposal_list = self.rpn_head.forward_train( + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_bboxes_ignore=gt_bboxes_ignore, + proposal_cfg=proposal_cfg, + **kwargs) + losses.update(rpn_losses) + + roi_losses = self.roi_head.forward_train(x, img_metas, proposal_list, + gt_bboxes, gt_labels, + gt_bboxes_ignore, gt_masks, + **kwargs) + losses.update(roi_losses) + + return losses + + def forward_test(self, img, img_metas, proposals=None, rescale=True): + + x = self.extract_feat(img) + if proposals is None: + proposal_list = self.rpn_head.simple_test_rpn(x, img_metas) + else: + proposal_list = proposals + + result = self.roi_head.simple_test( + x, proposal_list, img_metas, rescale=rescale) + return dict(eval_result=result, img_metas=img_metas) + + def forward(self, img, img_metas, **kwargs): + + # currently only support cpu or single gpu + if isinstance(img, self.data_container): + img = img.data[0] + if isinstance(img_metas, self.data_container): + img_metas = img_metas.data[0] + for k, w in kwargs.items(): + if isinstance(w, self.data_container): + w = w.data[0] + kwargs[k] = w + + if next(self.parameters()).is_cuda: + device = next(self.parameters()).device + img = self.scatter(img, [device])[0] + img_metas = self.scatter(img_metas, [device])[0] + for k, w in kwargs.items(): + kwargs[k] = self.scatter(w, [device])[0] + + if self.training: + losses = self.forward_train(img, img_metas, **kwargs) + loss, log_vars = self._parse_losses(losses) + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(img_metas)) + return outputs + else: + return self.forward_test(img, img_metas, **kwargs) + + def _parse_losses(self, losses): + + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def train_step(self, data, optimizer): + + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs + + def val_step(self, data, optimizer=None): + + losses = self(**data) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, log_vars=log_vars, num_samples=len(data['img_metas'])) + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/__init__.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1b096fb33dc6d964bb9f325cc43ef282f82d3744 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .transforms import build_preprocess_transform diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/transforms.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..f0dde75990903ecd5a042715cfc5d6d27e16581e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/datasets/transforms.py @@ -0,0 +1,114 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np + +from modelscope.fileio import File + + +def build_preprocess_transform(cfg): + assert isinstance(cfg, dict) + cfg = cfg.copy() + type = cfg.pop('type') + if type == 'LoadImageFromFile': + return LoadImageFromFile(**cfg) + elif type == 'LoadAnnotations': + from mmdet.datasets.pipelines import LoadAnnotations + return LoadAnnotations(**cfg) + elif type == 'Resize': + if 'img_scale' in cfg: + if isinstance(cfg.img_scale[0], list): + elems = [] + for elem in cfg.img_scale: + elems.append(tuple(elem)) + cfg.img_scale = elems + else: + cfg.img_scale = tuple(cfg.img_scale) + from mmdet.datasets.pipelines import Resize + return Resize(**cfg) + elif type == 'RandomFlip': + from mmdet.datasets.pipelines import RandomFlip + return RandomFlip(**cfg) + elif type == 'Normalize': + from mmdet.datasets.pipelines import Normalize + return Normalize(**cfg) + elif type == 'Pad': + from mmdet.datasets.pipelines import Pad + return Pad(**cfg) + elif type == 'DefaultFormatBundle': + from mmdet.datasets.pipelines import DefaultFormatBundle + return DefaultFormatBundle(**cfg) + elif type == 'ImageToTensor': + from mmdet.datasets.pipelines import ImageToTensor + return ImageToTensor(**cfg) + elif type == 'Collect': + from mmdet.datasets.pipelines import Collect + return Collect(**cfg) + else: + raise ValueError(f'preprocess transform \'{type}\' is not supported.') + + +class LoadImageFromFile: + """Load an image from file. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename", "ann_file", and "classes"). Added or updated keys are + "filename", "ori_filename", "img", "img_shape", "ori_shape" (same as `img_shape`), + "img_fields", "ann_file" (path to annotation file) and "classes". + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + """ + + def __init__(self, to_float32=False, mode='rgb'): + self.to_float32 = to_float32 + self.mode = mode + + from mmcv import imfrombytes + + self.imfrombytes = imfrombytes + + def __call__(self, results): + """Call functions to load image and get image meta information. + + Args: + results (dict): Result dict from :obj:`ImageInstanceSegmentationCocoDataset`. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + if 'img' in results and isinstance(results['img'], np.ndarray): + img = results['img'] + filename = results['img_info']['filename'] + else: + if results['img_prefix'] is not None: + filename = osp.join(results['img_prefix'], + results['img_info']['filename']) + else: + filename = results['img_info']['filename'] + + img_bytes = File.read(filename) + + img = self.imfrombytes(img_bytes, 'color', 'bgr', backend='pillow') + + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + results['img_fields'] = ['img'] + results['ann_file'] = results['img_info']['ann_file'] + results['classes'] = results['img_info']['classes'] + return results + + def __repr__(self): + repr_str = (f'{self.__class__.__name__}(' + f'to_float32={self.to_float32}, ' + f"mode='{self.mode}'") + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/__init__.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..aa4300f6f6be5d4e5bfdd986ae9ee178bf4855e1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py @@ -0,0 +1,351 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_colorization.ddcolor.utils.transformer_utils import ( + MLP, CrossAttentionLayer, FFNLayer, SelfAttentionLayer) + + +class QueryProposal(nn.Module): + + def __init__(self, num_features, num_queries, num_classes): + super().__init__() + self.topk = num_queries + self.num_classes = num_classes + + self.conv_proposal_cls_logits = nn.Sequential( + nn.Conv2d( + num_features, num_features, kernel_size=3, stride=1, + padding=1), + nn.ReLU(inplace=True), + nn.Conv2d( + num_features, + num_classes + 1, + kernel_size=1, + stride=1, + padding=0), + ) + + @torch.no_grad() + def compute_coordinates(self, x): + h, w = x.size(2), x.size(3) + y_loc = torch.linspace(0, 1, h, device=x.device) + x_loc = torch.linspace(0, 1, w, device=x.device) + y_loc, x_loc = torch.meshgrid(y_loc, x_loc) + locations = torch.stack([x_loc, y_loc], 0).unsqueeze(0) + return locations + + def seek_local_maximum(self, x, epsilon=1e-6): + """ + inputs: + x: torch.tensor, shape [b, c, h, w] + return: + torch.tensor, shape [b, c, h, w] + """ + x_pad = F.pad(x, (1, 1, 1, 1), 'constant', 0) + # top, bottom, left, right, top-left, top-right, bottom-left, bottom-right + maximum = (x >= x_pad[:, :, :-2, 1:-1]) & \ + (x >= x_pad[:, :, 2:, 1:-1]) & \ + (x >= x_pad[:, :, 1:-1, :-2]) & \ + (x >= x_pad[:, :, 1:-1, 2:]) & \ + (x >= x_pad[:, :, :-2, :-2]) & \ + (x >= x_pad[:, :, :-2, 2:]) & \ + (x >= x_pad[:, :, 2:, :-2]) & \ + (x >= x_pad[:, :, 2:, 2:]) & \ + (x >= epsilon) + return maximum.to(x) + + def forward(self, x, pos_embeddings): + + proposal_cls_logits = self.conv_proposal_cls_logits(x) # b, c, h, w + proposal_cls_probs = proposal_cls_logits.softmax(dim=1) # b, c, h, w + proposal_cls_one_hot = F.one_hot( + proposal_cls_probs[:, :-1, :, :].max(1)[1], + num_classes=self.num_classes + 1).permute(0, 3, 1, 2) # b, c, h, w + proposal_cls_probs = proposal_cls_probs.mul(proposal_cls_one_hot) + proposal_local_maximum_map = self.seek_local_maximum( + proposal_cls_probs) # b, c, h, w + proposal_cls_probs = proposal_cls_probs + proposal_local_maximum_map # b, c, h, w + + # top-k indices + topk_indices = torch.topk( + proposal_cls_probs[:, :-1, :, :].flatten(2).max(1)[0], + self.topk, + dim=1)[1] # b, q + topk_indices = topk_indices.unsqueeze(1) # b, 1, q + + # topk queries + topk_proposals = torch.gather( + x.flatten(2), dim=2, index=topk_indices.repeat(1, x.shape[1], + 1)) # b, c, q + pos_embeddings = pos_embeddings.repeat(x.shape[0], 1, 1, 1).flatten(2) + topk_pos_embeddings = torch.gather( + pos_embeddings, + dim=2, + index=topk_indices.repeat(1, pos_embeddings.shape[1], + 1)) # b, c, q + if self.training: + locations = self.compute_coordinates(x).repeat(x.shape[0], 1, 1, 1) + topk_locations = torch.gather( + locations.flatten(2), + dim=2, + index=topk_indices.repeat(1, locations.shape[1], 1)) + topk_locations = topk_locations.transpose(-1, -2) # b, q, 2 + else: + topk_locations = None + return topk_proposals, topk_pos_embeddings, topk_locations, proposal_cls_logits + + +class FastInstDecoder(nn.Module): + + def __init__(self, in_channels, *, num_classes: int, hidden_dim: int, + num_queries: int, num_aux_queries: int, nheads: int, + dim_feedforward: int, dec_layers: int, pre_norm: bool, + mask_dim: int): + """ + Args: + in_channels: channels of the input features + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + num_aux_queries: number of auxiliary queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + dec_layers: number of Transformer decoder layers + pre_norm: whether to use pre-LayerNorm or not + mask_dim: mask feature dimension + """ + super().__init__() + self.num_heads = nheads + self.num_layers = dec_layers + self.num_queries = num_queries + self.num_aux_queries = num_aux_queries + self.num_classes = num_classes + + meta_pos_size = int(round(math.sqrt(self.num_queries))) + self.meta_pos_embed = nn.Parameter( + torch.empty(1, hidden_dim, meta_pos_size, meta_pos_size)) + if num_aux_queries > 0: + self.empty_query_features = nn.Embedding(num_aux_queries, + hidden_dim) + self.empty_query_pos_embed = nn.Embedding(num_aux_queries, + hidden_dim) + + self.query_proposal = QueryProposal(hidden_dim, num_queries, + num_classes) + + self.transformer_query_cross_attention_layers = nn.ModuleList() + self.transformer_query_self_attention_layers = nn.ModuleList() + self.transformer_query_ffn_layers = nn.ModuleList() + self.transformer_mask_cross_attention_layers = nn.ModuleList() + self.transformer_mask_ffn_layers = nn.ModuleList() + for idx in range(self.num_layers): + self.transformer_query_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + + self.decoder_query_norm_layers = nn.ModuleList() + self.class_embed_layers = nn.ModuleList() + self.mask_embed_layers = nn.ModuleList() + self.mask_features_layers = nn.ModuleList() + for idx in range(self.num_layers + 1): + self.decoder_query_norm_layers.append(nn.LayerNorm(hidden_dim)) + self.class_embed_layers.append( + MLP(hidden_dim, hidden_dim, num_classes + 1, 3)) + self.mask_embed_layers.append( + MLP(hidden_dim, hidden_dim, mask_dim, 3)) + self.mask_features_layers.append(nn.Linear(hidden_dim, mask_dim)) + + def forward(self, x, mask_features, targets=None): + bs = x[0].shape[0] + proposal_size = x[1].shape[-2:] + pixel_feature_size = x[2].shape[-2:] + + pixel_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=pixel_feature_size, + mode='bilinear', + align_corners=False) + proposal_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=proposal_size, + mode='bilinear', + align_corners=False) + + pixel_features = x[2].flatten(2).permute(2, 0, 1) + pixel_pos_embeds = pixel_pos_embeds.flatten(2).permute(2, 0, 1) + + query_features, query_pos_embeds, query_locations, proposal_cls_logits = self.query_proposal( + x[1], proposal_pos_embeds) + query_features = query_features.permute(2, 0, 1) + query_pos_embeds = query_pos_embeds.permute(2, 0, 1) + if self.num_aux_queries > 0: + aux_query_features = self.empty_query_features.weight.unsqueeze( + 1).repeat(1, bs, 1) + aux_query_pos_embed = self.empty_query_pos_embed.weight.unsqueeze( + 1).repeat(1, bs, 1) + query_features = torch.cat([query_features, aux_query_features], + dim=0) + query_pos_embeds = torch.cat( + [query_pos_embeds, aux_query_pos_embed], dim=0) + + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + -1, + return_attn_mask=True) + predictions_class = [outputs_class] + predictions_mask = [outputs_mask] + predictions_matching_index = [None] + query_feature_memory = [query_features] + pixel_feature_memory = [pixel_features] + + for i in range(self.num_layers): + query_features, pixel_features = self.forward_one_layer( + query_features, pixel_features, query_pos_embeds, + pixel_pos_embeds, attn_mask, i) + if i < self.num_layers - 1: + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + return_attn_mask=True, + ) + else: + outputs_class, outputs_mask, _, matching_indices, gt_attn_mask = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + ) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + predictions_matching_index.append(None) + query_feature_memory.append(query_features) + pixel_feature_memory.append(pixel_features) + + out = { + 'proposal_cls_logits': + proposal_cls_logits, + 'query_locations': + query_locations, + 'pred_logits': + predictions_class[-1], + 'pred_masks': + predictions_mask[-1], + 'pred_indices': + predictions_matching_index[-1], + 'aux_outputs': + self._set_aux_loss(predictions_class, predictions_mask, + predictions_matching_index, query_locations) + } + return out + + def forward_one_layer(self, query_features, pixel_features, + query_pos_embeds, pixel_pos_embeds, attn_mask, i): + pixel_features = self.transformer_mask_cross_attention_layers[i]( + pixel_features, + query_features, + query_pos=pixel_pos_embeds, + pos=query_pos_embeds) + pixel_features = self.transformer_mask_ffn_layers[i](pixel_features) + + query_features = self.transformer_query_cross_attention_layers[i]( + query_features, + pixel_features, + memory_mask=attn_mask, + query_pos=query_pos_embeds, + pos=pixel_pos_embeds) + query_features = self.transformer_query_self_attention_layers[i]( + query_features, query_pos=query_pos_embeds) + query_features = self.transformer_query_ffn_layers[i](query_features) + return query_features, pixel_features + + def forward_prediction_heads(self, + query_features, + pixel_features, + pixel_feature_size, + idx_layer, + return_attn_mask=False, + return_gt_attn_mask=False, + targets=None, + query_locations=None): + decoder_query_features = self.decoder_query_norm_layers[idx_layer + 1]( + query_features[:self.num_queries]) + decoder_query_features = decoder_query_features.transpose(0, 1) + if idx_layer + 1 == self.num_layers: + outputs_class = self.class_embed_layers[idx_layer + 1]( + decoder_query_features) + else: + outputs_class = None + outputs_mask_embed = self.mask_embed_layers[idx_layer + 1]( + decoder_query_features) + outputs_mask_features = self.mask_features_layers[idx_layer + 1]( + pixel_features.transpose(0, 1)) + + outputs_mask = torch.einsum('bqc,blc->bql', outputs_mask_embed, + outputs_mask_features) + outputs_mask = outputs_mask.reshape(-1, self.num_queries, + *pixel_feature_size) + + if return_attn_mask: + # outputs_mask.shape: b, q, h, w + attn_mask = F.pad(outputs_mask, + (0, 0, 0, 0, 0, self.num_aux_queries), + 'constant', 1) + attn_mask = (attn_mask < 0.).flatten(2) # b, q, hw + invalid_query = attn_mask.all(-1, keepdim=True) # b, q, 1 + attn_mask = (~invalid_query) & attn_mask # b, q, hw + attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1, + 1).flatten(0, 1) + attn_mask = attn_mask.detach() + else: + attn_mask = None + + matching_indices = None + gt_attn_mask = None + + return outputs_class, outputs_mask, attn_mask, matching_indices, gt_attn_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks, output_indices, + output_query_locations): + return [{ + 'query_locations': output_query_locations, + 'pred_logits': a, + 'pred_masks': b, + 'pred_matching_indices': c + } for a, b, c in zip(outputs_class[:-1], outputs_seg_masks[:-1], + output_indices[:-1])] diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..46b3f74d6bd3236681b9d440686447bc0721ddb9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py @@ -0,0 +1,180 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import logging +from typing import Callable, Optional, Union + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +# This is a modified FPN decoder. +class BaseFPN(nn.Module): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__() + + input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride']) + self.in_features = [k for k, v in input_shape + ] # starting from "res3" to "res5" + feature_channels = [v['channels'] for k, v in input_shape] + + lateral_convs = [] + output_convs = [] + + use_bias = norm == '' + for idx, in_channels in enumerate(feature_channels): + lateral_norm = nn.GroupNorm(32, convs_dim) + output_norm = nn.GroupNorm(32, convs_dim) + + lateral_conv = Conv2d( + in_channels, + convs_dim, + kernel_size=1, + bias=use_bias, + norm=lateral_norm) + output_conv = Conv2d( + convs_dim, + convs_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + self.add_module('adapter_{}'.format(idx + 1), lateral_conv) + self.add_module('layer_{}'.format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + self.convs_dim = convs_dim + self.num_feature_levels = 3 # always use 3 scales + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = lateral_conv(x) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + return None, multi_scale_features + + def forward(self, features, targets=None): + logger = logging.getLogger(__name__) + logger.warning( + 'Calling forward() may cause unpredicted behavior of PixelDecoder module.' + ) + return self.forward_features(features) + + +class PyramidPoolingModule(nn.Module): + + def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)): + super().__init__() + self.stages = [] + self.stages = nn.ModuleList( + [self._make_stage(in_channels, channels, size) for size in sizes]) + self.bottleneck = Conv2d(in_channels + len(sizes) * channels, + in_channels, 1) + + def _make_stage(self, features, out_features, size): + prior = nn.AdaptiveAvgPool2d(output_size=(size, size)) + conv = Conv2d(features, out_features, 1) + return nn.Sequential(prior, conv) + + def forward(self, feats): + h, w = feats.size(2), feats.size(3) + priors = [ + F.interpolate( + input=F.relu_(stage(feats)), + size=(h, w), + mode='bilinear', + align_corners=False) for stage in self.stages + ] + [feats] + out = F.relu_(self.bottleneck(torch.cat(priors, 1))) + return out + + +class PyramidPoolingModuleFPN(BaseFPN): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__( + input_shape, convs_dim=convs_dim, mask_dim=mask_dim, norm=norm) + self.ppm = PyramidPoolingModule(convs_dim, convs_dim // 4) + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = self.ppm(lateral_conv(x)) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + + return None, multi_scale_features diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst_model.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f9cfbc4fa4c862dc932275062f0d989757330ffa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/fastinst_model.py @@ -0,0 +1,221 @@ +# Part of implementation is borrowed and modified from Mask2Former, publicly available at +# https://github.com/facebookresearch/Mask2Former. +import os +from typing import Any, Dict, List + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation.maskdino_swin import \ + ImageList +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .backbones import build_resnet_backbone +from .fastinst.fastinst_decoder import FastInstDecoder +from .fastinst.fastinst_encoder import PyramidPoolingModuleFPN + +logger = get_logger() + + +@MODELS.register_module(Tasks.image_segmentation, module_name=Models.fastinst) +class FastInst(TorchModel): + + def __init__(self, + model_dir, + backbone=None, + encoder=None, + decoder=None, + pretrained=None, + classes=None, + **kwargs): + """ + Deep Learning Technique for Human Parsing: A Survey and Outlook. See https://arxiv.org/abs/2301.00394 + Args: + backbone (dict): backbone config. + encoder (dict): encoder config. + decoder (dict): decoder config. + pretrained (bool): whether to use pretrained model + classes (list): class names + """ + super(FastInst, self).__init__(model_dir, **kwargs) + + self.backbone = build_resnet_backbone( + **backbone, input_shape={'channels': 3}) + in_features = encoder.pop('in_features') + input_shape = { + k: v + for k, v in self.backbone.output_shape().items() + if k in in_features + } + encoder = PyramidPoolingModuleFPN(input_shape=input_shape, **encoder) + decoder = FastInstDecoder(in_channels=encoder.convs_dim, **decoder) + self.sem_seg_head = FastInstHead( + pixel_decoder=encoder, transformer_predictor=decoder) + + self.num_classes = decoder.num_classes + self.num_queries = decoder.num_queries + self.size_divisibility = 32 + self.register_buffer( + 'pixel_mean', + torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False) + self.register_buffer( + 'pixel_std', + torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False) + self.classes = classes + self.test_topk_per_image = 100 + + if pretrained: + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['model'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info( + f'size mismatch for {name} ' + f'({load_size} -> {tgt_size}), skip loading.') + del weight[name] + else: + logger.info( + f'{name} doesn\'t exist in current model, skip loading.' + ) + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + def forward(self, batched_inputs: List[dict]) -> Dict[str, Any]: + images = [x['image'].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + features = self.backbone(images.tensor) + outputs = self.sem_seg_head(features) + + return dict( + outputs=outputs, batched_inputs=batched_inputs, images=images) + + def postprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + outputs = input['outputs'] + batched_inputs = input['batched_inputs'] + images = input['images'] + if self.training: + raise NotImplementedError + else: + mask_cls_results = outputs['pred_logits'] # (B, Q, C+1) + mask_pred_results = outputs['pred_masks'] # (B, Q, H, W) + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode='bilinear', + align_corners=False, + ) + + del outputs + + processed_results = [] + for mask_cls_result, mask_pred_result, input_per_image, image_size in zip( + mask_cls_results, mask_pred_results, batched_inputs, + images.image_sizes): + height = input_per_image.get('height', image_size[0]) + width = input_per_image.get('width', image_size[1]) + processed_results.append({}) # for each image + + mask_pred_result = self.sem_seg_postprocess( + mask_pred_result, image_size, height, width) + mask_cls_result = mask_cls_result.to(mask_pred_result) + + instance_r = self.instance_inference(mask_cls_result, + mask_pred_result) + processed_results[-1]['instances'] = instance_r + + return dict(eval_result=processed_results) + + @property + def device(self): + return self.pixel_mean.device + + def sem_seg_postprocess(self, result, img_size, output_height, + output_width): + result = result[:, :img_size[0], :img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate( + result, + size=(output_height, output_width), + mode='bilinear', + align_corners=False)[0] + return result + + def instance_inference(self, mask_cls, mask_pred): + # mask_pred is already processed to have the same shape as original input + image_size = mask_pred.shape[-2:] + + # [Q, K] + scores = F.softmax(mask_cls, dim=-1)[:, :-1] + labels = torch.arange( + self.num_classes, + device=self.device).unsqueeze(0).repeat(self.num_queries, + 1).flatten(0, 1) + scores_per_image, topk_indices = scores.flatten(0, 1).topk( + self.test_topk_per_image, sorted=False) + labels_per_image = labels[topk_indices] + + topk_indices = topk_indices // self.num_classes + mask_pred = mask_pred[topk_indices] + + result = {'image_size': image_size} + # mask (before sigmoid) + mask_pred_sigmoid = mask_pred.sigmoid() + result['pred_masks'] = (mask_pred_sigmoid > 0.5).float() + + # calculate average mask prob + mask_scores_per_image = (mask_pred_sigmoid.flatten(1) + * result['pred_masks'].flatten(1)).sum(1) / ( + result['pred_masks'].flatten(1).sum(1) + + 1e-6) + result['scores'] = scores_per_image * mask_scores_per_image + result['pred_classes'] = labels_per_image + return result + + +class FastInstHead(nn.Module): + + def __init__( + self, + *, + pixel_decoder: nn.Module, + # extra parameters + transformer_predictor: nn.Module): + """ + NOTE: this interface is experimental. + Args: + pixel_decoder: the pixel decoder module + transformer_predictor: the transformer decoder that makes prediction + """ + super().__init__() + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + + def forward(self, features, targets=None): + return self.layers(features, targets) + + def layers(self, features, targets=None): + mask_features, multi_scale_features = self.pixel_decoder.forward_features( + features) + predictions = self.predictor(multi_scale_features, mask_features, + targets) + return predictions diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/__init__.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..703d04f1d8c07ff9fd16234ce7702fe893e65afa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .maskdino_encoder import MaskDINOEncoder + from .maskdino_decoder import MaskDINODecoder + +else: + _import_structure = { + 'maskdino_encoder': ['MaskDINOEncoder'], + 'maskdino_decoder': ['MaskDINODecoder'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/dino_decoder.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/dino_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..769922c5e995f51e5c112df73c64ae7e2771b801 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/dino_decoder.py @@ -0,0 +1,277 @@ +# The implementation is adopted from DINO, made publicly available under the Apache License, +# Version 2.0 at https://github.com/IDEA-Research/DINO + +from typing import Optional + +import torch +from torch import Tensor, nn +from torch.cuda.amp import autocast + +from .ms_deform_attn import MSDeformAttn +from .utils import (MLP, _get_activation_fn, _get_clones, + gen_sineembed_for_position, inverse_sigmoid) + + +class TransformerDecoder(nn.Module): + + def __init__( + self, + decoder_layer, + num_layers, + norm=None, + return_intermediate=False, + d_model=256, + query_dim=4, + modulate_hw_attn=True, + num_feature_levels=1, + deformable_decoder=True, + decoder_query_perturber=None, + dec_layer_number=None, # number of queries each layer in decoder + rm_dec_query_scale=True, + dec_layer_share=False, + dec_layer_dropout_prob=None, + ): + super().__init__() + if num_layers > 0: + self.layers = _get_clones( + decoder_layer, num_layers, layer_share=dec_layer_share) + else: + self.layers = [] + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + assert return_intermediate, 'support return_intermediate only' + self.query_dim = query_dim + assert query_dim in [ + 2, 4 + ], 'query_dim should be 2/4 but {}'.format(query_dim) + self.num_feature_levels = num_feature_levels + + self.ref_point_head = MLP(query_dim // 2 * d_model, d_model, d_model, + 2) + if not deformable_decoder: + self.query_pos_sine_scale = MLP(d_model, d_model, d_model, 2) + else: + self.query_pos_sine_scale = None + + if rm_dec_query_scale: + self.query_scale = None + else: + raise NotImplementedError + self.bbox_embed = None + self.class_embed = None + + self.d_model = d_model + self.modulate_hw_attn = modulate_hw_attn + self.deformable_decoder = deformable_decoder + + if not deformable_decoder and modulate_hw_attn: + self.ref_anchor_head = MLP(d_model, d_model, 2, 2) + else: + self.ref_anchor_head = None + + self.decoder_query_perturber = decoder_query_perturber + self.box_pred_damping = None + + self.dec_layer_number = dec_layer_number + if dec_layer_number is not None: + assert isinstance(dec_layer_number, list) + assert len(dec_layer_number) == num_layers + # assert dec_layer_number[0] == + + self.dec_layer_dropout_prob = dec_layer_dropout_prob + if dec_layer_dropout_prob is not None: + assert isinstance(dec_layer_dropout_prob, list) + assert len(dec_layer_dropout_prob) == num_layers + for i in dec_layer_dropout_prob: + assert 0.0 <= i <= 1.0 + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + + def forward( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + refpoints_unsigmoid: Optional[Tensor] = None, # num_queries, bs, 2 + # for memory + level_start_index: Optional[Tensor] = None, # num_levels + spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 + valid_ratios: Optional[Tensor] = None, + ): + + output = tgt + + intermediate = [] + reference_points = refpoints_unsigmoid.sigmoid() + ref_points = [reference_points] + + for layer_id, layer in enumerate(self.layers): + # preprocess ref points + if self.training and self.decoder_query_perturber is not None and layer_id != 0: + reference_points = self.decoder_query_perturber( + reference_points) + + reference_points_input = ( + reference_points[:, :, None] + * torch.cat([valid_ratios, valid_ratios], -1)[None, :] + ) # nq, bs, nlevel, 4 + query_sine_embed = gen_sineembed_for_position( + reference_points_input[:, :, 0, :]) # nq, bs, 256*2 + + raw_query_pos = self.ref_point_head( + query_sine_embed) # nq, bs, 256 + pos_scale = self.query_scale( + output) if self.query_scale is not None else 1 + query_pos = pos_scale * raw_query_pos + + output = layer( + tgt=output, + tgt_query_pos=query_pos, + tgt_query_sine_embed=query_sine_embed, + tgt_key_padding_mask=tgt_key_padding_mask, + tgt_reference_points=reference_points_input, + memory=memory, + memory_key_padding_mask=memory_key_padding_mask, + memory_level_start_index=level_start_index, + memory_spatial_shapes=spatial_shapes, + memory_pos=pos, + self_attn_mask=tgt_mask, + cross_attn_mask=memory_mask) + + # iter update + if self.bbox_embed is not None: + reference_before_sigmoid = inverse_sigmoid(reference_points) + delta_unsig = self.bbox_embed[layer_id](output) + outputs_unsig = delta_unsig + reference_before_sigmoid + new_reference_points = outputs_unsig.sigmoid() + + reference_points = new_reference_points.detach() + # if layer_id != self.num_layers - 1: + ref_points.append(new_reference_points) + + intermediate.append(self.norm(output)) + + return [[itm_out.transpose(0, 1) for itm_out in intermediate], + [itm_refpoint.transpose(0, 1) for itm_refpoint in ref_points]] + + +class DeformableTransformerDecoderLayer(nn.Module): + + def __init__( + self, + d_model=256, + d_ffn=1024, + dropout=0.1, + activation='relu', + n_levels=4, + n_heads=8, + n_points=4, + use_deformable_box_attn=False, + key_aware_type=None, + ): + super().__init__() + + # cross attention + if use_deformable_box_attn: + raise NotImplementedError + else: + self.cross_attn = MSDeformAttn(d_model, n_levels, n_heads, + n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # self attention + self.self_attn = nn.MultiheadAttention( + d_model, n_heads, dropout=dropout) + self.dropout2 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout3 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout4 = nn.Dropout(dropout) + self.norm3 = nn.LayerNorm(d_model) + + self.key_aware_type = key_aware_type + self.key_aware_proj = None + + def rm_self_attn_modules(self): + self.self_attn = None + self.dropout2 = None + self.norm2 = None + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, tgt): + tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout4(tgt2) + tgt = self.norm3(tgt) + return tgt + + @autocast(enabled=False) + def forward( + self, + tgt: Optional[Tensor], # nq, bs, d_model + tgt_query_pos: Optional[ + Tensor] = None, # pos for query. MLP(Sine(pos)) + tgt_query_sine_embed: Optional[ + Tensor] = None, # pos for query. Sine(pos) + tgt_key_padding_mask: Optional[Tensor] = None, + tgt_reference_points: Optional[Tensor] = None, # nq, bs, 4 + memory: Optional[Tensor] = None, # hw, bs, d_model + memory_key_padding_mask: Optional[Tensor] = None, + memory_level_start_index: Optional[Tensor] = None, # num_levels + memory_spatial_shapes: Optional[ + Tensor] = None, # bs, num_levels, 2 + memory_pos: Optional[Tensor] = None, # pos for memory + self_attn_mask: Optional[ + Tensor] = None, # mask used for self-attention + cross_attn_mask: Optional[ + Tensor] = None, # mask used for cross-attention + ): + + # self attention + if self.self_attn is not None: + q = k = self.with_pos_embed(tgt, tgt_query_pos) + tgt2 = self.self_attn(q, k, tgt, attn_mask=self_attn_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + + # cross attention + if self.key_aware_type is not None: + if self.key_aware_type == 'mean': + tgt = tgt + memory.mean(0, keepdim=True) + elif self.key_aware_type == 'proj_mean': + tgt = tgt + self.key_aware_proj(memory).mean(0, keepdim=True) + else: + raise NotImplementedError('Unknown key_aware_type: {}'.format( + self.key_aware_type)) + tgt2 = self.cross_attn( + self.with_pos_embed(tgt, tgt_query_pos).transpose(0, 1), + tgt_reference_points.transpose(0, 1).contiguous(), + memory.transpose(0, 1), memory_spatial_shapes, + memory_level_start_index, memory_key_padding_mask).transpose(0, 1) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + + # ffn + tgt = self.forward_ffn(tgt) + + return tgt diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_decoder.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..5e6c9ad22db06101aee478c24a3b0ccb7fc935ea --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_decoder.py @@ -0,0 +1,352 @@ +# The implementation is adopted from Mask DINO, made publicly available under the Apache License, +# Version 2.0 at https://github.com/IDEA-Research/MaskDINO +# Part of implementation is borrowed from Mask2Former, +# https://github.com/facebookresearch/Mask2Former, under MIT license. + +import torch +from torch import nn + +from .dino_decoder import DeformableTransformerDecoderLayer, TransformerDecoder +from .utils import (MLP, Conv2d, box_xyxy_to_cxcywh, + gen_encoder_output_proposals, get_bounding_boxes, + inverse_sigmoid) + + +class MaskDINODecoder(nn.Module): + + def __init__( + self, + in_channels, + mask_classification=True, + *, + num_classes: int, + hidden_dim: int, + num_queries: int, + nheads: int, + dim_feedforward: int, + dec_layers: int, + mask_dim: int, + enforce_input_project: bool, + two_stage: bool, + initialize_box_type: bool, + initial_pred: bool, + learn_tgt: bool, + total_num_feature_levels: int = 4, + dropout: float = 0.0, + activation: str = 'relu', + nhead: int = 8, + dec_n_points: int = 4, + return_intermediate_dec: bool = True, + query_dim: int = 4, + dec_layer_share: bool = False, + semantic_ce_loss: bool = False, + ): + """ + NOTE: this interface is experimental. + Args: + in_channels: channels of the input features + mask_classification: whether to add mask classifier or not + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + dec_layers: number of Transformer decoder layers + mask_dim: mask feature dimension + enforce_input_project: add input project 1x1 conv even if input + channels and hidden dim is identical + dropout: dropout rate + activation: activation function + nhead: num heads in multi-head attention + dec_n_points: number of sampling points in decoder + return_intermediate_dec: return the intermediate results of decoder + query_dim: 4 -> (x, y, w, h) + dec_layer_share: whether to share each decoder layer + semantic_ce_loss: use ce loss for semantic segmentation + """ + super().__init__() + + assert mask_classification, 'Only support mask classification model' + self.mask_classification = mask_classification + self.num_feature_levels = total_num_feature_levels + self.initial_pred = initial_pred + + # define Transformer decoder here + self.learn_tgt = learn_tgt + self.num_heads = nheads + self.num_layers = dec_layers + self.two_stage = two_stage + self.initialize_box_type = initialize_box_type + self.total_num_feature_levels = total_num_feature_levels + + self.num_queries = num_queries + self.semantic_ce_loss = semantic_ce_loss + # learnable query features + if not two_stage or self.learn_tgt: + self.query_feat = nn.Embedding(num_queries, hidden_dim) + if not two_stage and initialize_box_type == 'no': + self.query_embed = nn.Embedding(num_queries, 4) + if two_stage: + self.enc_output = nn.Linear(hidden_dim, hidden_dim) + self.enc_output_norm = nn.LayerNorm(hidden_dim) + + self.input_proj = nn.ModuleList() + for _ in range(self.num_feature_levels): + if in_channels != hidden_dim or enforce_input_project: + self.input_proj.append( + Conv2d(in_channels, hidden_dim, kernel_size=1)) + nn.init.kaiming_uniform_(self.input_proj[-1].weight, a=1) + nn.init.constant_(self.input_proj[-1].bias, 0) + else: + self.input_proj.append(nn.Sequential()) + self.num_classes = num_classes + # output FFNs + assert self.mask_classification, 'why not class embedding?' + if self.mask_classification: + if self.semantic_ce_loss: + self.class_embed = nn.Linear(hidden_dim, num_classes + 1) + else: + self.class_embed = nn.Linear(hidden_dim, num_classes) + self.label_enc = nn.Embedding(num_classes, hidden_dim) + self.mask_embed = MLP(hidden_dim, hidden_dim, mask_dim, 3) + + # init decoder + self.decoder_norm = decoder_norm = nn.LayerNorm(hidden_dim) + decoder_layer = DeformableTransformerDecoderLayer( + hidden_dim, dim_feedforward, dropout, activation, + self.num_feature_levels, nhead, dec_n_points) + self.decoder = TransformerDecoder( + decoder_layer, + self.num_layers, + decoder_norm, + return_intermediate=return_intermediate_dec, + d_model=hidden_dim, + query_dim=query_dim, + num_feature_levels=self.num_feature_levels, + dec_layer_share=dec_layer_share, + ) + + self.hidden_dim = hidden_dim + self._bbox_embed = _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3) + nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0) + nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0) + box_embed_layerlist = [_bbox_embed for i in range(self.num_layers) + ] # share box prediction each layer + self.bbox_embed = nn.ModuleList(box_embed_layerlist) + self.decoder.bbox_embed = self.bbox_embed + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def pred_box(self, reference, hs, ref0=None): + """ + Args: + reference: reference box coordinates from each decoder layer + hs: content + ref0: whether there are prediction from the first layer + """ + if ref0 is None: + outputs_coord_list = [] + else: + outputs_coord_list = [ref0] + for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate( + zip(reference[:-1], self.bbox_embed, hs)): + layer_delta_unsig = layer_bbox_embed(layer_hs) + layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid( + layer_ref_sig) + layer_outputs_unsig = layer_outputs_unsig.sigmoid() + outputs_coord_list.append(layer_outputs_unsig) + outputs_coord_list = torch.stack(outputs_coord_list) + return outputs_coord_list + + def forward(self, x, mask_features, masks, targets=None): + """ + Args: + x: input, a list of multi-scale feature + mask_features: is the per-pixel embeddings with resolution 1/4 of the original image, + obtained by fusing backbone encoder encoded features. This is used to produce binary masks. + masks: mask in the original image + targets: used for denoising training + """ + assert len(x) == self.num_feature_levels + size_list = [] + # disable mask, it does not affect performance + enable_mask = 0 + if masks is not None: + for src in x: + if src.size(2) % 32 or src.size(3) % 32: + enable_mask = 1 + if enable_mask == 0: + masks = [ + torch.zeros((src.size(0), src.size(2), src.size(3)), + device=src.device, + dtype=torch.bool) for src in x + ] + src_flatten = [] + mask_flatten = [] + spatial_shapes = [] + for i in range(self.num_feature_levels): + idx = self.num_feature_levels - 1 - i + bs, c, h, w = x[idx].shape + size_list.append(x[i].shape[-2:]) + spatial_shapes.append(x[idx].shape[-2:]) + src_flatten.append(self.input_proj[idx]( + x[idx]).flatten(2).transpose(1, 2)) + mask_flatten.append(masks[i].flatten(1)) + src_flatten = torch.cat(src_flatten, 1) # bs, \sum{hxw}, c + mask_flatten = torch.cat(mask_flatten, 1) # bs, \sum{hxw} + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=src_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + predictions_class = [] + predictions_mask = [] + if self.two_stage: + output_memory, output_proposals = gen_encoder_output_proposals( + src_flatten, mask_flatten, spatial_shapes) + output_memory = self.enc_output_norm( + self.enc_output(output_memory)) + enc_outputs_class_unselected = self.class_embed(output_memory) + enc_outputs_coord_unselected = self._bbox_embed( + output_memory + ) + output_proposals # (bs, \sum{hw}, 4) unsigmoid + topk = self.num_queries + topk_proposals = torch.topk( + enc_outputs_class_unselected.max(-1)[0], topk, dim=1)[1] + refpoint_embed_undetach = torch.gather( + enc_outputs_coord_unselected, 1, + topk_proposals.unsqueeze(-1).repeat(1, 1, 4)) # unsigmoid + refpoint_embed = refpoint_embed_undetach.detach() + + tgt_undetach = torch.gather(output_memory, 1, + topk_proposals.unsqueeze(-1).repeat( + 1, 1, + self.hidden_dim)) # unsigmoid + + outputs_class, outputs_mask = self.forward_prediction_heads( + tgt_undetach.transpose(0, 1), mask_features) + tgt = tgt_undetach.detach() + if self.learn_tgt: + tgt = self.query_feat.weight[None].repeat(bs, 1, 1) + interm_outputs = dict() + interm_outputs['pred_logits'] = outputs_class + interm_outputs['pred_boxes'] = refpoint_embed_undetach.sigmoid() + interm_outputs['pred_masks'] = outputs_mask + + if self.initialize_box_type != 'no': + # convert masks into boxes to better initialize box in the decoder + assert self.initial_pred + flatten_mask = outputs_mask.detach().flatten(0, 1) + h, w = outputs_mask.shape[-2:] + if self.initialize_box_type == 'bitmask': # slower, but more accurate + refpoint_embed = get_bounding_boxes(flatten_mask > 0) + else: + assert NotImplementedError + refpoint_embed = box_xyxy_to_cxcywh( + refpoint_embed) / torch.as_tensor( + [w, h, w, h], + dtype=torch.float, + device=refpoint_embed.device) + refpoint_embed = refpoint_embed.reshape( + outputs_mask.shape[0], outputs_mask.shape[1], 4) + refpoint_embed = inverse_sigmoid(refpoint_embed) + elif not self.two_stage: + tgt = self.query_feat.weight[None].repeat(bs, 1, 1) + refpoint_embed = self.query_embed.weight[None].repeat(bs, 1, 1) + + tgt_mask = None + mask_dict = None + + # direct prediction from the matching and denoising part in the begining + if self.initial_pred: + outputs_class, outputs_mask = self.forward_prediction_heads( + tgt.transpose(0, 1), mask_features, self.training) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + hs, references = self.decoder( + tgt=tgt.transpose(0, 1), + memory=src_flatten.transpose(0, 1), + memory_key_padding_mask=mask_flatten, + pos=None, + refpoints_unsigmoid=refpoint_embed.transpose(0, 1), + level_start_index=level_start_index, + spatial_shapes=spatial_shapes, + valid_ratios=valid_ratios, + tgt_mask=tgt_mask) + for i, output in enumerate(hs): + outputs_class, outputs_mask = self.forward_prediction_heads( + output.transpose(0, 1), mask_features, self.training + or (i == len(hs) - 1)) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + # iteratively box prediction + if self.initial_pred: + out_boxes = self.pred_box(references, hs, refpoint_embed.sigmoid()) + assert len(predictions_class) == self.num_layers + 1 + else: + out_boxes = self.pred_box(references, hs) + if mask_dict is not None: + predictions_mask = torch.stack(predictions_mask) + predictions_class = torch.stack(predictions_class) + predictions_class, out_boxes, predictions_mask = \ + self.dn_post_process(predictions_class, out_boxes, mask_dict, predictions_mask) + predictions_class, predictions_mask = list( + predictions_class), list(predictions_mask) + elif self.training: # this is to insure self.label_enc participate in the model + predictions_class[-1] += 0.0 * self.label_enc.weight.sum() + + out = { + 'pred_logits': + predictions_class[-1], + 'pred_masks': + predictions_mask[-1], + 'pred_boxes': + out_boxes[-1], + 'aux_outputs': + self._set_aux_loss( + predictions_class if self.mask_classification else None, + predictions_mask, out_boxes) + } + if self.two_stage: + out['interm_outputs'] = interm_outputs + return out, mask_dict + + def forward_prediction_heads(self, output, mask_features, pred_mask=True): + decoder_output = self.decoder_norm(output) + decoder_output = decoder_output.transpose(0, 1) + outputs_class = self.class_embed(decoder_output) + outputs_mask = None + if pred_mask: + mask_embed = self.mask_embed(decoder_output) + outputs_mask = torch.einsum('bqc,bchw->bqhw', mask_embed, + mask_features) + + return outputs_class, outputs_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks, out_boxes=None): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + if out_boxes is None: + return [{ + 'pred_logits': a, + 'pred_masks': b + } for a, b in zip(outputs_class[:-1], outputs_seg_masks[:-1])] + else: + return [{ + 'pred_logits': a, + 'pred_masks': b, + 'pred_boxes': c + } for a, b, c in zip(outputs_class[:-1], outputs_seg_masks[:-1], + out_boxes[:-1])] diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_encoder.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..0116ec0b6578bdc62ca17d58b3b6f2db832f21f5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/maskdino_encoder.py @@ -0,0 +1,473 @@ +# The implementation is adopted from Mask DINO, made publicly available under the Apache License, +# Version 2.0 at https://github.com/IDEA-Research/MaskDINO +# Part of implementation is borrowed from Mask2Former, +# https://github.com/facebookresearch/Mask2Former, under MIT license. + +from typing import Any, Dict, List + +import numpy as np +import torch +from torch import nn +from torch.cuda.amp import autocast +from torch.nn import functional as F +from torch.nn.init import constant_, kaiming_uniform_, normal_ + +from .ms_deform_attn import MSDeformAttn +from .position_encoding import PositionEmbeddingSine +from .utils import Conv2d, _get_activation_fn, _get_clones + + +# MSDeformAttn Transformer encoder in deformable detr +class MSDeformAttnTransformerEncoderOnly(nn.Module): + + def __init__( + self, + d_model=256, + nhead=8, + num_encoder_layers=6, + dim_feedforward=1024, + dropout=0.1, + activation='relu', + num_feature_levels=4, + enc_n_points=4, + ): + super().__init__() + + self.d_model = d_model + self.nhead = nhead + + encoder_layer = MSDeformAttnTransformerEncoderLayer( + d_model, dim_feedforward, dropout, activation, num_feature_levels, + nhead, enc_n_points) + self.encoder = MSDeformAttnTransformerEncoder(encoder_layer, + num_encoder_layers) + + self.level_embed = nn.Parameter( + torch.Tensor(num_feature_levels, d_model)) + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + normal_(self.level_embed) + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def forward(self, srcs, masks, pos_embeds): + + enable_mask = 0 + if masks is not None: + for src in srcs: + if src.size(2) % 32 or src.size(3) % 32: + enable_mask = 1 + if enable_mask == 0: + masks = [ + torch.zeros((x.size(0), x.size(2), x.size(3)), + device=x.device, + dtype=torch.bool) for x in srcs + ] + # prepare input for encoder + src_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (src, mask, + pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): + bs, c, h, w = src.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + src = src.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + src_flatten.append(src) + mask_flatten.append(mask) + src_flatten = torch.cat(src_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=src_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + # encoder + memory = self.encoder(src_flatten, spatial_shapes, level_start_index, + valid_ratios, lvl_pos_embed_flatten, + mask_flatten) + + return memory, spatial_shapes, level_start_index + + +class MSDeformAttnTransformerEncoderLayer(nn.Module): + + def __init__(self, + d_model=256, + d_ffn=1024, + dropout=0.1, + activation='relu', + n_levels=4, + n_heads=8, + n_points=4): + super().__init__() + + # self attention + self.self_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout2 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout3 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, src): + src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) + src = src + self.dropout3(src2) + src = self.norm2(src) + return src + + def forward(self, + src, + pos, + reference_points, + spatial_shapes, + level_start_index, + padding_mask=None): + # self attention + src2 = self.self_attn( + self.with_pos_embed(src, pos), reference_points, src, + spatial_shapes, level_start_index, padding_mask) + src = src + self.dropout1(src2) + src = self.norm1(src) + + # ffn + src = self.forward_ffn(src) + + return src + + +class MSDeformAttnTransformerEncoder(nn.Module): + + def __init__(self, encoder_layer, num_layers): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + + ref_y, ref_x = torch.meshgrid( + torch.linspace( + 0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace( + 0.5, W_ - 0.5, W_, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / ( + valid_ratios[:, None, lvl, 1] * H_) + ref_x = ref_x.reshape(-1)[None] / ( + valid_ratios[:, None, lvl, 0] * W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def forward(self, + src, + spatial_shapes, + level_start_index, + valid_ratios, + pos=None, + padding_mask=None): + output = src + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=src.device) + for _, layer in enumerate(self.layers): + output = layer(output, pos, reference_points, spatial_shapes, + level_start_index, padding_mask) + + return output + + +class MaskDINOEncoder(nn.Module): + """ + This is the multi-scale encoder in detection models, also named as pixel decoder in segmentation models. + """ + + def __init__( + self, + input_shape: Dict[str, Any], + *, + transformer_dropout: float, + transformer_nheads: int, + transformer_dim_feedforward: int, + transformer_enc_layers: int, + conv_dim: int, + mask_dim: int, + # deformable transformer encoder args + transformer_in_features: List[str], + common_stride: int, + num_feature_levels: int, + total_num_feature_levels: int, + feature_order: str, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + transformer_dropout: dropout probability in transformer + transformer_nheads: number of heads in transformer + transformer_dim_feedforward: dimension of feedforward network + transformer_enc_layers: number of transformer encoder layers + conv_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + num_feature_levels: feature scales used + total_num_feature_levels: total feautre scales used (include the downsampled features) + feature_order: 'low2high' or 'high2low', i.e., 'low2high' means low-resolution features + are put in the first. + """ + super().__init__() + transformer_input_shape = { + k: v + for k, v in input_shape.items() if k in transformer_in_features + } + # this is the input shape of pixel decoder + input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride']) + self.in_features = [k for k, v in input_shape + ] # starting from "res2" to "res5" + self.feature_strides = [v['stride'] for k, v in input_shape] + self.feature_channels = [v['channels'] for k, v in input_shape] + self.feature_order = feature_order + + if feature_order == 'low2high': + transformer_input_shape = sorted( + transformer_input_shape.items(), key=lambda x: -x[1]['stride']) + else: + transformer_input_shape = sorted( + transformer_input_shape.items(), key=lambda x: x[1]['stride']) + self.transformer_in_features = [k for k, v in transformer_input_shape + ] # starting from "res2" to "res5" + transformer_in_channels = [ + v['channels'] for k, v in transformer_input_shape + ] + self.transformer_feature_strides = [ + v['stride'] for k, v in transformer_input_shape + ] # to decide extra FPN layers + + self.maskdino_num_feature_levels = num_feature_levels # always use 3 scales + self.total_num_feature_levels = total_num_feature_levels + self.common_stride = common_stride + + self.transformer_num_feature_levels = len(self.transformer_in_features) + self.low_resolution_index = transformer_in_channels.index( + max(transformer_in_channels)) + self.high_resolution_index = 0 if self.feature_order == 'low2high' else -1 + if self.transformer_num_feature_levels > 1: + input_proj_list = [] + for in_channels in transformer_in_channels[::-1]: + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + )) + # input projectino for downsample + in_channels = max(transformer_in_channels) + for _ in range( + self.total_num_feature_levels + - self.transformer_num_feature_levels): # exclude the res2 + input_proj_list.append( + nn.Sequential( + nn.Conv2d( + in_channels, + conv_dim, + kernel_size=3, + stride=2, + padding=1), + nn.GroupNorm(32, conv_dim), + )) + in_channels = conv_dim + self.input_proj = nn.ModuleList(input_proj_list) + else: + self.input_proj = nn.ModuleList([ + nn.Sequential( + nn.Conv2d( + transformer_in_channels[-1], conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + ) + ]) + + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + self.transformer = MSDeformAttnTransformerEncoderOnly( + d_model=conv_dim, + dropout=transformer_dropout, + nhead=transformer_nheads, + dim_feedforward=transformer_dim_feedforward, + num_encoder_layers=transformer_enc_layers, + num_feature_levels=self.total_num_feature_levels, + ) + N_steps = conv_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + self.mask_dim = mask_dim + # use 1x1 conv instead + self.mask_features = Conv2d( + conv_dim, + mask_dim, + kernel_size=1, + stride=1, + padding=0, + ) + self.c2_xavier_fill(self.mask_features) + # extra fpn levels + stride = min(self.transformer_feature_strides) + self.num_fpn_levels = max( + int(np.log2(stride) - np.log2(self.common_stride)), 1) + + lateral_convs = [] + output_convs = [] + + use_bias = False + for idx, in_channels in enumerate( + self.feature_channels[:self.num_fpn_levels]): + lateral_norm = nn.GroupNorm(32, conv_dim) + output_norm = nn.GroupNorm(32, conv_dim) + + lateral_conv = Conv2d( + in_channels, + conv_dim, + kernel_size=1, + bias=use_bias, + norm=lateral_norm) + output_conv = Conv2d( + conv_dim, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + self.c2_xavier_fill(lateral_conv) + self.c2_xavier_fill(output_conv) + self.add_module('adapter_{}'.format(idx + 1), lateral_conv) + self.add_module('layer_{}'.format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + def c2_xavier_fill(self, module: nn.Module) -> None: + kaiming_uniform_(module.weight, a=1) + if module.bias is not None: + constant_(module.bias, 0) + + @autocast(enabled=False) + def forward_features(self, features, masks): + """ + Args: + features: multi-scale features from the backbone + masks: image mask + Returns: + enhanced multi-scale features and mask feature (1/4 resolution) for the decoder to produce binary mask + """ + # backbone features + srcs = [] + pos = [] + # additional downsampled features + srcsl = [] + posl = [] + if self.total_num_feature_levels > self.transformer_num_feature_levels: + smallest_feat = features[self.transformer_in_features[ + self.low_resolution_index]].float() + _len_srcs = self.transformer_num_feature_levels + for i in range(_len_srcs, self.total_num_feature_levels): + if i == _len_srcs: + src = self.input_proj[i](smallest_feat) + else: + src = self.input_proj[i](srcsl[-1]) + srcsl.append(src) + posl.append(self.pe_layer(src)) + srcsl = srcsl[::-1] + # Reverse feature maps + for idx, f in enumerate(self.transformer_in_features[::-1]): + x = features[f].float( + ) # deformable detr does not support half precision + srcs.append(self.input_proj[idx](x)) + pos.append(self.pe_layer(x)) + srcs.extend( + srcsl) if self.feature_order == 'low2high' else srcsl.extend(srcs) + pos.extend(posl) if self.feature_order == 'low2high' else posl.extend( + pos) + if self.feature_order != 'low2high': + srcs = srcsl + pos = posl + y, spatial_shapes, level_start_index = self.transformer( + srcs, masks, pos) + bs = y.shape[0] + + split_size_or_sections = [None] * self.total_num_feature_levels + for i in range(self.total_num_feature_levels): + if i < self.total_num_feature_levels - 1: + split_size_or_sections[i] = level_start_index[ + i + 1] - level_start_index[i] + else: + split_size_or_sections[i] = y.shape[1] - level_start_index[i] + y = torch.split(y, split_size_or_sections, dim=1) + + out = [] + multi_scale_features = [] + num_cur_levels = 0 + for i, z in enumerate(y): + out.append( + z.transpose(1, 2).view(bs, -1, spatial_shapes[i][0], + spatial_shapes[i][1])) + + # append `out` with extra FPN levels + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[:self.num_fpn_levels][::-1]): + x = features[f].float() + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + cur_fpn = lateral_conv(x) + # Following FPN implementation, we use nearest upsampling here + y = cur_fpn + F.interpolate( + out[self.high_resolution_index], + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + out.append(y) + for o in out: + if num_cur_levels < self.total_num_feature_levels: + multi_scale_features.append(o) + num_cur_levels += 1 + return self.mask_features(out[-1]), out[0], multi_scale_features diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/ms_deform_attn.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..4bcdb998186a842077c2f0655d8649640f3ff912 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/ms_deform_attn.py @@ -0,0 +1,160 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +# Copyright (c) Alibaba, Inc. and its affiliates. + +from __future__ import absolute_import, division, print_function +import math +import warnings + +import torch +import torch.nn.functional as F +from mmcv.ops.multi_scale_deform_attn import ( + MultiScaleDeformableAttnFunction, multi_scale_deformable_attn_pytorch) +from torch import nn +from torch.nn.init import constant_, xavier_uniform_ + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + 'invalid input for _is_power_of_2: {} (type: {})'.format( + n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """Multi-Scale Deformable Attention Module + + Args: + d_model: hidden dimension + n_levels: number of feature levels + n_heads: number of attention heads + n_points: number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError( + 'd_model must be divisible by n_heads, but got {} and {}'. + format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn( + "You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + 'which is more efficient in our CUDA implementation.') + + self.im2col_step = 128 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, + n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, + n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange( + self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init + / grid_init.abs().max(-1, keepdim=True)[0]).view( + self.n_heads, 1, 1, 2).repeat(1, self.n_levels, + self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, + query, + reference_points, + input_flatten, + input_spatial_shapes, + input_level_start_index, + input_padding_mask=None): + """ + Args: + query: (N, Length_{query}, C) + reference_points: (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area or (N, Length_{query}, n_levels, 4), + add additional (w, h) to form reference boxes + input_flatten: (N, H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}, C) + input_spatial_shapes: (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + input_level_start_index: (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, + H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + input_padding_mask: (N, H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}), True for padding elements, + False for non-padding elements + + Returns: + output: (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] + * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, + self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view( + N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view( + N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, + -1).view(N, Len_q, self.n_heads, + self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack( + [input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], + -1) + sampling_locations = ( + reference_points[:, :, None, :, None, :] + sampling_offsets + / offset_normalizer[None, None, None, :, None, :]) + elif reference_points.shape[-1] == 4: + sampling_locations = ( + reference_points[:, :, None, :, None, :2] + + sampling_offsets / self.n_points + * reference_points[:, :, None, :, None, 2:] * 0.5) + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.' + .format(reference_points.shape[-1])) + try: + output = MultiScaleDeformableAttnFunction.apply( + value, input_spatial_shapes, input_level_start_index, + sampling_locations, attention_weights, self.im2col_step) + except Exception: + # CPU + output = multi_scale_deformable_attn_pytorch( + value, input_spatial_shapes, sampling_locations, + attention_weights) + output = self.output_proj(output) + return output diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/position_encoding.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..923f68ece46f633aee7b7cd66cab408b78194ba9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/position_encoding.py @@ -0,0 +1,71 @@ +# The implementation is adopted from Mask DINO, made publicly available under the Apache License, +# Version 2.0 at https://github.com/IDEA-Research/MaskDINO +# Part of implementation is borrowed from Mask2Former, +# https://github.com/facebookresearch/Mask2Former, under MIT license. + +import math + +import torch +from torch import nn + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, + num_pos_feats=64, + temperature=10000, + normalize=False, + scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError('normalize should be True if scale is passed') + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, x, mask=None): + if mask is None: + mask = torch.zeros((x.size(0), x.size(2), x.size(3)), + device=x.device, + dtype=torch.bool) + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange( + self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature**(2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + def __repr__(self, _repr_indent=4): + head = 'Positional encoding ' + self.__class__.__name__ + body = [ + 'num_pos_feats: {}'.format(self.num_pos_feats), + 'temperature: {}'.format(self.temperature), + 'normalize: {}'.format(self.normalize), + 'scale: {}'.format(self.scale), + ] + # _repr_indent = 4 + lines = [head] + [' ' * _repr_indent + line for line in body] + return '\n'.join(lines) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/utils.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bc674d0be9415de2d612b525f351f9117774a764 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino/utils.py @@ -0,0 +1,184 @@ +# Part of the implementation is borrowed and modified from Mask DINO, publicly available at +# https://github.com/IDEA-Research/MaskDINO + +import copy +import math + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + + +class Conv2d(torch.nn.Conv2d): + + def __init__(self, *args, **kwargs): + norm = kwargs.pop('norm', None) + activation = kwargs.pop('activation', None) + super().__init__(*args, **kwargs) + + self.norm = norm + self.activation = activation + + def forward(self, x): + x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + +class MLP(nn.Module): + """ Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList( + nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +def get_bounding_boxes(mask_tensor): + boxes = torch.zeros( + mask_tensor.shape[0], + 4, + dtype=torch.float32, + device=mask_tensor.device) + x_any = torch.any(mask_tensor, dim=1) + y_any = torch.any(mask_tensor, dim=2) + for idx in range(mask_tensor.shape[0]): + x = torch.where(x_any[idx, :])[0] + y = torch.where(y_any[idx, :])[0] + if len(x) > 0 and len(y) > 0: + boxes[idx, :] = torch.as_tensor([x[0], y[0], x[-1] + 1, y[-1] + 1], + dtype=torch.float32, + device=mask_tensor.device) + return boxes + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def gen_encoder_output_proposals(memory: Tensor, memory_padding_mask: Tensor, + spatial_shapes: Tensor): + """ + Args: + memory: bs, h_0*w_0+...+h_{L-1}*w_{L-1}, d_model + memory_padding_mask: bs, h_0*w_0+...+h_{L-1}*w_{L-1} + spatial_shapes: nlevel, 2 + Returns: + output_memory: bs, h_0*w_0+...+h_{L-1}*w_{L-1}, d_model + output_proposals: bs, h_0*w_0+...+h_{L-1}*w_{L-1}, 4 + """ + N_, S_, C_ = memory.shape + proposals = [] + _cur = 0 + for lvl, (H_, W_) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur:(_cur + H_ * W_)].view( + N_, H_, W_, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + grid_y, grid_x = torch.meshgrid( + torch.linspace( + 0, H_ - 1, H_, dtype=torch.float32, device=memory.device), + torch.linspace( + 0, W_ - 1, W_, dtype=torch.float32, device=memory.device)) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) + + scale = torch.cat([valid_W.unsqueeze(-1), + valid_H.unsqueeze(-1)], 1).view(N_, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale + wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) + proposal = torch.cat((grid, wh), -1).view(N_, -1, 4) + proposals.append(proposal) + _cur += (H_ * W_) + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = (output_proposals > 0.01) & ( + output_proposals < 0.99) + output_proposals_valid = output_proposals_valid.all(-1, keepdim=True) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) + output_proposals = output_proposals.masked_fill( + memory_padding_mask.unsqueeze(-1), float('inf')) + output_proposals = output_proposals.masked_fill(~output_proposals_valid, + float('inf')) + + output_memory = memory + output_memory = output_memory.masked_fill( + memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, + float(0)) + return output_memory, output_proposals + + +def gen_sineembed_for_position(pos_tensor): + # n_query, bs, _ = pos_tensor.size() + # sineembed_tensor = torch.zeros(n_query, bs, 256) + scale = 2 * math.pi + dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) + dim_t = 10000**(2 * (dim_t // 2) / 128) + x_embed = pos_tensor[:, :, 0] * scale + y_embed = pos_tensor[:, :, 1] * scale + pos_x = x_embed[:, :, None] / dim_t + pos_y = y_embed[:, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), + dim=3).flatten(2) + pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), + dim=3).flatten(2) + if pos_tensor.size(-1) == 2: + pos = torch.cat((pos_y, pos_x), dim=2) + elif pos_tensor.size(-1) == 4: + w_embed = pos_tensor[:, :, 2] * scale + pos_w = w_embed[:, :, None] / dim_t + pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), + dim=3).flatten(2) + + h_embed = pos_tensor[:, :, 3] * scale + pos_h = h_embed[:, :, None] / dim_t + pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), + dim=3).flatten(2) + + pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) + else: + raise ValueError('Unknown pos_tensor shape(-1):{}'.format( + pos_tensor.size(-1))) + return pos + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == 'relu': + return F.relu + if activation == 'gelu': + return F.gelu + if activation == 'glu': + return F.glu + if activation == 'prelu': + return nn.PReLU() + if activation == 'selu': + return F.selu + raise RuntimeError(F'activation should be relu/gelu, not {activation}.') + + +def _get_clones(module, N, layer_share=False): + if layer_share: + return nn.ModuleList([module for i in range(N)]) + else: + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_model.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5cdf48c19a57fa25a819df81f7ccb28cf0f5a304 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_model.py @@ -0,0 +1,44 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation import MaskDINOSwin +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_segmentation, module_name=Models.maskdino_swin) +class MaskDINOSwinModel(TorchModel): + + def __init__(self, model_dir=None, *args, **kwargs): + """ + Args: + model_dir (str): model directory. + """ + super(MaskDINOSwinModel, self).__init__( + model_dir=model_dir, *args, **kwargs) + + if 'backbone' not in kwargs: + config_path = os.path.join(model_dir, ModelFile.CONFIGURATION) + cfg = Config.from_file(config_path) + model_cfg = cfg.model + kwargs.update(model_cfg) + + self.model = MaskDINOSwin(model_dir=model_dir, **kwargs) + + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + self.model.to(self.device) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + output = self.model(**input) + return output + + def postprocess(self, input: Dict[str, Any], **kwargs) -> Dict[str, Any]: + return input diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_swin.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_swin.py new file mode 100644 index 0000000000000000000000000000000000000000..8c2aa7d2d305be0b0bbe4d328b5c5d230d720adb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/maskdino_swin.py @@ -0,0 +1,282 @@ +# Part of the implementation is borrowed and modified from Mask DINO, publicly available at +# https://github.com/IDEA-Research/MaskDINO +# Part of implementation is borrowed and modified from Mask2Former, publicly available at +# https://github.com/facebookresearch/Mask2Former. + +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.image_instance_segmentation.backbones import \ + D2SwinTransformer +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger +from .maskdino.maskdino_decoder import MaskDINODecoder +from .maskdino.maskdino_encoder import MaskDINOEncoder + +logger = get_logger() + + +class MaskDINOSwin(nn.Module): + + def __init__(self, backbone, encoder, decoder, pretrained=None, **kwargs): + """ + Mask DINO: Towards A Unified Transformer-based Framework for Object + Detection and Segmentation. See https://arxiv.org/abs/2206.02777 + Args: + backbone (dict): backbone config. + encoder (dict): encoder config. + decoder (dict): decoder config. + pretrained (bool): whether to use pretrained model + """ + super(MaskDINOSwin, self).__init__() + self.register_buffer( + 'pixel_mean', + torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False) + self.register_buffer( + 'pixel_std', + torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False) + self.size_divisibility = 32 + + self.backbone = D2SwinTransformer(**backbone) + input_shape = { + k: v + for k, v in self.backbone.output_shape().items() + if k in encoder['transformer_in_features'] + } + encoder = MaskDINOEncoder(input_shape=input_shape, **encoder) + decoder = MaskDINODecoder(**decoder) + self.sem_seg_head = MaskDINOHead( + pixel_decoder=encoder, transformer_predictor=decoder) + self.num_classes = decoder.num_classes + self.num_queries = decoder.num_queries + self.test_topk_per_image = 100 + + self.classes = kwargs.pop('classes', None) + + if pretrained: + assert 'model_dir' in kwargs, 'pretrained model dir is missing.' + model_path = os.path.join(kwargs['model_dir'], + ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['model'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info(f'size mismatch for {name}, skip loading.') + del weight[name] + else: + logger.info( + f'{name} doesn\'t exist in current model, skip loading.' + ) + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + @property + def device(self): + return self.pixel_mean.device + + def forward(self, batched_inputs, **kwargs): + + images = [x['image'].to(self.device) for x in batched_inputs] + images = [(255. * x - self.pixel_mean) / self.pixel_std + for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + features = self.backbone(images.tensor) + + if self.training: + raise NotImplementedError + else: + outputs, _ = self.sem_seg_head(features) + mask_cls_results = outputs['pred_logits'] + mask_pred_results = outputs['pred_masks'] + mask_box_results = outputs['pred_boxes'] + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode='bilinear', + align_corners=False, + ) + + del outputs + + processed_results = [] + for mask_cls_result, mask_pred_result, mask_box_result, input_per_image, image_size in zip( + mask_cls_results, mask_pred_results, mask_box_results, + batched_inputs, images.image_sizes + ): # image_size is augmented size, not divisible to 32 + height = input_per_image.get('height', + image_size[0]) # real size + width = input_per_image.get('width', image_size[1]) + processed_results.append({}) + new_size = mask_pred_result.shape[ + -2:] # padded size (divisible to 32) + + # post process + mask_pred_result = mask_pred_result[:, :image_size[0], : + image_size[1]].expand( + 1, -1, -1, -1) + mask_pred_result = F.interpolate( + mask_pred_result, + size=(height, width), + mode='bilinear', + align_corners=False)[0] + + mask_cls_result = mask_cls_result.to(mask_pred_result) + mask_box_result = mask_box_result.to(mask_pred_result) + height = new_size[0] / image_size[0] * height + width = new_size[1] / image_size[1] * width + mask_box_result = self.box_postprocess(mask_box_result, height, + width) + + instance_r = self.instance_inference(mask_cls_result, + mask_pred_result, + mask_box_result) + processed_results[-1]['instances'] = instance_r + + return dict(eval_result=processed_results) + + def instance_inference(self, mask_cls, mask_pred, mask_box_result): + # mask_pred is already processed to have the same shape as original input + image_size = mask_pred.shape[-2:] + scores = mask_cls.sigmoid() # [100, 80] + labels = torch.arange( + self.num_classes, + device=self.device).unsqueeze(0).repeat(self.num_queries, + 1).flatten(0, 1) + scores_per_image, topk_indices = scores.flatten(0, 1).topk( + self.test_topk_per_image, sorted=False) # select 100 + labels_per_image = labels[topk_indices] + topk_indices = topk_indices // self.num_classes + mask_pred = mask_pred[topk_indices] + # if this is panoptic segmentation, we only keep the "thing" classes + + result = {'image_size': image_size} + # mask (before sigmoid) + result['pred_masks'] = (mask_pred > 0).float() + # half mask box half pred box + mask_box_result = mask_box_result[topk_indices] + result['pred_boxes'] = mask_box_result + + # calculate average mask prob + mask_scores_per_image = (mask_pred.sigmoid().flatten(1) + * result['pred_masks'].flatten(1)).sum(1) / ( + result['pred_masks'].flatten(1).sum(1) + + 1e-6) + result['scores'] = scores_per_image * mask_scores_per_image + result['pred_classes'] = labels_per_image + return result + + def box_postprocess(self, out_bbox, img_h, img_w): + # postprocess box height and width + x_c, y_c, w, h = out_bbox.unbind(-1) + boxes = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), + (y_c + 0.5 * h)] + boxes = torch.stack(boxes, dim=-1) + scale_fct = torch.tensor([img_w, img_h, img_w, img_h]) + scale_fct = scale_fct.to(out_bbox) + boxes = boxes * scale_fct + return boxes + + +class MaskDINOHead(nn.Module): + + def __init__( + self, + pixel_decoder: nn.Module, + transformer_predictor: nn.Module, + ): + super().__init__() + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + + def forward(self, features, mask=None, targets=None): + return self.layers(features, mask, targets=targets) + + def layers(self, features, mask=None, targets=None): + mask_features, transformer_encoder_features, multi_scale_features = self.pixel_decoder.forward_features( + features, mask) + predictions = self.predictor( + multi_scale_features, mask_features, mask, targets=targets) + return predictions + + +class ImageList(object): + + def __init__(self, tensor, image_sizes): + self.tensor = tensor + self.image_sizes = image_sizes + + def __len__(self): + return len(self.image_sizes) + + def __getitem__(self, idx): + size = self.image_sizes[idx] + return self.tensor[idx, ..., :size[0], :size[1]] + + @torch.jit.unused + def to(self, *args, **kwargs): + cast_tensor = self.tensor.to(*args, **kwargs) + return ImageList(cast_tensor, self.image_sizes) + + @property + def device(self): + return self.tensor.device + + @staticmethod + def from_tensors(tensors, size_divisibility=0, pad_value=0.0): + assert len(tensors) > 0 + assert isinstance(tensors, (tuple, list)) + for t in tensors: + assert isinstance(t, torch.Tensor), type(t) + assert t.shape[:-2] == tensors[0].shape[:-2], t.shape + + image_sizes = [(im.shape[-2], im.shape[-1]) for im in tensors] + image_sizes_tensor = [torch.as_tensor(x) for x in image_sizes] + max_size = torch.stack(image_sizes_tensor).max(0).values + + if size_divisibility > 1: + stride = size_divisibility + # the last two dims are H,W, both subject to divisibility requirement + max_size = (max_size + (stride - 1)) // stride * stride + + # handle weirdness of scripting and tracing ... + if torch.jit.is_scripting(): + max_size = max_size.to(dtype=torch.long).tolist() + else: + if torch.jit.is_tracing(): + image_sizes = image_sizes_tensor + + if len(tensors) == 1: + image_size = image_sizes[0] + padding_size = [ + 0, max_size[-1] - image_size[1], 0, + max_size[-2] - image_size[0] + ] + batched_imgs = F.pad( + tensors[0], padding_size, value=pad_value).unsqueeze_(0) + else: + # max_size can be a tensor in tracing mode, therefore convert to list + batch_shape = [len(tensors)] + list( + tensors[0].shape[:-2]) + list(max_size) + batched_imgs = tensors[0].new_full(batch_shape, pad_value) + for img, pad_img in zip(tensors, batched_imgs): + pad_img[..., :img.shape[-2], :img.shape[-1]].copy_(img) + + return ImageList(batched_imgs.contiguous(), image_sizes) diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/model.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..a56a160839d98a888b396084cdb3927746f1397b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/model.py @@ -0,0 +1,50 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation import \ + CascadeMaskRCNNSwin +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_segmentation, module_name=Models.cascade_mask_rcnn_swin) +class CascadeMaskRCNNSwinModel(TorchModel): + + def __init__(self, model_dir=None, *args, **kwargs): + """ + Args: + model_dir (str): model directory. + + """ + super(CascadeMaskRCNNSwinModel, self).__init__( + model_dir=model_dir, *args, **kwargs) + + if 'backbone' not in kwargs: + config_path = os.path.join(model_dir, ModelFile.CONFIGURATION) + cfg = Config.from_file(config_path) + model_cfg = cfg.model + kwargs.update(model_cfg) + + self.model = CascadeMaskRCNNSwin(model_dir=model_dir, **kwargs) + + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + self.model.to(self.device) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + output = self.model(**input) + return output + + def postprocess(self, input: Dict[str, Any], **kwargs) -> Dict[str, Any]: + + return input + + def compute_loss(self, outputs: Dict[str, Any], labels): + pass diff --git a/runtime/Lib/site_packages/models/cv/image_instance_segmentation/postprocess_utils.py b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/postprocess_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aad7d8e92f25ff2898027a16d33c20b544af65c7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_instance_segmentation/postprocess_utils.py @@ -0,0 +1,242 @@ +# Part of the implementation is borrowed and modified from MMDetection, publicly available at +# https://github.com/open-mmlab/mmdetection/blob/master/mmdet/core/visualization/image.py +import itertools + +import cv2 +import numpy as np +import pycocotools.mask as maskUtils +import torch + +from modelscope.outputs import OutputKeys + + +def get_seg_bboxes(bboxes, labels, segms=None, class_names=None, score_thr=0.): + assert bboxes.ndim == 2, \ + f' bboxes ndim should be 2, but its ndim is {bboxes.ndim}.' + assert labels.ndim == 1, \ + f' labels ndim should be 1, but its ndim is {labels.ndim}.' + assert bboxes.shape[0] == labels.shape[0], \ + 'bboxes.shape[0] and labels.shape[0] should have the same length.' + assert bboxes.shape[1] == 4 or bboxes.shape[1] == 5, \ + f' bboxes.shape[1] should be 4 or 5, but its {bboxes.shape[1]}.' + + if score_thr > 0: + assert bboxes.shape[1] == 5 + scores = bboxes[:, -1] + inds = scores > score_thr + bboxes = bboxes[inds, :] + labels = labels[inds] + if segms is not None: + segms = segms[inds, ...] + + bboxes_names = [] + for i, (bbox, label) in enumerate(zip(bboxes, labels)): + label_name = class_names[ + label] if class_names is not None else f'class {label}' + bbox = [0 if b < 0 else b for b in list(bbox)] + bbox.append(label_name) + bbox.append(segms[i].astype(bool)) + bboxes_names.append(bbox) + + return bboxes_names + + +def get_img_seg_results(det_rawdata=None, + class_names=None, + score_thr=0.3, + is_decode=True): + ''' + Get all boxes of one image. + score_thr: Classification probability threshold。 + output format: [ [x1,y1,x2,y2, prob, cls_name, mask], [x1,y1,x2,y2, prob, cls_name, mask], ... ] + ''' + assert det_rawdata is not None, 'det_rawdata should be not None.' + assert class_names is not None, 'class_names should be not None.' + + if isinstance(det_rawdata, tuple): + bbox_result, segm_result = det_rawdata + if isinstance(segm_result, tuple): + segm_result = segm_result[0] # ms rcnn + else: + bbox_result, segm_result = det_rawdata, None + bboxes = np.vstack(bbox_result) + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + + segms = None + if segm_result is not None and len(labels) > 0: # non empty + segms = list(itertools.chain(*segm_result)) + if is_decode: + segms = maskUtils.decode(segms) + segms = segms.transpose(2, 0, 1) + if isinstance(segms[0], torch.Tensor): + segms = torch.stack(segms, dim=0).detach().cpu().numpy() + else: + segms = np.stack(segms, axis=0) + + bboxes_names = get_seg_bboxes( + bboxes, + labels, + segms=segms, + class_names=class_names, + score_thr=score_thr) + + return bboxes_names + + +def get_img_ins_seg_result(img_seg_result=None, + class_names=None, + score_thr=0.3): + assert img_seg_result is not None, 'img_seg_result should be not None.' + assert class_names is not None, 'class_names should be not None.' + + img_seg_result = get_img_seg_results( + det_rawdata=(img_seg_result[0], img_seg_result[1]), + class_names=class_names, + score_thr=score_thr, + is_decode=False) + + results_dict = { + OutputKeys.BOXES: [], + OutputKeys.MASKS: [], + OutputKeys.LABELS: [], + OutputKeys.SCORES: [] + } + for seg_result in img_seg_result: + + box = [ + int(seg_result[0]), + int(seg_result[1]), + int(seg_result[2]), + int(seg_result[3]) + ] + score = float(seg_result[4]) + category = seg_result[5] + + mask = np.array(seg_result[6], order='F', dtype='uint8') + mask = mask.astype(float) + + results_dict[OutputKeys.BOXES].append(box) + results_dict[OutputKeys.MASKS].append(mask) + results_dict[OutputKeys.SCORES].append(score) + results_dict[OutputKeys.LABELS].append(category) + + return results_dict + + +def show_result( + img, + result, + out_file='result.jpg', + show_box=True, + show_label=True, + show_score=True, + alpha=0.5, + fontScale=0.5, + fontFace=cv2.FONT_HERSHEY_COMPLEX_SMALL, + thickness=1, +): + + assert isinstance(img, (str, np.ndarray)), \ + f'img must be str or np.ndarray, but got {type(img)}.' + + if isinstance(img, str): + img = cv2.imread(img) + if len(img.shape) == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = img.astype(np.float32) + + labels = result[OutputKeys.LABELS] + scores = result[OutputKeys.SCORES] + boxes = result[OutputKeys.BOXES] + masks = result[OutputKeys.MASKS] + + for label, score, box, mask in zip(labels, scores, boxes, masks): + + random_color = np.array([ + np.random.random() * 255.0, + np.random.random() * 255.0, + np.random.random() * 255.0 + ]) + + x1 = int(box[0]) + y1 = int(box[1]) + x2 = int(box[2]) + y2 = int(box[3]) + + if show_box: + cv2.rectangle( + img, (x1, y1), (x2, y2), random_color, thickness=thickness) + if show_label or show_score: + if show_label and show_score: + text = '{}|{}'.format(label, round(float(score), 2)) + elif show_label: + text = '{}'.format(label) + else: + text = '{}'.format(round(float(score), 2)) + + retval, baseLine = cv2.getTextSize( + text, + fontFace=fontFace, + fontScale=fontScale, + thickness=thickness) + cv2.rectangle( + img, (x1, y1 - retval[1] - baseLine), (x1 + retval[0], y1), + thickness=-1, + color=(0, 0, 0)) + cv2.putText( + img, + text, (x1, y1 - baseLine), + fontScale=fontScale, + fontFace=fontFace, + thickness=thickness, + color=random_color) + + idx = np.nonzero(mask) + img[idx[0], idx[1], :] *= 1.0 - alpha + img[idx[0], idx[1], :] += alpha * random_color + + cv2.imwrite(out_file, img) + + +def get_maskdino_ins_seg_result(maskdino_seg_result, + class_names, + score_thr=0.3): + scores = maskdino_seg_result['scores'].detach().cpu().numpy() + pred_masks = maskdino_seg_result['pred_masks'].detach().cpu().numpy() + pred_boxes = maskdino_seg_result['pred_boxes'].detach().cpu().numpy() + pred_classes = maskdino_seg_result['pred_classes'].detach().cpu().numpy() + + thresholded_idxs = np.array(scores) >= score_thr + scores = scores[thresholded_idxs] + pred_classes = pred_classes[thresholded_idxs] + pred_masks = pred_masks[thresholded_idxs] + pred_boxes = pred_boxes[thresholded_idxs] + + results_dict = { + OutputKeys.BOXES: [], + OutputKeys.MASKS: [], + OutputKeys.LABELS: [], + OutputKeys.SCORES: [] + } + for score, cls, mask, box in zip(scores, pred_classes, pred_masks, + pred_boxes): + score = np.float64(score) + label = class_names[int(cls)] + mask = np.array(mask, dtype=np.float64) + box = [ + np.int64(box[0]), + np.int64(box[1]), + np.int64(box[2]), + np.int64(box[3]) + ] + results_dict[OutputKeys.SCORES].append(score) + results_dict[OutputKeys.LABELS].append(label) + results_dict[OutputKeys.MASKS].append(mask) + results_dict[OutputKeys.BOXES].append(box) + + return results_dict diff --git a/runtime/Lib/site_packages/models/cv/image_matching/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..919c249c58a30a372635b2acab77b5d576a91eb6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .quadtree_attention_model import QuadTreeAttentionForImageMatching + +else: + _import_structure = { + 'quadtree_attention_model': ['QuadTreeAttentionForImageMatching'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_matching/config/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_matching/config/default.py b/runtime/Lib/site_packages/models/cv/image_matching/config/default.py new file mode 100644 index 0000000000000000000000000000000000000000..129d155977b872e8007644e8871442150b7e9d44 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/config/default.py @@ -0,0 +1,173 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +from yacs.config import CfgNode as CN + +_CN = CN() + +_CN.LOFTR = CN() +_CN.LOFTR.BACKBONE_TYPE = 'ResNetFPN' +_CN.LOFTR.RESOLUTION = (8, 2) # options: [(8, 2), (16, 4)] +_CN.LOFTR.FINE_WINDOW_SIZE = 5 # window_size in fine_level, must be odd +_CN.LOFTR.FINE_CONCAT_COARSE_FEAT = True + +# 1. LoFTR-backbone (local feature CNN) config +_CN.LOFTR.RESNETFPN = CN() +_CN.LOFTR.RESNETFPN.INITIAL_DIM = 128 +_CN.LOFTR.RESNETFPN.BLOCK_DIMS = [128, 196, 256] # s1, s2, s3 + +# 2. LoFTR-coarse module config +_CN.LOFTR.COARSE = CN() +_CN.LOFTR.COARSE.D_MODEL = 256 +_CN.LOFTR.COARSE.D_FFN = 256 +_CN.LOFTR.COARSE.NHEAD = 8 +_CN.LOFTR.COARSE.LAYER_NAMES = ['self', 'cross'] * 4 +_CN.LOFTR.COARSE.ATTENTION = 'linear' # options: ['linear', 'full'] +_CN.LOFTR.COARSE.TEMP_BUG_FIX = True +_CN.LOFTR.COARSE.BLOCK_TYPE = 'quadtree' +_CN.LOFTR.COARSE.ATTN_TYPE = 'B' +_CN.LOFTR.COARSE.TOPKS = [16, 8, 8] + +# 3. Coarse-Matching config +_CN.LOFTR.MATCH_COARSE = CN() +_CN.LOFTR.MATCH_COARSE.THR = 0.2 +_CN.LOFTR.MATCH_COARSE.BORDER_RM = 2 +_CN.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' # options: ['dual_softmax, 'sinkhorn'] +_CN.LOFTR.MATCH_COARSE.DSMAX_TEMPERATURE = 0.1 +_CN.LOFTR.MATCH_COARSE.SKH_ITERS = 3 +_CN.LOFTR.MATCH_COARSE.SKH_INIT_BIN_SCORE = 1.0 +_CN.LOFTR.MATCH_COARSE.SKH_PREFILTER = False +_CN.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 # training tricks: save GPU memory +_CN.LOFTR.MATCH_COARSE.TRAIN_PAD_NUM_GT_MIN = 200 # training tricks: avoid DDP deadlock +_CN.LOFTR.MATCH_COARSE.SPARSE_SPVS = False + +# 4. LoFTR-fine module config +_CN.LOFTR.FINE = CN() +_CN.LOFTR.FINE.D_MODEL = 128 +_CN.LOFTR.FINE.D_FFN = 128 +_CN.LOFTR.FINE.NHEAD = 8 +_CN.LOFTR.FINE.LAYER_NAMES = ['self', 'cross'] * 1 +_CN.LOFTR.FINE.ATTENTION = 'linear' +_CN.LOFTR.FINE.BLOCK_TYPE = 'loftr' + +# 5. LoFTR Losses +# -- # coarse-level +_CN.LOFTR.LOSS = CN() +_CN.LOFTR.LOSS.COARSE_TYPE = 'focal' # ['focal', 'cross_entropy'] +_CN.LOFTR.LOSS.COARSE_WEIGHT = 1.0 +# -- - -- # focal loss (coarse) +_CN.LOFTR.LOSS.FOCAL_ALPHA = 0.25 +_CN.LOFTR.LOSS.FOCAL_GAMMA = 2.0 +_CN.LOFTR.LOSS.POS_WEIGHT = 1.0 +_CN.LOFTR.LOSS.NEG_WEIGHT = 1.0 + +# -- # fine-level +_CN.LOFTR.LOSS.FINE_TYPE = 'l2_with_std' # ['l2_with_std', 'l2'] +_CN.LOFTR.LOSS.FINE_WEIGHT = 1.0 +_CN.LOFTR.LOSS.FINE_CORRECT_THR = 1.0 + +_CN.DATASET = CN() +# 1. data config +# training and validating +_CN.DATASET.TRAINVAL_DATA_SOURCE = None # options: ['ScanNet', 'MegaDepth'] +_CN.DATASET.TRAIN_DATA_ROOT = None +_CN.DATASET.TRAIN_POSE_ROOT = None # (optional directory for poses) +_CN.DATASET.TRAIN_NPZ_ROOT = None +_CN.DATASET.TRAIN_LIST_PATH = None +_CN.DATASET.TRAIN_INTRINSIC_PATH = None +_CN.DATASET.VAL_DATA_ROOT = None +_CN.DATASET.VAL_POSE_ROOT = None # (optional directory for poses) +_CN.DATASET.VAL_NPZ_ROOT = None +_CN.DATASET.VAL_LIST_PATH = None # None if val data from all scenes are bundled into a single npz file +_CN.DATASET.VAL_INTRINSIC_PATH = None +# testing +_CN.DATASET.TEST_DATA_SOURCE = None +_CN.DATASET.TEST_DATA_ROOT = None +_CN.DATASET.TEST_POSE_ROOT = None # (optional directory for poses) +_CN.DATASET.TEST_NPZ_ROOT = None +_CN.DATASET.TEST_LIST_PATH = None # None if test data from all scenes are bundled into a single npz file +_CN.DATASET.TEST_INTRINSIC_PATH = None + +# 2. dataset config +# general options +_CN.DATASET.MIN_OVERLAP_SCORE_TRAIN = 0.4 # discard data with overlap_score < min_overlap_score +_CN.DATASET.MIN_OVERLAP_SCORE_TEST = 0.0 +_CN.DATASET.AUGMENTATION_TYPE = None # options: [None, 'dark', 'mobile'] + +# MegaDepth options +_CN.DATASET.MGDPT_IMG_RESIZE = 640 # resize the longer side, zero-pad bottom-right to square. +_CN.DATASET.MGDPT_IMG_PAD = True # pad img to square with size = MGDPT_IMG_RESIZE +_CN.DATASET.MGDPT_DEPTH_PAD = True # pad depthmap to square with size = 2000 +_CN.DATASET.MGDPT_DF = 8 + +_CN.TRAINER = CN() +_CN.TRAINER.WORLD_SIZE = 1 +_CN.TRAINER.CANONICAL_BS = 64 +_CN.TRAINER.CANONICAL_LR = 8e-3 +_CN.TRAINER.SCALING = None # this will be calculated automatically +_CN.TRAINER.FIND_LR = False # use learning rate finder from pytorch-lightning + +# optimizer +_CN.TRAINER.OPTIMIZER = 'adamw' # [adam, adamw] +_CN.TRAINER.TRUE_LR = None # this will be calculated automatically at runtime +_CN.TRAINER.ADAM_DECAY = 0. # ADAM: for adam +_CN.TRAINER.ADAMW_DECAY = 0.1 + +# step-based warm-up +_CN.TRAINER.WARMUP_TYPE = 'linear' # [linear, constant] +_CN.TRAINER.WARMUP_RATIO = 0.1 +_CN.TRAINER.WARMUP_STEP = 1875 + +# learning rate scheduler +_CN.TRAINER.SCHEDULER = 'MultiStepLR' # [MultiStepLR, CosineAnnealing, ExponentialLR] +_CN.TRAINER.SCHEDULER_INTERVAL = 'epoch' # [epoch, step] +_CN.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] # MSLR: MultiStepLR +_CN.TRAINER.MSLR_GAMMA = 0.5 +_CN.TRAINER.COSA_TMAX = 30 # COSA: CosineAnnealing +_CN.TRAINER.ELR_GAMMA = 0.999992 # ELR: ExponentialLR, this value for 'step' interval + +# plotting related +_CN.TRAINER.ENABLE_PLOTTING = True +_CN.TRAINER.N_VAL_PAIRS_TO_PLOT = 32 # number of val/test paris for plotting +_CN.TRAINER.PLOT_MODE = 'evaluation' # ['evaluation', 'confidence'] +_CN.TRAINER.PLOT_MATCHES_ALPHA = 'dynamic' + +# geometric metrics and pose solver +_CN.TRAINER.EPI_ERR_THR = 5e-4 # recommendation: 5e-4 for ScanNet, 1e-4 for MegaDepth (from SuperGlue) +_CN.TRAINER.POSE_GEO_MODEL = 'E' # ['E', 'F', 'H'] +_CN.TRAINER.POSE_ESTIMATION_METHOD = 'RANSAC' # [RANSAC, DEGENSAC, MAGSAC] +_CN.TRAINER.RANSAC_PIXEL_THR = 0.5 +_CN.TRAINER.RANSAC_CONF = 0.99999 +_CN.TRAINER.RANSAC_MAX_ITERS = 10000 +_CN.TRAINER.USE_MAGSACPP = False + +# data sampler for train_dataloader +_CN.TRAINER.DATA_SAMPLER = 'scene_balance' # options: ['scene_balance', 'random', 'normal'] +# 'scene_balance' config +_CN.TRAINER.N_SAMPLES_PER_SUBSET = 200 +_CN.TRAINER.SB_SUBSET_SAMPLE_REPLACEMENT = True # whether sample each scene with replacement or not +_CN.TRAINER.SB_SUBSET_SHUFFLE = True # after sampling from scenes, whether shuffle within the epoch or not +_CN.TRAINER.SB_REPEAT = 1 # repeat N times for training the sampled data +# 'random' config +_CN.TRAINER.RDM_REPLACEMENT = True +_CN.TRAINER.RDM_NUM_SAMPLES = None + +# gradient clipping +_CN.TRAINER.GRADIENT_CLIPPING = 0.5 + +# reproducibility +# This seed affects the data sampling. With the same seed, the data sampling is promised +# to be the same. When resume training from a checkpoint, it's better to use a different +# seed, otherwise the sampled data will be exactly the same as before resuming, which will +# cause less unique data items sampled during the entire training. +# Use of different seed values might affect the final training result, since not all data items +# are used during training on ScanNet. (60M pairs of images sampled during traing from 230M pairs in total.) +_CN.TRAINER.SEED = 66 + + +def get_cfg_defaults(): + """Get a yacs CfgNode object with default values for my_project.""" + # Return a clone so that the defaults will not be altered + # This is for the "local variable" use pattern + return _CN.clone() diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1841a5c6a23f1ac409ef6b941fe7fe19359cfbce --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/__init__.py @@ -0,0 +1,5 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +from .loftr import LoFTR diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..48343c7a7537aa5df87974d1b2e257a700521ae8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/__init__.py @@ -0,0 +1,16 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +from .resnet_fpn import ResNetFPN_8_2, ResNetFPN_16_4 + + +def build_backbone(config): + if config['backbone_type'] == 'ResNetFPN': + if config['resolution'] == (8, 2): + return ResNetFPN_8_2(config['resnetfpn']) + elif config['resolution'] == (16, 4): + return ResNetFPN_16_4(config['resnetfpn']) + else: + raise ValueError( + f"LOFTR.BACKBONE_TYPE {config['backbone_type']} not supported.") diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/resnet_fpn.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/resnet_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..b635559acac739731d24e3dfb1161c9e12902755 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/backbone/resnet_fpn.py @@ -0,0 +1,223 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import torch.nn as nn +import torch.nn.functional as F + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution without padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=stride, + padding=0, + bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlock(nn.Module): + + def __init__(self, in_planes, planes, stride=1): + super().__init__() + self.conv1 = conv3x3(in_planes, planes, stride) + self.conv2 = conv3x3(planes, planes) + self.bn1 = nn.BatchNorm2d(planes) + self.bn2 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + + if stride == 1: + self.downsample = None + else: + self.downsample = nn.Sequential( + conv1x1(in_planes, planes, stride=stride), + nn.BatchNorm2d(planes)) + + def forward(self, x): + y = x + y = self.relu(self.bn1(self.conv1(y))) + y = self.bn2(self.conv2(y)) + + if self.downsample is not None: + x = self.downsample(x) + + return self.relu(x + y) + + +class ResNetFPN_8_2(nn.Module): + """ + ResNet+FPN, output resolution are 1/8 and 1/2. + Each block has 2 layers. + """ + + def __init__(self, config): + super().__init__() + # Config + block = BasicBlock + initial_dim = config['initial_dim'] + block_dims = config['block_dims'] + + # Class Variable + self.in_planes = initial_dim + + # Networks + self.conv1 = nn.Conv2d( + 1, initial_dim, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(initial_dim) + self.relu = nn.ReLU(inplace=True) + + self.layer1 = self._make_layer(block, block_dims[0], stride=1) # 1/2 + self.layer2 = self._make_layer(block, block_dims[1], stride=2) # 1/4 + self.layer3 = self._make_layer(block, block_dims[2], stride=2) # 1/8 + + # 3. FPN upsample + self.layer3_outconv = conv1x1(block_dims[2], block_dims[2]) + self.layer2_outconv = conv1x1(block_dims[1], block_dims[2]) + self.layer2_outconv2 = nn.Sequential( + conv3x3(block_dims[2], block_dims[2]), + nn.BatchNorm2d(block_dims[2]), + nn.LeakyReLU(), + conv3x3(block_dims[2], block_dims[1]), + ) + self.layer1_outconv = conv1x1(block_dims[0], block_dims[1]) + self.layer1_outconv2 = nn.Sequential( + conv3x3(block_dims[1], block_dims[1]), + nn.BatchNorm2d(block_dims[1]), + nn.LeakyReLU(), + conv3x3(block_dims[1], block_dims[0]), + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, dim, stride=1): + layer1 = block(self.in_planes, dim, stride=stride) + layer2 = block(dim, dim, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x): + # ResNet Backbone + x0 = self.relu(self.bn1(self.conv1(x))) + x1 = self.layer1(x0) # 1/2 + x2 = self.layer2(x1) # 1/4 + x3 = self.layer3(x2) # 1/8 + + # FPN + x3_out = self.layer3_outconv(x3) + + x3_out_2x = F.interpolate( + x3_out, scale_factor=2., mode='bilinear', align_corners=True) + x2_out = self.layer2_outconv(x2) + x2_out = self.layer2_outconv2(x2_out + x3_out_2x) + + x2_out_2x = F.interpolate( + x2_out, scale_factor=2., mode='bilinear', align_corners=True) + x1_out = self.layer1_outconv(x1) + x1_out = self.layer1_outconv2(x1_out + x2_out_2x) + + return [x3_out, x1_out] + + +class ResNetFPN_16_4(nn.Module): + """ + ResNet+FPN, output resolution are 1/16 and 1/4. + Each block has 2 layers. + """ + + def __init__(self, config): + super().__init__() + # Config + block = BasicBlock + initial_dim = config['initial_dim'] + block_dims = config['block_dims'] + + # Class Variable + self.in_planes = initial_dim + + # Networks + self.conv1 = nn.Conv2d( + 1, initial_dim, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(initial_dim) + self.relu = nn.ReLU(inplace=True) + + self.layer1 = self._make_layer(block, block_dims[0], stride=1) # 1/2 + self.layer2 = self._make_layer(block, block_dims[1], stride=2) # 1/4 + self.layer3 = self._make_layer(block, block_dims[2], stride=2) # 1/8 + self.layer4 = self._make_layer(block, block_dims[3], stride=2) # 1/16 + + # 3. FPN upsample + self.layer4_outconv = conv1x1(block_dims[3], block_dims[3]) + self.layer3_outconv = conv1x1(block_dims[2], block_dims[3]) + self.layer3_outconv2 = nn.Sequential( + conv3x3(block_dims[3], block_dims[3]), + nn.BatchNorm2d(block_dims[3]), + nn.LeakyReLU(), + conv3x3(block_dims[3], block_dims[2]), + ) + + self.layer2_outconv = conv1x1(block_dims[1], block_dims[2]) + self.layer2_outconv2 = nn.Sequential( + conv3x3(block_dims[2], block_dims[2]), + nn.BatchNorm2d(block_dims[2]), + nn.LeakyReLU(), + conv3x3(block_dims[2], block_dims[1]), + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, dim, stride=1): + layer1 = block(self.in_planes, dim, stride=stride) + layer2 = block(dim, dim, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x): + # ResNet Backbone + x0 = self.relu(self.bn1(self.conv1(x))) + x1 = self.layer1(x0) # 1/2 + x2 = self.layer2(x1) # 1/4 + x3 = self.layer3(x2) # 1/8 + x4 = self.layer4(x3) # 1/16 + + # FPN + x4_out = self.layer4_outconv(x4) + + x4_out_2x = F.interpolate( + x4_out, scale_factor=2., mode='bilinear', align_corners=True) + x3_out = self.layer3_outconv(x3) + x3_out = self.layer3_outconv2(x3_out + x4_out_2x) + + x3_out_2x = F.interpolate( + x3_out, scale_factor=2., mode='bilinear', align_corners=True) + x2_out = self.layer2_outconv(x2) + x2_out = self.layer2_outconv2(x2_out + x3_out_2x) + + return [x4_out, x2_out] diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr.py new file mode 100644 index 0000000000000000000000000000000000000000..16dd6521472ec489a3d9e252e00664d7caf908dc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr.py @@ -0,0 +1,98 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import torch +import torch.nn as nn +from einops.einops import rearrange + +from .backbone import build_backbone +from .loftr_module import FinePreprocess, LocalFeatureTransformer +from .utils.coarse_matching import CoarseMatching +from .utils.fine_matching import FineMatching +from .utils.position_encoding import PositionEncodingSine + + +class LoFTR(nn.Module): + + def __init__(self, config): + super().__init__() + # Misc + self.config = config + + # Modules + self.backbone = build_backbone(config) + self.pos_encoding = PositionEncodingSine( + config['coarse']['d_model'], + temp_bug_fix=config['coarse']['temp_bug_fix']) + self.loftr_coarse = LocalFeatureTransformer(config['coarse']) + self.coarse_matching = CoarseMatching(config['match_coarse']) + self.fine_preprocess = FinePreprocess(config) + self.loftr_fine = LocalFeatureTransformer(config['fine']) + self.fine_matching = FineMatching() + + def forward(self, data): + """ + Update: + data (dict): { + 'image0': (torch.Tensor): (N, 1, H, W) + 'image1': (torch.Tensor): (N, 1, H, W) + 'mask0'(optional) : (torch.Tensor): (N, H, W) '0' indicates a padded position + 'mask1'(optional) : (torch.Tensor): (N, H, W) + } + """ + # 1. Local Feature CNN + data.update({ + 'bs': data['image0'].size(0), + 'hw0_i': data['image0'].shape[2:], + 'hw1_i': data['image1'].shape[2:] + }) + + if data['hw0_i'] == data['hw1_i']: # faster & better BN convergence + feats_c, feats_f = self.backbone( + torch.cat([data['image0'], data['image1']], dim=0)) + (feat_c0, feat_c1), (feat_f0, feat_f1) = feats_c.split( + data['bs']), feats_f.split(data['bs']) + else: # handle different input shapes + (feat_c0, feat_f0), (feat_c1, feat_f1) = self.backbone( + data['image0']), self.backbone(data['image1']) + + data.update({ + 'hw0_c': feat_c0.shape[2:], + 'hw1_c': feat_c1.shape[2:], + 'hw0_f': feat_f0.shape[2:], + 'hw1_f': feat_f1.shape[2:] + }) + + # 2. coarse-level loftr module + # add featmap with positional encoding, then flatten it to sequence [N, HW, C] + + feat_c0 = self.pos_encoding(feat_c0) + feat_c1 = self.pos_encoding(feat_c1) + + mask_c0 = mask_c1 = None # mask is useful in training + if 'mask0' in data: + mask_c0, mask_c1 = data['mask0'].flatten( + -2), data['mask1'].flatten(-2) + feat_c0, feat_c1 = self.loftr_coarse(feat_c0, feat_c1, mask_c0, + mask_c1) + + # 3. match coarse-level + self.coarse_matching( + feat_c0, feat_c1, data, mask_c0=mask_c0, mask_c1=mask_c1) + + # 4. fine-level refinement + feat_f0_unfold, feat_f1_unfold = self.fine_preprocess( + feat_f0, feat_f1, feat_c0, feat_c1, data) + if feat_f0_unfold.size(0) != 0: # at least one coarse level predicted + feat_f0_unfold, feat_f1_unfold = self.loftr_fine( + feat_f0_unfold, feat_f1_unfold) + + # 5. match fine-level + self.fine_matching(feat_f0_unfold, feat_f1_unfold, data) + + def load_state_dict(self, state_dict, *args, **kwargs): + for k in list(state_dict.keys()): + if k.startswith('matcher.'): + state_dict[k.replace('matcher.', '', 1)] = state_dict.pop(k) + return super().load_state_dict(state_dict, *args, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d766a460c8d584bc9696fe52f9f9fd91ec8b9bb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/__init__.py @@ -0,0 +1,6 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +from .fine_preprocess import FinePreprocess +from .transformer import LocalFeatureTransformer diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/fine_preprocess.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/fine_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..1b1c159de0488a967a87254e17d106572e554b1f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/fine_preprocess.py @@ -0,0 +1,77 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops.einops import rearrange, repeat + + +class FinePreprocess(nn.Module): + + def __init__(self, config): + super().__init__() + + self.config = config + self.cat_c_feat = config['fine_concat_coarse_feat'] + self.W = self.config['fine_window_size'] + + d_model_c = self.config['coarse']['d_model'] + d_model_f = self.config['fine']['d_model'] + self.d_model_f = d_model_f + if self.cat_c_feat: + self.down_proj = nn.Linear(d_model_c, d_model_f, bias=True) + self.merge_feat = nn.Linear(2 * d_model_f, d_model_f, bias=True) + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.kaiming_normal_(p, mode='fan_out', nonlinearity='relu') + + def forward(self, feat_f0, feat_f1, feat_c0, feat_c1, data): + W = self.W + stride = data['hw0_f'][0] // data['hw0_c'][0] + + data.update({'W': W}) + if data['b_ids'].shape[0] == 0: + feat0 = torch.empty( + 0, self.W**2, self.d_model_f, device=feat_f0.device) + feat1 = torch.empty( + 0, self.W**2, self.d_model_f, device=feat_f0.device) + return feat0, feat1 + + # 1. unfold(crop) all local windows + feat_f0_unfold = F.unfold( + feat_f0, kernel_size=(W, W), stride=stride, padding=W // 2) + feat_f0_unfold = rearrange( + feat_f0_unfold, 'n (c ww) l -> n l ww c', ww=W**2) + feat_f1_unfold = F.unfold( + feat_f1, kernel_size=(W, W), stride=stride, padding=W // 2) + feat_f1_unfold = rearrange( + feat_f1_unfold, 'n (c ww) l -> n l ww c', ww=W**2) + + # 2. select only the predicted matches + feat_f0_unfold = feat_f0_unfold[data['b_ids'], + data['i_ids']] # [n, ww, cf] + feat_f1_unfold = feat_f1_unfold[data['b_ids'], data['j_ids']] + + # option: use coarse-level loftr feature as context: concat and linear + if self.cat_c_feat: + feat_c_win = self.down_proj( + torch.cat([ + feat_c0[data['b_ids'], data['i_ids']], + feat_c1[data['b_ids'], data['j_ids']] + ], 0)) # [2n, c] + feat_cf_win = self.merge_feat( + torch.cat( + [ + torch.cat([feat_f0_unfold, feat_f1_unfold], + 0), # [2n, ww, cf] + repeat(feat_c_win, 'n c -> n ww c', ww=W**2), # noqa + ], + -1)) # noqa + feat_f0_unfold, feat_f1_unfold = torch.chunk(feat_cf_win, 2, dim=0) + + return feat_f0_unfold, feat_f1_unfold diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/linear_attention.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/linear_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..76d58764bfdd6eab12a414df498a0a67692da117 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/linear_attention.py @@ -0,0 +1,89 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR +""" +Linear Transformer proposed in "Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention" +Modified from: https://github.com/idiap/fast-transformers/blob/master/fast_transformers/attention/linear_attention.py +""" + +import torch +from torch.nn import Dropout, Module + + +def elu_feature_map(x): + return torch.nn.functional.elu(x) + 1 + + +class LinearAttention(Module): + + def __init__(self, eps=1e-6): + super().__init__() + self.feature_map = elu_feature_map + self.eps = eps + + def forward(self, queries, keys, values, q_mask=None, kv_mask=None): + """ Multi-Head linear attention proposed in "Transformers are RNNs" + Args: + queries: [N, L, H, D] + keys: [N, S, H, D] + values: [N, S, H, D] + q_mask: [N, L] + kv_mask: [N, S] + Returns: + queried_values: (N, L, H, D) + """ + Q = self.feature_map(queries) + K = self.feature_map(keys) + + # set padded position to zero + if q_mask is not None: + Q = Q * q_mask[:, :, None, None] + if kv_mask is not None: + K = K * kv_mask[:, :, None, None] + values = values * kv_mask[:, :, None, None] + + v_length = values.size(1) + values = values / v_length # prevent fp16 overflow + KV = torch.einsum('nshd,nshv->nhdv', K, values) # (S,D)' @ S,V + Z = 1 / (torch.einsum('nlhd,nhd->nlh', Q, K.sum(dim=1)) + self.eps) + queried_values = torch.einsum('nlhd,nhdv,nlh->nlhv', Q, KV, + Z) * v_length + + return queried_values.contiguous() + + +class FullAttention(Module): + + def __init__(self, use_dropout=False, attention_dropout=0.1): + super().__init__() + self.use_dropout = use_dropout + self.dropout = Dropout(attention_dropout) + + def forward(self, queries, keys, values, q_mask=None, kv_mask=None): + """ Multi-head scaled dot-product attention, a.k.a full attention. + Args: + queries: [N, L, H, D] + keys: [N, S, H, D] + values: [N, S, H, D] + q_mask: [N, L] + kv_mask: [N, S] + Returns: + queried_values: (N, L, H, D) + """ + + # Compute the unnormalized attention and apply the masks + QK = torch.einsum('nlhd,nshd->nlsh', queries, keys) + if kv_mask is not None: + QK.masked_fill_( + ~(q_mask[:, :, None, None] * kv_mask[:, None, :, None]), + float('-inf')) + + # Compute the attention and the weighted average + softmax_temp = 1. / queries.size(3)**.5 # sqrt(D) + A = torch.softmax(softmax_temp * QK, dim=2) + if self.use_dropout: + A = self.dropout(A) + + queried_values = torch.einsum('nlsh,nshd->nlhd', A, values) + + return queried_values.contiguous() diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/quadtree_attention.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/quadtree_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a406152befd8a098760f5a44b1f247e6084488 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/quadtree_attention.py @@ -0,0 +1,98 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import trunc_normal_ + +from modelscope.ops.quadtree_attention import QTAttA, QTAttB + + +class QuadtreeAttention(nn.Module): + + def __init__( + self, + dim, + num_heads, + topks, + value_branch=False, + act=nn.GELU(), + qkv_bias=False, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + scale=1, + attn_type='B', + ): + super().__init__() + assert dim % num_heads == 0, f'dim {dim} should be divided by num_heads {num_heads}.' + + self.dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.q_proj = nn.Conv2d( + dim, dim, kernel_size=1, stride=1, bias=qkv_bias) + self.k_proj = nn.Conv2d( + dim, dim, kernel_size=1, stride=1, bias=qkv_bias) + self.v_proj = nn.Conv2d( + dim, dim, kernel_size=1, stride=1, bias=qkv_bias) + if attn_type == 'A': + self.py_att = QTAttA( + num_heads, dim // num_heads, scale=scale, topks=topks) + else: + self.py_att = QTAttB( + num_heads, dim // num_heads, scale=scale, topks=topks) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.scale = scale + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + trunc_normal_(m.weight, std=0.02) + m.init = True + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, target, H, W, msg=None): + + B, N, C = x.shape + x = x.permute(0, 2, 1).reshape(B, C, H, W) + target = target.permute(0, 2, 1).reshape(B, C, H, W) + keys = [] + values = [] + queries = [] + + q = self.q_proj(x) + k = self.k_proj(target) + v = self.v_proj(target) + for i in range(self.scale): + keys.append(k) + values.append(v) + queries.append(q) + + if i != self.scale - 1: + k = F.avg_pool2d(k, kernel_size=2, stride=2) + q = F.avg_pool2d(q, kernel_size=2, stride=2) + v = F.avg_pool2d(v, kernel_size=2, stride=2) + + msg = self.py_att(queries, keys, values).view(B, -1, C) + + x = self.proj(msg) + x = self.proj_drop(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/transformer.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..69a6f7da7b2c2720347c1ec04a45921f71765e54 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/loftr_module/transformer.py @@ -0,0 +1,287 @@ +# Part of the implementation is borrowed and modified from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import copy +import math + +import torch +import torch.nn as nn +from einops.einops import rearrange +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from .linear_attention import FullAttention, LinearAttention +from .quadtree_attention import QuadtreeAttention + + +class DWConv(nn.Module): + + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x, H, W): + B, N, C = x.shape + x = x.transpose(1, 2).view(B, C, H, W) + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + + return x + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + self.relu = nn.ReLU(inplace=True) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = self.fc1(x) + x = self.relu(x) + + x = self.dwconv(x, H, W) + + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + + return x + + +class LoFTREncoderLayer(nn.Module): + + def __init__(self, d_model, nhead, attention='linear'): + super(LoFTREncoderLayer, self).__init__() + + self.dim = d_model // nhead + self.nhead = nhead + + # multi-head attention + self.q_proj = nn.Linear(d_model, d_model, bias=False) + self.k_proj = nn.Linear(d_model, d_model, bias=False) + self.v_proj = nn.Linear(d_model, d_model, bias=False) + self.attention = LinearAttention( + ) if attention == 'linear' else FullAttention() + self.merge = nn.Linear(d_model, d_model, bias=False) + + # feed-forward network + self.mlp = nn.Sequential( + nn.Linear(d_model * 2, d_model * 2, bias=False), + nn.ReLU(True), + nn.Linear(d_model * 2, d_model, bias=False), + ) + + # norm and dropout + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + + def forward(self, x, source, x_mask=None, source_mask=None): + """ + Args: + x (torch.Tensor): [N, L, C] + source (torch.Tensor): [N, S, C] + x_mask (torch.Tensor): [N, L] (optional) + source_mask (torch.Tensor): [N, S] (optional) + """ + bs = x.size(0) + query, key, value = x, source, source + + # multi-head attention + query = self.q_proj(query).view(bs, -1, self.nhead, + self.dim) # [N, L, (H, D)] + key = self.k_proj(key).view(bs, -1, self.nhead, + self.dim) # [N, S, (H, D)] + value = self.v_proj(value).view(bs, -1, self.nhead, self.dim) + message = self.attention( + query, key, value, q_mask=x_mask, + kv_mask=source_mask) # [N, L, (H, D)] + message = self.merge(message.view(bs, -1, + self.nhead * self.dim)) # [N, L, C] + message = self.norm1(message) + + # feed-forward network + message = self.mlp(torch.cat([x, message], dim=2)) + message = self.norm2(message) + + return x + message + + +class QuadtreeBlock(nn.Module): + + def __init__(self, + dim, + num_heads, + topks, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + scale=1, + attn_type='B'): + + super().__init__() + + self.norm1 = norm_layer(dim) + + self.attn = QuadtreeAttention( + dim, + num_heads=num_heads, + topks=topks, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + scale=scale, + attn_type=attn_type) + + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + + self.norm2 = norm_layer(dim) + + mlp_hidden_dim = int(dim * mlp_ratio) + + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.apply(self._init_weights) + + def _init_weights(self, m): + if hasattr(m, 'init'): + return + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, target, H, W): + + x = x + self.drop_path( + self.attn(self.norm1(x), self.norm1(target), H, W)) + x = x + self.drop_path(self.mlp(self.norm2(x), H, W)) + + return x + + +class LocalFeatureTransformer(nn.Module): + """A Local Feature Transformer (LoFTR) module.""" + + def __init__(self, config): + super(LocalFeatureTransformer, self).__init__() + self.block_type = config['block_type'] + self.config = config + self.d_model = config['d_model'] + self.nhead = config['nhead'] + self.layer_names = config['layer_names'] + + if config['block_type'] == 'loftr': + encoder_layer = LoFTREncoderLayer(config['d_model'], + config['nhead'], + config['attention']) + self.layers = nn.ModuleList([ + copy.deepcopy(encoder_layer) + for _ in range(len(self.layer_names)) + ]) + self._reset_parameters() + elif config['block_type'] == 'quadtree': + encoder_layer = QuadtreeBlock( + config['d_model'], + config['nhead'], + attn_type=config['attn_type'], + topks=config['topks'], + scale=3) + self.layers = nn.ModuleList([ + copy.deepcopy(encoder_layer) + for _ in range(len(self.layer_names)) + ]) + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, feat0, feat1, mask0=None, mask1=None): + """ + Args: + feat0 (torch.Tensor): [N, L, C] + feat1 (torch.Tensor): [N, S, C] + mask0 (torch.Tensor): [N, L] (optional) + mask1 (torch.Tensor): [N, S] (optional) + """ + + if len(feat0.shape) == 4: + B, C, H, W = feat0.shape + feat0 = rearrange(feat0, 'b c h w -> b (h w) c') + feat1 = rearrange(feat1, 'b c h w -> b (h w) c') + + if self.block_type == 'loftr': + for layer, name in zip(self.layers, self.layer_names): + if name == 'self': + feat0 = layer(feat0, feat0, mask0, mask0) + feat1 = layer(feat1, feat1, mask1, mask1) + elif name == 'cross': + feat0 = layer(feat0, feat1, mask0, mask1) + feat1 = layer(feat1, feat0, mask1, mask0) + else: + raise KeyError + else: + for layer, name in zip(self.layers, self.layer_names): + if name == 'self': + feat0 = layer(feat0, feat0, H, W) + feat1 = layer(feat1, feat1, H, W) + elif name == 'cross': + if self.config['block_type'] == 'quadtree': + feat0, feat1 = layer(feat0, feat1, H, + W), layer(feat1, feat0, H, W) + else: + feat0 = layer(feat0, feat1, H, W) + feat1 = layer(feat1, feat0, H, W) + else: + raise KeyError + + return feat0, feat1 diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/coarse_matching.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/coarse_matching.py new file mode 100644 index 0000000000000000000000000000000000000000..d31686b2ba68886da6c701e1432c00912a1de470 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/coarse_matching.py @@ -0,0 +1,268 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops.einops import rearrange + +INF = 1e9 + + +def mask_border(m, b: int, v): + """ Mask borders with value + Args: + m (torch.Tensor): [N, H0, W0, H1, W1] + b (int) + v (m.dtype) + """ + if b <= 0: + return + + m[:, :b] = v + m[:, :, :b] = v + m[:, :, :, :b] = v + m[:, :, :, :, :b] = v + m[:, -b:] = v + m[:, :, -b:] = v + m[:, :, :, -b:] = v + m[:, :, :, :, -b:] = v + + +def mask_border_with_padding(m, bd, v, p_m0, p_m1): + if bd <= 0: + return + + m[:, :bd] = v + m[:, :, :bd] = v + m[:, :, :, :bd] = v + m[:, :, :, :, :bd] = v + + h0s, w0s = p_m0.sum(1).max(-1)[0].int(), p_m0.sum(-1).max(-1)[0].int() + h1s, w1s = p_m1.sum(1).max(-1)[0].int(), p_m1.sum(-1).max(-1)[0].int() + for b_idx, (h0, w0, h1, w1) in enumerate(zip(h0s, w0s, h1s, w1s)): + m[b_idx, h0 - bd:] = v + m[b_idx, :, w0 - bd:] = v + m[b_idx, :, :, h1 - bd:] = v + m[b_idx, :, :, :, w1 - bd:] = v + + +def compute_max_candidates(p_m0, p_m1): + """Compute the max candidates of all pairs within a batch + + Args: + p_m0, p_m1 (torch.Tensor): padded masks + """ + h0s, w0s = p_m0.sum(1).max(-1)[0], p_m0.sum(-1).max(-1)[0] + h1s, w1s = p_m1.sum(1).max(-1)[0], p_m1.sum(-1).max(-1)[0] + max_cand = torch.sum( + torch.min(torch.stack([h0s * w0s, h1s * w1s], -1), -1)[0]) + return max_cand + + +class CoarseMatching(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + # general config + self.thr = config['thr'] + self.border_rm = config['border_rm'] + # -- # for trainig fine-level LoFTR + self.train_coarse_percent = config['train_coarse_percent'] + self.train_pad_num_gt_min = config['train_pad_num_gt_min'] + + # we provide 2 options for differentiable matching + self.match_type = config['match_type'] + if self.match_type == 'dual_softmax': + self.temperature = config['dsmax_temperature'] + elif self.match_type == 'sinkhorn': + try: + from .superglue import log_optimal_transport + except ImportError: + raise ImportError('download superglue.py first!') + self.log_optimal_transport = log_optimal_transport + self.bin_score = nn.Parameter( + torch.tensor(config['skh_init_bin_score'], requires_grad=True)) + self.skh_iters = config['skh_iters'] + self.skh_prefilter = config['skh_prefilter'] + else: + raise NotImplementedError() + + def forward(self, feat_c0, feat_c1, data, mask_c0=None, mask_c1=None): + """ + Args: + feat0 (torch.Tensor): [N, L, C] + feat1 (torch.Tensor): [N, S, C] + data (dict) + mask_c0 (torch.Tensor): [N, L] (optional) + mask_c1 (torch.Tensor): [N, S] (optional) + Update: + data (dict): { + 'b_ids' (torch.Tensor): [M'], + 'i_ids' (torch.Tensor): [M'], + 'j_ids' (torch.Tensor): [M'], + 'gt_mask' (torch.Tensor): [M'], + 'mkpts0_c' (torch.Tensor): [M, 2], + 'mkpts1_c' (torch.Tensor): [M, 2], + 'mconf' (torch.Tensor): [M]} + NOTE: M' != M during training. + """ + _, L, S, _ = feat_c0.size(0), feat_c0.size(1), feat_c1.size( + 1), feat_c0.size(2) + + # normalize + feat_c0, feat_c1 = map(lambda feat: feat / feat.shape[-1]**.5, + [feat_c0, feat_c1]) + + if self.match_type == 'dual_softmax': + sim_matrix = torch.einsum('nlc,nsc->nls', feat_c0, + feat_c1) / self.temperature + if mask_c0 is not None: + sim_matrix.masked_fill_( + ~(mask_c0[..., None] * mask_c1[:, None]).bool(), -INF) + conf_matrix = F.softmax(sim_matrix, 1) * F.softmax(sim_matrix, 2) + + elif self.match_type == 'sinkhorn': + # sinkhorn, dustbin included + sim_matrix = torch.einsum('nlc,nsc->nls', feat_c0, feat_c1) + if mask_c0 is not None: + sim_matrix[:, :L, :S].masked_fill_( + ~(mask_c0[..., None] * mask_c1[:, None]).bool(), -INF) + + # build uniform prior & use sinkhorn + log_assign_matrix = self.log_optimal_transport( + sim_matrix, self.bin_score, self.skh_iters) + assign_matrix = log_assign_matrix.exp() + conf_matrix = assign_matrix[:, :-1, :-1] + + # filter prediction with dustbin score (only in evaluation mode) + if not self.training and self.skh_prefilter: + filter0 = (assign_matrix.max(dim=2)[1] == S)[:, :-1] # [N, L] + filter1 = (assign_matrix.max(dim=1)[1] == L)[:, :-1] # [N, S] + conf_matrix[filter0[..., None].repeat(1, 1, S)] = 0 + conf_matrix[filter1[:, None].repeat(1, L, 1)] = 0 + + if self.config['sparse_spvs']: + data.update({'conf_matrix_with_bin': assign_matrix.clone()}) + + data.update({'conf_matrix': conf_matrix}) + + # predict coarse matches from conf_matrix + data.update(**self.get_coarse_match(conf_matrix, data)) + + @torch.no_grad() + def get_coarse_match(self, conf_matrix, data): + """ + Args: + conf_matrix (torch.Tensor): [N, L, S] + data (dict): with keys ['hw0_i', 'hw1_i', 'hw0_c', 'hw1_c'] + Returns: + coarse_matches (dict): { + 'b_ids' (torch.Tensor): [M'], + 'i_ids' (torch.Tensor): [M'], + 'j_ids' (torch.Tensor): [M'], + 'gt_mask' (torch.Tensor): [M'], + 'm_bids' (torch.Tensor): [M], + 'mkpts0_c' (torch.Tensor): [M, 2], + 'mkpts1_c' (torch.Tensor): [M, 2], + 'mconf' (torch.Tensor): [M]} + """ + axes_lengths = { + 'h0c': data['hw0_c'][0], + 'w0c': data['hw0_c'][1], + 'h1c': data['hw1_c'][0], + 'w1c': data['hw1_c'][1] + } + _device = conf_matrix.device + # 1. confidence thresholding + mask = conf_matrix > self.thr + mask = rearrange(mask, 'b (h0c w0c) (h1c w1c) -> b h0c w0c h1c w1c', + **axes_lengths) + if 'mask0' not in data: + mask_border(mask, self.border_rm, False) + else: + mask_border_with_padding(mask, self.border_rm, False, + data['mask0'], data['mask1']) + mask = rearrange(mask, 'b h0c w0c h1c w1c -> b (h0c w0c) (h1c w1c)', + **axes_lengths) + + # 2. mutual nearest + mask = mask \ + * (conf_matrix == conf_matrix.max(dim=2, keepdim=True)[0]) \ + * (conf_matrix == conf_matrix.max(dim=1, keepdim=True)[0]) + + # 3. find all valid coarse matches + # this only works when at most one `True` in each row + mask_v, all_j_ids = mask.max(dim=2) + b_ids, i_ids = torch.where(mask_v) + j_ids = all_j_ids[b_ids, i_ids] + mconf = conf_matrix[b_ids, i_ids, j_ids] + + # 4. Random sampling of training samples for fine-level LoFTR + # (optional) pad samples with gt coarse-level matches + if self.training: + # NOTE: + # The sampling is performed across all pairs in a batch without manually balancing + # #samples for fine-level increases w.r.t. batch_size + if 'mask0' not in data: + num_candidates_max = mask.size(0) * max( + mask.size(1), mask.size(2)) + else: + num_candidates_max = compute_max_candidates( + data['mask0'], data['mask1']) + num_matches_train = int(num_candidates_max + * self.train_coarse_percent) + num_matches_pred = len(b_ids) + assert self.train_pad_num_gt_min < num_matches_train, 'min-num-gt-pad should be less than num-train-matches' + + # pred_indices is to select from prediction + if num_matches_pred <= num_matches_train - self.train_pad_num_gt_min: + pred_indices = torch.arange(num_matches_pred, device=_device) + else: + pred_indices = torch.randint( + num_matches_pred, + (num_matches_train - self.train_pad_num_gt_min, ), + device=_device) + + # gt_pad_indices is to select from gt padding. e.g. max(3787-4800, 200) + gt_pad_indices = torch.randint( + len(data['spv_b_ids']), + (max(num_matches_train - num_matches_pred, + self.train_pad_num_gt_min), ), + device=_device) + mconf_gt = torch.zeros( + len(data['spv_b_ids']), + device=_device) # set conf of gt paddings to all zero + + b_ids, i_ids, j_ids, mconf = map( + lambda x, y: torch.cat([x[pred_indices], y[gt_pad_indices]], + dim=0), + *zip([b_ids, data['spv_b_ids']], [i_ids, data['spv_i_ids']], + [j_ids, data['spv_j_ids']], [mconf, mconf_gt])) + + # These matches select patches that feed into fine-level network + coarse_matches = {'b_ids': b_ids, 'i_ids': i_ids, 'j_ids': j_ids} + + # 4. Update with matches in original image resolution + scale = data['hw0_i'][0] / data['hw0_c'][0] + scale0 = scale * data['scale0'][b_ids] if 'scale0' in data else scale + scale1 = scale * data['scale1'][b_ids] if 'scale1' in data else scale + mkpts0_c = torch.stack( + [i_ids % data['hw0_c'][1], i_ids // data['hw0_c'][1]], + dim=1) * scale0 + mkpts1_c = torch.stack( + [j_ids % data['hw1_c'][1], j_ids // data['hw1_c'][1]], + dim=1) * scale1 + + # These matches is the current prediction (for visualization) + coarse_matches.update({ + 'gt_mask': mconf == 0, + 'm_bids': b_ids[mconf != 0], # mconf == 0 => gt matches + 'mkpts0_c': mkpts0_c[mconf != 0], + 'mkpts1_c': mkpts1_c[mconf != 0], + 'mconf': mconf[mconf != 0] + }) + + return coarse_matches diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/fine_matching.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/fine_matching.py new file mode 100644 index 0000000000000000000000000000000000000000..d0340eb8bb63d6700527321c4696458132a0bc44 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/fine_matching.py @@ -0,0 +1,86 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import math + +import torch +import torch.nn as nn +from kornia.geometry.subpix import dsnt +from kornia.utils.grid import create_meshgrid + + +class FineMatching(nn.Module): + """FineMatching with s2d paradigm""" + + def __init__(self): + super().__init__() + + def forward(self, feat_f0, feat_f1, data): + """ + Args: + feat0 (torch.Tensor): [M, WW, C] + feat1 (torch.Tensor): [M, WW, C] + data (dict) + Update: + data (dict):{ + 'expec_f' (torch.Tensor): [M, 3], + 'mkpts0_f' (torch.Tensor): [M, 2], + 'mkpts1_f' (torch.Tensor): [M, 2]} + """ + M, WW, C = feat_f0.shape + W = int(math.sqrt(WW)) + scale = data['hw0_i'][0] / data['hw0_f'][0] + self.M, self.W, self.WW, self.C, self.scale = M, W, WW, C, scale + + # corner case: if no coarse matches found + if M == 0: + assert self.training is False, 'M is always >0, when training, see coarse_matching.py' + # logger.warning('No matches found in coarse-level.') + data.update({ + 'expec_f': torch.empty(0, 3, device=feat_f0.device), + 'mkpts0_f': data['mkpts0_c'], + 'mkpts1_f': data['mkpts1_c'], + }) + return + + feat_f0_picked = feat_f0_picked = feat_f0[:, WW // 2, :] + sim_matrix = torch.einsum('mc,mrc->mr', feat_f0_picked, feat_f1) + softmax_temp = 1. / C**.5 + heatmap = torch.softmax( + softmax_temp * sim_matrix, dim=1).view(-1, W, W) + + # compute coordinates from heatmap + coords_normalized = dsnt.spatial_expectation2d(heatmap[None], + True)[0] # [M, 2] + grid_normalized = create_meshgrid(W, W, True, heatmap.device).reshape( + 1, -1, 2) # [1, WW, 2] + + # compute std over + var = torch.sum( + grid_normalized**2 * heatmap.view(-1, WW, 1), + dim=1) - coords_normalized**2 # [M, 2] + std = torch.sum(torch.sqrt(torch.clamp(var, min=1e-10)), + -1) # [M] clamp needed for numerical stability + + # for fine-level supervision + data.update( + {'expec_f': + torch.cat([coords_normalized, std.unsqueeze(1)], -1)}) + + # compute absolute kpt coords + self.get_fine_match(coords_normalized, data) + + @torch.no_grad() + def get_fine_match(self, coords_normed, data): + W, scale = self.W, self.scale + + # mkpts0_f and mkpts1_f + mkpts0_f = data['mkpts0_c'] + scale1 = scale * data['scale1'][ + data['b_ids']] if 'scale0' in data else scale + mkpts1_f = data['mkpts1_c'] + ( + coords_normed * # noqa + (W // 2) * scale1)[:len(data['mconf'])] # noqa + + data.update({'mkpts0_f': mkpts0_f, 'mkpts1_f': mkpts1_f}) diff --git a/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/position_encoding.py b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..4f3cfe971dfee61bb20bba7b6fa5a8aac62e43a5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/loftr_quadtree/utils/position_encoding.py @@ -0,0 +1,52 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +import math + +import torch +from torch import nn + + +class PositionEncodingSine(nn.Module): + """ + This is a sinusoidal position encoding that generalized to 2-dimensional images + """ + + def __init__(self, d_model, max_shape=(256, 256), temp_bug_fix=True): + """ + Args: + max_shape (tuple): for 1/8 featmap, the max length of 256 corresponds to 2048 pixels + temp_bug_fix (bool): As noted in this [issue](https://github.com/zju3dv/LoFTR/issues/41), + the original implementation of LoFTR includes a bug in the pos-enc impl, which has little impact + on the final performance. For now, we keep both impls for backward compatability. + We will remove the buggy impl after re-training all variants of our released models. + """ + super().__init__() + + pe = torch.zeros((d_model, *max_shape)) + y_position = torch.ones(max_shape).cumsum(0).float().unsqueeze(0) + x_position = torch.ones(max_shape).cumsum(1).float().unsqueeze(0) + if temp_bug_fix: + div_term = torch.exp( + torch.arange(0, d_model // 2, 2).float() * # noqa + (-math.log(10000.0) / (d_model // 2))) + else: # a buggy implementation (for backward compatability only) + div_term = torch.exp( + torch.arange(0, d_model // 2, 2).float() * # noqa + (-math.log(10000.0) / d_model // 2)) + div_term = div_term[:, None, None] # [C//4, 1, 1] + pe[0::4, :, :] = torch.sin(x_position * div_term) + pe[1::4, :, :] = torch.cos(x_position * div_term) + pe[2::4, :, :] = torch.sin(y_position * div_term) + pe[3::4, :, :] = torch.cos(y_position * div_term) + + self.register_buffer( + 'pe', pe.unsqueeze(0), persistent=False) # [1, C, H, W] + + def forward(self, x): + """ + Args: + x: [N, C, H, W] + """ + return x + self.pe[:, :, :x.size(2), :x.size(3)] diff --git a/runtime/Lib/site_packages/models/cv/image_matching/quadtree_attention_model.py b/runtime/Lib/site_packages/models/cv/image_matching/quadtree_attention_model.py new file mode 100644 index 0000000000000000000000000000000000000000..377c5c4fabbd7cfef95d5fec60a685fc543484a9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/quadtree_attention_model.py @@ -0,0 +1,78 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp +from pathlib import Path + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from .config.default import get_cfg_defaults +from .loftr_quadtree.loftr import LoFTR +from .utils.misc import lower_config + + +@MODELS.register_module( + Tasks.image_matching, module_name=Models.quadtree_attention_image_matching) +class QuadTreeAttentionForImageMatching(TorchModel): + ''' + Image matching with quadtree attention. This model is trained on outdoor images. + For more details, please refer to https://arxiv.org/abs/2201.02767 + ''' + + def __init__(self, model_dir: str, model_type='outdoor', **kwargs): + ''' + Args: + model_dir: model directory + model_type: model type, 'outdoor' or 'indoor'. Only support outdoor model for modelscope. + ''' + assert model_type == 'outdoor', 'Only support outdoor model for modelscope' + # Note: for indoor model, max_image_size should be 640 because scannet training image size is 640, + # and currently, this model is overfited on scannet. For outdoor model, larger image size will be better + + super().__init__(model_dir, **kwargs) + config = get_cfg_defaults() + _config = lower_config(config) + + matcher = LoFTR(config=_config['loftr']) + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + state_dict = torch.load( + str(model_path), map_location='cpu')['state_dict'] + + matcher.load_state_dict(state_dict, strict=True) + self.matcher = matcher + + self.matcher.eval() + self.matcher.to('cuda') + + def forward(self, Inputs): + ''' + Args: + Inputs: a dict with keys 'image0', 'image1' and 'preprocess_info'. + 'image0' and 'image1' are torch tensor with shape [1, 1, H1, W1] + and [1, 1, H2, W2]. 'preprocess_info' contains the information of + resizing, which will be used for postprocessing. + ''' + self.matcher(Inputs) + return { + 'kpts0': Inputs['mkpts0_f'], + 'kpts1': Inputs['mkpts1_f'], + 'conf': Inputs['mconf'], + 'preprocess_info': Inputs['preprocess_info'] + } + + def postprocess(self, Inputs): + matching_result = Inputs + + results = {OutputKeys.MATCHES: matching_result} + return results + + def inference(self, data): + results = self.forward(data) + + return results diff --git a/runtime/Lib/site_packages/models/cv/image_matching/utils/__init__.py b/runtime/Lib/site_packages/models/cv/image_matching/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_matching/utils/misc.py b/runtime/Lib/site_packages/models/cv/image_matching/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..6b5acc10cdfc32d4d3f1997b7e5e88d92641179e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_matching/utils/misc.py @@ -0,0 +1,11 @@ +# This implementation is adopted from LoFTR, +# made public available under the Apache License, Version 2.0, +# at https://github.com/zju3dv/LoFTR + +from yacs.config import CfgNode as CN + + +def lower_config(yacs_cfg): + if not isinstance(yacs_cfg, CN): + return yacs_cfg + return {k.lower(): lower_config(v) for k, v in yacs_cfg.items()} diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9137b4b14796248c97da4401f826d3c0ae49586d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .casmvs_model import ImageMultiViewDepthEstimation + +else: + _import_structure = { + 'casmvs_model': ['ImageMultiViewDepthEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/cas_mvsnet.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/cas_mvsnet.py new file mode 100644 index 0000000000000000000000000000000000000000..63dbf034f8ac017ef96850172397a82501a8eb59 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/cas_mvsnet.py @@ -0,0 +1,221 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .module import (CostRegNet, FeatureNet, RefineNet, depth_regression, + get_depth_range_samples, homo_warping) + +Align_Corners_Range = False + + +class DepthNet(nn.Module): + + def __init__(self): + super(DepthNet, self).__init__() + + def forward(self, + features, + proj_matrices, + depth_values, + num_depth, + cost_regularization, + prob_volume_init=None): + proj_matrices = torch.unbind(proj_matrices, 1) + assert len(features) == len( + proj_matrices + ), 'Different number of images and projection matrices' + assert depth_values.shape[ + 1] == num_depth, 'depth_values.shape[1]:{} num_depth:{}'.format( + depth_values.shapep[1], num_depth) + num_views = len(features) + + # step 1. feature extraction + # in: images; out: 32-channel feature maps + ref_feature, src_features = features[0], features[1:] + ref_proj, src_projs = proj_matrices[0], proj_matrices[1:] + + # step 2. differentiable homograph, build cost volume + ref_volume = ref_feature.unsqueeze(2).repeat(1, 1, num_depth, 1, 1) + volume_sum = ref_volume + volume_sq_sum = ref_volume**2 + del ref_volume + for src_fea, src_proj in zip(src_features, src_projs): + # warpped features + src_proj_new = src_proj[:, 0].clone() + src_proj_new[:, :3, :4] = torch.matmul(src_proj[:, 1, :3, :3], + src_proj[:, 0, :3, :4]) + ref_proj_new = ref_proj[:, 0].clone() + ref_proj_new[:, :3, :4] = torch.matmul(ref_proj[:, 1, :3, :3], + ref_proj[:, 0, :3, :4]) + warped_volume = homo_warping(src_fea, src_proj_new, ref_proj_new, + depth_values) + if self.training: + volume_sum = volume_sum + warped_volume + volume_sq_sum = volume_sq_sum + warped_volume**2 + else: + # TODO: this is only a temporary solution to save memory, better way? + volume_sum += warped_volume + volume_sq_sum += warped_volume.pow_( + 2) # the memory of warped_volume has been modified + del warped_volume + # aggregate multiple feature volumes by variance + volume_variance = volume_sq_sum.div_(num_views).sub_( + volume_sum.div_(num_views).pow_(2)) + + # step 3. cost volume regularization + cost_reg = cost_regularization(volume_variance) + prob_volume_pre = cost_reg.squeeze(1) + + if prob_volume_init is not None: + prob_volume_pre += prob_volume_init + + prob_volume = F.softmax(prob_volume_pre, dim=1) + depth = depth_regression(prob_volume, depth_values=depth_values) + + with torch.no_grad(): + # photometric confidence + prob_volume_sum4 = 4 * F.avg_pool3d( + F.pad(prob_volume.unsqueeze(1), pad=(0, 0, 0, 0, 1, 2)), + (4, 1, 1), + stride=1, + padding=0).squeeze(1) + depth_index = depth_regression( + prob_volume, + depth_values=torch.arange( + num_depth, device=prob_volume.device, + dtype=torch.float)).long() + depth_index = depth_index.clamp(min=0, max=num_depth - 1) + photometric_confidence = torch.gather( + prob_volume_sum4, 1, depth_index.unsqueeze(1)).squeeze(1) + + return { + 'depth': depth, + 'photometric_confidence': photometric_confidence + } + + +class CascadeMVSNet(nn.Module): + + def __init__(self, + refine=False, + ndepths=[48, 32, 8], + depth_interals_ratio=[4, 2, 1], + share_cr=False, + grad_method='detach', + arch_mode='fpn', + cr_base_chs=[8, 8, 8]): + super(CascadeMVSNet, self).__init__() + self.refine = refine + self.share_cr = share_cr + self.ndepths = ndepths + self.depth_interals_ratio = depth_interals_ratio + self.grad_method = grad_method + self.arch_mode = arch_mode + self.cr_base_chs = cr_base_chs + self.num_stage = len(ndepths) + + assert len(ndepths) == len(depth_interals_ratio) + + self.stage_infos = { + 'stage1': { + 'scale': 4.0, + }, + 'stage2': { + 'scale': 2.0, + }, + 'stage3': { + 'scale': 1.0, + } + } + + self.feature = FeatureNet( + base_channels=8, + stride=4, + num_stage=self.num_stage, + arch_mode=self.arch_mode) + if self.share_cr: + self.cost_regularization = CostRegNet( + in_channels=self.feature.out_channels, base_channels=8) + else: + self.cost_regularization = nn.ModuleList([ + CostRegNet( + in_channels=self.feature.out_channels[i], + base_channels=self.cr_base_chs[i]) + for i in range(self.num_stage) + ]) + if self.refine: + self.refine_network = RefineNet() + self.DepthNet = DepthNet() + + def forward(self, imgs, proj_matrices, depth_values): + depth_min = float(depth_values[0, 0].cpu().numpy()) + depth_max = float(depth_values[0, -1].cpu().numpy()) + depth_interval = (depth_max - depth_min) / depth_values.size(1) + + # step 1. feature extraction + features = [] + for nview_idx in range(imgs.size(1)): # imgs shape (B, N, C, H, W) + img = imgs[:, nview_idx] + features.append(self.feature(img)) + + outputs = {} + depth, cur_depth = None, None + for stage_idx in range(self.num_stage): + # stage feature, proj_mats, scales + features_stage = [ + feat['stage{}'.format(stage_idx + 1)] for feat in features + ] + proj_matrices_stage = proj_matrices['stage{}'.format(stage_idx + + 1)] + stage_scale = self.stage_infos['stage{}'.format(stage_idx + + 1)]['scale'] + + if depth is not None: + if self.grad_method == 'detach': + cur_depth = depth.detach() + else: + cur_depth = depth + cur_depth = F.interpolate( + cur_depth.unsqueeze(1), [img.shape[2], img.shape[3]], + mode='bilinear', + align_corners=Align_Corners_Range).squeeze(1) + else: + cur_depth = depth_values + depth_range_samples = get_depth_range_samples( + cur_depth=cur_depth, + ndepth=self.ndepths[stage_idx], + depth_inteval_pixel=self.depth_interals_ratio[stage_idx] + * depth_interval, + dtype=img[0].dtype, + device=img[0].device, + shape=[img.shape[0], img.shape[2], img.shape[3]], + max_depth=depth_max, + min_depth=depth_min) + + outputs_stage = self.DepthNet( + features_stage, + proj_matrices_stage, + depth_values=F.interpolate( + depth_range_samples.unsqueeze(1), [ + self.ndepths[stage_idx], img.shape[2] + // int(stage_scale), img.shape[3] // int(stage_scale) + ], + mode='trilinear', + align_corners=Align_Corners_Range).squeeze(1), + num_depth=self.ndepths[stage_idx], + cost_regularization=self.cost_regularization + if self.share_cr else self.cost_regularization[stage_idx]) + + depth = outputs_stage['depth'] + + outputs['stage{}'.format(stage_idx + 1)] = outputs_stage + outputs.update(outputs_stage) + + # depth map refinement + if self.refine: + refined_depth = self.refine_network( + torch.cat((imgs[:, 0], depth), 1)) + outputs['refined_depth'] = refined_depth + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/casmvs_model.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/casmvs_model.py new file mode 100644 index 0000000000000000000000000000000000000000..e5215607efe6186bed6db1d6ba6207a9a60a7416 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/casmvs_model.py @@ -0,0 +1,164 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import os.path as osp + +import cv2 +import numpy as np +import torch +from easydict import EasyDict as edict + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .cas_mvsnet import CascadeMVSNet +from .colmap2mvsnet import processing_single_scene +from .depth_filter import pcd_depth_filter +from .general_eval_dataset import MVSDataset, save_pfm +from .utils import (generate_pointcloud, numpy2torch, tensor2numpy, tocuda, + write_cam) + +logger = get_logger() + + +@MODELS.register_module( + Tasks.image_multi_view_depth_estimation, + module_name=Models.image_casmvs_depth_estimation) +class ImageMultiViewDepthEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + # build model + self.model = CascadeMVSNet( + refine=False, + ndepths=[48, 32, 8], + depth_interals_ratio=[float(d_i) for d_i in [4, 2, 1]], + share_cr=False, + cr_base_chs=[8, 8, 8], + grad_method='detach') + + # load checkpoint file + ckpt_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model {ckpt_path}') + state_dict = torch.load(ckpt_path, map_location=torch.device('cpu')) + self.model.load_state_dict(state_dict['model'], strict=True) + + if torch.cuda.is_available(): + self.device = 'cuda' + else: + self.device = 'cpu' + + self.model.to(self.device) + self.model.eval() + logger.info(f'model init done! Device:{self.device}') + + def preprocess_make_pair(self, inputs): + + data = inputs['input_dir'] + casmvs_inp_dir = inputs['casmvs_inp_dir'] + + args = edict() + args.dense_folder = data + args.save_folder = casmvs_inp_dir + args.max_d = 192 + args.interval_scale = 1.06 + args.theta0 = 5 + args.sigma1 = 1 + args.sigma2 = 10 + args.model_ext = '.bin' + + logger.info('preprocess of making pair data start') + processing_single_scene(args) + logger.info('preprocess of making pair data done') + + def forward(self, inputs): + test_dir = os.path.dirname(inputs['casmvs_inp_dir']) + scene = os.path.basename(inputs['casmvs_inp_dir']) + test_list = [scene] + save_dir = inputs['casmvs_res_dir'] + + logger.info('depth estimation start') + + test_dataset = MVSDataset( + test_dir, + test_list, + 'test', + 5, + 192, + 1.06, + max_h=1200, + max_w=1200, + fix_res=False) + + with torch.no_grad(): + for batch_idx, sample in enumerate(test_dataset): + sample = numpy2torch(sample) + + if self.device == 'cuda': + sample_cuda = tocuda(sample) + + proj_matrices_dict = sample_cuda['proj_matrices'] + proj_matrices_dict_new = {} + for k, v in proj_matrices_dict.items(): + proj_matrices_dict_new[k] = v.unsqueeze(0) + + outputs = self.model(sample_cuda['imgs'].unsqueeze(0), + proj_matrices_dict_new, + sample_cuda['depth_values'].unsqueeze(0)) + + outputs = tensor2numpy(outputs) + del sample_cuda + filenames = [sample['filename']] + cams = sample['proj_matrices']['stage{}'.format(3)].unsqueeze( + 0).numpy() + imgs = sample['imgs'].unsqueeze(0).numpy() + + # save depth maps and confidence maps + for filename, cam, img, depth_est, photometric_confidence in zip( + filenames, cams, imgs, outputs['depth'], + outputs['photometric_confidence']): + + img = img[0] # ref view + cam = cam[0] # ref cam + depth_filename = os.path.join( + save_dir, filename.format('depth_est', '.pfm')) + confidence_filename = os.path.join( + save_dir, filename.format('confidence', '.pfm')) + cam_filename = os.path.join( + save_dir, filename.format('cams', '_cam.txt')) + img_filename = os.path.join( + save_dir, filename.format('images', '.jpg')) + ply_filename = os.path.join( + save_dir, filename.format('ply_local', '.ply')) + os.makedirs( + depth_filename.rsplit('/', 1)[0], exist_ok=True) + os.makedirs( + confidence_filename.rsplit('/', 1)[0], exist_ok=True) + os.makedirs(cam_filename.rsplit('/', 1)[0], exist_ok=True) + os.makedirs(img_filename.rsplit('/', 1)[0], exist_ok=True) + os.makedirs(ply_filename.rsplit('/', 1)[0], exist_ok=True) + # save depth maps + save_pfm(depth_filename, depth_est) + # save confidence maps + save_pfm(confidence_filename, photometric_confidence) + # save cams, img + write_cam(cam_filename, cam) + img = np.clip(np.transpose(img, (1, 2, 0)) * 255, 0, + 255).astype(np.uint8) + img_bgr = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + cv2.imwrite(img_filename, img_bgr) + + logger.info('depth estimation end') + return inputs + + def postprocess(self, inputs): + test_dir = os.path.dirname(inputs['casmvs_inp_dir']) + scene = os.path.basename(inputs['casmvs_inp_dir']) + logger.info('depth fusion start') + pcd = pcd_depth_filter( + scene, test_dir, inputs['casmvs_res_dir'], thres_view=4) + logger.info('depth fusion end') + return pcd diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py new file mode 100644 index 0000000000000000000000000000000000000000..37d92c13a523835103b76a0664ec182ba221e164 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/colmap2mvsnet.py @@ -0,0 +1,472 @@ +# The implementation is borrowed from https://github.com/YoYo000/MVSNet. Model reading is provided by COLMAP. + +from __future__ import print_function +import collections +import multiprocessing as mp +import os +import shutil +import struct +from functools import partial + +import cv2 +import numpy as np + +# ============================ read_model.py ============================# +CameraModel = collections.namedtuple('CameraModel', + ['model_id', 'model_name', 'num_params']) +Camera = collections.namedtuple('Camera', + ['id', 'model', 'width', 'height', 'params']) +BaseImage = collections.namedtuple( + 'Image', ['id', 'qvec', 'tvec', 'camera_id', 'name', 'xys', 'point3D_ids']) +Point3D = collections.namedtuple( + 'Point3D', ['id', 'xyz', 'rgb', 'error', 'image_ids', 'point2D_idxs']) + + +class Image(BaseImage): + + def qvec2rotmat(self): + return qvec2rotmat(self.qvec) + + +CAMERA_MODELS = { + CameraModel(model_id=0, model_name='SIMPLE_PINHOLE', num_params=3), + CameraModel(model_id=1, model_name='PINHOLE', num_params=4), + CameraModel(model_id=2, model_name='SIMPLE_RADIAL', num_params=4), + CameraModel(model_id=3, model_name='RADIAL', num_params=5), + CameraModel(model_id=4, model_name='OPENCV', num_params=8), + CameraModel(model_id=5, model_name='OPENCV_FISHEYE', num_params=8), + CameraModel(model_id=6, model_name='FULL_OPENCV', num_params=12), + CameraModel(model_id=7, model_name='FOV', num_params=5), + CameraModel(model_id=8, model_name='SIMPLE_RADIAL_FISHEYE', num_params=4), + CameraModel(model_id=9, model_name='RADIAL_FISHEYE', num_params=5), + CameraModel(model_id=10, model_name='THIN_PRISM_FISHEYE', num_params=12) +} +CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) + for camera_model in CAMERA_MODELS]) + + +def read_next_bytes(fid, + num_bytes, + format_char_sequence, + endian_character='<'): + """Read and unpack the next bytes from a binary file. + :param fid: + :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + :param endian_character: Any of {@, =, <, >, !} + :return: Tuple of read and unpacked values. + """ + data = fid.read(num_bytes) + return struct.unpack(endian_character + format_char_sequence, data) + + +def read_cameras_text(path): + cameras = {} + with open(path, 'r', encoding='utf-8') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + camera_id = int(elems[0]) + model = elems[1] + width = int(elems[2]) + height = int(elems[3]) + params = np.array(tuple(map(float, elems[4:]))) + cameras[camera_id] = Camera( + id=camera_id, + model=model, + width=width, + height=height, + params=params) + return cameras + + +def read_cameras_binary(path_to_model_file): + cameras = {} + with open(path_to_model_file, 'rb') as fid: + num_cameras = read_next_bytes(fid, 8, 'Q')[0] + for camera_line_index in range(num_cameras): + camera_properties = read_next_bytes( + fid, num_bytes=24, format_char_sequence='iiQQ') + camera_id = camera_properties[0] + model_id = camera_properties[1] + model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name + width = camera_properties[2] + height = camera_properties[3] + num_params = CAMERA_MODEL_IDS[model_id].num_params + params = read_next_bytes( + fid, + num_bytes=8 * num_params, + format_char_sequence='d' * num_params) + cameras[camera_id] = Camera( + id=camera_id, + model=model_name, + width=width, + height=height, + params=np.array(params)) + assert len(cameras) == num_cameras + return cameras + + +def read_images_text(path): + images = {} + with open(path, 'r', encoding='utf-8') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + image_id = int(elems[0]) + qvec = np.array(tuple(map(float, elems[1:5]))) + tvec = np.array(tuple(map(float, elems[5:8]))) + camera_id = int(elems[8]) + image_name = elems[9] + elems = fid.readline().split() + xys = np.column_stack([ + tuple(map(float, elems[0::3])), + tuple(map(float, elems[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, elems[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def read_images_binary(path_to_model_file): + images = {} + with open(path_to_model_file, 'rb') as fid: + num_reg_images = read_next_bytes(fid, 8, 'Q')[0] + for image_index in range(num_reg_images): + binary_image_properties = read_next_bytes( + fid, num_bytes=64, format_char_sequence='idddddddi') + image_id = binary_image_properties[0] + qvec = np.array(binary_image_properties[1:5]) + tvec = np.array(binary_image_properties[5:8]) + camera_id = binary_image_properties[8] + image_name = '' + current_char = read_next_bytes(fid, 1, 'c')[0] + while current_char != b'\x00': # look for the ASCII 0 entry + image_name += current_char.decode('utf-8') + current_char = read_next_bytes(fid, 1, 'c')[0] + num_points2D = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + x_y_id_s = read_next_bytes( + fid, + num_bytes=24 * num_points2D, + format_char_sequence='ddq' * num_points2D) + xys = np.column_stack([ + tuple(map(float, x_y_id_s[0::3])), + tuple(map(float, x_y_id_s[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def read_points3D_text(path): + points3D = {} + with open(path, 'r', encoding='utf-8') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + point3D_id = int(elems[0]) + xyz = np.array(tuple(map(float, elems[1:4]))) + rgb = np.array(tuple(map(int, elems[4:7]))) + error = float(elems[7]) + image_ids = np.array(tuple(map(int, elems[8::2]))) + point2D_idxs = np.array(tuple(map(int, elems[9::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def read_points3d_binary(path_to_model_file): + points3D = {} + with open(path_to_model_file, 'rb') as fid: + num_points = read_next_bytes(fid, 8, 'Q')[0] + for point_line_index in range(num_points): + binary_point_line_properties = read_next_bytes( + fid, num_bytes=43, format_char_sequence='QdddBBBd') + point3D_id = binary_point_line_properties[0] + xyz = np.array(binary_point_line_properties[1:4]) + rgb = np.array(binary_point_line_properties[4:7]) + error = np.array(binary_point_line_properties[7]) + track_length = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + track_elems = read_next_bytes( + fid, + num_bytes=8 * track_length, + format_char_sequence='ii' * track_length) + image_ids = np.array(tuple(map(int, track_elems[0::2]))) + point2D_idxs = np.array(tuple(map(int, track_elems[1::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def read_model(path, ext): + if ext == '.txt': + cameras = read_cameras_text(os.path.join(path, 'cameras' + ext)) + images = read_images_text(os.path.join(path, 'images' + ext)) + points3D = read_points3D_text(os.path.join(path, 'points3D') + ext) + else: + cameras = read_cameras_binary(os.path.join(path, 'cameras' + ext)) + images = read_images_binary(os.path.join(path, 'images' + ext)) + points3D = read_points3d_binary(os.path.join(path, 'points3D') + ext) + return cameras, images, points3D + + +def qvec2rotmat(qvec): + return np.array([ + [ + 1 - 2 * qvec[2]**2 - 2 * qvec[3]**2, + 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], + 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2] + ], # noqa + [ + 2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], + 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2, + 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1] + ], # noqa + [ + 2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], + 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], + 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2 + ] + ]) # noqa + + +def rotmat2qvec(R): + Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat + K = np.array( + [[Rxx - Ryy - Rzz, 0, 0, 0], [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], + [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], + [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0 # noqa + eigvals, eigvecs = np.linalg.eigh(K) + qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)] + if qvec[0] < 0: + qvec *= -1 + return qvec + + +def calc_score(inputs, images, points3d, extrinsic, args): + i, j = inputs + id_i = images[i + 1].point3D_ids + id_j = images[j + 1].point3D_ids + id_intersect = [it for it in id_i if it in id_j] + cam_center_i = -np.matmul(extrinsic[i + 1][:3, :3].transpose(), + extrinsic[i + 1][:3, 3:4])[:, 0] + cam_center_j = -np.matmul(extrinsic[j + 1][:3, :3].transpose(), + extrinsic[j + 1][:3, 3:4])[:, 0] + score = 0 + for pid in id_intersect: + if pid == -1: + continue + p = points3d[pid].xyz + theta = (180 / np.pi) * np.arccos( + np.dot(cam_center_i - p, cam_center_j - p) + / np.linalg.norm(cam_center_i - p) + / np.linalg.norm(cam_center_j - p)) + tmp_value = ( + 2 * # noqa + (args.sigma1 if theta <= args.theta0 else args.sigma2)**2) + score += np.exp(-(theta - args.theta0) * # noqa + (theta - args.theta0) / tmp_value) + return i, j, score + + +def processing_single_scene(args): + + image_dir = os.path.join(args.dense_folder, 'images') + model_dir = os.path.join(args.dense_folder, 'sparse') + cam_dir = os.path.join(args.save_folder, 'cams') + image_converted_dir = os.path.join(args.save_folder, 'images_post') + + if os.path.exists(image_converted_dir): + shutil.rmtree(image_converted_dir) + os.makedirs(image_converted_dir) + if os.path.exists(cam_dir): + shutil.rmtree(cam_dir) + + cameras, images, points3d = read_model(model_dir, args.model_ext) + num_images = len(list(images.items())) + + param_type = { + 'SIMPLE_PINHOLE': ['f', 'cx', 'cy'], + 'PINHOLE': ['fx', 'fy', 'cx', 'cy'], + 'SIMPLE_RADIAL': ['f', 'cx', 'cy', 'k'], + 'SIMPLE_RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k'], + 'RADIAL': ['f', 'cx', 'cy', 'k1', 'k2'], + 'RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k1', 'k2'], + 'OPENCV': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2'], + 'OPENCV_FISHEYE': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'k3', 'k4'], + 'FULL_OPENCV': [ + 'fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'k5', + 'k6' + ], + 'FOV': ['fx', 'fy', 'cx', 'cy', 'omega'], + 'THIN_PRISM_FISHEYE': [ + 'fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'sx1', + 'sy1' + ] + } + + # intrinsic + intrinsic = {} + for camera_id, cam in cameras.items(): + params_dict = { + key: value + for key, value in zip(param_type[cam.model], cam.params) + } + if 'f' in param_type[cam.model]: + params_dict['fx'] = params_dict['f'] + params_dict['fy'] = params_dict['f'] + i = np.array([[params_dict['fx'], 0, params_dict['cx']], + [0, params_dict['fy'], params_dict['cy']], [0, 0, 1]]) + intrinsic[camera_id] = i + + new_images = {} + for i, image_id in enumerate(sorted(images.keys())): + new_images[i + 1] = images[image_id] + images = new_images + + # extrinsic + extrinsic = {} + for image_id, image in images.items(): + e = np.zeros((4, 4)) + e[:3, :3] = qvec2rotmat(image.qvec) + e[:3, 3] = image.tvec + e[3, 3] = 1 + extrinsic[image_id] = e + + # depth range and interval + depth_ranges = {} + for i in range(num_images): + zs = [] + for p3d_id in images[i + 1].point3D_ids: + if p3d_id == -1: + continue + transformed = np.matmul(extrinsic[i + 1], [ + points3d[p3d_id].xyz[0], points3d[p3d_id].xyz[1], + points3d[p3d_id].xyz[2], 1 + ]) + zs.append(transformed[2].item()) + zs_sorted = sorted(zs) + # relaxed depth range + max_ratio = 0.1 + min_ratio = 0.03 + num_max = max(5, int(len(zs) * max_ratio)) + num_min = max(1, int(len(zs) * min_ratio)) + depth_min = 1.0 * sum(zs_sorted[:num_min]) / len(zs_sorted[:num_min]) + depth_max = 1.0 * sum(zs_sorted[-num_max:]) / len(zs_sorted[-num_max:]) + if args.max_d == 0: + image_int = intrinsic[images[i + 1].camera_id] + image_ext = extrinsic[i + 1] + image_r = image_ext[0:3, 0:3] + image_t = image_ext[0:3, 3] + p1 = [image_int[0, 2], image_int[1, 2], 1] + p2 = [image_int[0, 2] + 1, image_int[1, 2], 1] + P1 = np.matmul(np.linalg.inv(image_int), p1) * depth_min + P1 = np.matmul(np.linalg.inv(image_r), (P1 - image_t)) + P2 = np.matmul(np.linalg.inv(image_int), p2) * depth_min + P2 = np.matmul(np.linalg.inv(image_r), (P2 - image_t)) + depth_num = (1 / depth_min - 1 / depth_max) / ( + 1 / depth_min - 1 / (depth_min + np.linalg.norm(P2 - P1))) + else: + depth_num = args.max_d + depth_interval = (depth_max - depth_min) / (depth_num + - 1) / args.interval_scale + depth_ranges[i + 1] = (depth_min, depth_interval, depth_num, depth_max) + + # view selection + score = np.zeros((len(images), len(images))) + queue = [] + for i in range(len(images)): + for j in range(i + 1, len(images)): + queue.append((i, j)) + + p = mp.Pool(processes=mp.cpu_count()) + func = partial( + calc_score, + images=images, + points3d=points3d, + args=args, + extrinsic=extrinsic) + result = p.map(func, queue) + for i, j, s in result: + score[i, j] = s + score[j, i] = s + view_sel = [] + for i in range(len(images)): + sorted_score = np.argsort(score[i])[::-1] + view_sel.append([(k, score[i, k]) for k in sorted_score[:10]]) + + # write + os.makedirs(cam_dir, exist_ok=True) + + for i in range(num_images): + with open(os.path.join(cam_dir, '%08d_cam.txt' % i), 'w') as f: + f.write('extrinsic\n') + for j in range(4): + for k in range(4): + f.write(str(extrinsic[i + 1][j, k]) + ' ') + f.write('\n') + f.write('\nintrinsic\n') + for j in range(3): + for k in range(3): + f.write( + str(intrinsic[images[i + 1].camera_id][j, k]) + ' ') + f.write('\n') + f.write('\n%f %f %f %f\n' % + (depth_ranges[i + 1][0], depth_ranges[i + 1][1], + depth_ranges[i + 1][2], depth_ranges[i + 1][3])) + with open(os.path.join(args.save_folder, 'pair.txt'), 'w') as f: + f.write('%d\n' % len(images)) + for i, sorted_score in enumerate(view_sel): + f.write('%d\n%d ' % (i, len(sorted_score))) + for image_id, s in sorted_score: + f.write('%d %f ' % (image_id, s)) + f.write('\n') + + # convert to jpg + for i in range(num_images): + img_path = os.path.join(image_dir, images[i + 1].name) + if not img_path.endswith('.jpg'): + img = cv2.imread(img_path) + cv2.imwrite(os.path.join(image_converted_dir, '%08d.jpg' % i), img) + else: + shutil.copyfile( + os.path.join(image_dir, images[i + 1].name), + os.path.join(image_converted_dir, '%08d.jpg' % i)) diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/depth_filter.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/depth_filter.py new file mode 100644 index 0000000000000000000000000000000000000000..4ef6275a249777a94516f628fc63fedd25d7f982 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/depth_filter.py @@ -0,0 +1,249 @@ +# The implementation here is modified based on https://github.com/xy-guo/MVSNet_pytorch +import os + +import cv2 +import numpy as np +from PIL import Image +from plyfile import PlyData, PlyElement + +from .general_eval_dataset import read_pfm + + +# read intrinsics and extrinsics +def read_camera_parameters(filename): + with open(filename) as f: + lines = f.readlines() + lines = [line.rstrip() for line in lines] + # extrinsics: line [1,5), 4x4 matrix + extrinsics = np.fromstring( + ' '.join(lines[1:5]), dtype=np.float32, sep=' ').reshape((4, 4)) + # intrinsics: line [7-10), 3x3 matrix + intrinsics = np.fromstring( + ' '.join(lines[7:10]), dtype=np.float32, sep=' ').reshape((3, 3)) + # assume the feature is 1/4 of the original image size + # intrinsics[:2, :] /= 4 + return intrinsics, extrinsics + + +# read an image +def read_img(filename): + img = Image.open(filename) + # scale 0~255 to 0~1 + np_img = np.array(img, dtype=np.float32) / 255. + return np_img + + +# read a binary mask +def read_mask(filename): + return read_img(filename) > 0.5 + + +# save a binary mask +def save_mask(filename, mask): + assert mask.dtype == bool + mask = mask.astype(np.uint8) * 255 + Image.fromarray(mask).save(filename) + + +# read a pair file, [(ref_view1, [src_view1-1, ...]), (ref_view2, [src_view2-1, ...]), ...] +def read_pair_file(filename): + data = [] + with open(filename) as f: + num_viewpoint = int(f.readline()) + # 49 viewpoints + for view_idx in range(num_viewpoint): + ref_view = int(f.readline().rstrip()) + src_views = [int(x) for x in f.readline().rstrip().split()[1::2]] + if len(src_views) > 0: + data.append((ref_view, src_views)) + return data + + +# project the reference point cloud into the source view, then project back +def reproject_with_depth(depth_ref, intrinsics_ref, extrinsics_ref, depth_src, + intrinsics_src, extrinsics_src): + width, height = depth_ref.shape[1], depth_ref.shape[0] + # step1. project reference pixels to the source view + # reference view x, y + x_ref, y_ref = np.meshgrid(np.arange(0, width), np.arange(0, height)) + x_ref, y_ref = x_ref.reshape([-1]), y_ref.reshape([-1]) + # reference 3D space + xyz_ref = np.matmul( + np.linalg.inv(intrinsics_ref), + np.vstack( + (x_ref, y_ref, np.ones_like(x_ref))) * depth_ref.reshape([-1])) + # source 3D space + xyz_src = np.matmul( + np.matmul(extrinsics_src, np.linalg.inv(extrinsics_ref)), + np.vstack((xyz_ref, np.ones_like(x_ref))))[:3] + # source view x, y + K_xyz_src = np.matmul(intrinsics_src, xyz_src) + xy_src = K_xyz_src[:2] / K_xyz_src[2:3] + + # step2. reproject the source view points with source view depth estimation + # find the depth estimation of the source view + x_src = xy_src[0].reshape([height, width]).astype(np.float32) + y_src = xy_src[1].reshape([height, width]).astype(np.float32) + sampled_depth_src = cv2.remap( + depth_src, x_src, y_src, interpolation=cv2.INTER_LINEAR) + + # source 3D space + # NOTE that we should use sampled source-view depth_here to project back + xyz_src = np.matmul( + np.linalg.inv(intrinsics_src), + np.vstack( + (xy_src, np.ones_like(x_ref))) * sampled_depth_src.reshape([-1])) + # reference 3D space + xyz_reprojected = np.matmul( + np.matmul(extrinsics_ref, np.linalg.inv(extrinsics_src)), + np.vstack((xyz_src, np.ones_like(x_ref))))[:3] + # source view x, y, depth + depth_reprojected = xyz_reprojected[2].reshape([height, + width]).astype(np.float32) + K_xyz_reprojected = np.matmul(intrinsics_ref, xyz_reprojected) + xy_reprojected = K_xyz_reprojected[:2] / K_xyz_reprojected[2:3] + x_reprojected = xy_reprojected[0].reshape([height, + width]).astype(np.float32) + y_reprojected = xy_reprojected[1].reshape([height, + width]).astype(np.float32) + + return depth_reprojected, x_reprojected, y_reprojected, x_src, y_src + + +def check_geometric_consistency(depth_ref, intrinsics_ref, extrinsics_ref, + depth_src, intrinsics_src, extrinsics_src): + width, height = depth_ref.shape[1], depth_ref.shape[0] + x_ref, y_ref = np.meshgrid(np.arange(0, width), np.arange(0, height)) + depth_reprojected, x2d_reprojected, y2d_reprojected, x2d_src, y2d_src = reproject_with_depth( + depth_ref, intrinsics_ref, extrinsics_ref, depth_src, intrinsics_src, + extrinsics_src) + # check |p_reproj-p_1| < 1 + dist = np.sqrt((x2d_reprojected - x_ref)**2 + (y2d_reprojected - y_ref)**2) + + # check |d_reproj-d_1| / d_1 < 0.01 + depth_diff = np.abs(depth_reprojected - depth_ref) + relative_depth_diff = depth_diff / depth_ref + + mask = np.logical_and(dist < 1, relative_depth_diff < 0.01) + depth_reprojected[~mask] = 0 + + return mask, depth_reprojected, x2d_src, y2d_src + + +def filter_depth(pair_folder, scan_folder, out_folder, thres_view): + # the pair file + pair_file = os.path.join(pair_folder, 'pair.txt') + # for the final point cloud + vertexs = [] + vertex_colors = [] + + pair_data = read_pair_file(pair_file) + + # for each reference view and the corresponding source views + for ref_view, src_views in pair_data: + # src_views = src_views[:args.num_view] + # load the camera parameters + ref_intrinsics, ref_extrinsics = read_camera_parameters( + os.path.join(scan_folder, 'cams/{:0>8}_cam.txt'.format(ref_view))) + # load the reference image + ref_img = read_img( + os.path.join(scan_folder, 'images/{:0>8}.jpg'.format(ref_view))) + # load the estimated depth of the reference view + ref_depth_est = read_pfm( + os.path.join(out_folder, + 'depth_est/{:0>8}.pfm'.format(ref_view)))[0] + # load the photometric mask of the reference view + confidence = read_pfm( + os.path.join(out_folder, + 'confidence/{:0>8}.pfm'.format(ref_view)))[0] + photo_mask = confidence > 0.9 + + all_srcview_depth_ests = [] + all_srcview_x = [] + all_srcview_y = [] + all_srcview_geomask = [] + + # compute the geometric mask + geo_mask_sum = 0 + for src_view in src_views: + # camera parameters of the source view + src_intrinsics, src_extrinsics = read_camera_parameters( + os.path.join(scan_folder, + 'cams/{:0>8}_cam.txt'.format(src_view))) + # the estimated depth of the source view + src_depth_est = read_pfm( + os.path.join(out_folder, + 'depth_est/{:0>8}.pfm'.format(src_view)))[0] + + geo_mask, depth_reprojected, x2d_src, y2d_src = check_geometric_consistency( + ref_depth_est, ref_intrinsics, ref_extrinsics, src_depth_est, + src_intrinsics, src_extrinsics) + geo_mask_sum += geo_mask.astype(np.int32) + all_srcview_depth_ests.append(depth_reprojected) + all_srcview_x.append(x2d_src) + all_srcview_y.append(y2d_src) + all_srcview_geomask.append(geo_mask) + + depth_est_averaged = (sum(all_srcview_depth_ests) + ref_depth_est) / ( + geo_mask_sum + 1) + # at least 3 source views matched + geo_mask = geo_mask_sum >= thres_view + final_mask = np.logical_and(photo_mask, geo_mask) + + os.makedirs(os.path.join(out_folder, 'mask'), exist_ok=True) + save_mask( + os.path.join(out_folder, 'mask/{:0>8}_photo.png'.format(ref_view)), + photo_mask) + save_mask( + os.path.join(out_folder, 'mask/{:0>8}_geo.png'.format(ref_view)), + geo_mask) + save_mask( + os.path.join(out_folder, 'mask/{:0>8}_final.png'.format(ref_view)), + final_mask) + + height, width = depth_est_averaged.shape[:2] + x, y = np.meshgrid(np.arange(0, width), np.arange(0, height)) + valid_points = final_mask + x, y, depth = x[valid_points], y[valid_points], depth_est_averaged[ + valid_points] + + color = ref_img[valid_points] + + xyz_ref = np.matmul( + np.linalg.inv(ref_intrinsics), + np.vstack((x, y, np.ones_like(x))) * depth) + xyz_world = np.matmul( + np.linalg.inv(ref_extrinsics), np.vstack( + (xyz_ref, np.ones_like(x))))[:3] + vertexs.append(xyz_world.transpose((1, 0))) + vertex_colors.append((color * 255).astype(np.uint8)) + + vertexs = np.concatenate(vertexs, axis=0) + vertex_colors = np.concatenate(vertex_colors, axis=0) + vertexs = np.array([tuple(v) for v in vertexs], + dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) + vertex_colors = np.array([tuple(v) for v in vertex_colors], + dtype=[('red', 'u1'), ('green', 'u1'), + ('blue', 'u1')]) + + vertex_all = np.empty( + len(vertexs), vertexs.dtype.descr + vertex_colors.dtype.descr) + for prop in vertexs.dtype.names: + vertex_all[prop] = vertexs[prop] + for prop in vertex_colors.dtype.names: + vertex_all[prop] = vertex_colors[prop] + + el = PlyElement.describe(vertex_all, 'vertex') + # PlyData([el]).write(plyfilename) + pcd = PlyData([el]) + + return pcd + + +def pcd_depth_filter(scene, test_dir, save_dir, thres_view): + old_scene_folder = os.path.join(test_dir, scene) + new_scene_folder = os.path.join(save_dir, scene) + out_folder = os.path.join(save_dir, scene) + pcd = filter_depth(old_scene_folder, new_scene_folder, out_folder, + thres_view) + return pcd diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/general_eval_dataset.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/general_eval_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..87ab311d1ec03f87b1094519533bf5f231e4652c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/general_eval_dataset.py @@ -0,0 +1,284 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import re +import sys + +import cv2 +import numpy as np +from PIL import Image +from torch.utils.data import Dataset + + +def read_pfm(filename): + file = open(filename, 'rb') + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().decode('utf-8').rstrip() + if header == 'PF': + color = True + elif header == 'Pf': + color = False + else: + raise Exception('Not a PFM file.') + + dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode('utf-8')) + if dim_match: + width, height = map(int, dim_match.groups()) + else: + raise Exception('Malformed PFM header.') + + scale = float(file.readline().rstrip()) + if scale < 0: # little-endian + endian = '<' + scale = -scale + else: + endian = '>' # big-endian + + data = np.fromfile(file, endian + 'f') + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + file.close() + return data, scale + + +def save_pfm(filename, image, scale=1): + file = open(filename, 'wb') + color = None + + image = np.flipud(image) + + if image.dtype.name != 'float32': + raise Exception('Image dtype must be float32.') + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif len(image.shape) == 2 or len( + image.shape) == 3 and image.shape[2] == 1: # greyscale + color = False + else: + raise Exception( + 'Image must have H x W x 3, H x W x 1 or H x W dimensions.') + + file.write('PF\n'.encode('utf-8') if color else 'Pf\n'.encode('utf-8')) + file.write('{} {}\n'.format(image.shape[1], + image.shape[0]).encode('utf-8')) + + endian = image.dtype.byteorder + + if endian == '<' or endian == '=' and sys.byteorder == 'little': + scale = -scale + + file.write(('%f\n' % scale).encode('utf-8')) + + image.tofile(file) + file.close() + + +S_H, S_W = 0, 0 + + +class MVSDataset(Dataset): + + def __init__(self, + datapath, + listfile, + mode, + nviews, + ndepths=192, + interval_scale=1.06, + **kwargs): + super(MVSDataset, self).__init__() + self.datapath = datapath + self.listfile = listfile + self.mode = mode + self.nviews = nviews + self.ndepths = ndepths + self.interval_scale = interval_scale + self.max_h, self.max_w = kwargs['max_h'], kwargs['max_w'] + self.fix_res = kwargs.get( + 'fix_res', False) # whether to fix the resolution of input image. + self.fix_wh = False + + assert self.mode == 'test' + self.metas = self.build_list() + + def build_list(self): + metas = [] + scans = self.listfile + + interval_scale_dict = {} + # scans + for scan in scans: + # determine the interval scale of each scene. default is 1.06 + if isinstance(self.interval_scale, float): + interval_scale_dict[scan] = self.interval_scale + else: + interval_scale_dict[scan] = self.interval_scale[scan] + + pair_file = '{}/pair.txt'.format(scan) + # read the pair file + with open(os.path.join(self.datapath, pair_file)) as f: + num_viewpoint = int(f.readline()) + # viewpoints + for view_idx in range(num_viewpoint): + ref_view = int(f.readline().rstrip()) + src_views = [ + int(x) for x in f.readline().rstrip().split()[1::2] + ] + # filter by no src view and fill to nviews + if len(src_views) > 0: + if len(src_views) < self.nviews: + src_views += [src_views[0]] * ( + self.nviews - len(src_views)) + metas.append((scan, ref_view, src_views, scan)) + + self.interval_scale = interval_scale_dict + return metas + + def __len__(self): + return len(self.metas) + + def read_cam_file(self, filename, interval_scale): + with open(filename) as f: + lines = f.readlines() + lines = [line.rstrip() for line in lines] + # extrinsics: line [1,5), 4x4 matrix + extrinsics = np.fromstring( + ' '.join(lines[1:5]), dtype=np.float32, sep=' ').reshape((4, 4)) + # intrinsics: line [7-10), 3x3 matrix + intrinsics = np.fromstring( + ' '.join(lines[7:10]), dtype=np.float32, sep=' ').reshape((3, 3)) + intrinsics[:2, :] /= 4.0 + # depth_min & depth_interval: line 11 + depth_min = float(lines[11].split()[0]) + depth_interval = float(lines[11].split()[1]) + + if len(lines[11].split()) >= 3: + num_depth = lines[11].split()[2] + depth_max = depth_min + int(float(num_depth)) * depth_interval + depth_interval = (depth_max - depth_min) / self.ndepths + + depth_interval *= interval_scale + + return intrinsics, extrinsics, depth_min, depth_interval + + def read_img(self, filename): + img = Image.open(filename) + # scale 0~255 to 0~1 + np_img = np.array(img, dtype=np.float32) / 255. + + return np_img + + def read_depth(self, filename): + # read pfm depth file + return np.array(read_pfm(filename)[0], dtype=np.float32) + + def scale_mvs_input(self, img, intrinsics, max_w, max_h, base=32): + h, w = img.shape[:2] + if h > max_h or w > max_w: + scale = 1.0 * max_h / h + if scale * w > max_w: + scale = 1.0 * max_w / w + new_w, new_h = scale * w // base * base, scale * h // base * base + else: + new_w, new_h = 1.0 * w // base * base, 1.0 * h // base * base + + scale_w = 1.0 * new_w / w + scale_h = 1.0 * new_h / h + intrinsics[0, :] *= scale_w + intrinsics[1, :] *= scale_h + + img = cv2.resize(img, (int(new_w), int(new_h))) + + return img, intrinsics + + def __getitem__(self, idx): + global S_H, S_W + meta = self.metas[idx] + scan, ref_view, src_views, scene_name = meta + # use only the reference view and first nviews-1 source views + view_ids = [ref_view] + src_views[:self.nviews - 1] + + imgs = [] + depth_values = None + proj_matrices = [] + + for i, vid in enumerate(view_ids): + img_filename = os.path.join( + self.datapath, '{}/images_post/{:0>8}.jpg'.format(scan, vid)) + if not os.path.exists(img_filename): + img_filename = os.path.join( + self.datapath, '{}/images/{:0>8}.jpg'.format(scan, vid)) + + proj_mat_filename = os.path.join( + self.datapath, '{}/cams/{:0>8}_cam.txt'.format(scan, vid)) + + img = self.read_img(img_filename) + intrinsics, extrinsics, depth_min, depth_interval = self.read_cam_file( + proj_mat_filename, + interval_scale=self.interval_scale[scene_name]) + # scale input + img, intrinsics = self.scale_mvs_input(img, intrinsics, self.max_w, + self.max_h) + + if self.fix_res: + # using the same standard height or width in entire scene. + S_H, S_W = img.shape[:2] + self.fix_res = False + self.fix_wh = True + + if i == 0: + if not self.fix_wh: + # using the same standard height or width in each nviews. + S_H, S_W = img.shape[:2] + + # resize to standard height or width + c_h, c_w = img.shape[:2] + if (c_h != S_H) or (c_w != S_W): + scale_h = 1.0 * S_H / c_h + scale_w = 1.0 * S_W / c_w + img = cv2.resize(img, (S_W, S_H)) + intrinsics[0, :] *= scale_w + intrinsics[1, :] *= scale_h + + imgs.append(img) + # extrinsics, intrinsics + proj_mat = np.zeros(shape=(2, 4, 4), dtype=np.float32) # + proj_mat[0, :4, :4] = extrinsics + proj_mat[1, :3, :3] = intrinsics + proj_matrices.append(proj_mat) + + if i == 0: # reference view + depth_values = np.arange( + depth_min, + depth_interval * (self.ndepths - 0.5) + depth_min, + depth_interval, + dtype=np.float32) + + # all + imgs = np.stack(imgs).transpose([0, 3, 1, 2]) + proj_matrices = np.stack(proj_matrices) + + stage2_pjmats = proj_matrices.copy() + stage2_pjmats[:, 1, :2, :] = proj_matrices[:, 1, :2, :] * 2 + stage3_pjmats = proj_matrices.copy() + stage3_pjmats[:, 1, :2, :] = proj_matrices[:, 1, :2, :] * 4 + + proj_matrices_ms = { + 'stage1': proj_matrices, + 'stage2': stage2_pjmats, + 'stage3': stage3_pjmats + } + + return { + 'imgs': imgs, + 'proj_matrices': proj_matrices_ms, + 'depth_values': depth_values, + 'filename': scan + '/{}/' + '{:0>8}'.format(view_ids[0]) + '{}' + } diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/module.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/module.py new file mode 100644 index 0000000000000000000000000000000000000000..2ffda232c83b6426ba485c02937e473345b4d3d0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/module.py @@ -0,0 +1,678 @@ +# The implementation here is modified based on https://github.com/xy-guo/MVSNet_pytorch + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def init_bn(module): + if module.weight is not None: + nn.init.ones_(module.weight) + if module.bias is not None: + nn.init.zeros_(module.bias) + return + + +def init_uniform(module, init_method): + if module.weight is not None: + if init_method == 'kaiming': + nn.init.kaiming_uniform_(module.weight) + elif init_method == 'xavier': + nn.init.xavier_uniform_(module.weight) + return + + +class Conv2d(nn.Module): + """Applies a 2D convolution (optionally with batch normalization and relu activation) + over an input signal composed of several input planes. + + Attributes: + conv (nn.Module): convolution module + bn (nn.Module): batch normalization module + relu (bool): whether to activate by relu + + Notes: + Default momentum for batch normalization is set to be 0.01, + + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + relu=True, + bn=True, + bn_momentum=0.1, + init_method='xavier', + **kwargs): + super(Conv2d, self).__init__() + + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + bias=(not bn), + **kwargs) + self.kernel_size = kernel_size + self.stride = stride + self.bn = nn.BatchNorm2d( + out_channels, momentum=bn_momentum) if bn else None + self.relu = relu + + def forward(self, x): + x = self.conv(x) + if self.bn is not None: + x = self.bn(x) + if self.relu: + x = F.relu(x, inplace=True) + return x + + def init_weights(self, init_method): + """default initialization""" + init_uniform(self.conv, init_method) + if self.bn is not None: + init_bn(self.bn) + + +class Deconv2d(nn.Module): + """Applies a 2D deconvolution (optionally with batch normalization and relu activation) + over an input signal composed of several input planes. + + Attributes: + conv (nn.Module): convolution module + bn (nn.Module): batch normalization module + relu (bool): whether to activate by relu + + Notes: + Default momentum for batch normalization is set to be 0.01, + + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + relu=True, + bn=True, + bn_momentum=0.1, + init_method='xavier', + **kwargs): + super(Deconv2d, self).__init__() + self.out_channels = out_channels + assert stride in [1, 2] + self.stride = stride + + self.conv = nn.ConvTranspose2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + bias=(not bn), + **kwargs) + self.bn = nn.BatchNorm2d( + out_channels, momentum=bn_momentum) if bn else None + self.relu = relu + + def forward(self, x): + y = self.conv(x) + if self.stride == 2: + h, w = list(x.size())[2:] + y = y[:, :, :2 * h, :2 * w].contiguous() + if self.bn is not None: + x = self.bn(y) + if self.relu: + x = F.relu(x, inplace=True) + return x + + def init_weights(self, init_method): + """default initialization""" + init_uniform(self.conv, init_method) + if self.bn is not None: + init_bn(self.bn) + + +class Conv3d(nn.Module): + """Applies a 3D convolution (optionally with batch normalization and relu activation) + over an input signal composed of several input planes. + + Attributes: + conv (nn.Module): convolution module + bn (nn.Module): batch normalization module + relu (bool): whether to activate by relu + + Notes: + Default momentum for batch normalization is set to be 0.01, + + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + relu=True, + bn=True, + bn_momentum=0.1, + init_method='xavier', + **kwargs): + super(Conv3d, self).__init__() + self.out_channels = out_channels + self.kernel_size = kernel_size + assert stride in [1, 2] + self.stride = stride + + self.conv = nn.Conv3d( + in_channels, + out_channels, + kernel_size, + stride=stride, + bias=(not bn), + **kwargs) + self.bn = nn.BatchNorm3d( + out_channels, momentum=bn_momentum) if bn else None + self.relu = relu + + def forward(self, x): + x = self.conv(x) + if self.bn is not None: + x = self.bn(x) + if self.relu: + x = F.relu(x, inplace=True) + return x + + def init_weights(self, init_method): + """default initialization""" + init_uniform(self.conv, init_method) + if self.bn is not None: + init_bn(self.bn) + + +class Deconv3d(nn.Module): + """Applies a 3D deconvolution (optionally with batch normalization and relu activation) + over an input signal composed of several input planes. + + Attributes: + conv (nn.Module): convolution module + bn (nn.Module): batch normalization module + relu (bool): whether to activate by relu + + Notes: + Default momentum for batch normalization is set to be 0.01, + + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + relu=True, + bn=True, + bn_momentum=0.1, + init_method='xavier', + **kwargs): + super(Deconv3d, self).__init__() + self.out_channels = out_channels + assert stride in [1, 2] + self.stride = stride + + self.conv = nn.ConvTranspose3d( + in_channels, + out_channels, + kernel_size, + stride=stride, + bias=(not bn), + **kwargs) + self.bn = nn.BatchNorm3d( + out_channels, momentum=bn_momentum) if bn else None + self.relu = relu + + def forward(self, x): + y = self.conv(x) + if self.bn is not None: + x = self.bn(y) + if self.relu: + x = F.relu(x, inplace=True) + return x + + def init_weights(self, init_method): + """default initialization""" + init_uniform(self.conv, init_method) + if self.bn is not None: + init_bn(self.bn) + + +class ConvBnReLU(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + pad=1): + super(ConvBnReLU, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=pad, + bias=False) + self.bn = nn.BatchNorm2d(out_channels) + + def forward(self, x): + return F.relu(self.bn(self.conv(x)), inplace=True) + + +class ConvBn(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + pad=1): + super(ConvBn, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=pad, + bias=False) + self.bn = nn.BatchNorm2d(out_channels) + + def forward(self, x): + return self.bn(self.conv(x)) + + +def homo_warping(src_fea, src_proj, ref_proj, depth_values): + """ + src_fea: [B, C, H, W] + src_proj: [B, 4, 4] + ref_proj: [B, 4, 4] + depth_values: [B, Ndepth] o [B, Ndepth, H, W] + out: [B, C, Ndepth, H, W] + """ + batch, channels = src_fea.shape[0], src_fea.shape[1] + num_depth = depth_values.shape[1] + height, width = src_fea.shape[2], src_fea.shape[3] + + with torch.no_grad(): + proj = torch.matmul(src_proj, torch.inverse(ref_proj)) + rot = proj[:, :3, :3] # [B,3,3] + trans = proj[:, :3, 3:4] # [B,3,1] + + y, x = torch.meshgrid([ + torch.arange( + 0, height, dtype=torch.float32, device=src_fea.device), + torch.arange(0, width, dtype=torch.float32, device=src_fea.device) + ]) + y, x = y.contiguous(), x.contiguous() + y, x = y.view(height * width), x.view(height * width) + xyz = torch.stack((x, y, torch.ones_like(x))) # [3, H*W] + xyz = torch.unsqueeze(xyz, 0).repeat(batch, 1, 1) # [B, 3, H*W] + rot_xyz = torch.matmul(rot, xyz) # [B, 3, H*W] + rot_depth_xyz = rot_xyz.unsqueeze(2).repeat( + 1, 1, num_depth, 1) * depth_values.view(batch, 1, num_depth, + -1) # [B, 3, Ndepth, H*W] + proj_xyz = rot_depth_xyz + trans.view(batch, 3, 1, + 1) # [B, 3, Ndepth, H*W] + proj_xy = proj_xyz[:, : + 2, :, :] / proj_xyz[:, 2: + 3, :, :] # [B, 2, Ndepth, H*W] + proj_x_normalized = proj_xy[:, 0, :, :] / ((width - 1) / 2) - 1 + proj_y_normalized = proj_xy[:, 1, :, :] / ((height - 1) / 2) - 1 + proj_xy = torch.stack((proj_x_normalized, proj_y_normalized), + dim=3) # [B, Ndepth, H*W, 2] + grid = proj_xy + + warped_src_fea = F.grid_sample( + src_fea, + grid.view(batch, num_depth * height, width, 2), + mode='bilinear', + padding_mode='zeros', + align_corners=True) + warped_src_fea = warped_src_fea.view(batch, channels, num_depth, height, + width) + + return warped_src_fea + + +class DeConv2dFuse(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + relu=True, + bn=True, + bn_momentum=0.1): + super(DeConv2dFuse, self).__init__() + + self.deconv = Deconv2d( + in_channels, + out_channels, + kernel_size, + stride=2, + padding=1, + output_padding=1, + bn=True, + relu=relu, + bn_momentum=bn_momentum) + + self.conv = Conv2d( + 2 * out_channels, + out_channels, + kernel_size, + stride=1, + padding=1, + bn=bn, + relu=relu, + bn_momentum=bn_momentum) + + def forward(self, x_pre, x): + x = self.deconv(x) + x = torch.cat((x, x_pre), dim=1) + x = self.conv(x) + return x + + +class FeatureNet(nn.Module): + + def __init__(self, base_channels, num_stage=3, stride=4, arch_mode='unet'): + super(FeatureNet, self).__init__() + assert arch_mode in [ + 'unet', 'fpn' + ], f"mode must be in 'unet' or 'fpn', but get:{arch_mode}" + self.arch_mode = arch_mode + self.stride = stride + self.base_channels = base_channels + self.num_stage = num_stage + + self.conv0 = nn.Sequential( + Conv2d(3, base_channels, 3, 1, padding=1), + Conv2d(base_channels, base_channels, 3, 1, padding=1), + ) + + self.conv1 = nn.Sequential( + Conv2d(base_channels, base_channels * 2, 5, stride=2, padding=2), + Conv2d(base_channels * 2, base_channels * 2, 3, 1, padding=1), + Conv2d(base_channels * 2, base_channels * 2, 3, 1, padding=1), + ) + + self.conv2 = nn.Sequential( + Conv2d( + base_channels * 2, base_channels * 4, 5, stride=2, padding=2), + Conv2d(base_channels * 4, base_channels * 4, 3, 1, padding=1), + Conv2d(base_channels * 4, base_channels * 4, 3, 1, padding=1), + ) + + self.out1 = nn.Conv2d( + base_channels * 4, base_channels * 4, 1, bias=False) + self.out_channels = [4 * base_channels] + + if self.arch_mode == 'unet': + if num_stage == 3: + self.deconv1 = DeConv2dFuse(base_channels * 4, + base_channels * 2, 3) + self.deconv2 = DeConv2dFuse(base_channels * 2, base_channels, + 3) + + self.out2 = nn.Conv2d( + base_channels * 2, base_channels * 2, 1, bias=False) + self.out3 = nn.Conv2d( + base_channels, base_channels, 1, bias=False) + self.out_channels.append(2 * base_channels) + self.out_channels.append(base_channels) + + elif num_stage == 2: + self.deconv1 = DeConv2dFuse(base_channels * 4, + base_channels * 2, 3) + + self.out2 = nn.Conv2d( + base_channels * 2, base_channels * 2, 1, bias=False) + self.out_channels.append(2 * base_channels) + elif self.arch_mode == 'fpn': + final_chs = base_channels * 4 + if num_stage == 3: + self.inner1 = nn.Conv2d( + base_channels * 2, final_chs, 1, bias=True) + self.inner2 = nn.Conv2d( + base_channels * 1, final_chs, 1, bias=True) + + self.out2 = nn.Conv2d( + final_chs, base_channels * 2, 3, padding=1, bias=False) + self.out3 = nn.Conv2d( + final_chs, base_channels, 3, padding=1, bias=False) + self.out_channels.append(base_channels * 2) + self.out_channels.append(base_channels) + + elif num_stage == 2: + self.inner1 = nn.Conv2d( + base_channels * 2, final_chs, 1, bias=True) + + self.out2 = nn.Conv2d( + final_chs, base_channels, 3, padding=1, bias=False) + self.out_channels.append(base_channels) + + def forward(self, x): + conv0 = self.conv0(x) + conv1 = self.conv1(conv0) + conv2 = self.conv2(conv1) + + intra_feat = conv2 + outputs = {} + out = self.out1(intra_feat) + outputs['stage1'] = out + if self.arch_mode == 'unet': + if self.num_stage == 3: + intra_feat = self.deconv1(conv1, intra_feat) + out = self.out2(intra_feat) + outputs['stage2'] = out + + intra_feat = self.deconv2(conv0, intra_feat) + out = self.out3(intra_feat) + outputs['stage3'] = out + + elif self.num_stage == 2: + intra_feat = self.deconv1(conv1, intra_feat) + out = self.out2(intra_feat) + outputs['stage2'] = out + + elif self.arch_mode == 'fpn': + if self.num_stage == 3: + intra_feat = F.interpolate( + intra_feat, scale_factor=2, + mode='nearest') + self.inner1(conv1) + out = self.out2(intra_feat) + outputs['stage2'] = out + + intra_feat = F.interpolate( + intra_feat, scale_factor=2, + mode='nearest') + self.inner2(conv0) + out = self.out3(intra_feat) + outputs['stage3'] = out + + elif self.num_stage == 2: + intra_feat = F.interpolate( + intra_feat, scale_factor=2, + mode='nearest') + self.inner1(conv1) + out = self.out2(intra_feat) + outputs['stage2'] = out + + return outputs + + +class CostRegNet(nn.Module): + + def __init__(self, in_channels, base_channels): + super(CostRegNet, self).__init__() + self.conv0 = Conv3d(in_channels, base_channels, padding=1) + + self.conv1 = Conv3d( + base_channels, base_channels * 2, stride=2, padding=1) + self.conv2 = Conv3d(base_channels * 2, base_channels * 2, padding=1) + + self.conv3 = Conv3d( + base_channels * 2, base_channels * 4, stride=2, padding=1) + self.conv4 = Conv3d(base_channels * 4, base_channels * 4, padding=1) + + self.conv5 = Conv3d( + base_channels * 4, base_channels * 8, stride=2, padding=1) + self.conv6 = Conv3d(base_channels * 8, base_channels * 8, padding=1) + + self.conv7 = Deconv3d( + base_channels * 8, + base_channels * 4, + stride=2, + padding=1, + output_padding=1) + + self.conv9 = Deconv3d( + base_channels * 4, + base_channels * 2, + stride=2, + padding=1, + output_padding=1) + + self.conv11 = Deconv3d( + base_channels * 2, + base_channels * 1, + stride=2, + padding=1, + output_padding=1) + + self.prob = nn.Conv3d( + base_channels, 1, 3, stride=1, padding=1, bias=False) + + def forward(self, x): + conv0 = self.conv0(x) + conv2 = self.conv2(self.conv1(conv0)) + conv4 = self.conv4(self.conv3(conv2)) + x = self.conv6(self.conv5(conv4)) + x = conv4 + self.conv7(x) + x = conv2 + self.conv9(x) + x = conv0 + self.conv11(x) + x = self.prob(x) + return x + + +class RefineNet(nn.Module): + + def __init__(self): + super(RefineNet, self).__init__() + self.conv1 = ConvBnReLU(4, 32) + self.conv2 = ConvBnReLU(32, 32) + self.conv3 = ConvBnReLU(32, 32) + self.res = ConvBnReLU(32, 1) + + def forward(self, img, depth_init): + concat = F.cat((img, depth_init), dim=1) + depth_residual = self.res(self.conv3(self.conv2(self.conv1(concat)))) + depth_refined = depth_init + depth_residual + return depth_refined + + +def depth_regression(p, depth_values): + if depth_values.dim() <= 2: + depth_values = depth_values.view(*depth_values.shape, 1, 1) + depth = torch.sum(p * depth_values, 1) + + return depth + + +def cas_mvsnet_loss(inputs, depth_gt_ms, mask_ms, **kwargs): + depth_loss_weights = kwargs.get('dlossw', None) + + total_loss = torch.tensor( + 0.0, + dtype=torch.float32, + device=mask_ms['stage1'].device, + requires_grad=False) + + for (stage_inputs, stage_key) in [(inputs[k], k) for k in inputs.keys() + if 'stage' in k]: + depth_est = stage_inputs['depth'] + depth_gt = depth_gt_ms[stage_key] + mask = mask_ms[stage_key] + mask = mask > 0.5 + + depth_loss = F.smooth_l1_loss( + depth_est[mask], depth_gt[mask], reduction='mean') + + if depth_loss_weights is not None: + stage_idx = int(stage_key.replace('stage', '')) - 1 + total_loss += depth_loss_weights[stage_idx] * depth_loss + else: + total_loss += 1.0 * depth_loss + + return total_loss, depth_loss + + +def get_cur_depth_range_samples(cur_depth, + ndepth, + depth_inteval_pixel, + shape, + max_depth=192.0, + min_depth=0.0): + """ + shape, (B, H, W) + cur_depth: (B, H, W) + return depth_range_values: (B, D, H, W) + """ + cur_depth_min = (cur_depth - ndepth / 2 * depth_inteval_pixel) # (B, H, W) + cur_depth_max = (cur_depth + ndepth / 2 * depth_inteval_pixel) + + assert cur_depth.shape == torch.Size( + shape), 'cur_depth:{}, input shape:{}'.format(cur_depth.shape, shape) + new_interval = (cur_depth_max - cur_depth_min) / (ndepth - 1) # (B, H, W) + + depth_range_samples = cur_depth_min.unsqueeze(1) + ( + torch.arange( + 0, + ndepth, + device=cur_depth.device, + dtype=cur_depth.dtype, + requires_grad=False).reshape(1, -1, 1, 1) + * new_interval.unsqueeze(1)) + + return depth_range_samples + + +def get_depth_range_samples(cur_depth, + ndepth, + depth_inteval_pixel, + device, + dtype, + shape, + max_depth=192.0, + min_depth=0.0): + """ + shape: (B, H, W) + cur_depth: (B, H, W) or (B, D) + return depth_range_samples: (B, D, H, W) + """ + if cur_depth.dim() == 2: + cur_depth_min = cur_depth[:, 0] # (B,) + cur_depth_max = cur_depth[:, -1] + new_interval = (cur_depth_max - cur_depth_min) / (ndepth - 1) # (B, ) + + depth_range_samples = cur_depth_min.unsqueeze(1) + (torch.arange( + 0, ndepth, device=device, dtype=dtype, + requires_grad=False).reshape(1, -1) * new_interval.unsqueeze(1) + ) # noqa # (B, D) + + depth_range_samples = depth_range_samples.unsqueeze(-1).unsqueeze( + -1).repeat(1, 1, shape[1], shape[2]) # (B, D, H, W) + + else: + + depth_range_samples = get_cur_depth_range_samples( + cur_depth, ndepth, depth_inteval_pixel, shape, max_depth, + min_depth) + + return depth_range_samples diff --git a/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/utils.py b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aeab02b36dbbb78dcf070e46333dd0f86c8cdd4c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_mvs_depth_estimation/utils.py @@ -0,0 +1,118 @@ +# The implementation here is modified based on https://github.com/xy-guo/MVSNet_pytorch +import random + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.utils as vutils + + +# convert a function into recursive style to handle nested dict/list/tuple variables +def make_recursive_func(func): + + def wrapper(vars): + if isinstance(vars, list): + return [wrapper(x) for x in vars] + elif isinstance(vars, tuple): + return tuple([wrapper(x) for x in vars]) + elif isinstance(vars, dict): + return {k: wrapper(v) for k, v in vars.items()} + else: + return func(vars) + + return wrapper + + +@make_recursive_func +def tensor2numpy(vars): + if isinstance(vars, np.ndarray): + return vars + elif isinstance(vars, torch.Tensor): + return vars.detach().cpu().numpy().copy() + else: + raise NotImplementedError( + 'invalid input type {} for tensor2numpy'.format(type(vars))) + + +@make_recursive_func +def numpy2torch(vars): + if isinstance(vars, np.ndarray): + return torch.from_numpy(vars) + elif isinstance(vars, torch.Tensor): + return vars + elif isinstance(vars, str): + return vars + else: + raise NotImplementedError( + 'invalid input type {} for numpy2torch'.format(type(vars))) + + +@make_recursive_func +def tocuda(vars): + if isinstance(vars, torch.Tensor): + return vars.to(torch.device('cuda')) + elif isinstance(vars, str): + return vars + else: + raise NotImplementedError( + 'invalid input type {} for tensor2numpy'.format(type(vars))) + + +def generate_pointcloud(rgb, depth, ply_file, intr, scale=1.0): + """ + Generate a colored point cloud in PLY format from a color and a depth image. + + Input: + rgb_file -- filename of color image + depth_file -- filename of depth image + ply_file -- filename of ply file + + """ + fx, fy, cx, cy = intr[0, 0], intr[1, 1], intr[0, 2], intr[1, 2] + points = [] + for v in range(rgb.shape[0]): + for u in range(rgb.shape[1]): + color = rgb[v, u] # rgb.getpixel((u, v)) + Z = depth[v, u] / scale + if Z == 0: + continue + X = (u - cx) * Z / fx + Y = (v - cy) * Z / fy + points.append('%f %f %f %d %d %d 0\n' % + (X, Y, Z, color[0], color[1], color[2])) + file = open(ply_file, 'w') + file.write('''ply + format ascii 1.0 + element vertex %d + property float x + property float y + property float z + property uchar red + property uchar green + property uchar blue + property uchar alpha + end_header + %s + ''' % (len(points), ''.join(points))) + file.close() + + +def write_cam(file, cam): + f = open(file, 'w') + f.write('extrinsic\n') + for i in range(0, 4): + for j in range(0, 4): + f.write(str(cam[0][i][j]) + ' ') + f.write('\n') + f.write('\n') + + f.write('intrinsic\n') + for i in range(0, 3): + for j in range(0, 3): + f.write(str(cam[1][i][j]) + ' ') + f.write('\n') + + f.write('\n' + str(cam[1][3][0]) + ' ' + str(cam[1][3][1]) + ' ' + + str(cam[1][3][2]) + ' ' + str(cam[1][3][3]) + '\n') + + f.close() diff --git a/runtime/Lib/site_packages/models/cv/image_paintbyexample/__init__.py b/runtime/Lib/site_packages/models/cv/image_paintbyexample/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b58d4305469534996adc6737980318f4001c5966 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_paintbyexample/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import StablediffusionPaintbyexample + +else: + _import_structure = { + 'model': ['StablediffusionPaintbyexample'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_paintbyexample/model.py b/runtime/Lib/site_packages/models/cv/image_paintbyexample/model.py new file mode 100644 index 0000000000000000000000000000000000000000..1cddafc6ae5d0778b3cd7c5b243979a749ced3f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_paintbyexample/model.py @@ -0,0 +1,49 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Optional, Union + +import torch +from omegaconf import OmegaConf +from paint_ldm.util import instantiate_from_config + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +LOGGER = get_logger() + + +def load_model_from_config(config, ckpt, verbose=False): + LOGGER.info(f'Loading model from {ckpt}') + pl_sd = torch.load(ckpt, map_location='cpu') + if 'global_step' in pl_sd: + LOGGER.info(f"Global Step: {pl_sd['global_step']}") + sd = pl_sd['state_dict'] + model = instantiate_from_config(config.model) + m, u = model.load_state_dict(sd, strict=False) + if len(m) > 0 and verbose: + LOGGER.info('missing keys:') + LOGGER.info(m) + if len(u) > 0 and verbose: + LOGGER.info('unexpected keys:') + LOGGER.info(u) + + return model + + +@MODELS.register_module( + Tasks.image_paintbyexample, module_name=Models.image_paintbyexample) +class StablediffusionPaintbyexample(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + super().__init__(model_dir, **kwargs) + + config = OmegaConf.load(os.path.join(model_dir, 'v1.yaml')) + model = load_model_from_config( + config, os.path.join(model_dir, 'pytorch_model.pt')) + self.model = model + + def forward(self, inputs): + return self.model(inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2b2be4b71a98ecb5169062541590087d60d60f70 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .panseg_model import SwinLPanopticSegmentation + +else: + _import_structure = { + 'panseg_model': ['SwinLPanopticSegmentation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/panseg_model.py b/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/panseg_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f44c01e8edc36c531ccda92235cc4eaaa1dff8f6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_panoptic_segmentation/panseg_model.py @@ -0,0 +1,53 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_segmentation, module_name=Models.panoptic_segmentation) +class SwinLPanopticSegmentation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + from mmcv.runner import load_checkpoint + import mmcv + from mmdet.models import build_detector + + config = osp.join(model_dir, 'config.py') + + cfg = mmcv.Config.fromfile(config) + if 'pretrained' in cfg.model: + cfg.model.pretrained = None + elif 'init_cfg' in cfg.model.backbone: + cfg.model.backbone.init_cfg = None + + # build model + cfg.model.train_cfg = None + self.model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoint = load_checkpoint( + self.model, model_path, map_location='cpu') + + self.CLASSES = checkpoint['meta']['CLASSES'] + self.num_classes = len(self.CLASSES) + self.cfg = cfg + + def inference(self, data): + """data is dict,contain img and img_metas,follow with mmdet.""" + + with torch.no_grad(): + results = self.model(return_loss=False, rescale=True, **data) + return results + + def forward(self, Inputs): + return self.model(**Inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/__init__.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4014bb15e8708143354cf3069632318f9981c602 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_portrait_enhancement import ImagePortraitEnhancement + +else: + _import_structure = { + 'image_portrait_enhancement': ['ImagePortraitEnhancement'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/align_faces.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/align_faces.py new file mode 100644 index 0000000000000000000000000000000000000000..e6852f8ca172c37a0f38685f55d0be5beea5707f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/align_faces.py @@ -0,0 +1,254 @@ +# Part of the implementation is borrowed and modified from Face-Alignment, +# publicly available at https://github.com/foamliu/Face-Alignment/blob/master/align_faces.py +import cv2 +import numpy as np +from skimage import transform as trans + +from modelscope.utils.logger import get_logger + +logger = get_logger() + +# reference facial points, a list of coordinates (x,y) +REFERENCE_FACIAL_POINTS = [[30.29459953, 51.69630051], + [65.53179932, 51.50139999], + [48.02519989, + 71.73660278], [33.54930115, 92.3655014], + [62.72990036, 92.20410156]] + +DEFAULT_CROP_SIZE = (96, 112) + + +def _umeyama(src, dst, estimate_scale=True, scale=1.0): + """Estimate N-D similarity transformation with or without scaling. + Parameters + ---------- + src : (M, N) array + Source coordinates. + dst : (M, N) array + Destination coordinates. + estimate_scale : bool + Whether to estimate scaling factor. + Returns + ------- + T : (N + 1, N + 1) + The homogeneous similarity transformation matrix. The matrix contains + NaN values only if the problem is not well-conditioned. + References + ---------- + .. [1] "Least-squares estimation of transformation parameters between two + point patterns", Shinji Umeyama, PAMI 1991, :DOI:`10.1109/34.88573` + """ + + num = src.shape[0] + dim = src.shape[1] + + # Compute mean of src and dst. + src_mean = src.mean(axis=0) + dst_mean = dst.mean(axis=0) + + # Subtract mean from src and dst. + src_demean = src - src_mean + dst_demean = dst - dst_mean + + # Eq. (38). + A = dst_demean.T @ src_demean / num + + # Eq. (39). + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + + T = np.eye(dim + 1, dtype=np.double) + + U, S, V = np.linalg.svd(A) + + # Eq. (40) and (43). + rank = np.linalg.matrix_rank(A) + if rank == 0: + return np.nan * T + elif rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = U @ V + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = U @ np.diag(d) @ V + d[dim - 1] = s + else: + T[:dim, :dim] = U @ np.diag(d) @ V + + if estimate_scale: + # Eq. (41) and (42). + scale = 1.0 / src_demean.var(axis=0).sum() * (S @ d) + else: + scale = scale + + T[:dim, dim] = dst_mean - scale * (T[:dim, :dim] @ src_mean.T) + T[:dim, :dim] *= scale + + return T, scale + + +class FaceWarpException(Exception): + + def __str__(self): + return 'In File {}:{}'.format(__file__, super.__str__(self)) + + +def get_reference_facial_points(output_size=None, + inner_padding_factor=0.0, + outer_padding=(0, 0), + default_square=False): + ref_5pts = np.array(REFERENCE_FACIAL_POINTS) + ref_crop_size = np.array(DEFAULT_CROP_SIZE) + + # 0) make the inner region a square + if default_square: + size_diff = max(ref_crop_size) - ref_crop_size + ref_5pts += size_diff / 2 + ref_crop_size += size_diff + + if (output_size and output_size[0] == ref_crop_size[0] + and output_size[1] == ref_crop_size[1]): + return ref_5pts + + if (inner_padding_factor == 0 and outer_padding == (0, 0)): + if output_size is None: + logger.info('No paddings to do: return default reference points') + return ref_5pts + else: + raise FaceWarpException( + 'No paddings to do, output_size must be None or {}'.format( + ref_crop_size)) + + # check output size + if not (0 <= inner_padding_factor <= 1.0): + raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)') + + if ((inner_padding_factor > 0 or outer_padding[0] > 0 + or outer_padding[1] > 0) and output_size is None): + output_size = ref_crop_size * (1 + inner_padding_factor * 2).astype( + np.int32) + output_size += np.array(outer_padding) + logger.info('deduced from paddings, output_size = ', output_size) + + if not (outer_padding[0] < output_size[0] + and outer_padding[1] < output_size[1]): + raise FaceWarpException('Not (outer_padding[0] < output_size[0]' + 'and outer_padding[1] < output_size[1])') + + # 1) pad the inner region according inner_padding_factor + if inner_padding_factor > 0: + size_diff = ref_crop_size * inner_padding_factor * 2 + ref_5pts += size_diff / 2 + ref_crop_size += np.round(size_diff).astype(np.int32) + + # 2) resize the padded inner region + size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2 + + if size_bf_outer_pad[0] * ref_crop_size[1] != size_bf_outer_pad[ + 1] * ref_crop_size[0]: + raise FaceWarpException( + 'Must have (output_size - outer_padding)' + '= some_scale * (crop_size * (1.0 + inner_padding_factor)') + + scale_factor = size_bf_outer_pad[0].astype(np.float32) / ref_crop_size[0] + ref_5pts = ref_5pts * scale_factor + ref_crop_size = size_bf_outer_pad + + # 3) add outer_padding to make output_size + reference_5point = ref_5pts + np.array(outer_padding) + ref_crop_size = output_size + + return reference_5point + + +def get_affine_transform_matrix(src_pts, dst_pts): + tfm = np.float32([[1, 0, 0], [0, 1, 0]]) + n_pts = src_pts.shape[0] + ones = np.ones((n_pts, 1), src_pts.dtype) + src_pts_ = np.hstack([src_pts, ones]) + dst_pts_ = np.hstack([dst_pts, ones]) + + A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_) + + if rank == 3: + tfm = np.float32([[A[0, 0], A[1, 0], A[2, 0]], + [A[0, 1], A[1, 1], A[2, 1]]]) + elif rank == 2: + tfm = np.float32([[A[0, 0], A[1, 0], 0], [A[0, 1], A[1, 1], 0]]) + + return tfm + + +def get_params(reference_pts, facial_pts, align_type): + ref_pts = np.float32(reference_pts) + ref_pts_shp = ref_pts.shape + if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: + raise FaceWarpException( + 'reference_pts.shape must be (K,2) or (2,K) and K>2') + + if ref_pts_shp[0] == 2: + ref_pts = ref_pts.T + + src_pts = np.float32(facial_pts) + src_pts_shp = src_pts.shape + if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: + raise FaceWarpException( + 'facial_pts.shape must be (K,2) or (2,K) and K>2') + + if src_pts_shp[0] == 2: + src_pts = src_pts.T + + if src_pts.shape != ref_pts.shape: + raise FaceWarpException( + 'facial_pts and reference_pts must have the same shape') + + if align_type == 'cv2_affine': + tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) + tfm_inv = cv2.getAffineTransform(ref_pts[0:3], src_pts[0:3]) + elif align_type == 'affine': + tfm = get_affine_transform_matrix(src_pts, ref_pts) + tfm_inv = get_affine_transform_matrix(ref_pts, src_pts) + else: + params, scale = _umeyama(src_pts, ref_pts) + tfm = params[:2, :] + + params, _ = _umeyama(ref_pts, src_pts, False, scale=1.0 / scale) + tfm_inv = params[:2, :] + + return tfm, tfm_inv + + +def warp_and_crop_face(src_img, + facial_pts, + reference_pts=None, + crop_size=(96, 112), + align_type='smilarity'): # smilarity cv2_affine affine + + reference_pts_112 = get_reference_facial_points((112, 112), 0.25, (0, 0), + True) + if reference_pts is None: + if crop_size[0] == 96 and crop_size[1] == 112: + reference_pts = REFERENCE_FACIAL_POINTS + else: + default_square = True # False + inner_padding_factor = 0.25 # 0 + outer_padding = (0, 0) + output_size = crop_size + reference_pts = get_reference_facial_points( + output_size, inner_padding_factor, outer_padding, + default_square) + + tfm, tfm_inv = get_params(reference_pts, facial_pts, align_type) + tfm_112, tfm_inv_112 = get_params(reference_pts_112, facial_pts, + align_type) + + if src_img is not None: + face_img = cv2.warpAffine( + src_img, tfm, (crop_size[0], crop_size[1]), flags=3) + face_img_112 = cv2.warpAffine(src_img, tfm_112, (112, 112), flags=3) + + return face_img, face_img_112, tfm_inv + else: + return tfm, tfm_inv diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/__init__.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/fqa.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/fqa.py new file mode 100644 index 0000000000000000000000000000000000000000..51f2206e592ca2ba11af8a32edd0ccdaf32dae48 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/fqa.py @@ -0,0 +1,58 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import cv2 +import numpy as np +import torch + +from .model_resnet import FaceQuality, ResNet + + +class FQA(object): + + def __init__(self, backbone_path, quality_path, device='cuda', size=112): + self.BACKBONE = ResNet(num_layers=100, feature_dim=512) + self.QUALITY = FaceQuality(512 * 7 * 7) + self.size = size + self.device = device + + self.load_model(backbone_path, quality_path) + + def load_model(self, backbone_path, quality_path): + checkpoint = torch.load(backbone_path, map_location='cpu') + self.load_state_dict(self.BACKBONE, checkpoint) + + checkpoint = torch.load(quality_path, map_location='cpu') + self.load_state_dict(self.QUALITY, checkpoint) + + self.BACKBONE.to(self.device) + self.QUALITY.to(self.device) + self.BACKBONE.eval() + self.QUALITY.eval() + + def load_state_dict(self, model, state_dict): + all_keys = {k for k in state_dict.keys()} + for k in all_keys: + if k.startswith('module.'): + state_dict[k[7:]] = state_dict.pop(k) + model_dict = model.state_dict() + pretrained_dict = { + k: v + for k, v in state_dict.items() + if k in model_dict and v.size() == model_dict[k].size() + } + + model_dict.update(pretrained_dict) + model.load_state_dict(model_dict) + + def get_face_quality(self, img): + img = torch.from_numpy(img).permute(2, 0, 1).unsqueeze(0).flip(1).to( + self.device) + img = (img - 127.5) / 128.0 + + # extract features & predict quality + with torch.no_grad(): + feature, fc = self.BACKBONE(img.to(self.device), True) + s = self.QUALITY(fc)[0] + + return s.cpu().numpy()[0], feature.cpu().numpy()[0] diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/model_resnet.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/model_resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..e0e8e9d5fb99d9f1909f5bba7b3266034b98f082 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/eqface/model_resnet.py @@ -0,0 +1,132 @@ +# The implementation is adopted from FaceQuality, made publicly available under the MIT License +# at https://github.com/deepcam-cn/FaceQuality/blob/master/models/model_resnet.py +import torch +from torch import nn + + +class BottleNeck_IR(nn.Module): + + def __init__(self, in_channel, out_channel, stride, dim_match): + super(BottleNeck_IR, self).__init__() + self.res_layer = nn.Sequential( + nn.BatchNorm2d(in_channel), + nn.Conv2d(in_channel, out_channel, (3, 3), 1, 1, bias=False), + nn.BatchNorm2d(out_channel), nn.PReLU(out_channel), + nn.Conv2d(out_channel, out_channel, (3, 3), stride, 1, bias=False), + nn.BatchNorm2d(out_channel)) + if dim_match: + self.shortcut_layer = None + else: + self.shortcut_layer = nn.Sequential( + nn.Conv2d( + in_channel, + out_channel, + kernel_size=(1, 1), + stride=stride, + bias=False), nn.BatchNorm2d(out_channel)) + + def forward(self, x): + shortcut = x + res = self.res_layer(x) + + if self.shortcut_layer is not None: + shortcut = self.shortcut_layer(x) + + return shortcut + res + + +channel_list = [64, 64, 128, 256, 512] + + +def get_layers(num_layers): + if num_layers == 34: + return [3, 4, 6, 3] + if num_layers == 50: + return [3, 4, 14, 3] + elif num_layers == 100: + return [3, 13, 30, 3] + elif num_layers == 152: + return [3, 8, 36, 3] + + +class ResNet(nn.Module): + + def __init__(self, + num_layers=100, + feature_dim=512, + drop_ratio=0.4, + channel_list=channel_list): + super(ResNet, self).__init__() + assert num_layers in [34, 50, 100, 152] + layers = get_layers(num_layers) + block = BottleNeck_IR + + self.input_layer = nn.Sequential( + nn.Conv2d( + 3, channel_list[0], (3, 3), stride=1, padding=1, bias=False), + nn.BatchNorm2d(channel_list[0]), nn.PReLU(channel_list[0])) + self.layer1 = self._make_layer( + block, channel_list[0], channel_list[1], layers[0], stride=2) + self.layer2 = self._make_layer( + block, channel_list[1], channel_list[2], layers[1], stride=2) + self.layer3 = self._make_layer( + block, channel_list[2], channel_list[3], layers[2], stride=2) + self.layer4 = self._make_layer( + block, channel_list[3], channel_list[4], layers[3], stride=2) + + self.output_layer = nn.Sequential( + nn.BatchNorm2d(512), nn.Dropout(drop_ratio), nn.Flatten()) + self.feature_layer = nn.Sequential( + nn.Linear(512 * 7 * 7, feature_dim), nn.BatchNorm1d(feature_dim)) + + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0.0) + elif isinstance(m, nn.BatchNorm2d) or isinstance( + m, nn.BatchNorm1d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, in_channel, out_channel, blocks, stride): + layers = [] + layers.append(block(in_channel, out_channel, stride, False)) + for i in range(1, blocks): + layers.append(block(out_channel, out_channel, 1, True)) + return nn.Sequential(*layers) + + def forward(self, x, fc=False): + x = self.input_layer(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.output_layer(x) + feature = self.feature_layer(x) + if fc: + return feature, x + return feature + + +class FaceQuality(nn.Module): + + def __init__(self, feature_dim): + super(FaceQuality, self).__init__() + self.qualtiy = nn.Sequential( + nn.Linear(feature_dim, 512, bias=False), nn.BatchNorm1d(512), + nn.ReLU(inplace=True), nn.Linear(512, 2, bias=False), + nn.Softmax(dim=1)) + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0.0) + elif isinstance(m, nn.BatchNorm2d) or isinstance( + m, nn.BatchNorm1d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + x = self.qualtiy(x) + return x[:, 0:1] diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/gpen.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/gpen.py new file mode 100644 index 0000000000000000000000000000000000000000..86009a4150adf05a74a3310efdd64db88759fa77 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/gpen.py @@ -0,0 +1,815 @@ +# The GPEN implementation is also open-sourced by the authors, +# and available at https://github.com/yangxy/GPEN/blob/main/face_model/gpen_model.py +import functools +import itertools +import math +import operator +import random + +import torch +from torch import nn +from torch.autograd import Function +from torch.nn import functional as F + +from modelscope.models.cv.face_generation.op import (FusedLeakyReLU, + fused_leaky_relu, + upfirdn2d) + + +class PixelNorm(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, input): + return input * torch.rsqrt( + torch.mean(input**2, dim=1, keepdim=True) + 1e-8) + + +def make_kernel(k): + k = torch.tensor(k, dtype=torch.float32) + + if k.ndim == 1: + k = k[None, :] * k[:, None] + + k /= k.sum() + + return k + + +class Upsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) * (factor**2) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=self.factor, down=1, pad=self.pad) + + return out + + +class Downsample(nn.Module): + + def __init__(self, kernel, factor=2): + super().__init__() + + self.factor = factor + kernel = make_kernel(kernel) + self.register_buffer('kernel', kernel) + + p = kernel.shape[0] - factor + + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.pad = (pad0, pad1) + + def forward(self, input): + out = upfirdn2d( + input, self.kernel, up=1, down=self.factor, pad=self.pad) + + return out + + +class Blur(nn.Module): + + def __init__(self, kernel, pad, upsample_factor=1): + super().__init__() + + kernel = make_kernel(kernel) + + if upsample_factor > 1: + kernel = kernel * (upsample_factor**2) + + self.register_buffer('kernel', kernel) + + self.pad = pad + + def forward(self, input): + out = upfirdn2d(input, self.kernel, pad=self.pad) + + return out + + +class EqualConv2d(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + stride=1, + padding=0, + bias=True): + super().__init__() + + self.weight = nn.Parameter( + torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size**2) + + self.stride = stride + self.padding = padding + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + + else: + self.bias = None + + def forward(self, input): + out = F.conv2d( + input, + self.weight * self.scale, + bias=self.bias, + stride=self.stride, + padding=self.padding, + ) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class EqualLinear(nn.Module): + + def __init__(self, + in_dim, + out_dim, + bias=True, + bias_init=0, + lr_mul=1, + activation=None): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) + + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) + + else: + self.bias = None + + self.activation = activation + + self.scale = (1 / math.sqrt(in_dim)) * lr_mul + self.lr_mul = lr_mul + + def forward(self, input): + if self.activation: + out = F.linear(input, self.weight * self.scale) + out = fused_leaky_relu(out, self.bias * self.lr_mul) + + else: + out = F.linear( + input, self.weight * self.scale, bias=self.bias * self.lr_mul) + + return out + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})' + ) + + +class ScaledLeakyReLU(nn.Module): + + def __init__(self, negative_slope=0.2): + super().__init__() + + self.negative_slope = negative_slope + + def forward(self, input): + out = F.leaky_relu(input, negative_slope=self.negative_slope) + + return out * math.sqrt(2) + + +class ModulatedConv2d(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + demodulate=True, + upsample=False, + downsample=False, + blur_kernel=[1, 3, 3, 1], + ): + super().__init__() + + self.eps = 1e-8 + self.kernel_size = kernel_size + self.in_channel = in_channel + self.out_channel = out_channel + self.upsample = upsample + self.downsample = downsample + + if upsample: + factor = 2 + p = (len(blur_kernel) - factor) - (kernel_size - 1) + pad0 = (p + 1) // 2 + factor - 1 + pad1 = p // 2 + 1 + + self.blur = Blur( + blur_kernel, pad=(pad0, pad1), upsample_factor=factor) + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + self.blur = Blur(blur_kernel, pad=(pad0, pad1)) + + fan_in = in_channel * kernel_size**2 + self.scale = 1 / math.sqrt(fan_in) + self.padding = kernel_size // 2 + + self.weight = nn.Parameter( + torch.randn(1, out_channel, in_channel, kernel_size, kernel_size)) + + self.modulation = EqualLinear(style_dim, in_channel, bias_init=1) + + self.demodulate = demodulate + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, ' + f'upsample={self.upsample}, downsample={self.downsample})') + + def forward(self, input, style): + batch, in_channel, height, width = input.shape + + style = self.modulation(style).view(batch, 1, in_channel, 1, 1) + weight = self.scale * self.weight * style + + if self.demodulate: + demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8) + weight = weight * demod.view(batch, self.out_channel, 1, 1, 1) + + weight = weight.view(batch * self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + + if self.upsample: + input = input.view(1, batch * in_channel, height, width) + weight = weight.view(batch, self.out_channel, in_channel, + self.kernel_size, self.kernel_size) + weight = weight.transpose(1, 2).reshape(batch * in_channel, + self.out_channel, + self.kernel_size, + self.kernel_size) + out = F.conv_transpose2d( + input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + out = self.blur(out) + + elif self.downsample: + input = self.blur(input) + _, _, height, width = input.shape + input = input.view(1, batch * in_channel, height, width) + out = F.conv2d(input, weight, padding=0, stride=2, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + else: + input = input.view(1, batch * in_channel, height, width) + out = F.conv2d(input, weight, padding=self.padding, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + + return out + + +class NoiseInjection(nn.Module): + + def __init__(self, isconcat=True): + super().__init__() + + self.isconcat = isconcat + self.weight = nn.Parameter(torch.zeros(1)) + + def forward(self, image, noise=None): + if noise is None: + batch, channel, height, width = image.shape + noise = image.new_empty(batch, channel, height, width).normal_() + + if self.isconcat: + return torch.cat((image, self.weight * noise), dim=1) + else: + return image + self.weight * noise + + +class ConstantInput(nn.Module): + + def __init__(self, channel, size=4): + super().__init__() + + self.input = nn.Parameter(torch.randn(1, channel, size, size)) + + def forward(self, input): + batch = input.shape[0] + out = self.input.repeat(batch, 1, 1, 1) + + return out + + +class StyledConv(nn.Module): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=False, + blur_kernel=[1, 3, 3, 1], + demodulate=True, + isconcat=True, + ): + super().__init__() + + self.conv = ModulatedConv2d( + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=upsample, + blur_kernel=blur_kernel, + demodulate=demodulate, + ) + + self.noise = NoiseInjection(isconcat) + # self.bias = nn.Parameter(torch.zeros(1, out_channel, 1, 1)) + # self.activate = ScaledLeakyReLU(0.2) + feat_multiplier = 2 if isconcat else 1 + self.activate = FusedLeakyReLU(out_channel * feat_multiplier) + + def forward(self, input, style, noise=None): + out = self.conv(input, style) + out = self.noise(out, noise=noise) + # out = out + self.bias + out = self.activate(out) + + return out + + +class ToRGB(nn.Module): + + def __init__(self, + in_channel, + style_dim, + upsample=True, + blur_kernel=[1, 3, 3, 1]): + super().__init__() + + if upsample: + self.upsample = Upsample(blur_kernel) + + self.conv = ModulatedConv2d( + in_channel, 3, 1, style_dim, demodulate=False) + self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1)) + + def forward(self, input, style, skip=None): + out = self.conv(input, style) + out = out + self.bias + + if skip is not None: + skip = self.upsample(skip) + + out = out + skip + + return out + + +class Generator(nn.Module): + + def __init__( + self, + size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + isconcat=True, + narrow=1, + ): + super().__init__() + + self.size = size + self.n_mlp = n_mlp + self.style_dim = style_dim + self.feat_multiplier = 2 if isconcat else 1 + + layers = [PixelNorm()] + + for i in range(n_mlp): + layers.append( + EqualLinear( + style_dim, + style_dim, + lr_mul=lr_mlp, + activation='fused_lrelu')) + + self.style = nn.Sequential(*layers) + + self.channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + self.input = ConstantInput(self.channels[4]) + self.conv1 = StyledConv( + self.channels[4], + self.channels[4], + 3, + style_dim, + blur_kernel=blur_kernel, + isconcat=isconcat) + self.to_rgb1 = ToRGB( + self.channels[4] * self.feat_multiplier, style_dim, upsample=False) + + self.log_size = int(math.log(size, 2)) + + self.convs = nn.ModuleList() + self.upsamples = nn.ModuleList() + self.to_rgbs = nn.ModuleList() + + in_channel = self.channels[4] + + for i in range(3, self.log_size + 1): + out_channel = self.channels[2**i] + + self.convs.append( + StyledConv( + in_channel * self.feat_multiplier, + out_channel, + 3, + style_dim, + upsample=True, + blur_kernel=blur_kernel, + isconcat=isconcat, + )) + + self.convs.append( + StyledConv( + out_channel * self.feat_multiplier, + out_channel, + 3, + style_dim, + blur_kernel=blur_kernel, + isconcat=isconcat)) + + self.to_rgbs.append( + ToRGB(out_channel * self.feat_multiplier, style_dim)) + + in_channel = out_channel + + self.n_latent = self.log_size * 2 - 2 + + def make_noise(self): + device = self.input.input.device + + noises = [torch.randn(1, 1, 2**2, 2**2, device=device)] + + for i in range(3, self.log_size + 1): + for _ in range(2): + noises.append(torch.randn(1, 1, 2**i, 2**i, device=device)) + + return noises + + def mean_latent(self, n_latent): + latent_in = torch.randn( + n_latent, self.style_dim, device=self.input.input.device) + latent = self.style(latent_in).mean(0, keepdim=True) + + return latent + + def get_latent(self, input): + return self.style(input) + + def forward( + self, + styles, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + noise=None, + ): + if not input_is_latent: + styles = [self.style(s) for s in styles] + + if noise is None: + ''' + noise = [None] * (2 * (self.log_size - 2) + 1) + ''' + noise = [] + batch = styles[0].shape[0] + for i in range(self.n_mlp + 1): + size = 2**(i + 2) + noise.append( + torch.randn( + batch, + self.channels[size], + size, + size, + device=styles[0].device)) + + if truncation < 1: + style_t = [] + + for style in styles: + style_t.append(truncation_latent + + truncation * (style - truncation_latent)) + + styles = style_t + + if len(styles) < 2: + inject_index = self.n_latent + + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + + else: + if inject_index is None: + inject_index = random.randint(1, self.n_latent - 1) + + latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) + latent2 = styles[1].unsqueeze(1).repeat( + 1, self.n_latent - inject_index, 1) + + latent = torch.cat([latent, latent2], 1) + + out = self.input(latent) + out = self.conv1(out, latent[:, 0], noise=noise[0]) + + skip = self.to_rgb1(out, latent[:, 1]) + + i = 1 + for conv1, conv2, noise1, noise2, to_rgb in zip( + self.convs[::2], self.convs[1::2], noise[1::2], noise[2::2], + self.to_rgbs): + out = conv1(out, latent[:, i], noise=noise1) + out = conv2(out, latent[:, i + 1], noise=noise2) + skip = to_rgb(out, latent[:, i + 2], skip) + + i += 2 + + image = skip + + if return_latents: + return image, latent + + else: + return image, None + + +class ConvLayer(nn.Sequential): + + def __init__( + self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + ): + layers = [] + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + layers.append(Blur(blur_kernel, pad=(pad0, pad1))) + + stride = 2 + self.padding = 0 + + else: + stride = 1 + self.padding = kernel_size // 2 + + layers.append( + EqualConv2d( + in_channel, + out_channel, + kernel_size, + padding=self.padding, + stride=stride, + bias=bias and not activate, + )) + + if activate: + if bias: + layers.append(FusedLeakyReLU(out_channel)) + + else: + layers.append(ScaledLeakyReLU(0.2)) + + super().__init__(*layers) + + +class ResBlock(nn.Module): + + def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + self.conv1 = ConvLayer(in_channel, in_channel, 3) + self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True) + + self.skip = ConvLayer( + in_channel, + out_channel, + 1, + downsample=True, + activate=False, + bias=False) + + def forward(self, input): + out = self.conv1(input) + out = self.conv2(out) + + skip = self.skip(input) + out = (out + skip) / math.sqrt(2) + + return out + + +class FullGenerator(nn.Module): + + def __init__( + self, + size, + style_dim, + n_mlp, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + isconcat=True, + narrow=1, + ): + super().__init__() + channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + self.log_size = int(math.log(size, 2)) + self.generator = Generator( + size, + style_dim, + n_mlp, + channel_multiplier=channel_multiplier, + blur_kernel=blur_kernel, + lr_mlp=lr_mlp, + isconcat=isconcat, + narrow=narrow) + + conv = [ConvLayer(3, channels[size], 1)] + self.ecd0 = nn.Sequential(*conv) + in_channel = channels[size] + + self.names = ['ecd%d' % i for i in range(self.log_size - 1)] + for i in range(self.log_size, 2, -1): + out_channel = channels[2**(i - 1)] + # conv = [ResBlock(in_channel, out_channel, blur_kernel)] + conv = [ConvLayer(in_channel, out_channel, 3, downsample=True)] + setattr(self, self.names[self.log_size - i + 1], + nn.Sequential(*conv)) + in_channel = out_channel + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, style_dim, activation='fused_lrelu')) + + def forward( + self, + inputs, + return_latents=False, + inject_index=None, + truncation=1, + truncation_latent=None, + input_is_latent=False, + ): + noise = [] + for i in range(self.log_size - 1): + ecd = getattr(self, self.names[i]) + inputs = ecd(inputs) + noise.append(inputs) + inputs = inputs.view(inputs.shape[0], -1) + outs = self.final_linear(inputs) + noise = list( + itertools.chain.from_iterable( + itertools.repeat(x, 2) for x in noise))[::-1] + outs = self.generator([outs], + return_latents, + inject_index, + truncation, + truncation_latent, + input_is_latent, + noise=noise[1:]) + return outs + + +class Discriminator(nn.Module): + + def __init__(self, + size, + channel_multiplier=2, + blur_kernel=[1, 3, 3, 1], + narrow=1): + super().__init__() + + channels = { + 4: int(512 * narrow), + 8: int(512 * narrow), + 16: int(512 * narrow), + 32: int(512 * narrow), + 64: int(256 * channel_multiplier * narrow), + 128: int(128 * channel_multiplier * narrow), + 256: int(64 * channel_multiplier * narrow), + 512: int(32 * channel_multiplier * narrow), + 1024: int(16 * channel_multiplier * narrow), + 2048: int(8 * channel_multiplier * narrow) + } + + convs = [ConvLayer(3, channels[size], 1)] + + log_size = int(math.log(size, 2)) + + in_channel = channels[size] + + for i in range(log_size, 2, -1): + out_channel = channels[2**(i - 1)] + + convs.append(ResBlock(in_channel, out_channel, blur_kernel)) + + in_channel = out_channel + + self.convs = nn.Sequential(*convs) + + self.stddev_group = 4 + self.stddev_feat = 1 + + self.final_conv = ConvLayer(in_channel + 1, channels[4], 3) + self.final_linear = nn.Sequential( + EqualLinear( + channels[4] * 4 * 4, channels[4], activation='fused_lrelu'), + EqualLinear(channels[4], 1), + ) + + def forward(self, input): + out = self.convs(input) + + batch, channel, height, width = out.shape + group = min(batch, self.stddev_group) + stddev = out.view(group, -1, self.stddev_feat, + channel // self.stddev_feat, height, width) + stddev = torch.sqrt(stddev.var(0, unbiased=False) + 1e-8) + stddev = stddev.mean([2, 3, 4], keepdims=True).squeeze(2) + stddev = stddev.repeat(group, 1, height, width) + out = torch.cat([out, stddev], 1) + + out = self.final_conv(out) + + out = out.view(batch, -1) + out = self.final_linear(out) + return out diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/image_portrait_enhancement.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/image_portrait_enhancement.py new file mode 100644 index 0000000000000000000000000000000000000000..c442562ba3e5a481d219c60fbf7f8fff3c9ce6bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/image_portrait_enhancement.py @@ -0,0 +1,204 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os.path as osp +from typing import Any, Dict, List, Union + +import torch +import torch.nn.functional as F +from torch import autograd, nn + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .gpen import Discriminator, FullGenerator +from .losses.losses import IDLoss, L1Loss + +logger = get_logger() + +__all__ = ['ImagePortraitEnhancement'] + + +@MODELS.register_module( + Tasks.image_portrait_enhancement, module_name=Models.gpen) +class ImagePortraitEnhancement(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the face enhancement model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + self.size = 256 + self.style_dim = 512 + self.n_mlp = 8 + self.mean_path_length = 0 + self.accum = 0.5**(32 / (10 * 1000)) + + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + + self.l1_loss = L1Loss() + self.id_loss = IDLoss(f'{model_dir}/arcface/model_ir_se50.pth', + self._device) + self.generator = FullGenerator( + self.size, self.style_dim, self.n_mlp, + isconcat=True).to(self._device) + self.g_ema = FullGenerator( + self.size, self.style_dim, self.n_mlp, + isconcat=True).to(self._device) + self.discriminator = Discriminator(self.size).to(self._device) + + if self.size == 512: + self.load_pretrained(model_dir) + + def load_pretrained(self, model_dir): + g_path = f'{model_dir}/{ModelFile.TORCH_MODEL_FILE}' + g_dict = torch.load(g_path, map_location=torch.device('cpu')) + self.generator.load_state_dict(g_dict) + self.g_ema.load_state_dict(g_dict) + + d_path = f'{model_dir}/net_d.pt' + d_dict = torch.load(d_path, map_location=torch.device('cpu')) + self.discriminator.load_state_dict(d_dict) + + logger.info('load model done.') + + def accumulate(self): + par1 = dict(self.g_ema.named_parameters()) + par2 = dict(self.generator.named_parameters()) + + for k in par1.keys(): + par1[k].data.mul_(self.accum).add_(1 - self.accum, par2[k].data) + + def requires_grad(self, model, flag=True): + for p in model.parameters(): + p.requires_grad = flag + + def d_logistic_loss(self, real_pred, fake_pred): + real_loss = F.softplus(-real_pred) + fake_loss = F.softplus(fake_pred) + + return real_loss.mean() + fake_loss.mean() + + def d_r1_loss(self, real_pred, real_img): + grad_real, = autograd.grad( + outputs=real_pred.sum(), inputs=real_img, create_graph=True) + grad_penalty = grad_real.pow(2).view(grad_real.shape[0], + -1).sum(1).mean() + + return grad_penalty + + def g_nonsaturating_loss(self, + fake_pred, + fake_img=None, + real_img=None, + input_img=None): + loss = F.softplus(-fake_pred).mean() + loss_l1 = self.l1_loss(fake_img, real_img) + loss_id, __, __ = self.id_loss(fake_img, real_img, input_img) + loss_id = 0 + loss += 1.0 * loss_l1 + 1.0 * loss_id + + return loss + + def g_path_regularize(self, + fake_img, + latents, + mean_path_length, + decay=0.01): + noise = torch.randn_like(fake_img) / math.sqrt( + fake_img.shape[2] * fake_img.shape[3]) + grad, = autograd.grad( + outputs=(fake_img * noise).sum(), + inputs=latents, + create_graph=True) + path_lengths = torch.sqrt(grad.pow(2).sum(2).mean(1)) + + path_mean = mean_path_length + decay * ( + path_lengths.mean() - mean_path_length) + + path_penalty = (path_lengths - path_mean).pow(2).mean() + + return path_penalty, path_mean.detach(), path_lengths + + @torch.no_grad() + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + preds, _ = self.generator(input) + preds = list(torch.split(preds, 1, 0)) + targets = list(torch.split(target, 1, 0)) + + preds = [((pred.data * 0.5 + 0.5) * 255.).squeeze(0).type( + torch.uint8).permute(1, 2, 0).cpu().numpy() for pred in preds] + targets = [((target.data * 0.5 + 0.5) * 255.).squeeze(0).type( + torch.uint8).permute(1, 2, 0).cpu().numpy() for target in targets] + + return {'pred': preds, 'target': targets} + + def _train_forward_d(self, input: Tensor, target: Tensor) -> Tensor: + self.requires_grad(self.generator, False) + self.requires_grad(self.discriminator, True) + + preds, _ = self.generator(input) + fake_pred = self.discriminator(preds) + real_pred = self.discriminator(target) + + d_loss = self.d_logistic_loss(real_pred, fake_pred) + + return d_loss + + def _train_forward_d_r1(self, input: Tensor, target: Tensor) -> Tensor: + input.requires_grad = True + target.requires_grad = True + real_pred = self.discriminator(target) + r1_loss = self.d_r1_loss(real_pred, target) + + return r1_loss + + def _train_forward_g(self, input: Tensor, target: Tensor) -> Tensor: + self.requires_grad(self.generator, True) + self.requires_grad(self.discriminator, False) + + preds, _ = self.generator(input) + fake_pred = self.discriminator(preds) + + g_loss = self.g_nonsaturating_loss(fake_pred, preds, target, input) + + return g_loss + + def _train_forward_g_path(self, input: Tensor, target: Tensor) -> Tensor: + fake_img, latents = self.generator(input, return_latents=True) + + path_loss, self.mean_path_length, path_lengths = self.g_path_regularize( + fake_img, latents, self.mean_path_length) + + return path_loss + + @torch.no_grad() + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'outputs': (self.generator(input)[0] * 0.5 + 0.5).clamp(0, 1)} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + for key, value in input.items(): + input[key] = input[key].to(self._device) + + if 'target' in input: + return self._evaluate_postprocess(**input) + else: + return self._inference_forward(**input) diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/__init__.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/helpers.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..86f6f227d580f320720a651bc7ce948794258a7a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/helpers.py @@ -0,0 +1,131 @@ +# The implementation is adopted from InsightFace_Pytorch, +# made publicly available under the MIT License at https://github.com/TreB1eN/InsightFace_Pytorch/blob/master/model.py +from collections import namedtuple + +import torch +from torch.nn import (AdaptiveAvgPool2d, BatchNorm2d, Conv2d, MaxPool2d, + Module, PReLU, ReLU, Sequential, Sigmoid) + + +class Flatten(Module): + + def forward(self, input): + return input.view(input.size(0), -1) + + +def l2_norm(input, axis=1): + norm = torch.norm(input, 2, axis, True) + output = torch.div(input, norm) + return output + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + """ A named tuple describing a ResNet block. """ + + +def get_block(in_channel, depth, num_units, stride=2): + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + else: + raise ValueError( + 'Invalid number of layers: {}. Must be one of [50, 100, 152]'. + format(num_layers)) + return blocks + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + return module_input * x + + +class bottleneck_IR(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut + + +class bottleneck_IR_SE(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR_SE, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth), SEModule(depth, 16)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/losses.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..0f5198c38b4e8a4e0fb6a3b94dfae48ab772bdba --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/losses.py @@ -0,0 +1,92 @@ +# The GPEN implementation is also open-sourced by the authors, +# and available at https://github.com/yangxy/GPEN/tree/main/training/loss/id_loss.py +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .model_irse import Backbone + + +class L1Loss(nn.Module): + """L1 (mean absolute error, MAE) loss. + + Args: + loss_weight (float): Loss weight for L1 loss. Default: 1.0. + reduction (str): Specifies the reduction to apply to the output. + Supported choices are 'none' | 'mean' | 'sum'. Default: 'mean'. + """ + + def __init__(self, loss_weight=1.0, reduction='mean'): + super(L1Loss, self).__init__() + if reduction not in ['none', 'mean', 'sum']: + raise ValueError( + f'Unsupported reduction mode: {reduction}. Supported ones are: {_reduction_modes}' + ) + + self.loss_weight = loss_weight + self.reduction = reduction + + def forward(self, pred, target, weight=None, **kwargs): + """ + Args: + pred (Tensor): of shape (N, C, H, W). Predicted tensor. + target (Tensor): of shape (N, C, H, W). Ground truth tensor. + weight (Tensor, optional): of shape (N, C, H, W). Element-wise weights. Default: None. + """ + return self.loss_weight * F.l1_loss( + pred, target, reduction=self.reduction) + + +class IDLoss(nn.Module): + + def __init__(self, model_path, device='cuda', ckpt_dict=None): + super(IDLoss, self).__init__() + print('Loading ResNet ArcFace') + self.facenet = Backbone( + input_size=112, num_layers=50, drop_ratio=0.6, + mode='ir_se').to(device) + if ckpt_dict is None: + self.facenet.load_state_dict( + torch.load(model_path, map_location=torch.device('cpu'))) + else: + self.facenet.load_state_dict(ckpt_dict) + self.pool = torch.nn.AdaptiveAvgPool2d((256, 256)) + self.face_pool = torch.nn.AdaptiveAvgPool2d((112, 112)) + self.facenet.eval() + + def extract_feats(self, x): + _, _, h, w = x.shape + assert h == w + if h != 256: + x = self.pool(x) + x = x[:, :, 35:-33, 32:-36] # crop roi + x = self.face_pool(x) + x_feats = self.facenet(x) + return x_feats + + @torch.no_grad() + def forward(self, y_hat, y, x): + n_samples = x.shape[0] + x_feats = self.extract_feats(x) + y_feats = self.extract_feats(y) # Otherwise use the feature from there + y_hat_feats = self.extract_feats(y_hat) + y_feats = y_feats.detach() + loss = 0 + sim_improvement = 0 + id_logs = [] + count = 0 + for i in range(n_samples): + diff_target = y_hat_feats[i].dot(y_feats[i]) + diff_input = y_hat_feats[i].dot(x_feats[i]) + diff_views = y_feats[i].dot(x_feats[i]) + id_logs.append({ + 'diff_target': float(diff_target), + 'diff_input': float(diff_input), + 'diff_views': float(diff_views) + }) + loss += 1 - diff_target + id_diff = float(diff_target) - float(diff_views) + sim_improvement += id_diff + count += 1 + + return loss / count, sim_improvement / count, id_logs diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/model_irse.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/model_irse.py new file mode 100644 index 0000000000000000000000000000000000000000..00dc7c527355dfa78b86b2527f38bf2005be5fb9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/losses/model_irse.py @@ -0,0 +1,94 @@ +# The implementation is adopted from InsightFace_Pytorch, +# made publicly available under the MIT License at https://github.com/TreB1eN/InsightFace_Pytorch/blob/master/model.py +from torch.nn import (BatchNorm1d, BatchNorm2d, Conv2d, Dropout, Linear, + Module, PReLU, Sequential) + +from .helpers import (Flatten, bottleneck_IR, bottleneck_IR_SE, get_blocks, + l2_norm) + + +class Backbone(Module): + + def __init__(self, + input_size, + num_layers, + mode='ir', + drop_ratio=0.4, + affine=True): + super(Backbone, self).__init__() + assert input_size in [112, 224], 'input_size should be 112 or 224' + assert num_layers in [50, 100, + 152], 'num_layers should be 50, 100 or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = bottleneck_IR + elif mode == 'ir_se': + unit_module = bottleneck_IR_SE + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + if input_size == 112: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(drop_ratio), Flatten(), + Linear(512 * 7 * 7, 512), BatchNorm1d(512, affine=affine)) + else: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(drop_ratio), Flatten(), + Linear(512 * 14 * 14, 512), BatchNorm1d(512, affine=affine)) + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return l2_norm(x) + + +def IR_50(input_size): + """Constructs a ir-50 model.""" + model = Backbone( + input_size, num_layers=50, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_101(input_size): + """Constructs a ir-101 model.""" + model = Backbone( + input_size, num_layers=100, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_152(input_size): + """Constructs a ir-152 model.""" + model = Backbone( + input_size, num_layers=152, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_50(input_size): + """Constructs a ir_se-50 model.""" + model = Backbone( + input_size, num_layers=50, mode='ir_se', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_101(input_size): + """Constructs a ir_se-101 model.""" + model = Backbone( + input_size, num_layers=100, mode='ir_se', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_152(input_size): + """Constructs a ir_se-152 model.""" + model = Backbone( + input_size, num_layers=152, mode='ir_se', drop_ratio=0.4, affine=False) + return model diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/__init__.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/detection.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/detection.py new file mode 100644 index 0000000000000000000000000000000000000000..e377d3fc714cb7f88a180a29d2e7f9110085fe5d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/detection.py @@ -0,0 +1,218 @@ +# The GPEN implementation is also open-sourced by the authors, +# and available at https://github.com/yangxy/GPEN/blob/main/face_detect/retinaface_detection.py +import os + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F + +from .models.retinaface import RetinaFace +from .utils import PriorBox, decode, decode_landm, py_cpu_nms + +cfg_re50 = { + 'name': 'Resnet50', + 'min_sizes': [[16, 32], [64, 128], [256, 512]], + 'steps': [8, 16, 32], + 'variance': [0.1, 0.2], + 'clip': False, + 'pretrain': False, + 'return_layers': { + 'layer2': 1, + 'layer3': 2, + 'layer4': 3 + }, + 'in_channel': 256, + 'out_channel': 256 +} + + +class RetinaFaceDetection(object): + + def __init__(self, model_path, device='cuda'): + cudnn.benchmark = True + self.model_path = model_path + self.device = device + self.cfg = cfg_re50 + self.net = RetinaFace(cfg=self.cfg) + self.load_model() + self.net = self.net.to(device) + + self.mean = torch.tensor([[[[104]], [[117]], [[123]]]]).to(device) + + def check_keys(self, pretrained_state_dict): + ckpt_keys = set(pretrained_state_dict.keys()) + model_keys = set(self.net.state_dict().keys()) + used_pretrained_keys = model_keys & ckpt_keys + assert len( + used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint' + return True + + def remove_prefix(self, state_dict, prefix): + new_state_dict = dict() + # remove unnecessary 'module.' + for k, v in state_dict.items(): + if k.startswith(prefix): + new_state_dict[k[len(prefix):]] = v + else: + new_state_dict[k] = v + return new_state_dict + + def load_model(self, load_to_cpu=False): + pretrained_dict = torch.load( + self.model_path, map_location=torch.device('cpu')) + if 'state_dict' in pretrained_dict.keys(): + pretrained_dict = self.remove_prefix(pretrained_dict['state_dict'], + 'module.') + else: + pretrained_dict = self.remove_prefix(pretrained_dict, 'module.') + self.check_keys(pretrained_dict) + self.net.load_state_dict(pretrained_dict, strict=False) + self.net.eval() + + def detect(self, + img_raw, + resize=1, + confidence_threshold=0.9, + nms_threshold=0.4, + top_k=5000, + keep_top_k=750, + save_image=False): + img = np.float32(img_raw) + + im_height, im_width = img.shape[:2] + ss = 1.0 + # tricky + if max(im_height, im_width) > 1500: + ss = 1000.0 / max(im_height, im_width) + img = cv2.resize(img, (0, 0), fx=ss, fy=ss) + im_height, im_width = img.shape[:2] + + scale = torch.Tensor( + [img.shape[1], img.shape[0], img.shape[1], img.shape[0]]) + img -= (104, 117, 123) + img = img.transpose(2, 0, 1) + img = torch.from_numpy(img).unsqueeze(0) + img = img.to(self.device) + scale = scale.to(self.device) + + loc, conf, landms = self.net(img) # forward pass + del img + + priorbox = PriorBox(self.cfg, image_size=(im_height, im_width)) + priors = priorbox.forward() + priors = priors.to(self.device) + prior_data = priors.data + boxes = decode(loc.data.squeeze(0), prior_data, self.cfg['variance']) + boxes = boxes * scale / resize + boxes = boxes.cpu().numpy() + scores = conf.squeeze(0).data.cpu().numpy()[:, 1] + landms = decode_landm( + landms.data.squeeze(0), prior_data, self.cfg['variance']) + scale1 = torch.Tensor([ + im_width, im_height, im_width, im_height, im_width, im_height, + im_width, im_height, im_width, im_height + ]) + scale1 = scale1.to(self.device) + landms = landms * scale1 / resize + landms = landms.cpu().numpy() + + # ignore low scores + inds = np.where(scores > confidence_threshold)[0] + boxes = boxes[inds] + landms = landms[inds] + scores = scores[inds] + + # keep top-K before NMS + order = scores.argsort()[::-1][:top_k] + boxes = boxes[order] + landms = landms[order] + scores = scores[order] + + # do NMS + dets = np.hstack((boxes, scores[:, np.newaxis])).astype( + np.float32, copy=False) + keep = py_cpu_nms(dets, nms_threshold) + dets = dets[keep, :] + landms = landms[keep] + + # keep top-K faster NMS + dets = dets[:keep_top_k, :] + landms = landms[:keep_top_k, :] + + landms = landms.reshape((-1, 5, 2)) + landms = landms.transpose((0, 2, 1)) + landms = landms.reshape( + -1, + 10, + ) + return dets / ss, landms / ss + + def detect_tensor(self, + img, + resize=1, + confidence_threshold=0.9, + nms_threshold=0.4, + top_k=5000, + keep_top_k=750, + save_image=False): + im_height, im_width = img.shape[-2:] + ss = 1000 / max(im_height, im_width) + img = F.interpolate(img, scale_factor=ss) + im_height, im_width = img.shape[-2:] + scale = torch.Tensor([im_width, im_height, im_width, + im_height]).to(self.device) + img -= self.mean + + loc, conf, landms = self.net(img) # forward pass + + priorbox = PriorBox(self.cfg, image_size=(im_height, im_width)) + priors = priorbox.forward() + priors = priors.to(self.device) + prior_data = priors.data + boxes = decode(loc.data.squeeze(0), prior_data, self.cfg['variance']) + boxes = boxes * scale / resize + boxes = boxes.cpu().numpy() + scores = conf.squeeze(0).data.cpu().numpy()[:, 1] + landms = decode_landm( + landms.data.squeeze(0), prior_data, self.cfg['variance']) + scale1 = torch.Tensor([ + img.shape[3], img.shape[2], img.shape[3], img.shape[2], + img.shape[3], img.shape[2], img.shape[3], img.shape[2], + img.shape[3], img.shape[2] + ]) + scale1 = scale1.to(self.device) + landms = landms * scale1 / resize + landms = landms.cpu().numpy() + + # ignore low scores + inds = np.where(scores > confidence_threshold)[0] + boxes = boxes[inds] + landms = landms[inds] + scores = scores[inds] + + # keep top-K before NMS + order = scores.argsort()[::-1][:top_k] + boxes = boxes[order] + landms = landms[order] + scores = scores[order] + + # do NMS + dets = np.hstack((boxes, scores[:, np.newaxis])).astype( + np.float32, copy=False) + keep = py_cpu_nms(dets, nms_threshold) + dets = dets[keep, :] + landms = landms[keep] + + # keep top-K faster NMS + dets = dets[:keep_top_k, :] + landms = landms[:keep_top_k, :] + + landms = landms.reshape((-1, 5, 2)) + landms = landms.transpose((0, 2, 1)) + landms = landms.reshape( + -1, + 10, + ) + return dets / ss, landms / ss diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/__init__.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/net.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/net.py new file mode 100644 index 0000000000000000000000000000000000000000..24451e96c7368c20066301d36689cd05a8d08474 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/net.py @@ -0,0 +1,150 @@ +# The implementation is adopted from Pytorch_Retinaface, made pubicly available under the MIT License +# at https://github.com/biubug6/Pytorch_Retinaface/tree/master/models/net.py +import time + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models +import torchvision.models._utils as _utils +from torch.autograd import Variable + + +def conv_bn(inp, oup, stride=1, leaky=0): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True)) + + +def conv_bn_no_relu(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + ) + + +def conv_bn1X1(inp, oup, stride, leaky=0): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, stride, padding=0, bias=False), + nn.BatchNorm2d(oup), nn.LeakyReLU(negative_slope=leaky, inplace=True)) + + +def conv_dw(inp, oup, stride, leaky=0.1): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + ) + + +class SSH(nn.Module): + + def __init__(self, in_channel, out_channel): + super(SSH, self).__init__() + assert out_channel % 4 == 0 + leaky = 0 + if (out_channel <= 64): + leaky = 0.1 + self.conv3X3 = conv_bn_no_relu(in_channel, out_channel // 2, stride=1) + + self.conv5X5_1 = conv_bn( + in_channel, out_channel // 4, stride=1, leaky=leaky) + self.conv5X5_2 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + self.conv7X7_2 = conv_bn( + out_channel // 4, out_channel // 4, stride=1, leaky=leaky) + self.conv7x7_3 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + def forward(self, input): + conv3X3 = self.conv3X3(input) + + conv5X5_1 = self.conv5X5_1(input) + conv5X5 = self.conv5X5_2(conv5X5_1) + + conv7X7_2 = self.conv7X7_2(conv5X5_1) + conv7X7 = self.conv7x7_3(conv7X7_2) + + out = torch.cat([conv3X3, conv5X5, conv7X7], dim=1) + out = F.relu(out) + return out + + +class FPN(nn.Module): + + def __init__(self, in_channels_list, out_channels): + super(FPN, self).__init__() + leaky = 0 + if (out_channels <= 64): + leaky = 0.1 + self.output1 = conv_bn1X1( + in_channels_list[0], out_channels, stride=1, leaky=leaky) + self.output2 = conv_bn1X1( + in_channels_list[1], out_channels, stride=1, leaky=leaky) + self.output3 = conv_bn1X1( + in_channels_list[2], out_channels, stride=1, leaky=leaky) + + self.merge1 = conv_bn(out_channels, out_channels, leaky=leaky) + self.merge2 = conv_bn(out_channels, out_channels, leaky=leaky) + + def forward(self, input): + # names = list(input.keys()) + input = list(input.values()) + + output1 = self.output1(input[0]) + output2 = self.output2(input[1]) + output3 = self.output3(input[2]) + + up3 = F.interpolate( + output3, size=[output2.size(2), output2.size(3)], mode='nearest') + output2 = output2 + up3 + output2 = self.merge2(output2) + + up2 = F.interpolate( + output2, size=[output1.size(2), output1.size(3)], mode='nearest') + output1 = output1 + up2 + output1 = self.merge1(output1) + + out = [output1, output2, output3] + return out + + +class MobileNetV1(nn.Module): + + def __init__(self): + super(MobileNetV1, self).__init__() + self.stage1 = nn.Sequential( + conv_bn(3, 8, 2, leaky=0.1), # 3 + conv_dw(8, 16, 1), # 7 + conv_dw(16, 32, 2), # 11 + conv_dw(32, 32, 1), # 19 + conv_dw(32, 64, 2), # 27 + conv_dw(64, 64, 1), # 43 + ) + self.stage2 = nn.Sequential( + conv_dw(64, 128, 2), # 43 + 16 = 59 + conv_dw(128, 128, 1), # 59 + 32 = 91 + conv_dw(128, 128, 1), # 91 + 32 = 123 + conv_dw(128, 128, 1), # 123 + 32 = 155 + conv_dw(128, 128, 1), # 155 + 32 = 187 + conv_dw(128, 128, 1), # 187 + 32 = 219 + ) + self.stage3 = nn.Sequential( + conv_dw(128, 256, 2), # 219 +3 2 = 241 + conv_dw(256, 256, 1), # 241 + 64 = 301 + ) + self.avg = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(256, 1000) + + def forward(self, x): + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = self.avg(x) + x = x.view(-1, 256) + x = self.fc(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/retinaface.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/retinaface.py new file mode 100644 index 0000000000000000000000000000000000000000..64d959713ef329ff98fb917e29bae391be03e6fb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/models/retinaface.py @@ -0,0 +1,146 @@ +# The implementation is adopted from Pytorch_Retinaface, made pubicly available under the MIT License +# at https://github.com/biubug6/Pytorch_Retinaface/tree/master/models/retinaface.py +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models +import torchvision.models._utils as _utils +import torchvision.models.detection.backbone_utils as backbone_utils + +from .net import FPN, SSH, MobileNetV1 + + +class ClassHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(ClassHead, self).__init__() + self.num_anchors = num_anchors + self.conv1x1 = nn.Conv2d( + inchannels, + self.num_anchors * 2, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 2) + + +class BboxHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(BboxHead, self).__init__() + self.conv1x1 = nn.Conv2d( + inchannels, + num_anchors * 4, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 4) + + +class LandmarkHead(nn.Module): + + def __init__(self, inchannels=512, num_anchors=3): + super(LandmarkHead, self).__init__() + self.conv1x1 = nn.Conv2d( + inchannels, + num_anchors * 10, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x): + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + + return out.view(out.shape[0], -1, 10) + + +class RetinaFace(nn.Module): + + def __init__(self, cfg=None): + """ + :param cfg: Network related settings. + """ + super(RetinaFace, self).__init__() + backbone = None + if cfg['name'] == 'Resnet50': + backbone = models.resnet50(pretrained=cfg['pretrain']) + else: + raise Exception('Invalid name') + + self.body = _utils.IntermediateLayerGetter(backbone, + cfg['return_layers']) + in_channels_stage2 = cfg['in_channel'] + in_channels_list = [ + in_channels_stage2 * 2, + in_channels_stage2 * 4, + in_channels_stage2 * 8, + ] + out_channels = cfg['out_channel'] + self.fpn = FPN(in_channels_list, out_channels) + self.ssh1 = SSH(out_channels, out_channels) + self.ssh2 = SSH(out_channels, out_channels) + self.ssh3 = SSH(out_channels, out_channels) + + self.ClassHead = self._make_class_head( + fpn_num=3, inchannels=cfg['out_channel']) + self.BboxHead = self._make_bbox_head( + fpn_num=3, inchannels=cfg['out_channel']) + self.LandmarkHead = self._make_landmark_head( + fpn_num=3, inchannels=cfg['out_channel']) + + def _make_class_head(self, fpn_num=3, inchannels=64, anchor_num=2): + classhead = nn.ModuleList() + for i in range(fpn_num): + classhead.append(ClassHead(inchannels, anchor_num)) + return classhead + + def _make_bbox_head(self, fpn_num=3, inchannels=64, anchor_num=2): + bboxhead = nn.ModuleList() + for i in range(fpn_num): + bboxhead.append(BboxHead(inchannels, anchor_num)) + return bboxhead + + def _make_landmark_head(self, fpn_num=3, inchannels=64, anchor_num=2): + landmarkhead = nn.ModuleList() + for i in range(fpn_num): + landmarkhead.append(LandmarkHead(inchannels, anchor_num)) + return landmarkhead + + def forward(self, inputs): + out = self.body(inputs) + + # FPN + fpn = self.fpn(out) + + # SSH + feature1 = self.ssh1(fpn[0]) + feature2 = self.ssh2(fpn[1]) + feature3 = self.ssh3(fpn[2]) + features = [feature1, feature2, feature3] + + bbox_regressions = torch.cat( + [self.BboxHead[i](feature) for i, feature in enumerate(features)], + dim=1) + classifications = torch.cat( + [self.ClassHead[i](feature) for i, feature in enumerate(features)], + dim=1) + ldm_regressions = torch.cat( + [self.LandmarkHead[i](feat) for i, feat in enumerate(features)], + dim=1) + + output = (bbox_regressions, F.softmax(classifications, + dim=-1), ldm_regressions) + return output diff --git a/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/utils.py b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..60c9e2dded749ec77a905a60c1ac3e5535002cc2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_portrait_enhancement/retinaface/utils.py @@ -0,0 +1,123 @@ +# -------------------------------------------------------- +# Modified from https://github.com/biubug6/Pytorch_Retinaface +# -------------------------------------------------------- + +from itertools import product as product +from math import ceil + +import numpy as np +import torch + + +class PriorBox(object): + + def __init__(self, cfg, image_size=None, phase='train'): + super(PriorBox, self).__init__() + self.min_sizes = cfg['min_sizes'] + self.steps = cfg['steps'] + self.clip = cfg['clip'] + self.image_size = image_size + self.feature_maps = [[ + ceil(self.image_size[0] / step), + ceil(self.image_size[1] / step) + ] for step in self.steps] + self.name = 's' + + def forward(self): + anchors = [] + for k, f in enumerate(self.feature_maps): + min_sizes = self.min_sizes[k] + for i, j in product(range(f[0]), range(f[1])): + for min_size in min_sizes: + s_kx = min_size / self.image_size[1] + s_ky = min_size / self.image_size[0] + dense_cx = [ + x * self.steps[k] / self.image_size[1] + for x in [j + 0.5] + ] + dense_cy = [ + y * self.steps[k] / self.image_size[0] + for y in [i + 0.5] + ] + for cy, cx in product(dense_cy, dense_cx): + anchors += [cx, cy, s_kx, s_ky] + + # back to torch land + output = torch.Tensor(anchors).view(-1, 4) + if self.clip: + output.clamp_(max=1, min=0) + return output + + +def py_cpu_nms(dets, thresh): + """Pure Python NMS baseline.""" + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = dets[:, 2] + y2 = dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + + return keep + + +# Adapted from https://github.com/Hakuyume/chainer-ssd +def decode(loc, priors, variances): + """Decode locations from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + loc (tensor): location predictions for loc layers, + Shape: [num_priors,4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat( + (priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def decode_landm(pre, priors, variances): + """Decode landm from predictions using priors to undo + the encoding we did for offset regression at train time. + Args: + pre (tensor): landm predictions for loc layers, + Shape: [num_priors,10] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + decoded landm predictions + """ + a = priors[:, :2] + pre[:, :2] * variances[0] * priors[:, 2:] + b = priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:] + c = priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:] + d = priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:] + e = priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:] + landms = torch.cat((a, b, c, d, e), dim=1) + return landms diff --git a/runtime/Lib/site_packages/models/cv/image_probing_model/__init__.py b/runtime/Lib/site_packages/models/cv/image_probing_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e97a1b77ad79875d91d074a89612f69d9796120c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_probing_model/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model import StructuredProbingModel + +else: + _import_structure = { + 'model': ['StructuredProbingModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_probing_model/backbone.py b/runtime/Lib/site_packages/models/cv/image_probing_model/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..8f3ed5b6f9ce2a074c8788e9d5aa9c69da0251b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_probing_model/backbone.py @@ -0,0 +1,308 @@ +# The implementation is adopted from OpenAI-CLIP, +# made pubicly available under the MIT License at https://github.com/openai/CLIP + +import math +import sys +from collections import OrderedDict +from functools import reduce +from operator import mul + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image +from torchvision import models + +from .utils import convert_weights, load_pretrained + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed + # after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, + # and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, 1) + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) + x = x + self.positional_embedding[:, None, :].to(x.dtype) + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor, idx): + features = {} + x_norm = self.ln_1(x) + features['layer_{}_pre_attn'.format(idx)] = x_norm.permute(1, 0, 2) + attn = self.attention(x_norm) + features['layer_{}_attn'.format(idx)] = attn.permute(1, 0, 2) + x = x + attn + mlp = self.mlp(self.ln_2(x)) + features['layer_{}_mlp'.format(idx)] = mlp.permute(1, 0, 2) + x = x + mlp + return x, features + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.ModuleList() + for i in range(layers): + block = ResidualAttentionBlock(width, heads, attn_mask) + self.resblocks.append(block) + + def forward(self, x: torch.Tensor): + features = {} + for idx, block in enumerate(self.resblocks): + x, block_feats = block(x, idx) + features.update(block_feats) + return x, features + + +class VisualTransformer(nn.Module): + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int): + super().__init__() + print(input_resolution, patch_size, width, layers, heads, output_dim) + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor, return_all=True): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + zeros = torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + # shape = [*, grid ** 2 + 1, width] + x = torch.cat([self.class_embedding.to(x.dtype) + zeros, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x, features = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if return_all: + features['pre_logits'] = x + return features + + if self.proj is not None: + x = x @ self.proj + + return x + + +class CLIPNet(nn.Module): + + def __init__(self, arch_name, pretrained, **kwargs): + super(CLIPNet, self).__init__() + + if arch_name == 'CLIP_ViTB32': + self.clip = VisualTransformer( + input_resolution=224, + patch_size=32, + width=768, + layers=12, + heads=12, + output_dim=512) + + elif arch_name in ('CLIP_ViTB16', 'CLIP_ViTB16_FP16'): + self.clip = VisualTransformer( + input_resolution=224, + patch_size=16, + width=768, + layers=12, + heads=12, + output_dim=512) + + elif arch_name in ('CLIP_ViTL14', 'CLIP_ViTL14_FP16'): + self.clip = VisualTransformer( + input_resolution=224, + patch_size=14, + width=1024, + layers=24, + heads=16, + output_dim=768) + + else: + raise KeyError(f'Unsupported arch_name for CLIP, {arch_name}') + + def forward(self, input_data): + output = self.clip(input_data) + return output + + +def CLIP(arch_name='CLIP_RN50', + use_pretrain=False, + load_from='', + state_dict=None, + **kwargs): + model = CLIPNet(arch_name=arch_name, pretrained=None, **kwargs) + if use_pretrain: + if arch_name.endswith('FP16'): + convert_weights(model.clip) + load_pretrained(model.clip, state_dict, load_from) + return model + + +class ProbingModel(torch.nn.Module): + + def __init__(self, feat_size, num_classes): + super(ProbingModel, self).__init__() + self.linear = torch.nn.Linear(feat_size, num_classes) + + def forward(self, x): + return self.linear(x) diff --git a/runtime/Lib/site_packages/models/cv/image_probing_model/model.py b/runtime/Lib/site_packages/models/cv/image_probing_model/model.py new file mode 100644 index 0000000000000000000000000000000000000000..e7636f4052aa6490a9a69041de5d47a03ca86b77 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_probing_model/model.py @@ -0,0 +1,93 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os +from typing import Any, Dict + +import json +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import CLIP, ProbingModel + + +@MODELS.register_module( + Tasks.image_classification, module_name=Models.image_probing_model) +class StructuredProbingModel(TorchModel): + """ + The implementation of 'Structured Model Probing: Empowering + Efficient Adaptation by Structured Regularization'. + """ + + def __init__(self, model_dir, *args, **kwargs): + """ + Initialize a probing model. + Args: + model_dir: model id or path + """ + super(StructuredProbingModel, self).__init__() + model_dir = os.path.join(model_dir, 'food101-clip-vitl14-full.pt') + model_file = torch.load(model_dir) + self.feature_size = model_file['meta_info']['feature_size'] + self.num_classes = model_file['meta_info']['num_classes'] + self.backbone = CLIP( + 'CLIP_ViTL14_FP16', + use_pretrain=True, + state_dict=model_file['backbone_model_state_dict']) + self.probing_model = ProbingModel(self.feature_size, self.num_classes) + self.probing_model.load_state_dict( + model_file['probing_model_state_dict']) + + def forward(self, x): + """ + Forward Function of SMP. + Args: + x: the input images (B, 3, H, W) + """ + + keys = [] + for idx in range(0, 24): + keys.append('layer_{}_pre_attn'.format(idx)) + keys.append('layer_{}_attn'.format(idx)) + keys.append('layer_{}_mlp'.format(idx)) + keys.append('pre_logits') + features = self.backbone(x.half()) + features_agg = [] + for i in keys: + aggregated_feature = self.aggregate_token(features[i], 1024) + features_agg.append(aggregated_feature) + features_agg = torch.cat((features_agg), dim=1) + outputs = self.probing_model(features_agg.float()) + return outputs + + def aggregate_token(self, output, target_size): + """ + Aggregating features from tokens. + Args: + output: the output of intermidiant features + from a ViT model + target_size: target aggregated feature size + """ + if len(output.shape) == 3: + _, n_token, channels = output.shape + if channels >= target_size: + pool_size = 0 + else: + n_groups = target_size / channels + pool_size = int(n_token / n_groups) + + if pool_size > 0: + output = torch.permute(output, (0, 2, 1)) + output = torch.nn.AvgPool1d( + kernel_size=pool_size, stride=pool_size)( + output) + output = torch.flatten(output, start_dim=1) + else: + output = torch.mean(output, dim=1) + output = torch.nn.functional.normalize(output, dim=1) + return output diff --git a/runtime/Lib/site_packages/models/cv/image_probing_model/utils.py b/runtime/Lib/site_packages/models/cv/image_probing_model/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2b13ae5d9efa103bd0197aca639fd4d67d79d37 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_probing_model/utils.py @@ -0,0 +1,148 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import re + +import torch +import torch.nn as nn + + +def load_pretrained(model: torch.nn.Module, + state_dict, + local_path: str, + map_location='cpu', + logger=None, + sub_level=None): + return load_pretrained_dict(model, state_dict, logger, sub_level=sub_level) + + +def load_pretrained_dict(model: torch.nn.Module, + state_dict: dict, + logger=None, + sub_level=None): + """ + Load parameters to model with + 1. Sub name by revise_keys For DataParallelModel or DistributeParallelModel. + 2. Load 'state_dict' again if possible by key 'state_dict' or 'model_state'. + 3. Take sub level keys from source, e.g. load 'backbone' part from a classifier into a backbone model. + 4. Auto remove invalid parameters from source. + 5. Log or warning if unexpected key exists or key misses. + + Args: + model (torch.nn.Module): + state_dict (dict): dict of parameters + logger (logging.Logger, None): + sub_level (str, optional): If not None, parameters with key startswith sub_level will remove the prefix + to fit actual model keys. This action happens if user want to load sub module parameters + into a sub module model. + """ + revise_keys = [(r'^module\.', '')] + + if 'state_dict' in state_dict: + state_dict = state_dict['state_dict'] + if 'model_state' in state_dict: + state_dict = state_dict['model_state'] + + for p, r in revise_keys: + state_dict = {re.sub(p, r, k): v for k, v in state_dict.items()} + + if sub_level: + sub_level = sub_level if sub_level.endswith('.') else (sub_level + '.') + sub_level_len = len(sub_level) + state_dict = { + key[sub_level_len:]: value + for key, value in state_dict.items() if key.startswith(sub_level) + } + + state_dict = _auto_drop_invalid(model, state_dict, logger=logger) + + load_status = model.load_state_dict(state_dict, strict=False) + unexpected_keys = load_status.unexpected_keys + missing_keys = load_status.missing_keys + err_msgs = [] + if unexpected_keys: + err_msgs.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msgs.append('missing key in source ' + f'state_dict: {", ".join(missing_keys)}\n') + err_msgs = '\n'.join(err_msgs) + + if len(err_msgs) > 0: + if logger: + logger.warning(err_msgs) + else: + import warnings + warnings.warn(err_msgs) + + +def convert_weights(model: nn.Module): + """ + Convert applicable model parameters to fp16. + """ + + def _convert_weights_to_fp16(layer): + if isinstance(layer, (nn.Conv1d, nn.Conv2d, nn.Linear)): + layer.weight.data = layer.weight.data.half() + if layer.bias is not None: + layer.bias.data = layer.bias.data.half() + + if isinstance(layer, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(layer, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ['text_projection', 'proj']: + if hasattr(layer, name): + attr = getattr(layer, name) + if attr is not None: + attr.data = attr.data.half() + + for name in ['prompt_embeddings']: + if hasattr(layer, name): + attr = getattr(layer, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +def _auto_drop_invalid(model: torch.nn.Module, state_dict: dict, logger=None): + """ + Strip unmatched parameters in state_dict, e.g. shape not matched, type not matched. + + Args: + model (torch.nn.Module): + state_dict (dict): + logger (logging.Logger, None): + + Returns: + A new state dict. + """ + ret_dict = state_dict.copy() + invalid_msgs = [] + for key, value in model.state_dict().items(): + if key in state_dict: + # Check shape + new_value = state_dict[key] + if value.shape != new_value.shape: + invalid_msgs.append( + f'{key}: invalid shape, dst {value.shape} vs. src {new_value.shape}' + ) + ret_dict.pop(key) + elif value.dtype != new_value.dtype: + invalid_msgs.append( + f'{key}: invalid dtype, dst {value.dtype} vs. src {new_value.dtype}' + ) + ret_dict.pop(key) + if len(invalid_msgs) > 0: + warning_msg = 'ignore keys from source: \n' + '\n'.join(invalid_msgs) + if logger: + logger.warning(warning_msg) + else: + import warnings + warnings.warn(warning_msg) + return ret_dict diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/__init__.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd102430dc85b6a8bd56388db9add3c5138b6ef --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_quality_assessment_degradation import ImageQualityAssessmentDegradation + +else: + _import_structure = { + 'image_quality_assessment_degradation': + ['ImageQualityAssessmentDegradation'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/degradation_model.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/degradation_model.py new file mode 100644 index 0000000000000000000000000000000000000000..639e349f22ea52e8d24bee5c3adfb7f87c23477e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/degradation_model.py @@ -0,0 +1,127 @@ +import time +from collections import defaultdict + +import cv2 +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torchvision import models + + +class DegradationIQA(nn.Module): + + def __init__(self): + super(DegradationIQA, self).__init__() + # [64, 128, 128] + features = list( + models.__dict__['resnet50'](pretrained=False).children())[:-2] + features = nn.Sequential(*features) + # features = list(models.__dict__['vgg16'](pretrained=True).children())[:-2][0][:-1] + self.features = features + self.lcn_radius = 7 + self.classifier = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self.noise_regression = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self.blur_regression = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self.compression_regression = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self.bright_regression = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self.color_regression = nn.Sequential( + nn.Conv2d(2048, 1024, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(1024, 256, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + # nn.BatchNorm1d(256), + nn.Conv2d(256, 64, kernel_size=3, padding=1, dilation=1), + nn.ReLU(True), + nn.Conv2d(64, 1, kernel_size=3, padding=1, dilation=1)) + + self._initialize_weights() + # activation function + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def _initialize_weights(self): + initialize_layers = [ + x for j in [ + self.classifier, self.noise_regression, self.blur_regression, + self.compression_regression, self.bright_regression, + self.color_regression + ] for x in j + ] + for m in initialize_layers: + if isinstance(m, nn.Conv2d): + nn.init.xavier_uniform_(m.weight.data) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, require_map=False): + + for model in self.features: + x = model(x) + + fea = x + out_map = self.classifier(fea) + noise_map = self.noise_regression(fea) + blur_map = self.blur_regression(fea) + comp_map = self.compression_regression(fea) + bright_map = self.bright_regression(fea) + color_map = self.color_regression(fea) + out = torch.mean(torch.mean(out_map, dim=2), dim=2) + noise_out = torch.mean(torch.mean(noise_map, dim=2), dim=2) + blur_out = torch.mean(torch.mean(blur_map, dim=2), dim=2) + comp_out = torch.mean(torch.mean(comp_map, dim=2), dim=2) + bright_out = torch.mean(torch.mean(bright_map, dim=2), dim=2) + color_out = torch.mean(torch.mean(color_map, dim=2), dim=2) + + if not require_map: + return out, [noise_out, blur_out, comp_out, bright_out, color_out] + else: + return out, [ + noise_out, blur_out, comp_out, bright_out, color_out + ], [noise_map, blur_map, comp_map, bright_map, color_map] diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/image_quality_assessment_degradation.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/image_quality_assessment_degradation.py new file mode 100644 index 0000000000000000000000000000000000000000..ddc73177befd54170789f9394e5c69a9f6cfaf01 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_degradation/image_quality_assessment_degradation.py @@ -0,0 +1,121 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch.cuda +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_quality_assessment_degradation.degradation_model import \ + DegradationIQA +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ImageQualityAssessmentDegradation'] + + +@MODELS.register_module( + Tasks.image_quality_assessment_degradation, + module_name=Models.image_quality_assessment_degradation) +class ImageQualityAssessmentDegradation(TorchModel): + """ + Its architecture is based on the modified resnet50, output with blur degree, noise degree, compression degree. + Reference: Rich features for perceptual quality assessment of UGC videos. + """ + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image_quality_assessment_degradation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.sigmoid_layer = nn.Sigmoid() + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.model = DegradationIQA() + self.model = self._load_pretrained(self.model, model_path) + self.model.eval() + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + losses = dict() + return losses + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + # Todo + # if img_tensor.shape[2]*img_tensor.shape[3] > 720*1280: + # img_tensor = torchvision.transforms.functional.resize(img_tensor, 720) + preds = self.model(input, require_map=False) + noise_degree, blur_degree, comp_degree = preds[1][:3] + noise_degree, blur_degree, comp_degree = self.sigmoid_layer( + noise_degree), self.sigmoid_layer(blur_degree), self.sigmoid_layer( + comp_degree) + if noise_degree > 0.3: + noise_degree = noise_degree + 0.1 + if noise_degree >= 0.2 and noise_degree <= 0.3: + noise_degree = (noise_degree - 0.2) * 2 + 0.2 + blur_degree = blur_degree + comp_degree / 2 + + return { + 'noise_degree': noise_degree, + 'blur_degree': blur_degree, + 'comp_degree': comp_degree + } + + def _evaluate_postprocess(self, input: Tensor, item_id: Tensor, + distortion_type: Tensor, target: Tensor, + **kwargs) -> Dict[str, list]: + torch.cuda.empty_cache() + with torch.no_grad(): + preds = self.model(input, require_map=False) + noise_degree, blur_degree, comp_degree = preds[1][:3] + noise_degree, blur_degree, comp_degree = self.sigmoid_layer( + noise_degree), self.sigmoid_layer( + blur_degree), self.sigmoid_layer(comp_degree) + noise_degree, blur_degree, comp_degree = noise_degree.cpu( + ), blur_degree.cpu(), comp_degree.cpu() + if noise_degree > 0.3: + noise_degree = noise_degree + 0.1 + if noise_degree >= 0.2 and noise_degree <= 0.3: + noise_degree = (noise_degree - 0.2) * 2 + 0.2 + blur_degree = blur_degree + comp_degree / 2 + del input + target = target.cpu() + torch.cuda.empty_cache() + return { + 'item_id': item_id, + 'distortion_type': distortion_type, + 'noise_degree': noise_degree, + 'blur_degree': blur_degree, + 'comp_degree': comp_degree, + 'target': target + } + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/__init__.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f29b90a77afbc7004b80154d27a2f1dfdae553ac --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_quality_assessment_man import ImageQualityAssessmentMAN + +else: + _import_structure = { + 'image_quality_assessment_man': ['ImageQualityAssessmentMAN'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/image_quality_assessment_man.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/image_quality_assessment_man.py new file mode 100644 index 0000000000000000000000000000000000000000..e290909ee0d81bbf5dbbac0fc3001b7fba1e52d0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/image_quality_assessment_man.py @@ -0,0 +1,79 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch.cuda +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_quality_assessment_man.maniqa import MANIQA +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ImageQualityAssessmentMAN'] + + +@MODELS.register_module( + Tasks.image_quality_assessment_mos, + module_name=Models.image_quality_assessment_man) +class ImageQualityAssessmentMAN(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image_quality_assessment_man model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.model = MANIQA() + self.model = self._load_pretrained(self.model, model_path) + self.model.eval() + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + losses = dict() + return losses + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'output': self.model(input).clamp(0, 1)} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + + torch.cuda.empty_cache() + with torch.no_grad(): + preds = self.model(input) + preds = preds.clamp(0, 1).cpu() + del input + target = target.cpu() + torch.cuda.empty_cache() + return {'pred': preds, 'target': target} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/maniqa.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/maniqa.py new file mode 100644 index 0000000000000000000000000000000000000000..8c9243096367b8e8a1cc902ab04f614180b2adea --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/maniqa.py @@ -0,0 +1,161 @@ +# This implementation is adopted from MANIQA, made pubicly available under the Apache License 2.0 at +# https://github.com/IIGROUP/MANIQA/blob/master/models/maniqa.py + +import timm +import torch +import torch.nn as nn +from einops import rearrange +from timm.models.vision_transformer import Block + +from .swin import SwinTransformer + + +class TABlock(nn.Module): + + def __init__(self, dim, drop=0.1): + super().__init__() + self.c_q = nn.Linear(dim, dim) + self.c_k = nn.Linear(dim, dim) + self.c_v = nn.Linear(dim, dim) + self.norm_fact = dim**-0.5 + self.softmax = nn.Softmax(dim=-1) + self.proj_drop = nn.Dropout(drop) + + def forward(self, x): + _x = x + B, C, N = x.shape + q = self.c_q(x) + k = self.c_k(x) + v = self.c_v(x) + + attn = q @ k.transpose(-2, -1) * self.norm_fact + attn = self.softmax(attn) + x = (attn @ v).transpose(1, 2).reshape(B, C, N) + x = self.proj_drop(x) + x = x + _x + return x + + +class SaveOutput: + + def __init__(self): + self.outputs = [] + + def __call__(self, module, module_in, module_out): + self.outputs.append(module_out) + + def clear(self): + self.outputs = [] + + +class MANIQA(nn.Module): + + def __init__(self, + embed_dim=768, + num_outputs=1, + patch_size=8, + drop=0.1, + depths=[2, 2], + window_size=4, + dim_mlp=768, + num_heads=[4, 4], + img_size=224, + num_tab=2, + scale=0.13, + **kwargs): + super().__init__() + self.img_size = img_size + self.patch_size = patch_size + self.input_size = img_size // patch_size + self.patches_resolution = (img_size // patch_size, + img_size // patch_size) + + self.vit = timm.create_model('vit_base_patch8_224', pretrained=False) + self.save_output = SaveOutput() + hook_handles = [] + for layer in self.vit.modules(): + if isinstance(layer, Block): + handle = layer.register_forward_hook(self.save_output) + hook_handles.append(handle) + + self.tablock1 = nn.ModuleList() + for i in range(num_tab): + tab = TABlock(self.input_size**2) + self.tablock1.append(tab) + + self.conv1 = nn.Conv2d(embed_dim * 4, embed_dim, 1, 1, 0) + self.swintransformer1 = SwinTransformer( + patches_resolution=self.patches_resolution, + depths=depths, + num_heads=num_heads, + embed_dim=embed_dim, + window_size=window_size, + dim_mlp=dim_mlp, + scale=scale) + + self.tablock2 = nn.ModuleList() + for i in range(num_tab): + tab = TABlock(self.input_size**2) + self.tablock2.append(tab) + + self.conv2 = nn.Conv2d(embed_dim, embed_dim // 2, 1, 1, 0) + self.swintransformer2 = SwinTransformer( + patches_resolution=self.patches_resolution, + depths=depths, + num_heads=num_heads, + embed_dim=embed_dim // 2, + window_size=window_size, + dim_mlp=dim_mlp, + scale=scale) + + self.fc_score = nn.Sequential( + nn.Linear(embed_dim // 2, embed_dim // 2), nn.ReLU(), + nn.Dropout(drop), nn.Linear(embed_dim // 2, num_outputs), + nn.ReLU()) + self.fc_weight = nn.Sequential( + nn.Linear(embed_dim // 2, embed_dim // 2), nn.ReLU(), + nn.Dropout(drop), nn.Linear(embed_dim // 2, num_outputs), + nn.Sigmoid()) + + def extract_feature(self, save_output): + x6 = save_output.outputs[6][:, 1:] + x7 = save_output.outputs[7][:, 1:] + x8 = save_output.outputs[8][:, 1:] + x9 = save_output.outputs[9][:, 1:] + x = torch.cat((x6, x7, x8, x9), dim=2) + return x + + def forward(self, x): + self.vit(x) + x = self.extract_feature(self.save_output) + self.save_output.outputs.clear() + + # stage 1 + x = rearrange( + x, 'b (h w) c -> b c (h w)', h=self.input_size, w=self.input_size) + for tab in self.tablock1: + x = tab(x) + x = rearrange( + x, 'b c (h w) -> b c h w', h=self.input_size, w=self.input_size) + x = self.conv1(x) + x = self.swintransformer1(x) + + # stage2 + x = rearrange( + x, 'b c h w -> b c (h w)', h=self.input_size, w=self.input_size) + for tab in self.tablock2: + x = tab(x) + x = rearrange( + x, 'b c (h w) -> b c h w', h=self.input_size, w=self.input_size) + x = self.conv2(x) + x = self.swintransformer2(x) + + x = rearrange( + x, 'b c h w -> b (h w) c', h=self.input_size, w=self.input_size) + score = torch.tensor([]).cuda() + for i in range(x.shape[0]): + f = self.fc_score(x[i]) + w = self.fc_weight(x[i]) + _s = torch.sum(f * w) / torch.sum(w) + score = torch.cat((score, _s.unsqueeze(0)), 0) + return score diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/swin.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..df58277f254a1ca04020e664c073fdf5d14ddc35 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_man/swin.py @@ -0,0 +1,632 @@ +# This implementation is adopted form SwinTransformer, made pubicly available under the MIT License at +# https://github.com/microsoft/Swin-Transformer/blob/main/models/swin_transformer.py + +import collections.abc +import math +import warnings +from itertools import repeat + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from einops import rearrange + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower = norm_cdf((a - mean) / std) + upper = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * lower - 1, 2 * upper - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +# From PyTorch internals +def _ntuple(n): + + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_2tuple = _ntuple(2) + + +def drop_path(x, + drop_prob: float = 0., + training: bool = False, + scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None, scale_by_keep=True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + r""" Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}' + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class SwinBlock(nn.Module): + r""" Swin Transformer Block. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + dim_mlp=1024., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.dim_mlp = dim_mlp + self.mlp_ratio = self.dim_mlp // dim + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = self.dim_mlp + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + if self.shift_size > 0: + # calculate attention mask for SW-MSA + H, W = self.input_resolution + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view( + -1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + else: + attn_mask = None + + self.register_buffer('attn_mask', attn_mask) + + def forward(self, x): + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, 'input feature has wrong size' + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=self.attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, + W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, ' \ + f'window_size={self.window_size}, shift_size={self.shift_size}' + + def flops(self): + flops = 0 + H, W = self.input_resolution + # norm1 + flops += self.dim * H * W + # W-MSA/SW-MSA + nW = H * W / self.window_size / self.window_size + flops += nW * self.attn.flops(self.window_size * self.window_size) + # mlp + flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + return flops + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + input_resolution, + depth, + num_heads, + window_size=7, + dim_mlp=1024, + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + + super().__init__() + self.dim = dim + self.conv = nn.Conv2d(dim, dim, 3, 1, 1) + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + dim_mlp=dim_mlp, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample( + input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x): + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x = rearrange( + x, + 'b (h w) c -> b c h w', + h=self.input_resolution[0], + w=self.input_resolution[1]) + x = F.relu(self.conv(x)) + x = rearrange(x, 'b c h w -> b (h w) c') + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}' + + def flops(self): + flops = 0 + for blk in self.blocks: + flops += blk.flops() + if self.downsample is not None: + flops += self.downsample.flops() + return flops + + +class SwinTransformer(nn.Module): + + def __init__(self, + patches_resolution, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + embed_dim=256, + drop=0.1, + drop_rate=0., + drop_path_rate=0.1, + dropout=0., + window_size=7, + dim_mlp=1024, + qkv_bias=True, + qk_scale=None, + attn_drop_rate=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + scale=0.8, + **kwargs): + super().__init__() + self.scale = scale + self.embed_dim = embed_dim + self.depths = depths + self.num_heads = num_heads + self.window_size = window_size + self.dropout = nn.Dropout(p=drop) + self.num_features = embed_dim + self.num_layers = len(depths) + self.patches_resolution = (patches_resolution[0], + patches_resolution[1]) + self.downsample = nn.Conv2d( + self.embed_dim, self.embed_dim, kernel_size=3, stride=2, padding=1) + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=self.embed_dim, + input_resolution=patches_resolution, + depth=self.depths[i_layer], + num_heads=self.num_heads[i_layer], + window_size=self.window_size, + dim_mlp=dim_mlp, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=dropout, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(self.depths[:i_layer] + ):sum(self.depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=downsample, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + def forward(self, x): + x = self.dropout(x) + x = rearrange(x, 'b c h w -> b (h w) c') + for layer in self.layers: + _x = x + x = layer(x) + x = self.scale * x + _x + x = rearrange( + x, + 'b (h w) c -> b c h w', + h=self.patches_resolution[0], + w=self.patches_resolution[1]) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/__init__.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..188702a2f042ca01928ecc5729dff9006bf5bc4e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_quality_assessment_mos import ImageQualityAssessmentMos + +else: + _import_structure = { + 'image_quality_assessment_mos': ['ImageQualityAssessmentMos'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fe55f4aa6958348971c8495586ff89b566d57193 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/__init__.py @@ -0,0 +1,8 @@ +# @Time : 2021/5/12 11:49 上午 +# @Author : shaoguowen +# @Email : shaoguowen@tencent.com +# @FileName: __init__.py +# @Software: PyCharm + +from .resnet import (resnet18, resnet34, resnet50, resnet101, resnet152, + resnext50_32x4d, resnext101_32x8d) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/resnet.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..e153e5f964361342c94eae6898805e7fb9d53c61 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/backbones/resnet.py @@ -0,0 +1,452 @@ +# The implementation is adopted from CenseoQoE, made pubicly available under the MIT License at +# https://github.com/Tencent/CenseoQoE +import os + +import torch +import torch.nn as nn + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + +model_urls = { + 'resnet18': + 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': + 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': + 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': + 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': + 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': + 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': + 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': + 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': + 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + """ + ResNet, ref from pytorch official website, + Differ from torch vidion resnet, this net : + a. delete some pooling and fc layers. + b. support output mid-layer feature. + """ + + def __init__(self, + block, + layers, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None, + input_channels=3, + out_indices=(3, ), + strides=(2, 2, 2)): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + self.block = block + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + input_channels, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.ouput_dims = [] + self.layer1 = self._make_layer(block, 64, layers[0]) + self.ouput_dims.append(block.expansion * 64) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=strides[0], + dilate=replace_stride_with_dilation[0]) + self.ouput_dims.append(block.expansion * 128) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=strides[1], + dilate=replace_stride_with_dilation[1]) + self.ouput_dims.append(block.expansion * 256) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=strides[2], + dilate=replace_stride_with_dilation[2]) + self.ouput_dims.append(block.expansion * 512) + self.out_indices = out_indices + + self.ouput_dims = [ + x for i, x in enumerate(self.ouput_dims) if i in out_indices + ] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def forward(self, x): + outs = [] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x1 = self.layer1(x) + outs.append(x1) + x2 = self.layer2(x1) + outs.append(x2) + x3 = self.layer3(x2) + outs.append(x3) + x4 = self.layer4(x3) + outs.append(x4) + + outs = [x for i, x in enumerate(outs) if i in self.out_indices] + return tuple(outs) + + +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + print('load pretrained model>>>>>') + if isinstance(pretrained, str) and os.path.exists(pretrained): + print('load pretrained from local:', pretrained) + state_dict = torch.load(pretrained) + else: + print('load pretrained from url:', model_urls[arch]) + state_dict = load_state_dict_from_url( + model_urls[arch], progress=progress) + net_dict = model.state_dict() + pretrained_dict = { + k: v + for k, v in state_dict.items() + if k in net_dict and v.size() == net_dict[k].size() + } + net_dict.update(pretrained_dict) + model.load_state_dict(net_dict) + return model + + +def resnet18(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/censeo_ivqa_model.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/censeo_ivqa_model.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe40e6ae8f05108bcfb3c9c89ddcd316c6fea74 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/censeo_ivqa_model.py @@ -0,0 +1,33 @@ +# The implementation is adopted from CenseoQoE, made pubicly available under the MIT License at +# https://github.com/Tencent/CenseoQoE + +import torch +from torch import nn + +from . import backbones, heads + + +class CenseoIVQAModel(nn.Module): + """ + A strong baseline model for image quality assessment. + Its architecture is based on the modified resnet18 and reach SOTA interms of PLCC and SRCC. + The reference papaer is https://arxiv.org/pdf/2111.07104.pdf + """ + + def __init__(self, pretrained=True): + super(CenseoIVQAModel, self).__init__() + input_channels = 3 + model_name = 'resnet18' + self.backbone = getattr(backbones, model_name)( + input_channels=input_channels, + pretrained=pretrained, + out_indices=(3, ), + strides=(2, 2, 2)) + self.head = getattr(heads, 'SimpleHead')( + self.backbone.ouput_dims, out_num=1) + + def forward(self, x): + feats = self.backbone(x) + out = self.head(feats) + out = torch.sigmoid(out) + return out diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/__init__.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..55d364b2a9229dd3fffedd863b8da19fbb97276d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/__init__.py @@ -0,0 +1 @@ +from .simple_head import SimpleHead diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/simple_head.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/simple_head.py new file mode 100644 index 0000000000000000000000000000000000000000..95153acabe596c90d61d255469e7a0c9fdf15a28 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/heads/simple_head.py @@ -0,0 +1,20 @@ +import torch +from torch import nn + + +class SimpleHead(torch.nn.Module): + + def __init__(self, feats_dims, out_num=1): + super(SimpleHead, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + out_num = out_num + self.fc_out = nn.Sequential( + nn.Linear(feats_dims[-1], 1024), nn.ReLU(inplace=True), + nn.Linear(1024, out_num)) + + def forward(self, x): + x = x[-1] + x = self.avg_pool(x) + x = torch.flatten(x, 1) + x = self.fc_out(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/image_quality_assessment_mos.py b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/image_quality_assessment_mos.py new file mode 100644 index 0000000000000000000000000000000000000000..8da84b0bd00ad7e08ad7e0aa729cbe7fbf1c5da7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_quality_assessment_mos/image_quality_assessment_mos.py @@ -0,0 +1,80 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch.cuda +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_quality_assessment_mos.censeo_ivqa_model import \ + CenseoIVQAModel +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ImageQualityAssessmentMos'] + + +@MODELS.register_module( + Tasks.image_quality_assessment_mos, + module_name=Models.image_quality_assessment_mos) +class ImageQualityAssessmentMos(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image_quality_assessment_mos model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.model = CenseoIVQAModel(pretrained=False) + self.model = self._load_pretrained(self.model, model_path) + self.model.eval() + + def _train_forward(self, input: Tensor, + target: Tensor) -> Dict[str, Tensor]: + losses = dict() + return losses + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'output': self.model(input).clamp(0, 1)} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + + torch.cuda.empty_cache() + with torch.no_grad(): + preds = self.model(input) + preds = preds.clamp(0, 1).cpu() + del input + target = target.cpu() + torch.cuda.empty_cache() + return {'pred': preds, 'target': target} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + if self.training: + return self._train_forward(**inputs) + elif 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/image_reid_person/__init__.py b/runtime/Lib/site_packages/models/cv/image_reid_person/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0fe0bede7cebe81c15bee2c7a9c89e65cffe7c2d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_reid_person/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .pass_model import PASS + +else: + _import_structure = { + 'pass_model': ['PASS'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_reid_person/pass_model.py b/runtime/Lib/site_packages/models/cv/image_reid_person/pass_model.py new file mode 100644 index 0000000000000000000000000000000000000000..3b032949d6316bcea83657882b19ba67e86d968b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_reid_person/pass_model.py @@ -0,0 +1,136 @@ +# The implementation is adopted from PASS-reID, made pubicly available under the Apache-2.0 License at +# https://github.com/CASIA-IVA-Lab/PASS-reID + +import os +from enum import Enum + +import torch +import torch.nn as nn + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .transreid_model import vit_base_patch16_224_TransReID + + +class Fusions(Enum): + CAT = 'cat' + MEAN = 'mean' + + +@MODELS.register_module( + Tasks.image_reid_person, module_name=Models.image_reid_person) +class PASS(TorchModel): + + def __init__(self, cfg: Config, model_dir: str, **kwargs): + super(PASS, self).__init__(model_dir=model_dir) + size_train = cfg.INPUT.SIZE_TRAIN + sie_coe = cfg.MODEL.SIE_COE + stride_size = cfg.MODEL.STRIDE_SIZE + drop_path = cfg.MODEL.DROP_PATH + drop_out = cfg.MODEL.DROP_OUT + att_drop_rate = cfg.MODEL.ATT_DROP_RATE + gem_pooling = cfg.MODEL.GEM_POOLING + stem_conv = cfg.MODEL.STEM_CONV + weight = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + self.neck_feat = cfg.TEST.NECK_FEAT + self.dropout_rate = cfg.MODEL.DROPOUT_RATE + self.num_classes = cfg.DATASETS.NUM_CLASSES + self.multi_neck = cfg.MODEL.MULTI_NECK + self.feat_fusion = cfg.MODEL.FEAT_FUSION + + self.base = vit_base_patch16_224_TransReID( + img_size=size_train, + sie_xishu=sie_coe, + stride_size=stride_size, + drop_path_rate=drop_path, + drop_rate=drop_out, + attn_drop_rate=att_drop_rate, + gem_pool=gem_pooling, + stem_conv=stem_conv) + self.in_planes = self.base.in_planes + + if self.feat_fusion == Fusions.CAT.value: + self.classifier = nn.Linear( + self.in_planes * 2, self.num_classes, bias=False) + elif self.feat_fusion == Fusions.MEAN.value: + self.classifier = nn.Linear( + self.in_planes, self.num_classes, bias=False) + + if self.multi_neck: + self.bottleneck = nn.BatchNorm1d(self.in_planes) + self.bottleneck.bias.requires_grad_(False) + self.bottleneck_1 = nn.BatchNorm1d(self.in_planes) + self.bottleneck_1.bias.requires_grad_(False) + self.bottleneck_2 = nn.BatchNorm1d(self.in_planes) + self.bottleneck_2.bias.requires_grad_(False) + self.bottleneck_3 = nn.BatchNorm1d(self.in_planes) + self.bottleneck_3.bias.requires_grad_(False) + else: + if self.feat_fusion == Fusions.CAT.value: + self.bottleneck = nn.BatchNorm1d(self.in_planes * 2) + self.bottleneck.bias.requires_grad_(False) + elif self.feat_fusion == Fusions.MEAN.value: + self.bottleneck = nn.BatchNorm1d(self.in_planes) + self.bottleneck.bias.requires_grad_(False) + + self.dropout = nn.Dropout(self.dropout_rate) + + self.load_param(weight) + + def forward(self, input): + + global_feat, local_feat_1, local_feat_2, local_feat_3 = self.base( + input) + + # single-neck, almost the same performance + if not self.multi_neck: + if self.feat_fusion == Fusions.MEAN.value: + local_feat = local_feat_1 / 3. + local_feat_2 / 3. + local_feat_3 / 3. + final_feat_before = (global_feat + local_feat) / 2 + elif self.feat_fusion == Fusions.CAT.value: + final_feat_before = torch.cat( + (global_feat, local_feat_1 / 3. + local_feat_2 / 3. + + local_feat_3 / 3.), + dim=1) + + final_feat_after = self.bottleneck(final_feat_before) + # multi-neck + else: + feat = self.bottleneck(global_feat) + local_feat_1_bn = self.bottleneck_1(local_feat_1) + local_feat_2_bn = self.bottleneck_2(local_feat_2) + local_feat_3_bn = self.bottleneck_3(local_feat_3) + + if self.feat_fusion == Fusions.MEAN.value: + final_feat_before = ((global_feat + local_feat_1 / 3 + + local_feat_2 / 3 + local_feat_3 / 3) + / 2.) + final_feat_after = (feat + local_feat_1_bn / 3 + + local_feat_2_bn / 3 + + local_feat_3_bn / 3) / 2. + elif self.feat_fusion == Fusions.CAT.value: + final_feat_before = torch.cat( + (global_feat, local_feat_1 / 3. + local_feat_2 / 3. + + local_feat_3 / 3.), + dim=1) + final_feat_after = torch.cat( + (feat, local_feat_1_bn / 3 + local_feat_2_bn / 3 + + local_feat_3_bn / 3), + dim=1) + + if self.neck_feat == 'after': + return final_feat_after + else: + return final_feat_before + + def load_param(self, trained_path): + param_dict = torch.load(trained_path, map_location='cpu') + for i in param_dict: + try: + self.state_dict()[i.replace('module.', + '')].copy_(param_dict[i]) + except Exception: + continue diff --git a/runtime/Lib/site_packages/models/cv/image_reid_person/transreid_model.py b/runtime/Lib/site_packages/models/cv/image_reid_person/transreid_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5bceb4685ca8fe73ace6a0e52ad9134182487c79 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_reid_person/transreid_model.py @@ -0,0 +1,418 @@ +# The implementation is adopted from PASS-reID, made pubicly available under the Apache-2.0 License at +# https://github.com/CASIA-IVA-Lab/PASS-reID + +import collections.abc as container_abcs +from functools import partial +from itertools import repeat + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +# From PyTorch internals +def _ntuple(n): + + def parse(x): + if isinstance(x, container_abcs.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_2tuple = _ntuple(2) + + +def vit_base_patch16_224_TransReID( + img_size=(256, 128), + stride_size=16, + drop_path_rate=0.1, + camera=0, + view=0, + local_feature=False, + sie_xishu=1.5, + **kwargs): + model = TransReID( + img_size=img_size, + patch_size=16, + stride_size=stride_size, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + qkv_bias=True, + camera=camera, + view=view, + drop_path_rate=drop_path_rate, + sie_xishu=sie_xishu, + local_feature=local_feature, + **kwargs) + return model + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class TransReID(nn.Module): + """Transformer-based Object Re-Identification + """ + + def __init__(self, + img_size=224, + patch_size=16, + stride_size=16, + in_chans=3, + num_classes=1000, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + camera=0, + view=0, + drop_path_rate=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6), + local_feature=False, + sie_xishu=1.0, + hw_ratio=1, + gem_pool=False, + stem_conv=False): + super().__init__() + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.local_feature = local_feature + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + stride_size=stride_size, + in_chans=in_chans, + embed_dim=embed_dim, + stem_conv=stem_conv) + + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part_token1 = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part_token2 = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part_token3 = nn.Parameter(torch.zeros(1, 1, embed_dim)) + + self.cls_pos = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part1_pos = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part2_pos = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.part3_pos = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim)) + self.cam_num = camera + self.view_num = view + self.sie_xishu = sie_xishu + self.in_planes = 768 + self.gem_pool = gem_pool + + # Initialize SIE Embedding + if camera > 1 and view > 1: + self.sie_embed = nn.Parameter( + torch.zeros(camera * view, 1, embed_dim)) + elif camera > 1: + self.sie_embed = nn.Parameter(torch.zeros(camera, 1, embed_dim)) + elif view > 1: + self.sie_embed = nn.Parameter(torch.zeros(view, 1, embed_dim)) + + self.pos_drop = nn.Dropout(p=drop_rate) + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer) for i in range(depth) + ]) + + self.norm = norm_layer(embed_dim) + + # Classifier head + self.fc = nn.Linear(embed_dim, + num_classes) if num_classes > 0 else nn.Identity() + + self.gem = GeneralizedMeanPooling() + + def forward_features(self, x, camera_id, view_id): + B = x.shape[0] + x = self.patch_embed(x) + + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + part_tokens1 = self.part_token1.expand(B, -1, -1) + part_tokens2 = self.part_token2.expand(B, -1, -1) + part_tokens3 = self.part_token3.expand(B, -1, -1) + x = torch.cat( + (cls_tokens, part_tokens1, part_tokens2, part_tokens3, x), dim=1) + + if self.cam_num > 0 and self.view_num > 0: + x = x + self.pos_embed + self.sie_xishu * self.sie_embed[ + camera_id * self.view_num + view_id] + elif self.cam_num > 0: + x = x + self.pos_embed + self.sie_xishu * self.sie_embed[camera_id] + elif self.view_num > 0: + x = x + self.pos_embed + self.sie_xishu * self.sie_embed[view_id] + else: + x = x + torch.cat((self.cls_pos, self.part1_pos, self.part2_pos, + self.part3_pos, self.pos_embed), + dim=1) + + x = self.pos_drop(x) + + if self.local_feature: + for blk in self.blocks[:-1]: + x = blk(x) + return x + else: + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + if self.gem_pool: + gf = self.gem(x[:, 1:].permute(0, 2, 1)).squeeze() + return x[:, 0] + gf + return x[:, 0], x[:, 1], x[:, 2], x[:, 3] + + def forward(self, x, cam_label=None, view_label=None): + global_feat, local_feat_1, local_feat_2, local_feat_3 = self.forward_features( + x, cam_label, view_label) + return global_feat, local_feat_1, local_feat_2, local_feat_3 + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding with overlapping patches + """ + + def __init__(self, + img_size=224, + patch_size=16, + stride_size=16, + in_chans=3, + embed_dim=768, + stem_conv=False): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + stride_size_tuple = to_2tuple(stride_size) + self.num_x = (img_size[1] - patch_size[1]) // stride_size_tuple[1] + 1 + self.num_y = (img_size[0] - patch_size[0]) // stride_size_tuple[0] + 1 + self.num_patches = self.num_x * self.num_y + self.img_size = img_size + self.patch_size = patch_size + + self.stem_conv = stem_conv + if self.stem_conv: + hidden_dim = 64 + stem_stride = 2 + stride_size = patch_size = patch_size[0] // stem_stride + self.conv = nn.Sequential( + nn.Conv2d( + in_chans, + hidden_dim, + kernel_size=7, + stride=stem_stride, + padding=3, + bias=False), + IBN(hidden_dim), + nn.ReLU(inplace=True), + nn.Conv2d( + hidden_dim, + hidden_dim, + kernel_size=3, + stride=1, + padding=1, + bias=False), + IBN(hidden_dim), + nn.ReLU(inplace=True), + nn.Conv2d( + hidden_dim, + hidden_dim, + kernel_size=3, + stride=1, + padding=1, + bias=False), + nn.BatchNorm2d(hidden_dim), + nn.ReLU(inplace=True), + ) + in_chans = hidden_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=stride_size) + + def forward(self, x): + if self.stem_conv: + x = self.conv(x) + x = self.proj(x) + x = x.flatten(2).transpose(1, 2) # [64, 8, 768] + + return x + + +class GeneralizedMeanPooling(nn.Module): + """Applies a 2D power-average adaptive pooling over an input signal composed of several input planes. + The function computed is: :math:`f(X) = pow(sum(pow(X, p)), 1/p)` + - At p = infinity, one gets Max Pooling + - At p = 1, one gets Average Pooling + The output is of size H x W, for any input size. + The number of output features is equal to the number of input planes. + Args: + output_size: the target output size of the image of the form H x W. + Can be a tuple (H, W) or a single H for a square image H x H + H and W can be either a ``int``, or ``None`` which means the size will + be the same as that of the input. + """ + + def __init__(self, norm=3, output_size=1, eps=1e-6): + super(GeneralizedMeanPooling, self).__init__() + assert norm > 0 + self.p = float(norm) + self.output_size = output_size + self.eps = eps + + def forward(self, x): + x = x.clamp(min=self.eps).pow(self.p) + return F.adaptive_avg_pool1d(x, self.output_size).pow(1. / self.p) + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_restoration/__init__.py b/runtime/Lib/site_packages/models/cv/image_restoration/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..91ee26360f7d3b4dde3230d9888bb31a9d583fba --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_restoration/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_restoration_model import ImageRestorationModel + +else: + _import_structure = { + 'image_restoration_model': ['ImageRestorationModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/__init__.py b/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..66328b3343b1a95bc94380964657b9ec53e8e49e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/__init__.py @@ -0,0 +1,3 @@ +from .nets import my_model as ESDNet + +model_map = {'ESDNet': ESDNet} diff --git a/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/nets.py b/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/nets.py new file mode 100644 index 0000000000000000000000000000000000000000..1f554b80dbb274f0ad84a300014b90073a3ac88c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_restoration/demoire_models/nets.py @@ -0,0 +1,310 @@ +# The implementation is adopted from UHDM work, made publicly available under the Apache 2.0 License +# source code avaiable via https://github.com/CVMI-Lab/UHDM/ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class my_model(nn.Module): + + def __init__( + self, + en_feature_num, + en_inter_num, + de_feature_num, + de_inter_num, + sam_number=1, + ): + super(my_model, self).__init__() + self.encoder = Encoder( + feature_num=en_feature_num, + inter_num=en_inter_num, + sam_number=sam_number) + self.decoder = Decoder( + en_num=en_feature_num, + feature_num=de_feature_num, + inter_num=de_inter_num, + sam_number=sam_number) + + def forward(self, x): + y_1, y_2, y_3 = self.encoder(x) + out_1, out_2, out_3 = self.decoder(y_1, y_2, y_3) + return out_1, out_2, out_3 + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + m.weight.data.normal_(0.0, 0.02) + if m.bias is not None: + m.bias.data.normal_(0.0, 0.02) + if isinstance(m, nn.ConvTranspose2d): + m.weight.data.normal_(0.0, 0.02) + + +class Decoder(nn.Module): + + def __init__(self, en_num, feature_num, inter_num, sam_number): + super(Decoder, self).__init__() + self.preconv_3 = conv_relu(4 * en_num, feature_num, 3, padding=1) + self.decoder_3 = Decoder_Level(feature_num, inter_num, sam_number) + self.preconv_2 = conv_relu( + 2 * en_num + feature_num, feature_num, 3, padding=1) + self.decoder_2 = Decoder_Level(feature_num, inter_num, sam_number) + self.preconv_1 = conv_relu( + en_num + feature_num, feature_num, 3, padding=1) + self.decoder_1 = Decoder_Level(feature_num, inter_num, sam_number) + + def forward(self, y_1, y_2, y_3): + x_3 = y_3 + x_3 = self.preconv_3(x_3) + out_3, feat_3 = self.decoder_3(x_3) + x_2 = torch.cat([y_2, feat_3], dim=1) + x_2 = self.preconv_2(x_2) + out_2, feat_2 = self.decoder_2(x_2) + x_1 = torch.cat([y_1, feat_2], dim=1) + x_1 = self.preconv_1(x_1) + out_1 = self.decoder_1(x_1, feat=False) + return out_1, out_2, out_3 + + +class Encoder(nn.Module): + + def __init__(self, feature_num, inter_num, sam_number): + super(Encoder, self).__init__() + self.conv_first = nn.Sequential( + nn.Conv2d( + 12, feature_num, kernel_size=5, stride=1, padding=2, + bias=True), nn.ReLU(inplace=True)) + self.encoder_1 = Encoder_Level( + feature_num, inter_num, level=1, sam_number=sam_number) + self.encoder_2 = Encoder_Level( + 2 * feature_num, inter_num, level=2, sam_number=sam_number) + self.encoder_3 = Encoder_Level( + 4 * feature_num, inter_num, level=3, sam_number=sam_number) + + def forward(self, x): + x = F.pixel_unshuffle(x, 2) + x = self.conv_first(x) + out_feature_1, down_feature_1 = self.encoder_1(x) + out_feature_2, down_feature_2 = self.encoder_2(down_feature_1) + out_feature_3 = self.encoder_3(down_feature_2) + return out_feature_1, out_feature_2, out_feature_3 + + +class Encoder_Level(nn.Module): + + def __init__(self, feature_num, inter_num, level, sam_number): + super(Encoder_Level, self).__init__() + self.rdb = RDB( + in_channel=feature_num, d_list=(1, 2, 1), inter_num=inter_num) + self.sam_blocks = nn.ModuleList() + for _ in range(sam_number): + sam_block = SAM( + in_channel=feature_num, + d_list=(1, 2, 3, 2, 1), + inter_num=inter_num) + self.sam_blocks.append(sam_block) + if level < 3: + self.down = nn.Sequential( + nn.Conv2d( + feature_num, + 2 * feature_num, + kernel_size=3, + stride=2, + padding=1, + bias=True), nn.ReLU(inplace=True)) + self.level = level + + def forward(self, x): + out_feature = self.rdb(x) + for sam_block in self.sam_blocks: + out_feature = sam_block(out_feature) + if self.level < 3: + down_feature = self.down(out_feature) + return out_feature, down_feature + return out_feature + + +class Decoder_Level(nn.Module): + + def __init__(self, feature_num, inter_num, sam_number): + super(Decoder_Level, self).__init__() + self.rdb = RDB(feature_num, (1, 2, 1), inter_num) + self.sam_blocks = nn.ModuleList() + for _ in range(sam_number): + sam_block = SAM( + in_channel=feature_num, + d_list=(1, 2, 3, 2, 1), + inter_num=inter_num) + self.sam_blocks.append(sam_block) + self.conv = conv( + in_channel=feature_num, out_channel=12, kernel_size=3, padding=1) + + def forward(self, x, feat=True): + x = self.rdb(x) + for sam_block in self.sam_blocks: + x = sam_block(x) + out = self.conv(x) + out = F.pixel_shuffle(out, 2) + if feat: + feature = F.interpolate(x, scale_factor=2, mode='bilinear') + return out, feature + else: + return out + + +class DB(nn.Module): + + def __init__(self, in_channel, d_list, inter_num): + super(DB, self).__init__() + self.d_list = d_list + self.conv_layers = nn.ModuleList() + c = in_channel + for i in range(len(d_list)): + dense_conv = conv_relu( + in_channel=c, + out_channel=inter_num, + kernel_size=3, + dilation_rate=d_list[i], + padding=d_list[i]) + self.conv_layers.append(dense_conv) + c = c + inter_num + self.conv_post = conv( + in_channel=c, out_channel=in_channel, kernel_size=1) + + def forward(self, x): + t = x + for conv_layer in self.conv_layers: + _t = conv_layer(t) + t = torch.cat([_t, t], dim=1) + t = self.conv_post(t) + return t + + +class SAM(nn.Module): + + def __init__(self, in_channel, d_list, inter_num): + super(SAM, self).__init__() + self.basic_block = DB( + in_channel=in_channel, d_list=d_list, inter_num=inter_num) + self.basic_block_2 = DB( + in_channel=in_channel, d_list=d_list, inter_num=inter_num) + self.basic_block_4 = DB( + in_channel=in_channel, d_list=d_list, inter_num=inter_num) + self.fusion = CSAF(3 * in_channel) + + def forward(self, x): + x_0 = x + x_2 = F.interpolate(x, scale_factor=0.5, mode='bilinear') + x_4 = F.interpolate(x, scale_factor=0.25, mode='bilinear') + y_0 = self.basic_block(x_0) + y_2 = self.basic_block_2(x_2) + y_4 = self.basic_block_4(x_4) + y_2 = F.interpolate(y_2, scale_factor=2, mode='bilinear') + y_4 = F.interpolate(y_4, scale_factor=4, mode='bilinear') + y = self.fusion(y_0, y_2, y_4) + y = x + y + return y + + +class CSAF(nn.Module): + + def __init__(self, in_chnls, ratio=4): + super(CSAF, self).__init__() + self.squeeze = nn.AdaptiveAvgPool2d((1, 1)) + self.compress1 = nn.Conv2d(in_chnls, in_chnls // ratio, 1, 1, 0) + self.compress2 = nn.Conv2d(in_chnls // ratio, in_chnls // ratio, 1, 1, + 0) + self.excitation = nn.Conv2d(in_chnls // ratio, in_chnls, 1, 1, 0) + + def forward(self, x0, x2, x4): + out0 = self.squeeze(x0) + out2 = self.squeeze(x2) + out4 = self.squeeze(x4) + out = torch.cat([out0, out2, out4], dim=1) + out = self.compress1(out) + out = F.relu(out) + out = self.compress2(out) + out = F.relu(out) + out = self.excitation(out) + out = F.sigmoid(out) + w0, w2, w4 = torch.chunk(out, 3, dim=1) + x = x0 * w0 + x2 * w2 + x4 * w4 + return x + + +class RDB(nn.Module): + + def __init__(self, in_channel, d_list, inter_num): + super(RDB, self).__init__() + self.d_list = d_list + self.conv_layers = nn.ModuleList() + c = in_channel + for i in range(len(d_list)): + dense_conv = conv_relu( + in_channel=c, + out_channel=inter_num, + kernel_size=3, + dilation_rate=d_list[i], + padding=d_list[i]) + self.conv_layers.append(dense_conv) + c = c + inter_num + self.conv_post = conv( + in_channel=c, out_channel=in_channel, kernel_size=1) + + def forward(self, x): + t = x + for conv_layer in self.conv_layers: + _t = conv_layer(t) + t = torch.cat([_t, t], dim=1) + t = self.conv_post(t) + return t + x + + +class conv(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + dilation_rate=1, + padding=0, + stride=1): + super(conv, self).__init__() + self.conv = nn.Conv2d( + in_channels=in_channel, + out_channels=out_channel, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=True, + dilation=dilation_rate) + + def forward(self, x_input): + out = self.conv(x_input) + return out + + +class conv_relu(nn.Module): + + def __init__(self, + in_channel, + out_channel, + kernel_size, + dilation_rate=1, + padding=0, + stride=1): + super(conv_relu, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d( + in_channels=in_channel, + out_channels=out_channel, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=True, + dilation=dilation_rate), nn.ReLU(inplace=True)) + + def forward(self, x_input): + out = self.conv(x_input) + return out diff --git a/runtime/Lib/site_packages/models/cv/image_restoration/image_restoration_model.py b/runtime/Lib/site_packages/models/cv/image_restoration/image_restoration_model.py new file mode 100644 index 0000000000000000000000000000000000000000..b60a360cc6351479c85a7a9233ad02927481b95d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_restoration/image_restoration_model.py @@ -0,0 +1,69 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import cv2 +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .demoire_models import model_map + + +@MODELS.register_module( + Tasks.image_demoireing, module_name=Models.image_restoration) +class ImageRestorationModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + config_path = osp.join(model_dir, ModelFile.CONFIGURATION) + config = Config.from_file(config_path) + model_name = config.model.network_type + model_class = model_map[model_name] + self.model = model_class(**config.model.network_param) + checkpoint = torch.load(model_path, map_location='cpu') + self.model.load_state_dict(checkpoint) + self.model.eval() + self.pad_32 = config.preprocessor.pad_32 + + def inference(self, data): + """data is tensor -1 * C * H * W ---> return tensor -1 * C * H * W .""" + if next(self.model.parameters()).is_cuda: + data = data.to( + torch.device([next(self.model.parameters()).device][0])) + with torch.no_grad(): + results = self.model(data) + if next(self.model.parameters()).is_cuda: + return results[0].cpu() + return results[0] + + def forward(self, inputs): + """inputs is dict""" + data = self.inference(inputs['img']) + outputs = inputs + outputs['img'] = data + return outputs + + def postprocess(self, inputs): + """ inputs is dict return is numpy""" + data = inputs['img'][0, :, :, :] + if self.pad_32: + h_pad = inputs['h_pad'] + h_odd_pad = inputs['h_odd_pad'] + w_pad = inputs['w_pad'] + w_odd_pad = inputs['w_odd_pad'] + if h_pad != 0: + data = data[:, h_pad:-h_odd_pad, :] + if w_pad != 0: + data = data[:, :, w_pad:-w_odd_pad] + data_norm_np = np.array(np.clip(data.numpy(), 0, 1) + * 255).astype('uint8').transpose(1, 2, 0) + if data_norm_np.shape[0] != inputs['img_h']: + data_norm_np = cv2.resize(data_norm_np, + (inputs['img_w'], inputs['img_h'])) + return data_norm_np diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..46eeeee4c870af8232c63aa547b999cfc2dc960c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .semantic_seg_model import SemanticSegmentation + from .segformer import Segformer + from .ddpm_segmentation_model import DDPMSegmentationModel + +else: + _import_structure = { + 'semantic_seg_model': ['SemanticSegmentation'], + 'segformer': ['Segformer'], + 'ddpm_segmentation_model': ['DDPMSegmentationModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/data_util.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/data_util.py new file mode 100644 index 0000000000000000000000000000000000000000..7fba552d69758813bc850ab0a0de0492fcd810ab --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/data_util.py @@ -0,0 +1,291 @@ +# The implementation here is adopted from ddpm-segmentation, +# originally Apache 2.0 License and publicly available at https://github.com/yandex-research/ddpm-segmentation + + +def get_palette(category): + if category == 'ffhq_34': + return ffhq_palette + elif category == 'bedroom_28': + return bedroom_palette + elif category == 'cat_15': + return cat_palette + elif category == 'horse_21': + return horse_palette + elif category == 'ade_bedroom_30': + return ade_bedroom_30_palette + elif category == 'celeba_19': + return celeba_palette + + +def get_class_names(category): + if category == 'ffhq_34': + return ffhq_class + elif category == 'bedroom_28': + return bedroom_class + elif category == 'cat_15': + return cat_class + elif category == 'horse_21': + return horse_class + elif category == 'ade_bedroom_30': + return ade_bedroom_30_class + elif category == 'celeba_19': + return celeba_class + + +############### +# Class names # +############### + +bedroom_class = [ + 'background', 'bed', 'bed***footboard', 'bed***headboard', + 'bed***side rail', 'carpet', 'ceiling', 'chandelier / ceiling fan blade', + 'curtain', 'cushion', 'floor', 'table/nightstand/dresser', + 'table/nightstand/dresser***top', 'picture / mirrow', 'pillow', + 'lamp***column', 'lamp***shade', 'wall', 'window', 'curtain rod', + 'window***frame', 'chair', 'picture / mirror***frame', 'plinth', + 'door / door frame', 'pouf', 'wardrobe', 'plant', 'table staff' +] + +ffhq_class = [ + 'background', 'head', 'head***cheek', 'head***chin', 'head***ear', + 'head***ear***helix', 'head***ear***lobule', 'head***eye***bottom lid', + 'head***eye***eyelashes', 'head***eye***iris', 'head***eye***pupil', + 'head***eye***sclera', 'head***eye***tear duct', 'head***eye***top lid', + 'head***eyebrow', 'head***forehead', 'head***frown', 'head***hair', + 'head***hair***sideburns', 'head***jaw', 'head***moustache', + 'head***mouth***inferior lip', 'head***mouth***oral commissure', + 'head***mouth***superior lip', 'head***mouth***teeth', 'head***neck', + 'head***nose', 'head***nose***ala of nose', 'head***nose***bridge', + 'head***nose***nose tip', 'head***nose***nostril', 'head***philtrum', + 'head***temple', 'head***wrinkles' +] + +cat_class = [ + 'background', 'back', 'belly', 'chest', 'leg', 'paw', 'head', 'ear', 'eye', + 'mouth', 'tongue', 'nose', 'tail', 'whiskers', 'neck' +] + +horse_class = [ + 'background', 'person', 'back', 'barrel', 'bridle', 'chest', 'ear', 'eye', + 'forelock', 'head', 'hoof', 'leg', 'mane', 'muzzle', 'neck', 'nostril', + 'tail', 'thigh', 'saddle', 'shoulder', 'leg protection' +] + +celeba_class = [ + 'background', 'cloth', 'ear_r', 'eye_g', 'hair', 'hat', 'l_brow', 'l_ear', + 'l_eye', 'l_lip', 'mouth', 'neck', 'neck_l', 'nose', 'r_brow', 'r_ear', + 'r_eye', 'skin', 'u_lip' +] + +ade_bedroom_50_class = [ + 'wall', 'bed', 'floor', 'table', 'lamp', 'ceiling', 'painting', + 'windowpane', 'pillow', 'curtain', 'cushion', 'door', 'chair', 'cabinet', + 'chest', 'mirror', 'rug', 'armchair', 'book', 'sconce', 'plant', + 'wardrobe', 'clock', 'light', 'flower', 'vase', 'fan', 'box', 'shelf', + 'television', 'blind', 'pot', 'ottoman', 'sofa', 'desk', 'basket', + 'blanket', 'coffee', 'plaything', 'radiator', 'tray', 'stool', 'bottle', + 'chandelier', 'fireplacel', 'towel', 'railing', 'canopy', 'glass', 'plate' +] + +ade_bedroom_40_class = ade_bedroom_50_class[:40] +ade_bedroom_30_class = ade_bedroom_50_class[:30] + +########### +# Palette # +########### + +ffhq_palette = [ + 1.0000, 1.0000, 1.0000, 0.4420, 0.5100, 0.4234, 0.8562, 0.9537, 0.3188, + 0.2405, 0.4699, 0.9918, 0.8434, 0.9329, 0.7544, 0.3748, 0.7917, 0.3256, + 0.0190, 0.4943, 0.3782, 0.7461, 0.0137, 0.5684, 0.1644, 0.2402, 0.7324, + 0.0200, 0.4379, 0.4100, 0.5853, 0.8880, 0.6137, 0.7991, 0.9132, 0.9720, + 0.6816, 0.6237, 0.8562, 0.9981, 0.4692, 0.3849, 0.5351, 0.8242, 0.2731, + 0.1747, 0.3626, 0.8345, 0.5323, 0.6668, 0.4922, 0.2122, 0.3483, 0.4707, + 0.6844, 0.1238, 0.1452, 0.3882, 0.4664, 0.1003, 0.2296, 0.0401, 0.3030, + 0.5751, 0.5467, 0.9835, 0.1308, 0.9628, 0.0777, 0.2849, 0.1846, 0.2625, + 0.9764, 0.9420, 0.6628, 0.3893, 0.4456, 0.6433, 0.8705, 0.3957, 0.0963, + 0.6117, 0.9702, 0.0247, 0.3668, 0.6694, 0.3117, 0.6451, 0.7302, 0.9542, + 0.6171, 0.1097, 0.9053, 0.3377, 0.4950, 0.7284, 0.1655, 0.9254, 0.6557, + 0.9450, 0.6721, 0.6162 +] + +ffhq_palette = [int(item * 255) for item in ffhq_palette] + +bedroom_palette = [ + 255, + 255, + 255, # bg + 238, + 229, + 102, # bed + 255, + 72, + 69, # bed footboard + 124, + 99, + 34, # bed headboard + 193, + 127, + 15, # bed side rail + 106, + 177, + 21, # carpet + 248, + 213, + 43, # ceiling + 252, + 155, + 83, # chandelier / ceiling fan blade + 220, + 147, + 77, # curtain + 99, + 83, + 3, # cushion + 116, + 116, + 138, # floor + 63, + 182, + 24, # table/nightstand/dresser + 200, + 226, + 37, # table/nightstand/dresser top + 225, + 184, + 161, # picture / mirrow + 233, + 5, + 219, # pillow + 142, + 172, + 248, # lamp column + 153, + 112, + 146, # lamp shade + 38, + 112, + 254, # wall + 229, + 30, + 141, # window + 99, + 205, + 255, # curtain rod + 74, + 59, + 83, # window frame + 186, + 9, + 0, # chair + 107, + 121, + 0, # picture / mirrow frame + 0, + 194, + 160, # plinth + 255, + 170, + 146, # door / door frame + 255, + 144, + 201, # pouf + 185, + 3, + 170, # wardrobe + 221, + 239, + 255, # plant + 0, + 0, + 53, # table staff +] + +cat_palette = [ + 255, 255, 255, 190, 153, 153, 250, 170, 30, 220, 220, 0, 107, 142, 35, 102, + 102, 156, 152, 251, 152, 119, 11, 32, 244, 35, 232, 220, 20, 60, 52, 83, + 84, 194, 87, 125, 143, 176, 255, 31, 102, 211, 104, 131, 101 +] + +horse_palette = [ + 255, 255, 255, 255, 74, 70, 0, 137, 65, 0, 111, 166, 163, 0, 89, 255, 219, + 229, 122, 73, 0, 0, 0, 166, 99, 255, 172, 183, 151, 98, 0, 77, 67, 143, + 176, 255, 241, 38, 110, 27, 210, 105, 128, 150, 147, 228, 230, 158, 160, + 136, 106, 79, 198, 1, 59, 93, 255, 115, 214, 209, 255, 47, 128 +] + +celeba_palette = [ + 255, + 255, + 255, # 0 background + 238, + 229, + 102, # 1 cloth + 250, + 150, + 50, # 2 ear_r + 124, + 99, + 34, # 3 eye_g + 193, + 127, + 15, # 4 hair + 225, + 96, + 18, # 5 hat + 220, + 147, + 77, # 6 l_brow + 99, + 83, + 3, # 7 l_ear + 116, + 116, + 138, # 8 l_eye + 200, + 226, + 37, # 9 l_lip + 225, + 184, + 161, # 10 mouth + 142, + 172, + 248, # 11 neck + 153, + 112, + 146, # 12 neck_l + 38, + 112, + 254, # 13 nose + 229, + 30, + 141, # 14 r_brow + 52, + 83, + 84, # 15 r_ear + 194, + 87, + 125, # 16 r_eye + 248, + 213, + 42, # 17 skin + 31, + 102, + 211, # 18 u_lip +] + +ade_bedroom_50_palette = [ + 240, 156, 206, 69, 88, 93, 240, 49, 184, 27, 107, 126, 50, 82, 241, 54, + 250, 147, 156, 213, 3, 176, 108, 79, 251, 150, 149, 66, 51, 34, 210, 97, + 53, 30, 53, 102, 232, 164, 118, 204, 150, 17, 101, 86, 178, 249, 20, 213, + 54, 35, 82, 157, 68, 216, 58, 161, 73, 174, 67, 67, 193, 181, 78, 169, 60, + 178, 220, 204, 166, 4, 127, 85, 245, 106, 216, 222, 172, 168, 84, 148, 105, + 137, 220, 89, 68, 252, 126, 29, 193, 187, 74, 40, 101, 52, 71, 61, 38, 92, + 205, 40, 104, 224, 146, 74, 160, 69, 43, 220, 70, 78, 213, 249, 93, 254, + 235, 71, 119, 193, 255, 102, 152, 55, 238, 133, 12, 223, 106, 116, 123, 86, + 14, 174, 244, 160, 161, 142, 105, 60, 153, 61, 124, 195, 156, 253, 241, 84, + 222, 202, 171, 227 +] + +ade_bedroom_40_palette = ade_bedroom_50_palette[:120] +ade_bedroom_30_palette = ade_bedroom_50_palette[:90] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/feature_extractors.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/feature_extractors.py new file mode 100644 index 0000000000000000000000000000000000000000..2cffe43ce7ab178f2fd9a5ee64681dac66b806f2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/feature_extractors.py @@ -0,0 +1,137 @@ +# The implementation here is modified based on ddpm-segmentation, +# originally Apache 2.0 License and publicly available at https://github.com/yandex-research/ddpm-segmentation +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import List + +import torch +from torch import nn + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def create_feature_extractor(model_type, **kwargs): + """ Create the feature extractor for architecture. """ + if model_type == 'ddpm': + logger.info('Creating DDPM Feature Extractor...') + feature_extractor = FeatureExtractorDDPM(**kwargs) + else: + raise Exception(f'Wrong model type: {model_type}') + return feature_extractor + + +def save_tensors(module: nn.Module, features, name: str): + """ Process and save activations in the module. """ + if type(features) in [list, tuple]: + features = [ + f.detach().float() if f is not None else None for f in features + ] + setattr(module, name, features) + elif isinstance(features, dict): + features = {k: f.detach().float() for k, f in features.items()} + setattr(module, name, features) + else: + setattr(module, name, features.detach().float()) + + +def save_out_hook(self, inp, out): + save_tensors(self, out, 'activations') + # return out + + +def save_input_hook(self, inp, out): + save_tensors(self, inp[0], 'activations') + # return out + + +class FeatureExtractor(nn.Module): + + def __init__(self, model_path: str, input_activations: bool, **kwargs): + ''' + Parent feature extractor class. + + param: model_path: path to the pretrained model + param: input_activations: + If True, features are input activations of the corresponding blocks + If False, features are output activations of the corresponding blocks + ''' + super().__init__() + self._load_pretrained_model(model_path, **kwargs) + logger.info( + f'Pretrained model is successfully loaded from {model_path}') + self.save_hook = save_input_hook if input_activations else save_out_hook + self.feature_blocks = [] + + def _load_pretrained_model(self, model_path: str, **kwargs): + pass + + +class FeatureExtractorDDPM(FeatureExtractor): + ''' + Wrapper to extract features from pretrained DDPMs. + + :param steps: list of diffusion steps t. + :param blocks: list of the UNet decoder blocks. + ''' + + def __init__(self, steps: List[int], blocks: List[int], **kwargs): + super().__init__(**kwargs) + self.steps = steps + + # Save decoder activations + for idx, block in enumerate(self.model.output_blocks): + if idx in blocks: + block.register_forward_hook(self.save_hook) + self.feature_blocks.append(block) + + def _load_pretrained_model(self, model_path, **kwargs): + import inspect + from ddpm_guided_diffusion.script_util import create_model_and_diffusion + + # Needed to pass only expected args to the function + argnames = inspect.getfullargspec(create_model_and_diffusion)[0] + expected_args = {name: kwargs[name] for name in argnames} + self.model, self.diffusion = create_model_and_diffusion( + **expected_args) + + state_dict = torch.load(model_path, map_location='cpu') + + self.model.load_state_dict(state_dict) + + if kwargs['use_fp16']: + self.model.convert_to_fp16() + self.model.eval() + + @torch.no_grad() + def forward(self, x, noise=None): + activations = [] + for t in self.steps: + # Compute x_t and run DDPM + t = torch.tensor([t]).to(x.device) + + noisy_x = self.diffusion.q_sample(x, t, noise=noise) + self.model(noisy_x, self.diffusion._scale_timesteps(t)) + + # Extract activations + for block in self.feature_blocks: + activations.append(block.activations) + block.activations = None + + # Per-layer list of activations [N, C, H, W] + return activations + + +def collect_features(cfg, activations: List[torch.Tensor], sample_idx=0): + """ Upsample activations and concatenate them to form a feature tensor """ + assert all([isinstance(acts, torch.Tensor) for acts in activations]) + size = tuple(cfg.mlp.dim[:-1]) + resized_activations = [] + for feats in activations: + feats = feats[sample_idx][None] + feats = nn.functional.interpolate( + feats, size=size, mode=cfg.ddpm.upsample_mode) + resized_activations.append(feats[0]) + + return torch.cat(resized_activations, dim=0) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/pixel_classifier.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/pixel_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..c25fb3641ecd3c3b42aee07eba03bf35dd485cc3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/pixel_classifier.py @@ -0,0 +1,151 @@ +# The implementation here is modified based on ddpm-segmentation, +# originally Apache 2.0 License and publicly available at https://github.com/yandex-research/ddpm-segmentation +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import Counter + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from torch.distributions import Categorical + +from .data_util import get_class_names, get_palette +from .utils import colorize_mask, oht_to_scalar + + +# Adopted from https://github.com/nv-tlabs/datasetGAN/train_interpreter.py +class pixel_classifier(nn.Module): + + def __init__(self, category, dim, **kwargs): + super(pixel_classifier, self).__init__() + category_cfg = kwargs.get(category, None) + assert category_cfg is not None + class_num = category_cfg['number_class'] + + dim = dim[-1] + + if class_num < 30: + self.layers = nn.Sequential( + nn.Linear(dim, + 128), nn.ReLU(), nn.BatchNorm1d(num_features=128), + nn.Linear(128, 32), nn.ReLU(), nn.BatchNorm1d(num_features=32), + nn.Linear(32, class_num)) + else: + self.layers = nn.Sequential( + nn.Linear(dim, + 256), nn.ReLU(), nn.BatchNorm1d(num_features=256), + nn.Linear(256, 128), nn.ReLU(), + nn.BatchNorm1d(num_features=128), nn.Linear(128, class_num)) + + def init_weights(self, init_type='normal', gain=0.02): + ''' + initialize network's weights + init_type: normal | xavier | kaiming | orthogonal + https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/9451e70673400885567d08a9e97ade2524c700d0/models/networks.py#L39 + ''' + + def init_func(m): + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + if init_type == 'normal': + nn.init.normal_(m.weight.data, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight.data, gain=gain) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight.data, gain=gain) + + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias.data, 0.0) + + elif classname.find('BatchNorm2d') != -1: + nn.init.normal_(m.weight.data, 1.0, gain) + nn.init.constant_(m.bias.data, 0.0) + + self.apply(init_func) + + def forward(self, x): + return self.layers(x) + + +def predict_labels(models, features, size): + if isinstance(features, np.ndarray): + features = torch.from_numpy(features) + + mean_seg = None + all_seg = [] + all_entropy = [] + seg_mode_ensemble = [] + + softmax_f = nn.Softmax(dim=1) + with torch.no_grad(): + for MODEL_NUMBER in range(len(models)): + models[MODEL_NUMBER].to(features.device) + preds = models[MODEL_NUMBER](features) + entropy = Categorical(logits=preds).entropy() + all_entropy.append(entropy) + all_seg.append(preds) + + if mean_seg is None: + mean_seg = softmax_f(preds) + else: + mean_seg += softmax_f(preds) + + img_seg = oht_to_scalar(preds) + img_seg = img_seg.reshape(*size) + img_seg = img_seg.cpu().detach() + + seg_mode_ensemble.append(img_seg) + + mean_seg = mean_seg / len(all_seg) + + full_entropy = Categorical(mean_seg).entropy() + + js = full_entropy - torch.mean(torch.stack(all_entropy), 0) + top_k = js.sort()[0][-int(js.shape[0] / 10):].mean() + + img_seg_final = torch.stack(seg_mode_ensemble, dim=-1) + img_seg_final = torch.mode(img_seg_final, 2)[0] + return img_seg_final, top_k + + +def load_ensemble(model_num=1, + model_path='', + device='cpu', + category='ffhq_34', + dim=[256, 256, 8448], + **kwargs): + models = [] + + for i in range(model_num): + per_model_path = os.path.join(model_path, f'model_{i}.pth') + state_dict = torch.load( + per_model_path, map_location='cpu')['model_state_dict'] + new_state_dict = { + k.replace('module.', ''): v + for k, v in state_dict.items() + } + model = pixel_classifier(category, dim, **kwargs) + model.load_state_dict(new_state_dict) + models.append(model.eval()) + return models + + +def save_predictions(preds, category='ffhq_34'): + palette = get_palette(category) + + masks = [] + out_imgs = [] + for i, pred in enumerate(preds['pred']): + + pred = np.squeeze(pred) + masks.append(pred) + + out_img = colorize_mask(pred, palette) + out_img = Image.fromarray(out_img) + out_imgs.append(out_img) + return masks, out_imgs diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/utils.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3eeafe96c87183c94cc9c8b2c97d5daa54279fe1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_seg/utils.py @@ -0,0 +1,71 @@ +""" +Copyright (C) 2021 NVIDIA Corporation. All rights reserved. +Licensed under The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import random + +import numpy as np +import torch +from PIL import Image + + +def multi_acc(y_pred, y_test): + y_pred_softmax = torch.log_softmax(y_pred, dim=1) + _, y_pred_tags = torch.max(y_pred_softmax, dim=1) + + correct_pred = (y_pred_tags == y_test).float() + acc = correct_pred.sum() / len(correct_pred) + + acc = acc * 100 + + return acc + + +def oht_to_scalar(y_pred): + y_pred_softmax = torch.log_softmax(y_pred, dim=1) + _, y_pred_tags = torch.max(y_pred_softmax, dim=1) + + return y_pred_tags + + +def colorize_mask(mask, palette): + # mask: numpy array of the mask + new_mask = Image.fromarray(mask.astype(np.uint8)).convert('P') + new_mask.putpalette(palette) + return np.array(new_mask.convert('RGB')) + + +def to_labels(masks, palette): + results = np.zeros((len(masks), 256, 256), dtype=np.int32) + label = 0 + for color in palette: + idxs = np.where((masks == color).all(-1)) + results[idxs] = label + label += 1 + return results + + +def setup_seed(seed): + print('Seed: ', seed) + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_segmentation_model.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_segmentation_model.py new file mode 100644 index 0000000000000000000000000000000000000000..1a76e4dc73e3913a86aca132ee4b35dba9eb8e43 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/ddpm_segmentation_model.py @@ -0,0 +1,80 @@ +# The implementation here is modified based on ddpm-segmentation, +# originally Apache 2.0 License and publicly available at https://github.com/yandex-research/ddpm-segmentation +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp +from typing import Any, Dict + +import torch +from ddpm_guided_diffusion.script_util import model_and_diffusion_defaults + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .ddpm_seg.feature_extractors import (collect_features, + create_feature_extractor) +from .ddpm_seg.pixel_classifier import (load_ensemble, pixel_classifier, + predict_labels, save_predictions) + +logger = get_logger() + + +@MODELS.register_module(Tasks.semantic_segmentation, module_name=Models.ddpm) +class DDPMSegmentationModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + + config_path = osp.join(model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(config_path) + self.cfg.model.mlp.category = self.cfg.pipeline.category + + self.cfg.model.ddpm.model_path = osp.join(model_dir, + ModelFile.TORCH_MODEL_FILE) + default_ddpm_args = model_and_diffusion_defaults() + default_ddpm_args.update(self.cfg.model.ddpm) + self.feature_extractor = create_feature_extractor(**default_ddpm_args) + + self.cfg.model.mlp.model_path = osp.join(model_dir, + self.cfg.model.mlp.category) + self.is_ensemble = kwargs.get('is_pipeline', True) + if self.is_ensemble: + logger.info('Load ensemble mlp ......') + self.seg_model = load_ensemble(**self.cfg.model.mlp) + else: + logger.info('Load single mlp ......') + self.seg_model = pixel_classifier(**self.cfg.model.mlp) + + def forward(self, inputs: Dict[str, Any]) -> Dict[str, torch.Tensor]: + re = self.inference(inputs) + return re + + def inference(self, batch, seed=0): + img = batch['input_img'] + img = img[None] + w, h, c = self.cfg.model.mlp.dim + + if self.cfg.model.ddpm.share_noise: + rnd_gen = torch.Generator().manual_seed(seed) + noise = torch.randn(1, 3, w, h, generator=rnd_gen) + noise = noise.to(img.device) + else: + noise = None + + features = self.feature_extractor(img, noise=noise) + features = collect_features(self.cfg.model, features) + + x = features.view(c, -1).permute(1, 0) + + pred, _ = predict_labels(self.seg_model, x, size=(w, h)) + + return {'pred': [pred.numpy()]} + + def postprocess(self, inputs: Dict[str, Any], **kwargs): + category = self.cfg.model.mlp.category + mask, out_img = save_predictions(inputs, category) + return mask, out_img diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a31a308d08fe56946ba838fa4c2fc0bbd37a0aa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .maskformer_semantic_head import MaskFormerSemanticHead diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/base_panoptic_fusion_head.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/base_panoptic_fusion_head.py new file mode 100644 index 0000000000000000000000000000000000000000..05e68d89a922ec349af92b9ed47633ca554cdd02 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/base_panoptic_fusion_head.py @@ -0,0 +1,47 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta, abstractmethod + +from mmcv.runner import BaseModule +from mmdet.models.builder import build_loss + + +class BasePanopticFusionHead(BaseModule, metaclass=ABCMeta): + """Base class for panoptic heads.""" + + def __init__(self, + num_things_classes=80, + num_stuff_classes=53, + test_cfg=None, + loss_panoptic=None, + init_cfg=None, + **kwargs): + super(BasePanopticFusionHead, self).__init__(init_cfg) + self.num_things_classes = num_things_classes + self.num_stuff_classes = num_stuff_classes + self.num_classes = num_things_classes + num_stuff_classes + self.test_cfg = test_cfg + + if loss_panoptic: + self.loss_panoptic = build_loss(loss_panoptic) + else: + self.loss_panoptic = None + + @property + def with_loss(self): + """bool: whether the panoptic head contains loss function.""" + return self.loss_panoptic is not None + + @abstractmethod + def forward_train(self, gt_masks=None, gt_semantic_seg=None, **kwargs): + """Forward function during training.""" + + @abstractmethod + def simple_test(self, + img_metas, + det_labels, + mask_preds, + seg_preds, + det_bboxes, + cfg=None, + **kwargs): + """Test without augmentation.""" diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/maskformer_semantic_head.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/maskformer_semantic_head.py new file mode 100644 index 0000000000000000000000000000000000000000..2f3364d007c87578c10bf9dabaa7167b58159bab --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/pan_merge/maskformer_semantic_head.py @@ -0,0 +1,58 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn.functional as F +from mmdet.models.builder import HEADS + +from .base_panoptic_fusion_head import BasePanopticFusionHead + + +@HEADS.register_module() +class MaskFormerSemanticHead(BasePanopticFusionHead): + + def __init__(self, + num_things_classes=80, + num_stuff_classes=53, + test_cfg=None, + loss_panoptic=None, + init_cfg=None, + **kwargs): + super().__init__(num_things_classes, num_stuff_classes, test_cfg, + loss_panoptic, init_cfg, **kwargs) + + def forward_train(self, **kwargs): + """MaskFormerFusionHead has no training loss.""" + return dict() + + def simple_test(self, + mask_cls_results, + mask_pred_results, + img_metas, + rescale=False, + **kwargs): + results = [] + for mask_cls_result, mask_pred_result, meta in zip( + mask_cls_results, mask_pred_results, img_metas): + # remove padding + img_height, img_width = meta['img_shape'][:2] + mask_pred_result = mask_pred_result[:, :img_height, :img_width] + + if rescale: + # return result in original resolution + ori_height, ori_width = meta['ori_shape'][:2] + mask_pred_result = F.interpolate( + mask_pred_result[:, None], + size=(ori_height, ori_width), + mode='bilinear', + align_corners=False)[:, 0] + + # semantic inference + cls_score = F.softmax(mask_cls_result, dim=-1)[..., :-1] + mask_pred = mask_pred_result.sigmoid() + seg_mask = torch.einsum('qc,qhw->chw', cls_score, mask_pred) + # still need softmax and argmax + seg_logit = F.softmax(seg_mask, dim=0) + seg_pred = seg_logit.argmax(dim=0) + seg_pred = seg_pred.cpu().numpy() + results.append(seg_pred) + + return results diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/semantic_seg_model.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/semantic_seg_model.py new file mode 100644 index 0000000000000000000000000000000000000000..455f29fb8f63556c1e3c6176a75cd99103712fdd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/semantic_seg_model.py @@ -0,0 +1,77 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_semantic_segmentation import (pan_merge, + vit_adapter) +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.image_segmentation, module_name=Models.swinL_semantic_segmentation) +@MODELS.register_module( + Tasks.image_segmentation, + module_name=Models.vitadapter_semantic_segmentation) +class SemanticSegmentation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + from mmcv.runner import load_checkpoint + import mmcv + from mmdet.models import build_detector + + config = osp.join(model_dir, 'mmcv_config.py') + cfg = mmcv.Config.fromfile(config) + if 'pretrained' in cfg.model: + cfg.model.pretrained = None + elif 'init_cfg' in cfg.model.backbone: + cfg.model.backbone.init_cfg = None + + # build model + cfg.model.train_cfg = None + self.model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + _ = load_checkpoint(self.model, model_path, map_location='cpu') + + self.CLASSES = cfg['CLASSES'] # list + self.PALETTE = cfg['PALETTE'] # list + + self.num_classes = len(self.CLASSES) + self.cfg = cfg + + def forward(self, Inputs): + return self.model(**Inputs) + + def postprocess(self, Inputs): + semantic_result = Inputs[0] + + ids = np.unique(semantic_result)[::-1] + legal_indices = ids != self.model.num_classes # for VOID label + ids = ids[legal_indices] + + segms = (semantic_result[None] == ids[:, None, None]) + masks = [it.astype(int) for it in segms] + labels_txt = np.array(self.CLASSES)[ids].tolist() + + results = { + OutputKeys.MASKS: masks, + OutputKeys.LABELS: labels_txt, + OutputKeys.SCORES: [0.999 for _ in range(len(labels_txt))] + } + return results + + def inference(self, data): + with torch.no_grad(): + results = self.model(return_loss=False, rescale=True, **data) + + return results diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b9a301cd2561a919f27bbd9952f78143b6a0e97 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/__init__.py @@ -0,0 +1,5 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .models import backbone, decode_heads, segmentors +from .utils import (ResizeToMultiple, add_prefix, build_pixel_sampler, + seg_resize) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..791dd26f287d8562644c3c013da324547b1d8944 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/__init__.py @@ -0,0 +1,5 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .backbone import BASEBEiT, BEiTAdapter +from .decode_heads import Mask2FormerHeadFromMMSeg +from .segmentors import EncoderDecoderMask2Former diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7abd0ef1b0366704149323ac4358d93196d02489 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/__init__.py @@ -0,0 +1,6 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .base import BASEBEiT +from .beit_adapter import BEiTAdapter + +__all__ = ['BEiTAdapter', 'BASEBEiT'] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/adapter_modules.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/adapter_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..cf30cca09d47b02d597a6a6c7659a9a48de3c086 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/adapter_modules.py @@ -0,0 +1,522 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git + +import logging +from functools import partial + +import torch +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmdet.models.utils.transformer import MultiScaleDeformableAttention +from timm.models.layers import DropPath + +_logger = logging.getLogger(__name__) + + +def get_reference_points(spatial_shapes, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace( + 0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace( + 0.5, W_ - 0.5, W_, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / H_ + ref_x = ref_x.reshape(-1)[None] / W_ + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] + return reference_points + + +def deform_inputs(x): + bs, c, h, w = x.shape + spatial_shapes = torch.as_tensor([(h // 8, w // 8), (h // 16, w // 16), + (h // 32, w // 32)], + dtype=torch.long, + device=x.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + reference_points = get_reference_points([(h // 16, w // 16)], x.device) + deform_inputs1 = [reference_points, spatial_shapes, level_start_index] + + spatial_shapes = torch.as_tensor([(h // 16, w // 16)], + dtype=torch.long, + device=x.device) + level_start_index = torch.cat((spatial_shapes.new_zeros( + (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + reference_points = get_reference_points([(h // 8, w // 8), + (h // 16, w // 16), + (h // 32, w // 32)], x.device) + deform_inputs2 = [reference_points, spatial_shapes, level_start_index] + + return deform_inputs1, deform_inputs2 + + +class ConvFFN(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x, H, W): + x = self.fc1(x) + x = self.dwconv(x, H, W) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class DWConv(nn.Module): + + def __init__(self, dim=768): + super().__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x, H, W): + B, N, C = x.shape + n = N // 21 + x1 = x[:, 0:16 * n, :].transpose(1, 2).view(B, C, H * 2, + W * 2).contiguous() + x2 = x[:, 16 * n:20 * n, :].transpose(1, 2).view(B, C, H, + W).contiguous() + x3 = x[:, 20 * n:, :].transpose(1, 2).view(B, C, H // 2, + W // 2).contiguous() + x1 = self.dwconv(x1).flatten(2).transpose(1, 2) + x2 = self.dwconv(x2).flatten(2).transpose(1, 2) + x3 = self.dwconv(x3).flatten(2).transpose(1, 2) + x = torch.cat([x1, x2, x3], dim=1) + return x + + +class Extractor(nn.Module): + + def __init__(self, + dim, + num_heads=6, + n_points=4, + n_levels=1, + deform_ratio=1.0, + with_cffn=True, + cffn_ratio=0.25, + drop=0., + drop_path=0., + norm_layer=partial(nn.LayerNorm, eps=1e-6), + with_cp=False): + super().__init__() + self.query_norm = norm_layer(dim) + self.feat_norm = norm_layer(dim) + self.attn = MultiScaleDeformableAttention( + embed_dims=dim, + num_heads=num_heads, + num_levels=n_levels, + num_points=n_points, + batch_first=True) + + # modify to fit the deform_ratio + value_proj_in_features = self.attn.value_proj.weight.shape[0] + value_proj_out_features = int(value_proj_in_features * deform_ratio) + self.attn.value_proj = nn.Linear(value_proj_in_features, + value_proj_out_features) + self.attn.output_proj = nn.Linear(value_proj_out_features, + value_proj_in_features) + + self.with_cffn = with_cffn + self.with_cp = with_cp + if with_cffn: + self.ffn = ConvFFN( + in_features=dim, + hidden_features=int(dim * cffn_ratio), + drop=drop) + self.ffn_norm = norm_layer(dim) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, query, reference_points, feat, spatial_shapes, + level_start_index, H, W): + + def _inner_forward(query, feat): + attn = self.attn( + query=self.query_norm(query), + key=None, + value=self.feat_norm(feat), + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index) + + query = query + attn + + if self.with_cffn: + query = query + self.drop_path( + self.ffn(self.ffn_norm(query), H, W)) + return query + + if self.with_cp and query.requires_grad: + query = cp.checkpoint(_inner_forward, query, feat) + else: + query = _inner_forward(query, feat) + + return query + + +class Injector(nn.Module): + + def __init__(self, + dim, + num_heads=6, + n_points=4, + n_levels=1, + deform_ratio=1.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + init_values=0., + with_cp=False): + super().__init__() + self.with_cp = with_cp + self.query_norm = norm_layer(dim) + self.feat_norm = norm_layer(dim) + self.attn = MultiScaleDeformableAttention( + embed_dims=dim, + num_heads=num_heads, + num_levels=n_levels, + num_points=n_points, + batch_first=True) + + # modify to fit the deform_ratio + value_proj_in_features = self.attn.value_proj.weight.shape[0] + value_proj_out_features = int(value_proj_in_features * deform_ratio) + self.attn.value_proj = nn.Linear(value_proj_in_features, + value_proj_out_features) + self.attn.output_proj = nn.Linear(value_proj_out_features, + value_proj_in_features) + + self.gamma = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + + def forward(self, query, reference_points, feat, spatial_shapes, + level_start_index): + + def _inner_forward(query, feat): + input_query = self.query_norm(query) + input_value = self.feat_norm(feat) + attn = self.attn( + query=input_query, + key=None, + value=input_value, + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index) + return query + self.gamma * attn + + if self.with_cp and query.requires_grad: + query = cp.checkpoint(_inner_forward, query, feat) + else: + query = _inner_forward(query, feat) + + return query + + +class InteractionBlock(nn.Module): + + def __init__(self, + dim, + num_heads=6, + n_points=4, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + drop=0., + drop_path=0., + with_cffn=True, + cffn_ratio=0.25, + init_values=0., + deform_ratio=1.0, + extra_extractor=False, + with_cp=False): + super().__init__() + + self.injector = Injector( + dim=dim, + n_levels=3, + num_heads=num_heads, + init_values=init_values, + n_points=n_points, + norm_layer=norm_layer, + deform_ratio=deform_ratio, + with_cp=with_cp) + self.extractor = Extractor( + dim=dim, + n_levels=1, + num_heads=num_heads, + n_points=n_points, + norm_layer=norm_layer, + deform_ratio=deform_ratio, + with_cffn=with_cffn, + cffn_ratio=cffn_ratio, + drop=drop, + drop_path=drop_path, + with_cp=with_cp) + if extra_extractor: + self.extra_extractors = nn.Sequential(*[ + Extractor( + dim=dim, + num_heads=num_heads, + n_points=n_points, + norm_layer=norm_layer, + with_cffn=with_cffn, + cffn_ratio=cffn_ratio, + deform_ratio=deform_ratio, + drop=drop, + drop_path=drop_path, + with_cp=with_cp) for _ in range(2) + ]) + else: + self.extra_extractors = None + + def forward(self, x, c, blocks, deform_inputs1, deform_inputs2, H, W): + x = self.injector( + query=x, + reference_points=deform_inputs1[0], + feat=c, + spatial_shapes=deform_inputs1[1], + level_start_index=deform_inputs1[2]) + for idx, blk in enumerate(blocks): + x = blk(x, H, W) + c = self.extractor( + query=c, + reference_points=deform_inputs2[0], + feat=x, + spatial_shapes=deform_inputs2[1], + level_start_index=deform_inputs2[2], + H=H, + W=W) + if self.extra_extractors is not None: + for extractor in self.extra_extractors: + c = extractor( + query=c, + reference_points=deform_inputs2[0], + feat=x, + spatial_shapes=deform_inputs2[1], + level_start_index=deform_inputs2[2], + H=H, + W=W) + return x, c + + +class InteractionBlockWithCls(nn.Module): + + def __init__(self, + dim, + num_heads=6, + n_points=4, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + drop=0., + drop_path=0., + with_cffn=True, + cffn_ratio=0.25, + init_values=0., + deform_ratio=1.0, + extra_extractor=False, + with_cp=False): + super().__init__() + + self.injector = Injector( + dim=dim, + n_levels=3, + num_heads=num_heads, + init_values=init_values, + n_points=n_points, + norm_layer=norm_layer, + deform_ratio=deform_ratio, + with_cp=with_cp) + self.extractor = Extractor( + dim=dim, + n_levels=1, + num_heads=num_heads, + n_points=n_points, + norm_layer=norm_layer, + deform_ratio=deform_ratio, + with_cffn=with_cffn, + cffn_ratio=cffn_ratio, + drop=drop, + drop_path=drop_path, + with_cp=with_cp) + if extra_extractor: + self.extra_extractors = nn.Sequential(*[ + Extractor( + dim=dim, + num_heads=num_heads, + n_points=n_points, + norm_layer=norm_layer, + with_cffn=with_cffn, + cffn_ratio=cffn_ratio, + deform_ratio=deform_ratio, + drop=drop, + drop_path=drop_path, + with_cp=with_cp) for _ in range(2) + ]) + else: + self.extra_extractors = None + + def forward(self, x, c, cls, blocks, deform_inputs1, deform_inputs2, H, W): + x = self.injector( + query=x, + reference_points=deform_inputs1[0], + feat=c, + spatial_shapes=deform_inputs1[1], + level_start_index=deform_inputs1[2]) + x = torch.cat((cls, x), dim=1) + for idx, blk in enumerate(blocks): + x = blk(x, H, W) + cls, x = x[:, :1, ], x[:, 1:, ] + c = self.extractor( + query=c, + reference_points=deform_inputs2[0], + feat=x, + spatial_shapes=deform_inputs2[1], + level_start_index=deform_inputs2[2], + H=H, + W=W) + if self.extra_extractors is not None: + for extractor in self.extra_extractors: + c = extractor( + query=c, + reference_points=deform_inputs2[0], + feat=x, + spatial_shapes=deform_inputs2[1], + level_start_index=deform_inputs2[2], + H=H, + W=W) + return x, c, cls + + +class SpatialPriorModule(nn.Module): + + def __init__(self, inplanes=64, embed_dim=384, with_cp=False): + super().__init__() + self.with_cp = with_cp + + self.stem = nn.Sequential(*[ + nn.Conv2d( + 3, inplanes, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(inplanes), + nn.ReLU(inplace=True), + nn.Conv2d( + inplanes, + inplanes, + kernel_size=3, + stride=1, + padding=1, + bias=False), + nn.BatchNorm2d(inplanes), + nn.ReLU(inplace=True), + nn.Conv2d( + inplanes, + inplanes, + kernel_size=3, + stride=1, + padding=1, + bias=False), + nn.BatchNorm2d(inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + ]) + self.conv2 = nn.Sequential(*[ + nn.Conv2d( + inplanes, + 2 * inplanes, + kernel_size=3, + stride=2, + padding=1, + bias=False), + nn.BatchNorm2d(2 * inplanes), + nn.ReLU(inplace=True) + ]) + self.conv3 = nn.Sequential(*[ + nn.Conv2d( + 2 * inplanes, + 4 * inplanes, + kernel_size=3, + stride=2, + padding=1, + bias=False), + nn.BatchNorm2d(4 * inplanes), + nn.ReLU(inplace=True) + ]) + self.conv4 = nn.Sequential(*[ + nn.Conv2d( + 4 * inplanes, + 4 * inplanes, + kernel_size=3, + stride=2, + padding=1, + bias=False), + nn.BatchNorm2d(4 * inplanes), + nn.ReLU(inplace=True) + ]) + self.fc1 = nn.Conv2d( + inplanes, embed_dim, kernel_size=1, stride=1, padding=0, bias=True) + self.fc2 = nn.Conv2d( + 2 * inplanes, + embed_dim, + kernel_size=1, + stride=1, + padding=0, + bias=True) + self.fc3 = nn.Conv2d( + 4 * inplanes, + embed_dim, + kernel_size=1, + stride=1, + padding=0, + bias=True) + self.fc4 = nn.Conv2d( + 4 * inplanes, + embed_dim, + kernel_size=1, + stride=1, + padding=0, + bias=True) + + def forward(self, x): + + def _inner_forward(x): + c1 = self.stem(x) + c2 = self.conv2(c1) + c3 = self.conv3(c2) + c4 = self.conv4(c3) + c1 = self.fc1(c1) + c2 = self.fc2(c2) + c3 = self.fc3(c3) + c4 = self.fc4(c4) + + bs, dim, _, _ = c1.shape + + c2 = c2.view(bs, dim, -1).transpose(1, 2) # 8s + c3 = c3.view(bs, dim, -1).transpose(1, 2) # 16s + c4 = c4.view(bs, dim, -1).transpose(1, 2) # 32s + + return c1, c2, c3, c4 + + if self.with_cp and x.requires_grad: + outs = cp.checkpoint(_inner_forward, x) + else: + outs = _inner_forward(x) + return outs diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5b33031fbbe10478cfaebcc0968be627e8074382 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/__init__.py @@ -0,0 +1,5 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .beit import BASEBEiT + +__all__ = ['BASEBEiT'] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/beit.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/beit.py new file mode 100644 index 0000000000000000000000000000000000000000..62f873ec901bd7ab9f5ccf748e93503b7b84ffdd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/base/beit.py @@ -0,0 +1,474 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +import math +from functools import partial + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.runner import _load_checkpoint +from mmdet.models.builder import BACKBONES +from mmdet.utils import get_root_logger +from timm.models.layers import drop_path, to_2tuple, trunc_normal_ + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + # commit dropout for the original BERT implement + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + window_size=None, + attn_head_dim=None): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, all_head_dim * 3, bias=False) + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(all_head_dim)) + self.v_bias = nn.Parameter(torch.zeros(all_head_dim)) + else: + self.q_bias = None + self.v_bias = None + + if window_size: + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] + - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter( + torch.zeros(self.num_relative_distance, + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, + coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = \ + torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, 1:] = relative_coords.sum( + -1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + self.register_buffer('relative_position_index', + relative_position_index) + + else: + self.window_size = None + self.relative_position_bias_table = None + self.relative_position_index = None + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, rel_pos_bias=None): + B, N, C = x.shape + qkv_bias = None + if self.q_bias is not None: + qkv_bias = torch.cat( + (self.q_bias, + torch.zeros_like(self.v_bias, + requires_grad=False), self.v_bias)) + + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + if self.relative_position_bias_table is not None: + relative_position_bias = \ + self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1] + 1, + self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + + attn = attn + relative_position_bias.unsqueeze(0) + + if rel_pos_bias is not None: + attn = attn + rel_pos_bias + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + init_values=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + window_size=None, + attn_head_dim=None, + with_cp=False): + super().__init__() + self.with_cp = with_cp + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + window_size=window_size, + attn_head_dim=attn_head_dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + if init_values is not None: + self.gamma_1 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + self.gamma_2 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + else: + self.gamma_1, self.gamma_2 = None, None + + def forward(self, x, H, W, rel_pos_bias=None): + + def _inner_forward(x): + if self.gamma_1 is None: + x = x + self.drop_path( + self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + x = x + self.drop_path(self.gamma_1 * self.attn( + self.norm1(x), rel_pos_bias=rel_pos_bias)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + if self.with_cp and x.requires_grad: + x = cp.checkpoint(_inner_forward, x) + else: + x = _inner_forward(x) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * ( + img_size[0] // patch_size[0]) + self.patch_shape = (img_size[0] // patch_size[0], + img_size[1] // patch_size[1]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x, **kwargs): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + # assert H == self.img_size[0] and W == self.img_size[1], \ + # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + Hp, Wp = x.shape[2], x.shape[3] + + x = x.flatten(2).transpose(1, 2) + return x, Hp, Wp + + +class HybridEmbed(nn.Module): + """ CNN Feature Map Embedding + Extract feature map from CNN, flatten, project to embedding dim. + """ + + def __init__(self, + backbone, + img_size=224, + feature_size=None, + in_chans=3, + embed_dim=768): + super().__init__() + assert isinstance(backbone, nn.Module) + img_size = to_2tuple(img_size) + self.img_size = img_size + self.backbone = backbone + if feature_size is None: + with torch.no_grad(): + # FIXME this is hacky, but most reliable way of determining the exact dim of the output feature + # map for all networks, the feature metadata has reliable channel and stride info, but using + # stride to calc feature dim requires info about padding of each stage that isn't captured. + training = backbone.training + if training: + backbone.eval() + o = self.backbone( + torch.zeros(1, in_chans, img_size[0], img_size[1]))[-1] + feature_size = o.shape[-2:] + feature_dim = o.shape[1] + backbone.train(training) + else: + feature_size = to_2tuple(feature_size) + feature_dim = self.backbone.feature_info.channels()[-1] + self.num_patches = feature_size[0] * feature_size[1] + self.proj = nn.Linear(feature_dim, embed_dim) + + def forward(self, x): + x = self.backbone(x)[-1] + x = x.flatten(2).transpose(1, 2) + x = self.proj(x) + return x + + +class RelativePositionBias(nn.Module): + + def __init__(self, window_size, num_heads): + super().__init__() + self.window_size = window_size + self.num_relative_distance = (2 * window_size[0] + - 1) * (2 * window_size[1] - 1) + 3 + self.relative_position_bias_table = nn.Parameter( + torch.zeros(self.num_relative_distance, + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + # cls to token & token 2 cls & cls to cls + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(window_size[0]) + coords_w = torch.arange(window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * window_size[1] - 1 + relative_position_index = \ + torch.zeros(size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype) + relative_position_index[1:, + 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = self.num_relative_distance - 3 + relative_position_index[0:, 0] = self.num_relative_distance - 2 + relative_position_index[0, 0] = self.num_relative_distance - 1 + + self.register_buffer('relative_position_index', + relative_position_index) + + def forward(self): + relative_position_bias = \ + self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1] + 1, + self.window_size[0] * self.window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + return relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + + +@BACKBONES.register_module() +class BASEBEiT(nn.Module): + """ Vision Transformer with support for patch or hybrid CNN input stage + """ + + def __init__(self, + img_size=512, + patch_size=16, + in_chans=3, + num_classes=80, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + hybrid_backbone=None, + norm_layer=None, + init_values=None, + use_checkpoint=False, + use_abs_pos_emb=False, + use_rel_pos_bias=True, + use_shared_rel_pos_bias=False, + pretrained=None, + with_cp=False): + super().__init__() + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + self.norm_layer = norm_layer + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.drop_path_rate = drop_path_rate + if hybrid_backbone is not None: + self.patch_embed = HybridEmbed( + hybrid_backbone, + img_size=img_size, + in_chans=in_chans, + embed_dim=embed_dim) + else: + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + if use_abs_pos_emb: + self.pos_embed = nn.Parameter( + torch.zeros(1, num_patches + 1, embed_dim)) + else: + self.pos_embed = None + self.pos_drop = nn.Dropout(p=drop_rate) + + if use_shared_rel_pos_bias: + self.rel_pos_bias = RelativePositionBias( + window_size=self.patch_embed.patch_shape, num_heads=num_heads) + else: + self.rel_pos_bias = None + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.use_rel_pos_bias = use_rel_pos_bias + self.use_checkpoint = use_checkpoint + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + with_cp=with_cp, + init_values=init_values, + window_size=self.patch_embed.patch_shape + if use_rel_pos_bias else None) for i in range(depth) + ]) + + trunc_normal_(self.cls_token, std=.02) + self.apply(self._init_weights) + self.init_weights(pretrained) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + init_cfg = dict(type='Pretrained', checkpoint=pretrained) + + checkpoint = _load_checkpoint( + init_cfg['checkpoint'], logger=logger, map_location='cpu') + state_dict = self.resize_rel_pos_embed(checkpoint) + self.load_state_dict(state_dict, False) + + def fix_init_weight(self): + + def rescale(param, layer_id): + param.div_(math.sqrt(2.0 * layer_id)) + + for layer_id, layer in enumerate(self.blocks): + rescale(layer.attn.proj.weight.data, layer_id + 1) + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def get_num_layers(self): + return len(self.blocks) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/beit_adapter.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/beit_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..182fc0c1ba098daa330253ad79ea8b928144724e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/backbone/beit_adapter.py @@ -0,0 +1,168 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +import logging +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmdet.models.builder import BACKBONES +from mmdet.models.utils.transformer import MultiScaleDeformableAttention +from timm.models.layers import DropPath, trunc_normal_ +from torch.nn.init import normal_ + +from .adapter_modules import InteractionBlockWithCls as InteractionBlock +from .adapter_modules import SpatialPriorModule, deform_inputs +from .base.beit import BASEBEiT + +_logger = logging.getLogger(__name__) + + +@BACKBONES.register_module() +class BEiTAdapter(BASEBEiT): + + def __init__(self, + pretrain_size=224, + conv_inplane=64, + n_points=4, + deform_num_heads=6, + init_values=0., + cffn_ratio=0.25, + deform_ratio=1.0, + with_cffn=True, + interaction_indexes=None, + add_vit_feature=True, + with_cp=False, + *args, + **kwargs): + + super().__init__( + init_values=init_values, with_cp=with_cp, *args, **kwargs) + + self.num_block = len(self.blocks) + self.pretrain_size = (pretrain_size, pretrain_size) + self.flags = [ + i for i in range(-1, self.num_block, self.num_block // 4) + ][1:] + self.interaction_indexes = interaction_indexes + self.add_vit_feature = add_vit_feature + embed_dim = self.embed_dim + + self.level_embed = nn.Parameter(torch.zeros(3, embed_dim)) + self.spm = SpatialPriorModule( + inplanes=conv_inplane, embed_dim=embed_dim, with_cp=False) + self.interactions = nn.Sequential(*[ + InteractionBlock( + dim=embed_dim, + num_heads=deform_num_heads, + n_points=n_points, + init_values=init_values, + drop_path=self.drop_path_rate, + norm_layer=self.norm_layer, + with_cffn=with_cffn, + cffn_ratio=cffn_ratio, + deform_ratio=deform_ratio, + extra_extractor=True if i == len(interaction_indexes) + - 1 else False, + with_cp=with_cp) for i in range(len(interaction_indexes)) + ]) + + self.up = nn.ConvTranspose2d(embed_dim, embed_dim, 2, 2) + self.norm1 = nn.BatchNorm2d(embed_dim) + self.norm2 = nn.BatchNorm2d(embed_dim) + self.norm3 = nn.BatchNorm2d(embed_dim) + self.norm4 = nn.BatchNorm2d(embed_dim) + + self.up.apply(self._init_weights) + self.spm.apply(self._init_weights) + self.interactions.apply(self._init_weights) + self.apply(self._init_deform_weights) + normal_(self.level_embed) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm) or isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def _get_pos_embed(self, pos_embed, H, W): + pos_embed = pos_embed.reshape(1, self.pretrain_size[0] // 16, + self.pretrain_size[1] // 16, + -1).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False). \ + reshape(1, -1, H * W).permute(0, 2, 1) + return pos_embed + + def _init_deform_weights(self, m): + if isinstance(m, MultiScaleDeformableAttention): + m.init_weights() + + def _add_level_embed(self, c2, c3, c4): + c2 = c2 + self.level_embed[0] + c3 = c3 + self.level_embed[1] + c4 = c4 + self.level_embed[2] + return c2, c3, c4 + + def forward(self, x): + deform_inputs1, deform_inputs2 = deform_inputs(x) + + # SPM forward + c1, c2, c3, c4 = self.spm(x) + c2, c3, c4 = self._add_level_embed(c2, c3, c4) + c = torch.cat([c2, c3, c4], dim=1) + + # Patch Embedding forward + x, H, W = self.patch_embed(x) + bs, n, dim = x.shape + cls = self.cls_token.expand( + bs, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + + if self.pos_embed is not None: + pos_embed = self._get_pos_embed(self.pos_embed, H, W) + x = x + pos_embed + x = self.pos_drop(x) + + # Interaction + outs = list() + for i, layer in enumerate(self.interactions): + indexes = self.interaction_indexes[i] + x, c, cls = layer(x, c, cls, + self.blocks[indexes[0]:indexes[-1] + 1], + deform_inputs1, deform_inputs2, H, W) + outs.append(x.transpose(1, 2).view(bs, dim, H, W).contiguous()) + + # Split & Reshape + c2 = c[:, 0:c2.size(1), :] + c3 = c[:, c2.size(1):c2.size(1) + c3.size(1), :] + c4 = c[:, c2.size(1) + c3.size(1):, :] + + c2 = c2.transpose(1, 2).view(bs, dim, H * 2, W * 2).contiguous() + c3 = c3.transpose(1, 2).view(bs, dim, H, W).contiguous() + c4 = c4.transpose(1, 2).view(bs, dim, H // 2, W // 2).contiguous() + c1 = self.up(c2) + c1 + + if self.add_vit_feature: + x1, x2, x3, x4 = outs + x1 = F.interpolate( + x1, scale_factor=4, mode='bilinear', align_corners=False) + x2 = F.interpolate( + x2, scale_factor=2, mode='bilinear', align_corners=False) + x4 = F.interpolate( + x4, scale_factor=0.5, mode='bilinear', align_corners=False) + c1, c2, c3, c4 = c1 + x1, c2 + x2, c3 + x3, c4 + x4 + + # Final Norm + f1 = self.norm1(c1) + f2 = self.norm2(c2) + f3 = self.norm3(c3) + f4 = self.norm4(c4) + return [f1, f2, f3, f4] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..12bf2a21853cc5380caae1094992c7a3664b29aa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/__init__.py @@ -0,0 +1,5 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .mask2former_head_from_mmseg import Mask2FormerHeadFromMMSeg + +__all__ = ['Mask2FormerHeadFromMMSeg'] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/base_decode_head.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/base_decode_head.py new file mode 100644 index 0000000000000000000000000000000000000000..ae7a041615bb23af482df9c89b0416ca0ae6d9d3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/base_decode_head.py @@ -0,0 +1,266 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn as nn +from mmcv.runner import BaseModule, auto_fp16, force_fp32 +from mmdet.models.builder import build_loss +from mmdet.models.losses import accuracy + +from ...utils import build_pixel_sampler, seg_resize + + +class BaseDecodeHead(BaseModule, metaclass=ABCMeta): + """Base class for BaseDecodeHead. + + Args: + in_channels (int|Sequence[int]): Input channels. + channels (int): Channels after modules, before conv_seg. + num_classes (int): Number of classes. + dropout_ratio (float): Ratio of dropout layer. Default: 0.1. + conv_cfg (dict|None): Config of conv layers. Default: None. + norm_cfg (dict|None): Config of norm layers. Default: None. + act_cfg (dict): Config of activation layers. + Default: dict(type='ReLU') + in_index (int|Sequence[int]): Input feature index. Default: -1 + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + Default: None. + loss_decode (dict | Sequence[dict]): Config of decode loss. + The `loss_name` is property of corresponding loss function which + could be shown in training log. If you want this loss + item to be included into the backward graph, `loss_` must be the + prefix of the name. Defaults to 'loss_ce'. + e.g. dict(type='CrossEntropyLoss'), + [dict(type='CrossEntropyLoss', loss_name='loss_ce'), + dict(type='DiceLoss', loss_name='loss_dice')] + Default: dict(type='CrossEntropyLoss'). + ignore_index (int | None): The label index to be ignored. When using + masked BCE loss, ignore_index should be set to None. Default: 255. + sampler (dict|None): The config of segmentation map sampler. + Default: None. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + init_cfg (dict or list[dict], optional): Initialization config dict. + """ + + def __init__(self, + in_channels, + channels, + *, + num_classes, + dropout_ratio=0.1, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + in_index=-1, + input_transform=None, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + ignore_index=255, + sampler=None, + align_corners=False, + init_cfg=dict( + type='Normal', std=0.01, override=dict(name='conv_seg'))): + super(BaseDecodeHead, self).__init__(init_cfg) + self._init_inputs(in_channels, in_index, input_transform) + self.channels = channels + self.num_classes = num_classes + self.dropout_ratio = dropout_ratio + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.in_index = in_index + + self.ignore_index = ignore_index + self.align_corners = align_corners + + if isinstance(loss_decode, dict): + self.loss_decode = build_loss(loss_decode) + elif isinstance(loss_decode, (list, tuple)): + self.loss_decode = nn.ModuleList() + for loss in loss_decode: + self.loss_decode.append(build_loss(loss)) + else: + raise TypeError(f'loss_decode must be a dict or sequence of dict,\ + but got {type(loss_decode)}') + + if sampler is not None: + self.sampler = build_pixel_sampler(sampler, context=self) + else: + self.sampler = None + + self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.fp16_enabled = False + + def extra_repr(self): + """Extra repr.""" + s = f'input_transform={self.input_transform}, ' \ + f'ignore_index={self.ignore_index}, ' \ + f'align_corners={self.align_corners}' + return s + + def _init_inputs(self, in_channels, in_index, input_transform): + """Check and initialize input transforms. + + The in_channels, in_index and input_transform must match. + Specifically, when input_transform is None, only single feature map + will be selected. So in_channels and in_index must be of type int. + When input_transform + + Args: + in_channels (int|Sequence[int]): Input channels. + in_index (int|Sequence[int]): Input feature index. + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + """ + + if input_transform is not None: + assert input_transform in ['resize_concat', 'multiple_select'] + self.input_transform = input_transform + self.in_index = in_index + if input_transform is not None: + assert isinstance(in_channels, (list, tuple)) + assert isinstance(in_index, (list, tuple)) + assert len(in_channels) == len(in_index) + if input_transform == 'resize_concat': + self.in_channels = sum(in_channels) + else: + self.in_channels = in_channels + else: + assert isinstance(in_channels, int) + assert isinstance(in_index, int) + self.in_channels = in_channels + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + + if self.input_transform == 'resize_concat': + inputs = [inputs[i] for i in self.in_index] + upsampled_inputs = [ + seg_resize( + input=x, + size=inputs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) for x in inputs + ] + inputs = torch.cat(upsampled_inputs, dim=1) + elif self.input_transform == 'multiple_select': + inputs = [inputs[i] for i in self.in_index] + else: + inputs = inputs[self.in_index] + + return inputs + + @auto_fp16() + @abstractmethod + def forward(self, inputs): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, img_metas, gt_semantic_seg, train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs) + losses = self.losses(seg_logits, gt_semantic_seg) + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs) + + def cls_seg(self, feat): + """Classify each pixel.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.conv_seg(feat) + return output + + @force_fp32(apply_to=('seg_logit', )) + def losses(self, seg_logit, seg_label): + """Compute segmentation loss.""" + loss = dict() + seg_logit = seg_resize( + input=seg_logit, + size=seg_label.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + if self.sampler is not None: + seg_weight = self.sampler.sample(seg_logit, seg_label) + else: + seg_weight = None + seg_label = seg_label.squeeze(1) + + if not isinstance(self.loss_decode, nn.ModuleList): + losses_decode = [self.loss_decode] + else: + losses_decode = self.loss_decode + for loss_decode in losses_decode: + if loss_decode.loss_name not in loss: + loss[loss_decode.loss_name] = loss_decode( + seg_logit, + seg_label, + weight=seg_weight, + ignore_index=self.ignore_index) + else: + loss[loss_decode.loss_name] += loss_decode( + seg_logit, + seg_label, + weight=seg_weight, + ignore_index=self.ignore_index) + + loss['acc_seg'] = accuracy( + seg_logit, seg_label, ignore_index=self.ignore_index) + return loss diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/mask2former_head_from_mmseg.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/mask2former_head_from_mmseg.py new file mode 100644 index 0000000000000000000000000000000000000000..c0681d2b151d01693ec8b2c3779cc3375747fac7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/decode_heads/mask2former_head_from_mmseg.py @@ -0,0 +1,580 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git + +import copy + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, build_plugin_layer, caffe2_xavier_init +from mmcv.cnn.bricks.transformer import (build_positional_encoding, + build_transformer_layer_sequence) +from mmcv.ops import point_sample +from mmcv.runner import ModuleList, force_fp32 +from mmdet.core import build_assigner, build_sampler, multi_apply, reduce_mean +from mmdet.models.builder import HEADS, build_loss +from mmdet.models.utils import get_uncertain_point_coords_with_randomness + +from .base_decode_head import BaseDecodeHead + + +@HEADS.register_module() +class Mask2FormerHeadFromMMSeg(BaseDecodeHead): + """Implements the Mask2Former head. + + See `Masked-attention Mask Transformer for Universal Image + Segmentation `_ for details. + + Args: + in_channels (list[int]): Number of channels in the input feature map. + feat_channels (int): Number of channels for features. + out_channels (int): Number of channels for output. + num_things_classes (int): Number of things. + num_stuff_classes (int): Number of stuff. + num_queries (int): Number of query in Transformer decoder. + pixel_decoder (:obj:`mmcv.ConfigDict` | dict): Config for pixel + decoder. Defaults to None. + enforce_decoder_input_project (bool, optional): Whether to add + a layer to change the embed_dim of tranformer encoder in + pixel decoder to the embed_dim of transformer decoder. + Defaults to False. + transformer_decoder (:obj:`mmcv.ConfigDict` | dict): Config for + transformer decoder. Defaults to None. + positional_encoding (:obj:`mmcv.ConfigDict` | dict): Config for + transformer decoder position encoding. Defaults to None. + loss_cls (:obj:`mmcv.ConfigDict` | dict): Config of the classification + loss. Defaults to None. + loss_mask (:obj:`mmcv.ConfigDict` | dict): Config of the mask loss. + Defaults to None. + loss_dice (:obj:`mmcv.ConfigDict` | dict): Config of the dice loss. + Defaults to None. + train_cfg (:obj:`mmcv.ConfigDict` | dict): Training config of + Mask2Former head. + test_cfg (:obj:`mmcv.ConfigDict` | dict): Testing config of + Mask2Former head. + init_cfg (dict or list[dict], optional): Initialization config dict. + Defaults to None. + """ + + def __init__(self, + in_channels, + feat_channels, + out_channels, + num_things_classes=80, + num_stuff_classes=53, + num_queries=100, + num_transformer_feat_level=3, + pixel_decoder=None, + enforce_decoder_input_project=False, + transformer_decoder=None, + positional_encoding=None, + loss_cls=None, + loss_mask=None, + loss_dice=None, + train_cfg=None, + test_cfg=None, + init_cfg=None, + **kwargs): + super(Mask2FormerHeadFromMMSeg, self).__init__( + in_channels=in_channels, + channels=feat_channels, + num_classes=(num_things_classes + num_stuff_classes), + init_cfg=init_cfg, + input_transform='multiple_select', + **kwargs) + self.num_things_classes = num_things_classes + self.num_stuff_classes = num_stuff_classes + self.num_classes = self.num_things_classes + self.num_stuff_classes + self.num_queries = num_queries + self.num_transformer_feat_level = num_transformer_feat_level + self.num_heads = transformer_decoder.transformerlayers. \ + attn_cfgs.num_heads + self.num_transformer_decoder_layers = transformer_decoder.num_layers + assert pixel_decoder.encoder.transformerlayers.attn_cfgs.num_levels == num_transformer_feat_level + pixel_decoder_ = copy.deepcopy(pixel_decoder) + pixel_decoder_.update( + in_channels=in_channels, + feat_channels=feat_channels, + out_channels=out_channels) + self.pixel_decoder = build_plugin_layer(pixel_decoder_)[1] + self.transformer_decoder = build_transformer_layer_sequence( + transformer_decoder) + self.decoder_embed_dims = self.transformer_decoder.embed_dims + + self.decoder_input_projs = ModuleList() + # from low resolution to high resolution + for _ in range(num_transformer_feat_level): + if (self.decoder_embed_dims != feat_channels + or enforce_decoder_input_project): + self.decoder_input_projs.append( + Conv2d( + feat_channels, self.decoder_embed_dims, kernel_size=1)) + else: + self.decoder_input_projs.append(nn.Identity()) + self.decoder_positional_encoding = build_positional_encoding( + positional_encoding) + self.query_embed = nn.Embedding(self.num_queries, feat_channels) + self.query_feat = nn.Embedding(self.num_queries, feat_channels) + # from low resolution to high resolution + self.level_embed = nn.Embedding(self.num_transformer_feat_level, + feat_channels) + + self.cls_embed = nn.Linear(feat_channels, self.num_classes + 1) + self.mask_embed = nn.Sequential( + nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), + nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), + nn.Linear(feat_channels, out_channels)) + self.conv_seg = None # fix a bug here (conv_seg is not used) + + self.test_cfg = test_cfg + self.train_cfg = train_cfg + if train_cfg: + self.assigner = build_assigner(self.train_cfg.assigner) + self.sampler = build_sampler(self.train_cfg.sampler, context=self) + self.num_points = self.train_cfg.get('num_points', 12544) + self.oversample_ratio = self.train_cfg.get('oversample_ratio', 3.0) + self.importance_sample_ratio = self.train_cfg.get( + 'importance_sample_ratio', 0.75) + + self.class_weight = loss_cls.class_weight + self.loss_cls = build_loss(loss_cls) + self.loss_mask = build_loss(loss_mask) + self.loss_dice = build_loss(loss_dice) + + def init_weights(self): + for m in self.decoder_input_projs: + if isinstance(m, Conv2d): + caffe2_xavier_init(m, bias=0) + + self.pixel_decoder.init_weights() + + for p in self.transformer_decoder.parameters(): + if p.dim() > 1: + nn.init.xavier_normal_(p) + + def get_targets(self, cls_scores_list, mask_preds_list, gt_labels_list, + gt_masks_list, img_metas): + """Compute classification and mask targets for all images for a decoder + layer. + + Args: + cls_scores_list (list[Tensor]): Mask score logits from a single + decoder layer for all images. Each with shape [num_queries, + cls_out_channels]. + mask_preds_list (list[Tensor]): Mask logits from a single decoder + layer for all images. Each with shape [num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for all + images. Each with shape (n, ), n is the sum of number of stuff + type and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[list[Tensor]]: a tuple containing the following targets. + + - labels_list (list[Tensor]): Labels of all images. + Each with shape [num_queries, ]. + - label_weights_list (list[Tensor]): Label weights of all + images.Each with shape [num_queries, ]. + - mask_targets_list (list[Tensor]): Mask targets of all images. + Each with shape [num_queries, h, w]. + - mask_weights_list (list[Tensor]): Mask weights of all images. + Each with shape [num_queries, ]. + - num_total_pos (int): Number of positive samples in all + images. + - num_total_neg (int): Number of negative samples in all + images. + """ + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + pos_inds_list, + neg_inds_list) = multi_apply(self._get_target_single, cls_scores_list, + mask_preds_list, gt_labels_list, + gt_masks_list, img_metas) + + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, mask_targets_list, + mask_weights_list, num_total_pos, num_total_neg) + + def _get_target_single(self, cls_score, mask_pred, gt_labels, gt_masks, + img_metas): + """Compute classification and mask targets for one image. + + Args: + cls_score (Tensor): Mask score logits from a single decoder layer + for one image. Shape (num_queries, cls_out_channels). + mask_pred (Tensor): Mask logits for a single decoder layer for one + image. Shape (num_queries, h, w). + gt_labels (Tensor): Ground truth class indices for one image with + shape (num_gts, ). + gt_masks (Tensor): Ground truth mask for each image, each with + shape (num_gts, h, w). + img_metas (dict): Image informtation. + + Returns: + tuple[Tensor]: A tuple containing the following for one image. + + - labels (Tensor): Labels of each image. \ + shape (num_queries, ). + - label_weights (Tensor): Label weights of each image. \ + shape (num_queries, ). + - mask_targets (Tensor): Mask targets of each image. \ + shape (num_queries, h, w). + - mask_weights (Tensor): Mask weights of each image. \ + shape (num_queries, ). + - pos_inds (Tensor): Sampled positive indices for each \ + image. + - neg_inds (Tensor): Sampled negative indices for each \ + image. + """ + # sample points + num_queries = cls_score.shape[0] + num_gts = gt_labels.shape[0] + + point_coords = torch.rand((1, self.num_points, 2), + device=cls_score.device) + # shape (num_queries, num_points) + mask_points_pred = point_sample( + mask_pred.unsqueeze(1), point_coords.repeat(num_queries, 1, + 1)).squeeze(1) + # shape (num_gts, num_points) + gt_points_masks = point_sample( + gt_masks.unsqueeze(1).float(), point_coords.repeat(num_gts, 1, + 1)).squeeze(1) + + # assign and sample + assign_result = self.assigner.assign(cls_score, mask_points_pred, + gt_labels, gt_points_masks, + img_metas) + sampling_result = self.sampler.sample(assign_result, mask_pred, + gt_masks) + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + # label target + labels = gt_labels.new_full((self.num_queries, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] + label_weights = gt_labels.new_ones((self.num_queries, )) + + # mask target + mask_targets = gt_masks[sampling_result.pos_assigned_gt_inds] + mask_weights = mask_pred.new_zeros((self.num_queries, )) + mask_weights[pos_inds] = 1.0 + + return (labels, label_weights, mask_targets, mask_weights, pos_inds, + neg_inds) + + def loss_single(self, cls_scores, mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function for outputs from a single decoder layer. + + Args: + cls_scores (Tensor): Mask score logits from a single decoder layer + for all images. Shape (batch_size, num_queries, + cls_out_channels). Note `cls_out_channels` should includes + background. + mask_preds (Tensor): Mask logits for a pixel decoder for all + images. Shape (batch_size, num_queries, h, w). + gt_labels_list (list[Tensor]): Ground truth class indices for each + image, each with shape (num_gts, ). + gt_masks_list (list[Tensor]): Ground truth mask for each image, + each with shape (num_gts, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + tuple[Tensor]: Loss components for outputs from a single \ + decoder layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + mask_preds_list = [mask_preds[i] for i in range(num_imgs)] + (labels_list, label_weights_list, mask_targets_list, mask_weights_list, + num_total_pos, + num_total_neg) = self.get_targets(cls_scores_list, mask_preds_list, + gt_labels_list, gt_masks_list, + img_metas) + # shape (batch_size, num_queries) + labels = torch.stack(labels_list, dim=0) + # shape (batch_size, num_queries) + label_weights = torch.stack(label_weights_list, dim=0) + # shape (num_total_gts, h, w) + mask_targets = torch.cat(mask_targets_list, dim=0) + # shape (batch_size, num_queries) + mask_weights = torch.stack(mask_weights_list, dim=0) + + # classfication loss + # shape (batch_size * num_queries, ) + cls_scores = cls_scores.flatten(0, 1) + labels = labels.flatten(0, 1) + label_weights = label_weights.flatten(0, 1) + + class_weight = cls_scores.new_tensor(self.class_weight) + loss_cls = self.loss_cls( + cls_scores, + labels, + label_weights, + avg_factor=class_weight[labels].sum()) + + num_total_masks = reduce_mean(cls_scores.new_tensor([num_total_pos])) + num_total_masks = max(num_total_masks, 1) + + # extract positive ones + # shape (batch_size, num_queries, h, w) -> (num_total_gts, h, w) + mask_preds = mask_preds[mask_weights > 0] + + if mask_targets.shape[0] == 0: + # zero match + loss_dice = mask_preds.sum() + loss_mask = mask_preds.sum() + return loss_cls, loss_mask, loss_dice + + with torch.no_grad(): + points_coords = get_uncertain_point_coords_with_randomness( + mask_preds.unsqueeze(1), None, self.num_points, + self.oversample_ratio, self.importance_sample_ratio) + # shape (num_total_gts, h, w) -> (num_total_gts, num_points) + mask_point_targets = point_sample( + mask_targets.unsqueeze(1).float(), points_coords).squeeze(1) + # shape (num_queries, h, w) -> (num_queries, num_points) + mask_point_preds = point_sample( + mask_preds.unsqueeze(1), points_coords).squeeze(1) + + # dice loss + loss_dice = self.loss_dice( + mask_point_preds, mask_point_targets, avg_factor=num_total_masks) + + # mask loss + # shape (num_queries, num_points) -> (num_queries * num_points, ) + mask_point_preds = mask_point_preds.reshape(-1, 1) + # shape (num_total_gts, num_points) -> (num_total_gts * num_points, ) + mask_point_targets = mask_point_targets.reshape(-1) + loss_mask = self.loss_mask( + mask_point_preds, + mask_point_targets, + avg_factor=num_total_masks * self.num_points) + + return loss_cls, loss_mask, loss_dice + + @force_fp32(apply_to=('all_cls_scores', 'all_mask_preds')) + def loss(self, all_cls_scores, all_mask_preds, gt_labels_list, + gt_masks_list, img_metas): + """Loss function. + + Args: + all_cls_scores (Tensor): Classification scores for all decoder + layers with shape [num_decoder, batch_size, num_queries, + cls_out_channels]. + all_mask_preds (Tensor): Mask scores for all decoder layers with + shape [num_decoder, batch_size, num_queries, h, w]. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (n, ). n is the sum of number of stuff type + and number of instance in a image. + gt_masks_list (list[Tensor]): Ground truth mask for each image with + shape (n, h, w). + img_metas (list[dict]): List of image meta information. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + num_dec_layers = len(all_cls_scores) + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_masks_list = [gt_masks_list for _ in range(num_dec_layers)] + img_metas_list = [img_metas for _ in range(num_dec_layers)] + losses_cls, losses_mask, losses_dice = multi_apply( + self.loss_single, all_cls_scores, all_mask_preds, + all_gt_labels_list, all_gt_masks_list, img_metas_list) + + loss_dict = dict() + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_mask'] = losses_mask[-1] + loss_dict['loss_dice'] = losses_dice[-1] + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_mask_i, loss_dice_i in zip( + losses_cls[:-1], losses_mask[:-1], losses_dice[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_mask'] = loss_mask_i + loss_dict[f'd{num_dec_layer}.loss_dice'] = loss_dice_i + num_dec_layer += 1 + return loss_dict + + def forward_head(self, decoder_out, mask_feature, attn_mask_target_size): + """Forward for head part which is called after every decoder layer. + + Args: + decoder_out (Tensor): in shape (num_queries, batch_size, c). + mask_feature (Tensor): in shape (batch_size, c, h, w). + attn_mask_target_size (tuple[int, int]): target attention + mask size. + + Returns: + tuple: A tuple contain three elements. + + - cls_pred (Tensor): Classification scores in shape \ + (batch_size, num_queries, cls_out_channels). \ + Note `cls_out_channels` should includes background. + - mask_pred (Tensor): Mask scores in shape \ + (batch_size, num_queries,h, w). + - attn_mask (Tensor): Attention mask in shape \ + (batch_size * num_heads, num_queries, h, w). + """ + decoder_out = self.transformer_decoder.post_norm(decoder_out) + decoder_out = decoder_out.transpose(0, 1) + # shape (num_queries, batch_size, c) + cls_pred = self.cls_embed(decoder_out) + # shape (num_queries, batch_size, c) + mask_embed = self.mask_embed(decoder_out) + # shape (num_queries, batch_size, h, w) + mask_pred = torch.einsum('bqc,bchw->bqhw', mask_embed, mask_feature) + attn_mask = F.interpolate( + mask_pred, + attn_mask_target_size, + mode='bilinear', + align_corners=False) + # shape (num_queries, batch_size, h, w) -> + # (batch_size * num_head, num_queries, h, w) + attn_mask = attn_mask.flatten(2).unsqueeze(1).repeat( + (1, self.num_heads, 1, 1)).flatten(0, 1) + attn_mask = attn_mask.sigmoid() < 0.5 + attn_mask = attn_mask.detach() + + return cls_pred, mask_pred, attn_mask + + def forward(self, feats, img_metas): + """Forward function. + + Args: + feats (list[Tensor]): Multi scale Features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + + Returns: + tuple: A tuple contains two elements. + + - cls_pred_list (list[Tensor)]: Classification logits \ + for each decoder layer. Each is a 3D-tensor with shape \ + (batch_size, num_queries, cls_out_channels). \ + Note `cls_out_channels` should includes background. + - mask_pred_list (list[Tensor]): Mask logits for each \ + decoder layer. Each with shape (batch_size, num_queries, \ + h, w). + """ + batch_size = len(img_metas) + mask_features, multi_scale_memorys = self.pixel_decoder(feats) + # multi_scale_memorys (from low resolution to high resolution) + decoder_inputs = [] + decoder_positional_encodings = [] + for i in range(self.num_transformer_feat_level): + decoder_input = self.decoder_input_projs[i](multi_scale_memorys[i]) + # shape (batch_size, c, h, w) -> (h*w, batch_size, c) + decoder_input = decoder_input.flatten(2).permute(2, 0, 1) + level_embed = self.level_embed.weight[i].view(1, 1, -1) + decoder_input = decoder_input + level_embed + # shape (batch_size, c, h, w) -> (h*w, batch_size, c) + mask = decoder_input.new_zeros( + (batch_size, ) + multi_scale_memorys[i].shape[-2:], + dtype=torch.bool) + decoder_positional_encoding = self.decoder_positional_encoding( + mask) + decoder_positional_encoding = decoder_positional_encoding.flatten( + 2).permute(2, 0, 1) + decoder_inputs.append(decoder_input) + decoder_positional_encodings.append(decoder_positional_encoding) + # shape (num_queries, c) -> (num_queries, batch_size, c) + query_feat = self.query_feat.weight.unsqueeze(1).repeat( + (1, batch_size, 1)) + query_embed = self.query_embed.weight.unsqueeze(1).repeat( + (1, batch_size, 1)) + + cls_pred_list = [] + mask_pred_list = [] + cls_pred, mask_pred, attn_mask = self.forward_head( + query_feat, mask_features, multi_scale_memorys[0].shape[-2:]) + cls_pred_list.append(cls_pred) + mask_pred_list.append(mask_pred) + + for i in range(self.num_transformer_decoder_layers): + level_idx = i % self.num_transformer_feat_level + # if a mask is all True(all background), then set it all False. + attn_mask[torch.where( + attn_mask.sum(-1) == attn_mask.shape[-1])] = False + + # cross_attn + self_attn + layer = self.transformer_decoder.layers[i] + attn_masks = [attn_mask, None] + query_feat = layer( + query=query_feat, + key=decoder_inputs[level_idx], + value=decoder_inputs[level_idx], + query_pos=query_embed, + key_pos=decoder_positional_encodings[level_idx], + attn_masks=attn_masks, + query_key_padding_mask=None, + # here we do not apply masking on padded region + key_padding_mask=None) + cls_pred, mask_pred, attn_mask = self.forward_head( + query_feat, mask_features, multi_scale_memorys[ + (i + 1) % self.num_transformer_feat_level].shape[-2:]) + + cls_pred_list.append(cls_pred) + mask_pred_list.append(mask_pred) + + return cls_pred_list, mask_pred_list + + def forward_train(self, x, img_metas, gt_semantic_seg, gt_labels, + gt_masks): + """Forward function for training mode. + + Args: + x (list[Tensor]): Multi-level features from the upstream network, + each is a 4D-tensor. + img_metas (list[Dict]): List of image information. + gt_semantic_seg (list[tensor]):Each element is the ground truth + of semantic segmentation with the shape (N, H, W). + train_cfg (dict): The training config, which not been used in + maskformer. + gt_labels (list[Tensor]): Each element is ground truth labels of + each box, shape (num_gts,). + gt_masks (list[BitmapMasks]): Each element is masks of instances + of a image, shape (num_gts, h, w). + + Returns: + losses (dict[str, Tensor]): a dictionary of loss components + """ + + # forward + all_cls_scores, all_mask_preds = self(x, img_metas) + + # loss + losses = self.loss(all_cls_scores, all_mask_preds, gt_labels, gt_masks, + img_metas) + + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Test segment without test-time aumengtation. + + Only the output of last decoder layers was used. + + Args: + inputs (list[Tensor]): Multi-level features from the + upstream network, each is a 4D-tensor. + img_metas (list[dict]): List of image information. + test_cfg (dict): Testing config. + + Returns: + seg_mask (Tensor): Predicted semantic segmentation logits. + """ + all_cls_scores, all_mask_preds = self(inputs, img_metas) + cls_score, mask_pred = all_cls_scores[-1], all_mask_preds[-1] + ori_h, ori_w, _ = img_metas[0]['ori_shape'] + + # semantic inference + cls_score = F.softmax(cls_score, dim=-1)[..., :-1] + mask_pred = mask_pred.sigmoid() + seg_mask = torch.einsum('bqc,bqhw->bchw', cls_score, mask_pred) + return seg_mask diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..18bbce0d65747e5705217adfcf5690238aee9c67 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/__init__.py @@ -0,0 +1,5 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .encoder_decoder_mask2former import EncoderDecoderMask2Former + +__all__ = ['EncoderDecoderMask2Former'] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/base_segmentor.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/base_segmentor.py new file mode 100644 index 0000000000000000000000000000000000000000..311352c2818fa82a0f2fbabea4ae595414695d52 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/base_segmentor.py @@ -0,0 +1,313 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +import warnings +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import mmcv +import numpy as np +import torch +import torch.distributed as dist +from mmcv.runner import BaseModule, auto_fp16 + + +class BaseSegmentor(BaseModule, metaclass=ABCMeta): + """Base class for segmentors.""" + + def __init__(self, init_cfg=None): + super(BaseSegmentor, self).__init__(init_cfg) + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the segmentor has neck""" + return hasattr(self, 'neck') and self.neck is not None + + @property + def with_auxiliary_head(self): + """bool: whether the segmentor has auxiliary head""" + return hasattr(self, + 'auxiliary_head') and self.auxiliary_head is not None + + @property + def with_decode_head(self): + """bool: whether the segmentor has decode head""" + return hasattr(self, 'decode_head') and self.decode_head is not None + + @abstractmethod + def extract_feat(self, imgs): + """Placeholder for extract features from images.""" + pass + + @abstractmethod + def encode_decode(self, img, img_metas): + """Placeholder for encode images with backbone and decode into a + semantic segmentation map of the same size as input.""" + pass + + @abstractmethod + def forward_train(self, imgs, img_metas, **kwargs): + """Placeholder for Forward function for training.""" + pass + + @abstractmethod + def simple_test(self, img, img_meta, **kwargs): + """Placeholder for single image test.""" + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Placeholder for augmentation test.""" + pass + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got ' + f'{type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) != ' + f'num of image meta ({len(img_metas)})') + + # all images in the same aug batch all of the same ori_shape and pad + # shape + def tensor_to_tuple(input_tensor): + return tuple(input_tensor.cpu().numpy()) + + for img_meta in img_metas: + ori_shapes = [_['ori_shape'] for _ in img_meta] + if isinstance(ori_shapes[0], torch.Tensor): + assert all( + tensor_to_tuple(shape) == tensor_to_tuple(ori_shapes[0]) + for shape in ori_shapes) + else: + assert all(shape == ori_shapes[0] for shape in ori_shapes) + + img_shapes = [_['img_shape'] for _ in img_meta] + if isinstance(img_shapes[0], torch.Tensor): + assert all( + tensor_to_tuple(shape) == tensor_to_tuple(img_shapes[0]) + for shape in img_shapes) + else: + assert all(shape == img_shapes[0] for shape in img_shapes) + + pad_shapes = [_['pad_shape'] for _ in img_meta] + if isinstance(pad_shapes[0], torch.Tensor): + assert all( + tensor_to_tuple(shape) == tensor_to_tuple(pad_shapes[0]) + for shape in pad_shapes) + else: + assert all(shape == pad_shapes[0] for shape in pad_shapes) + + if num_augs == 1: + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def train_step(self, data_batch, optimizer, **kwargs): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, + ``num_samples``. + ``loss`` is a tensor for back propagation, which can be a + weighted sum of multiple losses. + ``log_vars`` contains all the variables to be sent to the + logger. + ``num_samples`` indicates the batch size (when the model is + DDP, it means the batch size on each GPU), which is used for + averaging the logs. + """ + losses = self(**data_batch) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, + log_vars=log_vars, + num_samples=len(data_batch['img_metas'])) + + return outputs + + def val_step(self, data_batch, optimizer=None, **kwargs): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + losses = self(**data_batch) + loss, log_vars = self._parse_losses(losses) + + log_vars_ = dict() + for loss_name, loss_value in log_vars.items(): + k = loss_name + '_val' + log_vars_[k] = loss_value + + outputs = dict( + loss=loss, + log_vars=log_vars_, + num_samples=len(data_batch['img_metas'])) + + return outputs + + @staticmethod + def _parse_losses(losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary information. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor + which may be a weighted sum of all losses, log_vars contains + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + # If the loss_vars has different length, raise assertion error + # to prevent GPUs from infinite waiting. + if dist.is_available() and dist.is_initialized(): + log_var_length = torch.tensor(len(log_vars), device=loss.device) + dist.all_reduce(log_var_length) + message = (f'rank {dist.get_rank()}' + + f' len(log_vars): {len(log_vars)}' + ' keys: ' + + ','.join(log_vars.keys()) + '\n') + assert log_var_length == len(log_vars) * dist.get_world_size(), \ + 'loss log variables are different across GPUs!\n' + message + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def show_result(self, + img, + result, + palette=None, + win_name='', + show=False, + wait_time=0, + out_file=None, + opacity=0.5): + """Draw `result` over `img`. + + Args: + img (str or Tensor): The image to be displayed. + result (Tensor): The semantic segmentation results to draw over + `img`. + palette (list[list[int]]] | np.ndarray | None): The palette of + segmentation map. If None is given, random palette will be + generated. Default: None + win_name (str): The window name. + wait_time (int): Value of waitKey param. + Default: 0. + show (bool): Whether to show the image. + Default: False. + out_file (str or None): The filename to write the image. + Default: None. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + img (Tensor): Only if not `show` or `out_file` + """ + img = mmcv.imread(img) + img = img.copy() + seg = result[0] + if palette is None: + if self.PALETTE is None: + # Get random state before set seed, + # and restore random state later. + # It will prevent loss of randomness, as the palette + # may be different in each iteration if not specified. + # See: https://github.com/open-mmlab/mmdetection/issues/5844 + state = np.random.get_state() + np.random.seed(42) + # random palette + palette = np.random.randint( + 0, 255, size=(len(self.CLASSES), 3)) + np.random.set_state(state) + else: + palette = self.PALETTE + palette = np.array(palette) + assert palette.shape[0] == len(self.CLASSES) + assert palette.shape[1] == 3 + assert len(palette.shape) == 2 + assert 0 < opacity <= 1.0 + color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) + for label, color in enumerate(palette): + color_seg[seg == label, :] = color + # convert to BGR + color_seg = color_seg[..., ::-1] + + img = img * (1 - opacity) + color_seg * opacity + img = img.astype(np.uint8) + # if out_file specified, do not show image in window + if out_file is not None: + show = False + + if show: + mmcv.imshow(img, win_name, wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + + if not (show or out_file): + warnings.warn('show==False and out_file is not specified, only ' + 'result image will be returned') + return img diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/encoder_decoder_mask2former.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/encoder_decoder_mask2former.py new file mode 100644 index 0000000000000000000000000000000000000000..5049237462be075630fbea26787e9e71c72417be --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/models/segmentors/encoder_decoder_mask2former.py @@ -0,0 +1,302 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmdet.models import builder +from mmdet.models.builder import DETECTORS + +from ...utils import add_prefix, seg_resize +from .base_segmentor import BaseSegmentor + + +@DETECTORS.register_module() +class EncoderDecoderMask2Former(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None, + init_cfg=None): + super(EncoderDecoderMask2Former, self).__init__(init_cfg) + if pretrained is not None: + assert backbone.get('pretrained') is None, \ + 'both backbone and segmentor set pretrained weight' + backbone.pretrained = pretrained + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + decode_head.update(train_cfg=train_cfg) + decode_head.update(test_cfg=test_cfg) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = seg_resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg, + **kwargs): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, **kwargs) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg, **kwargs): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg, + **kwargs) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + + def tensor_to_tuple(input_tensor): + return tuple(input_tensor.cpu().numpy()) + + if rescale: + preds = seg_resize( + preds, + size=tensor_to_tuple(img_meta[0]['ori_shape'])[:2] + if isinstance(img_meta[0]['ori_shape'], torch.Tensor) else + img_meta[0]['ori_shape'], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = seg_resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + + def tensor_to_tuple(input_tensor): + return tuple(input_tensor.cpu().numpy()) + + if isinstance(ori_shape, torch.Tensor): + assert all( + tensor_to_tuple(_['ori_shape']) == tensor_to_tuple(ori_shape) + for _ in img_meta) + else: + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3, )) + elif flip_direction == 'vertical': + output = output.flip(dims=(2, )) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/__init__.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9c4d5c4c99ab792d9820c3677a9dd79707e6eb28 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/__init__.py @@ -0,0 +1,9 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from .builder import build_pixel_sampler +from .data_process_func import ResizeToMultiple +from .seg_func import add_prefix, seg_resize + +__all__ = [ + 'seg_resize', 'add_prefix', 'build_pixel_sampler', 'ResizeToMultiple' +] diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/builder.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..0603ef9487af04dfd50f4dc019833f6fe3e05bdb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/builder.py @@ -0,0 +1,10 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git +from mmcv.utils import Registry, build_from_cfg + +PIXEL_SAMPLERS = Registry('pixel sampler') + + +def build_pixel_sampler(cfg, **default_args): + """Build pixel sampler for segmentation map.""" + return build_from_cfg(cfg, PIXEL_SAMPLERS, default_args) diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/data_process_func.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/data_process_func.py new file mode 100644 index 0000000000000000000000000000000000000000..194361afb74ee077291a613b18f234c1dbf1a5be --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/data_process_func.py @@ -0,0 +1,60 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import mmcv +from mmdet.datasets.builder import PIPELINES + + +@PIPELINES.register_module() +class ResizeToMultiple(object): + """Resize images & seg to multiple of divisor. + + Args: + size_divisor (int): images and gt seg maps need to resize to multiple + of size_divisor. Default: 32. + interpolation (str, optional): The interpolation mode of image resize. + Default: None + """ + + def __init__(self, size_divisor=32, interpolation=None): + self.size_divisor = size_divisor + self.interpolation = interpolation + + def __call__(self, results): + """Call function to resize images, semantic segmentation map to + multiple of size divisor. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape' keys are updated. + """ + # Align image to multiple of size divisor. + img = results['img'] + img = mmcv.imresize_to_multiple( + img, + self.size_divisor, + scale_factor=1, + interpolation=self.interpolation + if self.interpolation else 'bilinear') + + results['img'] = img + results['img_shape'] = img.shape + results['pad_shape'] = img.shape + + # Align segmentation map to multiple of size divisor. + for key in results.get('seg_fields', []): + gt_seg = results[key] + gt_seg = mmcv.imresize_to_multiple( + gt_seg, + self.size_divisor, + scale_factor=1, + interpolation='nearest') + results[key] = gt_seg + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(size_divisor={self.size_divisor}, ' + f'interpolation={self.interpolation})') + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/seg_func.py b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/seg_func.py new file mode 100644 index 0000000000000000000000000000000000000000..3c072296108ec31e4de05f191c98b0fb7f6a7625 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_semantic_segmentation/vit_adapter/utils/seg_func.py @@ -0,0 +1,54 @@ +# The implementation is adopted from VitAdapter, +# made publicly available under the Apache License at https://github.com/czczup/ViT-Adapter.git + +import warnings + +import torch.nn.functional as F + + +def seg_resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > input_w: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + + try: + return F.interpolate(input, size, scale_factor, mode, align_corners) + except ValueError: + if isinstance(size, tuple): + if len(size) == 3: + size = size[:2] + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +def add_prefix(inputs, prefix): + """Add prefix for dict. + + Args: + inputs (dict): The input dict with str keys. + prefix (str): The prefix to add. + + Returns: + + dict: The dict with keys updated with ``prefix``. + """ + + outputs = dict() + for name, value in inputs.items(): + outputs[f'{prefix}.{name}'] = value + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/__init__.py b/runtime/Lib/site_packages/models/cv/image_skychange/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..955f26e6f799c369932813ab91a4128d527ce232 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .skychange_model import ImageSkychange + from .preprocessor import ImageSkyChangePreprocessor + +else: + _import_structure = {'skychange_model': ['ImageSkychange']} + _import_structure = {'preprocessor': ['ImageSkyChangePreprocessor']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/preprocessor.py b/runtime/Lib/site_packages/models/cv/image_skychange/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..87edca0ef72d50ef72b9fd46d27bc788b2bf5abc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/preprocessor.py @@ -0,0 +1,245 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numbers +import pdb +from typing import Any, Dict, Union + +import cv2 +import json +import numpy as np +import torch +from torchvision import transforms + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.preprocessors.image import LoadImage +from modelscope.utils.constant import Fields, ModeKeys + +_cv2_pad_to_str = { + 'constant': cv2.BORDER_CONSTANT, + 'edge': cv2.BORDER_REPLICATE, + 'reflect': cv2.BORDER_REFLECT_101, + 'symmetric': cv2.BORDER_REFLECT, +} + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.image_sky_change_preprocessor) +class ImageSkyChangePreprocessor(Preprocessor): + + def __init__(self, + model_dir: str = None, + mode: str = ModeKeys.INFERENCE, + coarse_model_width=640, + coarse_model_height=640, + refine_model_width=1280, + refine_model_height=1280, + mean_vec=[0.485, 0.456, 0.406], + std_vec=[0.229, 0.224, 0.225], + *args, + **kwargs): + """ + Args: + model_dir (str): model directory to initialize some resource. + mode: The mode for the preprocessor. + coarse_model_width: required width of input tensor of coarse model. + coarse_model_height: required height of input tensor of coarse model. + refine_model_width: required width of input tensor of refine model. + refine_model_height: required height of input tensor of refine model. + mean_vec: mean of dataset(for transforms.Normalize), default is mean of Imagenet dataset. + std_vec: standard deviation of dataset(for transforms.Normalize), default is std of Imagenet dataset. + """ + super().__init__(mode) + + # set preprocessor info + self.coarse_input_size = [coarse_model_width, coarse_model_height] + self.refine_input_size = [refine_model_width, refine_model_height] + self.normalize = transforms.Normalize(mean=mean_vec, std=std_vec) + + def __call__(self, data: Union[str, Dict], **kwargs) -> Dict[str, Any]: + """process the raw input data + Args: + data (dict): data dict containing following info: + sky_image, scene_image + + Example: + >>> { + >>> "sky_image": "xxx.jpg" # sky_image path(str) + >>> "scene_image": "xxx.jpg", # scene_image path(str) + >>> } + + Returns: + Dict[str, Any]: the preprocessed data + { + "sky_image": the preprocessed sky image(origin size) + "sky_image_refine": the preprocessed resized sky image + "scene_image": the preprocessed scene image(origin size) + "scene_image_refine": the preprocessed resized scene image + "img_metas": informations of preprocessed images, e.g. origin shape, pad information, resized shape. + } + """ + if 'sky_image' not in data.keys(): + raise Exception('sky_image not in input data') + if 'scene_image' not in data.keys(): + raise Exception('scene_image not in input data') + if isinstance(data['sky_image'], str): + sky_image = LoadImage.convert_to_ndarray(data['sky_image']) + sky_image = sky_image.astype(np.uint8) # RGB + sky_image = cv2.cvtColor(sky_image, cv2.COLOR_RGB2BGR) # BGR + if sky_image is not None: + sky_image = self.check_image(sky_image) + else: + raise Exception('sky_image is None') + else: + raise Exception('sky_image(path of sky image) is not valid') + if isinstance(data['scene_image'], str): + scene_image = LoadImage.convert_to_ndarray(data['scene_image']) + scene_image = scene_image.astype(np.uint8) # RGB + scene_image = cv2.cvtColor(scene_image, cv2.COLOR_RGB2BGR) # BGR + if scene_image is not None: + scene_image = self.check_image(scene_image) + else: + raise Exception('scene_image is None') + else: + raise Exception('scene_image(path of scene image) is not valid') + data = {} + sky_image_refine, sky_img_metas = self.process_single_img(sky_image) + scene_image_refine, scene_img_metas = self.process_single_img( + scene_image) + data['sky_image'] = sky_image + data['sky_image_refine'] = sky_image_refine + data['scene_image'] = scene_image + data['scene_image_refine'] = scene_image_refine + data['img_metas'] = { + 'sky_img_metas': sky_img_metas, + 'scene_img_metas': scene_img_metas, + 'input_size': { + 'coarse_input_size': self.coarse_input_size, + 'refine_input_size': self.refine_input_size + } + } + return data + + def process_single_img(self, img): + img_metas = {} + img_metas['ori_shape'] = img.shape[0:2] # img: (origin_h, origin_w, 3) + img, pad_direction = get_refine_input(img, self.refine_input_size) + img = image_transform( + img, self.normalize) # torch.Size([3, refine_net_h, refine_net_w]) + img = img.unsqueeze(0) + img_metas['pad_direction'] = pad_direction + img_metas['refine_shape'] = img.shape[ + 2:] # torch.Size([1, 3, refine_net_h, refine_net_w]) + return img, img_metas + + def check_image(self, input_img): + whole_temp_shape = input_img.shape + if len(whole_temp_shape) == 2: + input_img = np.stack([input_img, input_img, input_img], axis=2) + elif whole_temp_shape[2] == 1: + input_img = np.concatenate([input_img, input_img, input_img], + axis=2) + elif whole_temp_shape[2] == 4: + input_img = input_img[:, :, + 0:3] * 1.0 * input_img[:, :, + 3:4] * 1.0 / 255.0 + return input_img + + +def get_refine_input(mat, refine_input_size): + # maxDimMatch: resize + mat = max_dim_match(mat, refine_input_size) + # pad image to refine net input size + mat, pad_direction = center_pad_image_withwh(mat, refine_input_size, 0) + return mat, pad_direction + + +def max_dim_match(image, refine_model_size): + h, w, c = np.shape(image) + resize_w, resize_h = refine_model_size + if h != resize_h or w != resize_w: + h_scale = float(resize_h) / h + w_scale = float(resize_w) / w + resize_scale = min(w_scale, h_scale) + new_h = int(h * resize_scale + 0.5) + new_w = int(w * resize_scale + 0.5) + image = cv2.resize( + image, (new_w, new_h), interpolation=cv2.INTER_LINEAR) + return image + + +def center_pad_image_withwh(image, + crop_size, + padvalue, + padding_mode='constant'): + pad_image = image + h, w = image.shape[0], image.shape[1] + pad_h = max(crop_size[1] - h, 0) + pad_w = max(crop_size[0] - w, 0) + pad_direction = (0, 0, 0, 0) + if pad_h > 0 or pad_w > 0: + half_w = int(pad_w / 2 + 0.5) + half_h = int(pad_h / 2 + 0.5) + pad_direction = (half_w, half_h, pad_w - half_w, pad_h - half_h) + pad_image = pad( + image, pad_direction, padvalue, padding_mode=padding_mode) + return pad_image, pad_direction + + +def pad(img, padding, fill=0, padding_mode='constant'): + if not is_numpy_image(img): + raise TypeError('img should be numpy ndarray. Got {}'.format( + type(img))) + if not isinstance(padding, + (numbers.Number, tuple, list)) or len(padding) != 4: + raise TypeError('Got inappropriate padding arg') + + pad_left = padding[0] + pad_top = padding[1] + pad_right = padding[2] + pad_bottom = padding[3] + + shape_len = len(img.shape) + if shape_len == 2: + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_to_str[padding_mode], + value=fill, + ) + elif shape_len == 3 and img.shape[2] == 1: + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_to_str[padding_mode], + value=fill, + )[:, :, np.newaxis] + else: + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_to_str[padding_mode], + value=fill, + ) + + +def image_transform(img, normalize): + img = img[:, :, ::-1] # BGR-->RGB to pil format + img = img.transpose((2, 0, 1)) # h,w,c --> c,h,w + img = img.astype(np.float32) / 255 + img = normalize(torch.from_numpy(img.copy())) + return img + + +def is_numpy_image(img): + return isinstance(img, np.ndarray) and (img.ndim in {2, 3}) diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/BlockModules.py b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/BlockModules.py new file mode 100644 index 0000000000000000000000000000000000000000..507f3cc69f170334327dda78618881b415a035c0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/BlockModules.py @@ -0,0 +1,118 @@ +# The implementation is adopted from ASPP made publicly available under the MIT License License +# at https://github.com/jfzhang95/pytorch-deeplab-xception +import torch +import torch.nn.functional as F +from torch import nn + +BatchNorm2d = nn.BatchNorm2d + + +class ASPPModule(nn.Module): + + def __init__(self, inplanes, planes, kernel_size, padding, dilation, + BatchNorm): + super(ASPPModule, self).__init__() + self.atrous_conv = nn.Conv2d( + inplanes, + planes, + kernel_size=kernel_size, + stride=1, + padding=padding, + dilation=dilation, + bias=False) + self.bn = BatchNorm(planes) + self.relu = nn.ReLU() + + self._init_weight() + + def forward(self, x): + x = self.atrous_conv(x) + x = self.bn(x) + + return self.relu(x) + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +# this aspp is official version +# copy from :https://github.com/jfzhang95/pytorch-deeplab-xception +class ASPP(nn.Module): + + def __init__(self, inplanes, outplanes, dilations, drop_rate=0.1): + super(ASPP, self).__init__() + + self.aspp1 = ASPPModule( + inplanes, + outplanes, + 1, + padding=0, + dilation=dilations[0], + BatchNorm=BatchNorm2d) + self.aspp2 = ASPPModule( + inplanes, + outplanes, + 3, + padding=dilations[1], + dilation=dilations[1], + BatchNorm=BatchNorm2d) + self.aspp3 = ASPPModule( + inplanes, + outplanes, + 3, + padding=dilations[2], + dilation=dilations[2], + BatchNorm=BatchNorm2d) + self.aspp4 = ASPPModule( + inplanes, + outplanes, + 3, + padding=dilations[3], + dilation=dilations[3], + BatchNorm=BatchNorm2d) + + self.global_avg_pool = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(inplanes, outplanes, 1, stride=1, bias=False), + BatchNorm2d(outplanes), nn.ReLU()) + self.conv1 = nn.Conv2d(outplanes * 5, outplanes, 1, bias=False) + self.bn1 = BatchNorm2d(outplanes) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(drop_rate) + self._init_weight() + + def forward(self, x): # [1, 256, 320, 320] + x1 = self.aspp1(x) # [1, 128, 160, 160] + x2 = self.aspp2(x) # [1, 128, 160, 160] + x3 = self.aspp3(x) # [1, 128, 160, 160] + x4 = self.aspp4(x) # [1, 128, 160, 160] + x5 = self.global_avg_pool(x) # b,c,h,w [1, 128, 1, 1] + x5 = F.interpolate( + x5, size=x4.size()[2:], mode='bilinear', + align_corners=True) # [1, 128, 160, 160] + x = torch.cat((x1, x2, x3, x4, x5), dim=1) # [1, 640, 160, 160] + + x = self.conv1(x) # [1, 640, 160, 160] + x = self.bn1(x) + x = self.relu(x) + + return self.dropout(x) + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/__init__.py b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_backnone.py b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_backnone.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcb66252543dc23359f96bd175dc07e1bc03d76 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_backnone.py @@ -0,0 +1,620 @@ +# The implementation is adopted from HRNet, made publicly available under the MIT License License +# at https://github.com/HRNet/HRNet-Semantic-Segmentation +import logging +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +BatchNorm2d = nn.BatchNorm2d + +BN_MOMENTUM = 0.1 +logger = logging.getLogger(__name__) + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = BatchNorm2d(planes * self.expansion, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class HighResolutionModule(nn.Module): + + def __init__(self, + num_branches, + blocks, + num_blocks, + num_inchannels, + num_channels, + fuse_method, + multi_scale_output=True): + super(HighResolutionModule, self).__init__() + self._check_branches(num_branches, blocks, num_blocks, num_inchannels, + num_channels) + + self.num_inchannels = num_inchannels + self.fuse_method = fuse_method + self.num_branches = num_branches + + self.multi_scale_output = multi_scale_output + + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(inplace=True) + + def _check_branches(self, num_branches, blocks, num_blocks, num_inchannels, + num_channels): + if num_branches != len(num_blocks): + error_msg = 'NUM_BRANCHES({}) <> NUM_BLOCKS({})'.format( + num_branches, len(num_blocks)) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = 'NUM_BRANCHES({}) <> NUM_CHANNELS({})'.format( + num_branches, len(num_channels)) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_inchannels): + error_msg = 'NUM_BRANCHES({}) <> NUM_INCHANNELS({})'.format( + num_branches, len(num_inchannels)) + logger.error(error_msg) + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + downsample = None + if stride != 1 or self.num_inchannels[ + branch_index] != num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.num_inchannels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + BatchNorm2d( + num_channels[branch_index] * block.expansion, + momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index], stride, downsample)) + self.num_inchannels[ + branch_index] = num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block(self.num_inchannels[branch_index], + num_channels[branch_index])) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + if self.num_branches == 1: + return None + + num_branches = self.num_branches + num_inchannels = self.num_inchannels # tuple + fuse_layers = [] + for i in range(num_branches if self.multi_scale_output else 1): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_inchannels[i], + 1, + 1, + 0, + bias=False), + BatchNorm2d( + num_inchannels[i], momentum=BN_MOMENTUM))) + elif j == i: + fuse_layer.append(None) + else: + conv3x3s = [] + for k in range(i - j): + if k == i - j - 1: + num_outchannels_conv3x3 = num_inchannels[i] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + BatchNorm2d( + num_outchannels_conv3x3, + momentum=BN_MOMENTUM))) + else: + num_outchannels_conv3x3 = num_inchannels[j] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False), + BatchNorm2d( + num_outchannels_conv3x3, + momentum=BN_MOMENTUM), + nn.ReLU(inplace=True))) + fuse_layer.append(nn.Sequential(*conv3x3s)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def get_num_inchannels(self): + return self.num_inchannels + + def forward(self, x): + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = x[0] if i == 0 else self.fuse_layers[i][0](x[0]) + for j in range(1, self.num_branches): + if i == j: + y = y + x[j] + elif j > i: + width_output = x[i].shape[-1] + height_output = x[i].shape[-2] + y = y + F.interpolate( + self.fuse_layers[i][j](x[j]), + size=(height_output, width_output), + mode='bilinear', + align_corners=True) + else: + y = y + self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + + return x_fuse + + +model_w18v1 = { + 'STAGE1': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 1, + 'BLOCK': 'BOTTLENECK', + 'NUM_BLOCKS': (1), + 'NUM_CHANNELS': (32), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE2': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 2, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2), + 'NUM_CHANNELS': (16, 32), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE3': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 3, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2, 2), + 'NUM_CHANNELS': (16, 32, 64), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE4': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 4, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2, 2, 2), + 'NUM_CHANNELS': (16, 32, 64, 128), + 'FUSE_METHOD': 'SUM' + }, + 'FINAL_CONV_KERNEL': 1 +} + +model_w18v2 = { + 'STAGE1': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 1, + 'BLOCK': 'BOTTLENECK', + 'NUM_BLOCKS': (2), + 'NUM_CHANNELS': (64), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE2': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 2, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2), + 'NUM_CHANNELS': (18, 36), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE3': { + 'NUM_MODULES': 3, + 'NUM_BRANCHES': 3, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2, 2), + 'NUM_CHANNELS': (18, 36, 72), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE4': { + 'NUM_MODULES': 2, + 'NUM_BRANCHES': 4, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (2, 2, 2, 2), + 'NUM_CHANNELS': (18, 36, 72, 144), + 'FUSE_METHOD': 'SUM' + }, + 'FINAL_CONV_KERNEL': 1 +} + +model_w48 = { + 'STAGE1': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 1, + 'BLOCK': 'BOTTLENECK', + 'NUM_BLOCKS': (4), + 'NUM_CHANNELS': (64), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE2': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 2, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (4, 4), + 'NUM_CHANNELS': (48, 96), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE3': { + 'NUM_MODULES': 4, + 'NUM_BRANCHES': 3, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (4, 4, 4), + 'NUM_CHANNELS': (48, 96, 192), + 'FUSE_METHOD': 'SUM' + }, + 'STAGE4': { + 'NUM_MODULES': 3, + 'NUM_BRANCHES': 4, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': (4, 4, 4, 4), + 'NUM_CHANNELS': (48, 96, 192, 384), + 'FUSE_METHOD': 'SUM' + }, + 'FINAL_CONV_KERNEL': 1 +} + +model_version_dict = {} +model_version_dict['w48'] = model_w48 +model_version_dict['w18v1'] = model_w18v1 +model_version_dict['w18v2'] = model_w18v2 + +blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + +class HrnetBackBone(nn.Module): + + def __init__(self, **kwargs): + super(HrnetBackBone, self).__init__() + + assert 'version' in kwargs, 'hrnet not exist model version' + extra = model_version_dict[kwargs['version']] + + # stem net + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = BatchNorm2d(64, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + 64, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn2 = BatchNorm2d(64, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + + self.stage1_cfg = extra['STAGE1'] + num_channels = self.stage1_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage1_cfg['BLOCK']] + num_blocks = self.stage1_cfg['NUM_BLOCKS'] + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + stage1_out_channel = block.expansion * num_channels + + self.stage2_cfg = extra['STAGE2'] + num_channels = self.stage2_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage2_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition1 = self._make_transition_layer([stage1_out_channel], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + self.stage3_cfg = extra['STAGE3'] + num_channels = self.stage3_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage3_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + self.stage4_cfg = extra['STAGE4'] + num_channels = self.stage4_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage4_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels, multi_scale_output=True) + + self.backbone_last_inp_channels = int(np.sum(pre_stage_channels)) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + nn.Conv2d( + num_channels_pre_layer[i], + num_channels_cur_layer[i], + 3, + 1, + 1, + bias=False), + BatchNorm2d( + num_channels_cur_layer[i], + momentum=BN_MOMENTUM), nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv3x3s = [] + for j in range(i + 1 - num_branches_pre): + inchannels = num_channels_pre_layer[-1] + outchannels = num_channels_cur_layer[ + i] if j == i - num_branches_pre else inchannels + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + inchannels, outchannels, 3, 2, 1, bias=False), + BatchNorm2d(outchannels, momentum=BN_MOMENTUM), + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv3x3s)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(inplanes, planes, stride, downsample)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_stage(self, + layer_config, + num_inchannels, + multi_scale_output=True): + num_modules = layer_config['NUM_MODULES'] + num_branches = layer_config['NUM_BRANCHES'] + num_blocks = layer_config['NUM_BLOCKS'] + num_channels = layer_config['NUM_CHANNELS'] + block = blocks_dict[layer_config['BLOCK']] + fuse_method = layer_config['FUSE_METHOD'] + + modules = [] + for i in range(num_modules): + # multi_scale_output is only used last module + if not multi_scale_output and i == num_modules - 1: + reset_multi_scale_output = False + else: + reset_multi_scale_output = True + modules.append( + HighResolutionModule(num_branches, block, num_blocks, + num_inchannels, num_channels, fuse_method, + reset_multi_scale_output)) + num_inchannels = modules[-1].get_num_inchannels() + + return nn.Sequential(*modules), num_inchannels + + def _backbone_forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['NUM_BRANCHES']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['NUM_BRANCHES']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['NUM_BRANCHES']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + x = self.stage4(x_list) + + # Upsampling + x0_h, x0_w = x[0].size(2), x[0].size(3) + x1 = F.interpolate( + x[1], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + x2 = F.interpolate( + x[2], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + x3 = F.interpolate( + x[3], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + + x = torch.cat([x[0], x1, x2, x3], 1) + return x + + def init_weights(self, url, cache_file=''): + pretrained_dict = load_state(url, model_dir=cache_file) + model_dict = self.state_dict() + + model_len = len(model_dict) + pretrain_len = len(pretrained_dict) + common_dict = {} + valid_layer_num = 0 + for k, v in pretrained_dict.items(): + if k in model_dict: + common_dict[k] = v + valid_layer_num += 1 + + print('*' * 50) + print('Model Param Num:{} Pretrained Param Num:{} ' + 'Commmon Num:{}'.format(model_len, pretrain_len, + valid_layer_num)) + print('-' * 50) + print('Model Extra Param Names:\n\t{}'.format( + set(model_dict) - set(pretrained_dict))) + print('-' * 50) + print('Pretrained Extra Param Names:\n\t{}'.format( + set(pretrained_dict) - set(model_dict))) + print('*' * 50) + + model_dict.update(common_dict) + self.load_state_dict(model_dict) diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py new file mode 100644 index 0000000000000000000000000000000000000000..5dbef66e9715d3002c8d793b5b9914956eb38265 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/hrnet_super_and_ocr.py @@ -0,0 +1,510 @@ +# Part of the implementation is borrowed and modified from HRNet, +# publicly available under the MIT License License at https://github.com/HRNet/HRNet-Semantic-Segmentation +from __future__ import absolute_import, division, print_function + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .BlockModules import ASPP +from .hrnet_backnone import BatchNorm2d, HrnetBackBone, blocks_dict + +ALIGN_CORNERS = True +BN_MOMENTUM = 0.1 + + +class ModuleHelper: + + @staticmethod + def BNReLU(num_features, bn_type=None, **kwargs): + return nn.Sequential(BatchNorm2d(num_features, **kwargs), nn.ReLU()) + + @staticmethod + def BatchNorm2d(*args, **kwargs): + return BatchNorm2d + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class SpatialGatherModule(nn.Module): + """ + Aggregate the context features according to the initial + predicted probability distribution. + Employ the soft-weighted method to aggregate the context. + """ + + def __init__(self, cls_num=0, scale=1): + super(SpatialGatherModule, self).__init__() + self.cls_num = cls_num + self.scale = scale + + def forward(self, feats, probs): + batch_size, c, _, _ = probs.size(0), probs.size(1), probs.size( + 2), probs.size(3) + probs = probs.view(batch_size, c, -1) + feats = feats.view(batch_size, feats.size(1), -1) + feats = feats.permute(0, 2, 1) # batch x hw x c + probs = F.softmax(self.scale * probs, dim=2) # batch x k x hw + ocr_context = torch.matmul(probs, feats) # batch x k x c + + ocr_context = ocr_context.permute(0, 2, + 1).unsqueeze(3) # batch x c x k x 1 + return ocr_context + + +class ObjectAttentionBlock(nn.Module): + ''' + The basic implementation for object context block + Input: + N X C X H X W + Parameters: + in_channels : the dimension of the input feature map + key_channels : the dimension after the key/query transform + scale : choose the scale to downsample the input feature maps (save memory cost) + bn_type : specify the bn type + Return: + N X C X H X W + ''' + + def __init__(self, in_channels, key_channels, scale=1, bn_type=None): + super(ObjectAttentionBlock, self).__init__() + self.scale = scale + self.in_channels = in_channels + self.key_channels = key_channels + self.pool = nn.MaxPool2d(kernel_size=(scale, scale)) + self.f_pixel = nn.Sequential( + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.key_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), + nn.Conv2d( + in_channels=self.key_channels, + out_channels=self.key_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), + ) + self.f_object = nn.Sequential( + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.key_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), + nn.Conv2d( + in_channels=self.key_channels, + out_channels=self.key_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), + ) + self.f_down = nn.Sequential( + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.key_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.key_channels, bn_type=bn_type), + ) + self.f_up = nn.Sequential( + nn.Conv2d( + in_channels=self.key_channels, + out_channels=self.in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(self.in_channels, bn_type=bn_type), + ) + + def forward(self, x, proxy): + batch_size, h, w = x.size(0), x.size(2), x.size(3) + if self.scale > 1: + x = self.pool(x) + + query = self.f_pixel(x).view(batch_size, self.key_channels, -1) + query = query.permute(0, 2, 1) + key = self.f_object(proxy).view(batch_size, self.key_channels, -1) + value = self.f_down(proxy).view(batch_size, self.key_channels, -1) + value = value.permute(0, 2, 1) + + sim_map = torch.matmul(query, key) + sim_map = (self.key_channels**-.5) * sim_map + sim_map = F.softmax(sim_map, dim=-1) + + # add bg context ... + context = torch.matmul(sim_map, value) + context = context.permute(0, 2, 1).contiguous() + context = context.view(batch_size, self.key_channels, *x.size()[2:]) + context = self.f_up(context) + if self.scale > 1: + context = F.interpolate( + input=context, + size=(h, w), + mode='bilinear', + align_corners=ALIGN_CORNERS) + + return context + + +class ObjectAttentionBlock2D(ObjectAttentionBlock): + + def __init__(self, in_channels, key_channels, scale=1, bn_type=None): + super(ObjectAttentionBlock2D, self).__init__( + in_channels, key_channels, scale, bn_type=bn_type) + + +class SpatialOCRModule(nn.Module): + """ + Implementation of the OCR module: + We aggregate the global object representation to update the representation for each pixel. + """ + + def __init__(self, + in_channels, + key_channels, + out_channels, + scale=1, + dropout=0.1, + bn_type=None): + super(SpatialOCRModule, self).__init__() + self.object_context_block = ObjectAttentionBlock2D( + in_channels, key_channels, scale, bn_type) + _in_channels = 2 * in_channels + + self.conv_bn_dropout = nn.Sequential( + nn.Conv2d( + _in_channels, + out_channels, + kernel_size=1, + padding=0, + bias=False), + ModuleHelper.BNReLU(out_channels, bn_type=bn_type), + nn.Dropout2d(dropout)) + + def forward(self, feats, proxy_feats): + context = self.object_context_block(feats, proxy_feats) + + output = self.conv_bn_dropout(torch.cat([context, feats], 1)) + + return output + + +class HrnetSuperAndOcr(HrnetBackBone): + + def __init__(self, **kwargs): + super(HrnetSuperAndOcr, self).__init__(**kwargs) + if 'architecture' not in kwargs: + raise Exception('HrnetSuperAndOcr not exist architecture param!') + self.architecture = kwargs['architecture'] + + if 'class_num' not in kwargs: + raise Exception('HrnetSuperAndOcr not exist class_num param!') + self.class_num = kwargs['class_num'] + + if 'ocr' not in kwargs: + raise Exception('HrnetSuperAndOcr not exist ocr param!') + ocr_mid_channels = kwargs['ocr']['mid_channels'] + ocr_key_channels = kwargs['ocr']['key_channels'] + dropout_rate = kwargs['ocr']['dropout_rate'] + scale = kwargs['ocr']['scale'] + + if 'super_param' not in kwargs: + raise Exception('HrnetSuperAndOcr not exist super_param param!') + + self.super_dict = kwargs['super_param'] + + self.is_export_onnx = False + self.is_export_full_onnx = False + + self.is_contain_tail = True if 'tail_param' in kwargs else False + if self.is_contain_tail: + self.stage_tail_dict = kwargs['tail_param'] + num_channels = self.stage_tail_dict['NUM_CHANNELS'][0] + block = blocks_dict[self.stage_tail_dict['BLOCK']] + num_blocks = self.stage_tail_dict['NUM_BLOCKS'][0] + self.stage_tail = self._make_layer(block, + self.backbone_last_inp_channels, + num_channels, num_blocks) + last_inp_channels = block.expansion * num_channels + else: + last_inp_channels = self.backbone_last_inp_channels + + self.is_contain_aspp = True if 'aspp' in kwargs else False + + if self.architecture == 'hrnet_super_ocr': + self.is_ocr_first = False + num_channels = [64, last_inp_channels] + self.stage_super, super_stage_channels = self._make_stage( + self.super_dict, num_channels) + last_inp_channels = int(np.sum(super_stage_channels)) + + if self.is_contain_aspp: + aspp_param = kwargs['aspp'] + self.aspp_layer = ASPP( + inplanes=last_inp_channels, + outplanes=aspp_param['outplanes'], + dilations=aspp_param['dilations'], + drop_rate=aspp_param['drop_rate']) + last_inp_channels = aspp_param['outplanes'] + + self.aux_head = nn.Sequential( + nn.Conv2d( + last_inp_channels, + last_inp_channels, + kernel_size=1, + stride=1, + padding=0), BatchNorm2d(last_inp_channels), + nn.ReLU(inplace=True), + nn.Conv2d( + last_inp_channels, + self.class_num, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + + self.conv3x3_ocr = nn.Sequential( + nn.Conv2d( + last_inp_channels, + ocr_mid_channels, + kernel_size=3, + stride=1, + padding=1), + BatchNorm2d(ocr_mid_channels), + nn.ReLU(inplace=True), + ) + self.ocr_gather_head = SpatialGatherModule(self.class_num) + + self.ocr_distri_head = SpatialOCRModule( + in_channels=ocr_mid_channels, + key_channels=ocr_key_channels, + out_channels=ocr_mid_channels, + scale=scale, + dropout=dropout_rate, + ) + + self.cls_head = nn.Sequential( + nn.Conv2d( + ocr_mid_channels, + ocr_mid_channels, + kernel_size=1, + stride=1, + padding=0), BatchNorm2d(ocr_mid_channels), + nn.ReLU(inplace=True), + nn.Conv2d( + ocr_mid_channels, + self.class_num, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + else: + self.is_ocr_first = True + + if self.is_contain_aspp: + aspp_param = kwargs['aspp'] + self.aspp_layer = ASPP( + inplanes=last_inp_channels, + outplanes=aspp_param['outplanes'], + dilations=aspp_param['dilations'], + drop_rate=aspp_param['drop_rate']) + last_inp_channels = aspp_param['outplanes'] + + self.aux_head = nn.Sequential( + nn.Conv2d( + last_inp_channels, + last_inp_channels, + kernel_size=1, + stride=1, + padding=0), BatchNorm2d(last_inp_channels), + nn.ReLU(inplace=True), + nn.Conv2d( + last_inp_channels, + self.class_num, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + + self.conv3x3_ocr = nn.Sequential( + nn.Conv2d( + last_inp_channels, + ocr_mid_channels, + kernel_size=3, + stride=1, + padding=1), + BatchNorm2d(ocr_mid_channels), + nn.ReLU(inplace=True), + ) + self.ocr_gather_head = SpatialGatherModule(self.class_num) + + self.ocr_distri_head = SpatialOCRModule( + in_channels=ocr_mid_channels, + key_channels=ocr_key_channels, + out_channels=ocr_mid_channels, + scale=scale, + dropout=dropout_rate, + ) + + num_channels = [64, ocr_mid_channels] + self.stage_super, super_stage_channels = self._make_stage( + self.super_dict, num_channels) + last_inp_channels = int(np.sum(super_stage_channels)) + + self.cls_head = nn.Sequential( + nn.Conv2d( + last_inp_channels, + last_inp_channels, + kernel_size=1, + stride=1, + padding=0), BatchNorm2d(last_inp_channels), + nn.ReLU(inplace=True), + nn.Conv2d( + last_inp_channels, + self.class_num, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + + def forward(self, x): + if self.is_export_onnx: + x = x.permute(0, 3, 1, 2) + raw_h, raw_w = x.size(2), x.size(3) + if self.is_export_full_onnx: + raw_h, raw_w = x.size(2), x.size(3) + + x = self.conv1(x) + x = self.bn1(x) # 5, 64, 320, 320 + x_stem = self.relu(x) + x = self.conv2(x_stem) + x = self.bn2(x) + x = self.relu(x) # 5, 64, 160, 160 + + x = self.layer1(x) # 5, 256=64*4, 160, 160 + + x_list = [] + for i in range(self.stage2_cfg['NUM_BRANCHES']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) # [[5, 18, 160, 160],[5, 36, 80, 80]] + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['NUM_BRANCHES']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['NUM_BRANCHES']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + x = self.stage4(x_list) + + # Upsampling + x0_h, x0_w = x[0].size(2), x[0].size(3) + x1 = F.interpolate( + x[1], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + x2 = F.interpolate( + x[2], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + x3 = F.interpolate( + x[3], size=(x0_h, x0_w), mode='bilinear', align_corners=True) + + feats = torch.cat([x[0], x1, x2, x3], 1) + + if self.is_contain_tail: + feats = self.stage_tail(feats) + + if self.is_ocr_first: + + if self.is_contain_aspp: + feats = self.aspp_layer(feats) + # compute contrast feature + out_aux = self.aux_head(feats) + + feats = self.conv3x3_ocr(feats) + context = self.ocr_gather_head(feats, out_aux) + feats = self.ocr_distri_head(feats, context) + + feats = [x_stem, feats] # 320*320 2X + x_super = self.stage_super(feats) + + xsuper_h, xsuper_w = x_super[0].size(2), x_super[0].size(3) + x_super1 = F.interpolate( + x_super[1], + size=(xsuper_h, xsuper_w), + mode='bilinear', + align_corners=True) + x_super = torch.cat([x_super[0], x_super1], 1) + out = self.cls_head(x_super) + + else: + x_super = [x_stem, feats] # 320*320 2X, 160*160 4X + x_super = self.stage_super(x_super) + + xsuper_h, xsuper_w = x_super[0].size(2), x_super[0].size(3) + x_super1 = F.interpolate( + x_super[1], + size=(xsuper_h, xsuper_w), + mode='bilinear', + align_corners=True) + x_super = torch.cat([x_super[0], x_super1], 1) + + if self.is_contain_aspp: + x_super = self.aspp_layer(x_super) + out_aux = self.aux_head(x_super) + + feats = self.conv3x3_ocr(x_super) + context = self.ocr_gather_head(feats, out_aux) + feats = self.ocr_distri_head(feats, context) + + out = self.cls_head(feats) + + if self.is_export_onnx or self.is_export_full_onnx: + x_class = F.interpolate( + out, size=(raw_h, raw_w), mode='bilinear', align_corners=True) + x_class = torch.softmax(x_class, dim=1) + _, x_class = torch.max(x_class, dim=1, keepdim=True) + x_class = x_class.float() + return x_class + else: + out_aux_seg = [ + out_aux, out + ] # out_aux: 5, 2, 160, 160(HRNet origin res); out: 5, 2, 320, 320(HRNet res+tail+aspp+ocr) + return out_aux_seg + + +def get_seg_model(cfg, **kwargs): + model = HrnetSuperAndOcr(cfg, **kwargs) + model.init_weights(cfg.MODEL.PRETRAINED) + + return model diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/unet.py b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..20affbefd51d0b31258429b64392907870a2b669 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/ptsemseg/unet.py @@ -0,0 +1,229 @@ +# Copyright 2021-2022 The Alibaba Vision Team Authors. All rights reserved. +import torch +import torch.nn as nn + +from .BlockModules import ASPP + + +class Conv2DBatchNormRelu(nn.Module): + + def __init__(self, + in_channels, + n_filters, + k_size, + stride, + padding, + bias=True, + dilation=1, + with_bn=True, + with_relu=True): + super(Conv2DBatchNormRelu, self).__init__() + + conv_mod = nn.Conv2d( + int(in_channels), + int(n_filters), + kernel_size=k_size, + padding=padding, + stride=stride, + bias=bias, + dilation=dilation, + ) + + if with_bn: + if with_relu: + self.cbr_unit = nn.Sequential(conv_mod, + nn.BatchNorm2d(int(n_filters)), + nn.ReLU(inplace=True)) + else: + self.cbr_unit = nn.Sequential(conv_mod, + nn.BatchNorm2d(int(n_filters))) + else: + if with_relu: + self.cbr_unit = nn.Sequential(conv_mod, nn.ReLU(inplace=True)) + else: + self.cbr_unit = nn.Sequential(conv_mod) + + def forward(self, inputs): + outputs = self.cbr_unit(inputs) + return outputs + + +class SegnetDown2(nn.Module): + + def __init__(self, in_size, out_size): + super(SegnetDown2, self).__init__() + self.conv1 = Conv2DBatchNormRelu( + in_size, out_size, k_size=3, stride=1, padding=1) + self.conv2 = Conv2DBatchNormRelu( + out_size, out_size, k_size=3, stride=1, padding=1) + self.maxpool_with_argmax = nn.MaxPool2d(2, 2, return_indices=True) + + def forward(self, inputs): + outputs = self.conv1(inputs) + outputs = self.conv2(outputs) + unpooled_shape = outputs.size() + outputs, indices = self.maxpool_with_argmax(outputs) + return outputs, indices, unpooled_shape + + +class SegnetDown3(nn.Module): + + def __init__(self, in_size, out_size): + super(SegnetDown3, self).__init__() + self.conv1 = Conv2DBatchNormRelu( + in_size, out_size, k_size=3, stride=1, padding=1) + self.conv2 = Conv2DBatchNormRelu( + out_size, out_size, k_size=3, stride=1, padding=1) + self.conv3 = Conv2DBatchNormRelu( + out_size, out_size, k_size=3, stride=1, padding=1) + self.maxpool_with_argmax = nn.MaxPool2d(2, 2, return_indices=True) + + def forward(self, inputs): + outputs = self.conv1(inputs) + outputs = self.conv2(outputs) + outputs = self.conv3(outputs) + unpooled_shape = outputs.size() + outputs, indices = self.maxpool_with_argmax(outputs) + return outputs, indices, unpooled_shape + + +class SegnetUp1(nn.Module): + + def __init__(self, in_size, out_size): + super(SegnetUp1, self).__init__() + self.unpool = nn.MaxUnpool2d(2, 2) + self.conv = Conv2DBatchNormRelu( + in_size, out_size, k_size=5, stride=1, padding=2, with_relu=False) + + def forward(self, inputs, indices, output_shape): + outputs = self.unpool( + input=inputs, indices=indices, output_size=output_shape) + outputs = self.conv(outputs) + return outputs + + +class Unet(nn.Module): + + def __init__(self, + n_classes=2, + in_channels=4, + is_unpooling=True, + pretrain=True, + **kwargs): + super(Unet, self).__init__() + print('Load Unet') + self.in_channels = in_channels + self.is_unpooling = is_unpooling + self.pretrain = pretrain + self.is_contain_aspp = True if 'aspp' in kwargs else False + + if self.is_contain_aspp: + aspp_param = kwargs['aspp'] + self.aspp_layer = ASPP( + inplanes=128, + outplanes=aspp_param['outplanes'], + dilations=aspp_param['dilations'], + drop_rate=aspp_param['drop_rate']) + self.aspp_channels = aspp_param['outplanes'] + + self.down1 = SegnetDown2(self.in_channels, 64) + self.down2 = SegnetDown2(64, 128) + self.down3 = SegnetDown3(128, 256) + self.down4 = SegnetDown3(256, 512) + self.down5 = SegnetDown3(512, 512) + + self.up5 = SegnetUp1(512, 512) + self.up4 = SegnetUp1(512, 256) + self.up3 = SegnetUp1(256, 128) + + if self.is_contain_aspp: + self.conv_1x1_aspp = Conv2DBatchNormRelu( + 128 + self.aspp_channels, + 128, + k_size=1, + stride=1, + padding=0, + with_relu=False) + + self.up2 = SegnetUp1(128, 64) + self.up1 = SegnetUp1(64, n_classes) + self.sigmoid = nn.Sigmoid() + + if self.pretrain: + import torchvision.models as models + vgg16 = models.vgg16() + self.init_vgg16_params(vgg16) + + def forward(self, inputs): # [1, 4, 1346, 1152] [2, 4, 1280, 1280] + # inputs: [N, 4, 320, 320] + # outputs, indices, unpooled_shape + down1, indices_1, unpool_shape1 = self.down1( + inputs) # [1, 64, 673, 576] [2, 64, 640, 640] + down2, indices_2, unpool_shape2 = self.down2( + down1) # [1, 128, 336, 288] [2, 128, 320, 320] + down3, indices_3, unpool_shape3 = self.down3( + down2) # [1, 256, 168, 144] [2, 256, 160, 160] + torch.cuda.empty_cache() + if self.is_contain_aspp: # batchsize can not be 1 + aspp_output = self.aspp_layer(down2) + + down4, indices_4, unpool_shape4 = self.down4( + down3) # [1, 512, 84, 72] [2, 512, 80, 80] + down5, indices_5, unpool_shape5 = self.down5( + down4) # [1, 512, 42, 36] [2, 512, 80, 80] + torch.cuda.empty_cache() + up5 = self.up5(down5, indices_5, + unpool_shape5) # [1, 512, 84, 72] [2, 512, 80, 80] + up4 = self.up4(up5, indices_4, + unpool_shape4) # [1, 256, 168, 144] [2, 256, 160, 160] + torch.cuda.empty_cache() + up3 = self.up3( + up4, indices_3, + unpool_shape3) # [1, 128, 336, 288] [2, 128, 320, 320] + if self.is_contain_aspp: + up3 = torch.cat([up3, aspp_output], 1) # [2, 256, 320, 320] + up3 = self.conv_1x1_aspp(up3) # [2, 128, 320, 320] + + up2 = self.up2( + up3, indices_2, + unpool_shape2) # [1, 64, 673, 576] indices_2: [2, 128, 320, 320] + up1 = self.up1(up2, indices_1, unpool_shape1) # [1, 1, 1346, 1152] + + x = torch.squeeze(up1, dim=1) # [N, 1, 320, 320] -> [N, 320, 320] + x = self.sigmoid(x) + + return x # [2, 1280, 1280] + + def init_vgg16_params(self, vgg16): + blocks = [self.down1, self.down2, self.down3, self.down4, self.down5] + + features = list(vgg16.features.children()) + + vgg_layers = [] + for _layer in features: + if isinstance(_layer, nn.Conv2d): + vgg_layers.append(_layer) + + merged_layers = [] + for idx, conv_block in enumerate(blocks): + if idx < 2: + units = [conv_block.conv1.cbr_unit, conv_block.conv2.cbr_unit] + else: + units = [ + conv_block.conv1.cbr_unit, + conv_block.conv2.cbr_unit, + conv_block.conv3.cbr_unit, + ] + for _unit in units: + for _layer in _unit: + if isinstance(_layer, nn.Conv2d): + merged_layers.append(_layer) + + assert len(vgg_layers) == len(merged_layers) + + for l1, l2 in zip(vgg_layers, merged_layers): + if isinstance(l1, nn.Conv2d) and isinstance(l2, nn.Conv2d): + if l1.weight.size() == l2.weight.size() and l1.bias.size( + ) == l2.bias.size(): + l2.weight.data = l1.weight.data + l2.bias.data = l1.bias.data diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/skychange.py b/runtime/Lib/site_packages/models/cv/image_skychange/skychange.py new file mode 100644 index 0000000000000000000000000000000000000000..274753ac69768fc55f80edf6d92088da2439e05d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/skychange.py @@ -0,0 +1,310 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numbers +import os +import pdb +from collections import deque + +import cv2 +import json +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torchvision import transforms + +torch.backends.cudnn.enabled = True + +IMAGE_MAX_DIM = 3000 +IMAGE_MIN_DIM = 50 +IMAGE_MAX_RATIO = 10.0 +IMAGE_BLENDER_MASK_RESIZE_SCALE = 10.0 +IMAGE_BLENDER_INNER_RECT_MAX_DIM = 256 +IMAGE_BLENDER_DILATE_KERNEL_SIZE = 7 +IMAGE_BLENDER_VALID_MASK_THRESHOLD = 100 +IMAGE_BLENDER_MIN_VALID_SKY_AREA = 100 +IMAGE_BLENDER_MIN_RESIZE_DIM = 10 +IMAGE_BLENDER_BLUR_KERNEL_SIZE = 5 + + +def extract_sky_image(in_sky_image, in_sky_mask): + scale = 1.0 + resize_mask = in_sky_mask.copy() + + rows, cols = resize_mask.shape[0:2] + # src size: (512, 640), target size: (256,256), then scale to size (256, 320) + if (rows > IMAGE_BLENDER_INNER_RECT_MAX_DIM + or cols > IMAGE_BLENDER_INNER_RECT_MAX_DIM): + height_scale = IMAGE_BLENDER_INNER_RECT_MAX_DIM / float(rows) + width_scale = IMAGE_BLENDER_INNER_RECT_MAX_DIM / float(cols) + scale = height_scale if height_scale > width_scale else width_scale + new_size = (max(int(cols * scale), 1), max(int(rows * scale), + 1)) # w, h + resize_mask = cv2.resize(resize_mask, new_size, cv2.INTER_LINEAR) + + kernelSize = max(3, int(scale * IMAGE_BLENDER_DILATE_KERNEL_SIZE + 0.5)) + + element = cv2.getStructuringElement(cv2.MORPH_RECT, + (kernelSize, kernelSize)) + resize_mask = cv2.morphologyEx(resize_mask, cv2.MORPH_CLOSE, element) + + max_inner_rect, area = get_max_inner_rect( + resize_mask, IMAGE_BLENDER_VALID_MASK_THRESHOLD, True) + + if area < IMAGE_BLENDER_MIN_VALID_SKY_AREA: + raise Exception( + '[extractSkyImage]failed!! Valid sky region is too small') + + scale = 1.0 / scale + # max_inner_rect: left top(x,y), right bottome(x,y); raw_inner_rect:left top x,y,w(of bbox),h(of bbox) + raw_inner_rect = scale_rect(max_inner_rect, in_sky_mask, scale) + out_sky_image = in_sky_image[raw_inner_rect[1]:raw_inner_rect[1] + + raw_inner_rect[3] + 1, + raw_inner_rect[0]:raw_inner_rect[0] + + raw_inner_rect[2] + 1, ].copy() + return out_sky_image + + +def blend(scene_image, scene_mask, sky_image, sky_mask, inBlendLevelNum=10): + if torch.cuda.is_available(): + scene_image = scene_image.cpu().numpy() + sky_image = sky_image.cpu().numpy() + else: + scene_image = scene_image.numpy() + sky_image = sky_image.numpy() + sky_image_h, sky_image_w = sky_image.shape[0:2] + sky_mask_h, sky_mask_w = sky_mask.shape[0:2] + + scene_image_h, scene_image_w = scene_image.shape[0:2] + scene_mask_h, scene_mask_w = scene_mask.shape[0:2] + + if sky_image_h != sky_mask_h or sky_image_w != sky_mask_w: + raise Exception( + '[blend]failed!! sky_image shape not equal with sky_image_mask shape' + ) + + if scene_image_h != scene_mask_h or scene_image_w != scene_mask_w: + raise Exception( + '[blend]failed!! scene_image shape not equal with scene_image_mask shape' + ) + + valid_sky_image = extract_sky_image(sky_image, sky_mask) + out_blend_image = blend_merge(scene_image, scene_mask, valid_sky_image, + inBlendLevelNum) + return out_blend_image + + +def get_max_inner_rect(in_image_mask, in_alpha_threshold, is_bigger_valid): + res = 0 + row, col = in_image_mask.shape[0:2] + i0, j0, i1, j1 = 0, 0, 0, 0 + height = [0] * (col + 1) + + for i in range(0, row): + s = deque() + for j in range(0, col + 1): + if j < col: + if is_bigger_valid: + height[j] = ( + height[j] + + 1 if in_image_mask[i, j] > in_alpha_threshold else 0) + else: + height[j] = ( + height[j] + 1 + if in_image_mask[i, j] <= in_alpha_threshold else 0) + + while len(s) != 0 and height[s[-1]] >= height[j]: + cur = s[-1] + s.pop() + _h = height[cur] + _w = j if len(s) == 0 else j - s[-1] - 1 + curArea = _h * _w + if curArea > res: + res = curArea + i1 = i + i0 = i1 - _h + 1 + j1 = j - 1 + j0 = j1 - _w + 1 + s.append(j) + + out_rect = ( + j0, + i0, + j1 - j0 + 1, + i1 - i0 + 1, + ) + return out_rect, res + + +def scale_rect(in_rect, in_image_size, in_scale): + tlX = int(in_rect[0] * in_scale + 0.5) + tlY = int(in_rect[1] * in_scale + 0.5) + in_image_size_h, in_image_size_w = in_image_size.shape[0:2] + brX = min(int(in_rect[2] * in_scale + 0.5), in_image_size_w) + brY = min(int(in_rect[3] * in_scale + 0.5), in_image_size_h) + out_rect = (tlX, tlY, brX - tlX, brY - tlY) + return out_rect + + +def get_fast_valid_rect(in_mask, in_threshold=0): + # mask: np.array [0~1] + in_mask = in_mask > in_threshold + locations = cv2.findNonZero(in_mask.astype(np.uint8)) + output_rect = cv2.boundingRect(locations) # x,y,w,h + return output_rect + + +def min_size_match(in_image, in_min_size, type=cv2.INTER_LINEAR): + resize_image = in_image.copy() + width, height = in_min_size + resize_img_height, resize_img_width = in_image.shape[0:2] + height_scale = height / resize_img_height + widht_scale = width / resize_img_width + scale = height_scale if height_scale > widht_scale else widht_scale + new_size = ( + max(int(resize_img_width * scale + 0.5), 1), + max(int(resize_img_height * scale + 0.5), 1), + ) + + resize_image = cv2.resize(resize_image, new_size, 0, 0, type) + return resize_image + + +def center_crop(in_image, in_size): + in_size_w, in_size_h = in_size + in_image_h, in_image_w = in_image.shape[0:2] + + half_height = (in_image_h - in_size_h) // 2 + half_width = (in_image_w - in_size_w) // 2 + + out_crop_image = in_image.copy() + out_crop_image = out_crop_image[half_height:half_height + in_size_h, + half_width:half_width + in_size_w] + return out_crop_image + + +def safe_roi_pad(in_pad_image, in_rect, out_base_image): + in_rect_x, in_rect_y, in_rect_w, in_rect_h = in_rect + + if in_rect_x < 0 or in_rect_y < 0 or in_rect_w <= 0 or in_rect_h <= 0: + raise Exception('[safe_roi_pad] Failed!! x,y,w,h of rect are illegal') + + if in_rect_w != in_pad_image.shape[1] or in_rect_h != in_pad_image.shape[0]: + raise Exception('[safe_roi_pad] Failed!!') + + if (in_rect_x + in_rect_w > out_base_image.shape[1] + or in_rect_y + in_rect_h > out_base_image.shape[0]): + raise Exception('[safe_roi_pad] Failed!!') + + out_base_image[in_rect_y:in_rect_y + in_rect_h, + in_rect_x:in_rect_x + in_rect_w] = in_pad_image + + +def merge_image(in_base_image, in_merge_image, in_merge_mask, in_point): + if in_merge_image.shape[0:2] != in_merge_mask.shape[0:2]: + raise Exception( + '[merge_image] Failed!! in_merge_image.shape != in_merge_mask.shape!!' + ) + + in_point_x, in_point_y = in_point + in_merge_image_rows, in_merge_image_cols = in_merge_image.shape[0:2] + in_base_image_rows, in_base_image_cols = in_base_image.shape[0:2] + + if (in_point_x + in_merge_image_cols > in_base_image_cols + or in_point_y + in_merge_image_rows > in_base_image_rows): + raise Exception( + '[merge_image] Failed!! merge_image:image rect not in image') + + base_roi_image = in_base_image[in_point_y:in_point_y + in_merge_image_rows, + in_point_x:in_point_x + + in_merge_image_cols, ] + + merge_image = in_merge_image.copy() + merge_alpha = in_merge_mask.copy() + base_roi_image = np.float32(base_roi_image) + merge_alpha = np.repeat(merge_alpha[:, :, np.newaxis], 3, axis=2) + merge_alpha = merge_alpha / 255.0 + + base_roi_image = ( + 1 - merge_alpha) * base_roi_image + merge_alpha * merge_image + base_roi_image = np.clip(base_roi_image, 0, 255) + base_roi_image = base_roi_image.astype('uint8') + + roi_rect = (in_point_x, in_point_y, in_merge_image_cols, + in_merge_image_rows) + safe_roi_pad(base_roi_image, roi_rect, in_base_image) + return in_base_image + + +def blend_merge(in_scene_image, + in_scene_mask, + in_valid_sky_image, + inBlendLevelNum=5): + scene_sky_rect = get_fast_valid_rect(in_scene_mask, 1) + area = scene_sky_rect[2] * scene_sky_rect[3] + + if area < IMAGE_BLENDER_MIN_VALID_SKY_AREA: + raise Exception( + '[blend_merge] Failed!! Scene Image Valid sky region is too small') + + valid_sky_image = min_size_match(in_valid_sky_image, scene_sky_rect[2:]) + valid_sky_image = center_crop(valid_sky_image, scene_sky_rect[2:]) + + # resizeSceneMask + sky_size = ( + max( + int(in_scene_mask.shape[1] * IMAGE_BLENDER_MASK_RESIZE_SCALE + + 0.5), + IMAGE_BLENDER_MIN_RESIZE_DIM, + ), + max( + int(in_scene_mask.shape[0] * IMAGE_BLENDER_MASK_RESIZE_SCALE + + 0.5), + IMAGE_BLENDER_MIN_RESIZE_DIM, + ), + ) + + resize_scene_mask = cv2.resize(in_scene_mask, sky_size, cv2.INTER_LINEAR) + resize_scene_mask = cv2.blur( + resize_scene_mask, + (IMAGE_BLENDER_BLUR_KERNEL_SIZE, IMAGE_BLENDER_BLUR_KERNEL_SIZE), + ) + + element = cv2.getStructuringElement( + cv2.MORPH_RECT, + (IMAGE_BLENDER_BLUR_KERNEL_SIZE, IMAGE_BLENDER_BLUR_KERNEL_SIZE)) + sky_mask = cv2.dilate(resize_scene_mask, element) # enlarge sky region + scene_mask = cv2.erode(resize_scene_mask, element) # enlarge scene region + scene_mask = 255 - scene_mask + + sky_mask = cv2.resize(sky_mask, in_scene_mask.shape[0:2][::-1]) + scene_mask = cv2.resize(scene_mask, in_scene_mask.shape[0:2][::-1]) + + x, y, w, h = scene_sky_rect + valid_sky_mask = sky_mask[y:y + h, x:x + w] + + pano_sky_image = in_scene_image.copy() + + pano_sky_image = merge_image(pano_sky_image, valid_sky_image, + valid_sky_mask, scene_sky_rect[0:2]) + blend_images = [] + blend_images.append(in_scene_image) + blend_images.append(pano_sky_image) + + blend_masks = [] + blend_masks.append(scene_mask.astype(np.uint8)) + blend_masks.append(sky_mask.astype(np.uint8)) + + panorama_rect = (0, 0, in_scene_image.shape[1], in_scene_image.shape[0]) + + blender = cv2.detail_MultiBandBlender(1, inBlendLevelNum) + blender.prepare(panorama_rect) + + for i in range(0, len(blend_images)): + blender.feed(blend_images[i], blend_masks[i], (0, 0)) + pano_mask = ( + np.ones( + (in_scene_image.shape[1], in_scene_image.shape[0]), dtype='uint8') + * 255) + out_blend_image = np.zeros_like(in_scene_image) + result = blender.blend(out_blend_image, pano_mask) + return result[0] diff --git a/runtime/Lib/site_packages/models/cv/image_skychange/skychange_model.py b/runtime/Lib/site_packages/models/cv/image_skychange/skychange_model.py new file mode 100644 index 0000000000000000000000000000000000000000..fa16c38c04e1db19037ab326492f2a701775e52e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_skychange/skychange_model.py @@ -0,0 +1,199 @@ +import math +import os +import pdb +import time +from collections import OrderedDict +from typing import Any, Dict, List, Union + +import cv2 +import json +import torch +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .ptsemseg.hrnet_super_and_ocr import HrnetSuperAndOcr +from .ptsemseg.unet import Unet +from .skychange import blend + +logger = get_logger() + + +@MODELS.register_module( + Tasks.image_skychange, module_name=Models.image_skychange) +class ImageSkychange(TorchModel): + + def __init__(self, model_dir, refine_cfg, coarse_cfg, *args, **kwargs): + """ + Args: + model_dir (str): model directory to initialize some resource. + refine_cfg: configuration of refine model. + coarse_cfg: configuration of coarse model. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + if torch.cuda.is_available(): + self.device = torch.device('cuda') + logger.info('Use GPU: {}'.format(self.device)) + else: + self.device = torch.device('cpu') + logger.info('Use CPU: {}'.format(self.device)) + + coarse_model_path = '{}/{}'.format(model_dir, + ModelFile.TORCH_MODEL_FILE) + refine_model_path = '{}/{}'.format(model_dir, + 'unet_sky_matting_final_model.pkl') + + logger.info( + '####################### load refine models ################################' + ) + self.refine_model = Unet(**refine_cfg['Model']) + self.load_model(self.refine_model, refine_model_path) + self.refine_model.eval() + logger.info( + '####################### load refine models done ############################' + ) + + logger.info( + '####################### load coarse models ################################' + ) + self.coarse_model = HrnetSuperAndOcr(**coarse_cfg['Model']) + self.load_model(self.coarse_model, coarse_model_path) + self.coarse_model.eval() + logger.info( + '####################### load coarse models done ############################' + ) + + def load_model(self, seg_model, input_model_path): + if not os.path.isfile(input_model_path): + logger.error( + '[checkModelPath]:model path dose not exits!!! model Path:' + + input_model_path) + raise Exception('[checkModelPath]:model path dose not exits!') + + if torch.cuda.is_available(): + checkpoint = torch.load(input_model_path) + model_state = self.convert_state_dict(checkpoint['model_state']) + seg_model.load_state_dict(model_state) + seg_model.to(self.device) + else: + checkpoint = torch.load(input_model_path, map_location='cpu') + model_state = self.convert_state_dict(checkpoint['model_state']) + seg_model.load_state_dict(model_state) + + def convert_state_dict(self, state_dict): + """Converts a state dict saved from a dataParallel module to normal + module state_dict inplace + :param state_dict is the loaded DataParallel model_state + """ + if not next(iter(state_dict)).startswith('module.'): + return state_dict # abort if dict is not a DataParallel model_state + new_state_dict = OrderedDict() + + split_index = 0 + for cur_key, cur_value in state_dict.items(): + if cur_key.startswith('module.model'): + split_index = 13 + elif cur_key.startswith('module'): + split_index = 7 + + break + + for k, v in state_dict.items(): + name = k[split_index:] # remove `module.` + new_state_dict[name] = v + return new_state_dict + + def forward( + self, + sky_image: torch.Tensor, + sky_image_refine: torch.Tensor, + scene_image: torch.Tensor, + scene_image_refine: torch.Tensor, + img_metas: Dict[str, Any], + ): + """ + Args: + sky_image (`torch.Tensor`): batched image tensor, shape is [1, 3, h', w']. + sky_image_refine (`torch.Tensor`): batched image tensor, shape is [1, 3, refine_net_h, refine_net_w]. + scene_image (`torch.Tensor`): batched image tensor, shape is [1, 3, h, w]. + scene_image_refine (`torch.Tensor`): batched image tensor, shape is [1, 3, refine_net_h, refine_net_w]. + img_metas (`Dict[str, Any]`): image meta info. + Return: + `IMAGE: shape is [h, w, 3] (0~255)` + """ + start = time.time() + sky_img_metas, scene_img_metas, input_size = img_metas[ + 'sky_img_metas'], img_metas['scene_img_metas'], img_metas[ + 'input_size'] + sky_mask = self.inference_mask(sky_image_refine, sky_img_metas, + input_size) + scene_mask = self.inference_mask(scene_image_refine, scene_img_metas, + input_size) + end = time.time() + logger.info( + 'Time of inferencing mask of sky and scene images:{}'.format( + end - start)) + start = time.time() + scene_mask = scene_mask * 255 + sky_mask = sky_mask * 255 + res = blend(scene_image, scene_mask, sky_image, sky_mask) + end = time.time() + logger.info('Time of blending: {}'.format(end - start)) + return res + + @torch.no_grad() + def inference_mask(self, img, img_metas, input_size): + self.eval() + raw_h, raw_w = img_metas['ori_shape'] + pad_direction = img_metas['pad_direction'] + coarse_input_size = input_size['coarse_input_size'] + refine_input_size = input_size['refine_input_size'] + h, w = img_metas['refine_shape'] + resize_images = F.interpolate( + img, coarse_input_size, mode='bilinear', align_corners=True) + # get coarse result + pred_scores = self.coarse_model(resize_images) + if isinstance(pred_scores, (tuple, list)): + pred_scores = pred_scores[-1] + score = F.interpolate( + input=pred_scores, + size=refine_input_size, + mode='bilinear', + align_corners=True, + ) + _, coarse_pred = torch.max(score, dim=1) # [B, h, w] + coarse_pred = coarse_pred.unsqueeze(1).type(img.dtype) + img = torch.cat([img, coarse_pred], dim=1) # [B, c=4, h, w] + del resize_images + del pred_scores + del score + del coarse_pred + torch.cuda.empty_cache() + cur_scores = self.refine_model(img) + del img + torch.cuda.empty_cache() + cur_scores = torch.clip(cur_scores, 0, 1) + cur_scores = cur_scores.detach().cpu().numpy()[0] + + # resize if cur_scores shape are not compatible with origin image shape + ph, pw = cur_scores.shape + if ph != h or pw != w: + cur_scores = F.interpolate( + input=cur_scores, + size=(h, w), + mode='nearest', + align_corners=True) + # unpad to get valid area and resize to origin size + valid_cur_pred = cur_scores[pad_direction[1]:refine_input_size[0] + - pad_direction[3], + pad_direction[0]:refine_input_size[1] + - pad_direction[2], ] + valid_cur_pred = cv2.resize(valid_cur_pred, (raw_w, raw_h)) + del cur_scores + torch.cuda.empty_cache() + print('get refine mask done') + return valid_cur_pred diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/__init__.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1448c3485d173303637e29ccbf89b664371577cf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .unet_2d_condition import UNet2DConditionModel + from .controlnet import ControlNetModel + +else: + _import_structure = { + 'unet_2d_condition': ['UNet2DConditionModel'], + 'controlnet': ['ControlNetModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/attention.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..825a98a9c8f7eeae74d5780fd8fdbc6c3d558e04 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/attention.py @@ -0,0 +1,421 @@ +# Part of the implementation is borrowed and modified from diffusers, +# publicly available at https://github.com/huggingface/diffusers/tree/main/src/diffusers/models/attention.py +import math +from typing import Callable, Optional + +import torch +import torch.nn.functional as F +from diffusers.models.attention_processor import Attention +from diffusers.models.embeddings import CombinedTimestepLabelEmbeddings +from diffusers.utils.torch_utils import maybe_allow_in_graph +from torch import nn + + +@maybe_allow_in_graph +class BasicTransformerBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + pixelwise_cross_attention_dim: Optional[int] = None, + activation_fn: str = 'geglu', + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = 'layer_norm', + final_dropout: bool = False, + use_pixelwise_attention: bool = False, + ): + super().__init__() + self.only_cross_attention = only_cross_attention + + self.use_ada_layer_norm_zero = ( + num_embeds_ada_norm is not None) and norm_type == 'ada_norm_zero' + self.use_ada_layer_norm = (num_embeds_ada_norm + is not None) and norm_type == 'ada_norm' + + if norm_type in ('ada_norm', + 'ada_norm_zero') and num_embeds_ada_norm is None: + raise ValueError( + f'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to' + f' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.' + ) + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + if self.use_ada_layer_norm: + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif self.use_ada_layer_norm_zero: + self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm) + else: + self.norm1 = nn.LayerNorm( + dim, elementwise_affine=norm_elementwise_affine) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim + if only_cross_attention else None, + upcast_attention=upcast_attention, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + self.norm2 = ( + AdaLayerNorm(dim, num_embeds_ada_norm) + if self.use_ada_layer_norm else nn.LayerNorm( + dim, elementwise_affine=norm_elementwise_affine)) + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim + if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 2+. pixelwise-Attn + if use_pixelwise_attention: + self.norm2_plus = ( + AdaLayerNorm(dim, num_embeds_ada_norm) + if self.use_ada_layer_norm else nn.LayerNorm( + dim, elementwise_affine=norm_elementwise_affine)) + self.attn2_plus = Attention( + query_dim=dim, + cross_attention_dim=pixelwise_cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2_plus = None + self.attn2_plus = None + + # 3. Feed-forward + self.norm3 = nn.LayerNorm( + dim, elementwise_affine=norm_elementwise_affine) + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + encoder_pixelwise_hidden_states=None, + encoder_pixelwise_attention_mask=None, + timestep=None, + cross_attention_kwargs=None, + class_labels=None, + ): + # Notice that normalization is always applied before the real computation in the following blocks. + # 1. Self-Attention + if self.use_ada_layer_norm: + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.use_ada_layer_norm_zero: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, + timestep, + class_labels, + hidden_dtype=hidden_states.dtype) + else: + norm_hidden_states = self.norm1(hidden_states) + + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states + if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + if self.use_ada_layer_norm_zero: + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = attn_output + hidden_states + + # 2. Cross-Attention + if self.attn2 is not None: + norm_hidden_states = ( + self.norm2(hidden_states, timestep) + if self.use_ada_layer_norm else self.norm2(hidden_states)) + # TODO (Birch-San): Here we should prepare the encoder_attention mask correctly + # prepare attention mask here + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 2+. pixelwise-Attention + if self.attn2_plus is not None: + norm_hidden_states = ( + self.norm2_plus(hidden_states, timestep) + if self.use_ada_layer_norm else self.norm2_plus(hidden_states)) + + attn_output = self.attn2_plus( + norm_hidden_states, + encoder_hidden_states=encoder_pixelwise_hidden_states, + attention_mask=encoder_pixelwise_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 3. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + + if self.use_ada_layer_norm_zero: + norm_hidden_states = norm_hidden_states * ( + 1 + scale_mlp[:, None]) + shift_mlp[:, None] + + ff_output = self.ff(norm_hidden_states) + + if self.use_ada_layer_norm_zero: + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = ff_output + hidden_states + + return hidden_states + + +class FeedForward(nn.Module): + r""" + A feed-forward layer. + + Parameters: + dim (`int`): The number of channels in the input. + dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. + mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + final_dropout (`bool` *optional*, defaults to False): Apply a final dropout. + """ + + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: int = 4, + dropout: float = 0.0, + activation_fn: str = 'geglu', + final_dropout: bool = False, + ): + super().__init__() + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + + if activation_fn == 'gelu': + act_fn = GELU(dim, inner_dim) + if activation_fn == 'gelu-approximate': + act_fn = GELU(dim, inner_dim, approximate='tanh') + elif activation_fn == 'geglu': + act_fn = GEGLU(dim, inner_dim) + elif activation_fn == 'geglu-approximate': + act_fn = ApproximateGELU(dim, inner_dim) + + self.net = nn.ModuleList([]) + # project in + self.net.append(act_fn) + # project dropout + self.net.append(nn.Dropout(dropout)) + # project out + self.net.append(nn.Linear(inner_dim, dim_out)) + # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout + if final_dropout: + self.net.append(nn.Dropout(dropout)) + + def forward(self, hidden_states): + for module in self.net: + hidden_states = module(hidden_states) + return hidden_states + + +class GELU(nn.Module): + r""" + GELU activation function with tanh approximation support with `approximate="tanh"`. + """ + + def __init__(self, dim_in: int, dim_out: int, approximate: str = 'none'): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out) + self.approximate = approximate + + def gelu(self, gate): + if gate.device.type != 'mps': + return F.gelu(gate, approximate=self.approximate) + # mps: gelu is not implemented for float16 + return F.gelu( + gate.to(dtype=torch.float32), + approximate=self.approximate).to(dtype=gate.dtype) + + def forward(self, hidden_states): + hidden_states = self.proj(hidden_states) + hidden_states = self.gelu(hidden_states) + return hidden_states + + +class GEGLU(nn.Module): + r""" + A variant of the gated linear unit activation function from https://arxiv.org/abs/2002.05202. + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + """ + + def __init__(self, dim_in: int, dim_out: int): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def gelu(self, gate): + if gate.device.type != 'mps': + return F.gelu(gate) + # mps: gelu is not implemented for float16 + return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype) + + def forward(self, hidden_states): + hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1) + return hidden_states * self.gelu(gate) + + +class ApproximateGELU(nn.Module): + """ + The approximate form of Gaussian Error Linear Unit (GELU) + + For more details, see section 2: https://arxiv.org/abs/1606.08415 + """ + + def __init__(self, dim_in: int, dim_out: int): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out) + + def forward(self, x): + x = self.proj(x) + return x * torch.sigmoid(1.702 * x) + + +class AdaLayerNorm(nn.Module): + """ + Norm layer modified to incorporate timestep embeddings. + """ + + def __init__(self, embedding_dim, num_embeddings): + super().__init__() + self.emb = nn.Embedding(num_embeddings, embedding_dim) + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, embedding_dim * 2) + self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False) + + def forward(self, x, timestep): + emb = self.linear(self.silu(self.emb(timestep))) + scale, shift = torch.chunk(emb, 2) + x = self.norm(x) * (1 + scale) + shift + return x + + +class AdaLayerNormZero(nn.Module): + """ + Norm layer adaptive layer norm zero (adaLN-Zero). + """ + + def __init__(self, embedding_dim, num_embeddings): + super().__init__() + + self.emb = CombinedTimestepLabelEmbeddings(num_embeddings, + embedding_dim) + + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=True) + self.norm = nn.LayerNorm( + embedding_dim, elementwise_affine=False, eps=1e-6) + + def forward(self, x, timestep, class_labels, hidden_dtype=None): + emb = self.linear( + self.silu( + self.emb(timestep, class_labels, hidden_dtype=hidden_dtype))) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk( + 6, dim=1) + x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] + return x, gate_msa, shift_mlp, scale_mlp, gate_mlp + + +class AdaGroupNorm(nn.Module): + """ + GroupNorm layer modified to incorporate timestep embeddings. + """ + + def __init__(self, + embedding_dim: int, + out_dim: int, + num_groups: int, + act_fn: Optional[str] = None, + eps: float = 1e-5): + super().__init__() + self.num_groups = num_groups + self.eps = eps + self.act = None + if act_fn == 'swish': + self.act = lambda x: F.silu(x) + elif act_fn == 'mish': + self.act = nn.Mish() + elif act_fn == 'silu': + self.act = nn.SiLU() + elif act_fn == 'gelu': + self.act = nn.GELU() + + self.linear = nn.Linear(embedding_dim, out_dim * 2) + + def forward(self, x, emb): + if self.act: + emb = self.act(emb) + emb = self.linear(emb) + emb = emb[:, :, None, None] + scale, shift = emb.chunk(2, dim=1) + + x = F.group_norm(x, self.num_groups, eps=self.eps) + x = x * (1 + scale) + shift + return x diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/controlnet.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f2224a1b806f3b8f70684e9fa18b7a3dfbcb8c38 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/controlnet.py @@ -0,0 +1,738 @@ +# Part of the implementation is borrowed and modified from diffusers, +# publicly available at https://github.com/huggingface/diffusers/tree/main/src/diffusers/models/controlnet.py +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin # , ControlNetModel +from diffusers.models.attention_processor import (AttentionProcessor, + AttnProcessor) +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.utils import BaseOutput, logging +from torch import nn +from torch.nn import functional as F +from torchvision import utils + +from modelscope.models.cv.super_resolution.rrdbnet_arch import RRDB +from .unet_2d_blocks import (CrossAttnDownBlock2D, DownBlock2D, + UNetMidBlock2DCrossAttn, get_down_block) +from .unet_2d_condition import UNet2DConditionModel + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class ControlNetOutput(BaseOutput): + controlnet_cond_mid: torch.Tensor + down_block_res_samples: Tuple[torch.Tensor] + mid_block_res_sample: torch.Tensor + + +class ControlNetConditioningEmbedding(nn.Module): + """ + Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN + [11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized + training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the + convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides + (activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full + model) to encode image-space conditions ... into feature maps ..." + """ + + def __init__( + self, + conditioning_embedding_channels: int, + conditioning_channels: int = 3, + block_out_channels: Tuple[int] = (16, 32, 96, 256), + return_rgbs: bool = True, + use_rrdb: bool = False, + ): + super().__init__() + + self.return_rgbs = return_rgbs + self.use_rrdb = use_rrdb + + self.conv_in = nn.Conv2d( + conditioning_channels, + block_out_channels[0], + kernel_size=3, + padding=1) + + if self.use_rrdb: + num_rrdb_block = 2 + layers = ( + RRDB(block_out_channels[0], block_out_channels[0]) + for i in range(num_rrdb_block)) + self.preprocesser = nn.Sequential(*layers) + + self.blocks = nn.ModuleList([]) + if return_rgbs: + self.to_rgbs = nn.ModuleList([]) + + for i in range(len(block_out_channels) - 1): + channel_in = block_out_channels[i] + channel_out = block_out_channels[i + 1] + self.blocks.append( + nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) + self.blocks.append( + nn.Conv2d( + channel_in, + channel_out, + kernel_size=3, + padding=1, + stride=2)) + + if return_rgbs: + self.to_rgbs.append( + nn.Conv2d(channel_out, 3, kernel_size=3, padding=1)) + + self.conv_out = zero_module( + nn.Conv2d( + block_out_channels[-1], + conditioning_embedding_channels, + kernel_size=3, + padding=1)) + + def forward(self, conditioning): + embedding = self.conv_in(conditioning) + embedding = F.silu(embedding) + + if self.use_rrdb: + embedding = self.preprocesser(embedding) + + out_rgbs = [] + for i, block in enumerate(self.blocks): + embedding = block(embedding) + embedding = F.silu(embedding) + + if i % 2 and self.return_rgbs: # 0 + out_rgbs.append(self.to_rgbs[i // 2](embedding)) + + embedding = self.conv_out(embedding) + + if self.return_rgbs: + return embedding, out_rgbs + else: + return embedding + + +class ControlNetModel(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 4, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'DownBlock2D', + ), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = 'silu', + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: int = 1280, + attention_head_dim: Union[int, Tuple[int]] = 8, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = 'default', + projection_class_embeddings_input_dim: Optional[int] = None, + controlnet_conditioning_channel_order: str = 'rgb', + conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, + 32, + 96, + 256), + global_pool_conditions: bool = False, + return_rgbs: bool = False, + use_rrdb: bool = False): + super().__init__() + + # Check inputs + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `block_out_channels` as `down_block_types`. \ + `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + only_cross_attention, + bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `only_cross_attention` as `down_block_types`. \ + `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + attention_head_dim, + int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `attention_head_dim` as `down_block_types`. \ + `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}.' + ) + + # input + self.return_rgbs = return_rgbs + conv_in_kernel = 3 + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=conv_in_kernel, + padding=conv_in_padding) + + # time + time_embed_dim = block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, + freq_shift) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + ) + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, + time_embed_dim) + elif class_embed_type == 'timestep': + self.class_embedding = TimestepEmbedding(timestep_input_dim, + time_embed_dim) + elif class_embed_type == 'identity': + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == 'projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding( + projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + # control net conditioning embedding + self.controlnet_cond_embedding = ControlNetConditioningEmbedding( + conditioning_embedding_channels=block_out_channels[0], + block_out_channels=conditioning_embedding_out_channels, + return_rgbs=return_rgbs, + use_rrdb=use_rrdb, + ) + + self.down_blocks = nn.ModuleList([]) + self.controlnet_down_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + only_cross_attention = [only_cross_attention + ] * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim, ) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attention_head_dim[i], + downsample_padding=downsample_padding, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + self.down_blocks.append(down_block) + + for _ in range(layers_per_block): + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + if not is_final_block: + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + # mid + mid_block_channel = block_out_channels[-1] + + controlnet_block = nn.Conv2d( + mid_block_channel, mid_block_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_mid_block = controlnet_block + + self.mid_block = UNetMidBlock2DCrossAttn( + in_channels=mid_block_channel, + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attention_head_dim[-1], + resnet_groups=norm_num_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + + @classmethod + def from_unet( + cls, + unet: UNet2DConditionModel, + controlnet_conditioning_channel_order: str = 'rgb', + conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, + 96, 256), + load_weights_from_unet: bool = True, + ): + r""" + Instantiate Controlnet class from UNet2DConditionModel. + + Parameters: + unet (`UNet2DConditionModel`): + UNet model which weights are copied to the ControlNet. Note that all configuration options are also + copied where applicable. + """ + controlnet = cls( + in_channels=unet.config.in_channels, + flip_sin_to_cos=unet.config.flip_sin_to_cos, + freq_shift=unet.config.freq_shift, + down_block_types=unet.config.down_block_types, + only_cross_attention=unet.config.only_cross_attention, + block_out_channels=unet.config.block_out_channels, + layers_per_block=unet.config.layers_per_block, + downsample_padding=unet.config.downsample_padding, + mid_block_scale_factor=unet.config.mid_block_scale_factor, + act_fn=unet.config.act_fn, + norm_num_groups=unet.config.norm_num_groups, + norm_eps=unet.config.norm_eps, + cross_attention_dim=unet.config.cross_attention_dim, + attention_head_dim=unet.config.attention_head_dim, + use_linear_projection=unet.config.use_linear_projection, + class_embed_type=unet.config.class_embed_type, + num_class_embeds=unet.config.num_class_embeds, + upcast_attention=unet.config.upcast_attention, + resnet_time_scale_shift=unet.config.resnet_time_scale_shift, + projection_class_embeddings_input_dim=unet.config. + projection_class_embeddings_input_dim, + controlnet_conditioning_channel_order= + controlnet_conditioning_channel_order, + conditioning_embedding_out_channels= + conditioning_embedding_out_channels, + ) + + if load_weights_from_unet: + controlnet.conv_in.load_state_dict(unet.conv_in.state_dict()) + controlnet.time_proj.load_state_dict(unet.time_proj.state_dict()) + controlnet.time_embedding.load_state_dict( + unet.time_embedding.state_dict()) + + if controlnet.class_embedding: + controlnet.class_embedding.load_state_dict( + unet.class_embedding.state_dict()) + + controlnet.down_blocks.load_state_dict( + unet.down_blocks.state_dict()) + controlnet.mid_block.load_state_dict(unet.mid_block.state_dict()) + + if controlnet.sr_model is not None: + load_net = torch.load( + 'annotator/ckpts/RealESRNet_x4plus.pth', + map_location=lambda storage, loc: storage) + if 'params_ema' in load_net: + load_net = load_net['params_ema'] + elif 'params' in load_net: + load_net = load_net['params'] + # remove unnecessary 'module.' + for k, v in deepcopy(load_net).items(): + if k.startswith('module.'): + load_net[k[7:]] = v + load_net.pop(k) + controlnet.sr_model.load_state_dict(load_net, strict=True) + + return controlnet + + @property + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, + processors: Dict[str, + AttentionProcessor]): + if hasattr(module, 'set_processor'): + processors[f'{name}.processor'] = module.processor + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f'{name}.{sub_name}', child, + processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, + Dict[str, + AttentionProcessor]]): + r""" + Parameters: + `processor (`dict` of `AttentionProcessor` or `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + of **all** `Attention` layers. + In case `processor` is a dict, the key needs to define the path to + the corresponding cross attention processor. + This is strongly recommended when setting trainable attention processors.: + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f'A dict of processors was passed, but the number of processors {len(processor)} does not match the' + f' number of attention layers: {count}. Please make sure to pass {count} processor classes.' + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, + processor): + if hasattr(module, 'set_processor'): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f'{name}.processor')) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f'{name}.{sub_name}', child, + processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + self.set_attn_processor(AttnProcessor()) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attention_slice + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module will split the input tensor in slices, to compute attention + in several steps. This is useful to save some memory in exchange for a small speed decrease. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If + `"max"`, maximum amount of memory will be saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, 'set_attention_slice'): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == 'auto': + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == 'max': + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance( + slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f'You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different' + f' attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}.' + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError( + f'size {size} has to be smaller or equal to {dim}.') + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, + slice_size: List[int]): + if hasattr(module, 'set_attention_slice'): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (CrossAttnDownBlock2D, DownBlock2D)): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + controlnet_cond: torch.FloatTensor, + fg_mask: Optional[torch.FloatTensor] = None, + conditioning_scale_fg: float = 1.0, + conditioning_scale_bg: float = 1.0, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guess_mode: bool = False, + return_dict: bool = True, + ) -> Union[ControlNetOutput, Tuple]: + # check channel order + channel_order = self.config.controlnet_conditioning_channel_order + + if channel_order == 'rgb': + # in rgb order by default + ... + elif channel_order == 'bgr': + controlnet_cond = torch.flip(controlnet_cond, dims=[1]) + else: + raise ValueError( + f'unknown `controlnet_conditioning_channel_order`: {channel_order}' + ) + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == 'mps' + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], + dtype=dtype, + device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError( + 'class_labels should be provided when num_class_embeds > 0' + ) + + if self.config.class_embed_type == 'timestep': + class_labels = self.time_proj(class_labels) + + class_emb = self.class_embedding(class_labels).to(dtype=self.dtype) + emb = emb + class_emb + + # 2. pre-process + sample = self.conv_in(sample) + + controlnet_cond_mid = None + if self.return_rgbs: + controlnet_cond, controlnet_cond_mid = self.controlnet_cond_embedding( + controlnet_cond) + else: + controlnet_cond = self.controlnet_cond_embedding(controlnet_cond) + + sample = sample + controlnet_cond + + # 3. down + down_block_res_samples = (sample, ) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, 'has_cross_attention' + ) and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block( + hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + # 5. Control net blocks + + controlnet_down_block_res_samples = () + + for down_block_res_sample, controlnet_block in zip( + down_block_res_samples, self.controlnet_down_blocks): + down_block_res_sample = controlnet_block(down_block_res_sample) + controlnet_down_block_res_samples = controlnet_down_block_res_samples + ( + down_block_res_sample, ) + + down_block_res_samples = controlnet_down_block_res_samples + + mid_block_res_sample = self.controlnet_mid_block(sample) + + # 6. scaling + if guess_mode and not self.config.global_pool_conditions: + scales = torch.logspace( + -1, 0, len(down_block_res_samples) + 1, + device=sample.device) # 0.1 to 1.0 + + scales = scales * conditioning_scale_fg + down_block_res_samples = [ + sample * scale + for sample, scale in zip(down_block_res_samples, scales) + ] + mid_block_res_sample = mid_block_res_sample * scales[ + -1] # last one + else: + if fg_mask is None: + down_block_res_samples = [ + sample * conditioning_scale_fg + for sample in down_block_res_samples + ] + mid_block_res_sample = mid_block_res_sample * conditioning_scale_fg + else: + down_block_masks = [ + torch.zeros_like(sample) + conditioning_scale_bg + for i, sample in enumerate(down_block_res_samples) + ] + mid_block_mask = torch.zeros_like( + mid_block_res_sample) + conditioning_scale_bg + + for i, sample in enumerate(down_block_masks): + tmp_mask = F.interpolate( + fg_mask, + size=sample.shape[-2:]).repeat(sample.shape[0], + sample.shape[1], 1, + 1).bool() + down_block_masks[i] = sample.masked_fill( + tmp_mask, conditioning_scale_fg) + + tmp_mask = F.interpolate( + fg_mask, size=mid_block_mask.shape[-2:]).repeat( + mid_block_mask.shape[0], mid_block_mask.shape[1], 1, + 1).bool() + mid_block_mask = mid_block_mask.masked_fill( + tmp_mask, conditioning_scale_fg) + + down_block_res_samples = [ + sample * down_block_mask for sample, down_block_mask in + zip(down_block_res_samples, down_block_masks) + ] + mid_block_res_sample = mid_block_res_sample * mid_block_mask + + if self.config.global_pool_conditions: + down_block_res_samples = [ + torch.mean(sample, dim=(2, 3), keepdim=True) + for sample in down_block_res_samples + ] + mid_block_res_sample = torch.mean( + mid_block_res_sample, dim=(2, 3), keepdim=True) + + if not return_dict: + return (controlnet_cond_mid, down_block_res_samples, + mid_block_res_sample) + + return ControlNetOutput( + controlnet_cond_mid=controlnet_cond_mid, + down_block_res_samples=down_block_res_samples, + mid_block_res_sample=mid_block_res_sample) + + +def zero_module(module): + for p in module.parameters(): + nn.init.zeros_(p) + return module diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/misc.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..048690645d6bca11701c9fbc4c991da8e561e1fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/misc.py @@ -0,0 +1,159 @@ +# Part of the implementation is borrowed and modified from StableSR, +# publicly available at https://github.com/IceClear/StableSR/blob/main/scripts/wavelet_color_fix.py +import torch +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import ( + convert_ldm_unet_checkpoint, convert_ldm_vae_checkpoint) +from PIL import Image +from safetensors import safe_open +from torch import Tensor +from torch.nn import functional as F +from torchvision.transforms import ToPILImage, ToTensor + + +def adain_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply adaptive instance normalization + result_tensor = adaptive_instance_normalization(target_tensor, + source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + + +def wavelet_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply wavelet reconstruction + result_tensor = wavelet_reconstruction(target_tensor, source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + + +def calc_mean_std(feat: Tensor, eps=1e-5): + """Calculate mean and std for adaptive_instance_normalization. + Args: + feat (Tensor): 4D tensor. + eps (float): A small value added to the variance to avoid + divide-by-zero. Default: 1e-5. + """ + size = feat.size() + assert len(size) == 4, 'The input feature should be 4D tensor.' + b, c = size[:2] + feat_var = feat.reshape(b, c, -1).var(dim=2) + eps + feat_std = feat_var.sqrt().reshape(b, c, 1, 1) + feat_mean = feat.reshape(b, c, -1).mean(dim=2).reshape(b, c, 1, 1) + return feat_mean, feat_std + + +def adaptive_instance_normalization(content_feat: Tensor, style_feat: Tensor): + """Adaptive instance normalization. + Adjust the reference features to have the similar color and illuminations + as those in the degradate features. + Args: + content_feat (Tensor): The reference feature. + style_feat (Tensor): The degradate features. + """ + size = content_feat.size() + style_mean, style_std = calc_mean_std(style_feat) + content_mean, content_std = calc_mean_std(content_feat) + normalized_feat = (content_feat + - content_mean.expand(size)) / content_std.expand(size) + return normalized_feat * style_std.expand(size) + style_mean.expand(size) + + +def wavelet_blur(image: Tensor, radius: int): + """ + Apply wavelet blur to the input tensor. + """ + # input shape: (1, 3, H, W) + # convolution kernel + kernel_vals = [ + [0.0625, 0.125, 0.0625], + [0.125, 0.25, 0.125], + [0.0625, 0.125, 0.0625], + ] + kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device) + # add channel dimensions to the kernel to make it a 4D tensor + kernel = kernel[None, None] + # repeat the kernel across all input channels + kernel = kernel.repeat(3, 1, 1, 1) + image = F.pad(image, (radius, radius, radius, radius), mode='replicate') + # apply convolution + output = F.conv2d(image, kernel, groups=3, dilation=radius) + return output + + +def wavelet_decomposition(image: Tensor, levels=5): + """ + Apply wavelet decomposition to the input tensor. + This function only returns the low frequency & the high frequency. + """ + high_freq = torch.zeros_like(image) + for i in range(levels): + radius = 2**i + low_freq = wavelet_blur(image, radius) + high_freq += (image - low_freq) + image = low_freq + + return high_freq, low_freq + + +def wavelet_reconstruction(content_feat: Tensor, style_feat: Tensor): + """ + Apply wavelet decomposition, so that the content will have the same color as the style. + """ + # calculate the wavelet decomposition of the content feature + content_high_freq, content_low_freq = wavelet_decomposition(content_feat) + del content_low_freq + # calculate the wavelet decomposition of the style feature + style_high_freq, style_low_freq = wavelet_decomposition(style_feat) + del style_high_freq + # reconstruct the content feature with the style's high frequency + return content_high_freq + style_low_freq + + +def load_dreambooth_lora(unet, vae=None, model_path=None, model_base=''): + if model_path is None: + return unet + + if model_path.endswith('.ckpt'): + base_state_dict = torch.load(model_path)['state_dict'] + elif model_path.endswith('.safetensors'): + state_dict = {} + with safe_open(model_path, framework='pt', device='cpu') as f: + for key in f.keys(): + state_dict[key] = f.get_tensor(key) + + is_lora = all('lora' in k for k in state_dict.keys()) + if not is_lora: + base_state_dict = state_dict + else: + base_state_dict = {} + with safe_open(model_base, framework='pt', device='cpu') as f: + for key in f.keys(): + base_state_dict[key] = f.get_tensor(key) + + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + base_state_dict, unet.config) + unet.load_state_dict(converted_unet_checkpoint, strict=False) + + if vae is not None: + converted_vae_checkpoint = convert_ldm_vae_checkpoint( + base_state_dict, vae.config) + vae.load_state_dict(converted_vae_checkpoint) + + return unet, vae diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/transformer_2d.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/transformer_2d.py new file mode 100644 index 0000000000000000000000000000000000000000..ad22d21276c71cbdc5dc6c1195c1fca7f79d50ad --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/transformer_2d.py @@ -0,0 +1,365 @@ +# Part of the implementation is borrowed and modified from diffusers, +# publicly available at https://github.com/huggingface/diffusers/tree/main/src/diffusers/models/transformer_2d.py +from dataclasses import dataclass +from typing import Optional + +import torch +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from diffusers.models.embeddings import ImagePositionalEmbeddings, PatchEmbed +from diffusers.utils import BaseOutput, deprecate +from torch import nn + +from .attention import BasicTransformerBlock + + +@dataclass +class Transformer2DModelOutput(BaseOutput): + """ + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or + `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete): + Hidden states conditioned on `encoder_hidden_states` input. If discrete, returns probability distributions + for the unnoised latent pixels. + """ + + sample: torch.FloatTensor + + +class Transformer2DModel(ModelMixin, ConfigMixin): + """ + Transformer model for image-like data. Takes either discrete (classes of vector embeddings) or continuous (actual + embeddings) inputs. + + When input is continuous: First, project the input (aka embedding) and reshape to b, t, d. Then apply standard + transformer action. Finally, reshape to image. + + When input is discrete: First, input (classes of latent pixels) is converted to embeddings and has positional + embeddings applied, see `ImagePositionalEmbeddings`. Then apply standard transformer action. Finally, predict + classes of unnoised image. + + Note that it is assumed one of the input classes is the masked latent pixel. The predicted classes of the unnoised + image do not contain a prediction for the masked pixel as the unnoised image cannot be masked. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + Pass if the input is continuous. The number of channels in the input and output. + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The number of encoder_hidden_states dimensions to use. + sample_size (`int`, *optional*): Pass if the input is discrete. The width of the latent images. + Note that this is fixed at training time as it is used for learning a number of position embeddings. See + `ImagePositionalEmbeddings`. + num_vector_embeds (`int`, *optional*): + Pass if the input is discrete. The number of classes of the vector embeddings of the latent pixels. + Includes the class for the masked latent pixel. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): Pass if at least one of the norm_layers is `AdaLayerNorm`. + The number of diffusion steps used during training. Note that this is fixed at training time as it is used + to learn a number of embeddings that are added to the hidden states. During inference, you can denoise for + up to but not more than steps than `num_embeds_ada_norm`. + attention_bias (`bool`, *optional*): + Configure if the TransformerBlocks' attention should contain a bias parameter. + """ + + @register_to_config + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + pixelwise_cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + patch_size: Optional[int] = None, + activation_fn: str = 'geglu', + num_embeds_ada_norm: Optional[int] = None, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + norm_type: str = 'layer_norm', + norm_elementwise_affine: bool = True, + use_pixelwise_attention: bool = False, + ): + super().__init__() + self.use_linear_projection = use_linear_projection + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + + # 1. Transformer2DModel can process both standard continuous images of shape + # `(batch_size, num_channels, width, height)` + # as well as quantized image embeddings of shape `(batch_size, num_image_vectors)` + # Define whether input is continuous or discrete depending on configuration + self.is_input_continuous = (in_channels + is not None) and (patch_size is None) + self.is_input_vectorized = num_vector_embeds is not None + self.is_input_patches = in_channels is not None and patch_size is not None + + if norm_type == 'layer_norm' and num_embeds_ada_norm is not None: + deprecation_message = ( + f'The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or' + " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config." + ' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect' + ' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it' + ' would be very nice if you could open a Pull request for the `transformer/config.json` file' + ) + deprecate( + 'norm_type!=num_embeds_ada_norm', + '1.0.0', + deprecation_message, + standard_warn=False) + norm_type = 'ada_norm' + + if self.is_input_continuous and self.is_input_vectorized: + raise ValueError( + f'Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make' + ' sure that either `in_channels` or `num_vector_embeds` is None.' + ) + elif self.is_input_vectorized and self.is_input_patches: + raise ValueError( + f'Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make' + ' sure that either `num_vector_embeds` or `num_patches` is None.' + ) + elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches: + raise ValueError( + f'Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:' + f' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.' + ) + + # 2. Define input layers + if self.is_input_continuous: + self.in_channels = in_channels + + self.norm = torch.nn.GroupNorm( + num_groups=norm_num_groups, + num_channels=in_channels, + eps=1e-6, + affine=True) + if use_linear_projection: + self.proj_in = nn.Linear(in_channels, inner_dim) + if use_pixelwise_attention: + self.proj_in_plus = nn.Linear( + pixelwise_cross_attention_dim, + pixelwise_cross_attention_dim) + else: + self.proj_in_plus = None + else: + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + if use_pixelwise_attention: + self.proj_in_plus = nn.Conv2d( + pixelwise_cross_attention_dim, + pixelwise_cross_attention_dim, + kernel_size=1, + stride=1, + padding=0) + else: + self.proj_in_plus = None + + elif self.is_input_vectorized: + assert sample_size is not None, 'Transformer2DModel over discrete input must provide sample_size' + assert num_vector_embeds is not None, 'Transformer2DModel over discrete input must provide num_embed' + + self.height = sample_size + self.width = sample_size + self.num_vector_embeds = num_vector_embeds + self.num_latent_pixels = self.height * self.width + + self.latent_image_embedding = ImagePositionalEmbeddings( + num_embed=num_vector_embeds, + embed_dim=inner_dim, + height=self.height, + width=self.width) + elif self.is_input_patches: + assert sample_size is not None, 'Transformer2DModel over patched input must provide sample_size' + + self.height = sample_size + self.width = sample_size + + self.patch_size = patch_size + self.pos_embed = PatchEmbed( + height=sample_size, + width=sample_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=inner_dim, + ) + + # 3. Define transformers blocks + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + pixelwise_cross_attention_dim=pixelwise_cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + norm_elementwise_affine=norm_elementwise_affine, + use_pixelwise_attention=use_pixelwise_attention, + ) for d in range(num_layers) + ]) + + # 4. Define output layers + self.out_channels = in_channels if out_channels is None else out_channels + if self.is_input_continuous: + # TODO: should use out_channels for continuous projections + if use_linear_projection: + self.proj_out = nn.Linear(inner_dim, in_channels) + else: + self.proj_out = nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, padding=0) + elif self.is_input_vectorized: + self.norm_out = nn.LayerNorm(inner_dim) + self.out = nn.Linear(inner_dim, self.num_vector_embeds - 1) + elif self.is_input_patches: + self.norm_out = nn.LayerNorm( + inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out_1 = nn.Linear(inner_dim, 2 * inner_dim) + self.proj_out_2 = nn.Linear( + inner_dim, patch_size * patch_size * self.out_channels) + + def forward( + self, + hidden_states, + encoder_hidden_states=None, + encoder_pixelwise_hidden_states=None, + timestep=None, + class_labels=None, + cross_attention_kwargs=None, + return_dict: bool = True, + ): + """ + Args: + hidden_states ( When discrete, `torch.LongTensor` of shape `(batch size, num latent pixels)`. + When continuous, `torch.FloatTensor` of shape `(batch size, channel, height, width)`): Input + hidden_states + encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.long`, *optional*): + Optional timestep to be applied as an embedding in AdaLayerNorm's. Used to indicate denoising step. + class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): + Optional class labels to be applied as an embedding in AdaLayerZeroNorm. Used to indicate class labels + conditioning. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple. + + Returns: + [`~models.transformer_2d.Transformer2DModelOutput`] or `tuple`: + [`~models.transformer_2d.Transformer2DModelOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + """ + # 1. Input + if self.is_input_continuous: + batch, _, height, width = hidden_states.shape + residual = hidden_states + + hidden_states = self.norm(hidden_states) + if not self.use_linear_projection: + hidden_states = self.proj_in(hidden_states) + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape( + batch, height * width, inner_dim) + + if self.proj_in_plus is not None: + encoder_pixelwise_hidden_states = self.proj_in_plus( + encoder_pixelwise_hidden_states) + pixelwise_inner_dim = encoder_pixelwise_hidden_states.shape[ + 1] + encoder_pixelwise_hidden_states = encoder_pixelwise_hidden_states.permute( + 0, 2, 3, 1).reshape(batch, height * width, + pixelwise_inner_dim) + else: + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape( + batch, height * width, inner_dim) + hidden_states = self.proj_in(hidden_states) + if self.proj_in_plus is not None: + pixelwise_inner_dim = encoder_pixelwise_hidden_states.shape[ + 1] + encoder_pixelwise_hidden_states = encoder_pixelwise_hidden_states.permute( + 0, 2, 3, 1).reshape(batch, height * width, + pixelwise_inner_dim) + encoder_pixelwise_hidden_states = self.proj_in_plus( + encoder_pixelwise_hidden_states) + + elif self.is_input_vectorized: + hidden_states = self.latent_image_embedding(hidden_states) + elif self.is_input_patches: + hidden_states = self.pos_embed(hidden_states) + + # 2. Blocks + for block in self.transformer_blocks: + hidden_states = block( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_pixelwise_hidden_states=encoder_pixelwise_hidden_states, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + + # 3. Output + if self.is_input_continuous: + if not self.use_linear_projection: + hidden_states = hidden_states.reshape(batch, height, width, + inner_dim).permute( + 0, 3, 1, + 2).contiguous() + hidden_states = self.proj_out(hidden_states) + else: + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.reshape(batch, height, width, + inner_dim).permute( + 0, 3, 1, + 2).contiguous() + + output = hidden_states + residual + elif self.is_input_vectorized: + hidden_states = self.norm_out(hidden_states) + logits = self.out(hidden_states) + # (batch, self.num_vector_embeds - 1, self.num_latent_pixels) + logits = logits.permute(0, 2, 1) + + # log(p(x_0)) + output = F.log_softmax(logits.double(), dim=1).float() + elif self.is_input_patches: + # TODO: cleanup! + conditioning = self.transformer_blocks[0].norm1.emb( + timestep, class_labels, hidden_dtype=hidden_states.dtype) + shift, scale = self.proj_out_1(F.silu(conditioning)).chunk( + 2, dim=1) + hidden_states = self.norm_out(hidden_states) * ( + 1 + scale[:, None]) + shift[:, None] + hidden_states = self.proj_out_2(hidden_states) + + # unpatchify + height = width = int(hidden_states.shape[1]**0.5) + hidden_states = hidden_states.reshape( + shape=(-1, height, width, self.patch_size, self.patch_size, + self.out_channels)) + hidden_states = torch.einsum('nhwpqc->nchpwq', hidden_states) + output = hidden_states.reshape( + shape=(-1, self.out_channels, height * self.patch_size, + width * self.patch_size)) + + if not return_dict: + return (output, ) + + return Transformer2DModelOutput(sample=output) diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_blocks.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..c95639e69dc9ed7541c300c27374bca24e1c9b93 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_blocks.py @@ -0,0 +1,2983 @@ +# Part of the implementation is borrowed and modified from diffusers, +# publicly available at https://github.com/huggingface/diffusers/tree/main/src/diffusers/models/unet_2d_blocks.py +from typing import Optional + +import numpy as np +import torch +import torch.nn.functional as F +from diffusers.models.attention_processor import (Attention, + AttnAddedKVProcessor, + AttnAddedKVProcessor2_0) +from diffusers.models.dual_transformer_2d import DualTransformer2DModel +from diffusers.models.resnet import (Downsample2D, FirDownsample2D, + FirUpsample2D, KDownsample2D, KUpsample2D, + ResnetBlock2D, Upsample2D) +from torch import nn + +from .attention import AdaGroupNorm +from .transformer_2d import Transformer2DModel + + +def get_down_block( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + attn_num_head_channels, + resnet_groups=None, + cross_attention_dim=None, + downsample_padding=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift='default', + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, +): + down_block_type = down_block_type[7:] if down_block_type.startswith( + 'UNetRes') else down_block_type + if down_block_type == 'DownBlock2D': + return DownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'ResnetDownsampleBlock2D': + return ResnetDownsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif down_block_type == 'AttnDownBlock2D': + return AttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'CrossAttnDownBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for CrossAttnDownBlock2D' + ) + return CrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'SimpleCrossAttnDownBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for SimpleCrossAttnDownBlock2D' + ) + return SimpleCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif down_block_type == 'SkipDownBlock2D': + return SkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'AttnSkipDownBlock2D': + return AttnSkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + downsample_padding=downsample_padding, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'DownEncoderBlock2D': + return DownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'AttnDownEncoderBlock2D': + return AttnDownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'KDownBlock2D': + return KDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif down_block_type == 'KCrossAttnDownBlock2D': + return KCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + add_self_attention=True if not add_downsample else False, + ) + raise ValueError(f'{down_block_type} does not exist.') + + +def get_up_block( + up_block_type, + num_layers, + in_channels, + out_channels, + prev_output_channel, + temb_channels, + add_upsample, + resnet_eps, + resnet_act_fn, + attn_num_head_channels, + resnet_groups=None, + cross_attention_dim=None, + pixelwise_cross_attention_dim=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift='default', + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, +): + up_block_type = up_block_type[7:] if up_block_type.startswith( + 'UNetRes') else up_block_type + if up_block_type == 'UpBlock2D': + return UpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'ResnetUpsampleBlock2D': + return ResnetUpsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif up_block_type == 'CrossAttnUpBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for CrossAttnUpBlock2D') + return CrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + pixelwise_cross_attention_dim=pixelwise_cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + use_pixelwise_attention=True, + ) + elif up_block_type == 'SimpleCrossAttnUpBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for SimpleCrossAttnUpBlock2D' + ) + return SimpleCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif up_block_type == 'AttnUpBlock2D': + return AttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'SkipUpBlock2D': + return SkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'AttnSkipUpBlock2D': + return AttnSkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'UpDecoderBlock2D': + return UpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'AttnUpDecoderBlock2D': + return AttnUpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attn_num_head_channels=attn_num_head_channels, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'KUpBlock2D': + return KUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif up_block_type == 'KCrossAttnUpBlock2D': + return KCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attn_num_head_channels=attn_num_head_channels, + ) + + raise ValueError(f'{up_block_type} does not exist.') + + +class UNetMidBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + self.add_attention = add_attention + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else in_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + else: + attentions.append(None) + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states, temb=None): + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = attn(hidden_states) + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DCrossAttn(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + cross_attention_dim=1280, + dual_cross_attention=False, + use_linear_projection=False, + upcast_attention=False, + ): + super().__init__() + + self.has_cross_attention = True + self.attn_num_head_channels = attn_num_head_channels + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for _ in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + attn_num_head_channels, + in_channels // attn_num_head_channels, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + attn_num_head_channels, + in_channels // attn_num_head_channels, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, + hidden_states, + temb=None, + encoder_hidden_states=None, + attention_mask=None, + cross_attention_kwargs=None): + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DSimpleCrossAttn(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + cross_attention_dim=1280, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + + self.has_cross_attention = True + + self.attn_num_head_channels = attn_num_head_channels + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + + self.num_heads = in_channels // self.attn_num_head_channels + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ] + attentions = [] + + for _ in range(num_layers): + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=in_channels, + cross_attention_dim=in_channels, + heads=self.num_heads, + dim_head=attn_num_head_channels, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, + hidden_states, + temb=None, + encoder_hidden_states=None, + attention_mask=None, + cross_attention_kwargs=None): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + # resnet + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class AttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + downsample_padding=1, + add_downsample=True, + ): + super().__init__() + resnets = [] + attentions = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states, temb=None, upsample_size=None): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + output_states += (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states += (hidden_states, ) + + return hidden_states, output_states + + +class CrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + downsample_padding=1, + add_downsample=True, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attn_num_head_channels = attn_num_head_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + attn_num_head_channels, + out_channels // attn_num_head_channels, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + attn_num_head_channels, + out_channels // attn_num_head_channels, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + temb=None, + encoder_hidden_states=None, + attention_mask=None, + cross_attention_kwargs=None): + # TODO(Patrick, William) - attention mask is not used + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + cross_attention_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class DownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class DownEncoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnDownEncoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + attentions = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states): + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=None) + hidden_states = attn(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnSkipDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=np.sqrt(2.0), + downsample_padding=1, + add_downsample=True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel='fir', + ) + self.downsamplers = nn.ModuleList( + [FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d( + 3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward(self, hidden_states, temb=None, skip_sample=None): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + output_states += (hidden_states, ) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states, ) + + return hidden_states, output_states, skip_sample + + +class SkipDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + output_scale_factor=np.sqrt(2.0), + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel='fir', + ) + self.downsamplers = nn.ModuleList( + [FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d( + 3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward(self, hidden_states, temb=None, skip_sample=None): + output_states = () + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb) + output_states += (hidden_states, ) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states, ) + + return hidden_states, output_states, skip_sample + + +class ResnetDownsampleBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + skip_time_act=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class SimpleCrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_downsample=True, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + + self.has_cross_attention = True + + resnets = [] + attentions = [] + + self.attn_num_head_channels = attn_num_head_channels + self.num_heads = out_channels // self.attn_num_head_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=attn_num_head_channels, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + temb=None, + encoder_hidden_states=None, + attention_mask=None, + cross_attention_kwargs=None): + output_states = () + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + for resnet, attn in zip(self.resnets, self.attentions): + # resnet + hidden_states = resnet(hidden_states, temb) + + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class KDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: int = 32, + add_downsample=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + # YiYi's comments- might be able to use FirDownsample2D, look into details later + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states += (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class KCrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + cross_attention_dim: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_group_size: int = 32, + add_downsample=True, + attn_num_head_channels: int = 64, + add_self_attention: bool = False, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + attentions.append( + KAttentionBlock( + out_channels, + out_channels // attn_num_head_channels, + attn_num_head_channels, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm='layer_norm', + group_size=resnet_group_size, + )) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_downsample: + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + temb=None, + encoder_hidden_states=None, + attention_mask=None, + cross_attention_kwargs=None): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + attention_mask, + cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if self.downsamplers is None: + output_states += (None, ) + else: + output_states += (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class AttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + add_upsample=True, + ): + super().__init__() + resnets = [] + attentions = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class CrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + cross_attention_dim=1280, + pixelwise_cross_attention_dim=None, + output_scale_factor=1.0, + add_upsample=True, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + use_pixelwise_attention=False, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attn_num_head_channels = attn_num_head_channels + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + attn_num_head_channels, + out_channels // attn_num_head_channels, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + pixelwise_cross_attention_dim=res_skip_channels, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + use_pixelwise_attention=use_pixelwise_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + attn_num_head_channels, + out_channels // attn_num_head_channels, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + res_hidden_states_tuple, + temb=None, + encoder_hidden_states=None, + encoder_pixelwise_hidden_states_tuple=None, + cross_attention_kwargs=None, + upsample_size=None, + attention_mask=None, + ): + # TODO(Patrick, William) - attention mask is not used + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + encoder_pixelwise_hidden_states = encoder_pixelwise_hidden_states_tuple[ + -1] + encoder_pixelwise_hidden_states_tuple = encoder_pixelwise_hidden_states_tuple[: + -1] + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + encoder_pixelwise_hidden_states, + cross_attention_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_pixelwise_hidden_states= + encoder_pixelwise_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpDecoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + def forward(self, hidden_states): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnUpDecoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=1.0, + add_upsample=True, + ): + super().__init__() + resnets = [] + attentions = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + def forward(self, hidden_states): + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=None) + hidden_states = attn(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnSkipUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + output_scale_factor=np.sqrt(2.0), + upsample_padding=1, + add_upsample=True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(resnet_in_channels + res_skip_channels // 4, + 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attn_num_head_channels + if attn_num_head_channels is not None else 1, + dim_head=attn_num_head_channels + if attn_num_head_channels is not None else out_channels, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel='fir', + ) + self.skip_conv = nn.Conv2d( + out_channels, + 3, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), + num_channels=out_channels, + eps=resnet_eps, + affine=True) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + skip_sample=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + + hidden_states = self.attentions[0](hidden_states) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class SkipUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + output_scale_factor=np.sqrt(2.0), + add_upsample=True, + upsample_padding=1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min((resnet_in_channels + res_skip_channels) // 4, + 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel='fir', + ) + self.skip_conv = nn.Conv2d( + out_channels, + 3, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), + num_channels=out_channels, + eps=resnet_eps, + affine=True) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + skip_sample=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class ResnetUpsampleBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + skip_time_act=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class SimpleCrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attn_num_head_channels=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_upsample=True, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attn_num_head_channels = attn_num_head_channels + + self.num_heads = out_channels // self.attn_num_head_channels + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=attn_num_head_channels, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + res_hidden_states_tuple, + temb=None, + encoder_hidden_states=None, + upsample_size=None, + attention_mask=None, + cross_attention_kwargs=None, + ): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + for resnet, attn in zip(self.resnets, self.attentions): + # resnet + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class KUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 5, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: Optional[int] = 32, + add_upsample=True, + ): + super().__init__() + resnets = [] + k_in_channels = 2 * out_channels + k_out_channels = in_channels + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=k_out_channels if + (i == num_layers - 1) else out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], + dim=1) + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class KCrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: int = 32, + attn_num_head_channels=1, # attention dim_head + cross_attention_dim: int = 768, + add_upsample: bool = True, + upcast_attention: bool = False, + ): + super().__init__() + resnets = [] + attentions = [] + + is_first_block = in_channels == out_channels == temb_channels + is_middle_block = in_channels != out_channels + add_self_attention = True if is_first_block else False + + self.has_cross_attention = True + self.attn_num_head_channels = attn_num_head_channels + + # in_channels, and out_channels for the block (k-unet) + k_in_channels = out_channels if is_first_block else 2 * out_channels + k_out_channels = in_channels + + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + if is_middle_block and (i == num_layers - 1): + conv_2d_out_channels = k_out_channels + else: + conv_2d_out_channels = None + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + conv_2d_out_channels=conv_2d_out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + attentions.append( + KAttentionBlock( + k_out_channels if (i == num_layers - 1) else out_channels, + k_out_channels // attn_num_head_channels if + (i == num_layers + - 1) else out_channels // attn_num_head_channels, + attn_num_head_channels, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm='layer_norm', + upcast_attention=upcast_attention, + )) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + res_hidden_states_tuple, + temb=None, + encoder_hidden_states=None, + cross_attention_kwargs=None, + upsample_size=None, + attention_mask=None, + ): + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], + dim=1) + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + attention_mask, + cross_attention_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +# can potentially later be renamed to `No-feed-forward` attention +class KAttentionBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout: float = 0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + upcast_attention: bool = False, + temb_channels: int = 768, # for ada_group_norm + add_self_attention: bool = False, + cross_attention_norm: Optional[str] = None, + group_size: int = 32, + ): + super().__init__() + self.add_self_attention = add_self_attention + + # 1. Self-Attn + if add_self_attention: + self.norm1 = AdaGroupNorm(temb_channels, dim, + max(1, dim // group_size)) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=None, + cross_attention_norm=None, + ) + + # 2. Cross-Attn + self.norm2 = AdaGroupNorm(temb_channels, dim, + max(1, dim // group_size)) + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + cross_attention_norm=cross_attention_norm, + ) + + def _to_3d(self, hidden_states, height, weight): + return hidden_states.permute(0, 2, 3, + 1).reshape(hidden_states.shape[0], + height * weight, -1) + + def _to_4d(self, hidden_states, height, weight): + return hidden_states.permute(0, 2, 1).reshape(hidden_states.shape[0], + -1, height, weight) + + def forward( + self, + hidden_states, + encoder_hidden_states=None, + emb=None, + attention_mask=None, + cross_attention_kwargs=None, + ): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + # 1. Self-Attention + if self.add_self_attention: + norm_hidden_states = self.norm1(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, + weight) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=None, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + # 2. Cross-Attention/None + norm_hidden_states = self.norm2(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + return hidden_states diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_condition.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_condition.py new file mode 100644 index 0000000000000000000000000000000000000000..539d3d1c889a4350383bf69644d63114d84e0816 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd/unet_2d_condition.py @@ -0,0 +1,887 @@ +# Part of the implementation is borrowed and modified from diffusers, +# publicly available at https://github.com/huggingface/diffusers/tree/main/src/diffusers/models/unet_2d_condition.py +import os +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import json +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import UNet2DConditionLoadersMixin +from diffusers.models import ModelMixin +from diffusers.models.attention_processor import (AttentionProcessor, + AttnProcessor) +from diffusers.models.embeddings import (GaussianFourierProjection, + TextTimeEmbedding, TimestepEmbedding, + Timesteps) +from diffusers.utils import BaseOutput, logging + +from .unet_2d_blocks import (CrossAttnDownBlock2D, CrossAttnUpBlock2D, + DownBlock2D, UNetMidBlock2DCrossAttn, + UNetMidBlock2DSimpleCrossAttn, UpBlock2D, + get_down_block, get_up_block) + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNet2DConditionOutput(BaseOutput): + """ + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Hidden states conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor + + +class UNet2DConditionModel(ModelMixin, ConfigMixin, + UNet2DConditionLoadersMixin): + r""" + UNet2DConditionModel is a conditional 2D UNet model that takes in a noisy sample, conditional state, and a timestep + and returns sample shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for the generic methods the library + implements for all the models (such as downloading or saving, etc.) + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + The mid block type. Choose from `UNetMidBlock2DCrossAttn` or `UNetMidBlock2DSimpleCrossAttn`, will skip the + mid block layer if `None`. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", "CrossAttnUpBlock2D",)`): + The tuple of upsample blocks to use. + only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, it will skip the normalization and activation layers in post-processing + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + encoder_hid_dim (`int`, *optional*, defaults to None): + If given, `encoder_hidden_states` will be projected from this dimension to `cross_attention_dim`. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for resnet blocks, see [`~models.resnet.ResnetBlock2D`]. Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to None): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to None): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + num_class_embeds (`int`, *optional*, defaults to None): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, default to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, default to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, default to `None`): + Optional activation function to use on the time embeddings only one time before they as passed to the rest + of the unet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str, *optional*, default to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, default to `None`): + The dimension of `cond_proj` layer in timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. + conv_out_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_out` layer. + projection_class_embeddings_input_dim (`int`, *optional*): The dimension of the `class_labels` input when + using the "projection" `class_embed_type`. Required when using the "projection" `class_embed_type`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`): + Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If + `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is None, the + `only_cross_attention` value will be used as the value for `mid_block_only_cross_attention`. Else, it will + default to `False`. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + center_input_sample: bool = False, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'DownBlock2D', + ), + mid_block_type: Optional[str] = 'UNetMidBlock2DCrossAttn', + up_block_types: Tuple[str] = ('UpBlock2D', 'CrossAttnUpBlock2D', + 'CrossAttnUpBlock2D', + 'CrossAttnUpBlock2D'), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = 'silu', + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + encoder_hid_dim: Optional[int] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = 'default', + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: int = 1.0, + time_embedding_type: str = 'positional', + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + class_embeddings_concat: bool = False, + mid_block_only_cross_attention: Optional[bool] = None, + cross_attention_norm: Optional[str] = None, + addition_embed_type_num_heads=64, + ): + super().__init__() + + self.sample_size = sample_size + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f'Must provide the same number of `down_block_types` as `up_block_types`. \ + `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}.' + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `block_out_channels` as `down_block_types`. \ + `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + only_cross_attention, + bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `only_cross_attention` as `down_block_types`. \ + `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + attention_head_dim, + int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `attention_head_dim` as `down_block_types`. \ + `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}.' + ) + + if isinstance( + cross_attention_dim, + list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `cross_attention_dim` as `down_block_types`. \ + `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + layers_per_block, + int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `layers_per_block` as `down_block_types`. \ + `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}.' + ) + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=conv_in_kernel, + padding=conv_in_padding) + + # time + if time_embedding_type == 'fourier': + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError( + f'`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.' + ) + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, + set_W_to_weight=False, + log=False, + flip_sin_to_cos=flip_sin_to_cos) + timestep_input_dim = time_embed_dim + elif time_embedding_type == 'positional': + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, + freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f'{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`.' + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + if encoder_hid_dim is not None: + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, + cross_attention_dim) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, + time_embed_dim) + elif class_embed_type == 'timestep': + self.class_embedding = TimestepEmbedding( + timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == 'identity': + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == 'projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding( + projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == 'simple_projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear( + projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == 'text': + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, + time_embed_dim, + num_heads=addition_embed_type_num_heads) + elif addition_embed_type is not None: + raise ValueError( + f"addition_embed_type: {addition_embed_type} must be None or 'text'." + ) + + if time_embedding_act_fn is None: + self.time_embed_act = None + elif time_embedding_act_fn == 'swish': + self.time_embed_act = lambda x: F.silu(x) + elif time_embedding_act_fn == 'mish': + self.time_embed_act = nn.Mish() + elif time_embedding_act_fn == 'silu': + self.time_embed_act = nn.SiLU() + elif time_embedding_act_fn == 'gelu': + self.time_embed_act = nn.GELU() + else: + raise ValueError( + f'Unsupported activation function: {time_embedding_act_fn}') + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = only_cross_attention + + only_cross_attention = [only_cross_attention + ] * len(down_block_types) + + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = False + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim, ) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = ( + cross_attention_dim, ) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + attn_num_head_channels=attention_head_dim[i], + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + ) + self.down_blocks.append(down_block) + + # mid + if mid_block_type == 'UNetMidBlock2DCrossAttn': + self.mid_block = UNetMidBlock2DCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim[-1], + attn_num_head_channels=attention_head_dim[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + elif mid_block_type == 'UNetMidBlock2DSimpleCrossAttn': + self.mid_block = UNetMidBlock2DSimpleCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim[-1], + attn_num_head_channels=attention_head_dim[-1], + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type is None: + self.mid_block = None + else: + raise ValueError(f'unknown mid_block_type : {mid_block_type}') + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_attention_head_dim = list(reversed(attention_head_dim)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + only_cross_attention = list(reversed(only_cross_attention)) + + reversed_pixelwise_cross_attention_dim = [-1, 1280, 640, 320] + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min( + i + 1, + len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + pixelwise_cross_attention_dim= + reversed_pixelwise_cross_attention_dim[i], + attn_num_head_channels=reversed_attention_head_dim[i], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], + num_groups=norm_num_groups, + eps=norm_eps) + + if act_fn == 'swish': + self.conv_act = lambda x: F.silu(x) + elif act_fn == 'mish': + self.conv_act = nn.Mish() + elif act_fn == 'silu': + self.conv_act = nn.SiLU() + elif act_fn == 'gelu': + self.conv_act = nn.GELU() + else: + raise ValueError(f'Unsupported activation function: {act_fn}') + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], + out_channels, + kernel_size=conv_out_kernel, + padding=conv_out_padding) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, + processors: Dict[str, + AttentionProcessor]): + if hasattr(module, 'set_processor'): + processors[f'{name}.processor'] = module.processor + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f'{name}.{sub_name}', child, + processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, + Dict[str, + AttentionProcessor]]): + r""" + Parameters: + `processor (`dict` of `AttentionProcessor` or `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + of **all** `Attention` layers. + In case `processor` is a dict, the key needs to define the path to + the corresponding cross attention processor. + This is strongly recommended when setting trainable attention processors.: + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f'A dict of processors was passed, but the number of processors {len(processor)} does not match the' + f' number of attention layers: {count}. Please make sure to pass {count} processor classes.' + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, + processor): + if hasattr(module, 'set_processor'): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f'{name}.processor')) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f'{name}.{sub_name}', child, + processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + self.set_attn_processor(AttnProcessor()) + + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module will split the input tensor in slices, to compute attention + in several steps. This is useful to save some memory in exchange for a small speed decrease. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If + `"max"`, maximum amount of memory will be saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, 'set_attention_slice'): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == 'auto': + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == 'max': + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance( + slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f'You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different' + f' attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}.' + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError( + f'size {size} has to be smaller or equal to {dim}.') + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, + slice_size: List[int]): + if hasattr(module, 'set_attention_slice'): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (CrossAttnDownBlock2D, DownBlock2D, + CrossAttnUpBlock2D, UpBlock2D)): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + Args: + sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor + timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps + encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). + + Returns: + [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info( + 'Forward upsample size to force interpolation output size.') + forward_upsample_size = True + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == 'mps' + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], + dtype=dtype, + device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError( + 'class_labels should be provided when num_class_embeds > 0' + ) + + if self.config.class_embed_type == 'timestep': + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to( + dtype=sample.dtype) + + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + if self.config.addition_embed_type == 'text': + aug_emb = self.add_embedding(encoder_hidden_states) + emb = emb + aug_emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + if self.encoder_hid_proj is not None: + encoder_hidden_states = self.encoder_hid_proj( + encoder_hidden_states) + + # 2. pre-process + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample, ) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, 'has_cross_attention' + ) and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block( + hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + if down_block_additional_residuals is not None: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + ( + down_block_res_sample, ) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if mid_block_additional_residual is not None: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets):] + down_block_res_samples = down_block_res_samples[:-len( + upsample_block.resnets)] + + down_block_additional_residual = down_block_additional_residuals[ + -len(upsample_block.resnets):] + down_block_additional_residuals = down_block_additional_residuals[:-len( + upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, 'has_cross_attention' + ) and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + encoder_pixelwise_hidden_states_tuple= + down_block_additional_residual, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if not return_dict: + return (sample, ) + + return UNet2DConditionOutput(sample=sample) + + @classmethod + def from_pretrained_(cls, pretrained_model_path, subfolder=None, **kwargs): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, + subfolder) + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f'{config_file} does not exist') + with open(config_file, 'r') as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model = cls.from_config(config) + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + if not os.path.isfile(model_file): + raise RuntimeError(f'{model_file} does not exist') + state_dict = torch.load(model_file, map_location='cpu') + for k, v in model.state_dict().items(): + if 'attn2_plus' in k: + state_dict.update({k: v}) + model.load_state_dict(state_dict, strict=False) + + return model diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/__init__.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1448c3485d173303637e29ccbf89b664371577cf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .unet_2d_condition import UNet2DConditionModel + from .controlnet import ControlNetModel + +else: + _import_structure = { + 'unet_2d_condition': ['UNet2DConditionModel'], + 'controlnet': ['ControlNetModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/controlnet.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..0440c82d3523950018eddd1e761b8bf4c9adb01f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/controlnet.py @@ -0,0 +1,980 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention_processor import (AttentionProcessor, + AttnProcessor) +from diffusers.models.embeddings import (TextImageProjection, + TextImageTimeEmbedding, + TextTimeEmbedding, TimestepEmbedding, + Timesteps) +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import BaseOutput, logging +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.super_resolution.rrdbnet_arch import RRDB +from .unet_2d_blocks import (CrossAttnDownBlock2D, DownBlock2D, + UNetMidBlock2DCrossAttn, get_down_block) +from .unet_2d_condition import UNet2DConditionModel + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class ControlNetOutput(BaseOutput): + """ + The output of [`ControlNetModel`]. + + Args: + down_block_res_samples (`tuple[torch.Tensor]`): + A tuple of downsample activations at different resolutions for each downsampling block. Each tensor should + be of shape `(batch_size, channel * resolution, height //resolution, width // resolution)`. Output can be + used to condition the original UNet's downsampling activations. + mid_down_block_re_sample (`torch.Tensor`): + The activation of the midde block (the lowest sample resolution). Each tensor should be of shape + `(batch_size, channel * lowest_resolution, height // lowest_resolution, width // lowest_resolution)`. + Output can be used to condition the original UNet's middle block activation. + """ + + down_block_res_samples: Tuple[torch.Tensor] + mid_block_res_sample: torch.Tensor + + +class ControlNetConditioningEmbedding(nn.Module): + """ + Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN + [11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized + training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the + convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides + (activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full + model) to encode image-space conditions ... into feature maps ..." + """ + + def __init__( + self, + conditioning_embedding_channels: int, + conditioning_channels: int = 3, + block_out_channels: Tuple[int] = (16, 32, 96, 256), + return_rgbs: bool = True, + use_rrdb: bool = True, + ): + super().__init__() + + self.return_rgbs = return_rgbs + self.use_rrdb = use_rrdb + + self.conv_in = nn.Conv2d( + conditioning_channels, + block_out_channels[0], + kernel_size=3, + padding=1) + + if self.use_rrdb: + num_rrdb_block = 2 + layers = ( + RRDB(block_out_channels[0], block_out_channels[0]) + for i in range(num_rrdb_block)) + self.preprocesser = nn.Sequential(*layers) + + self.blocks = nn.ModuleList([]) + self.to_rgbs = nn.ModuleList([]) + + for i in range(len(block_out_channels) - 1): + channel_in = block_out_channels[i] + channel_out = block_out_channels[i + 1] + self.blocks.append( + nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) + self.blocks.append( + nn.Conv2d( + channel_in, + channel_out, + kernel_size=3, + padding=1, + stride=2)) + + if return_rgbs: + self.to_rgbs.append( + nn.Conv2d(channel_out, 3, kernel_size=3, + padding=1)) # channel_in + + self.conv_out = zero_module( + nn.Conv2d( + block_out_channels[-1], + conditioning_embedding_channels, + kernel_size=3, + padding=1)) + + def forward(self, conditioning): + embedding = self.conv_in(conditioning) + embedding = F.silu(embedding) + + if self.use_rrdb: + embedding = self.preprocesser(embedding) + + out_rgbs = [] + for i, block in enumerate(self.blocks): + embedding = block(embedding) + embedding = F.silu(embedding) + + if i % 2 and self.return_rgbs: # 0 + out_rgbs.append(self.to_rgbs[i // 2](embedding)) + + embedding = self.conv_out(embedding) + + return [embedding, out_rgbs] if self.return_rgbs else embedding + + +class ControlNetModel(ModelMixin, ConfigMixin): + """ + A ControlNet model. + + Args: + in_channels (`int`, defaults to 4): + The number of channels in the input sample. + flip_sin_to_cos (`bool`, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, defaults to 0): + The frequency shift to apply to the time embedding. + down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`): + block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, defaults to 2): + The number of layers per block. + downsample_padding (`int`, defaults to 1): + The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, defaults to 1): + The scale factor to use for the mid block. + act_fn (`str`, defaults to "silu"): + The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups to use for the normalization. If None, normalization and activation layers is skipped + in post-processing. + norm_eps (`float`, defaults to 1e-5): + The epsilon to use for the normalization. + cross_attention_dim (`int`, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8): + The dimension of the attention heads. + use_linear_projection (`bool`, defaults to `False`): + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from None, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + num_class_embeds (`int`, *optional*, defaults to 0): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + upcast_attention (`bool`, defaults to `False`): + resnet_time_scale_shift (`str`, defaults to `"default"`): + Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`. + projection_class_embeddings_input_dim (`int`, *optional*, defaults to `None`): + The dimension of the `class_labels` input when `class_embed_type="projection"`. Required when + `class_embed_type="projection"`. + controlnet_conditioning_channel_order (`str`, defaults to `"rgb"`): + The channel order of conditional image. Will convert to `rgb` if it's `bgr`. + conditioning_embedding_out_channels (`tuple[int]`, *optional*, defaults to `(16, 32, 96, 256)`): + The tuple of output channel for each block in the `conditioning_embedding` layer. + global_pool_conditions (`bool`, defaults to `False`): + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 4, + conditioning_channels: int = 3, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'DownBlock2D', + ), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = 'silu', + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: int = 1280, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = 'default', + projection_class_embeddings_input_dim: Optional[int] = None, + controlnet_conditioning_channel_order: str = 'rgb', + conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, + 96, 256), + global_pool_conditions: bool = False, + addition_embed_type_num_heads=64, + return_rgbs: bool = True, + use_rrdb: bool = False, + ): + super().__init__() + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in + # https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: \ + {block_out_channels}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + only_cross_attention, + bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: \ + {only_cross_attention}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + num_attention_heads, + int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: \ + {num_attention_heads}. `down_block_types`: {down_block_types}.' + ) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block + ] * len(down_block_types) + + # input + self.return_rgbs = return_rgbs + conv_in_kernel = 3 + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=conv_in_kernel, + padding=conv_in_padding) + + # time + time_embed_dim = block_out_channels[0] * 4 + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, + freq_shift) + timestep_input_dim = block_out_channels[0] + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + ) + + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = 'text_proj' + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info( + "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined." + ) + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f'`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}.' + ) + + if encoder_hid_dim_type == 'text_proj': + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, + cross_attention_dim) + elif encoder_hid_dim_type == 'text_image_proj': + # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the + # currently only use case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)` + self.encoder_hid_proj = TextImageProjection( + text_embed_dim=encoder_hid_dim, + image_embed_dim=cross_attention_dim, + cross_attention_dim=cross_attention_dim, + ) + + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, + time_embed_dim) + elif class_embed_type == 'timestep': + self.class_embedding = TimestepEmbedding(timestep_input_dim, + time_embed_dim) + elif class_embed_type == 'identity': + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == 'projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding( + projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == 'text': + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, + time_embed_dim, + num_heads=addition_embed_type_num_heads) + elif addition_embed_type == 'text_image': + # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter + # the __init__ too much they are set to `cross_attention_dim` here as this is exactly the + # required dimension for the currently only use + # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)` + self.add_embedding = TextImageTimeEmbedding( + text_embed_dim=cross_attention_dim, + image_embed_dim=cross_attention_dim, + time_embed_dim=time_embed_dim) + elif addition_embed_type == 'text_time': + self.add_time_proj = Timesteps(addition_time_embed_dim, + flip_sin_to_cos, freq_shift) + self.add_embedding = TimestepEmbedding( + projection_class_embeddings_input_dim, time_embed_dim) + + elif addition_embed_type is not None: + raise ValueError( + f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'." + ) + + # control net conditioning embedding + self.controlnet_cond_embedding = ControlNetConditioningEmbedding( + conditioning_embedding_channels=block_out_channels[0], + block_out_channels=conditioning_embedding_out_channels, + conditioning_channels=conditioning_channels, + return_rgbs=return_rgbs, + use_rrdb=use_rrdb, + ) + + self.down_blocks = nn.ModuleList([]) + self.controlnet_down_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + only_cross_attention = [only_cross_attention + ] * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim, ) * len(down_block_types) + + if isinstance(num_attention_heads, int): + num_attention_heads = ( + num_attention_heads, ) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[i], + attention_head_dim=attention_head_dim[i] + if attention_head_dim[i] is not None else output_channel, + downsample_padding=downsample_padding, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + self.down_blocks.append(down_block) + + for _ in range(layers_per_block): + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + if not is_final_block: + controlnet_block = nn.Conv2d( + output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + # mid + mid_block_channel = block_out_channels[-1] + + controlnet_block = nn.Conv2d( + mid_block_channel, mid_block_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_mid_block = controlnet_block + + self.mid_block = UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=mid_block_channel, + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + + @classmethod + def from_unet( + cls, + unet: UNet2DConditionModel, + controlnet_conditioning_channel_order: str = 'rgb', + conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, + 96, 256), + load_weights_from_unet: bool = True, + ): + r""" + Instantiate a [`ControlNetModel`] from [`UNet2DConditionModel`]. + + Parameters: + unet (`UNet2DConditionModel`): + The UNet model weights to copy to the [`ControlNetModel`]. All configuration options are also copied + where applicable. + """ + transformer_layers_per_block = ( + unet.config.transformer_layers_per_block + if 'transformer_layers_per_block' in unet.config else 1) + encoder_hid_dim = unet.config.encoder_hid_dim if 'encoder_hid_dim' in unet.config else None + encoder_hid_dim_type = unet.config.encoder_hid_dim_type if 'encoder_hid_dim_type' in unet.config else None + addition_embed_type = unet.config.addition_embed_type if 'addition_embed_type' in unet.config else None + addition_time_embed_dim = ( + unet.config.addition_time_embed_dim + if 'addition_time_embed_dim' in unet.config else None) + + controlnet = cls( + encoder_hid_dim=encoder_hid_dim, + encoder_hid_dim_type=encoder_hid_dim_type, + addition_embed_type=addition_embed_type, + addition_time_embed_dim=addition_time_embed_dim, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=unet.config.in_channels, + flip_sin_to_cos=unet.config.flip_sin_to_cos, + freq_shift=unet.config.freq_shift, + down_block_types=unet.config.down_block_types, + only_cross_attention=unet.config.only_cross_attention, + block_out_channels=unet.config.block_out_channels, + layers_per_block=unet.config.layers_per_block, + downsample_padding=unet.config.downsample_padding, + mid_block_scale_factor=unet.config.mid_block_scale_factor, + act_fn=unet.config.act_fn, + norm_num_groups=unet.config.norm_num_groups, + norm_eps=unet.config.norm_eps, + cross_attention_dim=unet.config.cross_attention_dim, + attention_head_dim=unet.config.attention_head_dim, + num_attention_heads=unet.config.num_attention_heads, + use_linear_projection=unet.config.use_linear_projection, + class_embed_type=unet.config.class_embed_type, + num_class_embeds=unet.config.num_class_embeds, + upcast_attention=unet.config.upcast_attention, + resnet_time_scale_shift=unet.config.resnet_time_scale_shift, + projection_class_embeddings_input_dim=unet.config. + projection_class_embeddings_input_dim, + controlnet_conditioning_channel_order= + controlnet_conditioning_channel_order, + conditioning_embedding_out_channels= + conditioning_embedding_out_channels, + ) + + if load_weights_from_unet: + controlnet.conv_in.load_state_dict(unet.conv_in.state_dict()) + controlnet.time_proj.load_state_dict(unet.time_proj.state_dict()) + controlnet.time_embedding.load_state_dict( + unet.time_embedding.state_dict()) + + if controlnet.class_embedding: + controlnet.class_embedding.load_state_dict( + unet.class_embedding.state_dict()) + + controlnet.down_blocks.load_state_dict( + unet.down_blocks.state_dict()) + controlnet.mid_block.load_state_dict(unet.mid_block.state_dict()) + + return controlnet + + @property + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, + processors: Dict[str, + AttentionProcessor]): + if hasattr(module, 'get_processor'): + processors[f'{name}.processor'] = module.get_processor( + return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f'{name}.{sub_name}', child, + processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, + Dict[str, + AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f'A dict of processors was passed, but the number of processors {len(processor)} does not match the' + f' number of attention layers: {count}. Please make sure to pass {count} processor classes.' + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, + processor): + if hasattr(module, 'set_processor'): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f'{name}.processor')) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f'{name}.{sub_name}', child, + processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + self.set_attn_processor(AttnProcessor()) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attention_slice + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, 'set_attention_slice'): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == 'auto': + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == 'max': + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance( + slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f'You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different' + f' attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}.' + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError( + f'size {size} has to be smaller or equal to {dim}.') + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, + slice_size: List[int]): + if hasattr(module, 'set_attention_slice'): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (CrossAttnDownBlock2D, DownBlock2D)): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + controlnet_cond: torch.FloatTensor, + fg_mask: Optional[torch.FloatTensor] = None, + conditioning_scale_fg: float = 1.0, + conditioning_scale_bg: float = 1.0, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guess_mode: bool = False, + return_dict: bool = True, + ) -> Union[ControlNetOutput, Tuple]: + """ + The [`ControlNetModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor. + timestep (`Union[torch.Tensor, float, int]`): + The number of timesteps to denoise an input. + encoder_hidden_states (`torch.Tensor`): + The encoder hidden states. + controlnet_cond (`torch.FloatTensor`): + The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`. + conditioning_scale (`float`, defaults to `1.0`): + The scale factor for ControlNet outputs. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond (`torch.Tensor`, *optional*, defaults to `None`): + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + added_cond_kwargs (`dict`): + Additional conditions for the Stable Diffusion XL UNet. + cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`): + A kwargs dictionary that if specified is passed along to the `AttnProcessor`. + guess_mode (`bool`, defaults to `False`): + In this mode, the ControlNet encoder tries its best to recognize the input content of the input even if + you remove all prompts. A `guidance_scale` between 3.0 and 5.0 is recommended. + return_dict (`bool`, defaults to `True`): + Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple. + + Returns: + [`~models.controlnet.ControlNetOutput`] **or** `tuple`: + If `return_dict` is `True`, a [`~models.controlnet.ControlNetOutput`] is returned, otherwise a tuple is + returned where the first element is the sample tensor. + """ + # check channel order + channel_order = self.config.controlnet_conditioning_channel_order + + if channel_order == 'rgb': + # in rgb order by default + ... + elif channel_order == 'bgr': + controlnet_cond = torch.flip(controlnet_cond, dims=[1]) + else: + raise ValueError( + f'unknown `controlnet_conditioning_channel_order`: {channel_order}' + ) + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == 'mps' + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], + dtype=dtype, + device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError( + 'class_labels should be provided when num_class_embeds > 0' + ) + + if self.config.class_embed_type == 'timestep': + class_labels = self.time_proj(class_labels) + + class_emb = self.class_embedding(class_labels).to(dtype=self.dtype) + emb = emb + class_emb + + if self.config.addition_embed_type is not None: + if self.config.addition_embed_type == 'text': + aug_emb = self.add_embedding(encoder_hidden_states) + + elif self.config.addition_embed_type == 'text_time': + if 'text_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' \ + which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get('text_embeds') + if 'time_ids' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' \ + which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get('time_ids') + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + + emb = emb + aug_emb if aug_emb is not None else emb + + # 2. pre-process + sample = self.conv_in(sample) + + controlnet_cond_mid = None + if self.return_rgbs: + controlnet_cond, controlnet_cond_mid = self.controlnet_cond_embedding( + controlnet_cond) + else: + controlnet_cond = self.controlnet_cond_embedding(controlnet_cond) + + sample = sample + controlnet_cond + + # 3. down + down_block_res_samples = (sample, ) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, 'has_cross_attention' + ) and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block( + hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + # 5. Control net blocks + + controlnet_down_block_res_samples = () + + for down_block_res_sample, controlnet_block in zip( + down_block_res_samples, self.controlnet_down_blocks): + down_block_res_sample = controlnet_block(down_block_res_sample) + controlnet_down_block_res_samples = controlnet_down_block_res_samples + ( + down_block_res_sample, ) + + down_block_res_samples = controlnet_down_block_res_samples + + mid_block_res_sample = self.controlnet_mid_block(sample) + + # 6. scaling + if guess_mode and not self.config.global_pool_conditions: + scales = torch.logspace( + -1, 0, len(down_block_res_samples) + 1, + device=sample.device) # 0.1 to 1.0 + + scales = scales * conditioning_scale_fg + down_block_res_samples = [ + sample * scale + for sample, scale in zip(down_block_res_samples, scales) + ] + mid_block_res_sample = mid_block_res_sample * scales[ + -1] # last one + else: + if fg_mask is None: + down_block_res_samples = [ + sample * conditioning_scale_fg + for sample in down_block_res_samples + ] + mid_block_res_sample = mid_block_res_sample * conditioning_scale_fg + else: + down_block_masks = [ + torch.zeros_like(sample) + conditioning_scale_bg + for i, sample in enumerate(down_block_res_samples) + ] + mid_block_mask = torch.zeros_like( + mid_block_res_sample) + conditioning_scale_bg + + for i, sample in enumerate(down_block_masks): + tmp_mask = F.interpolate( + fg_mask, + size=sample.shape[-2:]).repeat(sample.shape[0], + sample.shape[1], 1, + 1).bool() + down_block_masks[i] = sample.masked_fill( + tmp_mask, conditioning_scale_fg) + + tmp_mask = F.interpolate( + fg_mask, size=mid_block_mask.shape[-2:]).repeat( + mid_block_mask.shape[0], mid_block_mask.shape[1], 1, + 1).bool() + mid_block_mask = mid_block_mask.masked_fill( + tmp_mask, conditioning_scale_fg) + + down_block_res_samples = [ + sample * down_block_mask for sample, down_block_mask in + zip(down_block_res_samples, down_block_masks) + ] + mid_block_res_sample = mid_block_res_sample * mid_block_mask + + if self.config.global_pool_conditions: + down_block_res_samples = [ + torch.mean(sample, dim=(2, 3), keepdim=True) + for sample in down_block_res_samples + ] + mid_block_res_sample = torch.mean( + mid_block_res_sample, dim=(2, 3), keepdim=True) + + if not return_dict: + return (controlnet_cond_mid, down_block_res_samples, + mid_block_res_sample) + + return ControlNetOutput( + controlnet_cond_mid=controlnet_cond_mid, + down_block_res_samples=down_block_res_samples, + mid_block_res_sample=mid_block_res_sample) + + +def zero_module(module): + for p in module.parameters(): + nn.init.zeros_(p) + return module diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_blocks.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..33de31e6f70184c3bcb50fd45a97bcd94709643c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_blocks.py @@ -0,0 +1,3439 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from diffusers.models.activations import get_activation +from diffusers.models.attention import AdaGroupNorm +from diffusers.models.attention_processor import (Attention, + AttnAddedKVProcessor, + AttnAddedKVProcessor2_0) +from diffusers.models.dual_transformer_2d import DualTransformer2DModel +from diffusers.models.resnet import (Downsample2D, FirDownsample2D, + FirUpsample2D, KDownsample2D, KUpsample2D, + ResnetBlock2D, Upsample2D) +from diffusers.models.transformer_2d import Transformer2DModel +from diffusers.utils import is_torch_version, logging +from einops import rearrange +from torch import nn + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_down_block( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + transformer_layers_per_block=1, + num_attention_heads=None, + resnet_groups=None, + cross_attention_dim=None, + downsample_padding=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift='default', + attention_type='default', + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, + attention_head_dim=None, + downsample_type=None, +): + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warn( + f'It is recommended to provide `attention_head_dim` when calling `get_down_block`. \ + Defaulting `attention_head_dim` to {num_attention_heads}.') + attention_head_dim = num_attention_heads + + down_block_type = down_block_type[7:] if down_block_type.startswith( + 'UNetRes') else down_block_type + if down_block_type == 'DownBlock2D': + return DownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'ResnetDownsampleBlock2D': + return ResnetDownsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif down_block_type == 'AttnDownBlock2D': + if add_downsample is False: + downsample_type = None + else: + downsample_type = downsample_type or 'conv' # default to 'conv' + return AttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + downsample_type=downsample_type, + ) + elif down_block_type == 'CrossAttnDownBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for CrossAttnDownBlock2D' + ) + return CrossAttnDownBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif down_block_type == 'SimpleCrossAttnDownBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for SimpleCrossAttnDownBlock2D' + ) + return SimpleCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif down_block_type == 'SkipDownBlock2D': + return SkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'AttnSkipDownBlock2D': + return AttnSkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'DownEncoderBlock2D': + return DownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'AttnDownEncoderBlock2D': + return AttnDownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == 'KDownBlock2D': + return KDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif down_block_type == 'KCrossAttnDownBlock2D': + return KCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + add_self_attention=True if not add_downsample else False, + ) + raise ValueError(f'{down_block_type} does not exist.') + + +def get_up_block( + up_block_type, + num_layers, + in_channels, + out_channels, + prev_output_channel, + temb_channels, + add_upsample, + resnet_eps, + resnet_act_fn, + transformer_layers_per_block=1, + num_attention_heads=None, + resnet_groups=None, + cross_attention_dim=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift='default', + attention_type='default', + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, + attention_head_dim=None, + upsample_type=None, +): + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warn( + f'It is recommended to provide `attention_head_dim` when calling `get_up_block`. \ + Defaulting `attention_head_dim` to {num_attention_heads}.') + attention_head_dim = num_attention_heads + + up_block_type = up_block_type[7:] if up_block_type.startswith( + 'UNetRes') else up_block_type + if up_block_type == 'UpBlock2D': + return UpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + transformer_layers_per_block=transformer_layers_per_block, + num_attention_heads=num_attention_heads, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + ) + elif up_block_type == 'ResnetUpsampleBlock2D': + return ResnetUpsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif up_block_type == 'CrossAttnUpBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for CrossAttnUpBlock2D') + return CrossAttnUpBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif up_block_type == 'SimpleCrossAttnUpBlock2D': + if cross_attention_dim is None: + raise ValueError( + 'cross_attention_dim must be specified for SimpleCrossAttnUpBlock2D' + ) + return SimpleCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif up_block_type == 'AttnUpBlock2D': + if add_upsample is False: + upsample_type = None + else: + upsample_type = upsample_type or 'conv' # default to 'conv' + + return AttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + upsample_type=upsample_type, + ) + elif up_block_type == 'SkipUpBlock2D': + return SkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'AttnSkipUpBlock2D': + return AttnSkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == 'UpDecoderBlock2D': + return UpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == 'AttnUpDecoderBlock2D': + return AttnUpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == 'KUpBlock2D': + return KUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif up_block_type == 'KCrossAttnUpBlock2D': + return KCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + ) + + raise ValueError(f'{up_block_type} does not exist.') + + +class AutoencoderTinyBlock(nn.Module): + + def __init__(self, in_channels: int, out_channels: int, act_fn: str): + super().__init__() + act_fn = get_activation(act_fn) + self.conv = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + ) + self.skip = ( + nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False) + if in_channels != out_channels else nn.Identity()) + self.fuse = nn.ReLU() + + def forward(self, x): + return self.fuse(self.conv(x) + self.skip(x)) + + +class UNetMidBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', # default, spatial + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + self.add_attention = add_attention + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `in_channels`: {in_channels}.' + ) + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups + if resnet_time_scale_shift == 'default' else None, + spatial_norm_dim=temb_channels + if resnet_time_scale_shift == 'spatial' else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + else: + attentions.append(None) + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states, temb=None): + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = attn(hidden_states, temb=temb) + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DCrossAttn(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + output_scale_factor=1.0, + cross_attention_dim=1280, + dual_cross_attention=False, + use_linear_projection=False, + upcast_attention=False, + attention_type='default', + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for _ in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DSimpleCrossAttn(nn.Module): + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + cross_attention_dim=1280, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + + self.has_cross_attention = True + + self.attention_head_dim = attention_head_dim + resnet_groups = resnet_groups if resnet_groups is not None else min( + in_channels // 4, 32) + + self.num_heads = in_channels // self.attention_head_dim + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ] + attentions = [] + + for _ in range(num_layers): + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=in_channels, + cross_attention_dim=in_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, + # so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param + # for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of + # via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + # resnet + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class AttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + downsample_padding=1, + downsample_type='conv', + ): + super().__init__() + resnets = [] + attentions = [] + self.downsample_type = downsample_type + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `in_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if downsample_type == 'conv': + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + elif downsample_type == 'resnet': + self.downsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + down=True, + ) + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states, temb=None, upsample_size=None): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + if self.downsample_type == 'resnet': + hidden_states = downsampler(hidden_states, temb=temb) + else: + hidden_states = downsampler(hidden_states) + + output_states += (hidden_states, ) + + return hidden_states, output_states + + +class CrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + downsample_padding=1, + add_downsample=True, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + attention_type='default', + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + additional_residuals=None, + ): + output_states = () + + blocks = list(zip(self.resnets, self.attentions)) + + for i, (resnet, attn) in enumerate(blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + # apply additional residuals to the output of the last pair of resnet and attention blocks + if i == len(blocks) - 1 and additional_residuals is not None: + hidden_states = hidden_states + additional_residuals + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class DownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class DownEncoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnDownEncoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `in_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states): + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=None) + hidden_states = attn(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnSkipDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=np.sqrt(2.0), + add_downsample=True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `in_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel='fir', + ) + self.downsamplers = nn.ModuleList( + [FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d( + 3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward(self, hidden_states, temb=None, skip_sample=None): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + output_states += (hidden_states, ) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states, ) + + return hidden_states, output_states, skip_sample + + +class SkipDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + output_scale_factor=np.sqrt(2.0), + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel='fir', + ) + self.downsamplers = nn.ModuleList( + [FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d( + 3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward(self, hidden_states, temb=None, skip_sample=None): + output_states = () + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb) + output_states += (hidden_states, ) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states, ) + + return hidden_states, output_states, skip_sample + + +class ResnetDownsampleBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_downsample=True, + skip_time_act=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class SimpleCrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_downsample=True, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + + self.has_cross_attention = True + + resnets = [] + attentions = [] + + self.attention_head_dim = attention_head_dim + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + output_states = () + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, + # so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param + # for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param + # instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + output_states = output_states + (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states, ) + + return hidden_states, output_states + + +class KDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: int = 32, + add_downsample=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + # YiYi's comments- might be able to use FirDownsample2D, look into details later + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states, temb=None): + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + output_states += (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class KCrossAttnDownBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + cross_attention_dim: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_group_size: int = 32, + add_downsample=True, + attention_head_dim: int = 64, + add_self_attention: bool = False, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + attentions.append( + KAttentionBlock( + out_channels, + out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm='layer_norm', + group_size=resnet_group_size, + )) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_downsample: + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.downsamplers is None: + output_states += (None, ) + else: + output_states += (hidden_states, ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class AttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + upsample_type='conv', + ): + super().__init__() + resnets = [] + attentions = [] + + self.upsample_type = upsample_type + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `in_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if upsample_type == 'conv': + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + elif upsample_type == 'resnet': + self.upsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + up=True, + ) + ]) + else: + self.upsamplers = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + if self.upsample_type == 'resnet': + hidden_states = upsampler(hidden_states, temb=temb) + else: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class CrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_upsample=True, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + attention_type='default', + use_pixelwise_attention=True, + ): + super().__init__() + resnets = [] + attentions = [] + pixel_attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + )) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + )) + pixel_attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=res_skip_channels, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + ) if use_pixelwise_attention else None) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + self.pixel_attentions = nn.ModuleList(pixel_attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + pixelwise_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + for resnet, attn, pix_attn in zip(self.resnets, self.attentions, + self.pixel_attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if pixelwise_hidden_states is not None: + pixelwise_hidden_state = pixelwise_hidden_states[-1] + pixelwise_hidden_states = pixelwise_hidden_states[:-1] + pixelwise_hidden_state = rearrange(pixelwise_hidden_state, + 'b c h w -> b (h w) c') + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + if pixelwise_hidden_states is not None: + hidden_states = pix_attn( + hidden_states, + encoder_hidden_states=pixelwise_hidden_state, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=None, + encoder_attention_mask=None, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + if pixelwise_hidden_states is not None: + hidden_states = pix_attn( + hidden_states, + encoder_hidden_states=pixelwise_hidden_state, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=None, + encoder_attention_mask=None, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + transformer_layers_per_block: int = 1, + num_attention_heads=1, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + use_pixelwise_attention=True, + ): + super().__init__() + resnets = [] + pixel_attentions = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + pixel_attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=out_channels, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + ) if use_pixelwise_attention else None) + + self.resnets = nn.ModuleList(resnets) + self.pixel_attentions = nn.ModuleList(pixel_attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None, + pixelwise_hidden_states=None, + cross_attention_kwargs=None): + for resnet, pix_attn in zip(self.resnets, self.pixel_attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if pixelwise_hidden_states is not None: + pixelwise_hidden_state = pixelwise_hidden_states[-1] + pixelwise_hidden_states = pixelwise_hidden_states[:-1] + pixelwise_hidden_state = rearrange(pixelwise_hidden_state, + 'b c h w -> b (h w) c') + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if pixelwise_hidden_states is not None: + hidden_states = pix_attn( + hidden_states, + encoder_hidden_states=pixelwise_hidden_state, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=None, + encoder_attention_mask=None, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpDecoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', # default, spatial + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + temb_channels=None, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + def forward(self, hidden_states, temb=None): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnUpDecoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=1.0, + add_upsample=True, + temb_channels=None, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `out_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups + if resnet_time_scale_shift != 'spatial' else None, + spatial_norm_dim=temb_channels + if resnet_time_scale_shift == 'spatial' else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + Upsample2D( + out_channels, use_conv=True, out_channels=out_channels) + ]) + else: + self.upsamplers = None + + def forward(self, hidden_states, temb=None): + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=temb) + hidden_states = attn(hidden_states, temb=temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnSkipUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + attention_head_dim=1, + output_scale_factor=np.sqrt(2.0), + add_upsample=True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(resnet_in_channels + res_skip_channels // 4, + 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + if attention_head_dim is None: + logger.warn( + f'It is not recommend to pass `attention_head_dim=None`. \ + Defaulting `attention_head_dim` to `out_channels`: {out_channels}.' + ) + attention_head_dim = out_channels + + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + )) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel='fir', + ) + self.skip_conv = nn.Conv2d( + out_channels, + 3, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), + num_channels=out_channels, + eps=resnet_eps, + affine=True) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + skip_sample=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + + hidden_states = self.attentions[0](hidden_states) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class SkipUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_pre_norm: bool = True, + output_scale_factor=np.sqrt(2.0), + add_upsample=True, + upsample_padding=1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min((resnet_in_channels + res_skip_channels) // 4, + 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + )) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel='fir', + ) + self.skip_conv = nn.Conv2d( + out_channels, + 3, + kernel_size=(3, 3), + stride=(1, 1), + padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), + num_channels=out_channels, + eps=resnet_eps, + affine=True) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + skip_sample=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + hidden_states = resnet(hidden_states, temb) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class ResnetUpsampleBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor=1.0, + add_upsample=True, + skip_time_act=False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class SimpleCrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = 'default', + resnet_act_fn: str = 'swish', + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_upsample=True, + skip_time_act=False, + only_cross_attention=False, + cross_attention_norm=None, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers + - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + )) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr( + F, 'scaled_dot_product_attention') else + AttnAddedKVProcessor()) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + )) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, + # so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param + # for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param + # instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + # resnet + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class KUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 5, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: Optional[int] = 32, + add_upsample=True, + ): + super().__init__() + resnets = [] + k_in_channels = 2 * out_channels + k_out_channels = in_channels + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=k_out_channels if + (i == num_layers - 1) else out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, + hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], + dim=1) + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class KCrossAttnUpBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = 'gelu', + resnet_group_size: int = 32, + attention_head_dim=1, # attention dim_head + cross_attention_dim: int = 768, + add_upsample: bool = True, + upcast_attention: bool = False, + ): + super().__init__() + resnets = [] + attentions = [] + + is_first_block = in_channels == out_channels == temb_channels + is_middle_block = in_channels != out_channels + add_self_attention = True if is_first_block else False + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + # in_channels, and out_channels for the block (k-unet) + k_in_channels = out_channels if is_first_block else 2 * out_channels + k_out_channels = in_channels + + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + if is_middle_block and (i == num_layers - 1): + conv_2d_out_channels = k_out_channels + else: + conv_2d_out_channels = None + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + conv_2d_out_channels=conv_2d_out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm='ada_group', + conv_shortcut_bias=False, + )) + attentions.append( + KAttentionBlock( + k_out_channels if (i == num_layers - 1) else out_channels, + k_out_channels // attention_head_dim if + (i == num_layers + - 1) else out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm='layer_norm', + upcast_attention=upcast_attention, + )) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], + dim=1) + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +# can potentially later be renamed to `No-feed-forward` attention +class KAttentionBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout: float = 0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + upcast_attention: bool = False, + temb_channels: int = 768, # for ada_group_norm + add_self_attention: bool = False, + cross_attention_norm: Optional[str] = None, + group_size: int = 32, + ): + super().__init__() + self.add_self_attention = add_self_attention + + # 1. Self-Attn + if add_self_attention: + self.norm1 = AdaGroupNorm(temb_channels, dim, + max(1, dim // group_size)) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=None, + cross_attention_norm=None, + ) + + # 2. Cross-Attn + self.norm2 = AdaGroupNorm(temb_channels, dim, + max(1, dim // group_size)) + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + cross_attention_norm=cross_attention_norm, + ) + + def _to_3d(self, hidden_states, height, weight): + return hidden_states.permute(0, 2, 3, + 1).reshape(hidden_states.shape[0], + height * weight, -1) + + def _to_4d(self, hidden_states, height, weight): + return hidden_states.permute(0, 2, 1).reshape(hidden_states.shape[0], + -1, height, weight) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + # TODO: mark emb as non-optional (self.norm2 requires it). + # requires assessing impact of change to positional param interface. + emb: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + # 1. Self-Attention + if self.add_self_attention: + norm_hidden_states = self.norm1(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, + weight) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + # 2. Cross-Attention/None + norm_hidden_states = self.norm2(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask + if encoder_hidden_states is None else encoder_attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + return hidden_states diff --git a/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_condition.py b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_condition.py new file mode 100644 index 0000000000000000000000000000000000000000..0f68b91f711519df58fb785642e8547dc5c0c69d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_super_resolution_pasd_v2/unet_2d_condition.py @@ -0,0 +1,1101 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import json +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import UNet2DConditionLoadersMixin +from diffusers.models import ModelMixin +from diffusers.models.attention_processor import (AttentionProcessor, + AttnProcessor) +from diffusers.models.embeddings import (GaussianFourierProjection, + TextTimeEmbedding, TimestepEmbedding, + Timesteps) +from diffusers.utils import BaseOutput, logging + +from .unet_2d_blocks import (UNetMidBlock2DCrossAttn, + UNetMidBlock2DSimpleCrossAttn, get_down_block, + get_up_block) + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNet2DConditionOutput(BaseOutput): + """ + The output of [`UNet2DConditionModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor = None + + +class UNet2DConditionModel(ModelMixin, ConfigMixin, + UNet2DConditionLoadersMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + Block type for middle of UNet, it can be either `UNetMidBlock2DCrossAttn` or + `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): + The number of attention heads. If not defined, defaults to `attention_head_dim` + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + addition_time_embed_dim: (`int`, *optional*, defaults to `None`): + Dimension for the timestep embeddings. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, defaults to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, defaults to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, defaults to `None`): + Optional activation function to use only once on the time embeddings before they are passed to the rest of + the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str`, *optional*, defaults to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, defaults to `None`): + The dimension of `cond_proj` layer in the timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. + conv_out_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_out` layer. + projection_class_embeddings_input_dim (`int`, *optional*): The dimension of the `class_labels` input when + `class_embed_type="projection"`. Required when `class_embed_type="projection"`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`): + Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If + `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the + `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False` + otherwise. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + center_input_sample: bool = False, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'DownBlock2D', + ), + mid_block_type: Optional[str] = 'UNetMidBlock2DCrossAttn', + up_block_types: Tuple[str] = ('UpBlock2D', 'CrossAttnUpBlock2D', + 'CrossAttnUpBlock2D', + 'CrossAttnUpBlock2D'), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = 'silu', + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = 'default', + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: int = 1.0, + time_embedding_type: str = 'positional', + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + attention_type: str = 'default', + class_embeddings_concat: bool = False, + mid_block_only_cross_attention: Optional[bool] = None, + cross_attention_norm: Optional[str] = None, + addition_embed_type_num_heads=64, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise ValueError( + 'At the moment it is not possible to define the number of attention heads via \ + `num_attention_heads` because of a naming issue as described in\ + https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. \ + Passing `num_attention_heads` will only be supported in diffusers v0.19.' + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in + # https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f'Must provide the same number of `down_block_types` as `up_block_types`. \ + `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}.' + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `block_out_channels` as `down_block_types`. \ + `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + only_cross_attention, + bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `only_cross_attention` as `down_block_types`. \ + `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + num_attention_heads, + int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `num_attention_heads` as `down_block_types`. \ + `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + attention_head_dim, + int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `attention_head_dim` as `down_block_types`. \ + `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}.' + ) + + if isinstance( + cross_attention_dim, + list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `cross_attention_dim` as `down_block_types`. \ + `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}.' + ) + + if not isinstance( + layers_per_block, + int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f'Must provide the same number of `layers_per_block` as `down_block_types`. \ + `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}.' + ) + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=conv_in_kernel, + padding=conv_in_padding) + + # time + if time_embedding_type == 'fourier': + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError( + f'`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.' + ) + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, + set_W_to_weight=False, + log=False, + flip_sin_to_cos=flip_sin_to_cos) + timestep_input_dim = time_embed_dim + elif time_embedding_type == 'positional': + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, + freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f'{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`.' + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = 'text_proj' + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info( + "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined." + ) + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f'`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}.' + ) + + if encoder_hid_dim_type == 'text_proj': + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, + cross_attention_dim) + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, + time_embed_dim) + elif class_embed_type == 'timestep': + self.class_embedding = TimestepEmbedding( + timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == 'identity': + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == 'projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding( + projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == 'simple_projection': + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear( + projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == 'text': + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, + time_embed_dim, + num_heads=addition_embed_type_num_heads) + elif addition_embed_type is not None: + raise ValueError( + f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'." + ) + + if time_embedding_act_fn is None: + self.time_embed_act = None + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = only_cross_attention + + only_cross_attention = [only_cross_attention + ] * len(down_block_types) + + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = False + + if isinstance(num_attention_heads, int): + num_attention_heads = ( + num_attention_heads, ) * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim, ) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = ( + cross_attention_dim, ) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block + ] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] + if attention_head_dim[i] is not None else output_channel, + ) + self.down_blocks.append(down_block) + + # mid + if mid_block_type == 'UNetMidBlock2DCrossAttn': + self.mid_block = UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim[-1], + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + elif mid_block_type == 'UNetMidBlock2DSimpleCrossAttn': + self.mid_block = UNetMidBlock2DSimpleCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim[-1], + attention_head_dim=attention_head_dim[-1], + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type is None: + self.mid_block = None + else: + raise ValueError(f'unknown mid_block_type : {mid_block_type}') + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = list( + reversed(transformer_layers_per_block)) + only_cross_attention = list(reversed(only_cross_attention)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min( + i + 1, + len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block= + reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] + if attention_head_dim[i] is not None else output_channel, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], + num_groups=norm_num_groups, + eps=norm_eps) + + if act_fn == 'swish': + self.conv_act = lambda x: F.silu(x) + elif act_fn == 'mish': + self.conv_act = nn.Mish() + elif act_fn == 'silu': + self.conv_act = nn.SiLU() + elif act_fn == 'gelu': + self.conv_act = nn.GELU() + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], + out_channels, + kernel_size=conv_out_kernel, + padding=conv_out_padding) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, + processors: Dict[str, + AttentionProcessor]): + if hasattr(module, 'get_processor'): + processors[f'{name}.processor'] = module.get_processor( + return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f'{name}.{sub_name}', child, + processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, + Dict[str, + AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f'A dict of processors was passed, but the number of processors {len(processor)} does not match the' + f' number of attention layers: {count}. Please make sure to pass {count} processor classes.' + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, + processor): + if hasattr(module, 'set_processor'): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f'{name}.processor')) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f'{name}.{sub_name}', child, + processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + self.set_attn_processor(AttnProcessor()) + + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, 'set_attention_slice'): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == 'auto': + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == 'max': + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance( + slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f'You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different' + f' attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}.' + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError( + f'size {size} has to be smaller or equal to {dim}.') + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, + slice_size: List[int]): + if hasattr(module, 'set_attention_slice'): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, 'gradient_checkpointing'): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + The [`UNet2DConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + encoder_attention_mask (`torch.Tensor`): + A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If + `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + + Returns: + [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + forward_upsample_size = True + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = ( + 1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == 'mps' + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], + dtype=dtype, + device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError( + 'class_labels should be provided when num_class_embeds > 0' + ) + + if self.config.class_embed_type == 'timestep': + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to( + dtype=sample.dtype) + + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + if self.config.addition_embed_type == 'text': + aug_emb = self.add_embedding(encoder_hidden_states) + elif self.config.addition_embed_type == 'text_image': + # Kandinsky 2.1 - style + if 'image_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' \ + which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + + image_embs = added_cond_kwargs.get('image_embeds') + text_embs = added_cond_kwargs.get('text_embeds', + encoder_hidden_states) + aug_emb = self.add_embedding(text_embs, image_embs) + elif self.config.addition_embed_type == 'text_time': + # SDXL - style + if 'text_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' \ + which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get('text_embeds') + if 'time_ids' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' \ + which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get('time_ids') + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + elif self.config.addition_embed_type == 'image': + # Kandinsky 2.2 - style + if 'image_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image' \ + which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get('image_embeds') + aug_emb = self.add_embedding(image_embs) + elif self.config.addition_embed_type == 'image_hint': + # Kandinsky 2.2 - style + if 'image_embeds' not in added_cond_kwargs or 'hint' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which \ + requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get('image_embeds') + hint = added_cond_kwargs.get('hint') + aug_emb, hint = self.add_embedding(image_embs, hint) + sample = torch.cat([sample, hint], dim=1) + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == 'text_proj': + encoder_hidden_states = self.encoder_hid_proj( + encoder_hidden_states) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == 'text_image_proj': + # Kadinsky 2.1 - style + if 'image_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' \ + which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + + image_embeds = added_cond_kwargs.get('image_embeds') + encoder_hidden_states = self.encoder_hid_proj( + encoder_hidden_states, image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == 'image_proj': + # Kandinsky 2.2 - style + if 'image_embeds' not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' \ + which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get('image_embeds') + encoder_hidden_states = self.encoder_hid_proj(image_embeds) + # 2. pre-process + sample = self.conv_in(sample) + + # 2.5 GLIGEN position net + if cross_attention_kwargs is not None and cross_attention_kwargs.get( + 'gligen', None) is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + gligen_args = cross_attention_kwargs.pop('gligen') + cross_attention_kwargs['gligen'] = { + 'objs': self.position_net(**gligen_args) + } + + # 3. down + + is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None + is_adapter = mid_block_additional_residual is None and down_block_additional_residuals is not None + + down_block_res_samples = (sample, ) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, 'has_cross_attention' + ) and downsample_block.has_cross_attention: + # For t2i-adapter CrossAttnDownBlock2D + additional_residuals = {} + if is_adapter and len(down_block_additional_residuals) > 0: + additional_residuals[ + 'additional_residuals'] = down_block_additional_residuals.pop( + 0) + + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + **additional_residuals, + ) + else: + sample, res_samples = downsample_block( + hidden_states=sample, temb=emb) + + if is_adapter and len(down_block_additional_residuals) > 0: + sample += down_block_additional_residuals.pop(0) + + down_block_res_samples += res_samples + + if is_controlnet: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + ( + down_block_res_sample, ) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + # To support T2I-Adapter-XL + if all([ + is_adapter, + len(down_block_additional_residuals) > 0, + sample.shape == down_block_additional_residuals[0].shape + ]): + sample += down_block_additional_residuals.pop(0) + + if is_controlnet: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets):] + down_block_res_samples = down_block_res_samples[:-len( + upsample_block.resnets)] + + down_block_additional_residual = down_block_additional_residuals[ + -len(upsample_block.resnets):] + down_block_additional_residuals = down_block_additional_residuals[:-len( + upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, 'has_cross_attention' + ) and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + pixelwise_hidden_states=down_block_additional_residual, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + pixelwise_hidden_states=down_block_additional_residual, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if not return_dict: + return (sample, ) + + return UNet2DConditionOutput(sample=sample) + + @classmethod + def from_pretrained_orig(cls, + pretrained_model_path, + subfolder=None, + **kwargs): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, + subfolder) + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f'{config_file} does not exist') + with open(config_file, 'r') as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model = cls.from_config(config) + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + if not os.path.isfile(model_file): + raise RuntimeError(f'{model_file} does not exist') + state_dict = torch.load(model_file, map_location='cpu') + for k, v in model.state_dict().items(): + if 'attn2_plus' in k: + state_dict.update({k: v}) + model.load_state_dict(state_dict, strict=False) + + return model diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1af3e55f09e65120758ecdc237602d407c100bed --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from . import data, models, ops diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..22b9d22c52243e0c93c7fd02390470263dd8bd2b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .transforms import PadToSquare + +else: + _import_structure = { + 'transforms': ['PadToSquare'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) + +# from .transforms import * # noqa F403 diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/transforms.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..29a25b4b224fc83b66ce4049627d237d69975cce --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/data/transforms.py @@ -0,0 +1,122 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math +import random + +import torchvision.transforms.functional as TF +from PIL import Image, ImageFilter + +__all__ = [ + 'Identity', 'PadToSquare', 'RandomScale', 'RandomRotate', + 'RandomGaussianBlur', 'RandomCrop' +] + + +class Identity(object): + + def __call__(self, *args): + if len(args) == 0: + return None + elif len(args) == 1: + return args[0] + else: + return args + + +class PadToSquare(object): + + def __init__(self, fill=(255, 255, 255)): + self.fill = fill + + def __call__(self, img): + w, h = img.size + if w != h: + if w > h: + t = (w - h) // 2 + b = w - h - t + padding = (0, t, 0, b) + else: + left = (h - w) // 2 + right = h - w - l + padding = (left, 0, right, 0) + img = TF.pad(img, padding, fill=self.fill) + return img + + +class RandomScale(object): + + def __init__(self, + min_scale=0.5, + max_scale=2.0, + min_ratio=0.8, + max_ratio=1.25): + self.min_scale = min_scale + self.max_scale = max_scale + self.min_ratio = min_ratio + self.max_ratio = max_ratio + + def __call__(self, img): + w, h = img.size + scale = 2**random.uniform( + math.log2(self.min_scale), math.log2(self.max_scale)) + ratio = 2**random.uniform( + math.log2(self.min_ratio), math.log2(self.max_ratio)) + ow = int(w * scale * math.sqrt(ratio)) + oh = int(h * scale / math.sqrt(ratio)) + img = img.resize((ow, oh), Image.BILINEAR) + return img + + +class RandomRotate(object): + + def __init__(self, + min_angle=-10.0, + max_angle=10.0, + padding=(255, 255, 255), + p=0.5): + self.min_angle = min_angle + self.max_angle = max_angle + self.padding = padding + self.p = p + + def __call__(self, img): + if random.random() < self.p: + angle = random.uniform(self.min_angle, self.max_angle) + img = img.rotate(angle, Image.BILINEAR, fillcolor=self.padding) + return img + + +class RandomGaussianBlur(object): + + def __init__(self, radius=5, p=0.5): + self.radius = radius + self.p = p + + def __call__(self, img): + if random.random() < self.p: + img = img.filter(ImageFilter.GaussianBlur(radius=self.radius)) + return img + + +class RandomCrop(object): + + def __init__(self, size, padding=(255, 255, 255)): + self.size = size + self.padding = padding + + def __call__(self, img): + # pad + w, h = img.size + pad_w = max(0, self.size - w) + pad_h = max(0, self.size - h) + if pad_w > 0 or pad_h > 0: + half_w = pad_w // 2 + half_h = pad_h // 2 + pad = (half_w, half_h, pad_w - half_w, pad_h - half_h) + img = TF.pad(img, pad, fill=self.padding) + + # crop + w, h = img.size + x1 = random.randint(0, w - self.size) + y1 = random.randint(0, h - self.size) + img = img.crop((x1, y1, x1 + self.size, y1 + self.size)) + return img diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/model.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..94e5dd7bb4c7aa34129d92d2a2ab835d3983ab49 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/model.py @@ -0,0 +1,323 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['UNet'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, scale_factor=1.0): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.scale_factor = scale_factor + + def forward(self, x): + if self.scale_factor == 2.0: + x = F.interpolate(x, scale_factor=2, mode='nearest') + elif self.scale_factor == 0.5: + x = F.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, embed_dim, out_dim, dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.embedding = nn.Sequential(nn.SiLU(), + nn.Linear(embed_dim, out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, y): + identity = x + x = self.layer1(x) + x = x + self.embedding(y).unsqueeze(-1).unsqueeze(-1) + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class MultiHeadAttention(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=8, dropout=0.0): + assert dim % num_heads == 0 + assert context_dim is None or context_dim % num_heads == 0 + context_dim = context_dim or dim + super(MultiHeadAttention, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = math.pow(self.head_dim, -0.25) + + # layers + self.q = nn.Linear(dim, dim, bias=False) + self.k = nn.Linear(context_dim, dim, bias=False) + self.v = nn.Linear(context_dim, dim, bias=False) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, context=None): + # check inputs + context = x if context is None else context + b, n, c = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, c) + k = self.k(context).view(b, -1, n, c) + v = self.v(context).view(b, -1, n, c) + + # compute attention + attn = torch.einsum('binc,bjnc->bnij', q * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + attn = self.dropout(attn) + + # gather context + x = torch.einsum('bnij,bjnc->binc', attn, v) + x = x.reshape(b, -1, n * c) + + # output + x = self.o(x) + x = self.dropout(x) + return x + + +class GLU(nn.Module): + + def __init__(self, in_dim, out_dim): + super(GLU, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.proj = nn.Linear(in_dim, out_dim * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +class TransformerBlock(nn.Module): + + def __init__(self, dim, context_dim, num_heads, dropout=0.0): + super(TransformerBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + + # input + self.norm1 = nn.GroupNorm(32, dim, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d(dim, dim, 1) + + # self attention + self.norm2 = nn.LayerNorm(dim) + self.self_attn = MultiHeadAttention(dim, None, num_heads, dropout) + + # cross attention + self.norm3 = nn.LayerNorm(dim) + self.cross_attn = MultiHeadAttention(dim, context_dim, num_heads, + dropout) + + # ffn + self.norm4 = nn.LayerNorm(dim) + self.ffn = nn.Sequential( + GLU(dim, dim * 4), nn.Dropout(dropout), nn.Linear(dim * 4, dim)) + + # output + self.conv2 = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.conv2.weight) + + def forward(self, x, context): + b, c, h, w = x.size() + identity = x + + # input + x = self.norm1(x) + x = self.conv1(x).view(b, c, -1).transpose(1, 2) + + # attention + x = x + self.self_attn(self.norm2(x)) + x = x + self.cross_attn(self.norm3(x), context) + x = x + self.ffn(self.norm4(x)) + + # output + x = x.transpose(1, 2).view(b, c, h, w) + x = self.conv2(x) + return x + identity + + +class UNet(nn.Module): + + def __init__(self, + resolution=64, + in_dim=3, + dim=192, + label_dim=512, + context_dim=512, + out_dim=3, + dim_mult=[1, 2, 3, 5], + num_heads=1, + head_dim=None, + num_res_blocks=2, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + dropout=0.0): + embed_dim = dim * 4 + super(UNet, self).__init__() + self.resolution = resolution + self.in_dim = in_dim + self.dim = dim + self.context_dim = context_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.clip_embedding = nn.Sequential( + nn.Linear(label_dim, context_dim), nn.SiLU(), + nn.Linear(context_dim, context_dim)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList( + [ResidualBlock(in_dim, embed_dim, out_dim, dropout)]) + if scale in attn_scales: + block.append( + TransformerBlock(out_dim, context_dim, num_heads)) + in_dim = out_dim + self.encoder.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + self.encoder.append( + nn.Conv2d(out_dim, out_dim, 3, stride=2, padding=1)) + shortcut_dims.append(out_dim) + scale /= 2.0 + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, dropout), + TransformerBlock(out_dim, context_dim, num_heads), + ResidualBlock(out_dim, embed_dim, out_dim, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, dropout) + ]) + if scale in attn_scales: + block.append( + TransformerBlock(out_dim, context_dim, num_heads, + dropout)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + block.append( + nn.Sequential( + Resample(scale_factor=2.0), + nn.Conv2d(out_dim, out_dim, 3, padding=1))) + scale *= 2.0 + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y): + # embeddings + t = self.time_embedding(sinusoidal_embedding(t, self.dim)) + y = self.clip_embedding(y) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, t, y) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, t, y) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, t, y) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, t, y): + if isinstance(module, ResidualBlock): + x = module(x, t) + elif isinstance(module, TransformerBlock): + x = module(x, y) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, t, y) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e98421f2796b59acda627a44415581269f6b4c03 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .autoencoder import VQAutoencoder + from .clip import VisionTransformer + +else: + _import_structure = { + 'autoencoder': ['VQAutoencoder'], + 'clip': ['VisionTransformer'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/autoencoder.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..dce256f619912c6a6781268b9827fab54b1c428f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/autoencoder.py @@ -0,0 +1,413 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['VQAutoencoder', 'KLAutoencoder', 'PatchDiscriminator'] + + +def group_norm(dim): + return nn.GroupNorm(32, dim, eps=1e-6, affine=True) + + +class Resample(nn.Module): + + def __init__(self, dim, scale_factor): + super(Resample, self).__init__() + self.dim = dim + self.scale_factor = scale_factor + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(dim, dim, 3, padding=1)) + elif scale_factor == 0.5: + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=2, padding=0)) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + group_norm(in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1), group_norm(out_dim), + nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Conv2d(in_dim, out_dim, + 1) if in_dim != out_dim else nn.Identity() + + # zero out the last layer params + nn.init.zeros_(self.residual[-1].weight) + + def forward(self, x): + return self.residual(x) + self.shortcut(x) + + +class AttentionBlock(nn.Module): + + def __init__(self, dim): + super(AttentionBlock, self).__init__() + self.dim = dim + self.scale = math.pow(dim, -0.25) + + # layers + self.norm = group_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, h, w = x.size() + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, c * 3, -1).chunk(3, dim=1) + + # compute attention + attn = torch.einsum('bci,bcj->bij', q * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.einsum('bij,bcj->bci', attn, v) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class Encoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(Encoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # params + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = nn.Conv2d(3, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + downsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + downsamples.append(Resample(out_dim, scale_factor=0.5)) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout)) + + # output blocks + self.head = nn.Sequential( + group_norm(out_dim), nn.SiLU(), + nn.Conv2d(out_dim, z_dim, 3, padding=1)) + + def forward(self, x): + x = self.conv1(x) + x = self.downsamples(x) + x = self.middle(x) + x = self.head(x) + return x + + +class Decoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(Decoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # params + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + + # init block + self.conv1 = nn.Conv2d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(dims[0], dims[0], dropout), AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks + 1): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + upsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # upsample block + if i != len(dim_mult) - 1: + upsamples.append(Resample(out_dim, scale_factor=2.0)) + scale *= 2.0 + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential( + group_norm(out_dim), nn.SiLU(), + nn.Conv2d(out_dim, 3, 3, padding=1)) + + def forward(self, x): + x = self.conv1(x) + x = self.middle(x) + x = self.upsamples(x) + x = self.head(x) + return x + + +class VectorQuantizer(nn.Module): + + def __init__(self, codebook_size=8192, z_dim=3, beta=0.25): + super(VectorQuantizer, self).__init__() + self.codebook_size = codebook_size + self.z_dim = z_dim + self.beta = beta + + # init codebook + eps = math.sqrt(1.0 / codebook_size) + self.codebook = nn.Parameter( + torch.empty(codebook_size, z_dim).uniform_(-eps, eps)) + + def forward(self, z): + # preprocess + b, c, h, w = z.size() + flatten = z.permute(0, 2, 3, 1).reshape(-1, c) + + # quantization + with torch.no_grad(): + tokens = torch.cdist(flatten, self.codebook).argmin(dim=1) + quantized = F.embedding(tokens, + self.codebook).view(b, h, w, + c).permute(0, 3, 1, 2) + + # compute loss + codebook_loss = F.mse_loss(quantized, z.detach()) + commitment_loss = F.mse_loss(quantized.detach(), z) + loss = codebook_loss + self.beta * commitment_loss + + # perplexity + counts = F.one_hot(tokens, self.codebook_size).sum(dim=0).to(z.dtype) + # dist.all_reduce(counts) + p = counts / counts.sum() + perplexity = torch.exp(-torch.sum(p * torch.log(p + 1e-10))) + + # postprocess + tokens = tokens.view(b, h, w) + quantized = z + (quantized - z).detach() + return quantized, tokens, loss, perplexity + + +class VQAutoencoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0, + codebook_size=8192, + beta=0.25): + super(VQAutoencoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.codebook_size = codebook_size + self.beta = beta + + # blocks + self.encoder = Encoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + self.conv1 = nn.Conv2d(z_dim, z_dim, 1) + self.quantizer = VectorQuantizer(codebook_size, z_dim, beta) + self.conv2 = nn.Conv2d(z_dim, z_dim, 1) + self.decoder = Decoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + + def forward(self, x): + z = self.encoder(x) + z = self.conv1(z) + z, tokens, loss, perplexity = self.quantizer(z) + z = self.conv2(z) + x = self.decoder(z) + return x, tokens, loss, perplexity + + def encode(self, imgs): + z = self.encoder(imgs) + z = self.conv1(z) + return z + + def decode(self, z): + r"""Absort the quantizer in the decoder. + """ + z = self.quantizer(z)[0] + z = self.conv2(z) + imgs = self.decoder(z) + return imgs + + @torch.no_grad() + def encode_to_tokens(self, imgs): + # preprocess + z = self.encoder(imgs) + z = self.conv1(z) + + # quantization + b, c, h, w = z.size() + flatten = z.permute(0, 2, 3, 1).reshape(-1, c) + tokens = torch.cdist(flatten, self.quantizer.codebook).argmin(dim=1) + return tokens.view(b, -1) + + @torch.no_grad() + def decode_from_tokens(self, tokens): + # dequantization + z = F.embedding(tokens, self.quantizer.codebook) + + # postprocess + b, l, c = z.size() + h = w = int(math.sqrt(l)) + z = z.view(b, h, w, c).permute(0, 3, 1, 2) + z = self.conv2(z) + imgs = self.decoder(z) + return imgs + + +class KLAutoencoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(KLAutoencoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # blocks + self.encoder = Encoder(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, dropout) + self.conv1 = nn.Conv2d(z_dim * 2, z_dim * 2, 1) + self.conv2 = nn.Conv2d(z_dim, z_dim, 1) + self.decoder = Decoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + + def forward(self, x): + mu, log_var = self.encode(x) + z = self.reparameterize(mu, log_var) + x = self.decode(z) + return x, mu, log_var + + def encode(self, x): + x = self.encoder(x) + mu, log_var = self.conv1(x).chunk(2, dim=1) + return mu, log_var + + def decode(self, z): + x = self.conv2(z) + x = self.decoder(x) + return x + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + +class PatchDiscriminator(nn.Module): + + def __init__(self, in_dim=3, dim=64, num_layers=3): + super(PatchDiscriminator, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.num_layers = num_layers + + # params + dims = [dim * min(8, 2**u) for u in range(num_layers + 1)] + + # layers + layers = [ + nn.Conv2d(in_dim, dim, 4, stride=2, padding=1), + nn.LeakyReLU(0.2) + ] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + stride = 1 if i == num_layers - 1 else 2 + layers += [ + nn.Conv2d( + in_dim, out_dim, 4, stride=stride, padding=1, bias=False), + nn.BatchNorm2d(out_dim), + nn.LeakyReLU(0.2) + ] + layers += [nn.Conv2d(out_dim, 1, 4, stride=1, padding=1)] + self.layers = nn.Sequential(*layers) + + # initialize weights + self.apply(self.init_weights) + + def forward(self, x): + return self.layers(x) + + def init_weights(self, m): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, 0.0, 0.02) + elif isinstance(m, nn.BatchNorm2d): + nn.init.normal_(m.weight, 1.0, 0.02) + nn.init.zeros_(m.bias) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/clip.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..8cd5e592e970e8f216f0dd189951cc66630b4b3f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/models/clip.py @@ -0,0 +1,420 @@ +# Part of the implementation is borrowed and modified from CLIP, publicly available at https://github.com/openai/CLIP. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import modelscope.models.cv.image_to_image_translation.ops as ops # for using differentiable all_gather + +__all__ = [ + 'CLIP', 'clip_vit_b_32', 'clip_vit_b_16', 'clip_vit_l_14', + 'clip_vit_l_14_336px', 'clip_vit_h_16' +] + + +def to_fp16(m): + if isinstance(m, (nn.Linear, nn.Conv2d)): + m.weight.data = m.weight.data.half() + if m.bias is not None: + m.bias.data = m.bias.data.half() + elif hasattr(m, 'head'): + p = getattr(m, 'head') + p.data = p.data.half() + + +class QuickGELU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class LayerNorm(nn.LayerNorm): + r"""Subclass of nn.LayerNorm to handle fp16. + """ + + def forward(self, x): + return super(LayerNorm, self).forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.attn_dropout = nn.Dropout(attn_dropout) + self.proj = nn.Linear(dim, dim) + self.proj_dropout = nn.Dropout(proj_dropout) + + def forward(self, x, mask=None): + r"""x: [B, L, C]. + mask: [*, L, L]. + """ + b, l, _, n = *x.size(), self.num_heads + + # compute query, key, and value + q, k, v = self.to_qkv(x.transpose(0, 1)).chunk(3, dim=-1) + q = q.reshape(l, b * n, -1).transpose(0, 1) + k = k.reshape(l, b * n, -1).transpose(0, 1) + v = v.reshape(l, b * n, -1).transpose(0, 1) + + # compute attention + attn = self.scale * torch.bmm(q, k.transpose(1, 2)) + if mask is not None: + attn = attn.masked_fill(mask[:, :l, :l] == 0, float('-inf')) + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + attn = self.attn_dropout(attn) + + # gather context + x = torch.bmm(attn, v) + x = x.view(b, n, l, -1).transpose(1, 2).reshape(b, l, -1) + + # output + x = self.proj(x) + x = self.proj_dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads, attn_dropout, proj_dropout) + self.norm2 = LayerNorm(dim) + self.mlp = nn.Sequential( + nn.Linear(dim, dim * 4), QuickGELU(), nn.Linear(dim * 4, dim), + nn.Dropout(proj_dropout)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.mlp(self.norm2(x)) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=224, + patch_size=16, + dim=768, + out_dim=512, + num_heads=12, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + assert image_size % patch_size == 0 + super(VisionTransformer, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + gain = 1.0 / math.sqrt(dim) + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.pos_embedding = nn.Parameter( + gain * torch.randn(1, self.num_patches + 1, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.pre_norm = LayerNorm(dim) + self.transformer = nn.Sequential(*[ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.post_norm = LayerNorm(dim) + + # head + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + def forward(self, x): + b, dtype = x.size(0), self.head.dtype + x = x.type(dtype) + + # patch-embedding + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) # [b, n, c] + x = torch.cat([self.cls_embedding.repeat(b, 1, 1).type(dtype), x], + dim=1) + x = self.dropout(x + self.pos_embedding.type(dtype)) + x = self.pre_norm(x) + + # transformer + x = self.transformer(x) + + # head + x = self.post_norm(x) + x = torch.mm(x[:, 0, :], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class TextTransformer(nn.Module): + + def __init__(self, + vocab_size, + text_len, + dim=512, + out_dim=512, + num_heads=8, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(TextTransformer, self).__init__() + self.vocab_size = vocab_size + self.text_len = text_len + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + + # embeddings + self.token_embedding = nn.Embedding(vocab_size, dim) + self.pos_embedding = nn.Parameter(0.01 * torch.randn(1, text_len, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.transformer = nn.ModuleList([ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.norm = LayerNorm(dim) + + # head + gain = 1.0 / math.sqrt(dim) + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + # causal attention mask + self.register_buffer('attn_mask', + torch.tril(torch.ones(1, text_len, text_len))) + + def forward(self, x): + eot, dtype = x.argmax(dim=-1), self.head.dtype + + # embeddings + x = self.dropout( + self.token_embedding(x).type(dtype) + + self.pos_embedding.type(dtype)) + + # transformer + for block in self.transformer: + x = block(x, self.attn_mask) + + # head + x = self.norm(x) + x = torch.mm(x[torch.arange(x.size(0)), eot], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class CLIP(nn.Module): + + def __init__(self, + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(CLIP, self).__init__() + self.embed_dim = embed_dim + self.image_size = image_size + self.patch_size = patch_size + self.vision_dim = vision_dim + self.vision_heads = vision_heads + self.vision_layers = vision_layers + self.vocab_size = vocab_size + self.text_len = text_len + self.text_dim = text_dim + self.text_heads = text_heads + self.text_layers = text_layers + + # models + self.visual = VisionTransformer( + image_size=image_size, + patch_size=patch_size, + dim=vision_dim, + out_dim=embed_dim, + num_heads=vision_heads, + num_layers=vision_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.textual = TextTransformer( + vocab_size=vocab_size, + text_len=text_len, + dim=text_dim, + out_dim=embed_dim, + num_heads=text_heads, + num_layers=text_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.log_scale = nn.Parameter(math.log(1 / 0.07) * torch.ones([])) + + def forward(self, imgs, txt_tokens): + r"""imgs: [B, C, H, W] of torch.float32. + txt_tokens: [B, T] of torch.long. + """ + xi = self.visual(imgs) + xt = self.textual(txt_tokens) + + # normalize features + xi = F.normalize(xi, p=2, dim=1) + xt = F.normalize(xt, p=2, dim=1) + + # gather features from all ranks + full_xi = ops.diff_all_gather(xi) + full_xt = ops.diff_all_gather(xt) + + # logits + scale = self.log_scale.exp() + logits_i2t = scale * torch.mm(xi, full_xt.t()) + logits_t2i = scale * torch.mm(xt, full_xi.t()) + + # labels + labels = torch.arange( + len(xi) * ops.get_rank(), + len(xi) * (ops.get_rank() + 1), + dtype=torch.long, + device=xi.device) + return logits_i2t, logits_t2i, labels + + def init_weights(self): + # embeddings + nn.init.normal_(self.textual.token_embedding.weight, std=0.02) + nn.init.normal_(self.visual.patch_embedding.weight, tsd=0.1) + + # attentions + for modality in ['visual', 'textual']: + dim = self.vision_dim if modality == 'visual' else 'textual' + transformer = getattr(self, modality).transformer + proj_gain = (1.0 / math.sqrt(dim)) * ( + 1.0 / math.sqrt(2 * transformer.num_layers)) + attn_gain = 1.0 / math.sqrt(dim) + mlp_gain = 1.0 / math.sqrt(2.0 * dim) + for block in transformer.layers: + nn.init.normal_(block.attn.to_qkv.weight, std=attn_gain) + nn.init.normal_(block.attn.proj.weight, std=proj_gain) + nn.init.normal_(block.mlp[0].weight, std=mlp_gain) + nn.init.normal_(block.mlp[2].weight, std=proj_gain) + + def param_groups(self): + groups = [{ + 'params': [ + p for n, p in self.named_parameters() + if 'norm' in n or n.endswith('bias') + ], + 'weight_decay': + 0.0 + }, { + 'params': [ + p for n, p in self.named_parameters() + if not ('norm' in n or n.endswith('bias')) + ] + }] + return groups + + def fp16(self): + return self.apply(to_fp16) + + +def clip_vit_b_32(**kwargs): + return CLIP( + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + text_dim=512, + text_heads=8, + text_layers=12, + **kwargs) + + +def clip_vit_b_16(**kwargs): + return CLIP( + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + text_dim=512, + text_heads=8, + text_layers=12, + **kwargs) + + +def clip_vit_l_14(**kwargs): + return CLIP( + embed_dim=768, + image_size=224, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + text_dim=768, + text_heads=12, + text_layers=12, + **kwargs) + + +def clip_vit_l_14_336px(**kwargs): + return CLIP( + embed_dim=768, + image_size=336, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + text_dim=768, + text_heads=12, + text_layers=12, + **kwargs) + + +def clip_vit_h_16(**kwargs): + return CLIP( + embed_dim=1024, + image_size=256, + patch_size=16, + vision_dim=1280, + vision_heads=16, + vision_layers=32, + text_dim=1024, + text_heads=16, + text_layers=24, + **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e3dac584b56b9ca471a3c255aea42b3d47bdc172 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .diffusion import GaussianDiffusion, beta_schedule + +else: + _import_structure = { + 'diffusion': ['GaussianDiffusion', 'beta_schedule'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/diffusion.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..b8ffbbbb75db6b059c8818fbef37476d96a49e38 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/diffusion.py @@ -0,0 +1,599 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch + +from .losses import discretized_gaussian_log_likelihood, kl_divergence + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + + # fn = lambda u: math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + def fn(u): + return math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + + betas.append(min(1.0 - fn(t2) / fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + noise = torch.randn_like(x0) if noise is None else noise + return _i(self.sqrt_alphas_cumprod, t, x0) * x0 + _i( + self.sqrt_one_minus_alphas_cumprod, t, x0) * noise + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + # no noise when t == 0 + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + assert self.mean_type == 'eps' + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + out = torch.cat( + [ + u_out[:, :3] + guide_scale * # noqa W504 + (y_out[:, :3] - u_out[:, :3]), + y_out[:, 3:] + ], + dim=1) + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - _i( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + xt) * xt + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + sigmas = eta * torch.sqrt((1 - alphas_prev) / # noqa W504 + (1 - alphas) * # noqa W504 + (1 - alphas / alphas_prev)) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b, c, h, w = x0.size() + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + - x0) / _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + # mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def loss(self, x0, t, model, model_kwargs={}, noise=None): + noise = torch.randn_like(x0) if noise is None else noise + xt = self.q_sample(x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', 'rescaled_l1']: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = (out - target).pow(1 if self.loss_type.endswith('l1') else 2 + ).abs().flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) + kl = kl.flatten(1).mean(dim=1) / math.log(2.0) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) + nll = nll.flatten(1).mean(dim=1) / math.log(2.0) + + # NLL for p(x0 | x1) and KL otherwise + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + r"""Compute the entire variational lower bound, measured in bits-per-dim. + """ + # prepare input and output + b, c, h, w = x0.size() + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/losses.py b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..46b9540afa35342db1ad696980150d5a8ed0b352 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_generation/ops/losses.py @@ -0,0 +1,36 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch + +__all__ = ['kl_divergence', 'discretized_gaussian_log_likelihood'] + + +def kl_divergence(mu1, logvar1, mu2, logvar2): + return 0.5 * ( + -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + # noqa W504 + ((mu1 - mu2)**2) * torch.exp(-logvar2)) + + +def standard_normal_cdf(x): + r"""A fast approximation of the cumulative distribution function of the standard normal. + """ + return 0.5 * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x0, mean, log_scale): + assert x0.shape == mean.shape == log_scale.shape + cx = x0 - mean + inv_stdv = torch.exp(-log_scale) + cdf_plus = standard_normal_cdf(inv_stdv * (cx + 1.0 / 255.0)) + cdf_min = standard_normal_cdf(inv_stdv * (cx - 1.0 / 255.0)) + log_cdf_plus = torch.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = torch.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = torch.where( + x0 < -0.999, log_cdf_plus, + torch.where(x0 > 0.999, log_one_minus_cdf_min, + torch.log(cdf_delta.clamp(min=1e-12)))) + assert log_probs.shape == x0.shape + return log_probs diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35aab6be14f94025852c2e7d92a7e85ff5c68a79 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model_translation import UNet + +else: + _import_structure = { + 'image_to_image_translation_model': ['UNet'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..724bca040407c59305d6fa3f9b3dd114adadf782 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .transforms import * # noqa F403 diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/transforms.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..29a25b4b224fc83b66ce4049627d237d69975cce --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/data/transforms.py @@ -0,0 +1,122 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math +import random + +import torchvision.transforms.functional as TF +from PIL import Image, ImageFilter + +__all__ = [ + 'Identity', 'PadToSquare', 'RandomScale', 'RandomRotate', + 'RandomGaussianBlur', 'RandomCrop' +] + + +class Identity(object): + + def __call__(self, *args): + if len(args) == 0: + return None + elif len(args) == 1: + return args[0] + else: + return args + + +class PadToSquare(object): + + def __init__(self, fill=(255, 255, 255)): + self.fill = fill + + def __call__(self, img): + w, h = img.size + if w != h: + if w > h: + t = (w - h) // 2 + b = w - h - t + padding = (0, t, 0, b) + else: + left = (h - w) // 2 + right = h - w - l + padding = (left, 0, right, 0) + img = TF.pad(img, padding, fill=self.fill) + return img + + +class RandomScale(object): + + def __init__(self, + min_scale=0.5, + max_scale=2.0, + min_ratio=0.8, + max_ratio=1.25): + self.min_scale = min_scale + self.max_scale = max_scale + self.min_ratio = min_ratio + self.max_ratio = max_ratio + + def __call__(self, img): + w, h = img.size + scale = 2**random.uniform( + math.log2(self.min_scale), math.log2(self.max_scale)) + ratio = 2**random.uniform( + math.log2(self.min_ratio), math.log2(self.max_ratio)) + ow = int(w * scale * math.sqrt(ratio)) + oh = int(h * scale / math.sqrt(ratio)) + img = img.resize((ow, oh), Image.BILINEAR) + return img + + +class RandomRotate(object): + + def __init__(self, + min_angle=-10.0, + max_angle=10.0, + padding=(255, 255, 255), + p=0.5): + self.min_angle = min_angle + self.max_angle = max_angle + self.padding = padding + self.p = p + + def __call__(self, img): + if random.random() < self.p: + angle = random.uniform(self.min_angle, self.max_angle) + img = img.rotate(angle, Image.BILINEAR, fillcolor=self.padding) + return img + + +class RandomGaussianBlur(object): + + def __init__(self, radius=5, p=0.5): + self.radius = radius + self.p = p + + def __call__(self, img): + if random.random() < self.p: + img = img.filter(ImageFilter.GaussianBlur(radius=self.radius)) + return img + + +class RandomCrop(object): + + def __init__(self, size, padding=(255, 255, 255)): + self.size = size + self.padding = padding + + def __call__(self, img): + # pad + w, h = img.size + pad_w = max(0, self.size - w) + pad_h = max(0, self.size - h) + if pad_w > 0 or pad_h > 0: + half_w = pad_w // 2 + half_h = pad_h // 2 + pad = (half_w, half_h, pad_w - half_w, pad_h - half_h) + img = TF.pad(img, pad, fill=self.padding) + + # crop + w, h = img.size + x1 = random.randint(0, w - self.size) + y1 = random.randint(0, h - self.size) + img = img.crop((x1, y1, x1 + self.size, y1 + self.size)) + return img diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/model_translation.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/model_translation.py new file mode 100644 index 0000000000000000000000000000000000000000..f2a9e7db2423301da80f53beb9e04d6d1452d110 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/model_translation.py @@ -0,0 +1,324 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['UNet'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, scale_factor=1.0): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.scale_factor = scale_factor + + def forward(self, x): + if self.scale_factor == 2.0: + x = F.interpolate(x, scale_factor=2, mode='nearest') + elif self.scale_factor == 0.5: + x = F.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, embed_dim, out_dim, dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.embedding = nn.Sequential(nn.SiLU(), + nn.Linear(embed_dim, out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, y): + identity = x + x = self.layer1(x) + x = x + self.embedding(y).unsqueeze(-1).unsqueeze(-1) + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class MultiHeadAttention(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=8, dropout=0.0): + assert dim % num_heads == 0 + assert context_dim is None or context_dim % num_heads == 0 + context_dim = context_dim or dim + super(MultiHeadAttention, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = math.pow(self.head_dim, -0.25) + + # layers + self.q = nn.Linear(dim, dim, bias=False) + self.k = nn.Linear(context_dim, dim, bias=False) + self.v = nn.Linear(context_dim, dim, bias=False) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, context=None): + # check inputs + context = x if context is None else context + b, n, c = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, c) + k = self.k(context).view(b, -1, n, c) + v = self.v(context).view(b, -1, n, c) + + # compute attention + attn = torch.einsum('binc,bjnc->bnij', q * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + attn = self.dropout(attn) + + # gather context + x = torch.einsum('bnij,bjnc->binc', attn, v) + x = x.reshape(b, -1, n * c) + + # output + x = self.o(x) + x = self.dropout(x) + return x + + +class GLU(nn.Module): + + def __init__(self, in_dim, out_dim): + super(GLU, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.proj = nn.Linear(in_dim, out_dim * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +class TransformerBlock(nn.Module): + + def __init__(self, dim, context_dim, num_heads, dropout=0.0): + super(TransformerBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + + # input + self.norm1 = nn.GroupNorm(32, dim, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d(dim, dim, 1) + + # self attention + self.norm2 = nn.LayerNorm(dim) + self.self_attn = MultiHeadAttention(dim, None, num_heads, dropout) + + # cross attention + self.norm3 = nn.LayerNorm(dim) + self.cross_attn = MultiHeadAttention(dim, context_dim, num_heads, + dropout) + + # ffn + self.norm4 = nn.LayerNorm(dim) + self.ffn = nn.Sequential( + GLU(dim, dim * 4), nn.Dropout(dropout), nn.Linear(dim * 4, dim)) + + # output + self.conv2 = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.conv2.weight) + + def forward(self, x, context): + b, c, h, w = x.size() + identity = x + + # input + x = self.norm1(x) + x = self.conv1(x).view(b, c, -1).transpose(1, 2) + + # attention + x = x + self.self_attn(self.norm2(x)) + x = x + self.cross_attn(self.norm3(x), context) + x = x + self.ffn(self.norm4(x)) + + # output + x = x.transpose(1, 2).view(b, c, h, w) + x = self.conv2(x) + return x + identity + + +class UNet(nn.Module): + + def __init__(self, + resolution=64, + in_dim=3, + dim=192, + context_dim=512, + out_dim=3, + dim_mult=[1, 2, 3, 5], + num_heads=1, + head_dim=None, + num_res_blocks=2, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + num_classes=1001, + dropout=0.0): + embed_dim = dim * 4 + super(UNet, self).__init__() + self.resolution = resolution + self.in_dim = in_dim + self.dim = dim + self.context_dim = context_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.num_classes = num_classes + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.label_embedding = nn.Embedding(num_classes, context_dim) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList( + [ResidualBlock(in_dim, embed_dim, out_dim, dropout)]) + if scale in attn_scales: + block.append( + TransformerBlock(out_dim, context_dim, num_heads)) + in_dim = out_dim + self.encoder.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + self.encoder.append( + nn.Conv2d(out_dim, out_dim, 3, stride=2, padding=1)) + shortcut_dims.append(out_dim) + scale /= 2.0 + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, dropout), + TransformerBlock(out_dim, context_dim, num_heads), + ResidualBlock(out_dim, embed_dim, out_dim, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, dropout) + ]) + if scale in attn_scales: + block.append( + TransformerBlock(out_dim, context_dim, num_heads, + dropout)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + block.append( + nn.Sequential( + Resample(scale_factor=2.0), + nn.Conv2d(out_dim, out_dim, 3, padding=1))) + scale *= 2.0 + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y, concat=None): + # embeddings + if concat is not None: + x = torch.cat([x, concat], dim=1) + t = self.time_embedding(sinusoidal_embedding(t, self.dim)) + y = self.label_embedding(y) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, t, y) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, t, y) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, t, y) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, t, y): + if isinstance(module, ResidualBlock): + x = module(x, t) + elif isinstance(module, TransformerBlock): + x = module(x, y) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, t, y) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7fdd81891268985f9e8b967cc4be68a6335372c4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .autoencoder import * # noqa F403 +from .clip import * # noqa F403 diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/autoencoder.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..dce256f619912c6a6781268b9827fab54b1c428f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/autoencoder.py @@ -0,0 +1,413 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['VQAutoencoder', 'KLAutoencoder', 'PatchDiscriminator'] + + +def group_norm(dim): + return nn.GroupNorm(32, dim, eps=1e-6, affine=True) + + +class Resample(nn.Module): + + def __init__(self, dim, scale_factor): + super(Resample, self).__init__() + self.dim = dim + self.scale_factor = scale_factor + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(dim, dim, 3, padding=1)) + elif scale_factor == 0.5: + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=2, padding=0)) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + group_norm(in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1), group_norm(out_dim), + nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Conv2d(in_dim, out_dim, + 1) if in_dim != out_dim else nn.Identity() + + # zero out the last layer params + nn.init.zeros_(self.residual[-1].weight) + + def forward(self, x): + return self.residual(x) + self.shortcut(x) + + +class AttentionBlock(nn.Module): + + def __init__(self, dim): + super(AttentionBlock, self).__init__() + self.dim = dim + self.scale = math.pow(dim, -0.25) + + # layers + self.norm = group_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, h, w = x.size() + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, c * 3, -1).chunk(3, dim=1) + + # compute attention + attn = torch.einsum('bci,bcj->bij', q * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.einsum('bij,bcj->bci', attn, v) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class Encoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(Encoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # params + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = nn.Conv2d(3, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + downsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + downsamples.append(Resample(out_dim, scale_factor=0.5)) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout)) + + # output blocks + self.head = nn.Sequential( + group_norm(out_dim), nn.SiLU(), + nn.Conv2d(out_dim, z_dim, 3, padding=1)) + + def forward(self, x): + x = self.conv1(x) + x = self.downsamples(x) + x = self.middle(x) + x = self.head(x) + return x + + +class Decoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(Decoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # params + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + + # init block + self.conv1 = nn.Conv2d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(dims[0], dims[0], dropout), AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks + 1): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + upsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # upsample block + if i != len(dim_mult) - 1: + upsamples.append(Resample(out_dim, scale_factor=2.0)) + scale *= 2.0 + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential( + group_norm(out_dim), nn.SiLU(), + nn.Conv2d(out_dim, 3, 3, padding=1)) + + def forward(self, x): + x = self.conv1(x) + x = self.middle(x) + x = self.upsamples(x) + x = self.head(x) + return x + + +class VectorQuantizer(nn.Module): + + def __init__(self, codebook_size=8192, z_dim=3, beta=0.25): + super(VectorQuantizer, self).__init__() + self.codebook_size = codebook_size + self.z_dim = z_dim + self.beta = beta + + # init codebook + eps = math.sqrt(1.0 / codebook_size) + self.codebook = nn.Parameter( + torch.empty(codebook_size, z_dim).uniform_(-eps, eps)) + + def forward(self, z): + # preprocess + b, c, h, w = z.size() + flatten = z.permute(0, 2, 3, 1).reshape(-1, c) + + # quantization + with torch.no_grad(): + tokens = torch.cdist(flatten, self.codebook).argmin(dim=1) + quantized = F.embedding(tokens, + self.codebook).view(b, h, w, + c).permute(0, 3, 1, 2) + + # compute loss + codebook_loss = F.mse_loss(quantized, z.detach()) + commitment_loss = F.mse_loss(quantized.detach(), z) + loss = codebook_loss + self.beta * commitment_loss + + # perplexity + counts = F.one_hot(tokens, self.codebook_size).sum(dim=0).to(z.dtype) + # dist.all_reduce(counts) + p = counts / counts.sum() + perplexity = torch.exp(-torch.sum(p * torch.log(p + 1e-10))) + + # postprocess + tokens = tokens.view(b, h, w) + quantized = z + (quantized - z).detach() + return quantized, tokens, loss, perplexity + + +class VQAutoencoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=3, + dim_mult=[1, 2, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0, + codebook_size=8192, + beta=0.25): + super(VQAutoencoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.codebook_size = codebook_size + self.beta = beta + + # blocks + self.encoder = Encoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + self.conv1 = nn.Conv2d(z_dim, z_dim, 1) + self.quantizer = VectorQuantizer(codebook_size, z_dim, beta) + self.conv2 = nn.Conv2d(z_dim, z_dim, 1) + self.decoder = Decoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + + def forward(self, x): + z = self.encoder(x) + z = self.conv1(z) + z, tokens, loss, perplexity = self.quantizer(z) + z = self.conv2(z) + x = self.decoder(z) + return x, tokens, loss, perplexity + + def encode(self, imgs): + z = self.encoder(imgs) + z = self.conv1(z) + return z + + def decode(self, z): + r"""Absort the quantizer in the decoder. + """ + z = self.quantizer(z)[0] + z = self.conv2(z) + imgs = self.decoder(z) + return imgs + + @torch.no_grad() + def encode_to_tokens(self, imgs): + # preprocess + z = self.encoder(imgs) + z = self.conv1(z) + + # quantization + b, c, h, w = z.size() + flatten = z.permute(0, 2, 3, 1).reshape(-1, c) + tokens = torch.cdist(flatten, self.quantizer.codebook).argmin(dim=1) + return tokens.view(b, -1) + + @torch.no_grad() + def decode_from_tokens(self, tokens): + # dequantization + z = F.embedding(tokens, self.quantizer.codebook) + + # postprocess + b, l, c = z.size() + h = w = int(math.sqrt(l)) + z = z.view(b, h, w, c).permute(0, 3, 1, 2) + z = self.conv2(z) + imgs = self.decoder(z) + return imgs + + +class KLAutoencoder(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + dropout=0.0): + super(KLAutoencoder, self).__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + + # blocks + self.encoder = Encoder(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, dropout) + self.conv1 = nn.Conv2d(z_dim * 2, z_dim * 2, 1) + self.conv2 = nn.Conv2d(z_dim, z_dim, 1) + self.decoder = Decoder(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, dropout) + + def forward(self, x): + mu, log_var = self.encode(x) + z = self.reparameterize(mu, log_var) + x = self.decode(z) + return x, mu, log_var + + def encode(self, x): + x = self.encoder(x) + mu, log_var = self.conv1(x).chunk(2, dim=1) + return mu, log_var + + def decode(self, z): + x = self.conv2(z) + x = self.decoder(x) + return x + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + +class PatchDiscriminator(nn.Module): + + def __init__(self, in_dim=3, dim=64, num_layers=3): + super(PatchDiscriminator, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.num_layers = num_layers + + # params + dims = [dim * min(8, 2**u) for u in range(num_layers + 1)] + + # layers + layers = [ + nn.Conv2d(in_dim, dim, 4, stride=2, padding=1), + nn.LeakyReLU(0.2) + ] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + stride = 1 if i == num_layers - 1 else 2 + layers += [ + nn.Conv2d( + in_dim, out_dim, 4, stride=stride, padding=1, bias=False), + nn.BatchNorm2d(out_dim), + nn.LeakyReLU(0.2) + ] + layers += [nn.Conv2d(out_dim, 1, 4, stride=1, padding=1)] + self.layers = nn.Sequential(*layers) + + # initialize weights + self.apply(self.init_weights) + + def forward(self, x): + return self.layers(x) + + def init_weights(self, m): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, 0.0, 0.02) + elif isinstance(m, nn.BatchNorm2d): + nn.init.normal_(m.weight, 1.0, 0.02) + nn.init.zeros_(m.bias) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/clip.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..8cd5e592e970e8f216f0dd189951cc66630b4b3f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/models/clip.py @@ -0,0 +1,420 @@ +# Part of the implementation is borrowed and modified from CLIP, publicly available at https://github.com/openai/CLIP. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import modelscope.models.cv.image_to_image_translation.ops as ops # for using differentiable all_gather + +__all__ = [ + 'CLIP', 'clip_vit_b_32', 'clip_vit_b_16', 'clip_vit_l_14', + 'clip_vit_l_14_336px', 'clip_vit_h_16' +] + + +def to_fp16(m): + if isinstance(m, (nn.Linear, nn.Conv2d)): + m.weight.data = m.weight.data.half() + if m.bias is not None: + m.bias.data = m.bias.data.half() + elif hasattr(m, 'head'): + p = getattr(m, 'head') + p.data = p.data.half() + + +class QuickGELU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class LayerNorm(nn.LayerNorm): + r"""Subclass of nn.LayerNorm to handle fp16. + """ + + def forward(self, x): + return super(LayerNorm, self).forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.attn_dropout = nn.Dropout(attn_dropout) + self.proj = nn.Linear(dim, dim) + self.proj_dropout = nn.Dropout(proj_dropout) + + def forward(self, x, mask=None): + r"""x: [B, L, C]. + mask: [*, L, L]. + """ + b, l, _, n = *x.size(), self.num_heads + + # compute query, key, and value + q, k, v = self.to_qkv(x.transpose(0, 1)).chunk(3, dim=-1) + q = q.reshape(l, b * n, -1).transpose(0, 1) + k = k.reshape(l, b * n, -1).transpose(0, 1) + v = v.reshape(l, b * n, -1).transpose(0, 1) + + # compute attention + attn = self.scale * torch.bmm(q, k.transpose(1, 2)) + if mask is not None: + attn = attn.masked_fill(mask[:, :l, :l] == 0, float('-inf')) + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + attn = self.attn_dropout(attn) + + # gather context + x = torch.bmm(attn, v) + x = x.view(b, n, l, -1).transpose(1, 2).reshape(b, l, -1) + + # output + x = self.proj(x) + x = self.proj_dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads, attn_dropout, proj_dropout) + self.norm2 = LayerNorm(dim) + self.mlp = nn.Sequential( + nn.Linear(dim, dim * 4), QuickGELU(), nn.Linear(dim * 4, dim), + nn.Dropout(proj_dropout)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.mlp(self.norm2(x)) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=224, + patch_size=16, + dim=768, + out_dim=512, + num_heads=12, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + assert image_size % patch_size == 0 + super(VisionTransformer, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + gain = 1.0 / math.sqrt(dim) + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.pos_embedding = nn.Parameter( + gain * torch.randn(1, self.num_patches + 1, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.pre_norm = LayerNorm(dim) + self.transformer = nn.Sequential(*[ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.post_norm = LayerNorm(dim) + + # head + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + def forward(self, x): + b, dtype = x.size(0), self.head.dtype + x = x.type(dtype) + + # patch-embedding + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) # [b, n, c] + x = torch.cat([self.cls_embedding.repeat(b, 1, 1).type(dtype), x], + dim=1) + x = self.dropout(x + self.pos_embedding.type(dtype)) + x = self.pre_norm(x) + + # transformer + x = self.transformer(x) + + # head + x = self.post_norm(x) + x = torch.mm(x[:, 0, :], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class TextTransformer(nn.Module): + + def __init__(self, + vocab_size, + text_len, + dim=512, + out_dim=512, + num_heads=8, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(TextTransformer, self).__init__() + self.vocab_size = vocab_size + self.text_len = text_len + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + + # embeddings + self.token_embedding = nn.Embedding(vocab_size, dim) + self.pos_embedding = nn.Parameter(0.01 * torch.randn(1, text_len, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.transformer = nn.ModuleList([ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.norm = LayerNorm(dim) + + # head + gain = 1.0 / math.sqrt(dim) + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + # causal attention mask + self.register_buffer('attn_mask', + torch.tril(torch.ones(1, text_len, text_len))) + + def forward(self, x): + eot, dtype = x.argmax(dim=-1), self.head.dtype + + # embeddings + x = self.dropout( + self.token_embedding(x).type(dtype) + + self.pos_embedding.type(dtype)) + + # transformer + for block in self.transformer: + x = block(x, self.attn_mask) + + # head + x = self.norm(x) + x = torch.mm(x[torch.arange(x.size(0)), eot], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class CLIP(nn.Module): + + def __init__(self, + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(CLIP, self).__init__() + self.embed_dim = embed_dim + self.image_size = image_size + self.patch_size = patch_size + self.vision_dim = vision_dim + self.vision_heads = vision_heads + self.vision_layers = vision_layers + self.vocab_size = vocab_size + self.text_len = text_len + self.text_dim = text_dim + self.text_heads = text_heads + self.text_layers = text_layers + + # models + self.visual = VisionTransformer( + image_size=image_size, + patch_size=patch_size, + dim=vision_dim, + out_dim=embed_dim, + num_heads=vision_heads, + num_layers=vision_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.textual = TextTransformer( + vocab_size=vocab_size, + text_len=text_len, + dim=text_dim, + out_dim=embed_dim, + num_heads=text_heads, + num_layers=text_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.log_scale = nn.Parameter(math.log(1 / 0.07) * torch.ones([])) + + def forward(self, imgs, txt_tokens): + r"""imgs: [B, C, H, W] of torch.float32. + txt_tokens: [B, T] of torch.long. + """ + xi = self.visual(imgs) + xt = self.textual(txt_tokens) + + # normalize features + xi = F.normalize(xi, p=2, dim=1) + xt = F.normalize(xt, p=2, dim=1) + + # gather features from all ranks + full_xi = ops.diff_all_gather(xi) + full_xt = ops.diff_all_gather(xt) + + # logits + scale = self.log_scale.exp() + logits_i2t = scale * torch.mm(xi, full_xt.t()) + logits_t2i = scale * torch.mm(xt, full_xi.t()) + + # labels + labels = torch.arange( + len(xi) * ops.get_rank(), + len(xi) * (ops.get_rank() + 1), + dtype=torch.long, + device=xi.device) + return logits_i2t, logits_t2i, labels + + def init_weights(self): + # embeddings + nn.init.normal_(self.textual.token_embedding.weight, std=0.02) + nn.init.normal_(self.visual.patch_embedding.weight, tsd=0.1) + + # attentions + for modality in ['visual', 'textual']: + dim = self.vision_dim if modality == 'visual' else 'textual' + transformer = getattr(self, modality).transformer + proj_gain = (1.0 / math.sqrt(dim)) * ( + 1.0 / math.sqrt(2 * transformer.num_layers)) + attn_gain = 1.0 / math.sqrt(dim) + mlp_gain = 1.0 / math.sqrt(2.0 * dim) + for block in transformer.layers: + nn.init.normal_(block.attn.to_qkv.weight, std=attn_gain) + nn.init.normal_(block.attn.proj.weight, std=proj_gain) + nn.init.normal_(block.mlp[0].weight, std=mlp_gain) + nn.init.normal_(block.mlp[2].weight, std=proj_gain) + + def param_groups(self): + groups = [{ + 'params': [ + p for n, p in self.named_parameters() + if 'norm' in n or n.endswith('bias') + ], + 'weight_decay': + 0.0 + }, { + 'params': [ + p for n, p in self.named_parameters() + if not ('norm' in n or n.endswith('bias')) + ] + }] + return groups + + def fp16(self): + return self.apply(to_fp16) + + +def clip_vit_b_32(**kwargs): + return CLIP( + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + text_dim=512, + text_heads=8, + text_layers=12, + **kwargs) + + +def clip_vit_b_16(**kwargs): + return CLIP( + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + text_dim=512, + text_heads=8, + text_layers=12, + **kwargs) + + +def clip_vit_l_14(**kwargs): + return CLIP( + embed_dim=768, + image_size=224, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + text_dim=768, + text_heads=12, + text_layers=12, + **kwargs) + + +def clip_vit_l_14_336px(**kwargs): + return CLIP( + embed_dim=768, + image_size=336, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + text_dim=768, + text_heads=12, + text_layers=12, + **kwargs) + + +def clip_vit_h_16(**kwargs): + return CLIP( + embed_dim=1024, + image_size=256, + patch_size=16, + vision_dim=1280, + vision_heads=16, + vision_layers=32, + text_dim=1024, + text_heads=16, + text_layers=24, + **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/__init__.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..474c811b10c32d39432b5d5f65d165a08bb3b281 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/__init__.py @@ -0,0 +1,9 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .degradation import * # noqa F403 +from .diffusion import * # noqa F403 +from .losses import * # noqa F403 +from .metrics import * # noqa F403 +from .random_color import * # noqa F403 +from .random_mask import * # noqa F403 +from .svd import * # noqa F403 +from .utils import * # noqa F403 diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/apps.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..39d2e0157ae08f3d13dfd824e601847df9a78149 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/apps.py @@ -0,0 +1,664 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# APPs that facilitate the use of pretrained neural networks. + +import os.path as osp + +import artist.data as data +import artist.models as models +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn.functional as F +import torchvision.transforms as T +from artist import DOWNLOAD_TO_CACHE +from PIL import Image +from torch.utils.data import DataLoader, Dataset + +from .utils import parallel, read_image + +__all__ = [ + 'FeatureExtractor', 'Classifier', 'Text2Image', 'Sole2Shoe', 'ImageParser', + 'TextImageMatch', 'taobao_feature_extractor', 'singleton_classifier', + 'orientation_classifier', 'fashion_text2image', 'mindalle_text2image', + 'sole2shoe', 'sole_parser', 'sod_foreground_parser', + 'fashion_text_image_match' +] + + +class ImageFolder(Dataset): + + def __init__(self, paths, transforms=None): + self.paths = paths + self.transforms = transforms + + def __getitem__(self, index): + img = read_image(self.paths[index]) + if img.mode != 'RGB': + img = img.convert('RGB') + if self.transforms is not None: + img = self.transforms(img) + return img + + def __len__(self): + return len(self.paths) + + +class FeatureExtractor(object): + + def __init__( + self, + model='InceptionV1', + checkpoint='models/inception-v1/1218shoes.v9_7.140.0.1520000', + resolution=224, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + batch_size=64, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.resolution = resolution + self.batch_size = batch_size + self.device = device + + # init model + self.net = getattr( + models, + model)(num_classes=None).eval().requires_grad_(False).to(device) + self.net.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(checkpoint), map_location=device)) + + # data transforms + self.transforms = T.Compose([ + data.PadToSquare(), + T.Resize(resolution), + T.ToTensor(), + T.Normalize(mean, std) + ]) + + def __call__(self, imgs, num_workers=0): + r"""imgs: Either a PIL.Image or a list of PIL.Image instances. + """ + # preprocess + if isinstance(imgs, Image.Image): + imgs = [imgs] + assert isinstance(imgs, + (tuple, list)) and isinstance(imgs[0], Image.Image) + imgs = torch.stack(parallel(self.transforms, imgs, num_workers), dim=0) + + # forward + feats = [] + for batch in imgs.split(self.batch_size, dim=0): + batch = batch.to(self.device, non_blocking=True) + feats.append(self.net(batch)) + return torch.cat(feats, dim=0) + + def batch_process(self, paths): + # init dataloader + dataloader = DataLoader( + dataset=ImageFolder(paths, self.transforms), + batch_size=self.batch_size, + shuffle=False, + drop_last=False, + pin_memory=True, + num_workers=8, + prefetch_factor=2) + + # forward + feats = [] + for step, batch in enumerate(dataloader, 1): + print(f'Step: {step}/{len(dataloader)}', flush=True) + batch = batch.to(self.device, non_blocking=True) + feats.append(self.net(batch)) + return torch.cat(feats) + + +class Classifier(object): + + def __init__( + self, + model='InceptionV1', + checkpoint='models/classifier/shoes+apparel+bag-sgdetect-211230.pth', + num_classes=1, + resolution=224, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + batch_size=64, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.num_classes = num_classes + self.resolution = resolution + self.batch_size = batch_size + self.device = device + + # init model + self.net = getattr(models, model)( + num_classes=num_classes).eval().requires_grad_(False).to(device) + self.net.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(checkpoint), map_location=device)) + + # data transforms + self.transforms = T.Compose([ + data.PadToSquare(), + T.Resize(resolution), + T.ToTensor(), + T.Normalize(mean, std) + ]) + + def __call__(self, imgs, num_workers=0): + r"""imgs: Either a PIL.Image or a list of PIL.Image instances. + """ + # preprocess + if isinstance(imgs, Image.Image): + imgs = [imgs] + assert isinstance(imgs, + (tuple, list)) and isinstance(imgs[0], Image.Image) + imgs = torch.stack(parallel(self.transforms, imgs, num_workers), dim=0) + + # forward + scores = [] + for batch in imgs.split(self.batch_size, dim=0): + batch = batch.to(self.device, non_blocking=True) + logits = self.net(batch) + scores.append(logits.sigmoid() if self.num_classes == # noqa W504 + 1 else logits.softmax(dim=1)) + return torch.cat(scores, dim=0) + + +class Text2Image(object): + + def __init__( + self, + vqgan_dim=128, + vqgan_z_dim=256, + vqgan_dim_mult=[1, 1, 2, 2, 4], + vqgan_num_res_blocks=2, + vqgan_attn_scales=[1.0 / 16], + vqgan_codebook_size=975, + vqgan_beta=0.25, + gpt_txt_vocab_size=21128, + gpt_txt_seq_len=64, + gpt_img_seq_len=1024, + gpt_dim=1024, + gpt_num_heads=16, + gpt_num_layers=24, + vqgan_checkpoint='models/vqgan/vqgan_shoes+apparels_step10k_vocab975.pth', + gpt_checkpoint='models/seq2seq_gpt/text2image_shoes+apparels_step400k.pth', + tokenizer=data.BertTokenizer(name='bert-base-chinese', length=64), + batch_size=16, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.tokenizer = tokenizer + self.batch_size = batch_size + self.device = device + + # init VQGAN model + self.vqgan = models.VQGAN( + dim=vqgan_dim, + z_dim=vqgan_z_dim, + dim_mult=vqgan_dim_mult, + num_res_blocks=vqgan_num_res_blocks, + attn_scales=vqgan_attn_scales, + codebook_size=vqgan_codebook_size, + beta=vqgan_beta).eval().requires_grad_(False).to(device) + self.vqgan.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE(vqgan_checkpoint), map_location=device)) + + # init GPT model + self.gpt = models.Seq2SeqGPT( + src_vocab_size=gpt_txt_vocab_size, + tar_vocab_size=vqgan_codebook_size, + src_seq_len=gpt_txt_seq_len, + tar_seq_len=gpt_img_seq_len, + dim=gpt_dim, + num_heads=gpt_num_heads, + num_layers=gpt_num_layers).eval().requires_grad_(False).to(device) + self.gpt.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(gpt_checkpoint), map_location=device)) + + def __call__(self, + txts, + top_k=64, + top_p=None, + temperature=0.6, + use_fp16=True): + # preprocess + if isinstance(txts, str): + txts = [txts] + assert isinstance(txts, (tuple, list)) and isinstance(txts[0], str) + txt_tokens = torch.LongTensor([self.tokenizer(u) for u in txts]) + + # forward + out_imgs = [] + for batch in txt_tokens.split(self.batch_size, dim=0): + # sample + batch = batch.to(self.device, non_blocking=True) + with amp.autocast(enabled=use_fp16): + img_tokens = self.gpt.sample(batch, top_k, top_p, temperature) + + # decode + imgs = self.vqgan.decode_from_tokens(img_tokens) + imgs = self._whiten_borders(imgs) + imgs = imgs.clamp_(-1, 1).add_(1).mul_(125.0).permute( + 0, 2, 3, 1).cpu().numpy().astype(np.uint8) + imgs = [Image.fromarray(u) for u in imgs] + + # append + out_imgs += imgs + return out_imgs + + def _whiten_borders(self, imgs): + r"""Remove border artifacts. + """ + imgs[:, :, :18, :] = 1 + imgs[:, :, :, :18] = 1 + imgs[:, :, -18:, :] = 1 + imgs[:, :, :, -18:] = 1 + return imgs + + +class Sole2Shoe(object): + + def __init__( + self, + vqgan_dim=128, + vqgan_z_dim=256, + vqgan_dim_mult=[1, 1, 2, 2, 4], + vqgan_num_res_blocks=2, + vqgan_attn_scales=[1.0 / 16], + vqgan_codebook_size=975, + vqgan_beta=0.25, + src_resolution=256, + tar_resolution=512, + gpt_dim=1024, + gpt_num_heads=16, + gpt_num_layers=24, + vqgan_checkpoint='models/vqgan/vqgan_shoes+apparels_step10k_vocab975.pth', + gpt_checkpoint='models/seq2seq_gpt/sole2shoe-step300k-220104.pth', + batch_size=12, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.batch_size = batch_size + self.device = device + src_seq_len = (src_resolution // 16)**2 + tar_seq_len = (tar_resolution // 16)**2 + + # init VQGAN model + self.vqgan = models.VQGAN( + dim=vqgan_dim, + z_dim=vqgan_z_dim, + dim_mult=vqgan_dim_mult, + num_res_blocks=vqgan_num_res_blocks, + attn_scales=vqgan_attn_scales, + codebook_size=vqgan_codebook_size, + beta=vqgan_beta).eval().requires_grad_(False).to(device) + self.vqgan.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE(vqgan_checkpoint), map_location=device)) + + # init GPT model + self.gpt = models.Seq2SeqGPT( + src_vocab_size=vqgan_codebook_size, + tar_vocab_size=vqgan_codebook_size, + src_seq_len=src_seq_len, + tar_seq_len=tar_seq_len, + dim=gpt_dim, + num_heads=gpt_num_heads, + num_layers=gpt_num_layers).eval().requires_grad_(False).to(device) + self.gpt.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(gpt_checkpoint), map_location=device)) + + # data transforms + self.transforms = T.Compose([ + data.PadToSquare(), + T.Resize(src_resolution), + T.ToTensor(), + T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + ]) + + def __call__(self, + sole_imgs, + top_k=64, + top_p=None, + temperature=0.6, + use_fp16=True, + num_workers=0): + # preprocess + if isinstance(sole_imgs, Image.Image): + sole_imgs = [sole_imgs] + assert isinstance(sole_imgs, (tuple, list)) and isinstance( + sole_imgs[0], Image.Image) + sole_imgs = torch.stack( + parallel(self.transforms, sole_imgs, num_workers), dim=0) + + # forward + out_imgs = [] + for batch in sole_imgs.split(self.batch_size, dim=0): + # sample + batch = batch.to(self.device) + with amp.autocast(enabled=use_fp16): + sole_tokens = self.vqgan.encode_to_tokens(batch) + shoe_tokens = self.gpt.sample(sole_tokens, top_k, top_p, + temperature) + + # decode + shoe_imgs = self.vqgan.decode_from_tokens(shoe_tokens) + shoe_imgs = self._whiten_borders(shoe_imgs) + shoe_imgs = shoe_imgs.clamp_(-1, 1).add_(1).mul_(125.0).permute( + 0, 2, 3, 1).cpu().numpy().astype(np.uint8) + shoe_imgs = [Image.fromarray(u) for u in shoe_imgs] + + # append + out_imgs += shoe_imgs + return out_imgs + + def _whiten_borders(self, imgs): + r"""Remove border artifacts. + """ + imgs[:, :, :18, :] = 1 + imgs[:, :, :, :18] = 1 + imgs[:, :, -18:, :] = 1 + imgs[:, :, :, -18:] = 1 + return imgs + + +class ImageParser(object): + + def __init__( + self, + model='SPNet', + num_classes=2, + resolution=800, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + model_with_softmax=False, + checkpoint='models/spnet/sole_segmentation_211219.pth', + batch_size=16, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.batch_size = batch_size + self.device = device + + # init model + if checkpoint.endswith('.pt'): + self.net = torch.jit.load( + DOWNLOAD_TO_CACHE(checkpoint)).eval().to(device) + [p.requires_grad_(False) for p in self.net.parameters()] + else: + self.net = getattr(models, model)( + num_classes=num_classes, + pretrained=False).eval().requires_grad_(False).to(device) + self.net.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(checkpoint), map_location=device)) + self.softmax = (lambda x, dim: x) if model_with_softmax else F.softmax + + # data transforms + self.transforms = T.Compose([ + data.PadToSquare(), + T.Resize(resolution), + T.ToTensor(), + T.Normalize(mean, std) + ]) + + def __call__(self, imgs, num_workers=0): + # preprocess + if isinstance(imgs, Image.Image): + imgs = [imgs] + assert isinstance(imgs, + (tuple, list)) and isinstance(imgs[0], Image.Image) + sizes = [u.size for u in imgs] + imgs = torch.stack(parallel(self.transforms, imgs, num_workers), dim=0) + + # forward + masks = [] + for batch in imgs.split(self.batch_size, dim=0): + batch = batch.to(self.device, non_blocking=True) + masks.append(self.softmax(self.net(batch), dim=1)) + + # postprocess + masks = torch.cat(masks, dim=0).unsqueeze(1) + masks = [ + F.interpolate(u, v, mode='bilinear', align_corners=False) + for u, v in zip(masks, sizes) + ] + return masks + + +class TextImageMatch(object): + + def __init__( + self, + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=21128, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + mean=[0.48145466, 0.4578275, 0.40821073], + std=[0.26862954, 0.26130258, 0.27577711], + checkpoint='models/clip/clip_shoes+apparels_step84k_210105.pth', + tokenizer=data.BertTokenizer(name='bert-base-chinese', length=77), + batch_size=64, + device=torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu')): # noqa E125 + self.tokenizer = tokenizer + self.batch_size = batch_size + self.device = device + + # init model + self.clip = models.CLIP( + embed_dim=embed_dim, + image_size=image_size, + patch_size=patch_size, + vision_dim=vision_dim, + vision_heads=vision_heads, + vision_layers=vision_layers, + vocab_size=vocab_size, + text_len=text_len, + text_dim=text_dim, + text_heads=text_heads, + text_layers=text_layers).eval().requires_grad_(False).to(device) + self.clip.load_state_dict( + torch.load(DOWNLOAD_TO_CACHE(checkpoint), map_location=device)) + + # transforms + scale_size = int(image_size * 8 / 7) + self.transforms = T.Compose([ + data.PadToSquare(), + T.Resize(scale_size), + T.CenterCrop(image_size), + T.ToTensor(), + T.Normalize(mean, std) + ]) + + def __call__(self, imgs, txts, num_workers=0): + # preprocess + assert isinstance(imgs, + (tuple, list)) and isinstance(imgs[0], Image.Image) + assert isinstance(txts, (tuple, list)) and isinstance(txts[0], str) + txt_tokens = torch.LongTensor([self.tokenizer(u) for u in txts]) + imgs = torch.stack(parallel(self.transforms, imgs, num_workers), dim=0) + + # forward + scores = [] + for img_batch, txt_batch in zip( + imgs.split(self.batch_size, dim=0), + txt_tokens.split(self.batch_size, dim=0)): + img_batch = img_batch.to(self.device) + txt_batch = txt_batch.to(self.device) + xi = F.normalize(self.clip.visual(img_batch), p=2, dim=1) + xt = F.normalize(self.clip.textual(txt_batch), p=2, dim=1) + scores.append((xi * xt).sum(dim=1)) + return torch.cat(scores, dim=0) + + +def taobao_feature_extractor(category='shoes', **kwargs): + r"""Pretrained taobao-search feature extractors. + """ + assert category in ['softall', 'shoes', 'bag'] + checkpoint = osp.join( + 'models/inception-v1', { + 'softall': '1214softall_10.10.0.5000', + 'shoes': '1218shoes.v9_7.140.0.1520000', + 'bag': '0926bag.v9_6.29.0.140000' + }[category]) + app = FeatureExtractor( + model='InceptionV1', + checkpoint=checkpoint, + resolution=224, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + **kwargs) + return app + + +def singleton_classifier(**kwargs): + r"""Pretrained classifier that finds single-object images. + Supports shoes, apparel, and bag images. + """ + app = Classifier( + model='InceptionV1', + checkpoint='models/classifier/shoes+apparel+bag-sgdetect-211230.pth', + num_classes=1, + resolution=224, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + **kwargs) + return app + + +def orientation_classifier(**kwargs): + r"""Shoes orientation classifier. + """ + app = Classifier( + model='InceptionV1', + checkpoint='models/classifier/shoes-oriendetect-20211026.pth', + num_classes=1, + resolution=224, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + **kwargs) + return app + + +def fashion_text2image(**kwargs): + r"""Fashion text-to-image generator. + Supports shoe and apparel image generation. + """ + app = Text2Image( + vqgan_dim=128, + vqgan_z_dim=256, + vqgan_dim_mult=[1, 1, 2, 2, 4], + vqgan_num_res_blocks=2, + vqgan_attn_scales=[1.0 / 16], + vqgan_codebook_size=975, + vqgan_beta=0.25, + gpt_txt_vocab_size=21128, + gpt_txt_seq_len=64, + gpt_img_seq_len=1024, + gpt_dim=1024, + gpt_num_heads=16, + gpt_num_layers=24, + vqgan_checkpoint= # noqa E251 + 'models/vqgan/vqgan_shoes+apparels_step10k_vocab975.pth', + gpt_checkpoint= # noqa E251 + 'models/seq2seq_gpt/text2image_shoes+apparels_step400k.pth', + tokenizer=data.BertTokenizer(name='bert-base-chinese', length=64), + **kwargs) + return app + + +def mindalle_text2image(**kwargs): + r"""Pretrained text2image generator with weights copied from minDALL-E. + """ + app = Text2Image( + vqgan_dim=128, + vqgan_z_dim=256, + vqgan_dim_mult=[1, 1, 2, 2, 4], + vqgan_num_res_blocks=2, + vqgan_attn_scales=[1.0 / 16], + vqgan_codebook_size=16384, + vqgan_beta=0.25, + gpt_txt_vocab_size=16384, + gpt_txt_seq_len=64, + gpt_img_seq_len=256, + gpt_dim=1536, + gpt_num_heads=24, + gpt_num_layers=42, + vqgan_checkpoint='models/minDALLE/1.3B_vqgan.pth', + gpt_checkpoint='models/minDALLE/1.3B_gpt.pth', + tokenizer=data.BPETokenizer(length=64), + **kwargs) + return app + + +def sole2shoe(**kwargs): + app = Sole2Shoe( + vqgan_dim=128, + vqgan_z_dim=256, + vqgan_dim_mult=[1, 1, 2, 2, 4], + vqgan_num_res_blocks=2, + vqgan_attn_scales=[1.0 / 16], + vqgan_codebook_size=975, + vqgan_beta=0.25, + src_resolution=256, + tar_resolution=512, + gpt_dim=1024, + gpt_num_heads=16, + gpt_num_layers=24, + vqgan_checkpoint= # noqa E251 + 'models/vqgan/vqgan_shoes+apparels_step10k_vocab975.pth', + gpt_checkpoint='models/seq2seq_gpt/sole2shoe-step300k-220104.pth', + **kwargs) + return app + + +def sole_parser(**kwargs): + app = ImageParser( + model='SPNet', + num_classes=2, + resolution=800, + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225], + model_with_softmax=False, + checkpoint='models/spnet/sole_segmentation_211219.pth', + **kwargs) + return app + + +def sod_foreground_parser(**kwargs): + app = ImageParser( + model=None, + num_classes=None, + resolution=448, + mean=[0.488431, 0.466275, 0.403686], + std=[0.222627, 0.21949, 0.22549], + model_with_softmax=True, + checkpoint='models/semseg/sod_model_20201228.pt', + **kwargs) + return app + + +def fashion_text_image_match(**kwargs): + app = TextImageMatch( + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=21128, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + mean=[0.48145466, 0.4578275, 0.40821073], + std=[0.26862954, 0.26130258, 0.27577711], + checkpoint='models/clip/clip_shoes+apparels_step84k_210105.pth', + tokenizer=data.BertTokenizer(name='bert-base-chinese', length=77), + **kwargs) + return app diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/degradation.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/degradation.py new file mode 100644 index 0000000000000000000000000000000000000000..9061e7bec87362722c450e2c904a84a76d2fbf50 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/degradation.py @@ -0,0 +1,1075 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math +import os +import random + +import cv2 +import numpy as np +import scipy +import scipy.stats as stats +import torch +from scipy import ndimage +from scipy.interpolate import interp2d +from scipy.linalg import orth + +os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' + +__all__ = ['degradation_bsrgan_light', 'degradation_bsrgan'] + + +# -------------------------------------------- +# get uint8 image of size HxWxn_channles (RGB) +# -------------------------------------------- +def imread_uint(path, n_channels=3): + # input: path + # output: HxWx3(RGB or GGG), or HxWx1 (G) + if n_channels == 1: + img = cv2.imread(path, 0) # cv2.IMREAD_GRAYSCALE + img = np.expand_dims(img, axis=2) # HxWx1 + elif n_channels == 3: + img = cv2.imread(path, cv2.IMREAD_UNCHANGED) # BGR or G + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) # GGG + else: + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # RGB + return img + + +# -------------------------------------------- +# numpy(single) [0, 1] <---> numpy(unit) +# -------------------------------------------- + + +def uint2single(img): + return np.float32(img / 255.) + + +def single2uint(img): + return np.uint8((img.clip(0, 1) * 255.).round()) + + +def uint162single(img): + return np.float32(img / 65535.) + + +def single2uint16(img): + return np.uint16((img.clip(0, 1) * 65535.).round()) + + +def rgb2ycbcr(img, only_y=True): + '''same as matlab rgb2ycbcr + only_y: only return Y channel + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + if only_y: + rlt = np.dot(img, [65.481, 128.553, 24.966]) / 255.0 + 16.0 + else: + rlt = np.matmul(img, + [[65.481, -37.797, 112.0], [128.553, -74.203, -93.786], + [24.966, 112.0, -18.214]]) / 255.0 + [16, 128, 128] + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def ycbcr2rgb(img): + '''same as matlab ycbcr2rgb + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + rlt = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0, -0.00153632, 0.00791071], + [0.00625893, -0.00318811, 0]]) * 255.0 + [ + -222.921, 135.576, -276.836 + ] # noqa E126 + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def bgr2ycbcr(img, only_y=True): + '''bgr version of rgb2ycbcr + only_y: only return Y channel + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + if only_y: + rlt = np.dot(img, [24.966, 128.553, 65.481]) / 255.0 + 16.0 + else: + rlt = np.matmul(img, + [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786], + [65.481, -37.797, 112.0]]) / 255.0 + [16, 128, 128] + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def channel_convert(in_c, tar_type, img_list): + # conversion among BGR, gray and y + if in_c == 3 and tar_type == 'gray': # BGR to gray + gray_list = [cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) for img in img_list] + return [np.expand_dims(img, axis=2) for img in gray_list] + elif in_c == 3 and tar_type == 'y': # BGR to y + y_list = [bgr2ycbcr(img, only_y=True) for img in img_list] + return [np.expand_dims(img, axis=2) for img in y_list] + elif in_c == 1 and tar_type == 'RGB': # gray/y to BGR + return [cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) for img in img_list] + else: + return img_list + + +''' +# -------------------------------------------- +# metric, PSNR and SSIM +# -------------------------------------------- +''' + + +# -------------------------------------------- +# PSNR +# -------------------------------------------- +def calculate_psnr(img1, img2, border=0): + # img1 and img2 have range [0, 255] + # img1 = img1.squeeze() + # img2 = img2.squeeze() + if not img1.shape == img2.shape: + raise ValueError('Input images must have the same dimensions.') + h, w = img1.shape[:2] + img1 = img1[border:h - border, border:w - border] + img2 = img2[border:h - border, border:w - border] + + img1 = img1.astype(np.float64) + img2 = img2.astype(np.float64) + mse = np.mean((img1 - img2)**2) + if mse == 0: + return float('inf') + return 20 * math.log10(255.0 / math.sqrt(mse)) + + +# -------------------------------------------- +# SSIM +# -------------------------------------------- +def calculate_ssim(img1, img2, border=0): + '''calculate SSIM + the same outputs as MATLAB's + img1, img2: [0, 255] + ''' + # img1 = img1.squeeze() + # img2 = img2.squeeze() + if not img1.shape == img2.shape: + raise ValueError('Input images must have the same dimensions.') + h, w = img1.shape[:2] + img1 = img1[border:h - border, border:w - border] + img2 = img2[border:h - border, border:w - border] + + if img1.ndim == 2: + return ssim(img1, img2) + elif img1.ndim == 3: + if img1.shape[2] == 3: + ssims = [] + for i in range(3): + ssims.append(ssim(img1[:, :, i], img2[:, :, i])) + return np.array(ssims).mean() + elif img1.shape[2] == 1: + return ssim(np.squeeze(img1), np.squeeze(img2)) + else: + raise ValueError('Wrong input image dimensions.') + + +def ssim(img1, img2): + C1 = (0.01 * 255)**2 + C2 = (0.03 * 255)**2 + + img1 = img1.astype(np.float64) + img2 = img2.astype(np.float64) + kernel = cv2.getGaussianKernel(11, 1.5) + window = np.outer(kernel, kernel.transpose()) + + mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5] # valid + mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5] + mu1_sq = mu1**2 + mu2_sq = mu2**2 + mu1_mu2 = mu1 * mu2 + sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq + sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq + sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2 + + ssim_map = ((2 * mu1_mu2 + C1) * # noqa W504 + (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * # noqa W504 + (sigma1_sq + sigma2_sq + C2)) + return ssim_map.mean() + + +''' +# -------------------------------------------- +# matlab's bicubic imresize (numpy and torch) [0, 1] +# -------------------------------------------- +''' + + +# matlab 'imresize' function, now only support 'bicubic' +def cubic(x): + absx = torch.abs(x) + absx2 = absx**2 + absx3 = absx**3 + return (1.5 * absx3 - 2.5 * absx2 + 1) * ( + (absx <= 1).type_as(absx)) + (-0.5 * absx3 + 2.5 * absx2 - 4 * absx + + 2) * (((absx > 1) * # noqa W504 + (absx <= 2)).type_as(absx)) + + +def calculate_weights_indices(in_length, out_length, scale, kernel, + kernel_width, antialiasing): + if (scale < 1) and (antialiasing): + # Use a modified kernel to simultaneously interpolate and antialias- larger kernel width + kernel_width = kernel_width / scale + + # Output-space coordinates + x = torch.linspace(1, out_length, out_length) + + # Input-space coordinates. Calculate the inverse mapping such that 0.5 + # in output space maps to 0.5 in input space, and 0.5+scale in output + # space maps to 1.5 in input space. + u = x / scale + 0.5 * (1 - 1 / scale) + + # What is the left-most pixel that can be involved in the computation? + left = torch.floor(u - kernel_width / 2) + + # What is the maximum number of pixels that can be involved in the + # computation? Note: it's OK to use an extra pixel here; if the + # corresponding weights are all zero, it will be eliminated at the end + # of this function. + P = math.ceil(kernel_width) + 2 + + # The indices of the input pixels involved in computing the k-th output + # pixel are in row k of the indices matrix. + indices = left.view(out_length, 1).expand(out_length, P) + torch.linspace( + 0, P - 1, P).view(1, P).expand(out_length, P) + + # The weights used to compute the k-th output pixel are in row k of the + # weights matrix. + distance_to_center = u.view(out_length, 1).expand(out_length, P) - indices + # apply cubic kernel + if (scale < 1) and (antialiasing): + weights = scale * cubic(distance_to_center * scale) + else: + weights = cubic(distance_to_center) + # Normalize the weights matrix so that each row sums to 1. + weights_sum = torch.sum(weights, 1).view(out_length, 1) + weights = weights / weights_sum.expand(out_length, P) + + # If a column in weights is all zero, get rid of it. only consider the first and last column. + weights_zero_tmp = torch.sum((weights == 0), 0) + if not math.isclose(weights_zero_tmp[0], 0, rel_tol=1e-6): + indices = indices.narrow(1, 1, P - 2) + weights = weights.narrow(1, 1, P - 2) + if not math.isclose(weights_zero_tmp[-1], 0, rel_tol=1e-6): + indices = indices.narrow(1, 0, P - 2) + weights = weights.narrow(1, 0, P - 2) + weights = weights.contiguous() + indices = indices.contiguous() + sym_len_s = -indices.min() + 1 + sym_len_e = indices.max() - in_length + indices = indices + sym_len_s - 1 + return weights, indices, int(sym_len_s), int(sym_len_e) + + +# -------------------------------------------- +# imresize for tensor image [0, 1] +# -------------------------------------------- +def imresize(img, scale, antialiasing=True): + # Now the scale should be the same for H and W + # input: img: pytorch tensor, CHW or HW [0,1] + # output: CHW or HW [0,1] w/o round + need_squeeze = True if img.dim() == 2 else False + if need_squeeze: + img.unsqueeze_(0) + in_C, in_H, in_W = img.size() + out_C, out_H, out_W = in_C, math.ceil(in_H * scale), math.ceil(in_W + * scale) + kernel_width = 4 + kernel = 'cubic' + + # Return the desired dimension order for performing the resize. The + # strategy is to perform the resize first along the dimension with the + # smallest scale factor. + # Now we do not support this. + + # get weights and indices + weights_H, indices_H, sym_len_Hs, sym_len_He = calculate_weights_indices( + in_H, out_H, scale, kernel, kernel_width, antialiasing) + weights_W, indices_W, sym_len_Ws, sym_len_We = calculate_weights_indices( + in_W, out_W, scale, kernel, kernel_width, antialiasing) + # process H dimension + # symmetric copying + img_aug = torch.FloatTensor(in_C, in_H + sym_len_Hs + sym_len_He, in_W) + img_aug.narrow(1, sym_len_Hs, in_H).copy_(img) + + sym_patch = img[:, :sym_len_Hs, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + img_aug.narrow(1, 0, sym_len_Hs).copy_(sym_patch_inv) + + sym_patch = img[:, -sym_len_He:, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + img_aug.narrow(1, sym_len_Hs + in_H, sym_len_He).copy_(sym_patch_inv) + + out_1 = torch.FloatTensor(in_C, out_H, in_W) + kernel_width = weights_H.size(1) + for i in range(out_H): + idx = int(indices_H[i][0]) + for j in range(out_C): + out_1[j, i, :] = img_aug[j, idx:idx + kernel_width, :].transpose( + 0, 1).mv(weights_H[i]) + + # process W dimension + # symmetric copying + out_1_aug = torch.FloatTensor(in_C, out_H, in_W + sym_len_Ws + sym_len_We) + out_1_aug.narrow(2, sym_len_Ws, in_W).copy_(out_1) + + sym_patch = out_1[:, :, :sym_len_Ws] + inv_idx = torch.arange(sym_patch.size(2) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(2, inv_idx) + out_1_aug.narrow(2, 0, sym_len_Ws).copy_(sym_patch_inv) + + sym_patch = out_1[:, :, -sym_len_We:] + inv_idx = torch.arange(sym_patch.size(2) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(2, inv_idx) + out_1_aug.narrow(2, sym_len_Ws + in_W, sym_len_We).copy_(sym_patch_inv) + + out_2 = torch.FloatTensor(in_C, out_H, out_W) + kernel_width = weights_W.size(1) + for i in range(out_W): + idx = int(indices_W[i][0]) + for j in range(out_C): + out_2[j, :, i] = out_1_aug[j, :, + idx:idx + kernel_width].mv(weights_W[i]) + if need_squeeze: + out_2.squeeze_() + return out_2 + + +# -------------------------------------------- +# imresize for numpy image [0, 1] +# -------------------------------------------- +def imresize_np(img, scale, antialiasing=True): + # Now the scale should be the same for H and W + # input: img: Numpy, HWC or HW [0,1] + # output: HWC or HW [0,1] w/o round + img = torch.from_numpy(img) + need_squeeze = True if img.dim() == 2 else False + if need_squeeze: + img.unsqueeze_(2) + + in_H, in_W, in_C = img.size() + out_C, out_H, out_W = in_C, math.ceil(in_H * scale), math.ceil(in_W + * scale) + kernel_width = 4 + kernel = 'cubic' + + # Return the desired dimension order for performing the resize. The + # strategy is to perform the resize first along the dimension with the + # smallest scale factor. + # Now we do not support this. + + # get weights and indices + weights_H, indices_H, sym_len_Hs, sym_len_He = calculate_weights_indices( + in_H, out_H, scale, kernel, kernel_width, antialiasing) + weights_W, indices_W, sym_len_Ws, sym_len_We = calculate_weights_indices( + in_W, out_W, scale, kernel, kernel_width, antialiasing) + # process H dimension + # symmetric copying + img_aug = torch.FloatTensor(in_H + sym_len_Hs + sym_len_He, in_W, in_C) + img_aug.narrow(0, sym_len_Hs, in_H).copy_(img) + + sym_patch = img[:sym_len_Hs, :, :] + inv_idx = torch.arange(sym_patch.size(0) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(0, inv_idx) + img_aug.narrow(0, 0, sym_len_Hs).copy_(sym_patch_inv) + + sym_patch = img[-sym_len_He:, :, :] + inv_idx = torch.arange(sym_patch.size(0) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(0, inv_idx) + img_aug.narrow(0, sym_len_Hs + in_H, sym_len_He).copy_(sym_patch_inv) + + out_1 = torch.FloatTensor(out_H, in_W, in_C) + kernel_width = weights_H.size(1) + for i in range(out_H): + idx = int(indices_H[i][0]) + for j in range(out_C): + out_1[i, :, j] = img_aug[idx:idx + kernel_width, :, + j].transpose(0, 1).mv(weights_H[i]) + + # process W dimension + # symmetric copying + out_1_aug = torch.FloatTensor(out_H, in_W + sym_len_Ws + sym_len_We, in_C) + out_1_aug.narrow(1, sym_len_Ws, in_W).copy_(out_1) + + sym_patch = out_1[:, :sym_len_Ws, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + out_1_aug.narrow(1, 0, sym_len_Ws).copy_(sym_patch_inv) + + sym_patch = out_1[:, -sym_len_We:, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + out_1_aug.narrow(1, sym_len_Ws + in_W, sym_len_We).copy_(sym_patch_inv) + + out_2 = torch.FloatTensor(out_H, out_W, in_C) + kernel_width = weights_W.size(1) + for i in range(out_W): + idx = int(indices_W[i][0]) + for j in range(out_C): + out_2[:, i, j] = out_1_aug[:, idx:idx + kernel_width, + j].mv(weights_W[i]) + if need_squeeze: + out_2.squeeze_() + + return out_2.numpy() + + +""" +# -------------------------------------------- +# Super-Resolution +# -------------------------------------------- +# +# Kai Zhang (cskaizhang@gmail.com) +# https://github.com/cszn +# From 2019/03--2021/08 +# -------------------------------------------- +""" + + +def modcrop_np(img, sf): + ''' + Args: + img: numpy image, WxH or WxHxC + sf: scale factor + Return: + cropped image + ''' + w, h = img.shape[:2] + im = np.copy(img) + return im[:w - w % sf, :h - h % sf, ...] + + +""" +# -------------------------------------------- +# anisotropic Gaussian kernels +# -------------------------------------------- +""" + + +def analytic_kernel(k): + """Calculate the X4 kernel from the X2 kernel (for proof see appendix in paper)""" + k_size = k.shape[0] + # Calculate the big kernels size + big_k = np.zeros((3 * k_size - 2, 3 * k_size - 2)) + # Loop over the small kernel to fill the big one + for r in range(k_size): + for c in range(k_size): + big_k[2 * r:2 * r + k_size, 2 * c:2 * c + k_size] += k[r, c] * k + # Crop the edges of the big kernel to ignore very small values and increase run time of SR + crop = k_size // 2 + cropped_big_k = big_k[crop:-crop, crop:-crop] + # Normalize to 1 + return cropped_big_k / cropped_big_k.sum() + + +def anisotropic_Gaussian(ksize=15, theta=np.pi, l1=6, l2=6): + """ generate an anisotropic Gaussian kernel + Args: + ksize : e.g., 15, kernel size + theta : [0, pi], rotation angle range + l1 : [0.1,50], scaling of eigenvalues + l2 : [0.1,l1], scaling of eigenvalues + If l1 = l2, will get an isotropic Gaussian kernel. + Returns: + k : kernel + """ + + v = np.dot( + np.array([[np.cos(theta), -np.sin(theta)], + [np.sin(theta), np.cos(theta)]]), np.array([1., 0.])) + V = np.array([[v[0], v[1]], [v[1], -v[0]]]) + D = np.array([[l1, 0], [0, l2]]) + Sigma = np.dot(np.dot(V, D), np.linalg.inv(V)) + k = gm_blur_kernel(mean=[0, 0], cov=Sigma, size=ksize) + return k + + +def gm_blur_kernel(mean, cov, size=15): + center = size / 2.0 + 0.5 + k = np.zeros([size, size]) + for y in range(size): + for x in range(size): + cy = y - center + 1 + cx = x - center + 1 + k[y, x] = stats.multivariate_normal.pdf([cx, cy], + mean=mean, + cov=cov) + + k = k / np.sum(k) + return k + + +def shift_pixel(x, sf, upper_left=True): + """shift pixel for super-resolution with different scale factors + Args: + x: WxHxC or WxH + sf: scale factor + upper_left: shift direction + """ + h, w = x.shape[:2] + shift = (sf - 1) * 0.5 + xv, yv = np.arange(0, w, 1.0), np.arange(0, h, 1.0) + if upper_left: + x1 = xv + shift + y1 = yv + shift + else: + x1 = xv - shift + y1 = yv - shift + + x1 = np.clip(x1, 0, w - 1) + y1 = np.clip(y1, 0, h - 1) + + if x.ndim == 2: + x = interp2d(xv, yv, x)(x1, y1) + if x.ndim == 3: + for i in range(x.shape[-1]): + x[:, :, i] = interp2d(xv, yv, x[:, :, i])(x1, y1) + + return x + + +def blur(x, k): + ''' + x: image, NxcxHxW + k: kernel, Nx1xhxw + ''' + n, c = x.shape[:2] + p1, p2 = (k.shape[-2] - 1) // 2, (k.shape[-1] - 1) // 2 + x = torch.nn.functional.pad(x, pad=(p1, p2, p1, p2), mode='replicate') + k = k.repeat(1, c, 1, 1) + k = k.view(-1, 1, k.shape[2], k.shape[3]) + x = x.view(1, -1, x.shape[2], x.shape[3]) + x = torch.nn.functional.conv2d( + x, k, bias=None, stride=1, padding=0, groups=n * c) + x = x.view(n, c, x.shape[2], x.shape[3]) + + return x + + +def gen_kernel( + k_size=np.array([15, 15]), + scale_factor=np.array([4, 4]), + min_var=0.6, + max_var=10., + noise_level=0): + """" + # modified version of https://github.com/assafshocher/BlindSR_dataset_generator + # Kai Zhang + # min_var = 0.175 * sf # variance of the gaussian kernel will be sampled between min_var and max_var + # max_var = 2.5 * sf + """ + # Set random eigen-vals (lambdas) and angle (theta) for COV matrix + lambda_1 = min_var + np.random.rand() * (max_var - min_var) + lambda_2 = min_var + np.random.rand() * (max_var - min_var) + theta = np.random.rand() * np.pi # random theta + noise = -noise_level + np.random.rand(*k_size) * noise_level * 2 + + # Set COV matrix using Lambdas and Theta + LAMBDA = np.diag([lambda_1, lambda_2]) + Q = np.array([[np.cos(theta), -np.sin(theta)], + [np.sin(theta), np.cos(theta)]]) + SIGMA = Q @ LAMBDA @ Q.T + INV_SIGMA = np.linalg.inv(SIGMA)[None, None, :, :] + + # Set expectation position (shifting kernel for aligned image) + MU = k_size // 2 - 0.5 * (scale_factor - 1 + ) # - 0.5 * (scale_factor - k_size % 2) + MU = MU[None, None, :, None] + + # Create meshgrid for Gaussian + [X, Y] = np.meshgrid(range(k_size[0]), range(k_size[1])) + Z = np.stack([X, Y], 2)[:, :, :, None] + + # Calcualte Gaussian for every pixel of the kernel + ZZ = Z - MU + ZZ_t = ZZ.transpose(0, 1, 3, 2) + raw_kernel = np.exp(-0.5 * np.squeeze(ZZ_t @ INV_SIGMA @ ZZ)) * (1 + noise) + + # shift the kernel so it will be centered + # raw_kernel_centered = kernel_shift(raw_kernel, scale_factor) + + # Normalize the kernel and return + # kernel = raw_kernel_centered / np.sum(raw_kernel_centered) + kernel = raw_kernel / np.sum(raw_kernel) + return kernel + + +def fspecial_gaussian(hsize, sigma): + hsize = [hsize, hsize] + siz = [(hsize[0] - 1.0) / 2.0, (hsize[1] - 1.0) / 2.0] + std = sigma + [x, y] = np.meshgrid( + np.arange(-siz[1], siz[1] + 1), np.arange(-siz[0], siz[0] + 1)) + arg = -(x * x + y * y) / (2 * std * std) + h = np.exp(arg) + h[h < scipy.finfo(float).eps * h.max()] = 0 + sumh = h.sum() + if sumh != 0: + h = h / sumh + return h + + +def fspecial_laplacian(alpha): + alpha = max([0, min([alpha, 1])]) + h1 = alpha / (alpha + 1) + h2 = (1 - alpha) / (alpha + 1) + h = [[h1, h2, h1], [h2, -4 / (alpha + 1), h2], [h1, h2, h1]] + h = np.array(h) + return h + + +def fspecial(filter_type, *args, **kwargs): + ''' + python code from: + https://github.com/ronaldosena/imagens-medicas-2/blob/40171a6c259edec7827a6693a93955de2bd39e76/Aulas/aula_2_-_uniform_filter/matlab_fspecial.py + ''' + if filter_type == 'gaussian': + return fspecial_gaussian(*args, **kwargs) + if filter_type == 'laplacian': + return fspecial_laplacian(*args, **kwargs) + + +""" +# -------------------------------------------- +# degradation models +# -------------------------------------------- +""" + + +def bicubic_degradation(x, sf=3): + ''' + Args: + x: HxWxC image, [0, 1] + sf: down-scale factor + Return: + bicubicly downsampled LR image + ''' + x = imresize_np(x, scale=1 / sf) + return x + + +def srmd_degradation(x, k, sf=3): + ''' blur + bicubic downsampling + Args: + x: HxWxC image, [0, 1] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + Reference: + @inproceedings{zhang2018learning, + title={Learning a single convolutional super-resolution network for multiple degradations}, + author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, + booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, + pages={3262--3271}, + year={2018} + } + ''' + x = ndimage.filters.convolve( + x, np.expand_dims(k, axis=2), mode='wrap') # 'nearest' | 'mirror' + x = bicubic_degradation(x, sf=sf) + return x + + +def dpsr_degradation(x, k, sf=3): + ''' bicubic downsampling + blur + Args: + x: HxWxC image, [0, 1] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + Reference: + @inproceedings{zhang2019deep, + title={Deep Plug-and-Play Super-Resolution for Arbitrary Blur Kernels}, + author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, + booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, + pages={1671--1681}, + year={2019} + } + ''' + x = bicubic_degradation(x, sf=sf) + x = ndimage.filters.convolve(x, np.expand_dims(k, axis=2), mode='wrap') + return x + + +def classical_degradation(x, k, sf=3): + ''' blur + downsampling + Args: + x: HxWxC image, [0, 1]/[0, 255] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + ''' + x = ndimage.filters.convolve(x, np.expand_dims(k, axis=2), mode='wrap') + # x = filters.correlate(x, np.expand_dims(np.flip(k), axis=2)) + st = 0 + return x[st::sf, st::sf, ...] + + +def add_sharpening(img, weight=0.5, radius=50, threshold=10): + """USM sharpening. borrowed from real-ESRGAN + Input image: I; Blurry image: B. + 1. K = I + weight * (I - B) + 2. Mask = 1 if abs(I - B) > threshold, else: 0 + 3. Blur mask: + 4. Out = Mask * K + (1 - Mask) * I + Args: + img (Numpy array): Input image, HWC, BGR; float32, [0, 1]. + weight (float): Sharp weight. Default: 1. + radius (float): Kernel size of Gaussian blur. Default: 50. + threshold (int): + """ + if radius % 2 == 0: + radius += 1 + blur = cv2.GaussianBlur(img, (radius, radius), 0) + residual = img - blur + mask = np.abs(residual) * 255 > threshold + mask = mask.astype('float32') + soft_mask = cv2.GaussianBlur(mask, (radius, radius), 0) + + K = img + weight * residual + K = np.clip(K, 0, 1) + return soft_mask * K + (1 - soft_mask) * img + + +def add_blur_1(img, sf=4): + wd2 = 4.0 + sf + wd = 2.0 + 0.2 * sf + + wd2 = wd2 / 4 + wd = wd / 4 + + if random.random() < 0.5: + l1 = wd2 * random.random() + l2 = wd2 * random.random() + k = anisotropic_Gaussian( + ksize=random.randint(2, 11) + 3, + theta=random.random() * np.pi, + l1=l1, + l2=l2) + else: + k = fspecial('gaussian', + random.randint(2, 4) + 3, wd * random.random()) + img = ndimage.filters.convolve( + img, np.expand_dims(k, axis=2), mode='mirror') + + return img + + +def add_resize(img, sf=4): + rnum = np.random.rand() + if rnum > 0.8: # up + sf1 = random.uniform(1, 2) + elif rnum < 0.7: # down + sf1 = random.uniform(0.5 / sf, 1) + else: + sf1 = 1.0 + img = cv2.resize( + img, (int(sf1 * img.shape[1]), int(sf1 * img.shape[0])), + interpolation=random.choice([1, 2, 3])) + img = np.clip(img, 0.0, 1.0) + + return img + + +def add_Gaussian_noise(img, noise_level1=2, noise_level2=25): + noise_level = random.randint(noise_level1, noise_level2) + rnum = np.random.rand() + if rnum > 0.6: # add color Gaussian noise + img = img + np.random.normal(0, noise_level / 255.0, img.shape).astype( + np.float32) + elif rnum < 0.4: # add grayscale Gaussian noise + img = img + np.random.normal(0, noise_level / 255.0, + (*img.shape[:2], 1)).astype(np.float32) + else: # add noise + L = noise_level2 / 255. + D = np.diag(np.random.rand(3)) + U = orth(np.random.rand(3, 3)) + conv = np.dot(np.dot(np.transpose(U), D), U) + img = img + np.random.multivariate_normal([0, 0, 0], np.abs( + L**2 * conv), img.shape[:2]).astype(np.float32) + img = np.clip(img, 0.0, 1.0) + return img + + +def add_speckle_noise(img, noise_level1=2, noise_level2=25): + noise_level = random.randint(noise_level1, noise_level2) + img = np.clip(img, 0.0, 1.0) + rnum = random.random() + if rnum > 0.6: + img += img * np.random.normal(0, noise_level / 255.0, + img.shape).astype(np.float32) + elif rnum < 0.4: + img += img * np.random.normal(0, noise_level / 255.0, + (*img.shape[:2], 1)).astype(np.float32) + else: + L = noise_level2 / 255. + D = np.diag(np.random.rand(3)) + U = orth(np.random.rand(3, 3)) + conv = np.dot(np.dot(np.transpose(U), D), U) + img += img * np.random.multivariate_normal( + [0, 0, 0], np.abs(L**2 * conv), img.shape[:2]).astype(np.float32) + img = np.clip(img, 0.0, 1.0) + return img + + +def add_Poisson_noise(img): + img = np.clip((img * 255.0).round(), 0, 255) / 255. + vals = 10**(2 * random.random() + 2.0) # [2, 4] + if random.random() < 0.5: + img = np.random.poisson(img * vals).astype(np.float32) / vals + else: + img_gray = np.dot(img[..., :3], [0.299, 0.587, 0.114]) + img_gray = np.clip((img_gray * 255.0).round(), 0, 255) / 255. + noise_gray = np.random.poisson(img_gray * vals).astype( + np.float32) / vals - img_gray + img += noise_gray[:, :, np.newaxis] + img = np.clip(img, 0.0, 1.0) + return img + + +def add_JPEG_noise(img): + quality_factor = random.randint(80, 95) + img = cv2.cvtColor(single2uint(img), cv2.COLOR_RGB2BGR) + result, encimg = cv2.imencode( + '.jpg', img, [int(cv2.IMWRITE_JPEG_QUALITY), quality_factor]) + img = cv2.imdecode(encimg, 1) + img = cv2.cvtColor(uint2single(img), cv2.COLOR_BGR2RGB) + return img + + +def random_crop(lq, hq, sf=4, lq_patchsize=64): + h, w = lq.shape[:2] + rnd_h = random.randint(0, h - lq_patchsize) + rnd_w = random.randint(0, w - lq_patchsize) + lq = lq[rnd_h:rnd_h + lq_patchsize, rnd_w:rnd_w + lq_patchsize, :] + + rnd_h_H, rnd_w_H = int(rnd_h * sf), int(rnd_w * sf) + hq = hq[rnd_h_H:rnd_h_H + lq_patchsize * sf, + rnd_w_H:rnd_w_H + lq_patchsize * sf, :] + return lq, hq + + +def degradation_bsrgan_light(image, sf=4, isp_model=None): + """ + This is the variant of the degradation model of BSRGAN from the paper + "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" + ---------- + sf: scale factor + isp_model: camera ISP model + Returns + ------- + img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] + hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] + """ + image = uint2single(image) + _, jpeg_prob, scale2_prob = 0.25, 0.9, 0.25 + # sf_ori = sf + + h1, w1 = image.shape[:2] + image = image.copy()[:w1 - w1 % sf, :h1 - h1 % sf, ...] # mod crop + h, w = image.shape[:2] + + # hq = image.copy() + + if sf == 4 and random.random() < scale2_prob: # downsample1 + if np.random.rand() < 0.5: + image = cv2.resize( + image, + (int(1 / 2 * image.shape[1]), int(1 / 2 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + image = imresize_np(image, 1 / 2, True) + image = np.clip(image, 0.0, 1.0) + sf = 2 + + shuffle_order = random.sample(range(7), 7) + idx1, idx2 = shuffle_order.index(2), shuffle_order.index(3) + if idx1 > idx2: # keep downsample3 last + shuffle_order[idx1], shuffle_order[idx2] = shuffle_order[ + idx2], shuffle_order[idx1] + + for i in shuffle_order: + + if i == 0: + image = add_blur_1(image, sf=sf) + elif i == 2: + a, b = image.shape[1], image.shape[0] + # downsample2 + if random.random() < 0.8: + sf1 = random.uniform(1, 2 * sf) + image = cv2.resize( + image, (int(1 / sf1 * image.shape[1]), + int(1 / sf1 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + k = fspecial('gaussian', 25, random.uniform(0.1, 0.6 * sf)) + k_shifted = shift_pixel(k, sf) + k_shifted = k_shifted / k_shifted.sum( + ) # blur with shifted kernel + image = ndimage.filters.convolve( + image, np.expand_dims(k_shifted, axis=2), mode='mirror') + image = image[0::sf, 0::sf, ...] # nearest downsampling + image = np.clip(image, 0.0, 1.0) + elif i == 3: + # downsample3 + image = cv2.resize( + image, (int(1 / sf * a), int(1 / sf * b)), + interpolation=random.choice([1, 2, 3])) + image = np.clip(image, 0.0, 1.0) + elif i == 4: + # add Gaussian noise + image = add_Gaussian_noise(image, noise_level1=1, noise_level2=2) + elif i == 5: + # add JPEG noise + if random.random() < jpeg_prob: + image = add_JPEG_noise(image) + + # add final JPEG compression noise + image = add_JPEG_noise(image) + image = single2uint(image) + return image + + +def add_blur_2(img, sf=4): + wd2 = 4.0 + sf + wd = 2.0 + 0.2 * sf + if random.random() < 0.5: + l1 = wd2 * random.random() + l2 = wd2 * random.random() + k = anisotropic_Gaussian( + ksize=2 * random.randint(2, 11) + 3, + theta=random.random() * np.pi, + l1=l1, + l2=l2) + else: + k = fspecial('gaussian', 2 * random.randint(2, 11) + 3, + wd * random.random()) + img = ndimage.filters.convolve( + img, np.expand_dims(k, axis=2), mode='mirror') + return img + + +def degradation_bsrgan(image, sf=4, isp_model=None): + """ + This is the variant of the degradation model of BSRGAN from the paper + "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" + ---------- + sf: scale factor + isp_model: camera ISP model + Returns + ------- + img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] + hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] + """ + image = uint2single(image) + _, jpeg_prob, scale2_prob = 0.25, 0.9, 0.25 + # sf_ori = sf + + h1, w1 = image.shape[:2] + image = image.copy()[:w1 - w1 % sf, :h1 - h1 % sf, ...] # mod crop + h, w = image.shape[:2] + + # hq = image.copy() + + if sf == 4 and random.random() < scale2_prob: # downsample1 + if np.random.rand() < 0.5: + image = cv2.resize( + image, + (int(1 / 2 * image.shape[1]), int(1 / 2 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + image = imresize_np(image, 1 / 2, True) + image = np.clip(image, 0.0, 1.0) + sf = 2 + + shuffle_order = random.sample(range(7), 7) + idx1, idx2 = shuffle_order.index(2), shuffle_order.index(3) + if idx1 > idx2: # keep downsample3 last + shuffle_order[idx1], shuffle_order[idx2] = shuffle_order[ + idx2], shuffle_order[idx1] + + for i in shuffle_order: + + if i == 0: + image = add_blur_2(image, sf=sf) + elif i == 1: + image = add_blur_2(image, sf=sf) + elif i == 2: + a, b = image.shape[1], image.shape[0] + # downsample2 + if random.random() < 0.75: + sf1 = random.uniform(1, 2 * sf) + image = cv2.resize( + image, (int(1 / sf1 * image.shape[1]), + int(1 / sf1 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + k = fspecial('gaussian', 25, random.uniform(0.1, 0.6 * sf)) + k_shifted = shift_pixel(k, sf) + k_shifted = k_shifted / k_shifted.sum( + ) # blur with shifted kernel + image = ndimage.filters.convolve( + image, np.expand_dims(k_shifted, axis=2), mode='mirror') + image = image[0::sf, 0::sf, ...] # nearest downsampling + image = np.clip(image, 0.0, 1.0) + elif i == 3: + # downsample3 + image = cv2.resize( + image, (int(1 / sf * a), int(1 / sf * b)), + interpolation=random.choice([1, 2, 3])) + image = np.clip(image, 0.0, 1.0) + elif i == 4: + # add Gaussian noise + image = add_Gaussian_noise(image, noise_level1=2, noise_level2=25) + elif i == 5: + # add JPEG noise + if random.random() < jpeg_prob: + image = add_JPEG_noise(image) + + # add final JPEG compression noise + image = add_JPEG_noise(image) + image = single2uint(image) + return image diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/diffusion.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..673f1eea1f67cc39a93f06ad69ae79ec6338a8c2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/diffusion.py @@ -0,0 +1,601 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch + +from .losses import discretized_gaussian_log_likelihood, kl_divergence + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + + # fn = lambda u: math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + def fn(u): + return math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + + betas.append(min(1.0 - fn(t2) / fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + noise = torch.randn_like(x0) if noise is None else noise + return _i(self.sqrt_alphas_cumprod, t, x0) * x0 + _i( + self.sqrt_one_minus_alphas_cumprod, t, x0) * noise + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + # no noise when t == 0 + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + assert self.mean_type == 'eps' + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + out = torch.cat( + [ + u_out[:, :3] + guide_scale * # noqa W504 + (y_out[:, :3] - u_out[:, :3]), + y_out[:, 3:] + ], + dim=1) + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - _i( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + xt) * xt + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + sigmas = eta * torch.sqrt((1 - alphas_prev) / # noqa W504 + (1 - alphas) * # noqa W504 + (1 - alphas / alphas_prev)) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b, c, h, w = x0.size() + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + - x0) / _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + # mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def loss(self, x0, t, model, model_kwargs={}, noise=None): + noise = torch.randn_like(x0) if noise is None else noise + xt = self.q_sample(x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', 'rescaled_l1']: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = (out - target).pow(1 if self.loss_type.endswith('l1') else 2 + ).abs().flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) + kl = kl.flatten(1).mean(dim=1) / math.log(2.0) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) + nll = nll.flatten(1).mean(dim=1) / math.log(2.0) + + # NLL for p(x0 | x1) and KL otherwise + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + r"""Compute the entire variational lower bound, measured in bits-per-dim. + """ + # prepare input and output + b, c, h, w = x0.size() + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/losses.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..46b9540afa35342db1ad696980150d5a8ed0b352 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/losses.py @@ -0,0 +1,36 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch + +__all__ = ['kl_divergence', 'discretized_gaussian_log_likelihood'] + + +def kl_divergence(mu1, logvar1, mu2, logvar2): + return 0.5 * ( + -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + # noqa W504 + ((mu1 - mu2)**2) * torch.exp(-logvar2)) + + +def standard_normal_cdf(x): + r"""A fast approximation of the cumulative distribution function of the standard normal. + """ + return 0.5 * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x0, mean, log_scale): + assert x0.shape == mean.shape == log_scale.shape + cx = x0 - mean + inv_stdv = torch.exp(-log_scale) + cdf_plus = standard_normal_cdf(inv_stdv * (cx + 1.0 / 255.0)) + cdf_min = standard_normal_cdf(inv_stdv * (cx - 1.0 / 255.0)) + log_cdf_plus = torch.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = torch.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = torch.where( + x0 < -0.999, log_cdf_plus, + torch.where(x0 > 0.999, log_one_minus_cdf_min, + torch.log(cdf_delta.clamp(min=1e-12)))) + assert log_probs.shape == x0.shape + return log_probs diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/metrics.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..c1023fa0293cd914116047a4a58db9b855079b43 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/metrics.py @@ -0,0 +1,127 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import numpy as np +import scipy.linalg as linalg +import torch + +__all__ = [ + 'get_fid_net', 'get_is_net', 'compute_fid', 'compute_prdc', 'compute_is' +] + + +def get_fid_net(resize_input=True, normalize_input=True): + r"""InceptionV3 network for the evaluation of Fréchet Inception Distance (FID). + + Args: + resize_input: whether or not to resize the input to (299, 299). + normalize_input: whether or not to normalize the input from range (0, 1) to range(-1, 1). + """ + from artist.models import InceptionV3 + return InceptionV3( + output_blocks=(3, ), + resize_input=resize_input, + normalize_input=normalize_input, + requires_grad=False, + use_fid_inception=True).eval().requires_grad_(False) + + +def get_is_net(resize_input=True, normalize_input=True): + r"""InceptionV3 network for the evaluation of Inception Score (IS). + + Args: + resize_input: whether or not to resize the input to (299, 299). + normalize_input: whether or not to normalize the input from range (0, 1) to range(-1, 1). + """ + from artist.models import InceptionV3 + return InceptionV3( + output_blocks=(4, ), + resize_input=resize_input, + normalize_input=normalize_input, + requires_grad=False, + use_fid_inception=False).eval().requires_grad_(False) + + +@torch.no_grad() +def compute_fid(real_feats, fake_feats, eps=1e-6): + r"""Compute Fréchet Inception Distance (FID). + + Args: + real_feats: [N, C]. + fake_feats: [N, C]. + """ + # check inputs + if isinstance(real_feats, torch.Tensor): + real_feats = real_feats.cpu().numpy().astype(np.float_) + if isinstance(fake_feats, torch.Tensor): + fake_feats = fake_feats.cpu().numpy().astype(np.float_) + + # real statistics + mu1 = np.mean(real_feats, axis=0) + sigma1 = np.cov(real_feats, rowvar=False) + + # fake statistics + mu2 = np.mean(fake_feats, axis=0) + sigma2 = np.cov(fake_feats, rowvar=False) + + # compute covmean + covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False) + if not np.isfinite(covmean).all(): + print( + f'FID calculation produces singular product; adding {eps} to diagonal of cov', + flush=True) + offset = np.eye(sigma1.shape[0]) * eps + covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset)) + + # numerical error might give slight imaginary component + if np.iscomplexobj(covmean): + if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3): + m = np.max(np.abs(covmean.imag)) + raise ValueError('Imaginary component {}'.format(m)) + covmean = covmean.real + + # compute Fréchet distance + diff = mu1 - mu2 + fid = diff.dot(diff) + np.trace(sigma1) + np.trace( + sigma2) - 2 * np.trace(covmean) + return fid.item() + + +@torch.no_grad() +def compute_prdc(real_feats, fake_feats, knn=5): + r"""Compute precision, recall, density, and coverage given two manifolds. + + Args: + real_feats: [N, C]. + fake_feats: [N, C]. + knn: the number of nearest neighbors to consider. + """ + # distances + real_kth = -(-torch.cdist(real_feats, real_feats)).topk( + k=knn, dim=1)[0][:, -1] + fake_kth = -(-torch.cdist(fake_feats, fake_feats)).topk( + k=knn, dim=1)[0][:, -1] + dists = torch.cdist(real_feats, fake_feats) + + # metrics + precision = (dists < real_kth.unsqueeze(1)).any( + dim=0).float().mean().item() + recall = (dists < fake_kth.unsqueeze(0)).any(dim=1).float().mean().item() + density = (dists < real_kth.unsqueeze(1)).float().sum( + dim=0).mean().item() / knn + coverage = (dists.min(dim=1)[0] < real_kth).float().mean().item() + return precision, recall, density, coverage + + +@torch.no_grad() +def compute_is(logits, num_splits=10): + preds = logits.softmax(dim=1).cpu().numpy() + split_scores = [] + for k in range(num_splits): + part = preds[k * (len(logits) // num_splits):(k + 1) + * (len(logits) // num_splits), :] + py = np.mean(part, axis=0) + scores = [] + for i in range(part.shape[0]): + pyx = part[i, :] + scores.append(entropy(pyx, py)) + split_scores.append(np.exp(np.mean(scores))) + return np.mean(split_scores), np.std(split_scores) diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_color.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_color.py new file mode 100644 index 0000000000000000000000000000000000000000..75692836fc1703253c94e5f361b47b6199b09ef6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_color.py @@ -0,0 +1,221 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import colorsys +import random + +__all__ = ['RandomColor', 'rand_color'] + +COLORMAP = { + 'blue': { + 'hue_range': [179, 257], + 'lower_bounds': [[20, 100], [30, 86], [40, 80], [50, 74], [60, 60], + [70, 52], [80, 44], [90, 39], [100, 35]] + }, + 'green': { + 'hue_range': [63, 178], + 'lower_bounds': [[30, 100], [40, 90], [50, 85], [60, 81], [70, 74], + [80, 64], [90, 50], [100, 40]] + }, + 'monochrome': { + 'hue_range': [0, 0], + 'lower_bounds': [[0, 0], [100, 0]] + }, + 'orange': { + 'hue_range': [19, 46], + 'lower_bounds': [[20, 100], [30, 93], [40, 88], [50, 86], [60, 85], + [70, 70], [100, 70]] + }, + 'pink': { + 'hue_range': [283, 334], + 'lower_bounds': [[20, 100], [30, 90], [40, 86], [60, 84], [80, 80], + [90, 75], [100, 73]] + }, + 'purple': { + 'hue_range': [258, 282], + 'lower_bounds': [[20, 100], [30, 87], [40, 79], [50, 70], [60, 65], + [70, 59], [80, 52], [90, 45], [100, 42]] + }, + 'red': { + 'hue_range': [-26, 18], + 'lower_bounds': [[20, 100], [30, 92], [40, 89], [50, 85], [60, 78], + [70, 70], [80, 60], [90, 55], [100, 50]] + }, + 'yellow': { + 'hue_range': [47, 62], + 'lower_bounds': [[25, 100], [40, 94], [50, 89], [60, 86], [70, 84], + [80, 82], [90, 80], [100, 75]] + } +} + + +class RandomColor(object): + + def __init__(self, seed=None): + self.colormap = COLORMAP + self.random = random.Random(seed) + + for color_name, color_attrs in self.colormap.items(): + lower_bounds = color_attrs['lower_bounds'] + s_min = lower_bounds[0][0] + s_max = lower_bounds[len(lower_bounds) - 1][0] + + b_min = lower_bounds[len(lower_bounds) - 1][1] + b_max = lower_bounds[0][1] + + self.colormap[color_name]['saturation_range'] = [s_min, s_max] + self.colormap[color_name]['brightness_range'] = [b_min, b_max] + + def generate(self, hue=None, luminosity=None, count=1, format_='hex'): + colors = [] + for _ in range(count): + # First we pick a hue (H) + H = self.pick_hue(hue) + + # Then use H to determine saturation (S) + S = self.pick_saturation(H, hue, luminosity) + + # Then use S and H to determine brightness (B). + B = self.pick_brightness(H, S, luminosity) + + # Then we return the HSB color in the desired format + colors.append(self.set_format([H, S, B], format_)) + + return colors + + def pick_hue(self, hue): + hue_range = self.get_hue_range(hue) + hue = self.random_within(hue_range) + + # Instead of storing red as two seperate ranges, + # we group them, using negative numbers + if (hue < 0): + hue += 360 + + return hue + + def pick_saturation(self, hue, hue_name, luminosity): + + if luminosity == 'random': + return self.random_within([0, 100]) + + if hue_name == 'monochrome': + return 0 + + saturation_range = self.get_saturation_range(hue) + + s_min = saturation_range[0] + s_max = saturation_range[1] + + if luminosity == 'bright': + s_min = 55 + elif luminosity == 'dark': + s_min = s_max - 10 + elif luminosity == 'light': + s_max = 55 + + return self.random_within([s_min, s_max]) + + def pick_brightness(self, H, S, luminosity): + b_min = self.get_minimum_brightness(H, S) + b_max = 100 + + if luminosity == 'dark': + b_max = b_min + 20 + elif luminosity == 'light': + b_min = (b_max + b_min) / 2 + elif luminosity == 'random': + b_min = 0 + b_max = 100 + + return self.random_within([b_min, b_max]) + + def set_format(self, hsv, format_): + if 'hsv' in format_: + color = hsv + elif 'rgb' in format_: + color = self.hsv_to_rgb(hsv) + elif 'hex' in format_: + r, g, b = self.hsv_to_rgb(hsv) + return '#%02x%02x%02x' % (r, g, b) + else: + return 'unrecognized format' + + if 'Array' in format_ or format_ == 'hex': + return color + else: + prefix = format_[:3] + color_values = [str(x) for x in color] + return '%s(%s)' % (prefix, ', '.join(color_values)) + + def get_minimum_brightness(self, H, S): + lower_bounds = self.get_color_info(H)['lower_bounds'] + + for i in range(len(lower_bounds) - 1): + s1 = lower_bounds[i][0] + v1 = lower_bounds[i][1] + + s2 = lower_bounds[i + 1][0] + v2 = lower_bounds[i + 1][1] + + if s1 <= S <= s2: + m = (v2 - v1) / (s2 - s1) + b = v1 - m * s1 + + return m * S + b + + return 0 + + def get_hue_range(self, color_input): + if color_input and color_input.isdigit(): + number = int(color_input) + + if 0 < number < 360: + return [number, number] + + elif color_input and color_input in self.colormap: + color = self.colormap[color_input] + if 'hue_range' in color: + return color['hue_range'] + + else: + return [0, 360] + + def get_saturation_range(self, hue): + return self.get_color_info(hue)['saturation_range'] + + def get_color_info(self, hue): + # Maps red colors to make picking hue easier + if 334 <= hue <= 360: + hue -= 360 + + for color_name, color in self.colormap.items(): + if color['hue_range'] and color['hue_range'][0] <= hue <= color[ + 'hue_range'][1]: + return self.colormap[color_name] + + # this should probably raise an exception + return 'Color not found' + + def random_within(self, r): + return self.random.randint(int(r[0]), int(r[1])) + + @classmethod + def hsv_to_rgb(cls, hsv): + h, s, v = hsv + h = 1 if h == 0 else h + h = 359 if h == 360 else h + + h = float(h) / 360 + s = float(s) / 100 + v = float(v) / 100 + + rgb = colorsys.hsv_to_rgb(h, s, v) + return [int(c * 255) for c in rgb] + + +def rand_color(): + generator = RandomColor() + hue = random.choice(list(COLORMAP.keys())) + color = generator.generate(hue=hue, count=1, format_='rgb')[0] + color = color[color.find('(') + 1:color.find(')')] + color = tuple([int(u) for u in color.split(',')]) + return color diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_mask.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..bda1ec11acaf1ee6a54dbb34ed4cbd174c8a2c04 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/random_mask.py @@ -0,0 +1,80 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import cv2 +import numpy as np + +__all__ = ['make_irregular_mask', 'make_rectangle_mask', 'make_uncrop'] + + +def make_irregular_mask(w, + h, + max_angle=4, + max_length=200, + max_width=100, + min_strokes=1, + max_strokes=5, + mode='line'): + # initialize mask + assert mode in ['line', 'circle', 'square'] + mask = np.zeros((h, w), np.float32) + + # draw strokes + num_strokes = np.random.randint(min_strokes, max_strokes + 1) + for i in range(num_strokes): + x1 = np.random.randint(w) + y1 = np.random.randint(h) + for j in range(1 + np.random.randint(5)): + angle = 0.01 + np.random.randint(max_angle) + if i % 2 == 0: + angle = 2 * 3.1415926 - angle + length = 10 + np.random.randint(max_length) + radius = 5 + np.random.randint(max_width) + x2 = np.clip((x1 + length * np.sin(angle)).astype(np.int32), 0, w) + y2 = np.clip((y1 + length * np.cos(angle)).astype(np.int32), 0, h) + if mode == 'line': + cv2.line(mask, (x1, y1), (x2, y2), 1.0, radius) + elif mode == 'circle': + cv2.circle( + mask, (x1, y1), radius=radius, color=1.0, thickness=-1) + elif mode == 'square': + radius = radius // 2 + mask[y1 - radius:y1 + radius, x1 - radius:x1 + radius] = 1 + x1, y1 = x2, y2 + return mask + + +def make_rectangle_mask(w, + h, + margin=10, + min_size=30, + max_size=150, + min_strokes=1, + max_strokes=4): + # initialize mask + mask = np.zeros((h, w), np.float32) + + # draw rectangles + num_strokes = np.random.randint(min_strokes, max_strokes + 1) + for i in range(num_strokes): + box_w = np.random.randint(min_size, max_size) + box_h = np.random.randint(min_size, max_size) + x1 = np.random.randint(margin, w - margin - box_w + 1) + y1 = np.random.randint(margin, h - margin - box_h + 1) + mask[y1:y1 + box_h, x1:x1 + box_w] = 1 + return mask + + +def make_uncrop(w, h): + # initialize mask + mask = np.zeros((h, w), np.float32) + + # randomly halve the image + side = np.random.choice([0, 1, 2, 3]) + if side == 0: + mask[:h // 2, :] = 1 + elif side == 1: + mask[h // 2:, :] = 1 + elif side == 2: + mask[:, :w // 2] = 1 + elif side == 2: + mask[:, w // 2:] = 1 + return mask diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/svd.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/svd.py new file mode 100644 index 0000000000000000000000000000000000000000..96f7e825bde0557e425a69071a65526c6fe0bdf9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/svd.py @@ -0,0 +1,153 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +r"""SVD of linear degradation matrices described in the paper + ``Denoising Diffusion Restoration Models.'' + @article{kawar2022denoising, + title={Denoising Diffusion Restoration Models}, + author={Bahjat Kawar and Michael Elad and Stefano Ermon and Jiaming Song}, + year={2022}, + journal={arXiv preprint arXiv:2201.11793}, + } +""" +import torch + +__all__ = ['SVD', 'IdentitySVD', 'DenoiseSVD', 'ColorizationSVD'] + + +class SVD(object): + r"""SVD decomposition of a matrix, i.e., H = UDV^T. + NOTE: assume that all inputs (i.e., h, x) are of shape [B, CHW]. + """ + + def __init__(self, h): + self.u, self.d, self.v = torch.svd(h, some=False) + self.ut = self.u.t() + self.vt = self.v.t() + self.d[self.d < 1e-3] = 0 + + def U(self, x): + return torch.matmul(self.u, x) + + def Ut(self, x): + return torch.matmul(self.ut, x) + + def V(self, x): + return torch.matmul(self.v, x) + + def Vt(self, x): + return torch.matmul(self.vt, x) + + @property + def D(self): + return self.d + + def H(self, x): + return self.U(self.D * self.Vt(x)[:, :self.D.size(0)]) + + def Ht(self, x): + return self.V(self._pad(self.D * self.Ut(x)[:, :self.D.size(0)])) + + def Hinv(self, x): + r"""Multiplies x by the pseudo inverse of H. + """ + x = self.Ut(x) + x[:, :self.D.size(0)] = x[:, :self.D.size(0)] / self.D + return self.V(self._pad(x)) + + def _pad(self, x): + o = x.new_zeros(x.size(0), self.v.size(0)) + o[:, :self.u.size(0)] = x.view(x.size(0), -1) + return o + + def to(self, *args, **kwargs): + r"""Update the data type and device of UDV matrices. + """ + for k, v in self.__dict__.items(): + if isinstance(v, torch.Tensor): + setattr(self, k, v.to(*args, **kwargs)) + return self + + +class IdentitySVD(SVD): + + def __init__(self, c, h, w): + self.d = torch.ones(c * h * w) + + def U(self, x): + return x.clone() + + def Ut(self, x): + return x.clone() + + def V(self, x): + return x.clone() + + def Vt(self, x): + return x.clone() + + def H(self, x): + return x.clone() + + def Ht(self, x): + return x.clone() + + def Hinv(self, x): + return x.clone() + + def _pad(self, x): + return x.clone() + + +class DenoiseSVD(SVD): + + def __init__(self, c, h, w): + self.num_entries = c * h * w + self.d = torch.ones(self.num_entries) + + def U(self, x): + return x.clone() + + def Ut(self, x): + return x.clone() + + def V(self, x): + return x.clone() + + def Vt(self, x): + return x.clone() + + def _pad(self, x): + return x.clone() + + +class ColorizationSVD(SVD): + + def __init__(self, c, h, w): + self.color_dim = c + self.num_pixels = h * w + self.u, self.d, self.v = torch.svd(torch.ones(1, c) / c, some=False) + self.vt = self.v.t() + + def U(self, x): + return self.u[0, 0] * x + + def Ut(self, x): + return self.u[0, 0] * x + + def V(self, x): + return torch.einsum('ij,bjn->bin', self.v, + x.view(x.size(0), self.color_dim, + self.num_pixels)).flatten(1) + + def Vt(self, x): + return torch.einsum('ij,bjn->bin', self.vt, + x.view(x.size(0), self.color_dim, + self.num_pixels)).flatten(1) + + @property + def D(self): + return self.d.repeat(self.num_pixels) + + def _pad(self, x): + o = x.new_zeros(x.size(0), self.color_dim * self.num_pixels) + o[:, :self.num_pixels] = x + return o diff --git a/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/utils.py b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2aacedcd480862098e3c00bd3183b3715dc8e72 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_to_image_translation/ops/utils.py @@ -0,0 +1,225 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import base64 +import binascii +import hashlib +import math +import os +import os.path as osp +import zipfile +from io import BytesIO +from multiprocessing.pool import ThreadPool as Pool + +import cv2 +import json +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + +from .random_color import rand_color + +__all__ = [ + 'ceil_divide', 'to_device', 'rand_name', 'ema', 'parallel', 'unzip', + 'load_state_dict', 'inverse_indices', 'detect_duplicates', 'md5', 'rope', + 'format_state', 'breakup_grid', 'viz_anno_geometry', 'image_to_base64' +] + +TFS_CLIENT = None + + +def ceil_divide(a, b): + return int(math.ceil(a / b)) + + +def to_device(batch, device, non_blocking=False): + if isinstance(batch, (list, tuple)): + return type(batch)([to_device(u, device, non_blocking) for u in batch]) + elif isinstance(batch, dict): + return type(batch)([(k, to_device(v, device, non_blocking)) + for k, v in batch.items()]) + elif isinstance(batch, torch.Tensor): + return batch.to(device, non_blocking=non_blocking) + return batch + + +def rand_name(length=8, suffix=''): + name = binascii.b2a_hex(os.urandom(length)).decode('utf-8') + if suffix: + if not suffix.startswith('.'): + suffix = '.' + suffix + name += suffix + return name + + +@torch.no_grad() +def ema(net_ema, net, beta, copy_buffer=False): + assert 0.0 <= beta <= 1.0 + for p_ema, p in zip(net_ema.parameters(), net.parameters()): + p_ema.copy_(p.lerp(p_ema, beta)) + if copy_buffer: + for b_ema, b in zip(net_ema.buffers(), net.buffers()): + b_ema.copy_(b) + + +def parallel(func, args_list, num_workers=32, timeout=None): + assert isinstance(args_list, list) + if not isinstance(args_list[0], tuple): + args_list = [(args, ) for args in args_list] + if num_workers == 0: + return [func(*args) for args in args_list] + with Pool(processes=num_workers) as pool: + results = [pool.apply_async(func, args) for args in args_list] + results = [res.get(timeout=timeout) for res in results] + return results + + +def unzip(filename, dst_dir=None): + if dst_dir is None: + dst_dir = osp.dirname(filename) + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(dst_dir) + + +def load_state_dict(module, state_dict, drop_prefix=''): + # find incompatible key-vals + src, dst = state_dict, module.state_dict() + if drop_prefix: + src = type(src)([ + (k[len(drop_prefix):] if k.startswith(drop_prefix) else k, v) + for k, v in src.items() + ]) + missing = [k for k in dst if k not in src] + unexpected = [k for k in src if k not in dst] + unmatched = [ + k for k in src.keys() & dst.keys() if src[k].shape != dst[k].shape + ] + + # keep only compatible key-vals + incompatible = set(unexpected + unmatched) + src = type(src)([(k, v) for k, v in src.items() if k not in incompatible]) + module.load_state_dict(src, strict=False) + + # report incompatible key-vals + if len(missing) != 0: + print(' Missing: ' + ', '.join(missing), flush=True) + if len(unexpected) != 0: + print(' Unexpected: ' + ', '.join(unexpected), flush=True) + if len(unmatched) != 0: + print(' Shape unmatched: ' + ', '.join(unmatched), flush=True) + + +def inverse_indices(indices): + r"""Inverse map of indices. + E.g., if A[indices] == B, then B[inv_indices] == A. + """ + inv_indices = torch.empty_like(indices) + inv_indices[indices] = torch.arange(len(indices)).to(indices) + return inv_indices + + +def detect_duplicates(feats, thr=0.9): + assert feats.ndim == 2 + + # compute simmat + feats = F.normalize(feats, p=2, dim=1) + simmat = torch.mm(feats, feats.T) + simmat.triu_(1) + torch.cuda.synchronize() + + # detect duplicates + mask = ~simmat.gt(thr).any(dim=0) + return torch.where(mask)[0] + + +def md5(filename): + with open(filename, 'rb') as f: + return hashlib.md5(f.read()).hexdigest() + + +def rope(x): + r"""Apply rotary position embedding on x of shape [B, *(spatial dimensions), C]. + """ + # reshape + shape = x.shape + x = x.view(x.size(0), -1, x.size(-1)) + l, c = x.shape[-2:] + assert c % 2 == 0 + half = c // 2 + + # apply rotary position embedding on x + sinusoid = torch.outer( + torch.arange(l).to(x), + torch.pow(10000, -torch.arange(half).to(x).div(half))) + sin, cos = torch.sin(sinusoid), torch.cos(sinusoid) + x1, x2 = x.chunk(2, dim=-1) + x = torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1) + + # reshape back + return x.view(shape) + + +def format_state(state, filename=None): + r"""For comparing/aligning state_dict. + """ + content = '\n'.join([f'{k}\t{tuple(v.shape)}' for k, v in state.items()]) + if filename: + with open(filename, 'w') as f: + f.write(content) + + +def breakup_grid(img, grid_size): + r"""The inverse operator of ``torchvision.utils.make_grid``. + """ + # params + nrow = img.height // grid_size + ncol = img.width // grid_size + wrow = wcol = 2 # NOTE: use default values here + + # collect grids + grids = [] + for i in range(nrow): + for j in range(ncol): + x1 = j * grid_size + (j + 1) * wcol + y1 = i * grid_size + (i + 1) * wrow + grids.append(img.crop((x1, y1, x1 + grid_size, y1 + grid_size))) + return grids + + +def viz_anno_geometry(item): + r"""Visualize an annotation item from SmartLabel. + """ + if isinstance(item, str): + item = json.loads(item) + assert isinstance(item, dict) + + # read image + orig_img = read_image(item['image_url'], retry=100) + img = cv2.cvtColor(np.asarray(orig_img), cv2.COLOR_BGR2RGB) + + # loop over geometries + for geometry in item['sd_result']['items']: + # params + poly_img = img.copy() + color = rand_color() + points = np.array(geometry['meta']['geometry']).round().astype(int) + line_color = tuple([int(u * 0.55) for u in color]) + + # draw polygons + poly_img = cv2.fillPoly(poly_img, pts=[points], color=color) + poly_img = cv2.polylines( + poly_img, + pts=[points], + isClosed=True, + color=line_color, + thickness=2) + + # mixing + img = np.clip(0.25 * img + 0.75 * poly_img, 0, 255).astype(np.uint8) + return orig_img, Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + + +def image_to_base64(img, format='JPEG'): + buffer = BytesIO() + img.save(buffer, format=format) + code = base64.b64encode(buffer.getvalue()).decode('utf-8') + return code diff --git a/runtime/Lib/site_packages/models/cv/image_try_on/__init__.py b/runtime/Lib/site_packages/models/cv/image_try_on/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..352d21b90f0a7819f3d2a760a9ef758bf9977858 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_try_on/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .try_on_infer import SALForImageTryOn + +else: + _import_structure = {'try_on_infer': ['SALForImageTryOn']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_try_on/generator.py b/runtime/Lib/site_packages/models/cv/image_try_on/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..47e2bc1a5540d017cb3e5f141d384806d9476190 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_try_on/generator.py @@ -0,0 +1,474 @@ +# The implementation here is modified based on spade, +# originally Apache 2.0 License and publicly avaialbe at https://github.com/NVlabs/SPADE + +import functools +import os + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.parallel +from torchvision import models + + +class ResidualBlock(nn.Module): + + def __init__(self, in_features=64, norm_layer=nn.BatchNorm2d): + super(ResidualBlock, self).__init__() + self.relu = nn.PReLU() + if norm_layer is None: + self.block = nn.Sequential( + nn.Conv2d(in_features, in_features, 3, 1, 1, bias=False), + nn.PReLU(), + nn.Conv2d(in_features, in_features, 3, 1, 1, bias=False), + ) + else: + self.block = nn.Sequential( + nn.Conv2d(in_features, in_features, 3, 1, 1, bias=False), + norm_layer(in_features), nn.PReLU(), + nn.Conv2d(in_features, in_features, 3, 1, 1, bias=False), + norm_layer(in_features)) + + def forward(self, x): + residual = x + out = self.block(x) + out += residual + out = self.relu(out) + return out + + +# Defines the submodule with skip connection. +# X -------------------identity---------------------- X +# |-- downsampling -- |submodule| -- upsampling --| +class ResUnetSkipConnectionBlock(nn.Module): + + def __init__(self, + outer_nc, + inner_nc, + input_nc=None, + submodule=None, + outermost=False, + innermost=False, + norm_layer=nn.BatchNorm2d, + use_dropout=False): + super(ResUnetSkipConnectionBlock, self).__init__() + self.outermost = outermost + use_bias = norm_layer == nn.InstanceNorm2d + + if input_nc is None: + input_nc = outer_nc + downconv = nn.Conv2d( + input_nc, + inner_nc, + kernel_size=3, + stride=2, + padding=1, + bias=use_bias) + # add two resblock + res_downconv = [ + ResidualBlock(inner_nc, norm_layer), + ResidualBlock(inner_nc, norm_layer), + ResidualBlock(inner_nc, norm_layer) + ] + res_upconv = [ + ResidualBlock(outer_nc, norm_layer), + ResidualBlock(outer_nc, norm_layer), + ResidualBlock(outer_nc, norm_layer) + ] + + downrelu = nn.PReLU() + uprelu = nn.PReLU() + if norm_layer is not None: + downnorm = norm_layer(inner_nc) + upnorm = norm_layer(outer_nc) + + if outermost: + upsample = nn.Upsample(scale_factor=2, mode='nearest') + upconv = nn.Conv2d( + inner_nc * 2, + outer_nc, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias) + down = [downconv, downrelu] + res_downconv + up = [upsample, upconv] + model = down + [submodule] + up + elif innermost: + upsample = nn.Upsample(scale_factor=2, mode='nearest') + upconv = nn.Conv2d( + inner_nc, + outer_nc, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias) + down = [downconv, downrelu] + res_downconv + if norm_layer is None: + up = [upsample, upconv, uprelu] + res_upconv + else: + up = [upsample, upconv, upnorm, uprelu] + res_upconv + model = down + up + else: + upsample = nn.Upsample(scale_factor=2, mode='nearest') + upconv = nn.Conv2d( + inner_nc * 2, + outer_nc, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias) + if norm_layer is None: + down = [downconv, downrelu] + res_downconv + up = [upsample, upconv, uprelu] + res_upconv + else: + down = [downconv, downnorm, downrelu] + res_downconv + up = [upsample, upconv, upnorm, uprelu] + res_upconv + + if use_dropout: + model = down + [submodule] + up + [nn.Dropout(0.5)] + else: + model = down + [submodule] + up + + self.model = nn.Sequential(*model) + + def forward(self, x): + if self.outermost: + return self.model(x) + else: + return torch.cat([x, self.model(x)], 1) + + +class LandmarkNorm(nn.Module): + + def __init__(self, param_free_norm_type, norm_nc, label_nc): + super().__init__() + + if param_free_norm_type == 'instance': + self.param_free_norm = nn.InstanceNorm2d(norm_nc, affine=False) + elif param_free_norm_type == 'syncbatch': + self.param_free_norm = SynchronizedBatchNorm2d( + norm_nc, affine=False) + elif param_free_norm_type == 'batch': + self.param_free_norm = nn.BatchNorm2d(norm_nc, affine=False) + else: + raise ValueError( + '%s is not a recognized param-free norm type in LandmarkNorm' + % param_free_norm_type) + + nhidden = 128 + ks = 3 + pw = ks // 2 + self.mlp_shared = nn.Sequential( + nn.Conv2d(label_nc, nhidden, kernel_size=ks, padding=pw), + nn.ReLU()) + self.mlp_gamma = nn.Conv2d( + nhidden, norm_nc, kernel_size=ks, padding=pw) + self.mlp_beta = nn.Conv2d(nhidden, norm_nc, kernel_size=ks, padding=pw) + + def forward(self, x, segmap): + + # Part 1. generate parameter-free normalized activations + normalized = self.param_free_norm(x) + + # Part 2. produce scaling and bias conditioned on semantic map + segmap = F.interpolate(segmap, size=x.size()[2:], mode='nearest') + actv = self.mlp_shared(segmap) + gamma = self.mlp_gamma(actv) + beta = self.mlp_beta(actv) + + # apply scale and bias + out = normalized * (1 + gamma) + beta + + return out + + +class LandmarkNormResnetBlock(nn.Module): + + def __init__(self, fin, fout): + super().__init__() + # Attributes + self.learned_shortcut = (fin != fout) + fmiddle = min(fin, fout) + + # create conv layers + self.conv_0 = nn.Conv2d(fin, fmiddle, kernel_size=3, padding=1) + self.conv_1 = nn.Conv2d(fmiddle, fout, kernel_size=3, padding=1) + if self.learned_shortcut: + self.conv_s = nn.Conv2d(fin, fout, kernel_size=1, bias=False) + + landmarknorm_config_str = 'batch' + semantic_nc = 32 + self.norm_0 = LandmarkNorm(landmarknorm_config_str, fin, semantic_nc) + self.norm_1 = LandmarkNorm(landmarknorm_config_str, fmiddle, + semantic_nc) + if self.learned_shortcut: + self.norm_s = LandmarkNorm(landmarknorm_config_str, fin, + semantic_nc) + + def forward(self, x, seg): + x_s = self.shortcut(x, seg) + + dx = self.conv_0(self.actvn(self.norm_0(x, seg))) + dx = self.conv_1(self.actvn(self.norm_1(dx, seg))) + out = x_s + dx + return out + + def shortcut(self, x, seg): + if self.learned_shortcut: + x_s = self.conv_s(self.norm_s(x, seg)) + else: + x_s = x + return x_s + + def actvn(self, x): + return F.leaky_relu(x, 2e-1) + + +class VTONGenerator(nn.Module): + """ initialize the try on generator model + """ + + def __init__(self, + input_nc, + output_nc, + num_downs, + ngf=64, + norm_layer=nn.BatchNorm2d, + use_dropout=False): + super(VTONGenerator, self).__init__() + use_bias = norm_layer == nn.InstanceNorm2d + + ngf_list = [ngf * 1, ngf * 2, ngf * 4, ngf * 8, ngf * 8] + self.num_downs = num_downs + self.Encoder = [] + self.Decoder = [] + self.LMnorm = [] + + for i in range(num_downs): + # Encoder + if i == 0: + in_nc = input_nc + inner_nc = ngf_list[i] + else: + in_nc, inner_nc = ngf_list[i - 1], ngf_list[i] + + downconv = nn.Conv2d( + in_nc, + inner_nc, + kernel_size=3, + stride=2, + padding=1, + bias=use_bias) + downnorm = norm_layer(inner_nc) + downrelu = nn.PReLU() + res_downconv = [ + ResidualBlock(inner_nc, norm_layer), + ResidualBlock(inner_nc, norm_layer), + ResidualBlock(inner_nc, norm_layer) + ] + + # Decoder + if i == (num_downs - 1): + outer_nc = ngf // 2 + inner_nc = 2 * ngf_list[0] + elif i == 0: + inner_nc, outer_nc = ngf_list[num_downs - i + - 1], ngf_list[num_downs - i - 1] + else: + inner_nc, outer_nc = 2 * ngf_list[num_downs - i + - 1], ngf_list[num_downs - i + - 2] + + upsample = nn.Upsample(scale_factor=2, mode='nearest') + upconv = nn.Conv2d( + inner_nc, + outer_nc, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias) + upnorm = norm_layer(outer_nc) + uprelu = nn.PReLU() + res_upconv = [ + ResidualBlock(outer_nc, norm_layer), + ResidualBlock(outer_nc, norm_layer), + ResidualBlock(outer_nc, norm_layer) + ] + + if i == 0: + encoderLayer = [downconv, downrelu] + res_downconv + decoderLayer = [upsample, upconv, upnorm, uprelu] + res_upconv + elif i == (num_downs - 1): + encoderLayer = [downconv, downrelu] + res_downconv + decoderLayer = [upsample, upconv] + else: + encoderLayer = [downconv, downnorm, downrelu] + res_downconv + decoderLayer = [upsample, upconv, upnorm, uprelu] + res_upconv + + encoderLayer = nn.Sequential(*encoderLayer) + decoderLayer = nn.Sequential(*decoderLayer) + self.Encoder.append(encoderLayer) + self.Decoder.append(decoderLayer) + + LMnorm = LandmarkNormResnetBlock(outer_nc, outer_nc) + self.LMnorm.append(LMnorm) + + self.Encoder = nn.ModuleList(self.Encoder) + self.Decoder = nn.ModuleList(self.Decoder) + self.LMnorm = nn.ModuleList(self.LMnorm) + + self.conv_img = nn.Conv2d(ngf // 2, 3, kernel_size=3, padding=1) + self.act = nn.PReLU() + self.tanh = nn.Tanh() + + def forward(self, inputs, p_point_heatmap): + en_fea = [] + x = inputs + for i in range(self.num_downs): + x = self.Encoder[i](x) + if i < (self.num_downs - 1): + en_fea.append(x) + + for i in range(self.num_downs): + if i != 0: + x = torch.cat([en_fea[-i], x], 1) + x = self.Decoder[i](x) + x = self.LMnorm[i](x, p_point_heatmap) + + x = self.conv_img(self.act(x)) + x = self.tanh(x) + return x + + +class ResUnetGenerator(nn.Module): + + def __init__(self, + input_nc, + output_nc, + num_downs, + ngf=64, + norm_layer=nn.BatchNorm2d, + use_dropout=False): + super(ResUnetGenerator, self).__init__() + # construct unet structure + unet_block = ResUnetSkipConnectionBlock( + ngf * 8, + ngf * 8, + input_nc=None, + submodule=None, + norm_layer=norm_layer, + innermost=True) + + for i in range(num_downs - 5): + unet_block = ResUnetSkipConnectionBlock( + ngf * 8, + ngf * 8, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer, + use_dropout=use_dropout) + unet_block = ResUnetSkipConnectionBlock( + ngf * 4, + ngf * 8, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + unet_block = ResUnetSkipConnectionBlock( + ngf * 2, + ngf * 4, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + unet_block = ResUnetSkipConnectionBlock( + ngf, + ngf * 2, + input_nc=None, + submodule=unet_block, + norm_layer=norm_layer) + unet_block = ResUnetSkipConnectionBlock( + output_nc, + ngf, + input_nc=input_nc, + submodule=unet_block, + outermost=True, + norm_layer=norm_layer) + + self.model = unet_block + + def forward(self, input): + return self.model(input) + + +class Vgg19(nn.Module): + + def __init__(self, requires_grad=False): + super(Vgg19, self).__init__() + vgg_pretrained_features = models.vgg19(pretrained=False) + # for torchvision >= 0.4.0 or torch >= 1.2.0 + for x in vgg_pretrained_features.modules(): + if isinstance(x, nn.MaxPool2d) or isinstance( + x, nn.AdaptiveAvgPool2d): + x.ceil_mode = True + vgg_pretrained_features.load_state_dict(torch.load(vgg_path)) + vgg_pretrained_features = vgg_pretrained_features.features + + self.slice1 = nn.Sequential() + self.slice2 = nn.Sequential() + self.slice3 = nn.Sequential() + self.slice4 = nn.Sequential() + self.slice5 = nn.Sequential() + for x in range(2): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(2, 7): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(7, 12): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(12, 21): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(21, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h_relu1 = self.slice1(X) + h_relu2 = self.slice2(h_relu1) + h_relu3 = self.slice3(h_relu2) + h_relu4 = self.slice4(h_relu3) + h_relu5 = self.slice5(h_relu4) + out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5] + return out + + +class VGGLoss(nn.Module): + + def __init__(self, layids=None): + super(VGGLoss, self).__init__() + self.vgg = Vgg19() + self.vgg.cuda() + self.criterion = nn.L1Loss() + self.weights = [1.0 / 32, 1.0 / 16, 1.0 / 8, 1.0 / 4, 1.0] + self.layids = layids + + def forward(self, x, y): + x_vgg, y_vgg = self.vgg(x), self.vgg(y) + loss = 0 + if self.layids is None: + self.layids = list(range(len(x_vgg))) + for i in self.layids: + loss += self.weights[i] * self.criterion(x_vgg[i], + y_vgg[i].detach()) + return loss + + +def load_checkpoint_parallel(model, checkpoint_path): + + checkpoint = torch.load( + checkpoint_path, map_location=lambda storage, loc: storage) + checkpoint_new = model.state_dict() + for param in checkpoint_new: + checkpoint_new[param] = checkpoint[param] + model.load_state_dict(checkpoint_new) diff --git a/runtime/Lib/site_packages/models/cv/image_try_on/landmark.py b/runtime/Lib/site_packages/models/cv/image_try_on/landmark.py new file mode 100644 index 0000000000000000000000000000000000000000..f74416d5408004ebb343696d98910771566d9321 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_try_on/landmark.py @@ -0,0 +1,431 @@ +# The implementation here is modified based on hrnet, +# originally Apache 2.0 License and publicly avaialbe at https://github.com/HRNet/HigherHRNet-Human-Pose-Estimation + +import logging +import os + +import torch +import torch.nn as nn +from torch.nn import functional as F + +from modelscope.models.cv.body_2d_keypoints.hrnet_basic_modules import ( + BasicBlock, Bottleneck, HighResolutionModule, conv3x3) + +BN_MOMENTUM = 0.1 +logger = logging.getLogger(__name__) + +blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + +class DownSample(nn.Module): + + def __init__(self, in_channels, out_channels): + super(DownSample, self).__init__() + self.block = nn.Sequential( + nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False), nn.BatchNorm2d(in_channels), nn.PReLU()) + + def forward(self, x): + return self.block(x) + + +class LandmarkNet(nn.Module): + + def __init__(self, cfg, in_channel=3, class_num=3, **kwargs): + self.inplanes = 64 + extra = cfg['MODEL']['EXTRA'] + super(LandmarkNet, self).__init__() + + # stem net + self.conv1 = nn.Conv2d( + in_channel, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d( + 64, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(Bottleneck, 64, 4) + + self.stage2_cfg = extra['STAGE2'] + num_channels = self.stage2_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage2_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition1 = self._make_transition_layer([256], num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + self.stage3_cfg = extra['STAGE3'] + num_channels = self.stage3_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage3_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + self.stage4_cfg = extra['STAGE4'] + num_channels = self.stage4_cfg['NUM_CHANNELS'] + block = blocks_dict[self.stage4_cfg['BLOCK']] + num_channels = [ + num_channels[i] * block.expansion + for i in range(len(num_channels)) + ] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels, multi_scale_output=True) + + self.final_layer = nn.Conv2d( + in_channels=pre_stage_channels[0], + out_channels=cfg['MODEL']['NUM_JOINTS'], + kernel_size=extra['FINAL_CONV_KERNEL'], + stride=1, + padding=1 if extra['FINAL_CONV_KERNEL'] == 3 else 0) + + self.pretrained_layers = extra['PRETRAINED_LAYERS'] + self.active_func = nn.Sigmoid() + + self.downsample = nn.Sequential( + DownSample(384, 384), DownSample(384, 384), + nn.AdaptiveAvgPool2d((1, class_num))) + + self.property_conv = nn.Sequential( + nn.Conv2d( + 384, out_channels=192, kernel_size=1, stride=1, padding=0), + nn.Conv2d( + 192, out_channels=32, kernel_size=1, stride=1, padding=0)) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + nn.Conv2d( + num_channels_pre_layer[i], + num_channels_cur_layer[i], + 3, + 1, + 1, + bias=False), + nn.BatchNorm2d(num_channels_cur_layer[i]), + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv3x3s = [] + for j in range(i + 1 - num_branches_pre): + inchannels = num_channels_pre_layer[-1] + outchannels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else inchannels + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + inchannels, outchannels, 3, 2, 1, bias=False), + nn.BatchNorm2d(outchannels), + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv3x3s)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_stage(self, + layer_config, + num_inchannels, + multi_scale_output=True): + num_modules = layer_config['NUM_MODULES'] + num_branches = layer_config['NUM_BRANCHES'] + num_blocks = layer_config['NUM_BLOCKS'] + num_channels = layer_config['NUM_CHANNELS'] + block = blocks_dict[layer_config['BLOCK']] + fuse_method = layer_config['FUSE_METHOD'] + + modules = [] + for i in range(num_modules): + # multi_scale_output is only used last module + if not multi_scale_output and i == num_modules - 1: + reset_multi_scale_output = False + else: + reset_multi_scale_output = True + + modules.append( + HighResolutionModule(num_branches, block, num_blocks, + num_inchannels, num_channels, fuse_method, + reset_multi_scale_output)) + num_inchannels = modules[-1].get_num_inchannels() + + return nn.Sequential(*modules), num_inchannels + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['NUM_BRANCHES']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['NUM_BRANCHES']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['NUM_BRANCHES']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + + y_list = self.stage4(x_list) + property_x = y_list[3] + + x = self.final_layer(y_list[0]) + x = self.active_func(x) + + property_x = self.downsample(property_x) + property_x = torch.squeeze(self.property_conv(property_x), + 2).permute(0, 2, 1) + + return x, property_x + + def init_weights(self, pretrained=''): + logger.info('=> init weights from normal distribution') + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, std=0.001) + for name, _ in m.named_parameters(): + if name in ['bias']: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + nn.init.normal_(m.weight, std=0.001) + for name, _ in m.named_parameters(): + if name in ['bias']: + nn.init.constant_(m.bias, 0) + + if os.path.isfile(pretrained): + pretrained_state_dict = torch.load(pretrained) + logger.info('=> loading pretrained model {}'.format(pretrained)) + + need_init_state_dict = {} + for name, m in pretrained_state_dict.items(): + if name.split('.')[0] in self.pretrained_layers \ + or self.pretrained_layers[0] == '*': + need_init_state_dict[name] = m + self.load_state_dict(need_init_state_dict, strict=False) + elif pretrained: + logger.error('=> please download pre-trained models first!') + raise ValueError('{} is not exist!'.format(pretrained)) + + +class VTONLandmark(nn.Module): + """initialize the try on landmark model + """ + + def __init__(self, **kwargs): + super(VTONLandmark, self).__init__() + cfg = { + 'AUTO_RESUME': True, + 'CUDNN': { + 'BENCHMARK': True, + 'DETERMINISTIC': False, + 'ENABLED': True + }, + 'DATA_DIR': '', + 'GPUS': '(0,1,2,3)', + 'OUTPUT_DIR': 'output', + 'LOG_DIR': 'log', + 'WORKERS': 24, + 'PRINT_FREQ': 100, + 'DATASET': { + 'COLOR_RGB': True, + 'DATASET': 'mpii', + 'DATA_FORMAT': 'jpg', + 'FLIP': True, + 'NUM_JOINTS_HALF_BODY': 8, + 'PROB_HALF_BODY': -1.0, + 'ROOT': 'data/mpii/', + 'ROT_FACTOR': 30, + 'SCALE_FACTOR': 0.25, + 'TEST_SET': 'valid', + 'TRAIN_SET': 'train' + }, + 'MODEL': { + 'INIT_WEIGHTS': True, + 'NAME': 'pose_hrnet', + 'NUM_JOINTS': 32, + 'PRETRAINED': 'models/pytorch/imagenet/hrnet_w48-8ef0771d.pth', + 'TARGET_TYPE': 'gaussian', + 'IMAGE_SIZE': [256, 256], + 'HEATMAP_SIZE': [64, 64], + 'SIGMA': 2, + 'EXTRA': { + 'PRETRAINED_LAYERS': [ + 'conv1', 'bn1', 'conv2', 'bn2', 'layer1', + 'transition1', 'stage2', 'transition2', 'stage3', + 'transition3', 'stage4' + ], + 'FINAL_CONV_KERNEL': + 1, + 'STAGE2': { + 'NUM_MODULES': 1, + 'NUM_BRANCHES': 2, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4], + 'NUM_CHANNELS': [48, 96], + 'FUSE_METHOD': 'SUM' + }, + 'STAGE3': { + 'NUM_MODULES': 4, + 'NUM_BRANCHES': 3, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4, 4], + 'NUM_CHANNELS': [48, 96, 192], + 'FUSE_METHOD': 'SUM' + }, + 'STAGE4': { + 'NUM_MODULES': 3, + 'NUM_BRANCHES': 4, + 'BLOCK': 'BASIC', + 'NUM_BLOCKS': [4, 4, 4, 4], + 'NUM_CHANNELS': [48, 96, 192, 384], + 'FUSE_METHOD': 'SUM' + } + } + }, + 'LOSS': { + 'USE_TARGET_WEIGHT': True + }, + 'TRAIN': { + 'BATCH_SIZE_PER_GPU': 32, + 'SHUFFLE': True, + 'BEGIN_EPOCH': 0, + 'END_EPOCH': 210, + 'OPTIMIZER': 'adam', + 'LR': 0.001, + 'LR_FACTOR': 0.1, + 'LR_STEP': [170, 200], + 'WD': 0.0001, + 'GAMMA1': 0.99, + 'GAMMA2': 0.0, + 'MOMENTUM': 0.9, + 'NESTEROV': False + }, + 'TEST': { + 'BATCH_SIZE_PER_GPU': 32, + 'MODEL_FILE': '', + 'FLIP_TEST': True, + 'POST_PROCESS': True, + 'SHIFT_HEATMAP': True + }, + 'DEBUG': { + 'DEBUG': True, + 'SAVE_BATCH_IMAGES_GT': True, + 'SAVE_BATCH_IMAGES_PRED': True, + 'SAVE_HEATMAPS_GT': True, + 'SAVE_HEATMAPS_PRED': True + } + } + + # stem net + self.stage1Net = LandmarkNet(cfg, in_channel=3, class_num=2) + self.stage2Net = LandmarkNet(cfg, in_channel=38) + + self.stage = 2 + + def forward(self, cloth, person): + c_landmark, c_property = self.stage1Net(cloth) + if self.stage == 2: + pred_class = torch.argmax(c_property, dim=1) + c_heatmap = F.upsample( + c_landmark, + scale_factor=4, + mode='bilinear', + align_corners=True) + c_heatmap = c_heatmap * pred_class.unsqueeze(2).unsqueeze(2) + input2 = torch.cat([person, cloth, c_heatmap], 1) + p_landmark, p_property = self.stage2Net(input2) + return c_landmark, c_property, p_landmark, p_property + else: + return c_landmark, c_property + + def init_weights(self, pretrained=''): + logger.info('=> init weights from normal distribution') + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, std=0.001) + for name, _ in m.named_parameters(): + if name in ['bias']: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + nn.init.normal_(m.weight, std=0.001) + for name, _ in m.named_parameters(): + if name in ['bias']: + nn.init.constant_(m.bias, 0) + + if os.path.isfile(pretrained): + pretrained_state_dict = torch.load(pretrained) + logger.info('=> loading pretrained model {}'.format(pretrained)) + + need_init_state_dict = {} + for name, m in pretrained_state_dict.items(): + if name.split('.')[0] in self.pretrained_layers \ + or self.pretrained_layers[0] == '*': + need_init_state_dict[name] = m + self.load_state_dict(need_init_state_dict, strict=False) + elif pretrained: + logger.error('=> please download pre-trained models first!') + raise ValueError('{} is not exist!'.format(pretrained)) diff --git a/runtime/Lib/site_packages/models/cv/image_try_on/try_on_infer.py b/runtime/Lib/site_packages/models/cv/image_try_on/try_on_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..0323a6efaa397266a7e620cab4c51aa472ef28a7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_try_on/try_on_infer.py @@ -0,0 +1,222 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import argparse +import os + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torchvision.transforms as transforms +import yaml +from PIL import Image +from torch.nn import functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .generator import VTONGenerator +from .landmark import VTONLandmark +from .warping import Warping + +logger = get_logger() + + +def load_checkpoint(model, checkpoint_path, device): + params = torch.load(checkpoint_path, map_location=device) + model.load_state_dict(params, strict=False) + model.to(device) + model.eval() + return model + + +@MODELS.register_module(Tasks.image_try_on, module_name=Models.image_try_on) +class SALForImageTryOn(TorchModel): + """initialize the image try on model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + if torch.cuda.is_available(): + self.device = 'cuda' + logger.info('Use GPU') + else: + self.device = 'cpu' + logger.info('Use CPU') + + self.model = VTONGenerator(12, 3, 5, ngf=96, norm_layer=nn.BatchNorm2d) + self.model = load_checkpoint( + self.model, model_dir + '/' + ModelFile.TORCH_MODEL_BIN_FILE, + self.device) + + def forward(self, x, y): + pred_result = self.model(x, y) + return pred_result + + +def infer(ourgen_model, model_path, person_img, garment_img, mask_img, device): + + ourwarp_model = Warping() + landmark_model = VTONLandmark() + ourwarp_model = load_checkpoint(ourwarp_model, model_path + '/warp.pth', + device) + landmark_model.load_state_dict( + torch.load(model_path + '/landmark.pth', map_location=device)) + landmark_model.to(device).eval() + input_scale = 4 + with torch.no_grad(): + garment_img = cv2.imread(garment_img) + garment_img = cv2.cvtColor(garment_img, cv2.COLOR_BGR2RGB) + clothes = cv2.resize(garment_img, (768, 1024)) + + mask_img = cv2.imread(mask_img) + person_img = cv2.imread(person_img) + person_img = cv2.cvtColor(person_img, cv2.COLOR_BGR2RGB) + cm = mask_img[:, :, 0] + input_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + clothes = input_transform(clothes).unsqueeze(0).to(device) + + cm_array = np.array(cm) + cm_array = (cm_array >= 128).astype(np.float32) + cm = torch.from_numpy(cm_array) + cm = cm.unsqueeze(0).unsqueeze(0) + cm = torch.FloatTensor((cm.numpy() > 0.5).astype(float)).to(device) + + im = person_img + h_ori, w_ori = im.shape[0:2] + im = cv2.resize(im, (768, 1024)) + im = input_transform(im).unsqueeze(0).to(device) + + h, w = 512, 384 + p_down = F.interpolate(im, size=(h, w), mode='bilinear') + c_down = F.interpolate(clothes, size=(h, w), mode='bilinear') + c_heatmap, c_property, p_heatmap, p_property = landmark_model( + c_down, p_down) + + N = c_heatmap.shape[0] + paired_cloth = clothes[0].cpu() + color_map = {'1': (0, 0, 255), '0': (255, 0, 0)} + c_im = (np.array(paired_cloth.permute(1, 2, 0)).copy() + 1) / 2 * 255 + c_im = cv2.cvtColor(c_im, cv2.COLOR_RGB2BGR) + pred_class = torch.argmax(c_property, dim=1) + point_ind = torch.argmax( + c_heatmap.view(N, 32, -1), dim=2).cpu().numpy() + pred_y, pred_x = 8 * (point_ind // 96), 8 * (point_ind % 96) + for ind in range(32): + point_class = int(pred_class[0, ind]) + if point_class < 0.9: + continue + point_color = color_map[str(point_class)] + y, x = pred_y[0][ind], pred_x[0][ind] + cv2.circle(c_im, (x, y), 2, point_color, 4) + cv2.putText( + c_im, + str(ind), (x + 4, y + 4), + cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.75, + color=point_color, + thickness=1) + paired_im = im[0].cpu() + color_map = {'2': (0, 0, 255), '1': (0, 255, 0), '0': (255, 0, 0)} + p_im = (np.array(paired_im.permute(1, 2, 0)).copy() + 1) / 2 * 255 + p_im = cv2.cvtColor(p_im, cv2.COLOR_RGB2BGR) + pred_class = torch.argmax(p_property, dim=1) + point_ind = torch.argmax( + p_heatmap.view(N, 32, -1), dim=2).cpu().numpy() + pred_y, pred_x = 8 * (point_ind // 96), 8 * (point_ind % 96) + for ind in range(32): + point_class = int(pred_class[0, ind]) + if point_class < 0.9: + continue + point_color = color_map[str(point_class)] + y, x = pred_y[0][ind], pred_x[0][ind] + cv2.circle(p_im, (x, y), 2, point_color, 4) + cv2.putText( + p_im, + str(ind), (x + 4, y + 4), + cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.75, + color=point_color, + thickness=1) + valid_c_point = np.zeros((32, 2)).astype(np.float32) + valid_p_point = np.zeros((32, 2)).astype(np.float32) + c_point_heatmap = -1 * torch.ones(32, 1024, 768) + p_point_heatmap = -1 * torch.ones(32, 1024, 768) + + cloth_property, person_property = torch.argmax( + c_property, dim=1), torch.argmax( + p_property, dim=1) + cloth_point_ind = torch.argmax( + c_heatmap.view(N, 32, -1), dim=2).cpu().numpy() + cloth_y, cloth_x = 8 * (cloth_point_ind // 96), 8 * ( + cloth_point_ind % 96) + person_point_ind = torch.argmax( + p_heatmap.view(N, 32, -1), dim=2).cpu().numpy() + person_y, person_x = 8 * (person_point_ind // 96), 8 * ( + person_point_ind % 96) + r = 20 + for k in range(32): + property_c, property_p = cloth_property[0, + k], person_property[0, + k] - 1 + if property_c > 0.1: + c_x, c_y = cloth_x[0, k], cloth_y[0, k] + x_min, y_min, x_max, y_max = max(c_x - r - 1, 0), max( + c_y - r - 1, 0), min(c_x + r, 768), min(c_y + r, 1024) + c_point_heatmap[k, y_min:y_max, + x_min:x_max] = torch.tensor(property_c) + valid_c_point[k, 0], valid_c_point[k, 1] = c_x, c_y + if property_p > -0.99: + p_x, p_y = person_x[0, k], person_y[0, k] + x_min, y_min, x_max, y_max = max(p_x - r - 1, 0), max( + p_y - r - 1, 0), min(p_x + r, 768), min(p_y + r, 1024) + p_point_heatmap[k, y_min:y_max, + x_min:x_max] = torch.tensor(property_p) + if property_p > 0: + valid_p_point[k, 0], valid_p_point[k, 1] = p_x, p_y + + c_point_plane = torch.tensor(valid_c_point).unsqueeze(0).to(device) + p_point_plane = torch.tensor(valid_p_point).unsqueeze(0).to(device) + c_point_heatmap = c_point_heatmap.unsqueeze(0).to(device) + p_point_heatmap = p_point_heatmap.unsqueeze(0).to(device) + + if input_scale > 1: + h, w = 1024 // input_scale, 768 // input_scale + c_point_plane = c_point_plane // input_scale + p_point_plane = p_point_plane // input_scale + c_point_heatmap = F.interpolate( + c_point_heatmap, size=(h, w), mode='nearest') + p_point_heatmap = F.interpolate( + p_point_heatmap, size=(h, w), mode='nearest') + + im_down = F.interpolate(im, size=(h, w), mode='bilinear') + c_down = F.interpolate(cm * clothes, size=(h, w), mode='bilinear') + cm_down = F.interpolate(cm, size=(h, w), mode='nearest') + + warping_input = [ + c_down, im_down, c_point_heatmap, p_point_heatmap, c_point_plane, + p_point_plane, cm_down, cm * clothes, device + ] + final_warped_cloth, last_flow, last_flow_all, flow_all, delta_list, x_all, x_edge_all, delta_x_all, \ + delta_y_all, local_warped_cloth_list, fuse_cloth, globalmap, up_cloth = ourwarp_model(warping_input) + + gen_inputs = torch.cat([im, up_cloth], 1) + gen_outputs = ourgen_model(gen_inputs, p_point_heatmap) + + combine = torch.cat([gen_outputs[0]], 2).squeeze() + cv_img = (combine.permute(1, 2, 0).detach().cpu().numpy() + 1) / 2 + rgb = (cv_img * 255).astype(np.uint8) + bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + bgr = cv2.resize(bgr, (w_ori, h_ori)) + return bgr diff --git a/runtime/Lib/site_packages/models/cv/image_try_on/warping.py b/runtime/Lib/site_packages/models/cv/image_try_on/warping.py new file mode 100644 index 0000000000000000000000000000000000000000..6c9cf18cd18fec5f4559e67f4bffe6e00eff2ca8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_try_on/warping.py @@ -0,0 +1,1260 @@ +# The implementation here is modified based on flow-style-vton, +# originally Apache 2.0 License and publicly avaialbe at https://github.com/SenHe/Flow-Style-VTON + +from collections import OrderedDict +from math import sqrt + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + + +def apply_offset(offset): + sizes = list(offset.size()[2:]) + grid_list = torch.meshgrid( + [torch.arange(size, device=offset.device) for size in sizes]) + grid_list = reversed(grid_list) + grid_list = [ + grid.float().unsqueeze(0) + offset[:, dim, ...] + for dim, grid in enumerate(grid_list) + ] + grid_list = [ + grid / ((size - 1.0) / 2.0) - 1.0 + for grid, size in zip(grid_list, reversed(sizes)) + ] + + return torch.stack(grid_list, dim=-1) + + +class Conv2dBlock(nn.Module): + + def __init__(self, + input_dim, + output_dim, + kernel_size, + stride, + padding=0, + norm='batch', + activation='prelu', + pad_type='zero', + bias=True): + super().__init__() + self.use_bias = bias + # initialize padding + if pad_type == 'reflect': + self.pad = nn.ReflectionPad2d(padding) + elif pad_type == 'replicate': + self.pad = nn.ReplicationPad2d(padding) + elif pad_type == 'zero': + self.pad = nn.ZeroPad2d(padding) + else: + assert 0, 'Unsupported padding type: {}'.format(pad_type) + + norm_dim = input_dim + if norm == 'batch': + self.norm = nn.BatchNorm2d(norm_dim) + elif norm == 'instance': + self.norm = nn.InstanceNorm2d(norm_dim) + elif norm == 'none': + self.norm = None + else: + assert 0, 'Unsupported normalization: {}'.format(norm) + + # initialize activation + if activation == 'relu': + self.activation = nn.ReLU(inplace=True) + elif activation == 'lrelu': + self.activation = nn.LeakyReLU(0.1, inplace=True) + elif activation == 'prelu': + self.activation = nn.PReLU() + elif activation == 'selu': + self.activation = nn.SELU(inplace=True) + elif activation == 'tanh': + self.activation = nn.Tanh() + elif activation == 'sigmoid': + self.activation = nn.Sigmoid() + + elif activation == 'none': + self.activation = None + else: + assert 0, 'Unsupported activation: {}'.format(activation) + + # initialize convolution + self.conv = nn.Conv2d( + input_dim, output_dim, kernel_size, stride, bias=self.use_bias) + + def forward(self, x): + if self.norm: + x = self.norm(x) + if self.activation: + x = self.activation(x) + x = self.conv(self.pad(x)) + return x + + +class ResBlock(nn.Module): + + def __init__(self, in_channels): + super(ResBlock, self).__init__() + + self.block = nn.Sequential( + nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.Conv2d( + in_channels, in_channels, kernel_size=3, padding=1, + bias=False), nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.Conv2d( + in_channels, in_channels, kernel_size=3, padding=1, + bias=False)) + + def forward(self, x): + return self.block(x) + x + + +class ResBlock_2(nn.Module): + + def __init__(self, in_channels): + super(ResBlock_2, self).__init__() + self.block = nn.Sequential( + nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.Conv2d( + in_channels, in_channels, kernel_size=3, padding=1, + bias=False), nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.Conv2d( + in_channels, in_channels, kernel_size=3, padding=1, + bias=False)) + + def forward(self, x1, x2): + return self.block(x1) + x2 + + +class DownSample(nn.Module): + + def __init__(self, in_channels, out_channels): + super(DownSample, self).__init__() + self.block = nn.Sequential( + nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False)) + + def forward(self, x): + return self.block(x) + + +class UpSample(nn.Module): + + def __init__(self, in_channels, out_channels): + super(UpSample, self).__init__() + self.block = nn.Sequential( + nn.BatchNorm2d(in_channels), nn.PReLU(), + nn.ConvTranspose2d( + in_channels, out_channels, kernel_size=2, stride=2, + bias=False)) + + def forward(self, x): + return self.block(x) + + def upsample(self, F, scale): + """[2x nearest neighbor upsampling] + Arguments: + F {[torch tensor]} -- [tensor to be upsampled, (B, C, H, W)] + """ + upsample = torch.nn.Upsample(scale_factor=scale, mode='nearest') + return upsample(F) + + +class FeatureEncoder(nn.Module): + + def __init__(self, in_channels, chns=[64, 128, 256, 256, 256]): + # in_channels = 3 for images, and is larger (e.g., 17+1+1) for agnositc representation + super(FeatureEncoder, self).__init__() + self.encoders = [] + for i, out_chns in enumerate(chns): + if i == 0: + encoder = nn.Sequential( + DownSample(in_channels, out_chns), ResBlock(out_chns), + ResBlock(out_chns)) + else: + encoder = nn.Sequential( + DownSample(chns[i - 1], out_chns), ResBlock(out_chns), + ResBlock(out_chns)) + + self.encoders.append(encoder) + + self.encoders = nn.ModuleList(self.encoders) + + def forward(self, x): + encoder_features = [] + for encoder in self.encoders: + x = encoder(x) + encoder_features.append(x) + return encoder_features + + +class RefinePyramid(nn.Module): + + def __init__(self, chns=[64, 128, 256, 256, 256], fpn_dim=256): + super(RefinePyramid, self).__init__() + self.chns = chns + + # adaptive + self.adaptive = [] + for in_chns in list(reversed(chns)): + adaptive_layer = nn.Conv2d(in_chns, fpn_dim, kernel_size=1) + self.adaptive.append(adaptive_layer) + self.adaptive = nn.ModuleList(self.adaptive) + # output conv + self.smooth = [] + for i in range(len(chns)): + smooth_layer = nn.Conv2d( + fpn_dim, fpn_dim, kernel_size=3, padding=1) + self.smooth.append(smooth_layer) + self.smooth = nn.ModuleList(self.smooth) + + def forward(self, x): + conv_ftr_list = x + + feature_list = [] + last_feature = None + for i, conv_ftr in enumerate(list(reversed(conv_ftr_list))): + # adaptive + feature = self.adaptive[i](conv_ftr) + # fuse + if last_feature is not None: + feature = feature + F.interpolate( + last_feature, scale_factor=2, mode='nearest') + # smooth + feature = self.smooth[i](feature) + last_feature = feature + feature_list.append(feature) + + return tuple(reversed(feature_list)) + + +def Round(x): + ''' x: tensor ''' + return torch.round(x) - x.detach() + x + + +def Camp(x, mi=0, ma=192): + ''' x: tensor ''' + if x < mi: + x = (mi - x.detach()) + x + elif x > ma: + x = x - (x.detach() - ma) + return x + + +class CorrelationLayer(nn.Module): + + def __init__(self, init_scale=5): + super(CorrelationLayer, self).__init__() + self.init_scale = init_scale + self.softmax3 = nn.Softmax(dim=3) + self.sig = nn.Sigmoid() + self.reduce1 = Conv2dBlock( + 64, 4, 1, 1, 0, activation='prelu', norm='none') + self.reduce2 = Conv2dBlock( + 64, 4, 1, 1, 0, activation='prelu', norm='none') + + init_h, init_w = 192 / self.init_scale, 256 / self.init_scale + self.step_h, self.step_w = [], [] + step_h_half = [i + 1 for i in range(0, int(init_h + 1), 2)] + step_h_half_re = list(reversed(step_h_half[1:])) + self.step_h.extend(step_h_half_re) + self.step_h.extend(step_h_half) + step_w_half = [i + 1 for i in range(0, int(init_w + 1), 2)] + step_w_half_re = list(reversed(step_w_half[1:])) + self.step_w.extend(step_w_half_re) + self.step_w.extend(step_w_half) + self.mask_h, self.mask_w = len(self.step_h), len(self.step_w) + + def forward(self, location, fea_c, fea_p, scale_param, H, W, c_landmark, + p_landmark): + init_h, init_w = torch.tensor( + W / self.init_scale, device=fea_c.device), torch.tensor( + H / self.init_scale, device=fea_c.device) + N, C, fea_H = fea_c.shape[0], fea_c.shape[1], fea_c.shape[2] + downsample_ratio = H / fea_H + landmark_flow = -1 * torch.ones((N, 64, H, W), device=fea_c.device) + mask_h, mask_w = self.mask_h, self.mask_w + + fea_cn = self.upsample(fea_c, scale=downsample_ratio) + fea_pn = self.upsample(fea_p, scale=downsample_ratio) + + fea_cn = self.reduce1(fea_cn) + fea_pn = self.reduce2(fea_pn) + C = fea_cn.shape[1] + + src_box_h0 = torch.tensor( + self.step_h, + device=fea_cn.device).unsqueeze(dim=0).unsqueeze(dim=0).repeat( + N, 32, 1) + src_box_w0 = torch.tensor( + self.step_w, + device=fea_cn.device).unsqueeze(dim=0).unsqueeze(dim=0).repeat( + N, 32, 1) + tar_box_h0 = torch.tensor( + self.step_h, + device=fea_cn.device).unsqueeze(dim=0).unsqueeze(dim=0).repeat( + N, 32, 1) + tar_box_w0 = torch.tensor( + self.step_w, + device=fea_cn.device).unsqueeze(dim=0).unsqueeze(dim=0).repeat( + N, 32, 1) + adj_cw, adj_ch, adj_pw, adj_ph = scale_param[:, :, 0, 0], scale_param[:, :, 0, 1],\ + scale_param[:, :, 1, 0], scale_param[:, :, 1, 1] + c_h, c_w = (init_h * self.sig(adj_ch)).unsqueeze( + dim=2), (init_w * self.sig(adj_cw)).unsqueeze(dim=2) + p_h, p_w = (init_h * self.sig(adj_ph)).unsqueeze( + dim=2), (init_w * self.sig(adj_pw)).unsqueeze(dim=2) + src_box_h = self.sig((c_h - src_box_h0) * 2) + src_box_w = self.sig((c_w - src_box_w0) * 2) + tar_box_h = self.sig((p_h - tar_box_h0) * 2) + tar_box_w = self.sig((p_w - tar_box_w0) * 2) + + src_box_h, src_box_w = src_box_h.unsqueeze(dim=3), src_box_w.unsqueeze( + dim=2) + tar_box_h, tar_box_w = tar_box_h.unsqueeze(dim=3), tar_box_w.unsqueeze( + dim=2) + src_mask = torch.matmul(src_box_h, src_box_w) + tar_mask = torch.matmul(tar_box_h, tar_box_w) + + cloth_patch_all = torch.zeros((N, 32 * C, mask_h, mask_w), + device=fea_cn.device) + person_patch_all = torch.zeros((N, 32 * C, mask_h, mask_w), + device=fea_cn.device) + + location_patch_all = -1 * torch.ones( + (N, 64, mask_h, mask_w), device=fea_cn.device) + one_flow_all = -1 * torch.ones( + (N, 64, mask_h, mask_w), device=fea_cn.device) + coord_info = [] + mask = torch.zeros((N, 32, H, W), device=fea_cn.device) + + for b in range(N): + for i in range(32): + c_center_x, c_center_y = c_landmark[b, i, 0], c_landmark[b, i, + 1] + p_center_x, p_center_y = p_landmark[b, i, 0], p_landmark[b, i, + 1] + if (c_center_x == 0 + and c_center_y == 0) or (p_center_x == 0 + and p_center_y == 0): + p_new_top_y, p_new_bottom_y, p_new_left_x, p_new_right_x, p_new_h, p_new_w = 0, 0, 0, 0, 0, 0 + else: + c_left_x = torch.floor(c_center_x.int() - mask_w / 2) + c_right_x = torch.floor(c_center_x.int() + mask_w / 2) + delta_x1 = int(torch.clamp(0 - c_left_x, min=0)) + delta_x2 = int(torch.clamp(c_right_x - W, min=0)) + c_left_x, c_right_x = int(c_left_x), int(c_right_x) + c_top_y = torch.floor(c_center_y.int() - mask_h / 2) + c_bottom_y = torch.floor(c_center_y.int() + mask_h / 2) + delta_y1 = int(torch.clamp(0 - c_top_y, min=0)) + delta_y2 = int(torch.clamp(c_bottom_y - H, min=0)) + c_top_y, c_bottom_y = int(c_top_y), int(c_bottom_y) + + p_left_x = torch.floor(p_center_x.int() - mask_w / 2) + p_right_x = torch.floor(p_center_x.int() + mask_w / 2) + delta_x3 = int(torch.clamp(0 - p_left_x, min=0)) + delta_x4 = int(torch.clamp(p_right_x - W, min=0)) + p_left_x, p_right_x = int(p_left_x), int(p_right_x) + p_top_y = torch.floor(p_center_y.int() - mask_h / 2) + p_bottom_y = torch.floor(p_center_y.int() + mask_h / 2) + delta_y3 = int(torch.clamp(0 - p_top_y, min=0)) + delta_y4 = int(torch.clamp(p_bottom_y - H, min=0)) + p_top_y, p_bottom_y = int(p_top_y), int(p_bottom_y) + + # Deformable Patch + c_new_top_y, c_new_bottom_y, c_new_left_x, c_new_right_x = \ + c_top_y + delta_y1, c_bottom_y - delta_y2, c_left_x + delta_x1, c_right_x - delta_x2 + p_new_top_y, p_new_bottom_y, p_new_left_x, p_new_right_x = \ + p_top_y + delta_y3, p_bottom_y - delta_y4, p_left_x + delta_x3, p_right_x - delta_x4 + c_new_h, c_new_w = c_new_bottom_y - c_new_top_y, c_new_right_x - c_new_left_x + p_new_h, p_new_w = p_new_bottom_y - p_new_top_y, p_new_right_x - p_new_left_x + + cloth_patch_all[b, i * C:(i + 1) * C, : c_new_h, : c_new_w] = \ + fea_cn[b, :, c_new_top_y: c_new_bottom_y, c_new_left_x: c_new_right_x] * \ + src_mask[b, i, 0 + delta_y1: mask_h - delta_y2, 0 + delta_x1: mask_w - delta_x2] + person_patch_all[b, i * C: (i + 1) * C, : p_new_h, : p_new_w] = \ + fea_pn[b, :, p_new_top_y: p_new_bottom_y, p_new_left_x: p_new_right_x] * \ + tar_mask[b, i, 0 + delta_y3: mask_h - delta_y4, 0 + delta_x3: mask_w - delta_x4] + location_patch_all[b, i * 2:i * 2 + + 2, :c_new_h, :c_new_w] = location[ + b, i * 2:i * 2 + 2, + c_new_top_y:c_new_bottom_y, + c_new_left_x:c_new_right_x] + + coord_info.append([ + p_new_top_y, p_new_bottom_y, p_new_left_x, p_new_right_x, + p_new_h, p_new_w + ]) + + Q = person_patch_all.view(N, 32, C, mask_h, + mask_w).view(N, 32, C, + -1).permute(0, 1, 3, + 2).contiguous() + K = cloth_patch_all.view(N, 32, C, mask_h, mask_w).view(N, 32, C, -1) + correlation = self.softmax3(torch.matmul(Q, K)) + V = location_patch_all.view(N, 32, 2, mask_h, + mask_w).view(N, 32, 2, + -1).permute(0, 1, 3, + 2).contiguous() + one_flow_all = torch.matmul(correlation, + V).permute(0, 1, 3, 2).contiguous().view( + N, 32 * 2, mask_h, mask_w) + + for b in range(N): + for i in range(32): + p_new_top_y, p_new_bottom_y, p_new_left_x, p_new_right_x, p_new_h, p_new_w = coord_info[ + b * 32 + i] + if p_new_h != 0 and p_new_w != 0: + landmark_flow[b, i * 2: i * 2 + 2, p_new_top_y: p_new_bottom_y, p_new_left_x: p_new_right_x] = \ + one_flow_all[b, i * 2: i * 2 + 2, : p_new_h, : p_new_w] + mask[b, i, p_new_top_y:p_new_bottom_y, + p_new_left_x:p_new_right_x] = torch.tensor( + 1, device=fea_cn.device) + + return landmark_flow, mask + + def upsample(self, F, scale): + """[2x nearest neighbor upsampling] + Arguments: + F {[torch tensor]} -- [tensor to be upsampled, (B, C, H, W)] + """ + upsample = torch.nn.Upsample(scale_factor=scale, mode='nearest') + return upsample(F) + + +class LocalFlow(nn.Module): + + def __init__(self, fpn_dim=256, use_num=3, init_scale=6): + super(LocalFlow, self).__init__() + self.use_num = use_num + self.fusers = [] + self.fusers_refine = [] + self.location_preds = [] + self.patch_preds = [] + self.map_preds = [] + self.map_refine = [] + self.reduc = [] + + for i in range(use_num): + in_chns = fpn_dim + map_refine = nn.Conv2d( + 32, 32, kernel_size=3, stride=1, padding=1, groups=32) + + reduc = nn.Sequential( + Conv2dBlock( + in_chns, 64, 3, 1, 1, activation='prelu', bias=False)) + fusers = nn.Sequential( + Conv2dBlock( + fpn_dim * 2, + fpn_dim, + 3, + 1, + 1, + activation='prelu', + bias=False), + Conv2dBlock( + fpn_dim, 32, 3, 1, 1, activation='prelu', bias=False)) + fusers_refine_layer = [ + nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=True), + ResBlock(32) + ] * ( + i + 1) + fusers_refine = nn.Sequential(*fusers_refine_layer) + location_preds = nn.Sequential( + Conv2dBlock(32, 64, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock(64, 64, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock(64, 64, 3, 1, 1, activation='prelu', norm='none'), + ) + # patch + patch_layer = [DownSample(32, 32), ResBlock(32)] * (use_num + 2) + patch_layer.append(nn.AdaptiveAvgPool2d((2, 2))) + patch_preds = nn.Sequential(*patch_layer) + # attention map + map_layer = [] + map_layer = [ + ResBlock(32), + Conv2dBlock(32, 32, 3, 1, 1, activation='prelu', norm='none') + ] + map_preds = nn.Sequential(*map_layer) + + self.reduc.append(reduc) + self.fusers.append(fusers) + self.fusers_refine.append(fusers_refine) + self.location_preds.append(location_preds) + self.patch_preds.append(patch_preds) + self.map_preds.append(map_preds) + self.map_refine.append(map_refine) + + self.reduc = nn.ModuleList(self.reduc) + + self.fusers = nn.ModuleList(self.fusers) + self.fusers_refine = nn.ModuleList(self.fusers_refine) + self.location_preds = nn.ModuleList(self.location_preds) + self.patch_preds = nn.ModuleList(self.patch_preds) + self.map_preds = nn.ModuleList(self.map_preds) + self.map_refine = nn.ModuleList(self.map_refine) + self.CorrelationLayer = CorrelationLayer(init_scale=init_scale) + + def forward(self, cloth, fea_c, fea_p, c_landmark, p_landmark): + flow_list, map_list = [], [] + N, _, H, W = cloth.shape + for i in range(self.use_num): + fuse = self.fusers[i](torch.cat((fea_c[i], fea_p[i]), dim=1)) + fuse = self.fusers_refine[i](fuse) + location = self.location_preds[i](fuse) + patch = self.patch_preds[i](fuse) + att_map = self.map_preds[i](fuse) + fea_c_reduc = self.reduc[i](fea_c[i]) + fea_p_reduc = self.reduc[i](fea_p[i]) + flow, mask = self.CorrelationLayer(location, fea_c_reduc, + fea_p_reduc, patch, H, W, + c_landmark, p_landmark) + if i != 0: + flow = (flow + last_flow) / 2 + add_map = torch.add(last_att_map, att_map) + att_map = self.map_refine[i](add_map) + + last_flow = flow + flow_list.append(last_flow) + last_att_map = att_map * mask + map_list.append(last_att_map) + + return flow_list, map_list + + +class EqualLR: + + def __init__(self, name): + self.name = name + + def compute_weight(self, module): + weight = getattr(module, self.name + '_orig') + fan_in = weight.data.size(1) * weight.data[0][0].numel() + + return weight * sqrt(2 / fan_in) + + @staticmethod + def apply(module, name): + fn = EqualLR(name) + + weight = getattr(module, name) + del module._parameters[name] + module.register_parameter(name + '_orig', nn.Parameter(weight.data)) + module.register_forward_pre_hook(fn) + + return fn + + def __call__(self, module, input): + weight = self.compute_weight(module) + setattr(module, self.name, weight) + + +def equal_lr(module, name='weight'): + EqualLR.apply(module, name) + + return module + + +class EqualLinear(nn.Module): + + def __init__(self, in_dim, out_dim): + super().__init__() + + linear = nn.Linear(in_dim, out_dim) + linear.weight.data.normal_() + linear.bias.data.zero_() + + self.linear = equal_lr(linear) + + def forward(self, input): + return self.linear(input) + + +class ModulatedConv2d(nn.Module): + + def __init__(self, + fin, + fout, + kernel_size, + padding_type='zero', + upsample=False, + downsample=False, + latent_dim=512, + normalize_mlp=False): + super(ModulatedConv2d, self).__init__() + self.in_channels = fin + self.out_channels = fout + self.kernel_size = kernel_size + padding_size = kernel_size // 2 + + if kernel_size == 1: + self.demudulate = False + else: + self.demudulate = True + + self.weight = nn.Parameter( + torch.Tensor(fout, fin, kernel_size, kernel_size)) + self.bias = nn.Parameter(torch.Tensor(1, fout, 1, 1)) + + if normalize_mlp: + self.mlp_class_std = nn.Sequential( + EqualLinear(latent_dim, fin), PixelNorm()) + else: + self.mlp_class_std = EqualLinear(latent_dim, fin) + + if padding_type == 'reflect': + self.padding = nn.ReflectionPad2d(padding_size) + else: + self.padding = nn.ZeroPad2d(padding_size) + + self.weight.data.normal_() + self.bias.data.zero_() + + def forward(self, input, latent): + fan_in = self.weight.data.size(1) * self.weight.data[0][0].numel() + weight = self.weight * sqrt(2 / fan_in) + weight = weight.view(1, self.out_channels, self.in_channels, + self.kernel_size, self.kernel_size) + + s = self.mlp_class_std(latent).view(-1, 1, self.in_channels, 1, 1) + weight = s * weight + if self.demudulate: + d = torch.rsqrt((weight**2).sum(4).sum(3).sum(2) + 1e-5).view( + -1, self.out_channels, 1, 1, 1) + weight = (d * weight).view(-1, self.in_channels, self.kernel_size, + self.kernel_size) + else: + weight = weight.view(-1, self.in_channels, self.kernel_size, + self.kernel_size) + + batch, _, height, width = input.shape + + input = input.view(1, -1, height, width) + input = self.padding(input) + out = F.conv2d( + input, weight, groups=batch).view(batch, self.out_channels, height, + width) + self.bias + + return out + + +class StyledConvBlock(nn.Module): + + def __init__(self, + fin, + fout, + latent_dim=256, + padding='zero', + actvn='lrelu', + normalize_affine_output=False, + modulated_conv=False): + super(StyledConvBlock, self).__init__() + if not modulated_conv: + if padding == 'reflect': + padding_layer = nn.ReflectionPad2d + else: + padding_layer = nn.ZeroPad2d + + if modulated_conv: + conv2d = ModulatedConv2d + else: + conv2d = EqualConv2d + + if modulated_conv: + self.actvn_gain = sqrt(2) + else: + self.actvn_gain = 1.0 + + self.modulated_conv = modulated_conv + + if actvn == 'relu': + activation = nn.ReLU(True) + else: + activation = nn.LeakyReLU(0.2, True) + + if self.modulated_conv: + self.conv0 = conv2d( + fin, + fout, + kernel_size=3, + padding_type=padding, + upsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv0 = conv2d(fin, fout, kernel_size=3) + + seq0 = [padding_layer(1), conv0] + self.conv0 = nn.Sequential(*seq0) + + self.actvn0 = activation + + if self.modulated_conv: + self.conv1 = conv2d( + fout, + fout, + kernel_size=3, + padding_type=padding, + downsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv1 = conv2d(fout, fout, kernel_size=3) + seq1 = [padding_layer(1), conv1] + self.conv1 = nn.Sequential(*seq1) + + self.actvn1 = activation + + def forward(self, input, latent=None): + if self.modulated_conv: + out = self.conv0(input, latent) + else: + out = self.conv0(input) + + out = self.actvn0(out) * self.actvn_gain + + if self.modulated_conv: + out = self.conv1(out, latent) + else: + out = self.conv1(out) + + out = self.actvn1(out) * self.actvn_gain + + return out + + +class Styled_F_ConvBlock(nn.Module): + + def __init__(self, + fin, + fout, + latent_dim=256, + padding='zero', + actvn='prelu', + normalize_affine_output=False, + modulated_conv=False): + super(Styled_F_ConvBlock, self).__init__() + if not modulated_conv: + if padding == 'reflect': + padding_layer = nn.ReflectionPad2d + else: + padding_layer = nn.ZeroPad2d + + if modulated_conv: + conv2d = ModulatedConv2d + else: + conv2d = EqualConv2d + + if modulated_conv: + self.actvn_gain = sqrt(2) + else: + self.actvn_gain = 1.0 + + self.modulated_conv = modulated_conv + + if actvn == 'relu': + activation = nn.ReLU(True) + else: + activation = nn.LeakyReLU(0.2, True) + + if self.modulated_conv: + self.conv0 = conv2d( + fin, + 128, + kernel_size=3, + padding_type=padding, + upsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv0 = conv2d(fin, 128, kernel_size=3) + + seq0 = [padding_layer(1), conv0] + self.conv0 = nn.Sequential(*seq0) + + self.actvn0 = activation + + if self.modulated_conv: + self.conv1 = conv2d( + 128, + fout, + kernel_size=3, + padding_type=padding, + downsample=False, + latent_dim=latent_dim, + normalize_mlp=normalize_affine_output) + else: + conv1 = conv2d(128, fout, kernel_size=3) + seq1 = [padding_layer(1), conv1] + self.conv1 = nn.Sequential(*seq1) + + def forward(self, input, latent=None): + if self.modulated_conv: + out = self.conv0(input, latent) + else: + out = self.conv0(input) + + out = self.actvn0(out) * self.actvn_gain + + if self.modulated_conv: + out = self.conv1(out, latent) + else: + out = self.conv1(out) + + return out + + +class AFlowNet(nn.Module): + + def __init__(self, num_pyramid, fpn_dim=256, use_num=3): + super(AFlowNet, self).__init__() + + padding_type = 'zero' + actvn = 'lrelu' + normalize_mlp = False + modulated_conv = True + self.use_num = use_num + + self.netRefine = [] + self.netStyle = [] + self.netF = [] + self.map_preds = [] + self.map_refine = [] + + for i in range(num_pyramid): + + netRefine_layer = torch.nn.Sequential( + torch.nn.Conv2d( + 2 * fpn_dim, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), ResBlock(128), ResBlock(128), + torch.nn.Conv2d( + in_channels=128, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=64, + out_channels=32, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=32, + out_channels=12, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=12, + out_channels=2, + kernel_size=3, + stride=1, + padding=1)) + + style_block = torch.nn.Sequential( + torch.nn.Conv2d( + 2 * fpn_dim, + out_channels=fpn_dim, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + fpn_dim, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=128, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=64, + out_channels=32, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=32, + out_channels=2, + kernel_size=3, + stride=1, + padding=1)) + + style_F_block = Styled_F_ConvBlock( + 49, + 2, + latent_dim=256, + padding=padding_type, + actvn=actvn, + normalize_affine_output=normalize_mlp, + modulated_conv=modulated_conv) + + map_preds = torch.nn.Sequential( + torch.nn.Conv2d( + 2 * fpn_dim, + out_channels=128, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=128, + out_channels=64, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=64, + out_channels=32, + kernel_size=3, + stride=1, + padding=1), nn.PReLU(), + torch.nn.Conv2d( + in_channels=32, + out_channels=2, + kernel_size=3, + stride=1, + padding=1)) + if i == 0: + map_refine = nn.Sequential( + Conv2dBlock( + 2, 2, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock( + 2, 2, 3, 1, 1, activation='prelu', norm='none')) + elif i >= 2: + map_refine = nn.Sequential( + Conv2dBlock( + 36, 16, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock( + 16, 4, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock( + 4, 2, 3, 1, 1, activation='prelu', norm='none')) + else: + map_refine = nn.Sequential( + Conv2dBlock( + 4, 2, 3, 1, 1, activation='prelu', norm='none'), + Conv2dBlock( + 2, 2, 3, 1, 1, activation='prelu', norm='none')) + + self.netRefine.append(netRefine_layer) + self.netStyle.append(style_block) + self.netF.append(style_F_block) + self.map_preds.append(map_preds) + self.map_refine.append(map_refine) + + self.netRefine = nn.ModuleList(self.netRefine) + self.netStyle = nn.ModuleList(self.netStyle) + self.netF = nn.ModuleList(self.netF) + self.map_preds = nn.ModuleList(self.map_preds) + self.map_refine = nn.ModuleList(self.map_refine) + + self.cond_style = torch.nn.Sequential( + torch.nn.Conv2d(256, 128, kernel_size=(8, 6), stride=1, padding=0), + nn.PReLU()) + self.image_style = torch.nn.Sequential( + torch.nn.Conv2d(256, 128, kernel_size=(8, 6), stride=1, padding=0), + nn.PReLU()) + + def forward(self, global_flow_input, device, warp_feature=True): + + x = global_flow_input[0] + x_edge = global_flow_input[1] + x_warps = global_flow_input[2] + x_conds = global_flow_input[3] + localmap_list = global_flow_input[5] + last_flow = None + last_gmap = None + gmap_all = [] + last_flow_all = [] + delta_list = [] + x_all = [] + x_edge_all = [] + cond_fea_all = [] + delta_x_all = [] + delta_y_all = [] + filter_x = [[0, 0, 0], [1, -2, 1], [0, 0, 0]] + filter_y = [[0, 1, 0], [0, -2, 0], [0, 1, 0]] + filter_diag1 = [[1, 0, 0], [0, -2, 0], [0, 0, 1]] + filter_diag2 = [[0, 0, 1], [0, -2, 0], [1, 0, 0]] + weight_array = np.ones([3, 3, 1, 4]) + weight_array[:, :, 0, 0] = filter_x + weight_array[:, :, 0, 1] = filter_y + weight_array[:, :, 0, 2] = filter_diag1 + weight_array[:, :, 0, 3] = filter_diag2 + + weight_array = torch.FloatTensor(weight_array).permute(3, 2, 0, + 1).to(device) + self.weight = nn.Parameter(data=weight_array, requires_grad=False) + + fea_num = len(x_warps) + + for i in range(fea_num): + x_warp = x_warps[fea_num - 1 - i] + x_cond = x_conds[fea_num - 1 - i] + cond_fea_all.append(x_cond) + + if last_flow is not None and warp_feature: + x_warp_after = F.grid_sample( + x_warp, + last_flow.detach().permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + else: + x_warp_after = x_warp + + flow = self.netStyle[i](torch.cat([x_warp_after, x_cond], 1)) + delta_list.append(flow) + flow = apply_offset(flow) + if last_flow is not None: + flow = F.grid_sample( + last_flow, flow, mode='bilinear', padding_mode='border') + else: + flow = flow.permute(0, 3, 1, 2) + + last_flow = flow + x_warp = F.grid_sample( + x_warp, + flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + concat = torch.cat([x_warp, x_cond], 1) + flow = self.netRefine[i](concat) + + g_map = self.map_preds[i](concat) + if i >= fea_num - self.use_num: + upsample_tmp = 0.5**(fea_num - i) + g_map = self.map_refine[i]( + torch.cat([ + g_map, last_gmap, + self.upsample( + localmap_list[i - (fea_num - self.use_num)], + upsample_tmp) + ], 1)) + elif last_gmap is not None: + g_map = self.map_refine[i](torch.cat([g_map, last_gmap], 1)) + elif i == 0: + g_map = self.map_refine[i](g_map) + last_gmap = self.upsample(g_map, 2) + gmap_all.append(last_gmap) + + delta_list.append(flow) + flow = apply_offset(flow) + flow = F.grid_sample( + last_flow, flow, mode='bilinear', padding_mode='border') + + last_flow = F.interpolate(flow, scale_factor=2, mode='bilinear') + last_flow_all.append(last_flow) + cur_x = F.interpolate( + x, scale_factor=0.5**(len(x_warps) - 1 - i), mode='bilinear') + cur_x_warp = F.grid_sample( + cur_x, + last_flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + + x_all.append(cur_x_warp) + cur_x_edge = F.interpolate( + x_edge, + scale_factor=0.5**(len(x_warps) - 1 - i), + mode='bilinear') + cur_x_warp_edge = F.grid_sample( + cur_x_edge, + last_flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='zeros') + x_edge_all.append(cur_x_warp_edge) + flow_x, flow_y = torch.split(last_flow, 1, dim=1) + delta_x = F.conv2d(flow_x, self.weight) + delta_y = F.conv2d(flow_y, self.weight) + delta_x_all.append(delta_x) + delta_y_all.append(delta_y) + + x_warp = F.grid_sample( + x, + last_flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + return x_warp, last_flow, cond_fea_all, last_flow_all, delta_list, x_all, \ + x_edge_all, delta_x_all, delta_y_all, gmap_all + + def upsample(self, F, scale): + """[2x nearest neighbor upsampling] + Arguments: + F {[torch tensor]} -- [tensor to be upsampled, (B, C, H, W)] + """ + upsample = torch.nn.Upsample(scale_factor=scale, mode='nearest') + return upsample(F) + + +class Warping(nn.Module): + """initialize the try on warping model + """ + + def __init__(self, input_nc=32 + 3, use_num=3): + super(Warping, self).__init__() + num_filters = [64, 128, 256, 256, 256] + self.num = len(num_filters) - 1 + self.cloth_input = nn.Conv2d( + 3, 32, kernel_size=3, padding=1, bias=False) + self.person_input = nn.Conv2d( + 3, 32, kernel_size=3, padding=1, bias=False) + self.c_input_refine = Conv2dBlock( + 64, 35, 3, 1, 1, activation='prelu', bias=False) + self.p_input_refine = Conv2dBlock( + 64, 35, 3, 1, 1, activation='prelu', bias=False) + + self.src_features = FeatureEncoder(input_nc, num_filters) + self.tar_features = FeatureEncoder(input_nc, num_filters) + self.src_FPN = RefinePyramid(num_filters) + self.tar_FPN = RefinePyramid(num_filters) + self.global_flow = AFlowNet(len(num_filters), use_num=use_num) + self.local_flow = LocalFlow(use_num=use_num, init_scale=5) + + self.softmax = nn.Softmax(dim=1) + + self.input_scale = 4 + + def forward(self, warping_input): + + cloth = warping_input[0] + person = warping_input[1] + c_heatmap = warping_input[2] + p_heatmap = warping_input[3] + c_landmark = warping_input[4] + p_landmark = warping_input[5] + x_edge = warping_input[6] + org_cloth = warping_input[7] + device = warping_input[8] + + N, _, H, W = cloth.shape + cf = self.cloth_input(cloth) + pf = self.person_input(person) + src_input = self.c_input_refine(torch.cat((cf, c_heatmap), dim=1)) + tar_input = self.p_input_refine(torch.cat((pf, p_heatmap), dim=1)) + + src_en_fea = self.src_features(src_input) + tar_en_fea = self.tar_features(tar_input) + src_c_fea = self.src_FPN(src_en_fea) + tar_c_fea = self.tar_FPN(tar_en_fea) + src_fuse_fea = src_c_fea + tar_fuse_fea = tar_c_fea + + localflow_list, localmap_list = self.local_flow( + cloth, src_fuse_fea, tar_fuse_fea, c_landmark, p_landmark) + global_flow_input = [ + cloth, x_edge, src_fuse_fea, tar_fuse_fea, localflow_list, + localmap_list + ] + x_warp, last_flow, last_flow_all, flow_all, delta_list, x_all, x_edge_all, \ + delta_x_all, delta_y_all, gmap_all = self.global_flow(global_flow_input, device) + + local_warped_cloth_list = [] + for i in range(len(localflow_list)): + localmap = self.softmax(localmap_list[i]) + warped_cloth = torch.zeros_like(cloth) + for j in range(32): + once_warped_cloth = F.grid_sample( + cloth, + localflow_list[i][:, j * 2:(j + 1) * 2, :, :].permute( + 0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + warped_cloth += once_warped_cloth * localmap[:, + j, :, :].unsqueeze( + dim=1) + local_warped_cloth_list.append(warped_cloth) + globalmap = self.softmax(gmap_all[-1]) + fuse_cloth = globalmap[:, 0, :, :].unsqueeze(dim=1) * x_warp + \ + globalmap[:, 1, :, :].unsqueeze(dim=1) * local_warped_cloth_list[-1] + + # global local fused + # global + _, _, h, w = org_cloth.shape + up_last_flow = F.interpolate(last_flow, size=(h, w), mode='bicubic') + up_warped_gcloth = F.grid_sample( + org_cloth, + up_last_flow.permute(0, 2, 3, 1), + mode='bicubic', + padding_mode='border') + # local + localmap = self.softmax(localmap_list[-1]) + up_map = F.interpolate(localmap, size=(h, w), mode='bicubic') + up_flow = F.interpolate( + localflow_list[-1], size=(h, w), mode='bicubic') + up_warped_lcloth = torch.zeros_like(org_cloth) + for j in range(32): + once_up_flow = F.interpolate( + up_flow[:, j * 2:(j + 1) * 2, :, :], + size=(h, w), + mode='bicubic') + once_warped_cloth = F.grid_sample( + org_cloth, + once_up_flow.permute(0, 2, 3, 1), + mode='bilinear', + padding_mode='border') + up_warped_lcloth += once_warped_cloth * up_map[:, + j, :, :].unsqueeze( + dim=1) + up_globalmap = F.interpolate(globalmap, size=(h, w), mode='bicubic') + up_fuse_cloth = up_globalmap[:, 0, :, :].unsqueeze( + dim=1) * up_warped_gcloth + up_globalmap[:, 1, :, :].unsqueeze( + dim=1) * up_warped_lcloth + up_cloth = torch.cat( + [up_warped_gcloth, up_warped_lcloth, up_fuse_cloth], 1) + + return x_warp, last_flow, last_flow_all, flow_all, delta_list, x_all, x_edge_all, \ + delta_x_all, delta_y_all, local_warped_cloth_list, fuse_cloth, gmap_all, up_cloth + + def warp(self, x, flo, device): + """ + warp an image/tensor (im2) back to im1, according to the optical flow + x: [B, C, H, W] (im2) + flo: [B, 2, H, W] flow + """ + import torch.autograd as autograd + from torch.autograd import Variable + B, C, H, W = x.size() + # mesh grid + xx = torch.arange(0, W).view(1, -1).repeat(H, 1) + yy = torch.arange(0, H).view(-1, 1).repeat(1, W) + xx = xx.view(1, 1, H, W).repeat(B, 1, 1, 1) + yy = yy.view(1, 1, H, W).repeat(B, 1, 1, 1) + grid = torch.cat((xx, yy), 1).float() + + if x.is_cuda: + grid = grid.cuda() + vgrid = Variable(grid) + flo + + # scale grid to [-1,1] + vgrid[:, + 0, :, :] = 2.0 * vgrid[:, 0, :, :].clone() / max(W - 1, 1) - 1.0 + vgrid[:, + 1, :, :] = 2.0 * vgrid[:, 1, :, :].clone() / max(H - 1, 1) - 1.0 + + vgrid = vgrid.permute(0, 2, 3, 1) + output = nn.functional.grid_sample(x, vgrid) + mask = torch.autograd.Variable(torch.ones(x.size())).to(device) + mask = nn.functional.grid_sample(mask, vgrid) + + mask[mask < 0.9999] = 0 + mask[mask > 0] = 1 + + return output * mask diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/__init__.py b/runtime/Lib/site_packages/models/cv/image_view_transform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d8f55b3eb479f0ffd42542040c4e8a409db2aebc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .image_view_transform_infer import ImageViewTransform + +else: + _import_structure = {'image_view_transform_infer': ['ImageViewTransform']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/image_view_transform_infer.py b/runtime/Lib/site_packages/models/cv/image_view_transform/image_view_transform_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..bc3221ec807ff9020ee0338cb877416783a72ca9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/image_view_transform_infer.py @@ -0,0 +1,219 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math +import os +import sys +import time +from contextlib import nullcontext +from functools import partial + +import cv2 +import diffusers # 0.12.1 +import fire +import numpy as np +import rich +import torch +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image +from rich import print +from torch import autocast +from torchvision import transforms + +from modelscope.fileio import load +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .ldm.ddim import DDIMSampler +from .util import instantiate_from_config, load_and_preprocess + +logger = get_logger() + + +def load_model_from_config(model, config, ckpt, device, verbose=False): + print(f'Loading model from {ckpt}') + pl_sd = torch.load(ckpt, map_location='cpu') + if 'global_step' in pl_sd: + print(f'Global Step: {pl_sd["global_step"]}') + sd = pl_sd['state_dict'] + model = instantiate_from_config(config.model) + m, u = model.load_state_dict(sd, strict=False) + model.to(device) + model.eval() + return model + + +@MODELS.register_module( + Tasks.image_view_transform, module_name=Models.image_view_transform) +class ImageViewTransform(TorchModel): + """initialize the image view translation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + + def __init__(self, model_dir, device='cpu', *args, **kwargs): + + super().__init__(model_dir=model_dir, device=device, *args, **kwargs) + + self.device = torch.device( + device if torch.cuda.is_available() else 'cpu') + + config = os.path.join(model_dir, + 'sd-objaverse-finetune-c_concat-256.yaml') + ckpt = os.path.join(model_dir, 'zero123-xl.ckpt') + config = OmegaConf.load(config) + self.model = None + self.model = load_model_from_config( + self.model, config, ckpt, device=self.device) + + def forward(self, model_path, x, y): + pred_results = _infer(self.model, model_path, x, y, self.device) + return pred_results + + +def infer(genmodel, model_path, image_path, target_view_path, device): + output_ims = genmodel(model_path, image_path, target_view_path) + return output_ims + + +@torch.no_grad() +def sample_model(input_im, model, sampler, precision, h, w, ddim_steps, + n_samples, scale, ddim_eta, x, y, z): + precision_scope = autocast if precision == 'autocast' else nullcontext + with precision_scope('cuda'): + with model.ema_scope(): + c = model.get_learned_conditioning(input_im).tile(n_samples, 1, 1) + T = torch.tensor([ + math.radians(x), + math.sin(math.radians(y)), + math.cos(math.radians(y)), z + ]) + T = T[None, None, :].repeat(n_samples, 1, 1).to(c.device) + c = torch.cat([c, T], dim=-1) + c = model.cc_projection(c) + cond = {} + cond['c_crossattn'] = [c] + cond['c_concat'] = [ + model.encode_first_stage( + (input_im.to(c.device))).mode().detach().repeat( + n_samples, 1, 1, 1) + ] + if scale != 1.0: + uc = {} + uc['c_concat'] = [ + torch.zeros(n_samples, 4, h // 8, w // 8).to(c.device) + ] + uc['c_crossattn'] = [torch.zeros_like(c).to(c.device)] + else: + uc = None + + shape = [4, h // 8, w // 8] + samples_ddim, _ = sampler.sample( + S=ddim_steps, + conditioning=cond, + batch_size=n_samples, + shape=shape, + verbose=False, + unconditional_guidance_scale=scale, + unconditional_conditioning=uc, + eta=ddim_eta, + x_T=None) + # samples_ddim = torch.nn.functional.interpolate(samples_ddim, 64, mode='nearest', antialias=False) + x_samples_ddim = model.decode_first_stage(samples_ddim) + return torch.clamp( + (x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0).cpu() + + +def preprocess_image(models, input_im, preprocess, carvekit_path): + ''' + :param input_im (PIL Image). + :return input_im (H, W, 3) array in [0, 1]. + ''' + + print('old input_im:', input_im.size) + + if preprocess: + + # model_carvekit = create_carvekit_interface() + model_carvekit = torch.load(carvekit_path) + input_im = load_and_preprocess(model_carvekit, input_im) + input_im = (input_im / 255.0).astype(np.float32) + # (H, W, 3) array in [0, 1]. + else: + input_im = input_im.resize([256, 256], Image.Resampling.LANCZOS) + input_im = np.asarray(input_im, dtype=np.float32) / 255.0 + alpha = input_im[:, :, 3:4] + white_im = np.ones_like(input_im) + input_im = alpha * input_im + (1.0 - alpha) * white_im + + input_im = input_im[:, :, 0:3] + # (H, W, 3) array in [0, 1]. + + return input_im + + +def main_run(models, + device, + return_what, + x=0.0, + y=0.0, + z=0.0, + raw_im=None, + carvekit_path=None, + preprocess=True, + scale=3.0, + n_samples=4, + ddim_steps=50, + ddim_eta=1.0, + precision='fp32', + h=256, + w=256): + ''' + :param raw_im (PIL Image). + ''' + + raw_im.thumbnail([1536, 1536], Image.Resampling.LANCZOS) + input_im = preprocess_image(models, raw_im, preprocess, carvekit_path) + + if 'gen' in return_what: + input_im = transforms.ToTensor()(input_im).unsqueeze(0).to(device) + input_im = input_im * 2 - 1 + input_im = transforms.functional.resize(input_im, [h, w]) + + sampler = DDIMSampler(models) + # used_x = -x # NOTE: Polar makes more sense in Basile's opinion this way! + used_x = x # NOTE: Set this way for consistency. + x_samples_ddim = sample_model(input_im, models, sampler, precision, h, + w, ddim_steps, n_samples, scale, + ddim_eta, used_x, y, z) + + output_ims = [] + for x_sample in x_samples_ddim: + image = x_sample.detach().cpu().squeeze().numpy() + image = np.transpose(image, (1, 2, 0)) * 255 + image = np.uint8(image) + bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + output_ims.append(bgr) + + return output_ims + + +def _infer(genmodel, model_path, image_path, target_view_path, device): + if isinstance(image_path, str): + raw_image = load(image_path) + print(type(raw_image)) + else: + raw_image = image_path + if isinstance(target_view_path, str): + views = load(target_view_path) + else: + views = target_view_path + # views = views.astype(np.float32) + carvekit_path = os.path.join(model_path, 'carvekit.pth') + output_ims = main_run(genmodel, device, 'angles_gen', views[0], views[1], + views[2], raw_image, carvekit_path, views[3], + views[4], views[5], views[6], views[7]) + return output_ims diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/__init__.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/attention.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..37f4317dfa260b28ac6d88fd973680a7bceb0d3f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/attention.py @@ -0,0 +1,294 @@ +import math +from inspect import isfunction + +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from torch import einsum, nn + +from .util_diffusion import checkpoint + + +def exists(val): + return val is not None + + +def uniq(arr): + return {el: True for el in arr}.keys() + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +def max_neg_value(t): + return -torch.finfo(t.dtype).max + + +def init_(tensor): + dim = tensor.shape[-1] + std = 1 / math.sqrt(dim) + tensor.uniform_(-std, std) + return tensor + + +# feedforward +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def Normalize(in_channels): + return torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + + +class LinearAttention(nn.Module): + + def __init__(self, dim, heads=4, dim_head=32): + super().__init__() + self.heads = heads + hidden_dim = dim_head * heads + self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias=False) + self.to_out = nn.Conv2d(hidden_dim, dim, 1) + + def forward(self, x): + b, c, h, w = x.shape + qkv = self.to_qkv(x) + q, k, v = rearrange( + qkv, + 'b (qkv heads c) h w -> qkv b heads c (h w)', + heads=self.heads, + qkv=3) + k = k.softmax(dim=-1) + context = torch.einsum('bhdn,bhen->bhde', k, v) + out = torch.einsum('bhde,bhdn->bhen', context, q) + out = rearrange( + out, + 'b heads c (h w) -> b (heads c) h w', + heads=self.heads, + h=h, + w=w) + return self.to_out(out) + + +class SpatialSelfAttention(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = rearrange(q, 'b c h w -> b (h w) c') + k = rearrange(k, 'b c h w -> b c (h w)') + w_ = torch.einsum('bij,bjk->bik', q, k) + + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = rearrange(v, 'b c h w -> b c (h w)') + w_ = rearrange(w_, 'b i j -> b j i') + h_ = torch.einsum('bij,bjk->bik', v, w_) + h_ = rearrange(h_, 'b c (h w) -> b c h w', h=h) + h_ = self.proj_out(h_) + + return x + h_ + + +class CrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + + sim = einsum('b i d, b j d -> b i j', q, k) * self.scale + + if exists(mask): + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + attn = sim.softmax(dim=-1) + + out = einsum('b i j, b j d -> b i d', attn, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + return self.to_out(out) + + +class BasicTransformerBlock(nn.Module): + + def __init__(self, + dim, + n_heads, + d_head, + dropout=0., + context_dim=None, + gated_ff=True, + checkpoint=True, + disable_self_attn=False): + super().__init__() + self.disable_self_attn = disable_self_attn + self.attn1 = CrossAttention( + query_dim=dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + context_dim=context_dim if self.disable_self_attn else + None) # is a self-attention if not self.disable_self_attn + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = CrossAttention( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout) # is self-attn if context is none + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward(self, x, context=None): + return checkpoint(self._forward, (x, context), self.parameters(), + self.checkpoint) + + def _forward(self, x, context=None): + x = self.attn1( + self.norm1(x), + context=context if self.disable_self_attn else None) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False): + super().__init__() + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = Normalize(in_channels) + + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim, + disable_self_attn=disable_self_attn) for d in range(depth) + ]) + + self.proj_out = zero_module( + nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, padding=0)) + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + x = self.proj_in(x) + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + for block in self.transformer_blocks: + x = block(x, context=context) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + x = self.proj_out(x) + return x + x_in diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/autoencoder.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..de702b354ea979a7ebbf0080778b606840cbcc5d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/autoencoder.py @@ -0,0 +1,555 @@ +from contextlib import contextmanager + +import pytorch_lightning as pl +import torch +import torch.nn.functional as F +from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer + +from ..util import instantiate_from_config +from .distributions import DiagonalGaussianDistribution +from .model import Decoder, Encoder + + +class VQModel(pl.LightningModule): + + def __init__( + self, + ddconfig, + lossconfig, + n_embed, + embed_dim, + ckpt_path=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + batch_resize_range=None, + scheduler_config=None, + lr_g_factor=1.0, + remap=None, + sane_index_shape=False, # tell vector quantizer to return indices as bhw + use_ema=False): + super().__init__() + self.embed_dim = embed_dim + self.n_embed = n_embed + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + self.loss = instantiate_from_config(lossconfig) + self.quantize = VectorQuantizer( + n_embed, + embed_dim, + beta=0.25, + remap=remap, + sane_index_shape=sane_index_shape) + self.quant_conv = torch.nn.Conv2d(ddconfig['z_channels'], embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + self.batch_resize_range = batch_resize_range + if self.batch_resize_range is not None: + print( + f'{self.__class__.__name__}: Using per-batch resizing in range {batch_resize_range}.' + ) + + self.use_ema = use_ema + if self.use_ema: + self.model_ema = LitEma(self) + print(f'Keeping EMAs of {len(list(self.model_ema.buffers()))}.') + + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + self.scheduler_config = scheduler_config + self.lr_g_factor = lr_g_factor + + @contextmanager + def ema_scope(self, context=None): + if self.use_ema: + self.model_ema.store(self.parameters()) + self.model_ema.copy_to(self) + if context is not None: + print(f'{context}: Switched to EMA weights') + try: + yield None + finally: + if self.use_ema: + self.model_ema.restore(self.parameters()) + if context is not None: + print(f'{context}: Restored training weights') + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + missing, unexpected = self.load_state_dict(sd, strict=False) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + print(f'Unexpected Keys: {unexpected}') + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + quant, emb_loss, info = self.quantize(h) + return quant, emb_loss, info + + def encode_to_prequant(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return h + + def decode(self, quant): + quant = self.post_quant_conv(quant) + dec = self.decoder(quant) + return dec + + def decode_code(self, code_b): + quant_b = self.quantize.embed_code(code_b) + dec = self.decode(quant_b) + return dec + + def forward(self, input, return_pred_indices=False): + quant, diff, (_, _, ind) = self.encode(input) + dec = self.decode(quant) + if return_pred_indices: + return dec, diff, ind + return dec, diff + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + if self.batch_resize_range is not None: + lower_size = self.batch_resize_range[0] + upper_size = self.batch_resize_range[1] + if self.global_step <= 4: + # do the first few batches with max size to avoid later oom + new_resize = upper_size + else: + new_resize = np.random.choice( + np.arange(lower_size, upper_size + 16, 16)) + if new_resize != x.shape[2]: + x = F.interpolate(x, size=new_resize, mode='bicubic') + x = x.detach() + return x + + def training_step(self, batch, batch_idx, optimizer_idx): + # https://github.com/pytorch/pytorch/issues/37142 + # try not to fool the heuristics + x = self.get_input(batch, self.image_key) + xrec, qloss, ind = self(x, return_pred_indices=True) + + if optimizer_idx == 0: + # autoencode + aeloss, log_dict_ae = self.loss( + qloss, + x, + xrec, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train', + predicted_indices=ind) + + self.log_dict( + log_dict_ae, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=True) + return aeloss + + if optimizer_idx == 1: + # discriminator + discloss, log_dict_disc = self.loss( + qloss, + x, + xrec, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train') + self.log_dict( + log_dict_disc, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=True) + return discloss + + def validation_step(self, batch, batch_idx): + log_dict = self._validation_step(batch, batch_idx) + with self.ema_scope(): + self._validation_step(batch, batch_idx, suffix='_ema') + return log_dict + + def _validation_step(self, batch, batch_idx, suffix=''): + x = self.get_input(batch, self.image_key) + xrec, qloss, ind = self(x, return_pred_indices=True) + aeloss, log_dict_ae = self.loss( + qloss, + x, + xrec, + 0, + self.global_step, + last_layer=self.get_last_layer(), + split='val' + suffix, + predicted_indices=ind) + + discloss, log_dict_disc = self.loss( + qloss, + x, + xrec, + 1, + self.global_step, + last_layer=self.get_last_layer(), + split='val' + suffix, + predicted_indices=ind) + rec_loss = log_dict_ae[f'val{suffix}/rec_loss'] + self.log( + f'val{suffix}/rec_loss', + rec_loss, + prog_bar=True, + logger=True, + on_step=False, + on_epoch=True, + sync_dist=True) + self.log( + f'val{suffix}/aeloss', + aeloss, + prog_bar=True, + logger=True, + on_step=False, + on_epoch=True, + sync_dist=True) + if version.parse(pl.__version__) >= version.parse('1.4.0'): + del log_dict_ae[f'val{suffix}/rec_loss'] + self.log_dict(log_dict_ae) + self.log_dict(log_dict_disc) + return self.log_dict + + def configure_optimizers(self): + lr_d = self.learning_rate + lr_g = self.lr_g_factor * self.learning_rate + print('lr_d', lr_d) + print('lr_g', lr_g) + opt_ae = torch.optim.Adam( + list(self.encoder.parameters()) + list(self.decoder.parameters()) + + list(self.quantize.parameters()) + + list(self.quant_conv.parameters()) + + list(self.post_quant_conv.parameters()), + lr=lr_g, + betas=(0.5, 0.9)) + opt_disc = torch.optim.Adam( + self.loss.discriminator.parameters(), lr=lr_d, betas=(0.5, 0.9)) + + if self.scheduler_config is not None: + scheduler = instantiate_from_config(self.scheduler_config) + + print('Setting up LambdaLR scheduler...') + scheduler = [ + { + 'scheduler': + LambdaLR(opt_ae, lr_lambda=scheduler.schedule), + 'interval': 'step', + 'frequency': 1 + }, + { + 'scheduler': + LambdaLR(opt_disc, lr_lambda=scheduler.schedule), + 'interval': 'step', + 'frequency': 1 + }, + ] + return [opt_ae, opt_disc], scheduler + return [opt_ae, opt_disc], [] + + def get_last_layer(self): + return self.decoder.conv_out.weight + + def log_images(self, batch, only_inputs=False, plot_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if only_inputs: + log['inputs'] = x + return log + xrec, _ = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['inputs'] = x + log['reconstructions'] = xrec + if plot_ema: + with self.ema_scope(): + xrec_ema, _ = self(x) + if x.shape[1] > 3: + xrec_ema = self.to_rgb(xrec_ema) + log['reconstructions_ema'] = xrec_ema + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class VQModelInterface(VQModel): + + def __init__(self, embed_dim, *args, **kwargs): + super().__init__(embed_dim=embed_dim, *args, **kwargs) + self.embed_dim = embed_dim + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return h + + def decode(self, h, force_not_quantize=False): + # also go through quantization layer + if not force_not_quantize: + quant, emb_loss, info = self.quantize(h) + else: + quant = h + quant = self.post_quant_conv(quant) + dec = self.decoder(quant) + return dec + + +class AutoencoderKL(pl.LightningModule): + + def __init__( + self, + ddconfig, + lossconfig, + embed_dim, + ckpt_path=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + ): + super().__init__() + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + self.loss = instantiate_from_config(lossconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d(2 * ddconfig['z_channels'], + 2 * embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + self.load_state_dict(sd, strict=False) + print(f'Restored from {path}') + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + return x + + def training_step(self, batch, batch_idx, optimizer_idx): + inputs = self.get_input(batch, self.image_key) + reconstructions, posterior = self(inputs) + + if optimizer_idx == 0: + # train encoder+decoder+logvar + aeloss, log_dict_ae = self.loss( + inputs, + reconstructions, + posterior, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train') + self.log( + 'aeloss', + aeloss, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=True) + self.log_dict( + log_dict_ae, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=False) + return aeloss + + if optimizer_idx == 1: + # train the discriminator + discloss, log_dict_disc = self.loss( + inputs, + reconstructions, + posterior, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train') + + self.log( + 'discloss', + discloss, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=True) + self.log_dict( + log_dict_disc, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=False) + return discloss + + def validation_step(self, batch, batch_idx): + inputs = self.get_input(batch, self.image_key) + reconstructions, posterior = self(inputs) + aeloss, log_dict_ae = self.loss( + inputs, + reconstructions, + posterior, + 0, + self.global_step, + last_layer=self.get_last_layer(), + split='val') + + discloss, log_dict_disc = self.loss( + inputs, + reconstructions, + posterior, + 1, + self.global_step, + last_layer=self.get_last_layer(), + split='val') + + self.log('val/rec_loss', log_dict_ae['val/rec_loss']) + self.log_dict(log_dict_ae) + self.log_dict(log_dict_disc) + return self.log_dict + + def configure_optimizers(self): + lr = self.learning_rate + opt_ae = torch.optim.Adam( + list(self.encoder.parameters()) + list(self.decoder.parameters()) + + list(self.quant_conv.parameters()) + + list(self.post_quant_conv.parameters()), + lr=lr, + betas=(0.5, 0.9)) + opt_disc = torch.optim.Adam( + self.loss.discriminator.parameters(), lr=lr, betas=(0.5, 0.9)) + return [opt_ae, opt_disc], [] + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class IdentityFirstStage(torch.nn.Module): + + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface # TODO: Should be true by default but check to not break older stuff + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddim.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddim.py new file mode 100644 index 0000000000000000000000000000000000000000..14b1ee42a61479c4eb5b9b2f0dcc6ea0787183e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddim.py @@ -0,0 +1,433 @@ +"""SAMPLING ONLY.""" + +from functools import partial + +import numpy as np +import torch +from einops import rearrange +from tqdm import tqdm + +from .sampling_util import (norm_thresholding, renorm_thresholding, + spatial_norm_thresholding) +from .util_diffusion import (extract_into_tensor, + make_ddim_sampling_parameters, + make_ddim_timesteps, noise_like) + + +class DDIMSampler(object): + + def __init__(self, model, schedule='linear', **kwargs): + super().__init__() + self.model = model + self.ddpm_num_timesteps = model.num_timesteps + self.schedule = schedule + + def to(self, device): + """Same as to in torch module + Don't really underestand why this isn't a module in the first place""" + for k, v in self.__dict__.items(): + if isinstance(v, torch.Tensor): + new_v = getattr(self, k).to(device) + setattr(self, k, new_v) + + def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device('cuda'): + attr = attr.to(torch.device('cuda')) + setattr(self, name, attr) + + def make_schedule(self, + ddim_num_steps, + ddim_discretize='uniform', + ddim_eta=0., + verbose=True): + self.ddim_timesteps = make_ddim_timesteps( + ddim_discr_method=ddim_discretize, + num_ddim_timesteps=ddim_num_steps, + num_ddpm_timesteps=self.ddpm_num_timesteps, + verbose=verbose) + alphas_cumprod = self.model.alphas_cumprod + assert alphas_cumprod.shape[ + 0] == self.ddpm_num_timesteps, 'alphas have to be defined for each timestep' + + def to_torch(x): + return x.clone().detach().to(torch.float32).to(self.model.device) + + self.register_buffer('betas', to_torch(self.model.betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer('alphas_cumprod_prev', + to_torch(self.model.alphas_cumprod_prev)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer('sqrt_alphas_cumprod', + to_torch(np.sqrt(alphas_cumprod.cpu()))) + self.register_buffer('sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1. - alphas_cumprod.cpu()))) + self.register_buffer('log_one_minus_alphas_cumprod', + to_torch(np.log(1. - alphas_cumprod.cpu()))) + self.register_buffer('sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod.cpu()))) + self.register_buffer('sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod.cpu() - 1))) + + # ddim sampling parameters + ddim_sigmas, ddim_alphas, ddim_alphas_prev = make_ddim_sampling_parameters( + alphacums=alphas_cumprod.cpu(), + ddim_timesteps=self.ddim_timesteps, + eta=ddim_eta, + verbose=verbose) + self.register_buffer('ddim_sigmas', ddim_sigmas) + self.register_buffer('ddim_alphas', ddim_alphas) + self.register_buffer('ddim_alphas_prev', ddim_alphas_prev) + self.register_buffer('ddim_sqrt_one_minus_alphas', + np.sqrt(1. - ddim_alphas)) + alpha_1 = (1 - self.alphas_cumprod_prev) / (1 - self.alphas_cumprod) + alpha_2 = (1 - self.alphas_cumprod / self.alphas_cumprod_prev) + sigmas_for_original_sampling_steps = ddim_eta * torch.sqrt( + alpha_1 * alpha_2) + self.register_buffer('ddim_sigmas_for_original_num_steps', + sigmas_for_original_sampling_steps) + + @torch.no_grad() + def sample(self, + S, + batch_size, + shape, + conditioning=None, + callback=None, + normals_sequence=None, + img_callback=None, + quantize_x0=False, + eta=0., + mask=None, + x0=None, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + verbose=True, + x_T=None, + log_every_t=100, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + dynamic_threshold=None, + **kwargs): + if conditioning is not None: + if isinstance(conditioning, dict): + ctmp = conditioning[list(conditioning.keys())[0]] + while isinstance(ctmp, list): + ctmp = ctmp[0] + cbs = ctmp.shape[0] + if cbs != batch_size: + print( + f'Warning: Got {cbs} conditionings but batch-size is {batch_size}' + ) + + else: + if conditioning.shape[0] != batch_size: + print( + f'Warning: Got {conditioning.shape[0]} conditionings but batch-size is {batch_size}' + ) + + self.make_schedule(ddim_num_steps=S, ddim_eta=eta, verbose=verbose) + # sampling + C, H, W = shape + size = (batch_size, C, H, W) + print(f'Data shape for DDIM sampling is {size}, eta {eta}') + + samples, intermediates = self.ddim_sampling( + conditioning, + size, + callback=callback, + img_callback=img_callback, + quantize_denoised=quantize_x0, + mask=mask, + x0=x0, + ddim_use_original_steps=False, + noise_dropout=noise_dropout, + temperature=temperature, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + x_T=x_T, + log_every_t=log_every_t, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + dynamic_threshold=dynamic_threshold, + ) + return samples, intermediates + + @torch.no_grad() + def ddim_sampling(self, + cond, + shape, + x_T=None, + ddim_use_original_steps=False, + callback=None, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + img_callback=None, + log_every_t=100, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + dynamic_threshold=None, + t_start=-1): + device = self.model.betas.device + b = shape[0] + if x_T is None: + img = torch.randn(shape, device=device) + else: + img = x_T + + if timesteps is None: + timesteps = self.ddpm_num_timesteps if ddim_use_original_steps else self.ddim_timesteps + elif timesteps is not None and not ddim_use_original_steps: + subset_end = int( + min(timesteps / self.ddim_timesteps.shape[0], 1) + * self.ddim_timesteps.shape[0]) - 1 + timesteps = self.ddim_timesteps[:subset_end] + + timesteps = timesteps[:t_start] + + intermediates = {'x_inter': [img], 'pred_x0': [img]} + time_range = reversed(range( + 0, timesteps)) if ddim_use_original_steps else np.flip(timesteps) + total_steps = timesteps if ddim_use_original_steps else timesteps.shape[ + 0] + print(f'Running DDIM Sampling with {total_steps} timesteps') + + iterator = tqdm(time_range, desc='DDIM Sampler', total=total_steps) + + for i, step in enumerate(iterator): + index = total_steps - i - 1 + ts = torch.full((b, ), step, device=device, dtype=torch.long) + + if mask is not None: + assert x0 is not None + img_orig = self.model.q_sample( + x0, ts) # TODO: deterministic forward pass? + img = img_orig * mask + (1. - mask) * img + + outs = self.p_sample_ddim( + img, + cond, + ts, + index=index, + use_original_steps=ddim_use_original_steps, + quantize_denoised=quantize_denoised, + temperature=temperature, + noise_dropout=noise_dropout, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + dynamic_threshold=dynamic_threshold) + img, pred_x0 = outs + if callback: + img = callback(i, img, pred_x0) + if img_callback: + img_callback(pred_x0, i) + + if index % log_every_t == 0 or index == total_steps - 1: + intermediates['x_inter'].append(img) + intermediates['pred_x0'].append(pred_x0) + + return img, intermediates + + @torch.no_grad() + def p_sample_ddim(self, + x, + c, + t, + index, + repeat_noise=False, + use_original_steps=False, + quantize_denoised=False, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + dynamic_threshold=None): + b, *_, device = *x.shape, x.device + + if unconditional_conditioning is None or unconditional_guidance_scale == 1.: + e_t = self.model.apply_model(x, t, c) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) + if isinstance(c, dict): + assert isinstance(unconditional_conditioning, dict) + c_in = dict() + for k in c: + if isinstance(c[k], list): + c_in[k] = [ + torch.cat( + [unconditional_conditioning[k][i], c[k][i]]) + for i in range(len(c[k])) + ] + else: + c_in[k] = torch.cat( + [unconditional_conditioning[k], c[k]]) + else: + c_in = torch.cat([unconditional_conditioning, c]) + e_t_uncond, e_t = self.model.apply_model(x_in, t_in, c_in).chunk(2) + e_t = e_t_uncond + unconditional_guidance_scale * ( + e_t - e_t_uncond) + + if score_corrector is not None: + assert self.model.parameterization == 'eps' + e_t = score_corrector.modify_score(self.model, e_t, x, t, c, + **corrector_kwargs) + + alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas + if use_original_steps: + alphas_prev = self.model.alphas_cumprod_prev + sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod + else: + alphas_prev = self.ddim_alphas_prev + sqrt_one_minus_alphas = self.ddim_sqrt_one_minus_alphas + sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas + # select parameters corresponding to the currently considered timestep + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full((b, 1, 1, 1), + sqrt_one_minus_alphas[index], + device=device) + + # current prediction for x_0 + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + + if dynamic_threshold is not None: + pred_x0 = norm_thresholding(pred_x0, dynamic_threshold) + + # direction pointing to x_t + dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t + noise = sigma_t * noise_like(x.shape, device, + repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + + @torch.no_grad() + def encode(self, + x0, + c, + t_enc, + use_original_steps=False, + return_intermediates=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None): + num_reference_steps = self.ddpm_num_timesteps if use_original_steps else self.ddim_timesteps.shape[ + 0] + + assert t_enc <= num_reference_steps + num_steps = t_enc + + if use_original_steps: + alphas_next = self.alphas_cumprod[:num_steps] + alphas = self.alphas_cumprod_prev[:num_steps] + else: + alphas_next = self.ddim_alphas[:num_steps] + alphas = torch.tensor(self.ddim_alphas_prev[:num_steps]) + + x_next = x0 + intermediates = [] + inter_steps = [] + for i in tqdm(range(num_steps), desc='Encoding Image'): + t = torch.full((x0.shape[0], ), + i, + device=self.model.device, + dtype=torch.long) + if unconditional_guidance_scale == 1.: + noise_pred = self.model.apply_model(x_next, t, c) + else: + assert unconditional_conditioning is not None + e_t_uncond, noise_pred = torch.chunk( + self.model.apply_model( + torch.cat((x_next, x_next)), torch.cat((t, t)), + torch.cat((unconditional_conditioning, c))), 2) + noise_pred = e_t_uncond + unconditional_guidance_scale * ( + noise_pred - e_t_uncond) + + xt_weighted = (alphas_next[i] / alphas[i]).sqrt() * x_next + alp_1 = (1 / alphas_next[i] - 1).sqrt() + alp_2 = (1 / alphas[i] - 1).sqrt() + weighted_noise_pred = alphas_next[i].sqrt() * ( + alp_1 - alp_2) * noise_pred + x_next = xt_weighted + weighted_noise_pred + if return_intermediates and i % (num_steps // return_intermediates + ) == 0 and i < num_steps - 1: + intermediates.append(x_next) + inter_steps.append(i) + elif return_intermediates and i >= num_steps - 2: + intermediates.append(x_next) + inter_steps.append(i) + + out = {'x_encoded': x_next, 'intermediate_steps': inter_steps} + if return_intermediates: + out.update({'intermediates': intermediates}) + return x_next, out + + @torch.no_grad() + def stochastic_encode(self, x0, t, use_original_steps=False, noise=None): + # fast, but does not allow for exact reconstruction + # t serves as an index to gather the correct alphas + if use_original_steps: + sqrt_alphas_cumprod = self.sqrt_alphas_cumprod + sqrt_one_minus_alphas_cumprod = self.sqrt_one_minus_alphas_cumprod + else: + sqrt_alphas_cumprod = torch.sqrt(self.ddim_alphas) + sqrt_one_minus_alphas_cumprod = self.ddim_sqrt_one_minus_alphas + + if noise is None: + noise = torch.randn_like(x0) + return ( + extract_into_tensor(sqrt_alphas_cumprod, t, x0.shape) * x0 + + extract_into_tensor(sqrt_one_minus_alphas_cumprod, t, x0.shape) + * noise) + + @torch.no_grad() + def decode(self, + x_latent, + cond, + t_start, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + use_original_steps=False): + + timesteps = np.arange(self.ddpm_num_timesteps + ) if use_original_steps else self.ddim_timesteps + timesteps = timesteps[:t_start] + + time_range = np.flip(timesteps) + total_steps = timesteps.shape[0] + print(f'Running DDIM Sampling with {total_steps} timesteps') + + iterator = tqdm(time_range, desc='Decoding image', total=total_steps) + x_dec = x_latent + for i, step in enumerate(iterator): + index = total_steps - i - 1 + ts = torch.full((x_latent.shape[0], ), + step, + device=x_latent.device, + dtype=torch.long) + x_dec, _ = self.p_sample_ddim( + x_dec, + cond, + ts, + index=index, + use_original_steps=use_original_steps, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning) + return x_dec diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddpm.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddpm.py new file mode 100644 index 0000000000000000000000000000000000000000..4f57d4567a0e6f8e32b400dd88182d1fbb30f183 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ddpm.py @@ -0,0 +1,2553 @@ +""" +wild mixture of +https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +https://github.com/openai/improved-diffusion/blob/e94489283bb876ac1477d5dd7709bbbd2d9902ce/improved_diffusion/gaussian_diffusion.py +https://github.com/CompVis/taming-transformers +-- merci +""" + +import itertools +from contextlib import contextmanager, nullcontext +from functools import partial + +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn as nn +from einops import rearrange, repeat +from omegaconf import ListConfig +from pytorch_lightning.utilities.distributed import rank_zero_only +from torch.optim.lr_scheduler import LambdaLR +from torchvision.utils import make_grid +from tqdm import tqdm + +from ..util import (count_params, default, exists, instantiate_from_config, + isimage, ismap, log_txt_as_img, mean_flat) +from .attention import CrossAttention +from .autoencoder import AutoencoderKL, IdentityFirstStage, VQModelInterface +from .ddim import DDIMSampler +from .distributions import DiagonalGaussianDistribution, normal_kl +from .ema import LitEma +from .util_diffusion import extract_into_tensor, make_beta_schedule, noise_like + +__conditioning_keys__ = { + 'concat': 'c_concat', + 'crossattn': 'c_crossattn', + 'adm': 'y' +} + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +def uniform_on_device(r1, r2, shape, device): + return (r1 - r2) * torch.rand(*shape, device=device) + r2 + + +class DDPM(pl.LightningModule): + # classic DDPM with Gaussian diffusion, in image space + def __init__( + self, + unet_config, + timesteps=1000, + beta_schedule='linear', + loss_type='l2', + ckpt_path=None, + ignore_keys=[], + load_only_unet=False, + monitor='val/loss', + use_ema=True, + first_stage_key='image', + image_size=256, + channels=3, + log_every_t=100, + clip_denoised=True, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3, + given_betas=None, + original_elbo_weight=0., + v_posterior=0., # weight for choosing posterior variance as sigma = (1-v) * beta_tilde + v * beta + l_simple_weight=1., + conditioning_key=None, + parameterization='eps', # all assuming fixed variance schedules + scheduler_config=None, + use_positional_encodings=False, + learn_logvar=False, + logvar_init=0., + make_it_fit=False, + ucg_training=None, + ): + super().__init__() + assert parameterization in [ + 'eps', 'x0' + ], 'currently only supporting "eps" and "x0"' + self.parameterization = parameterization + print( + f'{self.__class__.__name__}: Running in {self.parameterization}-prediction mode' + ) + self.cond_stage_model = None + self.clip_denoised = clip_denoised + self.log_every_t = log_every_t + self.first_stage_key = first_stage_key + self.image_size = image_size # try conv? + self.channels = channels + self.use_positional_encodings = use_positional_encodings + self.model = DiffusionWrapper(unet_config, conditioning_key) + count_params(self.model, verbose=True) + self.use_ema = use_ema + if self.use_ema: + self.model_ema = LitEma(self.model) + print(f'Keeping EMAs of {len(list(self.model_ema.buffers()))}.') + + self.use_scheduler = scheduler_config is not None + if self.use_scheduler: + self.scheduler_config = scheduler_config + + self.v_posterior = v_posterior + self.original_elbo_weight = original_elbo_weight + self.l_simple_weight = l_simple_weight + + if monitor is not None: + self.monitor = monitor + self.make_it_fit = make_it_fit + if ckpt_path is not None: + self.init_from_ckpt( + ckpt_path, ignore_keys=ignore_keys, only_model=load_only_unet) + + self.register_schedule( + given_betas=given_betas, + beta_schedule=beta_schedule, + timesteps=timesteps, + linear_start=linear_start, + linear_end=linear_end, + cosine_s=cosine_s) + + self.loss_type = loss_type + + self.learn_logvar = learn_logvar + self.logvar = torch.full( + fill_value=logvar_init, size=(self.num_timesteps, )) + if self.learn_logvar: + self.logvar = nn.Parameter(self.logvar, requires_grad=True) + + self.ucg_training = ucg_training or dict() + if self.ucg_training: + self.ucg_prng = np.random.RandomState() + + def register_schedule(self, + given_betas=None, + beta_schedule='linear', + timesteps=1000, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3): + if exists(given_betas): + betas = given_betas + else: + betas = make_beta_schedule( + beta_schedule, + timesteps, + linear_start=linear_start, + linear_end=linear_end, + cosine_s=cosine_s) + alphas = 1. - betas + alphas_cumprod = np.cumprod(alphas, axis=0) + alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1]) + + timesteps, = betas.shape + self.num_timesteps = int(timesteps) + self.linear_start = linear_start + self.linear_end = linear_end + assert alphas_cumprod.shape[ + 0] == self.num_timesteps, 'alphas have to be defined for each timestep' + + to_torch = partial(torch.tensor, dtype=torch.float32) + + self.register_buffer('betas', to_torch(betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer('alphas_cumprod_prev', + to_torch(alphas_cumprod_prev)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer('sqrt_alphas_cumprod', + to_torch(np.sqrt(alphas_cumprod))) + self.register_buffer('sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1. - alphas_cumprod))) + self.register_buffer('log_one_minus_alphas_cumprod', + to_torch(np.log(1. - alphas_cumprod))) + self.register_buffer('sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod))) + self.register_buffer('sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod - 1))) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + posterior_variance = (1 - self.v_posterior) * betas * ( + 1. - alphas_cumprod_prev) / ( + 1. - alphas_cumprod) + self.v_posterior * betas + # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) + self.register_buffer('posterior_variance', + to_torch(posterior_variance)) + # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain + self.register_buffer( + 'posterior_log_variance_clipped', + to_torch(np.log(np.maximum(posterior_variance, 1e-20)))) + beta_1 = betas * np.sqrt(alphas_cumprod_prev) + beta_2 = (1. - alphas_cumprod) + self.register_buffer('posterior_mean_coef1', to_torch(beta_1 / beta_2)) + alpha_1 = (1. - alphas_cumprod_prev) * np.sqrt(alphas) + alpha_2 = (1. - alphas_cumprod) + self.register_buffer('posterior_mean_coef2', + to_torch(alpha_1 / alpha_2)) + + if self.parameterization == 'eps': + p_1 = 2 * self.posterior_variance * to_torch(alphas) + p_2 = (1 - self.alphas_cumprod) + lvlb_weights = self.betas**2 / (p_1 * p_2) + elif self.parameterization == 'x0': + lvlb_weights = 0.5 * np.sqrt(torch.Tensor(alphas_cumprod)) / ( + 2. * 1 - torch.Tensor(alphas_cumprod)) + else: + raise NotImplementedError('mu not supported') + # TODO how to choose this term + lvlb_weights[0] = lvlb_weights[1] + self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) + assert not torch.isnan(self.lvlb_weights).all() + + @contextmanager + def ema_scope(self, context=None): + if self.use_ema: + self.model_ema.store(self.model.parameters()) + self.model_ema.copy_to(self.model) + if context is not None: + print(f'{context}: Switched to EMA weights') + try: + yield None + finally: + if self.use_ema: + self.model_ema.restore(self.model.parameters()) + if context is not None: + print(f'{context}: Restored training weights') + + @torch.no_grad() + def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): + sd = torch.load(path, map_location='cpu') + if 'state_dict' in list(sd.keys()): + sd = sd['state_dict'] + + if self.make_it_fit: + n_params = len([ + name for name, _ in itertools.chain(self.named_parameters(), + self.named_buffers()) + ]) + for name, param in tqdm( + itertools.chain(self.named_parameters(), + self.named_buffers()), + desc='Fitting old weights to new weights', + total=n_params): + if name not in sd: + continue + old_shape = sd[name].shape + new_shape = param.shape + assert len(old_shape) == len(new_shape) + if len(new_shape) > 2: + # we only modify first two axes + assert new_shape[2:] == old_shape[2:] + # assumes first axis corresponds to output dim + if not new_shape == old_shape: + new_param = param.clone() + old_param = sd[name] + if len(new_shape) == 1: + for i in range(new_param.shape[0]): + new_param[i] = old_param[i % old_shape[0]] + elif len(new_shape) >= 2: + for i in range(new_param.shape[0]): + for j in range(new_param.shape[1]): + new_param[i, j] = old_param[i % old_shape[0], + j % old_shape[1]] + + n_used_old = torch.ones(old_shape[1]) + for j in range(new_param.shape[1]): + n_used_old[j % old_shape[1]] += 1 + n_used_new = torch.zeros(new_shape[1]) + for j in range(new_param.shape[1]): + n_used_new[j] = n_used_old[j % old_shape[1]] + + n_used_new = n_used_new[None, :] + while len(n_used_new.shape) < len(new_shape): + n_used_new = n_used_new.unsqueeze(-1) + new_param /= n_used_new + + sd[name] = new_param + + missing, unexpected = self.load_state_dict( + sd, + strict=False) if not only_model else self.model.load_state_dict( + sd, strict=False) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + if len(unexpected) > 0: + print(f'Unexpected Keys: {unexpected}') + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start) + variance = extract_into_tensor(1.0 - self.alphas_cumprod, t, + x_start.shape) + log_variance = extract_into_tensor(self.log_one_minus_alphas_cumprod, + t, x_start.shape) + return mean, variance, log_variance + + def predict_start_from_noise(self, x_t, t, noise): + return ( + extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, + x_t.shape) * noise) + + def q_posterior(self, x_start, x_t, t): + p_1 = extract_into_tensor(self.posterior_mean_coef1, t, + x_t.shape) * x_start + p_2 = extract_into_tensor(self.posterior_mean_coef2, t, + x_t.shape) * x_t + posterior_mean = (p_1 + p_2) + posterior_variance = extract_into_tensor(self.posterior_variance, t, + x_t.shape) + posterior_log_variance_clipped = extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape) + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def p_mean_variance(self, x, t, clip_denoised: bool): + model_out = self.model(x, t) + if self.parameterization == 'eps': + x_recon = self.predict_start_from_noise(x, t=t, noise=model_out) + elif self.parameterization == 'x0': + x_recon = model_out + if clip_denoised: + x_recon.clamp_(-1., 1.) + + model_mean, posterior_variance, posterior_log_variance = self.q_posterior( + x_start=x_recon, x_t=x, t=t) + return model_mean, posterior_variance, posterior_log_variance + + @torch.no_grad() + def p_sample(self, x, t, clip_denoised=True, repeat_noise=False): + b, *_, device = *x.shape, x.device + model_mean, _, model_log_variance = self.p_mean_variance( + x=x, t=t, clip_denoised=clip_denoised) + noise = noise_like(x.shape, device, repeat_noise) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape( + b, *((1, ) * (len(x.shape) - 1))) + return model_mean + nonzero_mask * (0.5 + * model_log_variance).exp() * noise + + @torch.no_grad() + def p_sample_loop(self, shape, return_intermediates=False): + device = self.betas.device + b = shape[0] + img = torch.randn(shape, device=device) + intermediates = [img] + for i in tqdm( + reversed(range(0, self.num_timesteps)), + desc='Sampling t', + total=self.num_timesteps): + img = self.p_sample( + img, + torch.full((b, ), i, device=device, dtype=torch.long), + clip_denoised=self.clip_denoised) + if i % self.log_every_t == 0 or i == self.num_timesteps - 1: + intermediates.append(img) + if return_intermediates: + return img, intermediates + return img + + @torch.no_grad() + def sample(self, batch_size=16, return_intermediates=False): + image_size = self.image_size + channels = self.channels + return self.p_sample_loop( + (batch_size, channels, image_size, image_size), + return_intermediates=return_intermediates) + + def q_sample(self, x_start, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + + extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, + x_start.shape) * noise) + + def get_loss(self, pred, target, mean=True): + if self.loss_type == 'l1': + loss = (target - pred).abs() + if mean: + loss = loss.mean() + elif self.loss_type == 'l2': + if mean: + loss = torch.nn.functional.mse_loss(target, pred) + else: + loss = torch.nn.functional.mse_loss( + target, pred, reduction='none') + else: + raise NotImplementedError("unknown loss type '{loss_type}'") + + return loss + + def p_losses(self, x_start, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + model_out = self.model(x_noisy, t) + + loss_dict = {} + if self.parameterization == 'eps': + target = noise + elif self.parameterization == 'x0': + target = x_start + else: + raise NotImplementedError( + f'Paramterization {self.parameterization} not yet supported') + + loss = self.get_loss(model_out, target, mean=False).mean(dim=[1, 2, 3]) + + log_prefix = 'train' if self.training else 'val' + + loss_dict.update({f'{log_prefix}/loss_simple': loss.mean()}) + loss_simple = loss.mean() * self.l_simple_weight + + loss_vlb = (self.lvlb_weights[t] * loss).mean() + loss_dict.update({f'{log_prefix}/loss_vlb': loss_vlb}) + + loss = loss_simple + self.original_elbo_weight * loss_vlb + + loss_dict.update({f'{log_prefix}/loss': loss}) + + return loss, loss_dict + + def forward(self, x, *args, **kwargs): + # b, c, h, w, device, img_size, = *x.shape, x.device, self.image_size + # assert h == img_size and w == img_size, f'height and width of image must be {img_size}' + t = torch.randint( + 0, self.num_timesteps, (x.shape[0], ), device=self.device).long() + return self.p_losses(x, t, *args, **kwargs) + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = rearrange(x, 'b h w c -> b c h w') + x = x.to(memory_format=torch.contiguous_format).float() + return x + + def shared_step(self, batch): + x = self.get_input(batch, self.first_stage_key) + loss, loss_dict = self(x) + return loss, loss_dict + + def training_step(self, batch, batch_idx): + for k in self.ucg_training: + p = self.ucg_training[k]['p'] + val = self.ucg_training[k]['val'] + if val is None: + val = '' + for i in range(len(batch[k])): + if self.ucg_prng.choice(2, p=[1 - p, p]): + batch[k][i] = val + + loss, loss_dict = self.shared_step(batch) + + self.log_dict( + loss_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True) + + self.log( + 'global_step', + self.global_step, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=False) + + if self.use_scheduler: + lr = self.optimizers().param_groups[0]['lr'] + self.log( + 'lr_abs', + lr, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=False) + + return loss + + @torch.no_grad() + def validation_step(self, batch, batch_idx): + _, loss_dict_no_ema = self.shared_step(batch) + with self.ema_scope(): + _, loss_dict_ema = self.shared_step(batch) + loss_dict_ema = { + key + '_ema': loss_dict_ema[key] + for key in loss_dict_ema + } + self.log_dict( + loss_dict_no_ema, + prog_bar=False, + logger=True, + on_step=False, + on_epoch=True) + self.log_dict( + loss_dict_ema, + prog_bar=False, + logger=True, + on_step=False, + on_epoch=True) + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self.model) + + def _get_rows_from_list(self, samples): + n_imgs_per_row = len(samples) + denoise_grid = rearrange(samples, 'n b c h w -> b n c h w') + denoise_grid = rearrange(denoise_grid, 'b n c h w -> (b n) c h w') + denoise_grid = make_grid(denoise_grid, nrow=n_imgs_per_row) + return denoise_grid + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=2, + sample=True, + return_keys=None, + **kwargs): + log = dict() + x = self.get_input(batch, self.first_stage_key) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + x = x.to(self.device)[:N] + log['inputs'] = x + + # get diffusion row + diffusion_row = list() + x_start = x[:n_row] + + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(x_start) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + diffusion_row.append(x_noisy) + + log['diffusion_row'] = self._get_rows_from_list(diffusion_row) + + if sample: + # get denoise row + with self.ema_scope('Plotting'): + samples, denoise_row = self.sample( + batch_size=N, return_intermediates=True) + + log['samples'] = samples + log['denoise_row'] = self._get_rows_from_list(denoise_row) + + if return_keys: + if np.intersect1d(list(log.keys()), return_keys).shape[0] == 0: + return log + else: + return {key: log[key] for key in return_keys} + return log + + def configure_optimizers(self): + lr = self.learning_rate + params = list(self.model.parameters()) + if self.learn_logvar: + params = params + [self.logvar] + opt = torch.optim.AdamW(params, lr=lr) + return opt + + +class LatentDiffusion(DDPM): + """main class""" + + def __init__(self, + first_stage_config, + cond_stage_config, + num_timesteps_cond=None, + cond_stage_key='image', + cond_stage_trainable=False, + concat_mode=True, + cond_stage_forward=None, + conditioning_key=None, + scale_factor=1.0, + scale_by_std=False, + unet_trainable=True, + *args, + **kwargs): + self.num_timesteps_cond = default(num_timesteps_cond, 1) + self.scale_by_std = scale_by_std + assert self.num_timesteps_cond <= kwargs['timesteps'] + # for backwards compatibility after implementation of DiffusionWrapper + if conditioning_key is None: + conditioning_key = 'concat' if concat_mode else 'crossattn' + if cond_stage_config == '__is_unconditional__': + conditioning_key = None + ckpt_path = kwargs.pop('ckpt_path', None) + ignore_keys = kwargs.pop('ignore_keys', []) + super().__init__(conditioning_key=conditioning_key, *args, **kwargs) + self.concat_mode = concat_mode + self.cond_stage_trainable = cond_stage_trainable + self.unet_trainable = unet_trainable + self.cond_stage_key = cond_stage_key + try: + self.num_downs = len( + first_stage_config.params.ddconfig.ch_mult) - 1 + except Exception: + self.num_downs = 0 + if not scale_by_std: + self.scale_factor = scale_factor + else: + self.register_buffer('scale_factor', torch.tensor(scale_factor)) + self.instantiate_first_stage(first_stage_config) + self.instantiate_cond_stage(cond_stage_config) + self.cond_stage_forward = cond_stage_forward + + # construct linear projection layer for concatenating image CLIP embedding and RT + self.cc_projection = nn.Linear(772, 768) + nn.init.eye_(list(self.cc_projection.parameters())[0][:768, :768]) + nn.init.zeros_(list(self.cc_projection.parameters())[1]) + self.cc_projection.requires_grad_(True) + + self.clip_denoised = False + self.bbox_tokenizer = None + + self.restarted_from_ckpt = False + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys) + self.restarted_from_ckpt = True + + def make_cond_schedule(self, ): + self.cond_ids = torch.full( + size=(self.num_timesteps, ), + fill_value=self.num_timesteps - 1, + dtype=torch.long) + ids = torch.round( + torch.linspace(0, self.num_timesteps - 1, + self.num_timesteps_cond)).long() + self.cond_ids[:self.num_timesteps_cond] = ids + + @rank_zero_only + @torch.no_grad() + def on_train_batch_start(self, batch, batch_idx, dataloader_idx): + # only for very first batch + cond_1 = self.scale_by_std and self.current_epoch == 0 + cond_2 = self.global_step == 0 and batch_idx == 0 + if cond_1 and cond_2 and not self.restarted_from_ckpt: + assert self.scale_factor == 1., 'error' + print('### USING STD-RESCALING ###') + x = super().get_input(batch, self.first_stage_key) + x = x.to(self.device) + encoder_posterior = self.encode_first_stage(x) + z = self.get_first_stage_encoding(encoder_posterior).detach() + del self.scale_factor + self.register_buffer('scale_factor', 1. / z.flatten().std()) + print(f'setting self.scale_factor to {self.scale_factor}') + print('### USING STD-RESCALING ###') + + def register_schedule(self, + given_betas=None, + beta_schedule='linear', + timesteps=1000, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3): + super().register_schedule(given_betas, beta_schedule, timesteps, + linear_start, linear_end, cosine_s) + + self.shorten_cond_schedule = self.num_timesteps_cond > 1 + if self.shorten_cond_schedule: + self.make_cond_schedule() + + def instantiate_first_stage(self, config): + model = instantiate_from_config(config) + self.first_stage_model = model.eval() + self.first_stage_model.train = disabled_train + for param in self.first_stage_model.parameters(): + param.requires_grad = False + + def instantiate_cond_stage(self, config): + if not self.cond_stage_trainable: + if config == '__is_first_stage__': + print('Using first stage also as cond stage.') + self.cond_stage_model = self.first_stage_model + elif config == '__is_unconditional__': + print( + f'Training {self.__class__.__name__} as an unconditional model.' + ) + self.cond_stage_model = None + # self.be_unconditional = True + else: + model = instantiate_from_config(config) + self.cond_stage_model = model.eval() + self.cond_stage_model.train = disabled_train + for param in self.cond_stage_model.parameters(): + param.requires_grad = False + else: + assert config != '__is_first_stage__' + assert config != '__is_unconditional__' + model = instantiate_from_config(config) + self.cond_stage_model = model + + def _get_denoise_row_from_list(self, + samples, + desc='', + force_no_decoder_quantization=False): + denoise_row = [] + for zd in tqdm(samples, desc=desc): + denoise_row.append( + self.decode_first_stage( + zd.to(self.device), + force_not_quantize=force_no_decoder_quantization)) + n_imgs_per_row = len(denoise_row) + denoise_row = torch.stack(denoise_row) # n_log_step, n_row, C, H, W + denoise_grid = rearrange(denoise_row, 'n b c h w -> b n c h w') + denoise_grid = rearrange(denoise_grid, 'b n c h w -> (b n) c h w') + denoise_grid = make_grid(denoise_grid, nrow=n_imgs_per_row) + return denoise_grid + + def get_first_stage_encoding(self, encoder_posterior): + if isinstance(encoder_posterior, DiagonalGaussianDistribution): + z = encoder_posterior.sample() + elif isinstance(encoder_posterior, torch.Tensor): + z = encoder_posterior + else: + raise NotImplementedError( + f"encoder_posterior of type '{type(encoder_posterior)}' not yet implemented" + ) + return self.scale_factor * z + + def get_learned_conditioning(self, c): + if self.cond_stage_forward is None: + if hasattr(self.cond_stage_model, 'encode') and callable( + self.cond_stage_model.encode): + c = self.cond_stage_model.encode(c) + if isinstance(c, DiagonalGaussianDistribution): + c = c.mode() + else: + c = self.cond_stage_model(c) + else: + assert hasattr(self.cond_stage_model, self.cond_stage_forward) + c = getattr(self.cond_stage_model, self.cond_stage_forward)(c) + return c + + def meshgrid(self, h, w): + y = torch.arange(0, h).view(h, 1, 1).repeat(1, w, 1) + x = torch.arange(0, w).view(1, w, 1).repeat(h, 1, 1) + + arr = torch.cat([y, x], dim=-1) + return arr + + def delta_border(self, h, w): + """ + :param h: height + :param w: width + :return: normalized distance to image border, + wtith min distance = 0 at border and max dist = 0.5 at image center + """ + lower_right_corner = torch.tensor([h - 1, w - 1]).view(1, 1, 2) + arr = self.meshgrid(h, w) / lower_right_corner + dist_left_up = torch.min(arr, dim=-1, keepdims=True)[0] + dist_right_down = torch.min(1 - arr, dim=-1, keepdims=True)[0] + edge_dist = torch.min( + torch.cat([dist_left_up, dist_right_down], dim=-1), dim=-1)[0] + return edge_dist + + def get_weighting(self, h, w, Ly, Lx, device): + weighting = self.delta_border(h, w) + weighting = torch.clip( + weighting, + self.split_input_params['clip_min_weight'], + self.split_input_params['clip_max_weight'], + ) + weighting = weighting.view(1, h * w, 1).repeat(1, 1, + Ly * Lx).to(device) + + if self.split_input_params['tie_braker']: + L_weighting = self.delta_border(Ly, Lx) + L_weighting = torch.clip( + L_weighting, self.split_input_params['clip_min_tie_weight'], + self.split_input_params['clip_max_tie_weight']) + + L_weighting = L_weighting.view(1, 1, Ly * Lx).to(device) + weighting = weighting * L_weighting + return weighting + + def get_fold_unfold(self, + x, + kernel_size, + stride, + uf=1, + df=1): # todo load once not every time, shorten code + """ + :param x: img of size (bs, c, h, w) + :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1]) + """ + bs, nc, h, w = x.shape + + # number of crops in image + Ly = (h - kernel_size[0]) // stride[0] + 1 + Lx = (w - kernel_size[1]) // stride[1] + 1 + + if uf == 1 and df == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride) + unfold = torch.nn.Unfold(**fold_params) + + fold = torch.nn.Fold(output_size=x.shape[2:], **fold_params) + + weighting = self.get_weighting(kernel_size[0], kernel_size[1], Ly, + Lx, x.device).to(x.dtype) + normalization = fold(weighting).view(1, 1, h, + w) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0], kernel_size[1], Ly * Lx)) + + elif uf > 1 and df == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride) + unfold = torch.nn.Unfold(**fold_params) + + fold_params2 = dict( + kernel_size=(kernel_size[0] * uf, kernel_size[0] * uf), + dilation=1, + padding=0, + stride=(stride[0] * uf, stride[1] * uf)) + fold = torch.nn.Fold( + output_size=(x.shape[2] * uf, x.shape[3] * uf), **fold_params2) + + weighting = self.get_weighting(kernel_size[0] * uf, + kernel_size[1] * uf, Ly, Lx, + x.device).to(x.dtype) + normalization = fold(weighting).view( + 1, 1, h * uf, w * uf) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0] * uf, kernel_size[1] * uf, Ly * Lx)) + + elif df > 1 and uf == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride) + unfold = torch.nn.Unfold(**fold_params) + + fold_params2 = dict( + kernel_size=(kernel_size[0] // df, kernel_size[0] // df), + dilation=1, + padding=0, + stride=(stride[0] // df, stride[1] // df)) + fold = torch.nn.Fold( + output_size=(x.shape[2] // df, x.shape[3] // df), + **fold_params2) + + weighting = self.get_weighting(kernel_size[0] // df, + kernel_size[1] // df, Ly, Lx, + x.device).to(x.dtype) + normalization = fold(weighting).view( + 1, 1, h // df, w // df) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0] // df, kernel_size[1] // df, Ly * Lx)) + + else: + raise NotImplementedError + + return fold, unfold, normalization, weighting + + @torch.no_grad() + def get_input(self, + batch, + k, + return_first_stage_outputs=False, + force_c_encode=False, + cond_key=None, + return_original_cond=False, + bs=None, + uncond=0.05): + x = super().get_input(batch, k) + T = batch['T'].to(memory_format=torch.contiguous_format).float() + + if bs is not None: + x = x[:bs] + T = T[:bs].to(self.device) + + x = x.to(self.device) + encoder_posterior = self.encode_first_stage(x) + z = self.get_first_stage_encoding(encoder_posterior).detach() + cond_key = cond_key or self.cond_stage_key + xc = super().get_input(batch, cond_key).to(self.device) + if bs is not None: + xc = xc[:bs] + cond = {} + + random = torch.rand(x.size(0), device=x.device) + prompt_mask = rearrange(random < 2 * uncond, 'n -> n 1 1') + r_1 = (random >= uncond).float() + r_2 = (random < 3 * uncond).float() + input_mask = 1 - rearrange(r_1 * r_2, 'n -> n 1 1 1') + null_prompt = self.get_learned_conditioning(['']) + + # z.shape: [8, 4, 64, 64]; c.shape: [8, 1, 768] + # print('=========== xc shape ===========', xc.shape) + with torch.enable_grad(): + clip_emb = self.get_learned_conditioning(xc).detach() + null_prompt = self.get_learned_conditioning(['']).detach() + mask_used = torch.where(prompt_mask, null_prompt, clip_emb) + cond['c_crossattn'] = [ + self.cc_projection( + torch.cat([mask_used, T[:, None, :]], dim=-1)) + ] + cond['c_concat'] = [ + input_mask * self.encode_first_stage( + (xc.to(self.device))).mode().detach() + ] + out = [z, cond] + if return_first_stage_outputs: + xrec = self.decode_first_stage(z) + out.extend([x, xrec]) + if return_original_cond: + out.append(xc) + return out + + # @torch.no_grad() + def decode_first_stage(self, + z, + predict_cids=False, + force_not_quantize=False): + if predict_cids: + if z.dim() == 4: + z = torch.argmax(z.exp(), dim=1).long() + z = self.first_stage_model.quantize.get_codebook_entry( + z, shape=None) + z = rearrange(z, 'b h w c -> b c h w').contiguous() + + z = 1. / self.scale_factor * z + + if hasattr(self, 'split_input_params'): + if self.split_input_params['patch_distributed_vq']: + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + uf = self.split_input_params['vqf'] + bs, nc, h, w = z.shape + if ks[0] > h or ks[1] > w: + ks = (min(ks[0], h), min(ks[1], w)) + print('reducing Kernel') + + if stride[0] > h or stride[1] > w: + stride = (min(stride[0], h), min(stride[1], w)) + print('reducing stride') + + fold, unfold, normalization, weighting = self.get_fold_unfold( + z, ks, stride, uf=uf) + + z = unfold(z) # (bn, nc * prod(**ks), L) + # 1. Reshape to img shape + z = z.view((z.shape[0], -1, ks[0], ks[1], + z.shape[-1])) # (bn, nc, ks[0], ks[1], L ) + + # 2. apply model loop over last dim + if isinstance(self.first_stage_model, VQModelInterface): + output_list = [ + self.first_stage_model.decode( + z[:, :, :, :, i], + force_not_quantize=predict_cids + or force_not_quantize) for i in range(z.shape[-1]) + ] + else: + + output_list = [ + self.first_stage_model.decode(z[:, :, :, :, i]) + for i in range(z.shape[-1]) + ] + + o = torch.stack( + output_list, axis=-1) # # (bn, nc, ks[0], ks[1], L) + o = o * weighting + # Reverse 1. reshape to img shape + o = o.view((o.shape[0], -1, + o.shape[-1])) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + decoded = fold(o) + decoded = decoded / normalization # norm is shape (1, 1, h, w) + return decoded + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, + force_not_quantize=predict_cids or force_not_quantize) + else: + return self.first_stage_model.decode(z) + + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, force_not_quantize=predict_cids or force_not_quantize) + else: + return self.first_stage_model.decode(z) + + @torch.no_grad() + def encode_first_stage(self, x): + if hasattr(self, 'split_input_params'): + if self.split_input_params['patch_distributed_vq']: + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + df = self.split_input_params['vqf'] + self.split_input_params['original_image_size'] = x.shape[-2:] + bs, nc, h, w = x.shape + if ks[0] > h or ks[1] > w: + ks = (min(ks[0], h), min(ks[1], w)) + print('reducing Kernel') + + if stride[0] > h or stride[1] > w: + stride = (min(stride[0], h), min(stride[1], w)) + print('reducing stride') + + fold, unfold, normalization, weighting = self.get_fold_unfold( + x, ks, stride, df=df) + z = unfold(x) # (bn, nc * prod(**ks), L) + # Reshape to img shape + z = z.view((z.shape[0], -1, ks[0], ks[1], + z.shape[-1])) # (bn, nc, ks[0], ks[1], L ) + + output_list = [ + self.first_stage_model.encode(z[:, :, :, :, i]) + for i in range(z.shape[-1]) + ] + + o = torch.stack(output_list, axis=-1) + o = o * weighting + + # Reverse reshape to img shape + o = o.view((o.shape[0], -1, + o.shape[-1])) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + decoded = fold(o) + decoded = decoded / normalization + return decoded + + else: + return self.first_stage_model.encode(x) + else: + return self.first_stage_model.encode(x) + + def shared_step(self, batch, **kwargs): + x, c = self.get_input(batch, self.first_stage_key) + loss = self(x, c) + return loss + + def forward(self, x, c, *args, **kwargs): + t = torch.randint( + 0, self.num_timesteps, (x.shape[0], ), device=self.device).long() + if self.model.conditioning_key is not None: + assert c is not None + # if self.cond_stage_trainable: + # c = self.get_learned_conditioning(c) + if self.shorten_cond_schedule: # TODO: drop this option + tc = self.cond_ids[t].to(self.device) + c = self.q_sample( + x_start=c, t=tc, noise=torch.randn_like(c.float())) + return self.p_losses(x, c, t, *args, **kwargs) + + def _rescale_annotations(self, bboxes, + crop_coordinates): # TODO: move to dataset + + def rescale_bbox(bbox): + x0 = clamp((bbox[0] - crop_coordinates[0]) / crop_coordinates[2]) + y0 = clamp((bbox[1] - crop_coordinates[1]) / crop_coordinates[3]) + w = min(bbox[2] / crop_coordinates[2], 1 - x0) + h = min(bbox[3] / crop_coordinates[3], 1 - y0) + return x0, y0, w, h + + return [rescale_bbox(b) for b in bboxes] + + def apply_model(self, x_noisy, t, cond, return_ids=False): + + if isinstance(cond, dict): + # hybrid case, cond is exptected to be a dict + pass + else: + if not isinstance(cond, list): + cond = [cond] + key = 'c_concat' if self.model.conditioning_key == 'concat' else 'c_crossattn' + cond = {key: cond} + + if hasattr(self, 'split_input_params'): + assert len( + cond) == 1 # todo can only deal with one conditioning atm + assert not return_ids + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + + h, w = x_noisy.shape[-2:] + + fold, unfold, normalization, weighting = self.get_fold_unfold( + x_noisy, ks, stride) + + z = unfold(x_noisy) # (bn, nc * prod(**ks), L) + # Reshape to img shape + z = z.view((z.shape[0], -1, ks[0], ks[1], + z.shape[-1])) # (bn, nc, ks[0], ks[1], L ) + z_list = [z[:, :, :, :, i] for i in range(z.shape[-1])] + + if self.cond_stage_key in [ + 'image', 'LR_image', 'segmentation', 'bbox_img' + ] and self.model.conditioning_key: # todo check for completeness + c_key = next(iter(cond.keys())) # get key + c = next(iter(cond.values())) # get value + assert (len(c) == 1 + ) # todo extend to list with more than one elem + c = c[0] # get element + + c = unfold(c) + c = c.view((c.shape[0], -1, ks[0], ks[1], + c.shape[-1])) # (bn, nc, ks[0], ks[1], L ) + + cond_list = [{ + c_key: [c[:, :, :, :, i]] + } for i in range(c.shape[-1])] + + elif self.cond_stage_key == 'coordinates_bbox': + assert 'original_image_size' in self.split_input_params, 'wrong' + + # assuming padding of unfold is always 0 and its dilation is always 1 + n_patches_per_row = int((w - ks[0]) / stride[0] + 1) + full_img_h, full_img_w = self.split_input_params[ + 'original_image_size'] + # as we are operating on latents, we need the factor from the original image size to the + # spatial latent size to properly rescale the crops for regenerating the bbox annotations + num_downs = self.first_stage_model.encoder.num_resolutions - 1 + rescale_latent = 2**(num_downs) + + # get top left postions of patches as conforming for the bbbox tokenizer, therefore we + # need to rescale the tl patch coordinates to be in between (0,1) + res_1 = rescale_latent * stride[0] * ( + patch_nr % n_patches_per_row) / full_img_w + res_2 = rescale_latent * stride[1] * ( + patch_nr // n_patches_per_row) / full_img_h + tl_patch_coordinates = [(res_1, res_2) + for patch_nr in range(z.shape[-1])] + + # patch_limits are tl_coord, width and height coordinates as (x_tl, y_tl, h, w) + patch_limits = [ + (x_tl, y_tl, rescale_latent * ks[0] / full_img_w, + rescale_latent * ks[1] / full_img_h) + for x_tl, y_tl in tl_patch_coordinates + ] + + # tokenize crop coordinates for the bounding boxes of the respective patches + patch_limits_tknzd = [ + torch.LongTensor( + self.bbox_tokenizer._crop_encoder(bbox))[None].to( + self.device) for bbox in patch_limits + ] # list of length l with tensors of shape (1, 2) + # cut tknzd crop position from conditioning + assert isinstance( + cond, dict), 'cond must be dict to be fed into model' + cut_cond = cond['c_crossattn'][0][..., :-2].to(self.device) + + adapted_cond = torch.stack([ + torch.cat([cut_cond, p], dim=1) for p in patch_limits_tknzd + ]) + adapted_cond = rearrange(adapted_cond, 'l b n -> (l b) n') + adapted_cond = self.get_learned_conditioning(adapted_cond) + adapted_cond = rearrange( + adapted_cond, '(l b) n d -> l b n d', l=z.shape[-1]) + + cond_list = [{'c_crossattn': [e]} for e in adapted_cond] + + else: + cond_list = [cond for i in range(z.shape[-1]) + ] # Todo make this more efficient + + # apply model by loop over crops + output_list = [ + self.model(z_list[i], t, **cond_list[i]) + for i in range(z.shape[-1]) + ] + assert not isinstance( + output_list[0], tuple + ) # todo cant deal with multiple model outputs check this never happens + + o = torch.stack(output_list, axis=-1) + o = o * weighting + # Reverse reshape to img shape + o = o.view( + (o.shape[0], -1, o.shape[-1])) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + x_recon = fold(o) / normalization + + else: + x_recon = self.model(x_noisy, t, **cond) + + if isinstance(x_recon, tuple) and not return_ids: + return x_recon[0] + else: + return x_recon + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + ex_1 = ( + extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - pred_xstart) + ex_2 = extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, + x_t.shape) + return ex_1 / ex_2 + + def _prior_bpd(self, x_start): + """ + Get the prior KL term for the variational lower-bound, measured in + bits-per-dim. + This term can't be optimized, as it only depends on the encoder. + :param x_start: the [N x C x ...] tensor of inputs. + :return: a batch of [N] KL values (in bits), one per batch element. + """ + batch_size = x_start.shape[0] + t = torch.tensor( + [self.num_timesteps - 1] * batch_size, device=x_start.device) + qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t) + kl_prior = normal_kl( + mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0) + return mean_flat(kl_prior) / np.log(2.0) + + def p_losses(self, x_start, cond, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + model_output = self.apply_model(x_noisy, t, cond) + + loss_dict = {} + prefix = 'train' if self.training else 'val' + + if self.parameterization == 'x0': + target = x_start + elif self.parameterization == 'eps': + target = noise + else: + raise NotImplementedError() + + loss_simple = self.get_loss( + model_output, target, mean=False).mean([1, 2, 3]) + loss_dict.update({f'{prefix}/loss_simple': loss_simple.mean()}) + + logvar_t = self.logvar[t].to(self.device) + loss = loss_simple / torch.exp(logvar_t) + logvar_t + # loss = loss_simple / torch.exp(self.logvar) + self.logvar + if self.learn_logvar: + loss_dict.update({f'{prefix}/loss_gamma': loss.mean()}) + loss_dict.update({'logvar': self.logvar.data.mean()}) + + loss = self.l_simple_weight * loss.mean() + + loss_vlb = self.get_loss( + model_output, target, mean=False).mean(dim=(1, 2, 3)) + loss_vlb = (self.lvlb_weights[t] * loss_vlb).mean() + loss_dict.update({f'{prefix}/loss_vlb': loss_vlb}) + loss += (self.original_elbo_weight * loss_vlb) + loss_dict.update({f'{prefix}/loss': loss}) + + return loss, loss_dict + + def p_mean_variance(self, + x, + c, + t, + clip_denoised: bool, + return_codebook_ids=False, + quantize_denoised=False, + return_x0=False, + score_corrector=None, + corrector_kwargs=None): + t_in = t + model_out = self.apply_model( + x, t_in, c, return_ids=return_codebook_ids) + + if score_corrector is not None: + assert self.parameterization == 'eps' + model_out = score_corrector.modify_score(self, model_out, x, t, c, + **corrector_kwargs) + + if return_codebook_ids: + model_out, logits = model_out + + if self.parameterization == 'eps': + x_recon = self.predict_start_from_noise(x, t=t, noise=model_out) + elif self.parameterization == 'x0': + x_recon = model_out + else: + raise NotImplementedError() + + if clip_denoised: + x_recon.clamp_(-1., 1.) + if quantize_denoised: + x_recon, _, [_, _, + indices] = self.first_stage_model.quantize(x_recon) + model_mean, posterior_variance, posterior_log_variance = self.q_posterior( + x_start=x_recon, x_t=x, t=t) + if return_codebook_ids: + return model_mean, posterior_variance, posterior_log_variance, logits + elif return_x0: + return model_mean, posterior_variance, posterior_log_variance, x_recon + else: + return model_mean, posterior_variance, posterior_log_variance + + @torch.no_grad() + def p_sample(self, + x, + c, + t, + clip_denoised=False, + repeat_noise=False, + return_codebook_ids=False, + quantize_denoised=False, + return_x0=False, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None): + b, *_, device = *x.shape, x.device + outputs = self.p_mean_variance( + x=x, + c=c, + t=t, + clip_denoised=clip_denoised, + return_codebook_ids=return_codebook_ids, + quantize_denoised=quantize_denoised, + return_x0=return_x0, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs) + if return_codebook_ids: + raise DeprecationWarning('Support dropped.') + model_mean, _, model_log_variance, logits = outputs + elif return_x0: + model_mean, _, model_log_variance, x0 = outputs + else: + model_mean, _, model_log_variance = outputs + + noise = noise_like(x.shape, device, repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape( + b, *((1, ) * (len(x.shape) - 1))) + + if return_codebook_ids: + return model_mean + nonzero_mask * ( + 0.5 * model_log_variance).exp() * noise, logits.argmax(dim=1) + if return_x0: + return model_mean + nonzero_mask * ( + 0.5 * model_log_variance).exp() * noise, x0 + else: + return model_mean + nonzero_mask * ( + 0.5 * model_log_variance).exp() * noise + + @torch.no_grad() + def progressive_denoising(self, + cond, + shape, + verbose=True, + callback=None, + quantize_denoised=False, + img_callback=None, + mask=None, + x0=None, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + batch_size=None, + x_T=None, + start_T=None, + log_every_t=None): + if not log_every_t: + log_every_t = self.log_every_t + timesteps = self.num_timesteps + if batch_size is not None: + b = batch_size if batch_size is not None else shape[0] + shape = [batch_size] + list(shape) + else: + b = batch_size = shape[0] + if x_T is None: + img = torch.randn(shape, device=self.device) + else: + img = x_T + intermediates = [] + if cond is not None: + if isinstance(cond, dict): + cond = { + key: + cond[key][:batch_size] if not isinstance(cond[key], list) + else list(map(lambda x: x[:batch_size], cond[key])) + for key in cond + } + else: + cond = [c[:batch_size] for c in cond] if isinstance( + cond, list) else cond[:batch_size] + + if start_T is not None: + timesteps = min(timesteps, start_T) + iterator = tqdm( + reversed(range(0, timesteps)), + desc='Progressive Generation', + total=timesteps) if verbose else reversed(range(0, timesteps)) + if type(temperature) == float: + temperature = [temperature] * timesteps + + for i in iterator: + ts = torch.full((b, ), i, device=self.device, dtype=torch.long) + if self.shorten_cond_schedule: + assert self.model.conditioning_key != 'hybrid' + tc = self.cond_ids[ts].to(cond.device) + cond = self.q_sample( + x_start=cond, t=tc, noise=torch.randn_like(cond)) + + img, x0_partial = self.p_sample( + img, + cond, + ts, + clip_denoised=self.clip_denoised, + quantize_denoised=quantize_denoised, + return_x0=True, + temperature=temperature[i], + noise_dropout=noise_dropout, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs) + if mask is not None: + assert x0 is not None + img_orig = self.q_sample(x0, ts) + img = img_orig * mask + (1. - mask) * img + + if i % log_every_t == 0 or i == timesteps - 1: + intermediates.append(x0_partial) + if callback: + callback(i) + if img_callback: + img_callback(img, i) + return img, intermediates + + @torch.no_grad() + def p_sample_loop(self, + cond, + shape, + return_intermediates=False, + x_T=None, + verbose=True, + callback=None, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + img_callback=None, + start_T=None, + log_every_t=None): + + if not log_every_t: + log_every_t = self.log_every_t + device = self.betas.device + b = shape[0] + if x_T is None: + img = torch.randn(shape, device=device) + else: + img = x_T + + intermediates = [img] + if timesteps is None: + timesteps = self.num_timesteps + + if start_T is not None: + timesteps = min(timesteps, start_T) + iterator = tqdm( + reversed(range(0, timesteps)), desc='Sampling t', + total=timesteps) if verbose else reversed(range(0, timesteps)) + + if mask is not None: + assert x0 is not None + assert x0.shape[2:3] == mask.shape[2: + 3] # spatial size has to match + + for i in iterator: + ts = torch.full((b, ), i, device=device, dtype=torch.long) + if self.shorten_cond_schedule: + assert self.model.conditioning_key != 'hybrid' + tc = self.cond_ids[ts].to(cond.device) + cond = self.q_sample( + x_start=cond, t=tc, noise=torch.randn_like(cond)) + + img = self.p_sample( + img, + cond, + ts, + clip_denoised=self.clip_denoised, + quantize_denoised=quantize_denoised) + if mask is not None: + img_orig = self.q_sample(x0, ts) + img = img_orig * mask + (1. - mask) * img + + if i % log_every_t == 0 or i == timesteps - 1: + intermediates.append(img) + if callback: + callback(i) + if img_callback: + img_callback(img, i) + + if return_intermediates: + return img, intermediates + return img + + @torch.no_grad() + def sample(self, + cond, + batch_size=16, + return_intermediates=False, + x_T=None, + verbose=True, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + shape=None, + **kwargs): + if shape is None: + shape = (batch_size, self.channels, self.image_size, + self.image_size) + if cond is not None: + if isinstance(cond, dict): + cond = { + key: + cond[key][:batch_size] if not isinstance(cond[key], list) + else list(map(lambda x: x[:batch_size], cond[key])) + for key in cond + } + else: + cond = [c[:batch_size] for c in cond] if isinstance( + cond, list) else cond[:batch_size] + return self.p_sample_loop( + cond, + shape, + return_intermediates=return_intermediates, + x_T=x_T, + verbose=verbose, + timesteps=timesteps, + quantize_denoised=quantize_denoised, + mask=mask, + x0=x0) + + @torch.no_grad() + def sample_log(self, cond, batch_size, ddim, ddim_steps, **kwargs): + if ddim: + ddim_sampler = DDIMSampler(self) + shape = (self.channels, self.image_size, self.image_size) + samples, intermediates = ddim_sampler.sample( + ddim_steps, batch_size, shape, cond, verbose=False, **kwargs) + + else: + samples, intermediates = self.sample( + cond=cond, + batch_size=batch_size, + return_intermediates=True, + **kwargs) + + return samples, intermediates + + @torch.no_grad() + def get_unconditional_conditioning(self, + batch_size, + null_label=None, + image_size=512): + if null_label is not None: + xc = null_label + if isinstance(xc, ListConfig): + xc = list(xc) + if isinstance(xc, dict) or isinstance(xc, list): + c = self.get_learned_conditioning(xc) + else: + if hasattr(xc, 'to'): + xc = xc.to(self.device) + c = self.get_learned_conditioning(xc) + else: + # todo: get null label from cond_stage_model + raise NotImplementedError() + c = repeat(c, '1 ... -> b ...', b=batch_size).to(self.device) + cond = {} + cond['c_crossattn'] = [c] + cond['c_concat'] = [ + torch.zeros([batch_size, 4, image_size // 8, + image_size // 8]).to(self.device) + ] + return cond + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=200, + ddim_eta=1., + return_keys=None, + quantize_denoised=True, + inpaint=True, + plot_denoise_rows=False, + plot_progressive_rows=True, + plot_diffusion_rows=True, + unconditional_guidance_scale=1., + unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc = self.get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=N) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption', 'txt']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch[self.cond_stage_key], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch['human_label'], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if plot_diffusion_rows: + # get diffusion row + diffusion_row = list() + z_start = z[:n_row] + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(z_start) + z_noisy = self.q_sample(x_start=z_start, t=t, noise=noise) + diffusion_row.append(self.decode_first_stage(z_noisy)) + + diffusion_row = torch.stack( + diffusion_row) # n_log_step, n_row, C, H, W + diffusion_grid = rearrange(diffusion_row, 'n b c h w -> b n c h w') + diffusion_grid = rearrange(diffusion_grid, + 'b n c h w -> (b n) c h w') + diffusion_grid = make_grid( + diffusion_grid, nrow=diffusion_row.shape[0]) + log['diffusion_row'] = diffusion_grid + + if sample: + # get denoise row + with ema_scope('Sampling'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + if plot_denoise_rows: + denoise_grid = self._get_denoise_row_from_list(z_denoise_row) + log['denoise_row'] = denoise_grid + + if quantize_denoised and not isinstance( + self.first_stage_model, AutoencoderKL) and not isinstance( + self.first_stage_model, IdentityFirstStage): + # also display when quantizing x0 while sampling + with ema_scope('Plotting Quantized Denoised'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + quantize_denoised=True) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True, + # quantize_denoised=True) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_x0_quantized'] = x_samples + + if unconditional_guidance_scale > 1.0: + uc = self.get_unconditional_conditioning( + N, unconditional_guidance_label, image_size=x.shape[-1]) + # uc = torch.zeros_like(c) + with ema_scope('Sampling with classifier-free guidance'): + samples_cfg, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f'samples_cfg_scale_{unconditional_guidance_scale:.2f}'] = x_samples_cfg + + if inpaint: + # make a simple center square + h, w = z.shape[2], z.shape[3] + mask = torch.ones(N, h, w).to(self.device) + # zeros will be filled in + mask[:, h // 4:3 * h // 4, w // 4:3 * w // 4] = 0. + mask = mask[:, None, ...] + with ema_scope('Plotting Inpaint'): + + samples, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + eta=ddim_eta, + ddim_steps=ddim_steps, + x0=z[:N], + mask=mask) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_inpainting'] = x_samples + log['mask'] = mask + + # outpaint + mask = 1. - mask + with ema_scope('Plotting Outpaint'): + samples, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + eta=ddim_eta, + ddim_steps=ddim_steps, + x0=z[:N], + mask=mask) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_outpainting'] = x_samples + + if plot_progressive_rows: + with ema_scope('Plotting Progressives'): + img, progressives = self.progressive_denoising( + c, + shape=(self.channels, self.image_size, self.image_size), + batch_size=N) + prog_row = self._get_denoise_row_from_list( + progressives, desc='Progressive Generation') + log['progressive_row'] = prog_row + + if return_keys: + if np.intersect1d(list(log.keys()), return_keys).shape[0] == 0: + return log + else: + return {key: log[key] for key in return_keys} + return log + + def configure_optimizers(self): + lr = self.learning_rate + params = [] + if self.unet_trainable == 'attn': + print('Training only unet attention layers') + for n, m in self.model.named_modules(): + if isinstance(m, CrossAttention) and n.endswith('attn2'): + params.extend(m.parameters()) + if self.unet_trainable == 'conv_in': + print('Training only unet input conv layers') + params = list( + self.model.diffusion_model.input_blocks[0][0].parameters()) + elif self.unet_trainable is True or self.unet_trainable == 'all': + print('Training the full unet') + params = list(self.model.parameters()) + else: + raise ValueError( + f'Unrecognised setting for unet_trainable: {self.unet_trainable}' + ) + + if self.cond_stage_trainable: + print( + f'{self.__class__.__name__}: Also optimizing conditioner params!' + ) + params = params + list(self.cond_stage_model.parameters()) + if self.learn_logvar: + print('Diffusion model optimizing logvar') + params.append(self.logvar) + + if self.cc_projection is not None: + params = params + list(self.cc_projection.parameters()) + print('========== optimizing for cc projection weight ==========') + + param_1 = {'params': self.model.parameters(), 'lr': lr} + param_2 = {'params': self.cc_projection.parameters(), 'lr': 10. * lr} + opt = torch.optim.AdamW([param_1, param_2], lr=lr) + if self.use_scheduler: + assert 'target' in self.scheduler_config + scheduler = instantiate_from_config(self.scheduler_config) + + print('Setting up LambdaLR scheduler...') + scheduler = [{ + 'scheduler': + LambdaLR(opt, lr_lambda=scheduler.schedule), + 'interval': + 'step', + 'frequency': + 1 + }] + return [opt], scheduler + return opt + + @torch.no_grad() + def to_rgb(self, x): + x = x.float() + if not hasattr(self, 'colorize'): + self.colorize = torch.randn(3, x.shape[1], 1, 1).to(x) + x = nn.functional.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class DiffusionWrapper(pl.LightningModule): + + def __init__(self, diff_model_config, conditioning_key): + super().__init__() + self.diffusion_model = instantiate_from_config(diff_model_config) + self.conditioning_key = conditioning_key + assert self.conditioning_key in [ + None, 'concat', 'crossattn', 'hybrid', 'adm', 'hybrid-adm' + ] + + def forward(self, + x, + t, + c_concat: list = None, + c_crossattn: list = None, + c_adm=None): + if self.conditioning_key is None: + out = self.diffusion_model(x, t) + elif self.conditioning_key == 'concat': + xc = torch.cat([x] + c_concat, dim=1) + out = self.diffusion_model(xc, t) + elif self.conditioning_key == 'crossattn': + # c_crossattn dimension: torch.Size([8, 1, 768]) 1 + # cc dimension: torch.Size([8, 1, 768] + cc = torch.cat(c_crossattn, 1) + out = self.diffusion_model(x, t, context=cc) + elif self.conditioning_key == 'hybrid': + xc = torch.cat([x] + c_concat, dim=1) + cc = torch.cat(c_crossattn, 1) + out = self.diffusion_model(xc, t, context=cc) + elif self.conditioning_key == 'hybrid-adm': + assert c_adm is not None + xc = torch.cat([x] + c_concat, dim=1) + cc = torch.cat(c_crossattn, 1) + out = self.diffusion_model(xc, t, context=cc, y=c_adm) + elif self.conditioning_key == 'adm': + cc = c_crossattn[0] + out = self.diffusion_model(x, t, y=cc) + else: + raise NotImplementedError() + + return out + + +class LatentUpscaleDiffusion(LatentDiffusion): + + def __init__(self, *args, low_scale_config, low_scale_key='LR', **kwargs): + super().__init__(*args, **kwargs) + # assumes that neither the cond_stage nor the low_scale_model contain trainable params + assert not self.cond_stage_trainable + self.instantiate_low_stage(low_scale_config) + self.low_scale_key = low_scale_key + + def instantiate_low_stage(self, config): + model = instantiate_from_config(config) + self.low_scale_model = model.eval() + self.low_scale_model.train = disabled_train + for param in self.low_scale_model.parameters(): + param.requires_grad = False + + @torch.no_grad() + def get_input(self, batch, k, cond_key=None, bs=None, log_mode=False): + if not log_mode: + z, c = super().get_input(batch, k, force_c_encode=True, bs=bs) + else: + z, c, x, xrec, xc = super().get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=bs) + x_low = batch[self.low_scale_key][:bs] + x_low = rearrange(x_low, 'b h w c -> b c h w') + x_low = x_low.to(memory_format=torch.contiguous_format).float() + zx, noise_level = self.low_scale_model(x_low) + all_conds = { + 'c_concat': [zx], + 'c_crossattn': [c], + 'c_adm': noise_level + } + if log_mode: + interpretability = False + if interpretability: + zx = zx[:, :, ::2, ::2] + x_low_rec = self.low_scale_model.decode(zx) + return z, all_conds, x, xrec, xc, x_low, x_low_rec, noise_level + return z, all_conds + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=200, + ddim_eta=1., + return_keys=None, + plot_denoise_rows=False, + plot_progressive_rows=True, + plot_diffusion_rows=True, + unconditional_guidance_scale=1., + unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc, x_low, x_low_rec, noise_level = self.get_input( + batch, self.first_stage_key, bs=N, log_mode=True) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + log['x_lr'] = x_low + log[f"x_lr_rec_@noise_levels{'-'.join(map(lambda x: str(x), list(noise_level.cpu().numpy())))}"] = x_low_rec + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption', 'txt']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch[self.cond_stage_key], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch['human_label'], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if plot_diffusion_rows: + # get diffusion row + diffusion_row = list() + z_start = z[:n_row] + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(z_start) + z_noisy = self.q_sample(x_start=z_start, t=t, noise=noise) + diffusion_row.append(self.decode_first_stage(z_noisy)) + + diffusion_row = torch.stack( + diffusion_row) # n_log_step, n_row, C, H, W + diffusion_grid = rearrange(diffusion_row, 'n b c h w -> b n c h w') + diffusion_grid = rearrange(diffusion_grid, + 'b n c h w -> (b n) c h w') + diffusion_grid = make_grid( + diffusion_grid, nrow=diffusion_row.shape[0]) + log['diffusion_row'] = diffusion_grid + + if sample: + # get denoise row + with ema_scope('Sampling'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + if plot_denoise_rows: + denoise_grid = self._get_denoise_row_from_list(z_denoise_row) + log['denoise_row'] = denoise_grid + + if unconditional_guidance_scale > 1.0: + uc_tmp = self.get_unconditional_conditioning( + N, unconditional_guidance_label) + # TODO explore better "unconditional" choices for the other keys + # maybe guide away from empty text label and highest noise level and maximally degraded zx? + uc = dict() + for k in c: + if k == 'c_crossattn': + assert isinstance(c[k], list) and len(c[k]) == 1 + uc[k] = [uc_tmp] + elif k == 'c_adm': # todo: only run with text-based guidance? + assert isinstance(c[k], torch.Tensor) + uc[k] = torch.ones_like( + c[k]) * self.low_scale_model.max_noise_level + elif isinstance(c[k], list): + uc[k] = [c[k][i] for i in range(len(c[k]))] + else: + uc[k] = c[k] + + with ema_scope('Sampling with classifier-free guidance'): + samples_cfg, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f'samples_cfg_scale_{unconditional_guidance_scale:.2f}'] = x_samples_cfg + + if plot_progressive_rows: + with ema_scope('Plotting Progressives'): + img, progressives = self.progressive_denoising( + c, + shape=(self.channels, self.image_size, self.image_size), + batch_size=N) + prog_row = self._get_denoise_row_from_list( + progressives, desc='Progressive Generation') + log['progressive_row'] = prog_row + + return log + + +class LatentInpaintDiffusion(LatentDiffusion): + """ + can either run as pure inpainting model (only concat mode) or with mixed conditionings, + e.g. mask as concat and text via cross-attn. + To disable finetuning mode, set finetune_keys to None + """ + + def __init__( + self, + finetune_keys=('model.diffusion_model.input_blocks.0.0.weight', + 'model_ema.diffusion_modelinput_blocks00weight'), + concat_keys=('mask', 'masked_image'), + masked_image_key='masked_image', + keep_finetune_dims=4, + c_concat_log_start=None, # to log reconstruction of c_concat codes + c_concat_log_end=None, + *args, + **kwargs): + ckpt_path = kwargs.pop('ckpt_path', None) + ignore_keys = kwargs.pop('ignore_keys', list()) + super().__init__(*args, **kwargs) + self.masked_image_key = masked_image_key + assert self.masked_image_key in concat_keys + self.finetune_keys = finetune_keys + self.concat_keys = concat_keys + self.keep_dims = keep_finetune_dims + self.c_concat_log_start = c_concat_log_start + self.c_concat_log_end = c_concat_log_end + if exists(self.finetune_keys): + assert exists( + ckpt_path), 'can only finetune from a given checkpoint' + if exists(ckpt_path): + self.init_from_ckpt(ckpt_path, ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): + sd = torch.load(path, map_location='cpu') + if 'state_dict' in list(sd.keys()): + sd = sd['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + + # make it explicit, finetune by including extra input channels + if exists(self.finetune_keys) and k in self.finetune_keys: + new_entry = None + for name, param in self.named_parameters(): + if name in self.finetune_keys: + print( + f"modifying key '{name}' and keeping its original {self.keep_dims} dimensions only" + ) + new_entry = torch.zeros_like(param) # zero init + assert exists( + new_entry), 'did not find matching parameter to modify' + new_entry[:, :self.keep_dims, ...] = sd[k] + sd[k] = new_entry + + missing, unexpected = self.load_state_dict( + sd, + strict=False) if not only_model else self.model.load_state_dict( + sd, strict=False) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + if len(unexpected) > 0: + print(f'Unexpected Keys: {unexpected}') + + @torch.no_grad() + def get_input(self, + batch, + k, + cond_key=None, + bs=None, + return_first_stage_outputs=False): + # note: restricted to non-trainable encoders currently + assert not self.cond_stage_trainable, 'trainable cond stages not yet supported for inpainting' + z, c, x, xrec, xc = super().get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=bs) + + assert exists(self.concat_keys) + c_cat = list() + for ck in self.concat_keys: + cc = rearrange(batch[ck], 'b h w c -> b c h w').to( + memory_format=torch.contiguous_format).float() + if bs is not None: + cc = cc[:bs] + cc = cc.to(self.device) + bchw = z.shape + if ck != self.masked_image_key: + cc = torch.nn.functional.interpolate(cc, size=bchw[-2:]) + else: + cc = self.get_first_stage_encoding(self.encode_first_stage(cc)) + c_cat.append(cc) + c_cat = torch.cat(c_cat, dim=1) + all_conds = {'c_concat': [c_cat], 'c_crossattn': [c]} + if return_first_stage_outputs: + return z, all_conds, x, xrec, xc + return z, all_conds + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=200, + ddim_eta=1., + return_keys=None, + quantize_denoised=True, + inpaint=True, + plot_denoise_rows=False, + plot_progressive_rows=True, + plot_diffusion_rows=True, + unconditional_guidance_scale=1., + unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc = self.get_input( + batch, self.first_stage_key, bs=N, return_first_stage_outputs=True) + c_cat, c = c['c_concat'][0], c['c_crossattn'][0] + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption', 'txt']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch[self.cond_stage_key], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch['human_label'], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if not (self.c_concat_log_start is None + and self.c_concat_log_end is None): + log['c_concat_decoded'] = self.decode_first_stage( + c_cat[:, self.c_concat_log_start:self.c_concat_log_end]) + + if plot_diffusion_rows: + # get diffusion row + diffusion_row = list() + z_start = z[:n_row] + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(z_start) + z_noisy = self.q_sample(x_start=z_start, t=t, noise=noise) + diffusion_row.append(self.decode_first_stage(z_noisy)) + + diffusion_row = torch.stack( + diffusion_row) # n_log_step, n_row, C, H, W + diffusion_grid = rearrange(diffusion_row, 'n b c h w -> b n c h w') + diffusion_grid = rearrange(diffusion_grid, + 'b n c h w -> (b n) c h w') + diffusion_grid = make_grid( + diffusion_grid, nrow=diffusion_row.shape[0]) + log['diffusion_row'] = diffusion_grid + + if sample: + # get denoise row + with ema_scope('Sampling'): + samples, z_denoise_row = self.sample_log( + cond={ + 'c_concat': [c_cat], + 'c_crossattn': [c] + }, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + if plot_denoise_rows: + denoise_grid = self._get_denoise_row_from_list(z_denoise_row) + log['denoise_row'] = denoise_grid + + if unconditional_guidance_scale > 1.0: + uc_cross = self.get_unconditional_conditioning( + N, unconditional_guidance_label) + uc_cat = c_cat + uc_full = {'c_concat': [uc_cat], 'c_crossattn': [uc_cross]} + with ema_scope('Sampling with classifier-free guidance'): + samples_cfg, _ = self.sample_log( + cond={ + 'c_concat': [c_cat], + 'c_crossattn': [c] + }, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc_full, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f'samples_cfg_scale_{unconditional_guidance_scale:.2f}'] = x_samples_cfg + + log['masked_image'] = rearrange( + batch['masked_image'], 'b h w c -> b c h w').to( + memory_format=torch.contiguous_format).float() + return log + + +class Layout2ImgDiffusion(LatentDiffusion): + # TODO: move all layout-specific hacks to this class + def __init__(self, cond_stage_key, *args, **kwargs): + assert cond_stage_key == 'coordinates_bbox', 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"' + super().__init__(cond_stage_key=cond_stage_key, *args, **kwargs) + + def log_images(self, batch, N=8, *args, **kwargs): + logs = super().log_images(batch=batch, N=N, *args, **kwargs) + + key = 'train' if self.training else 'validation' + dset = self.trainer.datamodule.datasets[key] + mapper = dset.conditional_builders[self.cond_stage_key] + + bbox_imgs = [] + + def map_fn(catno): + return dset.get_textual_label(dset.get_category_id(catno)) + + for tknzd_bbox in batch[self.cond_stage_key][:N]: + bboximg = mapper.plot(tknzd_bbox.detach().cpu(), map_fn, + (256, 256)) + bbox_imgs.append(bboximg) + + cond_img = torch.stack(bbox_imgs, dim=0) + logs['bbox_image'] = cond_img + return logs + + +class SimpleUpscaleDiffusion(LatentDiffusion): + + def __init__(self, *args, low_scale_key='LR', **kwargs): + super().__init__(*args, **kwargs) + # assumes that neither the cond_stage nor the low_scale_model contain trainable params + assert not self.cond_stage_trainable + self.low_scale_key = low_scale_key + + @torch.no_grad() + def get_input(self, batch, k, cond_key=None, bs=None, log_mode=False): + if not log_mode: + z, c = super().get_input(batch, k, force_c_encode=True, bs=bs) + else: + z, c, x, xrec, xc = super().get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=bs) + x_low = batch[self.low_scale_key][:bs] + x_low = rearrange(x_low, 'b h w c -> b c h w') + x_low = x_low.to(memory_format=torch.contiguous_format).float() + + encoder_posterior = self.encode_first_stage(x_low) + zx = self.get_first_stage_encoding(encoder_posterior).detach() + all_conds = {'c_concat': [zx], 'c_crossattn': [c]} + + if log_mode: + # TODO: maybe disable if too expensive + interpretability = False + if interpretability: + zx = zx[:, :, ::2, ::2] + return z, all_conds, x, xrec, xc, x_low + return z, all_conds + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=200, + ddim_eta=1., + return_keys=None, + plot_denoise_rows=False, + plot_progressive_rows=True, + plot_diffusion_rows=True, + unconditional_guidance_scale=1., + unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc, x_low = self.get_input( + batch, self.first_stage_key, bs=N, log_mode=True) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + log['x_lr'] = x_low + + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption', 'txt']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch[self.cond_stage_key], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch['human_label'], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if sample: + # get denoise row + with ema_scope('Sampling'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + + if unconditional_guidance_scale > 1.0: + uc_tmp = self.get_unconditional_conditioning( + N, unconditional_guidance_label) + uc = dict() + for k in c: + if k == 'c_crossattn': + assert isinstance(c[k], list) and len(c[k]) == 1 + uc[k] = [uc_tmp] + elif isinstance(c[k], list): + uc[k] = [c[k][i] for i in range(len(c[k]))] + else: + uc[k] = c[k] + + with ema_scope('Sampling with classifier-free guidance'): + samples_cfg, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f'samples_cfg_scale_{unconditional_guidance_scale:.2f}'] = x_samples_cfg + return log + + +class MultiCatFrameDiffusion(LatentDiffusion): + + def __init__(self, *args, low_scale_key='LR', **kwargs): + super().__init__(*args, **kwargs) + # assumes that neither the cond_stage nor the low_scale_model contain trainable params + assert not self.cond_stage_trainable + self.low_scale_key = low_scale_key + + @torch.no_grad() + def get_input(self, batch, k, cond_key=None, bs=None, log_mode=False): + n = 2 + if not log_mode: + z, c = super().get_input(batch, k, force_c_encode=True, bs=bs) + else: + z, c, x, xrec, xc = super().get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=bs) + cat_conds = batch[self.low_scale_key][:bs] + cats = [] + for i in range(n): + x_low = cat_conds[:, :, :, 3 * i:3 * (i + 1)] + x_low = rearrange(x_low, 'b h w c -> b c h w') + x_low = x_low.to(memory_format=torch.contiguous_format).float() + encoder_posterior = self.encode_first_stage(x_low) + zx = self.get_first_stage_encoding(encoder_posterior).detach() + cats.append(zx) + + all_conds = {'c_concat': [torch.cat(cats, dim=1)], 'c_crossattn': [c]} + + if log_mode: + # TODO: maybe disable if too expensive + interpretability = False + if interpretability: + zx = zx[:, :, ::2, ::2] + return z, all_conds, x, xrec, xc, x_low + return z, all_conds + + @torch.no_grad() + def log_images(self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=200, + ddim_eta=1., + return_keys=None, + plot_denoise_rows=False, + plot_progressive_rows=True, + plot_diffusion_rows=True, + unconditional_guidance_scale=1., + unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc, x_low = self.get_input( + batch, self.first_stage_key, bs=N, log_mode=True) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + log['x_lr'] = x_low + + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption', 'txt']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch[self.cond_stage_key], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), + batch['human_label'], + size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if sample: + # get denoise row + with ema_scope('Sampling'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + + if unconditional_guidance_scale > 1.0: + uc_tmp = self.get_unconditional_conditioning( + N, unconditional_guidance_label) + uc = dict() + for k in c: + if k == 'c_crossattn': + assert isinstance(c[k], list) and len(c[k]) == 1 + uc[k] = [uc_tmp] + elif isinstance(c[k], list): + uc[k] = [c[k][i] for i in range(len(c[k]))] + else: + uc[k] = c[k] + + with ema_scope('Sampling with classifier-free guidance'): + samples_cfg, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f'samples_cfg_scale_{unconditional_guidance_scale:.2f}'] = x_samples_cfg + return log diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/distributions.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/distributions.py new file mode 100644 index 0000000000000000000000000000000000000000..cca3ff12e43b04759e5a0745c3c849a782209dc3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/distributions.py @@ -0,0 +1,92 @@ +import numpy as np +import torch + + +class AbstractDistribution: + + def sample(self): + raise NotImplementedError() + + def mode(self): + raise NotImplementedError() + + +class DiracDistribution(AbstractDistribution): + + def __init__(self, value): + self.value = value + + def sample(self): + return self.value + + def mode(self): + return self.value + + +class DiagonalGaussianDistribution(object): + + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean).to(device=self.parameters.device) + + def sample(self): + x = self.mean + self.std * torch.randn( + self.mean.shape).to(device=self.parameters.device) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3]) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var - 1.0 - self.logvar + other.logvar, + dim=[1, 2, 3]) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + + torch.pow(sample - self.mean, 2) / self.var, + dim=dims) + + def mode(self): + return self.mean + + +def normal_kl(mean1, logvar1, mean2, logvar2): + """ + Compute the KL divergence between two gaussians. + Shapes are automatically broadcasted, so batches can be compared to + scalars, among other use cases. + """ + tensor = None + for obj in (mean1, logvar1, mean2, logvar2): + if isinstance(obj, torch.Tensor): + tensor = obj + break + assert tensor is not None, 'at least one argument must be a Tensor' + + # Force variances to be Tensors. Broadcasting helps convert scalars to + # Tensors, but it does not work for torch.exp(). + logvar1, logvar2 = [ + x if isinstance(x, torch.Tensor) else torch.tensor(x).to(tensor) + for x in (logvar1, logvar2) + ] + comp_1 = -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + comp_2 = ((mean1 - mean2)**2) * torch.exp(-logvar2) + return 0.5 * (comp_1 + comp_2) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ema.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..01810e9cfb8de901ea876cfbaf5e55cf7058ce0f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/ema.py @@ -0,0 +1,84 @@ +import torch +from torch import nn + + +class LitEma(nn.Module): + + def __init__(self, model, decay=0.9999, use_num_upates=True): + super().__init__() + if decay < 0.0 or decay > 1.0: + raise ValueError('Decay must be between 0 and 1') + + self.m_name2s_name = {} + self.register_buffer('decay', torch.tensor(decay, dtype=torch.float32)) + self.register_buffer( + 'num_updates', + torch.tensor(0, dtype=torch.int) + if use_num_upates else torch.tensor(-1, dtype=torch.int)) + + for name, p in model.named_parameters(): + if p.requires_grad: + + s_name = name.replace('.', '') + self.m_name2s_name.update({name: s_name}) + self.register_buffer(s_name, p.clone().detach().data) + + self.collected_params = [] + + def forward(self, model): + decay = self.decay + + if self.num_updates >= 0: + self.num_updates += 1 + num_1 = (1 + self.num_updates) + num_2 = (10 + self.num_updates) + decay = min(self.decay, num_1 / num_2) + + one_minus_decay = 1.0 - decay + + with torch.no_grad(): + m_param = dict(model.named_parameters()) + shadow_params = dict(self.named_buffers()) + + for key in m_param: + if m_param[key].requires_grad: + sname = self.m_name2s_name[key] + shadow_params[sname] = shadow_params[sname].type_as( + m_param[key]) + param_1 = (shadow_params[sname] - m_param[key]) + shadow_params[sname].sub_(one_minus_decay * param_1) + else: + assert key not in self.m_name2s_name + + def copy_to(self, model): + m_param = dict(model.named_parameters()) + shadow_params = dict(self.named_buffers()) + for key in m_param: + if m_param[key].requires_grad: + m_param[key].data.copy_( + shadow_params[self.m_name2s_name[key]].data) + else: + assert key not in self.m_name2s_name + + def store(self, parameters): + """ + Save the current parameters for restoring later. + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + temporarily stored. + """ + self.collected_params = [param.clone() for param in parameters] + + def restore(self, parameters): + """ + Restore the parameters stored with the `store` method. + Useful to validate the model with EMA parameters without affecting the + original optimization process. Store the parameters before the + `copy_to` method. After validation (or model saving), use this to + restore the former parameters. + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored parameters. + """ + for c_param, param in zip(self.collected_params, parameters): + param.data.copy_(c_param.data) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/helpers.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..d1622d59c4f9599ddde7693a57dbd2bcf375cfdf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/helpers.py @@ -0,0 +1,131 @@ +# https://github.com/eladrich/pixel2style2pixel + +from collections import namedtuple + +import torch +from torch.nn import (AdaptiveAvgPool2d, BatchNorm2d, Conv2d, MaxPool2d, + Module, PReLU, ReLU, Sequential, Sigmoid) + + +class Flatten(Module): + + def forward(self, input): + return input.view(input.size(0), -1) + + +def l2_norm(input, axis=1): + norm = torch.norm(input, 2, axis, True) + output = torch.div(input, norm) + return output + + +class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): + pass + + +def get_block(in_channel, depth, num_units, stride=2): + return [Bottleneck(in_channel, depth, stride) + ] + [Bottleneck(depth, depth, 1) for i in range(num_units - 1)] + + +def get_blocks(num_layers): + if num_layers == 50: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=4), + get_block(in_channel=128, depth=256, num_units=14), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 100: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=13), + get_block(in_channel=128, depth=256, num_units=30), + get_block(in_channel=256, depth=512, num_units=3) + ] + elif num_layers == 152: + blocks = [ + get_block(in_channel=64, depth=64, num_units=3), + get_block(in_channel=64, depth=128, num_units=8), + get_block(in_channel=128, depth=256, num_units=36), + get_block(in_channel=256, depth=512, num_units=3) + ] + else: + raise ValueError( + 'Invalid number of layers: {}. Must be one of [50, 100, 152]'. + format(num_layers)) + return blocks + + +class SEModule(Module): + + def __init__(self, channels, reduction): + super(SEModule, self).__init__() + self.avg_pool = AdaptiveAvgPool2d(1) + self.fc1 = Conv2d( + channels, + channels // reduction, + kernel_size=1, + padding=0, + bias=False) + self.relu = ReLU(inplace=True) + self.fc2 = Conv2d( + channels // reduction, + channels, + kernel_size=1, + padding=0, + bias=False) + self.sigmoid = Sigmoid() + + def forward(self, x): + module_input = x + x = self.avg_pool(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.sigmoid(x) + return module_input * x + + +class bottleneck_IR(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut + + +class bottleneck_IR_SE(Module): + + def __init__(self, in_channel, depth, stride): + super(bottleneck_IR_SE, self).__init__() + if in_channel == depth: + self.shortcut_layer = MaxPool2d(1, stride) + else: + self.shortcut_layer = Sequential( + Conv2d(in_channel, depth, (1, 1), stride, bias=False), + BatchNorm2d(depth)) + self.res_layer = Sequential( + BatchNorm2d(in_channel), + Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), + PReLU(depth), Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + BatchNorm2d(depth), SEModule(depth, 16)) + + def forward(self, x): + shortcut = self.shortcut_layer(x) + res = self.res_layer(x) + return res + shortcut diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/id_loss.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/id_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..6371096805072875973055cf318499f0a0853ec7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/id_loss.py @@ -0,0 +1,27 @@ +# https://github.com/eladrich/pixel2style2pixel +import torch +from torch import nn + +from .model_irse import Backbone + + +class IDFeatures(nn.Module): + + def __init__(self, model_path): + super(IDFeatures, self).__init__() + print('Loading ResNet ArcFace') + self.facenet = Backbone( + input_size=112, num_layers=50, drop_ratio=0.6, mode='ir_se') + self.facenet.load_state_dict( + torch.load(model_path, map_location='cpu')) + self.face_pool = torch.nn.AdaptiveAvgPool2d((112, 112)) + self.facenet.eval() + + def forward(self, x, crop=False): + # Not sure of the image range here + if crop: + x = torch.nn.functional.interpolate(x, (256, 256), mode='area') + x = x[:, :, 35:223, 32:220] + x = self.face_pool(x) + x_feats = self.facenet(x) + return x_feats diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model.py new file mode 100644 index 0000000000000000000000000000000000000000..9063c31bdbf368c7cd9c95093d7ca84883a0775f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model.py @@ -0,0 +1,961 @@ +# pytorch_diffusion + derived encoder decoder +import math + +import numpy as np +import torch +import torch.nn as nn +from einops import rearrange + +from ..util import instantiate_from_config +from .attention import LinearAttention + + +def get_timestep_embedding(timesteps, embedding_dim): + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: + From Fairseq. + Build sinusoidal embeddings. + This matches the implementation in tensor2tensor, but differs slightly + from the description in Section 3.5 of "Attention Is All You Need". + """ + assert len(timesteps.shape) == 1 + + half_dim = embedding_dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb) + emb = emb.to(device=timesteps.device) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm( + num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +class Upsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x = torch.nn.functional.interpolate( + x, scale_factor=2.0, mode='nearest') + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResnetBlock(nn.Module): + + def __init__(self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class LinAttnBlock(LinearAttention): + """to match AttnBlock usage""" + + def __init__(self, in_channels): + super().__init__(dim=in_channels, heads=1, dim_head=in_channels) + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm( + v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +def make_attn(in_channels, attn_type='vanilla'): + assert attn_type in ['vanilla', 'linear', + 'none'], f'attn_type {attn_type} unknown' + print( + f"making attention of type '{attn_type}' with {in_channels} in_channels" + ) + if attn_type == 'vanilla': + return AttnBlock(in_channels) + elif attn_type == 'none': + return nn.Identity(in_channels) + else: + return LinAttnBlock(in_channels) + + +class Model(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + use_timestep=True, + use_linear_attn=False, + attn_type='vanilla'): + super().__init__() + if use_linear_attn: + attn_type = 'linear' + self.ch = ch + self.temb_ch = self.ch * 4 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + self.use_timestep = use_timestep + if self.use_timestep: + # timestep embedding + self.temb = nn.Module() + self.temb.dense = nn.ModuleList([ + torch.nn.Linear(self.ch, self.temb_ch), + torch.nn.Linear(self.temb_ch, self.temb_ch), + ]) + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + skip_in = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + if i_block == self.num_res_blocks: + skip_in = ch * in_ch_mult[i_level] + block.append( + ResnetBlock( + in_channels=block_in + skip_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, x, t=None, context=None): + if context is not None: + # assume aligned context, cat along channel axis + x = torch.cat((x, context), dim=1) + if self.use_timestep: + # timestep embedding + assert t is not None + temb = get_timestep_embedding(t, self.ch) + temb = self.temb.dense[0](temb) + temb = nonlinearity(temb) + temb = self.temb.dense[1](temb) + else: + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](torch.cat([h, hs.pop()], + dim=1), temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + def get_last_layer(self): + return self.conv_out.weight + + +class Encoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + use_linear_attn=False, + attn_type='vanilla', + **ignore_kwargs): + super().__init__() + if use_linear_attn: + attn_type = 'linear' + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type='vanilla', + **ignorekwargs): + super().__init__() + if use_linear_attn: + attn_type = 'linear' + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute in_ch_mult, block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2**(self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + print('Working with z of shape {} = {} dimensions.'.format( + self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class SimpleDecoder(nn.Module): + + def __init__(self, in_channels, out_channels, *args, **kwargs): + super().__init__() + self.model = nn.ModuleList([ + nn.Conv2d(in_channels, in_channels, 1), + ResnetBlock( + in_channels=in_channels, + out_channels=2 * in_channels, + temb_channels=0, + dropout=0.0), + ResnetBlock( + in_channels=2 * in_channels, + out_channels=4 * in_channels, + temb_channels=0, + dropout=0.0), + ResnetBlock( + in_channels=4 * in_channels, + out_channels=2 * in_channels, + temb_channels=0, + dropout=0.0), + nn.Conv2d(2 * in_channels, in_channels, 1), + Upsample(in_channels, with_conv=True) + ]) + # end + self.norm_out = Normalize(in_channels) + self.conv_out = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + for i, layer in enumerate(self.model): + if i in [1, 2, 3]: + x = layer(x, None) + else: + x = layer(x) + + h = self.norm_out(x) + h = nonlinearity(h) + x = self.conv_out(h) + return x + + +class UpsampleDecoder(nn.Module): + + def __init__(self, + in_channels, + out_channels, + ch, + num_res_blocks, + resolution, + ch_mult=(2, 2), + dropout=0.0): + super().__init__() + # upsampling + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + block_in = in_channels + curr_res = resolution // 2**(self.num_resolutions - 1) + self.res_blocks = nn.ModuleList() + self.upsample_blocks = nn.ModuleList() + for i_level in range(self.num_resolutions): + res_block = [] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + res_block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + self.res_blocks.append(nn.ModuleList(res_block)) + if i_level != self.num_resolutions - 1: + self.upsample_blocks.append(Upsample(block_in, True)) + curr_res = curr_res * 2 + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + # upsampling + h = x + for k, i_level in enumerate(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.res_blocks[i_level][i_block](h, None) + if i_level != self.num_resolutions - 1: + h = self.upsample_blocks[k](h) + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class LatentRescaler(nn.Module): + + def __init__(self, + factor, + in_channels, + mid_channels, + out_channels, + depth=2): + super().__init__() + # residual block, interpolate, residual block + self.factor = factor + self.conv_in = nn.Conv2d( + in_channels, mid_channels, kernel_size=3, stride=1, padding=1) + self.res_block1 = nn.ModuleList([ + ResnetBlock( + in_channels=mid_channels, + out_channels=mid_channels, + temb_channels=0, + dropout=0.0) for _ in range(depth) + ]) + self.attn = AttnBlock(mid_channels) + self.res_block2 = nn.ModuleList([ + ResnetBlock( + in_channels=mid_channels, + out_channels=mid_channels, + temb_channels=0, + dropout=0.0) for _ in range(depth) + ]) + + self.conv_out = nn.Conv2d( + mid_channels, + out_channels, + kernel_size=1, + ) + + def forward(self, x): + x = self.conv_in(x) + for block in self.res_block1: + x = block(x, None) + x = torch.nn.functional.interpolate( + x, + size=(int(round(x.shape[2] * self.factor)), + int(round(x.shape[3] * self.factor)))) + x = self.attn(x) + for block in self.res_block2: + x = block(x, None) + x = self.conv_out(x) + return x + + +class MergedRescaleEncoder(nn.Module): + + def __init__(self, + in_channels, + ch, + resolution, + out_ch, + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + ch_mult=(1, 2, 4, 8), + rescale_factor=1.0, + rescale_module_depth=1): + super().__init__() + intermediate_chn = ch * ch_mult[-1] + self.encoder = Encoder( + in_channels=in_channels, + num_res_blocks=num_res_blocks, + ch=ch, + ch_mult=ch_mult, + z_channels=intermediate_chn, + double_z=False, + resolution=resolution, + attn_resolutions=attn_resolutions, + dropout=dropout, + resamp_with_conv=resamp_with_conv, + out_ch=None) + self.rescaler = LatentRescaler( + factor=rescale_factor, + in_channels=intermediate_chn, + mid_channels=intermediate_chn, + out_channels=out_ch, + depth=rescale_module_depth) + + def forward(self, x): + x = self.encoder(x) + x = self.rescaler(x) + return x + + +class MergedRescaleDecoder(nn.Module): + + def __init__(self, + z_channels, + out_ch, + resolution, + num_res_blocks, + attn_resolutions, + ch, + ch_mult=(1, 2, 4, 8), + dropout=0.0, + resamp_with_conv=True, + rescale_factor=1.0, + rescale_module_depth=1): + super().__init__() + tmp_chn = z_channels * ch_mult[-1] + self.decoder = Decoder( + out_ch=out_ch, + z_channels=tmp_chn, + attn_resolutions=attn_resolutions, + dropout=dropout, + resamp_with_conv=resamp_with_conv, + in_channels=None, + num_res_blocks=num_res_blocks, + ch_mult=ch_mult, + resolution=resolution, + ch=ch) + self.rescaler = LatentRescaler( + factor=rescale_factor, + in_channels=z_channels, + mid_channels=tmp_chn, + out_channels=tmp_chn, + depth=rescale_module_depth) + + def forward(self, x): + x = self.rescaler(x) + x = self.decoder(x) + return x + + +class Upsampler(nn.Module): + + def __init__(self, + in_size, + out_size, + in_channels, + out_channels, + ch_mult=2): + super().__init__() + assert out_size >= in_size + num_blocks = int(np.log2(out_size // in_size)) + 1 + factor_up = 1. + (out_size % in_size) + print( + f'Building {self.__class__.__name__} with in_size: {in_size} --> out_size {out_size} and factor {factor_up}' + ) + self.rescaler = LatentRescaler( + factor=factor_up, + in_channels=in_channels, + mid_channels=2 * in_channels, + out_channels=in_channels) + self.decoder = Decoder( + out_ch=out_channels, + resolution=out_size, + z_channels=in_channels, + num_res_blocks=2, + attn_resolutions=[], + in_channels=None, + ch=in_channels, + ch_mult=[ch_mult for _ in range(num_blocks)]) + + def forward(self, x): + x = self.rescaler(x) + x = self.decoder(x) + return x + + +class Resize(nn.Module): + + def __init__(self, in_channels=None, learned=False, mode='bilinear'): + super().__init__() + self.with_conv = learned + self.mode = mode + if self.with_conv: + print( + f'Note: {self.__class__.__name} uses learned downsampling and will ignore the fixed {mode} mode' + ) + raise NotImplementedError() + assert in_channels is not None + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=4, stride=2, padding=1) + + def forward(self, x, scale_factor=1.0): + if scale_factor == 1.0: + return x + else: + x = torch.nn.functional.interpolate( + x, + mode=self.mode, + align_corners=False, + scale_factor=scale_factor) + return x + + +class FirstStagePostProcessor(nn.Module): + + def __init__(self, + ch_mult: list, + in_channels, + pretrained_model: nn.Module = None, + reshape=False, + n_channels=None, + dropout=0., + pretrained_config=None): + super().__init__() + if pretrained_config is None: + assert pretrained_model is not None, 'Either "pretrained_model" or "pretrained_config" must not be None' + self.pretrained_model = pretrained_model + else: + assert pretrained_config is not None, 'Either "pretrained_model" or "pretrained_config" must not be None' + self.instantiate_pretrained(pretrained_config) + + self.do_reshape = reshape + + if n_channels is None: + n_channels = self.pretrained_model.encoder.ch + + self.proj_norm = Normalize(in_channels, num_groups=in_channels // 2) + self.proj = nn.Conv2d( + in_channels, n_channels, kernel_size=3, stride=1, padding=1) + + blocks = [] + downs = [] + ch_in = n_channels + for m in ch_mult: + blocks.append( + ResnetBlock( + in_channels=ch_in, + out_channels=m * n_channels, + dropout=dropout)) + ch_in = m * n_channels + downs.append(Downsample(ch_in, with_conv=False)) + + self.model = nn.ModuleList(blocks) + self.downsampler = nn.ModuleList(downs) + + def instantiate_pretrained(self, config): + model = instantiate_from_config(config) + self.pretrained_model = model.eval() + # self.pretrained_model.train = False + for param in self.pretrained_model.parameters(): + param.requires_grad = False + + @torch.no_grad() + def encode_with_pretrained(self, x): + c = self.pretrained_model.encode(x) + if isinstance(c, DiagonalGaussianDistribution): + c = c.mode() + return c + + def forward(self, x): + z_fs = self.encode_with_pretrained(x) + z = self.proj_norm(z_fs) + z = self.proj(z) + z = nonlinearity(z) + + for submodel, downmodel in zip(self.model, self.downsampler): + z = submodel(z, temb=None) + z = downmodel(z) + + if self.do_reshape: + z = rearrange(z, 'b c h w -> b (h w) c') + return z diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model_irse.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model_irse.py new file mode 100644 index 0000000000000000000000000000000000000000..3b87d7fdf28d06f5a577bdeea294efd03b637a2c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/model_irse.py @@ -0,0 +1,92 @@ +from torch.nn import (BatchNorm1d, BatchNorm2d, Conv2d, Dropout, Linear, + Module, PReLU, Sequential) + +from .helpers import (Flatten, bottleneck_IR, bottleneck_IR_SE, get_blocks, + l2_norm) + + +class Backbone(Module): + + def __init__(self, + input_size, + num_layers, + mode='ir', + drop_ratio=0.4, + affine=True): + super(Backbone, self).__init__() + assert input_size in [112, 224], 'input_size should be 112 or 224' + assert num_layers in [50, 100, + 152], 'num_layers should be 50, 100 or 152' + assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' + blocks = get_blocks(num_layers) + if mode == 'ir': + unit_module = bottleneck_IR + elif mode == 'ir_se': + unit_module = bottleneck_IR_SE + self.input_layer = Sequential( + Conv2d(3, 64, (3, 3), 1, 1, bias=False), BatchNorm2d(64), + PReLU(64)) + if input_size == 112: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(drop_ratio), Flatten(), + Linear(512 * 7 * 7, 512), BatchNorm1d(512, affine=affine)) + else: + self.output_layer = Sequential( + BatchNorm2d(512), Dropout(drop_ratio), Flatten(), + Linear(512 * 14 * 14, 512), BatchNorm1d(512, affine=affine)) + + modules = [] + for block in blocks: + for bottleneck in block: + modules.append( + unit_module(bottleneck.in_channel, bottleneck.depth, + bottleneck.stride)) + self.body = Sequential(*modules) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return l2_norm(x) + + +def IR_50(input_size): + """Constructs a ir-50 model.""" + model = Backbone( + input_size, num_layers=50, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_101(input_size): + """Constructs a ir-101 model.""" + model = Backbone( + input_size, num_layers=100, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_152(input_size): + """Constructs a ir-152 model.""" + model = Backbone( + input_size, num_layers=152, mode='ir', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_50(input_size): + """Constructs a ir_se-50 model.""" + model = Backbone( + input_size, num_layers=50, mode='ir_se', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_101(input_size): + """Constructs a ir_se-101 model.""" + model = Backbone( + input_size, num_layers=100, mode='ir_se', drop_ratio=0.4, affine=False) + return model + + +def IR_SE_152(input_size): + """Constructs a ir_se-152 model.""" + model = Backbone( + input_size, num_layers=152, mode='ir_se', drop_ratio=0.4, affine=False) + return model diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/modules.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..5727f91b4d6e0de1caf860e9766c1013806d12d0 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/modules.py @@ -0,0 +1,668 @@ +import random +from functools import partial + +import clip +import kornia +import kornia.augmentation as K +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import transforms +from transformers import (CLIPTextModel, CLIPTokenizer, CLIPVisionModel, + T5EncoderModel, T5Tokenizer) + +from ..util import default, instantiate_from_config +from .id_loss import IDFeatures +from .util_diffusion import extract_into_tensor, make_beta_schedule, noise_like +from .x_transformer import Encoder, TransformerWrapper + + +class AbstractEncoder(nn.Module): + + def __init__(self): + super().__init__() + + def encode(self, *args, **kwargs): + raise NotImplementedError + + +class IdentityEncoder(AbstractEncoder): + + def encode(self, x): + return x + + +class FaceClipEncoder(AbstractEncoder): + + def __init__(self, augment=True, retreival_key=None): + super().__init__() + self.encoder = FrozenCLIPImageEmbedder() + self.augment = augment + self.retreival_key = retreival_key + + def forward(self, img): + encodings = [] + with torch.no_grad(): + x_offset = 125 + if self.retreival_key: + # Assumes retrieved image are packed into the second half of channels + face = img[:, 3:, 190:440, x_offset:(512 - x_offset)] + other = img[:, :3, ...].clone() + else: + face = img[:, :, 190:440, x_offset:(512 - x_offset)] + other = img.clone() + + if self.augment: + face = K.RandomHorizontalFlip()(face) + + other[:, :, 190:440, x_offset:(512 - x_offset)] *= 0 + encodings = [ + self.encoder.encode(face), + self.encoder.encode(other), + ] + + return torch.cat(encodings, dim=1) + + def encode(self, img): + if isinstance(img, list): + # Uncondition + return torch.zeros( + (1, 2, 768), + device=self.encoder.model.visual.conv1.weight.device) + + return self(img) + + +class FaceIdClipEncoder(AbstractEncoder): + + def __init__(self): + super().__init__() + self.encoder = FrozenCLIPImageEmbedder() + for p in self.encoder.parameters(): + p.requires_grad = False + self.id = FrozenFaceEncoder( + '/home/jpinkney/code/stable-diffusion/model_ir_se50.pth', + augment=True) + + def forward(self, img): + encodings = [] + with torch.no_grad(): + face = kornia.geometry.resize( + img, (256, 256), interpolation='bilinear', align_corners=True) + + other = img.clone() + other[:, :, 184:452, 122:396] *= 0 + encodings = [ + self.id.encode(face), + self.encoder.encode(other), + ] + + return torch.cat(encodings, dim=1) + + def encode(self, img): + if isinstance(img, list): + # Uncondition + return torch.zeros( + (1, 2, 768), + device=self.encoder.model.visual.conv1.weight.device) + + return self(img) + + +class ClassEmbedder(nn.Module): + + def __init__(self, embed_dim, n_classes=1000, key='class'): + super().__init__() + self.key = key + self.embedding = nn.Embedding(n_classes, embed_dim) + + def forward(self, batch, key=None): + if key is None: + key = self.key + # this is for use in crossattn + c = batch[key][:, None] + c = self.embedding(c) + return c + + +class TransformerEmbedder(AbstractEncoder): + """Some transformer encoder layers""" + + def __init__(self, + n_embed, + n_layer, + vocab_size, + max_seq_len=77, + device='cuda'): + super().__init__() + self.device = device + self.transformer = TransformerWrapper( + num_tokens=vocab_size, + max_seq_len=max_seq_len, + attn_layers=Encoder(dim=n_embed, depth=n_layer)) + + def forward(self, tokens): + tokens = tokens.to(self.device) # meh + z = self.transformer(tokens, return_embeddings=True) + return z + + def encode(self, x): + return self(x) + + +class BERTTokenizer(AbstractEncoder): + """ Uses a pretrained BERT tokenizer by huggingface. Vocab size: 30522 (?)""" + + def __init__(self, device='cuda', vq_interface=True, max_length=77): + super().__init__() + from transformers import BertTokenizerFast # TODO: add to reuquirements + self.tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased') + self.device = device + self.vq_interface = vq_interface + self.max_length = max_length + + def forward(self, text): + batch_encoding = self.tokenizer( + text, + truncation=True, + max_length=self.max_length, + return_length=True, + return_overflowing_tokens=False, + padding='max_length', + return_tensors='pt') + tokens = batch_encoding['input_ids'].to(self.device) + return tokens + + @torch.no_grad() + def encode(self, text): + tokens = self(text) + if not self.vq_interface: + return tokens + return None, None, [None, None, tokens] + + def decode(self, text): + return text + + +class BERTEmbedder(AbstractEncoder): + """Uses the BERT tokenizr model and add some transformer encoder layers""" + + def __init__(self, + n_embed, + n_layer, + vocab_size=30522, + max_seq_len=77, + device='cuda', + use_tokenizer=True, + embedding_dropout=0.0): + super().__init__() + self.use_tknz_fn = use_tokenizer + if self.use_tknz_fn: + self.tknz_fn = BERTTokenizer( + vq_interface=False, max_length=max_seq_len) + self.device = device + self.transformer = TransformerWrapper( + num_tokens=vocab_size, + max_seq_len=max_seq_len, + attn_layers=Encoder(dim=n_embed, depth=n_layer), + emb_dropout=embedding_dropout) + + def forward(self, text): + if self.use_tknz_fn: + tokens = self.tknz_fn(text) + else: + tokens = text + z = self.transformer(tokens, return_embeddings=True) + return z + + def encode(self, text): + # output of length 77 + return self(text) + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +class FrozenT5Embedder(AbstractEncoder): + """Uses the T5 transformer encoder for text""" + + def __init__(self, + version='google/t5-v1_1-large', + device='cuda', + max_length=77 + ): # others are google/t5-v1_1-xl and google/t5-v1_1-xxl + super().__init__() + self.tokenizer = T5Tokenizer.from_pretrained(version) + self.transformer = T5EncoderModel.from_pretrained(version) + self.device = device + self.max_length = max_length # TODO: typical value? + self.freeze() + + def freeze(self): + self.transformer = self.transformer.eval() + + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + batch_encoding = self.tokenizer( + text, + truncation=True, + max_length=self.max_length, + return_length=True, + return_overflowing_tokens=False, + padding='max_length', + return_tensors='pt') + tokens = batch_encoding['input_ids'].to(self.device) + outputs = self.transformer(input_ids=tokens) + + z = outputs.last_hidden_state + return z + + def encode(self, text): + return self(text) + + +class FrozenFaceEncoder(AbstractEncoder): + + def __init__(self, model_path, augment=False): + super().__init__() + self.loss_fn = IDFeatures(model_path) + # face encoder is frozen + for p in self.loss_fn.parameters(): + p.requires_grad = False + # Mapper is trainable + self.mapper = torch.nn.Linear(512, 768) + p = 0.25 + if augment: + self.augment = K.AugmentationSequential( + K.RandomHorizontalFlip(p=0.5), + K.RandomEqualize(p=p), + # K.RandomPlanckianJitter(p=p), + # K.RandomPlasmaBrightness(p=p), + # K.RandomPlasmaContrast(p=p), + # K.ColorJiggle(0.02, 0.2, 0.2, p=p), + ) + else: + self.augment = False + + def forward(self, img): + if isinstance(img, list): + # Uncondition + return torch.zeros((1, 1, 768), device=self.mapper.weight.device) + + if self.augment is not None: + # Transforms require 0-1 + img = self.augment((img + 1) / 2) + img = 2 * img - 1 + + feat = self.loss_fn(img, crop=True) + feat = self.mapper(feat.unsqueeze(1)) + return feat + + def encode(self, img): + return self(img) + + +class FrozenCLIPEmbedder(AbstractEncoder): + """Uses the CLIP transformer encoder for text (from huggingface)""" + + def __init__(self, + version='openai/clip-vit-large-patch14', + device='cuda', + max_length=77): # clip-vit-base-patch32 + super().__init__() + self.tokenizer = CLIPTokenizer.from_pretrained(version) + self.transformer = CLIPTextModel.from_pretrained(version) + self.device = device + self.max_length = max_length # TODO: typical value? + self.freeze() + + def freeze(self): + self.transformer = self.transformer.eval() + + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + batch_encoding = self.tokenizer( + text, + truncation=True, + max_length=self.max_length, + return_length=True, + return_overflowing_tokens=False, + padding='max_length', + return_tensors='pt') + tokens = batch_encoding['input_ids'].to(self.device) + outputs = self.transformer(input_ids=tokens) + + z = outputs.last_hidden_state + return z + + def encode(self, text): + return self(text) + + +class ClipImageProjector(AbstractEncoder): + """ + Uses the CLIP image encoder. + """ + + def __init__(self, + version='openai/clip-vit-large-patch14', + max_length=77): # clip-vit-base-patch32 + super().__init__() + self.model = CLIPVisionModel.from_pretrained(version) + self.model.train() + self.max_length = max_length # TODO: typical value? + self.antialias = True + self.mapper = torch.nn.Linear(1024, 768) + self.register_buffer( + 'mean', + torch.Tensor([0.48145466, 0.4578275, 0.40821073]), + persistent=False) + self.register_buffer( + 'std', + torch.Tensor([0.26862954, 0.26130258, 0.27577711]), + persistent=False) + null_cond = self.get_null_cond(version, max_length) + self.register_buffer('null_cond', null_cond) + + @torch.no_grad() + def get_null_cond(self, version, max_length): + device = self.mean.device + embedder = FrozenCLIPEmbedder( + version=version, device=device, max_length=max_length) + null_cond = embedder(['']) + return null_cond + + def preprocess(self, x): + # Expects inputs in the range -1, 1 + x = kornia.geometry.resize( + x, (224, 224), + interpolation='bicubic', + align_corners=True, + antialias=self.antialias) + x = (x + 1.) / 2. + # renormalize according to clip + x = kornia.enhance.normalize(x, self.mean, self.std) + return x + + def forward(self, x): + if isinstance(x, list): + return self.null_cond + # x is assumed to be in range [-1,1] + x = self.preprocess(x) + outputs = self.model(pixel_values=x) + last_hidden_state = outputs.last_hidden_state + last_hidden_state = self.mapper(last_hidden_state) + return F.pad( + last_hidden_state, + [0, 0, 0, self.max_length - last_hidden_state.shape[1], 0, 0]) + + def encode(self, im): + return self(im) + + +class ProjectedFrozenCLIPEmbedder(AbstractEncoder): + + def __init__(self, + version='openai/clip-vit-large-patch14', + device='cuda', + max_length=77): # clip-vit-base-patch32 + super().__init__() + self.embedder = FrozenCLIPEmbedder( + version=version, device=device, max_length=max_length) + self.projection = torch.nn.Linear(768, 768) + + def forward(self, text): + z = self.embedder(text) + return self.projection(z) + + def encode(self, text): + return self(text) + + +class FrozenCLIPImageEmbedder(AbstractEncoder): + """ + Uses the CLIP image encoder. + Not actually frozen... If you want that set cond_stage_trainable=False in cfg + """ + + def __init__( + self, + model='ViT-L/14', + jit=False, + device='cpu', + antialias=False, + ): + super().__init__() + self.model, _ = clip.load(name=model, device=device, jit=jit) + # We don't use the text part so delete it + del self.model.transformer + self.antialias = antialias + self.register_buffer( + 'mean', + torch.Tensor([0.48145466, 0.4578275, 0.40821073]), + persistent=False) + self.register_buffer( + 'std', + torch.Tensor([0.26862954, 0.26130258, 0.27577711]), + persistent=False) + + def preprocess(self, x): + # Expects inputs in the range -1, 1 + x = kornia.geometry.resize( + x, (224, 224), + interpolation='bicubic', + align_corners=True, + antialias=self.antialias) + x = (x + 1.) / 2. + # renormalize according to clip + x = kornia.enhance.normalize(x, self.mean, self.std) + return x + + def forward(self, x): + # x is assumed to be in range [-1,1] + if isinstance(x, list): + # [""] denotes condition dropout for ucg + device = self.model.visual.conv1.weight.device + return torch.zeros(1, 768, device=device) + return self.model.encode_image(self.preprocess(x)).float() + + def encode(self, im): + return self(im).unsqueeze(1) + + +class FrozenCLIPImageMutliEmbedder(AbstractEncoder): + """ + Uses the CLIP image encoder. + Not actually frozen... If you want that set cond_stage_trainable=False in cfg + """ + + def __init__( + self, + model='ViT-L/14', + jit=False, + device='cpu', + antialias=True, + max_crops=5, + ): + super().__init__() + self.model, _ = clip.load(name=model, device=device, jit=jit) + # We don't use the text part so delete it + del self.model.transformer + self.antialias = antialias + self.register_buffer( + 'mean', + torch.Tensor([0.48145466, 0.4578275, 0.40821073]), + persistent=False) + self.register_buffer( + 'std', + torch.Tensor([0.26862954, 0.26130258, 0.27577711]), + persistent=False) + self.max_crops = max_crops + + def preprocess(self, x): + + # Expects inputs in the range -1, 1 + randcrop = transforms.RandomResizedCrop( + 224, scale=(0.085, 1.0), ratio=(1, 1)) + max_crops = self.max_crops + patches = [] + crops = [randcrop(x) for _ in range(max_crops)] + patches.extend(crops) + x = torch.cat(patches, dim=0) + x = (x + 1.) / 2. + # renormalize according to clip + x = kornia.enhance.normalize(x, self.mean, self.std) + return x + + def forward(self, x): + # x is assumed to be in range [-1,1] + if isinstance(x, list): + # [""] denotes condition dropout for ucg + device = self.model.visual.conv1.weight.device + return torch.zeros(1, self.max_crops, 768, device=device) + batch_tokens = [] + for im in x: + patches = self.preprocess(im.unsqueeze(0)) + tokens = self.model.encode_image(patches).float() + for t in tokens: + if random.random() < 0.1: + t *= 0 + batch_tokens.append(tokens.unsqueeze(0)) + + return torch.cat(batch_tokens, dim=0) + + def encode(self, im): + return self(im) + + +class SpatialRescaler(nn.Module): + + def __init__(self, + n_stages=1, + method='bilinear', + multiplier=0.5, + in_channels=3, + out_channels=None, + bias=False): + super().__init__() + self.n_stages = n_stages + assert self.n_stages >= 0 + assert method in [ + 'nearest', 'linear', 'bilinear', 'trilinear', 'bicubic', 'area' + ] + self.multiplier = multiplier + self.interpolator = partial( + torch.nn.functional.interpolate, mode=method) + self.remap_output = out_channels is not None + if self.remap_output: + print( + f'Spatial Rescaler mapping from {in_channels} to {out_channels} channels after resizing.' + ) + self.channel_mapper = nn.Conv2d( + in_channels, out_channels, 1, bias=bias) + + def forward(self, x): + for stage in range(self.n_stages): + x = self.interpolator(x, scale_factor=self.multiplier) + + if self.remap_output: + x = self.channel_mapper(x) + return x + + def encode(self, x): + return self(x) + + +class LowScaleEncoder(nn.Module): + + def __init__(self, + model_config, + linear_start, + linear_end, + timesteps=1000, + max_noise_level=250, + output_size=64, + scale_factor=1.0): + super().__init__() + self.max_noise_level = max_noise_level + self.model = instantiate_from_config(model_config) + self.augmentation_schedule = self.register_schedule( + timesteps=timesteps, + linear_start=linear_start, + linear_end=linear_end) + self.out_size = output_size + self.scale_factor = scale_factor + + def register_schedule(self, + beta_schedule='linear', + timesteps=1000, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3): + betas = make_beta_schedule( + beta_schedule, + timesteps, + linear_start=linear_start, + linear_end=linear_end, + cosine_s=cosine_s) + alphas = 1. - betas + alphas_cumprod = np.cumprod(alphas, axis=0) + alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1]) + + timesteps, = betas.shape + self.num_timesteps = int(timesteps) + self.linear_start = linear_start + self.linear_end = linear_end + assert alphas_cumprod.shape[ + 0] == self.num_timesteps, 'alphas have to be defined for each timestep' + + to_torch = partial(torch.tensor, dtype=torch.float32) + + self.register_buffer('betas', to_torch(betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer('alphas_cumprod_prev', + to_torch(alphas_cumprod_prev)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer('sqrt_alphas_cumprod', + to_torch(np.sqrt(alphas_cumprod))) + self.register_buffer('sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1. - alphas_cumprod))) + self.register_buffer('log_one_minus_alphas_cumprod', + to_torch(np.log(1. - alphas_cumprod))) + self.register_buffer('sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod))) + self.register_buffer('sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod - 1))) + + def q_sample(self, x_start, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + + extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, + x_start.shape) * noise) + + def forward(self, x): + z = self.model.encode(x).sample() + z = z * self.scale_factor + noise_level = torch.randint( + 0, self.max_noise_level, (x.shape[0], ), device=x.device).long() + z = self.q_sample(z, noise_level) + if self.out_size is not None: + z = torch.nn.functional.interpolate( + z, size=self.out_size, + mode='nearest') # TODO: experiment with mode + # z = z.repeat_interleave(2, -2).repeat_interleave(2, -1) + return z, noise_level + + def decode(self, z): + z = z / self.scale_factor + return self.model.decode(z) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/openaimodel.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/openaimodel.py new file mode 100644 index 0000000000000000000000000000000000000000..dd372f424f9d52b80a4ccc401c5473917a39080d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/openaimodel.py @@ -0,0 +1,1010 @@ +import math +from abc import abstractmethod +from functools import partial +from typing import Iterable + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as F + +from ..util import exists +from .attention import SpatialTransformer +from .util_diffusion import (avg_pool_nd, checkpoint, conv_nd, linear, + normalization, timestep_embedding, zero_module) + + +# dummy replace +def convert_module_to_f16(x): + pass + + +def convert_module_to_f32(x): + pass + + +class AttentionPool2d(nn.Module): + """ + Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__( + self, + spacial_dim: int, + embed_dim: int, + num_heads_channels: int, + output_dim: int = None, + ): + super().__init__() + self.positional_embedding = nn.Parameter( + th.randn(embed_dim, spacial_dim**2 + 1) / embed_dim**0.5) + self.qkv_proj = conv_nd(1, embed_dim, 3 * embed_dim, 1) + self.c_proj = conv_nd(1, embed_dim, output_dim or embed_dim, 1) + self.num_heads = embed_dim // num_heads_channels + self.attention = QKVAttention(self.num_heads) + + def forward(self, x): + b, c, *_spatial = x.shape + x = x.reshape(b, c, -1) # NC(HW) + x = th.cat([x.mean(dim=-1, keepdim=True), x], dim=-1) # NC(HW+1) + x = x + self.positional_embedding[None, :, :].to(x.dtype) # NC(HW+1) + x = self.qkv_proj(x) + x = self.attention(x) + x = self.c_proj(x) + return x[:, :, 0] + + +class TimestepBlock(nn.Module): + """ + Any module where forward() takes timestep embeddings as a second argument. + """ + + @abstractmethod + def forward(self, x, emb): + """ + Apply the module to `x` given `emb` timestep embeddings. + """ + + +class TimestepEmbedSequential(nn.Sequential, TimestepBlock): + """ + A sequential module that passes timestep embeddings to the children that + support it as an extra input. + """ + + def forward(self, x, emb, context=None): + for layer in self: + if isinstance(layer, TimestepBlock): + x = layer(x, emb) + elif isinstance(layer, SpatialTransformer): + x = layer(x, context) + else: + x = layer(x) + return x + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = conv_nd( + dims, self.channels, self.out_channels, 3, padding=padding) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + if self.use_conv: + x = self.conv(x) + return x + + +class TransposedUpsample(nn.Module): + 'Learned 2x upsampling without padding' + + def __init__(self, channels, out_channels=None, ks=5): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + + self.up = nn.ConvTranspose2d( + self.channels, self.out_channels, kernel_size=ks, stride=2) + + def forward(self, x): + return self.up(x) + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = conv_nd( + dims, + self.channels, + self.out_channels, + 3, + stride=stride, + padding=padding) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class ResBlock(TimestepBlock): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + use_checkpoint=False, + up=False, + down=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_checkpoint = use_checkpoint + self.use_scale_shift_norm = use_scale_shift_norm + + self.in_layers = nn.Sequential( + normalization(channels), + nn.SiLU(), + conv_nd(dims, channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + normalization(self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + conv_nd( + dims, self.out_channels, self.out_channels, 3, padding=1)), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = conv_nd(dims, channels, self.out_channels, + 1) + + def forward(self, x, emb): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return checkpoint(self._forward, (x, emb), self.parameters(), + self.use_checkpoint) + + def _forward(self, x, emb): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + return self.skip_connection(x) + h + + +class AttentionBlock(nn.Module): + """ + An attention block that allows spatial positions to attend to each other. + Originally ported from here, but adapted to the N-d case. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66. + """ + + def __init__( + self, + channels, + num_heads=1, + num_head_channels=-1, + use_checkpoint=False, + use_new_attention_order=False, + ): + super().__init__() + self.channels = channels + if num_head_channels == -1: + self.num_heads = num_heads + else: + assert ( + channels % num_head_channels == 0 + ), f'q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}' + self.num_heads = channels // num_head_channels + self.use_checkpoint = use_checkpoint + self.norm = normalization(channels) + self.qkv = conv_nd(1, channels, channels * 3, 1) + if use_new_attention_order: + # split qkv before split heads + self.attention = QKVAttention(self.num_heads) + else: + # split heads before split qkv + self.attention = QKVAttentionLegacy(self.num_heads) + + self.proj_out = zero_module(conv_nd(1, channels, channels, 1)) + + def forward(self, x): + return checkpoint( + self._forward, (x, ), self.parameters(), True + ) # TODO: check checkpoint usage, is True # TODO: fix the .half call!!! + + def _forward(self, x): + b, c, *spatial = x.shape + x = x.reshape(b, c, -1) + qkv = self.qkv(self.norm(x)) + h = self.attention(qkv) + h = self.proj_out(h) + return (x + h).reshape(b, c, *spatial) + + +def count_flops_attn(model, _x, y): + """ + A counter for the `thop` package to count the operations in an + attention operation. + Meant to be used like: + macs, params = thop.profile( + model, + inputs=(inputs, timestamps), + custom_ops={QKVAttention: QKVAttention.count_flops}, + ) + """ + b, c, *spatial = y[0].shape + num_spatial = int(np.prod(spatial)) + # We perform two matmuls with the same number of ops. + # The first computes the weight matrix, the second computes + # the combination of the value vectors. + matmul_ops = 2 * b * (num_spatial**2) * c + model.total_ops += th.DoubleTensor([matmul_ops]) + + +class QKVAttentionLegacy(nn.Module): + """ + A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.reshape(bs * self.n_heads, ch * 3, length).split( + ch, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + 'bct,bcs->bts', q * scale, + k * scale) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum('bts,bcs->bct', weight, v) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class QKVAttention(nn.Module): + """ + A module which performs QKV attention and splits in a different order. + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.chunk(3, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + 'bct,bcs->bts', + (q * scale).view(bs * self.n_heads, ch, length), + (k * scale).view(bs * self.n_heads, ch, length), + ) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum('bts,bcs->bct', weight, + v.reshape(bs * self.n_heads, ch, length)) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class UNetModel(nn.Module): + """ + The full UNet model with attention and timestep embedding. + :param in_channels: channels in the input Tensor. + :param model_channels: base channel count for the model. + :param out_channels: channels in the output Tensor. + :param num_res_blocks: number of residual blocks per downsample. + :param attention_resolutions: a collection of downsample rates at which + attention will take place. May be a set, list, or tuple. + For example, if this contains 4, then at 4x downsampling, attention + will be used. + :param dropout: the dropout probability. + :param channel_mult: channel multiplier for each level of the UNet. + :param conv_resample: if True, use learned convolutions for upsampling and + downsampling. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param num_classes: if specified (as an int), then this model will be + class-conditional with `num_classes` classes. + :param use_checkpoint: use gradient checkpointing to reduce memory usage. + :param num_heads: the number of attention heads in each attention layer. + :param num_heads_channels: if specified, ignore num_heads and instead use + a fixed channel width per attention head. + :param num_heads_upsample: works with num_heads to set a different number + of heads for upsampling. Deprecated. + :param use_scale_shift_norm: use a FiLM-like conditioning mechanism. + :param resblock_updown: use residual blocks for up/downsampling. + :param use_new_attention_order: use a different attention pattern for potentially + increased efficiency. + """ + + def __init__( + self, + image_size, + in_channels, + model_channels, + out_channels, + num_res_blocks, + attention_resolutions, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + num_classes=None, + use_checkpoint=False, + use_fp16=False, + num_heads=-1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + use_spatial_transformer=False, # custom transformer support + transformer_depth=1, # custom transformer support + context_dim=None, # custom transformer support + n_embed=None, # custom support for prediction of discrete ids into codebook of first stage vq model + legacy=True, + disable_self_attentions=None, + num_attention_blocks=None): + super().__init__() + if use_spatial_transformer: + assert context_dim is not None, 'include the dimension of your cross-attention conditioning...' + + if context_dim is not None: + assert use_spatial_transformer, 'use the spatial transformer for your cross-attention conditioning...' + from omegaconf.listconfig import ListConfig + if type(context_dim) == ListConfig: + context_dim = list(context_dim) + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + if num_heads == -1: + assert num_head_channels != -1, 'Either num_heads or num_head_channels has to be set' + + if num_head_channels == -1: + assert num_heads != -1, 'Either num_heads or num_head_channels has to be set' + + self.image_size = image_size + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + if isinstance(num_res_blocks, int): + self.num_res_blocks = len(channel_mult) * [num_res_blocks] + else: + if len(num_res_blocks) != len(channel_mult): + raise ValueError( + 'provide num_res_blocks either as an int (globally constant) or ' + 'as a list/tuple (per-level) with the same length as channel_mult' + ) + self.num_res_blocks = num_res_blocks + + if disable_self_attentions is not None: + # should be a list of booleans, indicating whether to disable self-attention in TransformerBlocks or not + assert len(disable_self_attentions) == len(channel_mult) + if num_attention_blocks is not None: + assert len(num_attention_blocks) == len(self.num_res_blocks) + assert all( + map( + lambda i: self.num_res_blocks[i] >= num_attention_blocks[i + ], + range(len(num_attention_blocks)))) + print( + f'Constructor of UNetModel received num_attention_blocks={num_attention_blocks}. ' + f'This option has LESS priority than attention_resolutions {attention_resolutions}, ' + f'i.e., in cases where num_attention_blocks[i] > 0 but 2**i not in attention_resolutions, ' + f'attention will still not be set.' + ) # todo: convert to warning + + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + self.predict_codebook_ids = n_embed is not None + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + linear(model_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ) + + if self.num_classes is not None: + self.label_emb = nn.Embedding(num_classes, time_embed_dim) + + self.input_blocks = nn.ModuleList([ + TimestepEmbedSequential( + conv_nd(dims, in_channels, model_channels, 3, padding=1)) + ]) + self._feature_size = model_channels + input_block_chans = [model_channels] + ch = model_channels + ds = 1 + for level, mult in enumerate(channel_mult): + for nr in range(self.num_res_blocks[level]): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=mult * model_channels, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = mult * model_channels + if ds in attention_resolutions: + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + if exists(disable_self_attentions): + disabled_sa = disable_self_attentions[level] + else: + disabled_sa = False + + if not exists(num_attention_blocks + ) or nr < num_attention_blocks[level]: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=dim_head, + use_new_attention_order=use_new_attention_order, + ) if not use_spatial_transformer else + SpatialTransformer( + ch, + num_heads, + dim_head, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa)) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) if resblock_updown else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + self.middle_block = TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=dim_head, + use_new_attention_order=use_new_attention_order, + ) if not use_spatial_transformer else + SpatialTransformer( # always uses a self-attn + ch, + num_heads, + dim_head, + depth=transformer_depth, + context_dim=context_dim), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + ) + self._feature_size += ch + + self.output_blocks = nn.ModuleList([]) + for level, mult in list(enumerate(channel_mult))[::-1]: + for i in range(self.num_res_blocks[level] + 1): + ich = input_block_chans.pop() + layers = [ + ResBlock( + ch + ich, + time_embed_dim, + dropout, + out_channels=model_channels * mult, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = model_channels * mult + if ds in attention_resolutions: + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + if exists(disable_self_attentions): + disabled_sa = disable_self_attentions[level] + else: + disabled_sa = False + + if not exists(num_attention_blocks + ) or i < num_attention_blocks[level]: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads_upsample, + num_head_channels=dim_head, + use_new_attention_order=use_new_attention_order, + ) if not use_spatial_transformer else + SpatialTransformer( + ch, + num_heads, + dim_head, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa)) + if level and i == self.num_res_blocks[level]: + out_ch = ch + layers.append( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + up=True, + ) if resblock_updown else Upsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ds //= 2 + self.output_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + + self.out = nn.Sequential( + normalization(ch), + nn.SiLU(), + zero_module( + conv_nd(dims, model_channels, out_channels, 3, padding=1)), + ) + if self.predict_codebook_ids: + self.id_predictor = nn.Sequential( + normalization(ch), + conv_nd(dims, model_channels, n_embed, 1), + ) + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + self.output_blocks.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + self.output_blocks.apply(convert_module_to_f32) + + def forward(self, x, timesteps=None, context=None, y=None, **kwargs): + """ + Apply the model to an input batch. + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :param context: conditioning plugged in via crossattn + :param y: an [N] Tensor of labels, if class-conditional. + :return: an [N x C x ...] Tensor of outputs. + """ + assert (y is not None) == ( + self.num_classes is not None + ), 'must specify y if and only if the model is class-conditional' + hs = [] + t_emb = timestep_embedding( + timesteps, self.model_channels, repeat_only=False) + emb = self.time_embed(t_emb) + + if self.num_classes is not None: + assert y.shape == (x.shape[0], ) + emb = emb + self.label_emb(y) + + h = x.type(self.dtype) + for module in self.input_blocks: + h = module(h, emb, context) + hs.append(h) + h = self.middle_block(h, emb, context) + for module in self.output_blocks: + h = th.cat([h, hs.pop()], dim=1) + h = module(h, emb, context) + h = h.type(x.dtype) + if self.predict_codebook_ids: + return self.id_predictor(h) + else: + return self.out(h) + + +class EncoderUNetModel(nn.Module): + """ + The half UNet model with attention and timestep embedding. + For usage, see UNet. + """ + + def __init__(self, + image_size, + in_channels, + model_channels, + out_channels, + num_res_blocks, + attention_resolutions, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + use_checkpoint=False, + use_fp16=False, + num_heads=1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + pool='adaptive', + *args, + **kwargs): + super().__init__() + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + self.num_res_blocks = num_res_blocks + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.use_checkpoint = use_checkpoint + self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + linear(model_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ) + + self.input_blocks = nn.ModuleList([ + TimestepEmbedSequential( + conv_nd(dims, in_channels, model_channels, 3, padding=1)) + ]) + self._feature_size = model_channels + input_block_chans = [model_channels] + ch = model_channels + ds = 1 + for level, mult in enumerate(channel_mult): + for _ in range(num_res_blocks): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=mult * model_channels, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = mult * model_channels + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + )) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) if resblock_updown else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + self.middle_block = TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + ) + self._feature_size += ch + self.pool = pool + if pool == 'adaptive': + self.out = nn.Sequential( + normalization(ch), + nn.SiLU(), + nn.AdaptiveAvgPool2d((1, 1)), + zero_module(conv_nd(dims, ch, out_channels, 1)), + nn.Flatten(), + ) + elif pool == 'attention': + assert num_head_channels != -1 + self.out = nn.Sequential( + normalization(ch), + nn.SiLU(), + AttentionPool2d((image_size // ds), ch, num_head_channels, + out_channels), + ) + elif pool == 'spatial': + self.out = nn.Sequential( + nn.Linear(self._feature_size, 2048), + nn.ReLU(), + nn.Linear(2048, self.out_channels), + ) + elif pool == 'spatial_v2': + self.out = nn.Sequential( + nn.Linear(self._feature_size, 2048), + normalization(2048), + nn.SiLU(), + nn.Linear(2048, self.out_channels), + ) + else: + raise NotImplementedError(f'Unexpected {pool} pooling') + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + + def forward(self, x, timesteps): + """ + Apply the model to an input batch. + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :return: an [N x K] Tensor of outputs. + """ + emb = self.time_embed( + timestep_embedding(timesteps, self.model_channels)) + + results = [] + h = x.type(self.dtype) + for module in self.input_blocks: + h = module(h, emb) + if self.pool.startswith('spatial'): + results.append(h.type(x.dtype).mean(dim=(2, 3))) + h = self.middle_block(h, emb) + if self.pool.startswith('spatial'): + results.append(h.type(x.dtype).mean(dim=(2, 3))) + h = th.cat(results, axis=-1) + return self.out(h) + else: + h = h.type(x.dtype) + return self.out(h) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/plms.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/plms.py new file mode 100644 index 0000000000000000000000000000000000000000..72fd6da2ee0607d7c23fa558b61c10db29832e5d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/plms.py @@ -0,0 +1,349 @@ +"""SAMPLING ONLY.""" + +from functools import partial + +import numpy as np +import torch +from tqdm import tqdm + +from .sampling_util import norm_thresholding +from .util_diffusion import (make_ddim_sampling_parameters, + make_ddim_timesteps, noise_like) + + +class PLMSSampler(object): + + def __init__(self, model, schedule='linear', **kwargs): + super().__init__() + self.model = model + self.ddpm_num_timesteps = model.num_timesteps + self.schedule = schedule + + def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device('cuda'): + attr = attr.to(torch.device('cuda')) + setattr(self, name, attr) + + def make_schedule(self, + ddim_num_steps, + ddim_discretize='uniform', + ddim_eta=0., + verbose=True): + if ddim_eta != 0: + raise ValueError('ddim_eta must be 0 for PLMS') + self.ddim_timesteps = make_ddim_timesteps( + ddim_discr_method=ddim_discretize, + num_ddim_timesteps=ddim_num_steps, + num_ddpm_timesteps=self.ddpm_num_timesteps, + verbose=verbose) + alphas_cumprod = self.model.alphas_cumprod + assert alphas_cumprod.shape[ + 0] == self.ddpm_num_timesteps, 'alphas have to be defined for each timestep' + + def to_torch(x): + return x.clone().detach().to(torch.float32).to(self.model.device) + + self.register_buffer('betas', to_torch(self.model.betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer('alphas_cumprod_prev', + to_torch(self.model.alphas_cumprod_prev)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer('sqrt_alphas_cumprod', + to_torch(np.sqrt(alphas_cumprod.cpu()))) + self.register_buffer('sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1. - alphas_cumprod.cpu()))) + self.register_buffer('log_one_minus_alphas_cumprod', + to_torch(np.log(1. - alphas_cumprod.cpu()))) + self.register_buffer('sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod.cpu()))) + self.register_buffer('sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1. / alphas_cumprod.cpu() - 1))) + + # ddim sampling parameters + ddim_sigmas, ddim_alphas, ddim_alphas_prev = make_ddim_sampling_parameters( + alphacums=alphas_cumprod.cpu(), + ddim_timesteps=self.ddim_timesteps, + eta=ddim_eta, + verbose=verbose) + self.register_buffer('ddim_sigmas', ddim_sigmas) + self.register_buffer('ddim_alphas', ddim_alphas) + self.register_buffer('ddim_alphas_prev', ddim_alphas_prev) + self.register_buffer('ddim_sqrt_one_minus_alphas', + np.sqrt(1. - ddim_alphas)) + alp_1 = (1 - self.alphas_cumprod_prev) / (1 - self.alphas_cumprod) + alp_2 = (1 - self.alphas_cumprod / self.alphas_cumprod_prev) + sigmas_for_original_sampling_steps = ddim_eta * torch.sqrt( + alp_1 * alp_2) + self.register_buffer('ddim_sigmas_for_original_num_steps', + sigmas_for_original_sampling_steps) + + @torch.no_grad() + def sample( + self, + S, + batch_size, + shape, + conditioning=None, + callback=None, + normals_sequence=None, + img_callback=None, + quantize_x0=False, + eta=0., + mask=None, + x0=None, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + verbose=True, + x_T=None, + log_every_t=100, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + # this has to come in the same format as the conditioning, # e.g. as encoded tokens, ... + dynamic_threshold=None, + **kwargs): + if conditioning is not None: + if isinstance(conditioning, dict): + ctmp = conditioning[list(conditioning.keys())[0]] + while isinstance(ctmp, list): + ctmp = ctmp[0] + cbs = ctmp.shape[0] + if cbs != batch_size: + print( + f'Warning: Got {cbs} conditionings but batch-size is {batch_size}' + ) + else: + if conditioning.shape[0] != batch_size: + print( + f'Warning: Got {conditioning.shape[0]} conditionings but batch-size is {batch_size}' + ) + + self.make_schedule(ddim_num_steps=S, ddim_eta=eta, verbose=verbose) + # sampling + C, H, W = shape + size = (batch_size, C, H, W) + print(f'Data shape for PLMS sampling is {size}') + + samples, intermediates = self.plms_sampling( + conditioning, + size, + callback=callback, + img_callback=img_callback, + quantize_denoised=quantize_x0, + mask=mask, + x0=x0, + ddim_use_original_steps=False, + noise_dropout=noise_dropout, + temperature=temperature, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + x_T=x_T, + log_every_t=log_every_t, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + dynamic_threshold=dynamic_threshold, + ) + return samples, intermediates + + @torch.no_grad() + def plms_sampling(self, + cond, + shape, + x_T=None, + ddim_use_original_steps=False, + callback=None, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + img_callback=None, + log_every_t=100, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + dynamic_threshold=None): + device = self.model.betas.device + b = shape[0] + if x_T is None: + img = torch.randn(shape, device=device) + else: + img = x_T + + if timesteps is None: + timesteps = self.ddpm_num_timesteps if ddim_use_original_steps else self.ddim_timesteps + elif timesteps is not None and not ddim_use_original_steps: + subset_end = int( + min(timesteps / self.ddim_timesteps.shape[0], 1) + * self.ddim_timesteps.shape[0]) - 1 + timesteps = self.ddim_timesteps[:subset_end] + + intermediates = {'x_inter': [img], 'pred_x0': [img]} + time_range = list(reversed(range( + 0, timesteps))) if ddim_use_original_steps else np.flip(timesteps) + total_steps = timesteps if ddim_use_original_steps else timesteps.shape[ + 0] + print(f'Running PLMS Sampling with {total_steps} timesteps') + + iterator = tqdm(time_range, desc='PLMS Sampler', total=total_steps) + old_eps = [] + + for i, step in enumerate(iterator): + index = total_steps - i - 1 + ts = torch.full((b, ), step, device=device, dtype=torch.long) + ts_next = torch.full((b, ), + time_range[min(i + 1, + len(time_range) - 1)], + device=device, + dtype=torch.long) + + if mask is not None: + assert x0 is not None + img_orig = self.model.q_sample( + x0, ts) # TODO: deterministic forward pass? + img = img_orig * mask + (1. - mask) * img + + outs = self.p_sample_plms( + img, + cond, + ts, + index=index, + use_original_steps=ddim_use_original_steps, + quantize_denoised=quantize_denoised, + temperature=temperature, + noise_dropout=noise_dropout, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + old_eps=old_eps, + t_next=ts_next, + dynamic_threshold=dynamic_threshold) + img, pred_x0, e_t = outs + old_eps.append(e_t) + if len(old_eps) >= 4: + old_eps.pop(0) + if callback: + callback(i) + if img_callback: + img_callback(pred_x0, i) + + if index % log_every_t == 0 or index == total_steps - 1: + intermediates['x_inter'].append(img) + intermediates['pred_x0'].append(pred_x0) + + return img, intermediates + + @torch.no_grad() + def p_sample_plms(self, + x, + c, + t, + index, + repeat_noise=False, + use_original_steps=False, + quantize_denoised=False, + temperature=1., + noise_dropout=0., + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1., + unconditional_conditioning=None, + old_eps=None, + t_next=None, + dynamic_threshold=None): + b, *_, device = *x.shape, x.device + + def get_model_output(x, t): + if unconditional_conditioning is None or unconditional_guidance_scale == 1.: + e_t = self.model.apply_model(x, t, c) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) + if isinstance(c, dict): + assert isinstance(unconditional_conditioning, dict) + c_in = dict() + for k in c: + if isinstance(c[k], list): + c_in[k] = [ + torch.cat([ + unconditional_conditioning[k][i], c[k][i] + ]) for i in range(len(c[k])) + ] + else: + c_in[k] = torch.cat( + [unconditional_conditioning[k], c[k]]) + else: + c_in = torch.cat([unconditional_conditioning, c]) + e_t_uncond, e_t = self.model.apply_model(x_in, t_in, + c_in).chunk(2) + e_t = e_t_uncond + unconditional_guidance_scale * ( + e_t - e_t_uncond) + + if score_corrector is not None: + assert self.model.parameterization == 'eps' + e_t = score_corrector.modify_score(self.model, e_t, x, t, c, + **corrector_kwargs) + + return e_t + + alphas = self.model.alphas_cumprod if use_original_steps else self.ddim_alphas + if use_original_steps: + alphas_prev = self.model.alphas_cumprod_prev + sqrt_one_minus_alphas = self.model.sqrt_one_minus_alphas_cumprod + else: + alphas_prev = self.ddim_alphas_prev + sqrt_one_minus_alphas = self.ddim_sqrt_one_minus_alphas + sigmas = self.model.ddim_sigmas_for_original_num_steps if use_original_steps else self.ddim_sigmas + + def get_x_prev_and_pred_x0(e_t, index): + # select parameters corresponding to the currently considered timestep + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), + alphas_prev[index], + device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full((b, 1, 1, 1), + sqrt_one_minus_alphas[index], + device=device) + + # current prediction for x_0 + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + if dynamic_threshold is not None: + pred_x0 = norm_thresholding(pred_x0, dynamic_threshold) + # direction pointing to x_t + dir_xt = (1. - a_prev - sigma_t**2).sqrt() * e_t + noise = sigma_t * noise_like(x.shape, device, + repeat_noise) * temperature + if noise_dropout > 0.: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + + e_t = get_model_output(x, t) + if len(old_eps) == 0: + # Pseudo Improved Euler (2nd order) + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t, index) + e_t_next = get_model_output(x_prev, t_next) + e_t_prime = (e_t + e_t_next) / 2 + elif len(old_eps) == 1: + # 2nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (3 * e_t - old_eps[-1]) / 2 + elif len(old_eps) == 2: + # 3nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (23 * e_t - 16 * old_eps[-1] + 5 * old_eps[-2]) / 12 + elif len(old_eps) >= 3: + # 4nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (55 * e_t - 59 * old_eps[-1] + 37 * old_eps[-2] + - 9 * old_eps[-3]) / 24 + + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t_prime, index) + + return x_prev, pred_x0, e_t diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/sampling_util.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/sampling_util.py new file mode 100644 index 0000000000000000000000000000000000000000..e6c6293b027fccf2a5212912e998d4f16f93df14 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/sampling_util.py @@ -0,0 +1,51 @@ +import numpy as np +import torch + + +def append_dims(x, target_dims): + """Appends dimensions to the end of a tensor until it has target_dims dimensions. + From https://github.com/crowsonkb/k-diffusion/blob/master/k_diffusion/utils.py""" + dims_to_append = target_dims - x.ndim + if dims_to_append < 0: + raise ValueError( + f'input has {x.ndim} dims but target_dims is {target_dims}, which is less' + ) + return x[(..., ) + (None, ) * dims_to_append] + + +def renorm_thresholding(x0, value): + # renorm + pred_max = x0.max() + pred_min = x0.min() + pred_x0 = (x0 - pred_min) / (pred_max - pred_min) # 0 ... 1 + pred_x0 = 2 * pred_x0 - 1. # -1 ... 1 + + s = torch.quantile( + rearrange(pred_x0, 'b ... -> b (...)').abs(), value, dim=-1) + s.clamp_(min=1.0) + s = s.view(-1, *((1, ) * (pred_x0.ndim - 1))) + + # clip by threshold + # pred_x0 = pred_x0.clamp(-s, s) / s # needs newer pytorch # TODO bring back to pure-gpu with min/max + + # temporary hack: numpy on cpu + pred_x0 = np.clip(pred_x0.cpu().numpy(), -s.cpu().numpy(), + s.cpu().numpy()) / s.cpu().numpy() + pred_x0 = torch.tensor(pred_x0).to(self.model.device) + + # re.renorm + pred_x0 = (pred_x0 + 1.) / 2. # 0 ... 1 + pred_x0 = (pred_max - pred_min) * pred_x0 + pred_min # orig range + return pred_x0 + + +def norm_thresholding(x0, value): + s = append_dims( + x0.pow(2).flatten(1).mean(1).sqrt().clamp(min=value), x0.ndim) + return x0 * (value / s) + + +def spatial_norm_thresholding(x0, value): + # b c h w + s = x0.pow(2).mean(1, keepdim=True).sqrt().clamp(min=value) + return x0 * (value / s) diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/util_diffusion.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/util_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..1e5496ca9299d845c60e953b33d955f0c70eae5d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/util_diffusion.py @@ -0,0 +1,308 @@ +# adopted from +# https://github.com/openai/improved-diffusion/blob/main/improved_diffusion/gaussian_diffusion.py +# and +# https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +# and +# https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py +# +# thanks! + +import math +import os + +import numpy as np +import torch +import torch.nn as nn +from einops import repeat + +from ..util import instantiate_from_config + + +def make_beta_schedule(schedule, + n_timestep, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3): + if schedule == 'linear': + betas = ( + torch.linspace( + linear_start**0.5, + linear_end**0.5, + n_timestep, + dtype=torch.float64)**2) + + elif schedule == 'cosine': + timesteps = ( + torch.arange(n_timestep + 1, dtype=torch.float64) / n_timestep + + cosine_s) + alphas = timesteps / (1 + cosine_s) * np.pi / 2 + alphas = torch.cos(alphas).pow(2) + alphas = alphas / alphas[0] + betas = 1 - alphas[1:] / alphas[:-1] + betas = np.clip(betas, a_min=0, a_max=0.999) + + elif schedule == 'sqrt_linear': + betas = torch.linspace( + linear_start, linear_end, n_timestep, dtype=torch.float64) + elif schedule == 'sqrt': + betas = torch.linspace( + linear_start, linear_end, n_timestep, dtype=torch.float64)**0.5 + else: + raise ValueError(f"schedule '{schedule}' unknown.") + return betas.numpy() + + +def make_ddim_timesteps(ddim_discr_method, + num_ddim_timesteps, + num_ddpm_timesteps, + verbose=True): + if ddim_discr_method == 'uniform': + c = num_ddpm_timesteps // num_ddim_timesteps + ddim_timesteps = np.asarray(list(range(0, num_ddpm_timesteps, c))) + elif ddim_discr_method == 'quad': + ddim_timesteps = ((np.linspace(0, np.sqrt(num_ddpm_timesteps * .8), + num_ddim_timesteps))**2).astype(int) + else: + raise NotImplementedError( + f'There is no ddim discretization method called "{ddim_discr_method}"' + ) + + # assert ddim_timesteps.shape[0] == num_ddim_timesteps + # add one to get the final alpha values right (the ones from first scale to data during sampling) + steps_out = ddim_timesteps + 1 + if verbose: + print(f'Selected timesteps for ddim sampler: {steps_out}') + return steps_out + + +def make_ddim_sampling_parameters(alphacums, + ddim_timesteps, + eta, + verbose=True): + # select alphas for computing the variance schedule + alphas = alphacums[ddim_timesteps] + alphas_prev = np.asarray([alphacums[0]] + + alphacums[ddim_timesteps[:-1]].tolist()) + + # according the the formula provided in https://arxiv.org/abs/2010.02502 + alpha_1 = (1 - alphas_prev) / (1 - alphas) + alpha_2 = (1 - alphas / alphas_prev) + sigmas = eta * np.sqrt(alpha_1 * alpha_2) + if verbose: + print( + f'Selected alphas for ddim sampler: a_t: {alphas}; a_(t-1): {alphas_prev}' + ) + print( + f'For the chosen value of eta, which is {eta}, ' + f'this results in the following sigma_t schedule for ddim sampler {sigmas}' + ) + return sigmas, alphas, alphas_prev + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + + +def extract_into_tensor(a, t, x_shape): + b, *_ = t.shape + out = a.gather(-1, t) + return out.reshape(b, *((1, ) * (len(x_shape) - 1))) + + +def checkpoint(func, inputs, params, flag): + """ + Evaluate a function without caching intermediate activations, allowing for + reduced memory at the expense of extra compute in the backward pass. + :param func: the function to evaluate. + :param inputs: the argument sequence to pass to `func`. + :param params: a sequence of parameters `func` depends on but does not + explicitly take as arguments. + :param flag: if False, disable gradient checkpointing. + """ + if flag: + args = tuple(inputs) + tuple(params) + return CheckpointFunction.apply(func, len(inputs), *args) + else: + return func(*inputs) + + +class CheckpointFunction(torch.autograd.Function): + + @staticmethod + def forward(ctx, run_function, length, *args): + ctx.run_function = run_function + ctx.input_tensors = list(args[:length]) + ctx.input_params = list(args[length:]) + + with torch.no_grad(): + output_tensors = ctx.run_function(*ctx.input_tensors) + return output_tensors + + @staticmethod + def backward(ctx, *output_grads): + ctx.input_tensors = [ + x.detach().requires_grad_(True) for x in ctx.input_tensors + ] + with torch.enable_grad(): + # Fixes a bug where the first op in run_function modifies the + # Tensor storage in place, which is not allowed for detach()'d + # Tensors. + shallow_copies = [x.view_as(x) for x in ctx.input_tensors] + output_tensors = ctx.run_function(*shallow_copies) + input_grads = torch.autograd.grad( + output_tensors, + ctx.input_tensors + ctx.input_params, + output_grads, + allow_unused=True, + ) + del ctx.input_tensors + del ctx.input_params + del output_tensors + return (None, None) + input_grads + + +def timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False): + """ + Create sinusoidal timestep embeddings. + :param timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an [N x dim] Tensor of positional embeddings. + """ + if not repeat_only: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half).to(device=timesteps.device) + args = timesteps[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + else: + embedding = repeat(timesteps, 'b -> b d', d=dim) + return embedding + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def scale_module(module, scale): + """ + Scale the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().mul_(scale) + return module + + +def mean_flat(tensor): + """ + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +def normalization(channels): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: an nn.Module for normalization. + """ + return GroupNorm32(32, channels) + + +# PyTorch 1.7 has SiLU, but we support PyTorch 1.5. +class SiLU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(x) + + +class GroupNorm32(nn.GroupNorm): + + def forward(self, x): + return super().forward(x.float()).type(x.dtype) + + +def conv_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f'unsupported dimensions: {dims}') + + +def linear(*args, **kwargs): + """ + Create a linear module. + """ + return nn.Linear(*args, **kwargs) + + +def avg_pool_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D average pooling module. + """ + if dims == 1: + return nn.AvgPool1d(*args, **kwargs) + elif dims == 2: + return nn.AvgPool2d(*args, **kwargs) + elif dims == 3: + return nn.AvgPool3d(*args, **kwargs) + raise ValueError(f'unsupported dimensions: {dims}') + + +class HybridConditioner(nn.Module): + + def __init__(self, c_concat_config, c_crossattn_config): + super().__init__() + self.concat_conditioner = instantiate_from_config(c_concat_config) + self.crossattn_conditioner = instantiate_from_config( + c_crossattn_config) + + def forward(self, c_concat, c_crossattn): + c_concat = self.concat_conditioner(c_concat) + c_crossattn = self.crossattn_conditioner(c_crossattn) + return {'c_concat': [c_concat], 'c_crossattn': [c_crossattn]} + + +def noise_like(shape, device, repeat=False): + + def repeat_noise(): + return torch.randn((1, *shape[1:]), + device=device).repeat(shape[0], + *((1, ) * (len(shape) - 1))) + + def noise(): + return torch.randn(shape, device=device) + + return repeat_noise() if repeat else noise() diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/x_transformer.py b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/x_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..8bcd2e61c44a9574c3a11563650b30b17804ec32 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/ldm/x_transformer.py @@ -0,0 +1,680 @@ +"""shout-out to https://github.com/lucidrains/x-transformers/tree/main/x_transformers""" +from collections import namedtuple +from functools import partial +from inspect import isfunction + +import torch +import torch.nn.functional as F +from einops import rearrange, reduce, repeat +from torch import einsum, nn + +# constants + +DEFAULT_DIM_HEAD = 64 + +Intermediates = namedtuple('Intermediates', + ['pre_softmax_attn', 'post_softmax_attn']) + +LayerIntermediates = namedtuple('Intermediates', + ['hiddens', 'attn_intermediates']) + + +class AbsolutePositionalEmbedding(nn.Module): + + def __init__(self, dim, max_seq_len): + super().__init__() + self.emb = nn.Embedding(max_seq_len, dim) + self.init_() + + def init_(self): + nn.init.normal_(self.emb.weight, std=0.02) + + def forward(self, x): + n = torch.arange(x.shape[1], device=x.device) + return self.emb(n)[None, :, :] + + +class FixedPositionalEmbedding(nn.Module): + + def __init__(self, dim): + super().__init__() + inv_freq = 1. / (10000**(torch.arange(0, dim, 2).float() / dim)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, x, seq_dim=1, offset=0): + t = torch.arange( + x.shape[seq_dim], device=x.device).type_as(self.inv_freq) + offset + sinusoid_inp = torch.einsum('i , j -> i j', t, self.inv_freq) + emb = torch.cat((sinusoid_inp.sin(), sinusoid_inp.cos()), dim=-1) + return emb[None, :, :] + + +# helpers + + +def exists(val): + return val is not None + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +def always(val): + + def inner(*args, **kwargs): + return val + + return inner + + +def not_equals(val): + + def inner(x): + return x != val + + return inner + + +def equals(val): + + def inner(x): + return x == val + + return inner + + +def max_neg_value(tensor): + return -torch.finfo(tensor.dtype).max + + +# keyword argument helpers + + +def pick_and_pop(keys, d): + values = list(map(lambda key: d.pop(key), keys)) + return dict(zip(keys, values)) + + +def group_dict_by_key(cond, d): + return_val = [dict(), dict()] + for key in d.keys(): + match = bool(cond(key)) + ind = int(not match) + return_val[ind][key] = d[key] + return (*return_val, ) + + +def string_begins_with(prefix, str): + return str.startswith(prefix) + + +def group_by_key_prefix(prefix, d): + return group_dict_by_key(partial(string_begins_with, prefix), d) + + +def groupby_prefix_and_trim(prefix, d): + kwargs_with_prefix, kwargs = group_dict_by_key( + partial(string_begins_with, prefix), d) + kwargs_without_prefix = dict( + map(lambda x: (x[0][len(prefix):], x[1]), + tuple(kwargs_with_prefix.items()))) + return kwargs_without_prefix, kwargs + + +# classes +class Scale(nn.Module): + + def __init__(self, value, fn): + super().__init__() + self.value = value + self.fn = fn + + def forward(self, x, **kwargs): + x, *rest = self.fn(x, **kwargs) + return (x * self.value, *rest) + + +class Rezero(nn.Module): + + def __init__(self, fn): + super().__init__() + self.fn = fn + self.g = nn.Parameter(torch.zeros(1)) + + def forward(self, x, **kwargs): + x, *rest = self.fn(x, **kwargs) + return (x * self.g, *rest) + + +class ScaleNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.scale = dim**-0.5 + self.eps = eps + self.g = nn.Parameter(torch.ones(1)) + + def forward(self, x): + norm = torch.norm(x, dim=-1, keepdim=True) * self.scale + return x / norm.clamp(min=self.eps) * self.g + + +class RMSNorm(nn.Module): + + def __init__(self, dim, eps=1e-8): + super().__init__() + self.scale = dim**-0.5 + self.eps = eps + self.g = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + norm = torch.norm(x, dim=-1, keepdim=True) * self.scale + return x / norm.clamp(min=self.eps) * self.g + + +class Residual(nn.Module): + + def forward(self, x, residual): + return x + residual + + +class GRUGating(nn.Module): + + def __init__(self, dim): + super().__init__() + self.gru = nn.GRUCell(dim, dim) + + def forward(self, x, residual): + gated_output = self.gru( + rearrange(x, 'b n d -> (b n) d'), + rearrange(residual, 'b n d -> (b n) d')) + + return gated_output.reshape_as(x) + + +# feedforward + + +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +# attention. +class Attention(nn.Module): + + def __init__(self, + dim, + dim_head=DEFAULT_DIM_HEAD, + heads=8, + causal=False, + mask=None, + talking_heads=False, + sparse_topk=None, + use_entmax15=False, + num_mem_kv=0, + dropout=0., + on_attn=False): + super().__init__() + if use_entmax15: + raise NotImplementedError( + 'Check out entmax activation instead of softmax activation!') + self.scale = dim_head**-0.5 + self.heads = heads + self.causal = causal + self.mask = mask + + inner_dim = dim_head * heads + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_k = nn.Linear(dim, inner_dim, bias=False) + self.to_v = nn.Linear(dim, inner_dim, bias=False) + self.dropout = nn.Dropout(dropout) + + # talking heads + self.talking_heads = talking_heads + if talking_heads: + self.pre_softmax_proj = nn.Parameter(torch.randn(heads, heads)) + self.post_softmax_proj = nn.Parameter(torch.randn(heads, heads)) + + # explicit topk sparse attention + self.sparse_topk = sparse_topk + + # entmax + + self.attn_fn = F.softmax + + # add memory key / values + self.num_mem_kv = num_mem_kv + if num_mem_kv > 0: + self.mem_k = nn.Parameter(torch.randn(heads, num_mem_kv, dim_head)) + self.mem_v = nn.Parameter(torch.randn(heads, num_mem_kv, dim_head)) + + # attention on attention + self.attn_on_attn = on_attn + self.to_out = nn.Sequential(nn.Linear( + inner_dim, dim + * 2), nn.GLU()) if on_attn else nn.Linear(inner_dim, dim) + + def forward(self, + x, + context=None, + mask=None, + context_mask=None, + rel_pos=None, + sinusoidal_emb=None, + prev_attn=None, + mem=None): + b, n, _, h, talking_heads, device = *x.shape, self.heads, self.talking_heads, x.device + kv_input = default(context, x) + + q_input = x + k_input = kv_input + v_input = kv_input + + if exists(mem): + k_input = torch.cat((mem, k_input), dim=-2) + v_input = torch.cat((mem, v_input), dim=-2) + + if exists(sinusoidal_emb): + # in shortformer, the query would start at a position offset depending on the past cached memory + offset = k_input.shape[-2] - q_input.shape[-2] + q_input = q_input + sinusoidal_emb(q_input, offset=offset) + k_input = k_input + sinusoidal_emb(k_input) + + q = self.to_q(q_input) + k = self.to_k(k_input) + v = self.to_v(v_input) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), + (q, k, v)) + + input_mask = None + if any(map(exists, (mask, context_mask))): + q_mask = default(mask, lambda: torch.ones( + (b, n), device=device).bool()) + k_mask = q_mask if not exists(context) else context_mask + k_mask = default( + k_mask, lambda: torch.ones( + (b, k.shape[-2]), device=device).bool()) + q_mask = rearrange(q_mask, 'b i -> b () i ()') + k_mask = rearrange(k_mask, 'b j -> b () () j') + input_mask = q_mask * k_mask + + if self.num_mem_kv > 0: + mem_k, mem_v = map(lambda t: repeat(t, 'h n d -> b h n d', b=b), + (self.mem_k, self.mem_v)) + k = torch.cat((mem_k, k), dim=-2) + v = torch.cat((mem_v, v), dim=-2) + if exists(input_mask): + input_mask = F.pad( + input_mask, (self.num_mem_kv, 0), value=True) + + dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale + mask_value = max_neg_value(dots) + + if exists(prev_attn): + dots = dots + prev_attn + + pre_softmax_attn = dots + + if talking_heads: + dots = einsum('b h i j, h k -> b k i j', dots, + self.pre_softmax_proj).contiguous() + + if exists(rel_pos): + dots = rel_pos(dots) + + if exists(input_mask): + dots.masked_fill_(~input_mask, mask_value) + del input_mask + + if self.causal: + i, j = dots.shape[-2:] + r = torch.arange(i, device=device) + mask = rearrange(r, 'i -> () () i ()') < rearrange( + r, 'j -> () () () j') + mask = F.pad(mask, (j - i, 0), value=False) + dots.masked_fill_(mask, mask_value) + del mask + + if exists(self.sparse_topk) and self.sparse_topk < dots.shape[-1]: + top, _ = dots.topk(self.sparse_topk, dim=-1) + vk = top[..., -1].unsqueeze(-1).expand_as(dots) + mask = dots < vk + dots.masked_fill_(mask, mask_value) + del mask + + attn = self.attn_fn(dots, dim=-1) + post_softmax_attn = attn + + attn = self.dropout(attn) + + if talking_heads: + attn = einsum('b h i j, h k -> b k i j', attn, + self.post_softmax_proj).contiguous() + + out = einsum('b h i j, b h j d -> b h i d', attn, v) + out = rearrange(out, 'b h n d -> b n (h d)') + + intermediates = Intermediates( + pre_softmax_attn=pre_softmax_attn, + post_softmax_attn=post_softmax_attn) + + return self.to_out(out), intermediates + + +class AttentionLayers(nn.Module): + + def __init__(self, + dim, + depth, + heads=8, + causal=False, + cross_attend=False, + only_cross=False, + use_scalenorm=False, + use_rmsnorm=False, + use_rezero=False, + rel_pos_num_buckets=32, + rel_pos_max_distance=128, + position_infused_attn=False, + custom_layers=None, + sandwich_coef=None, + par_ratio=None, + residual_attn=False, + cross_residual_attn=False, + macaron=False, + pre_norm=True, + gate_residual=False, + **kwargs): + super().__init__() + ff_kwargs, kwargs = groupby_prefix_and_trim('ff_', kwargs) + attn_kwargs, _ = groupby_prefix_and_trim('attn_', kwargs) + + attn_kwargs.get('dim_head', DEFAULT_DIM_HEAD) + + self.dim = dim + self.depth = depth + self.layers = nn.ModuleList([]) + + self.has_pos_emb = position_infused_attn + self.pia_pos_emb = FixedPositionalEmbedding( + dim) if position_infused_attn else None + self.rotary_pos_emb = always(None) + + assert rel_pos_num_buckets <= rel_pos_max_distance, 'error' + self.rel_pos = None + + self.pre_norm = pre_norm + + self.residual_attn = residual_attn + self.cross_residual_attn = cross_residual_attn + + norm_class = ScaleNorm if use_scalenorm else nn.LayerNorm + norm_class = RMSNorm if use_rmsnorm else norm_class + norm_fn = partial(norm_class, dim) + + norm_fn = nn.Identity if use_rezero else norm_fn + branch_fn = Rezero if use_rezero else None + + if cross_attend and not only_cross: + default_block = ('a', 'c', 'f') + elif cross_attend and only_cross: + default_block = ('c', 'f') + else: + default_block = ('a', 'f') + + if macaron: + default_block = ('f', ) + default_block + + if exists(custom_layers): + layer_types = custom_layers + elif exists(par_ratio): + par_depth = depth * len(default_block) + assert 1 < par_ratio <= par_depth, 'par ratio out of range' + default_block = tuple(filter(not_equals('f'), default_block)) + par_attn = par_depth // par_ratio + depth_cut = par_depth * 2 // 3 # 2 / 3 attention layer cutoff suggested by PAR paper + par_width = (depth_cut + depth_cut // par_attn) // par_attn + assert len( + default_block + ) <= par_width, 'default block is too large for par_ratio' + par_block = default_block + ('f', ) * ( + par_width - len(default_block)) + par_head = par_block * par_attn + layer_types = par_head + ('f', ) * (par_depth - len(par_head)) + elif exists(sandwich_coef): + assert sandwich_coef > 0 and sandwich_coef <= depth, 'sandwich coefficient should be less than the depth' + layer_types = ('a', ) * sandwich_coef + default_block * ( + depth - sandwich_coef) + ('f', ) * sandwich_coef + else: + layer_types = default_block * depth + + self.layer_types = layer_types + self.num_attn_layers = len(list(filter(equals('a'), layer_types))) + + for layer_type in self.layer_types: + if layer_type == 'a': + layer = Attention( + dim, heads=heads, causal=causal, **attn_kwargs) + elif layer_type == 'c': + layer = Attention(dim, heads=heads, **attn_kwargs) + elif layer_type == 'f': + layer = FeedForward(dim, **ff_kwargs) + layer = layer if not macaron else Scale(0.5, layer) + else: + raise Exception(f'invalid layer type {layer_type}') + + if isinstance(layer, Attention) and exists(branch_fn): + layer = branch_fn(layer) + + if gate_residual: + residual_fn = GRUGating(dim) + else: + residual_fn = Residual() + + self.layers.append(nn.ModuleList([norm_fn(), layer, residual_fn])) + + def forward(self, + x, + context=None, + mask=None, + context_mask=None, + mems=None, + return_hiddens=False): + hiddens = [] + intermediates = [] + prev_attn = None + prev_cross_attn = None + + mems = mems.copy() if exists(mems) else [None] * self.num_attn_layers + + for ind, (layer_type, (norm, block, residual_fn)) in enumerate( + zip(self.layer_types, self.layers)): + is_last = ind == (len(self.layers) - 1) + + if layer_type == 'a': + hiddens.append(x) + layer_mem = mems.pop(0) + + residual = x + + if self.pre_norm: + x = norm(x) + + if layer_type == 'a': + out, inter = block( + x, + mask=mask, + sinusoidal_emb=self.pia_pos_emb, + rel_pos=self.rel_pos, + prev_attn=prev_attn, + mem=layer_mem) + elif layer_type == 'c': + out, inter = block( + x, + context=context, + mask=mask, + context_mask=context_mask, + prev_attn=prev_cross_attn) + elif layer_type == 'f': + out = block(x) + + x = residual_fn(out, residual) + + if layer_type in ('a', 'c'): + intermediates.append(inter) + + if layer_type == 'a' and self.residual_attn: + prev_attn = inter.pre_softmax_attn + elif layer_type == 'c' and self.cross_residual_attn: + prev_cross_attn = inter.pre_softmax_attn + + if not self.pre_norm and not is_last: + x = norm(x) + + if return_hiddens: + intermediates = LayerIntermediates( + hiddens=hiddens, attn_intermediates=intermediates) + + return x, intermediates + + return x + + +class Encoder(AttentionLayers): + + def __init__(self, **kwargs): + assert 'causal' not in kwargs, 'cannot set causality on encoder' + super().__init__(causal=False, **kwargs) + + +class TransformerWrapper(nn.Module): + + def __init__(self, + *, + num_tokens, + max_seq_len, + attn_layers, + emb_dim=None, + max_mem_len=0., + emb_dropout=0., + num_memory_tokens=None, + tie_embedding=False, + use_pos_emb=True): + super().__init__() + assert isinstance( + attn_layers, AttentionLayers + ), 'attention layers must be one of Encoder or Decoder' + + dim = attn_layers.dim + emb_dim = default(emb_dim, dim) + + self.max_seq_len = max_seq_len + self.max_mem_len = max_mem_len + self.num_tokens = num_tokens + + self.token_emb = nn.Embedding(num_tokens, emb_dim) + self.pos_emb = AbsolutePositionalEmbedding(emb_dim, max_seq_len) if ( + use_pos_emb and not attn_layers.has_pos_emb) else always(0) + self.emb_dropout = nn.Dropout(emb_dropout) + + self.project_emb = nn.Linear(emb_dim, + dim) if emb_dim != dim else nn.Identity() + self.attn_layers = attn_layers + self.norm = nn.LayerNorm(dim) + + self.init_() + + self.to_logits = nn.Linear( + dim, num_tokens + ) if not tie_embedding else lambda t: t @ self.token_emb.weight.t() + + # memory tokens (like [cls]) from Memory Transformers paper + num_memory_tokens = default(num_memory_tokens, 0) + self.num_memory_tokens = num_memory_tokens + if num_memory_tokens > 0: + self.memory_tokens = nn.Parameter( + torch.randn(num_memory_tokens, dim)) + + # let funnel encoder know number of memory tokens, if specified + if hasattr(attn_layers, 'num_memory_tokens'): + attn_layers.num_memory_tokens = num_memory_tokens + + def init_(self): + nn.init.normal_(self.token_emb.weight, std=0.02) + + def forward(self, + x, + return_embeddings=False, + mask=None, + return_mems=False, + return_attn=False, + mems=None, + **kwargs): + b, num_mem = *x.shape[0], self.num_memory_tokens + x = self.token_emb(x) + x += self.pos_emb(x) + x = self.emb_dropout(x) + + x = self.project_emb(x) + + if num_mem > 0: + mem = repeat(self.memory_tokens, 'n d -> b n d', b=b) + x = torch.cat((mem, x), dim=1) + + # auto-handle masking after appending memory tokens + if exists(mask): + mask = F.pad(mask, (num_mem, 0), value=True) + + x, intermediates = self.attn_layers( + x, mask=mask, mems=mems, return_hiddens=True, **kwargs) + x = self.norm(x) + + mem, x = x[:, :num_mem], x[:, num_mem:] + + out = self.to_logits(x) if not return_embeddings else x + + if return_mems: + hiddens = intermediates.hiddens + new_mems = list( + map(lambda pair: torch.cat(pair, dim=-2), zip( + mems, hiddens))) if exists(mems) else hiddens + new_mems = list( + map(lambda t: t[..., -self.max_mem_len:, :].detach(), + new_mems)) + return out, new_mems + + if return_attn: + attn_maps = list( + map(lambda t: t.post_softmax_attn, + intermediates.attn_intermediates)) + return out, attn_maps + + return out diff --git a/runtime/Lib/site_packages/models/cv/image_view_transform/util.py b/runtime/Lib/site_packages/models/cv/image_view_transform/util.py new file mode 100644 index 0000000000000000000000000000000000000000..269f62658c10b69678d1fc638813d7044c71fb29 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/image_view_transform/util.py @@ -0,0 +1,297 @@ +import importlib +import os +import time +from inspect import isfunction + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import PIL +import torch +import torchvision +from PIL import Image, ImageDraw, ImageFont +from torch import optim + + +def pil_rectangle_crop(im): + width, height = im.size # Get dimensions + + if width <= height: + left = 0 + right = width + top = (height - width) / 2 + bottom = (height + width) / 2 + else: + + top = 0 + bottom = height + left = (width - height) / 2 + bottom = (width + height) / 2 + + # Crop the center of the image + im = im.crop((left, top, right, bottom)) + return im + + +def add_margin(pil_img, color, size=256): + width, height = pil_img.size + result = Image.new(pil_img.mode, (size, size), color) + result.paste(pil_img, ((size - width) // 2, (size - height) // 2)) + return result + + +# def create_carvekit_interface(): +# # Check doc strings for more information +# interface = HiInterface( +# object_type='object', # Can be "object" or "hairs-like". +# batch_size_seg=5, +# batch_size_matting=1, +# device='cuda' if torch.cuda.is_available() else 'cpu', +# seg_mask_size=640, # Use 640 for Tracer B7 and 320 for U2Net +# matting_mask_size=2048, +# trimap_prob_threshold=231, +# trimap_dilation=30, +# trimap_erosion_iters=5, +# fp16=False) + +# return interface + + +def load_and_preprocess(interface, input_im): + ''' + :param input_im (PIL Image). + :return image (H, W, 3) array in [0, 1]. + ''' + # See https://github.com/Ir1d/image-background-remove-tool + image = input_im.convert('RGB') + + image_without_background = interface([image])[0] + image_without_background = np.array(image_without_background) + est_seg = image_without_background > 127 + image = np.array(image) + foreground = est_seg[:, :, -1].astype(np.bool_) + image[~foreground] = [255., 255., 255.] + x, y, w, h = cv2.boundingRect(foreground.astype(np.uint8)) + image = image[y:y + h, x:x + w, :] + image = PIL.Image.fromarray(np.array(image)) + + # resize image such that long edge is 512 + image.thumbnail([200, 200], Image.Resampling.LANCZOS) + image = add_margin(image, (255, 255, 255), size=256) + image = np.array(image) + + return image + + +def log_txt_as_img(wh, xc, size=10): + # wh a tuple of (width, height) + # xc a list of captions to plot + b = len(xc) + txts = list() + for bi in range(b): + txt = Image.new('RGB', wh, color='white') + draw = ImageDraw.Draw(txt) + font = ImageFont.truetype('data/DejaVuSans.ttf', size=size) + nc = int(40 * (wh[0] / 256)) + lines = '\n'.join(xc[bi][start:start + nc] + for start in range(0, len(xc[bi]), nc)) + + try: + draw.text((0, 0), lines, fill='black', font=font) + except UnicodeEncodeError: + print('Cant encode string for logging. Skipping.') + + txt = np.array(txt).transpose(2, 0, 1) / 127.5 - 1.0 + txts.append(txt) + txts = np.stack(txts) + txts = torch.tensor(txts) + return txts + + +def ismap(x): + if not isinstance(x, torch.Tensor): + return False + return (len(x.shape) == 4) and (x.shape[1] > 3) + + +def isimage(x): + if not isinstance(x, torch.Tensor): + return False + return (len(x.shape) == 4) and (x.shape[1] == 3 or x.shape[1] == 1) + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +def mean_flat(tensor): + """ + https://github.com/openai/guided-diffusion/blob/27c20a8fab9cb472df5d6bdd6c8d11c8f430b924/guided_diffusion/nn.py#L86 + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +def count_params(model, verbose=False): + total_params = sum(p.numel() for p in model.parameters()) + if verbose: + print( + f'{model.__class__.__name__} has {total_params*1.e-6:.2f} M params.' + ) + return total_params + + +def instantiate_from_config(config): + if 'target' not in config: + if config == '__is_first_stage__': + return None + elif config == '__is_unconditional__': + return None + raise KeyError('Expected key `target` to instantiate.') + return get_obj_from_str(config['target'])(**config.get('params', dict())) + + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit('.', 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) + + +class AdamWwithEMAandWings(optim.Optimizer): + # credit to https://gist.github.com/crowsonkb/65f7265353f403714fce3b2595e0b298 + def __init__(self, + params, + lr=1.e-3, + betas=(0.9, 0.999), + eps=1.e-8, + weight_decay=1.e-2, + amsgrad=False, + ema_decay=0.9999, + ema_power=1., + param_names=()): + + if not 0.0 <= lr: + raise ValueError('Invalid learning rate: {}'.format(lr)) + if not 0.0 <= eps: + raise ValueError('Invalid epsilon value: {}'.format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError('Invalid beta parameter at index 0: {}'.format( + betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError('Invalid beta parameter at index 1: {}'.format( + betas[1])) + if not 0.0 <= weight_decay: + raise ValueError( + 'Invalid weight_decay value: {}'.format(weight_decay)) + if not 0.0 <= ema_decay <= 1.0: + raise ValueError('Invalid ema_decay value: {}'.format(ema_decay)) + defaults = dict( + lr=lr, + betas=betas, + eps=eps, + weight_decay=weight_decay, + amsgrad=amsgrad, + ema_decay=ema_decay, + ema_power=ema_power, + param_names=param_names) + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault('amsgrad', False) + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step. + Args: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad = [] + grads = [] + exp_avgs = [] + exp_avg_sqs = [] + ema_params_with_grad = [] + max_exp_avg_sqs = [] + state_steps = [] + amsgrad = group['amsgrad'] + beta1, beta2 = group['betas'] + ema_decay = group['ema_decay'] + ema_power = group['ema_power'] + + for p in group['params']: + if p.grad is None: + continue + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError( + 'AdamW does not support sparse gradients') + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like( + p, memory_format=torch.preserve_format) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like( + p, memory_format=torch.preserve_format) + if amsgrad: + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_exp_avg_sq'] = torch.zeros_like( + p, memory_format=torch.preserve_format) + # Exponential moving average of parameter values + state['param_exp_avg'] = p.detach().float().clone() + + exp_avgs.append(state['exp_avg']) + exp_avg_sqs.append(state['exp_avg_sq']) + ema_params_with_grad.append(state['param_exp_avg']) + + if amsgrad: + max_exp_avg_sqs.append(state['max_exp_avg_sq']) + + # update the steps for each param group update + state['step'] += 1 + # record the step after step update + state_steps.append(state['step']) + + optim._functional.adamw( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + amsgrad=amsgrad, + beta1=beta1, + beta2=beta2, + lr=group['lr'], + weight_decay=group['weight_decay'], + eps=group['eps'], + maximize=False) + + cur_ema_decay = min(ema_decay, 1 - state['step']**-ema_power) + for param, ema_param in zip(params_with_grad, + ema_params_with_grad): + ema_param.mul_(cur_ema_decay).add_( + param.float(), alpha=1 - cur_ema_decay) + + return loss diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b1f83a8777ee63f129437258fb937f8b6677e4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .panovit import LayoutEstimation + +else: + _import_structure = { + 'panovit': ['LayoutEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/__init__.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed6c66a754c51f5ffbda6857b7e66e50e2916b83 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .resnet_DA import ResnetDA +from .vit_horizon_pry_image import ViTHorizonPryImage diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/resnet_DA.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/resnet_DA.py new file mode 100644 index 0000000000000000000000000000000000000000..26bfeb5856658d993640e420b13bff67336da68f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/resnet_DA.py @@ -0,0 +1,112 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torchvision.models as models + +from ..utils import StripPooling + + +class SPHead(nn.Module): + + def __init__(self, in_channels, out_channels, norm_layer): + super(SPHead, self).__init__() + inter_channels = in_channels // 2 + self.trans_layer = nn.Sequential( + nn.Conv2d(in_channels, inter_channels, 1, 1, 0, bias=False), + norm_layer(inter_channels), nn.ReLU(True)) + self.strip_pool1 = StripPooling(inter_channels, (20, 12), norm_layer, + up_kwargs) + self.strip_pool2 = StripPooling(inter_channels, (20, 12), norm_layer, + up_kwargs) + self.score_layer = nn.Sequential( + nn.Conv2d( + inter_channels, inter_channels // 2, 3, 1, 1, bias=False), + norm_layer(inter_channels // 2), nn.ReLU(True), + nn.Dropout2d(0.1, False), + nn.Conv2d(inter_channels // 2, out_channels, 1)) + + def forward(self, x): + x = self.trans_layer(x) + x = self.strip_pool1(x) + x = self.strip_pool2(x) + x = self.score_layer(x) + return x + + +class ResnetDA(nn.Module): + + def __init__(self, + backbone='resnet50', + coco='', + input_extra=0, + input_height=512): + super(ResnetDA, self).__init__() + self.encoder = getattr(models, backbone)(pretrained=True) + del self.encoder.fc, self.encoder.avgpool + if coco: + coco_pretrain = getattr(models.segmentation, + coco)(pretrained=True).backbone + self.encoder.load_state_dict(coco_pretrain.state_dict()) + self.out_channels = [256, 512, 1024, 2048] + self.feat_heights = [input_height // 4 // (2**i) for i in range(4)] + if int(backbone[6:]) < 50: + self.out_channels = [_ // 4 for _ in self.out_channels] + + # Patch for extra input channel + if input_extra > 0: + ori_conv1 = self.encoder.conv1 + new_conv1 = nn.Conv2d( + 3 + input_extra, + ori_conv1.out_channels, + kernel_size=ori_conv1.kernel_size, + stride=ori_conv1.stride, + padding=ori_conv1.padding, + bias=ori_conv1.bias) + with torch.no_grad(): + for i in range(0, 3 + input_extra, 3): + n = new_conv1.weight[:, i:i + 3].shape[1] + new_conv1.weight[:, i:i + n] = ori_conv1.weight[:, :n] + self.encoder.conv1 = new_conv1 + + # Prepare for pre/pose down height filtering + self.pre_down = None + self.post_down = None + # SPhead + self.strip_pool1 = StripPooling( + self.out_channels[0], [128, 256], (20, 12), + norm_layer=nn.BatchNorm2d) + self.strip_pool2 = StripPooling( + self.out_channels[1], [64, 128], (20, 12), + norm_layer=nn.BatchNorm2d) + self.strip_pool3 = StripPooling( + self.out_channels[2], [32, 64], (20, 12), + norm_layer=nn.BatchNorm2d) + self.strip_pool4 = StripPooling( + self.out_channels[3], [16, 32], (10, 12), + norm_layer=nn.BatchNorm2d) + + def forward(self, x): + features = [] + x = self.encoder.conv1(x) + x = self.encoder.bn1(x) + x = self.encoder.relu(x) + x = self.encoder.maxpool(x) + + if self.pre_down is not None: + x = self.pre_down(x) + x = self.encoder.layer1(x) + x = self.strip_pool1(x) + if self.post_down is not None: + x = self.post_down(x) + features.append(x) # 1/4 + x = self.encoder.layer2(x) + x = self.strip_pool2(x) + features.append(x) # 1/8 + x = self.encoder.layer3(x) + x = self.strip_pool3(x) + features.append(x) # 1/16 + x = self.encoder.layer4(x) + x = self.strip_pool4(x) + features.append(x) # 1/32 + return features diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/vit_horizon_pry_image.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/vit_horizon_pry_image.py new file mode 100644 index 0000000000000000000000000000000000000000..7ce22feebcdf7cc297d72f835a547b35fdd9b1b8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/backbone/vit_horizon_pry_image.py @@ -0,0 +1,181 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import timm +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + + def __init__(self, + img_size=224, + patch_size=16, + stride_size=16, + padding=[0, 1], + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True): + super().__init__() + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], + img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=stride_size, + padding=[0, 1]) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + B, C, H, W = x.shape + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +class ViTHorizonPryImage(nn.Module): + + def __init__(self, backbone, fourier, embedding): + super(ViTHorizonPryImage, self).__init__() + embed_dim = 192 + F_lens = [256, 128, 64, 32, 512] + position_encode = np.sum(np.array(F_lens)) + self.embedding = embedding + if fourier is False: + in_chans = 3 + else: + in_chans = 9 + self.pre_image = PatchEmbed([512, 1024], [32, 32], [32, 32], + in_chans=in_chans, + embed_dim=embed_dim) + self.pre_net = nn.ModuleList([ + PatchEmbed([128, 256], [128, 3], [128, 1], + padding=[0, 1], + in_chans=64, + embed_dim=embed_dim), + PatchEmbed([64, 128], [64, 3], [64, 1], + padding=[0, 1], + in_chans=128, + embed_dim=embed_dim), + PatchEmbed([32, 64], [32, 3], [32, 1], + padding=[0, 1], + in_chans=256, + embed_dim=embed_dim), + PatchEmbed([16, 32], [16, 3], [16, 1], + padding=[0, 1], + in_chans=512, + embed_dim=embed_dim) + ]) + + self.encoder = timm.create_model(backbone, pretrained=False) + del self.encoder.patch_embed, self.encoder.head + + self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter( + torch.zeros(1, position_encode + 1, embed_dim)) + + def EfficientConvCompressH(in_c, out_c, down_h): + + net1 = nn.Sequential( + nn.Conv2d(in_c, out_c, 3, padding=1, bias=False), + nn.BatchNorm2d(out_c), nn.ReLU(inplace=True)) + + net2 = nn.Sequential( + nn.Conv2d(out_c, out_c, 3, padding=1, bias=False), + nn.BatchNorm2d(out_c), + nn.ReLU(inplace=True), + nn.Conv2d(out_c, out_c, (down_h, 1), groups=out_c, bias=False), + ) + return net1, net2 + + self.ECH1, self.ECH2 = EfficientConvCompressH(embed_dim, 2 * embed_dim, + 4) + self.scales = [1, 2, 4, 8] + # self.L = nn.Linear(454,1024) + if self.embedding == 'sin': + import math + max_len, d_model = position_encode, embed_dim + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = \ + torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + self.register_buffer('pos', pe.T[None].contiguous()) + + elif self.embedding == 'recurrent': + import math + d_model = embed_dim + index = torch.randint(0, F_lens[0], [1]) + for i, max_len in enumerate(F_lens): + pe = torch.zeros(max_len, d_model) + position = torch.arange( + 0, max_len, dtype=torch.float).unsqueeze(1) + if i < len(F_lens) - 1: + index = torch.div(index, 2, rounding_mode='floor')**i + position = (index + position) % max_len + position = position + np.sum(np.array(F_lens[:i])) + div_term = \ + torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + if i == 0: + pe_re = pe + else: + pe_re = torch.cat((pe_re, pe), dim=0) + self.register_buffer('pos', pe_re.T[None].contiguous()) + + def forward(self, img, x): + for i, feat in enumerate(x): + pre = self.pre_net[i](feat) + if i == 0: + inputs = pre + else: + inputs = torch.cat((inputs, pre), 1) + pre = self.pre_image(img) + inputs = torch.cat((inputs, pre), 1) + if self.embedding == 'learnable': + inputs = torch.cat( + (self.dist_token.expand(inputs.shape[0], -1, -1), inputs), + dim=1) + inputs = inputs + self.pos_embed + if self.embedding == 'sin': + inputs = inputs + self.pos.permute(0, 2, 1) + if self.embedding == 'recurrent': + inputs = inputs + self.pos.permute(0, 2, 1) + + x = self.encoder.pos_drop(inputs) + for i in range(12): + x = self.encoder.blocks[i](x) + x = x.permute(0, 2, 1) + a1 = x[:, :, :256].reshape(x.shape[0], x.shape[1], 1, 256) + a1 = F.interpolate( + a1, scale_factor=(1, 4), mode='bilinear', align_corners=False) + a2 = x[:, :, 256:384].reshape(x.shape[0], x.shape[1], 1, 128) + a2 = F.interpolate( + a2, scale_factor=(1, 8), mode='bilinear', align_corners=False) + a3 = x[:, :, 384:448].reshape(x.shape[0], x.shape[1], 1, 64) + a3 = F.interpolate( + a3, scale_factor=(1, 16), mode='bilinear', align_corners=False) + a4 = x[:, :, 448:480].reshape(x.shape[0], x.shape[1], 1, 32) + a4 = F.interpolate( + a4, scale_factor=(1, 32), mode='bilinear', align_corners=False) + a = torch.cat((a1, a2, a3, a4), dim=2) + a = self.ECH1(a) + a = self.ECH2(a).flatten(2) + + feat = {} + feat['1D'] = a + return feat diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/__init__.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/fourier.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/fourier.py new file mode 100644 index 0000000000000000000000000000000000000000..92d6a7b1119952d5bcb203b9ede71f0048823764 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/fourier.py @@ -0,0 +1,99 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +from PIL import Image +from scipy.fft import fft2, ifft2 + +AL = 1 +pas = 50 +highpas = 250 +index = 550 + + +def gene_mask(f, angle, horizon): + alpha = AL + an = angle / 360 * (2 * np.pi) + W, H = f.shape[:2] + X = np.arange(0, W) - 255 + Y = np.arange(0, H) - 512 + x, y = np.meshgrid(Y, X) + dis = np.sqrt(x**2 + y**2) + dis[dis < pas] = 0 + dis[dis >= highpas] = 0 + dis[dis != 0] = 1 + + maskband = np.zeros(f.shape) + maskband[:, index:index + 200] = 1 + + angles = np.abs(np.arctan(y / (x + 0.00001))) + angles[angles > an] = 1 + angles[angles < an] = 0 + if not horizon: + mask = angles + else: + mask = 1 - angles + return alpha * mask * maskband + + +def normal(rgb_recons, edge=False): + rgb_recons = rgb_recons.astype(int) + rgb_recons = (rgb_recons - np.min(rgb_recons)) / ( + np.max(rgb_recons) - np.min(rgb_recons)) * 255 + return rgb_recons + + +def fourier_gray(img): + img = Image.fromarray(img.astype(np.uint8)).convert('L') + img = np.array(img) + im = img + x = im * 1 + y = fft2(x, axes=(0, 1)) + shift2center = np.fft.fftshift(y) + + mask = np.zeros(y.shape) + mask_angle = gene_mask(mask, 25, horizon=False) + mask = mask_angle + crop1 = shift2center * mask + iresult = np.fft.ifftshift(crop1) + recons = np.abs(ifft2(iresult, axes=(0, 1))) + rgb_recons = recons + + mask = np.zeros(y.shape) + mask_angle = gene_mask(mask, 20, horizon=True) + mask = mask_angle + crop1 = shift2center * mask + iresult = np.fft.ifftshift(crop1) + recons = np.abs(ifft2(iresult, axes=(0, 1))) + rgb_reconsH = recons + + rgb_recons = normal(rgb_recons, True) + + rgb_reconsH = normal(rgb_reconsH, True) + + rgb_reconsA = rgb_reconsH * 0 + + x = np.concatenate((rgb_recons[:, :, None], rgb_reconsH[:, :, None], + rgb_reconsA[:, :, None]), 2) + + return x + + +def fourier(img): + rgb_recons = np.zeros(img.shape) + index = 520 + for k in range(3): + im = img[:, :, k:k + 1] + x = im * 1 + y = fft2(x) + shift2center = np.fft.fftshift(y) + + mask = np.zeros(y.shape) + mask[:, index:index + 200, :] = 1 + + crop1 = shift2center * mask + iresult = np.fft.ifftshift(crop1) + recons = np.abs(ifft2(iresult)) + rgb_recons[:, :, k:k + 1] = recons + + rgb_recons = (rgb_recons - np.min(rgb_recons)) / ( + np.max(rgb_recons) - np.min(rgb_recons)) + return rgb_recons diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/panostretch.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/panostretch.py new file mode 100644 index 0000000000000000000000000000000000000000..a3b3b0b32e4b3f7b49ffb48e251f2773e452934e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/panostretch.py @@ -0,0 +1,114 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import functools + +import numpy as np +from scipy.ndimage import map_coordinates + + +def uv_meshgrid(w, h): + uv = np.stack(np.meshgrid(range(w), range(h)), axis=-1) + uv = uv.astype(np.float64) + uv[..., 0] = ((uv[..., 0] + 0.5) / w - 0.5) * 2 * np.pi + uv[..., 1] = ((uv[..., 1] + 0.5) / h - 0.5) * np.pi + return uv + + +@functools.lru_cache() +def _uv_tri(w, h): + uv = uv_meshgrid(w, h) + sin_u = np.sin(uv[..., 0]) + cos_u = np.cos(uv[..., 0]) + tan_v = np.tan(uv[..., 1]) + return sin_u, cos_u, tan_v + + +def uv_tri(w, h): + sin_u, cos_u, tan_v = _uv_tri(w, h) + return sin_u.copy(), cos_u.copy(), tan_v.copy() + + +def coorx2u(x, w=1024): + return ((x + 0.5) / w - 0.5) * 2 * np.pi + + +def coory2v(y, h=512): + return ((y + 0.5) / h - 0.5) * np.pi + + +def u2coorx(u, w=1024): + return (u / (2 * np.pi) + 0.5) * w - 0.5 + + +def v2coory(v, h=512): + return (v / np.pi + 0.5) * h - 0.5 + + +def uv2xy(u, v, z=-50): + c = z / np.tan(v) + x = c * np.cos(u) + y = c * np.sin(u) + return x, y + + +def pano_connect_points(p1, p2, z=-50, w=1024, h=512): + if p1[0] == p2[0]: + return np.array([p1, p2], np.float32) + + u1 = coorx2u(p1[0], w) + v1 = coory2v(p1[1], h) + u2 = coorx2u(p2[0], w) + v2 = coory2v(p2[1], h) + + x1, y1 = uv2xy(u1, v1, z) + x2, y2 = uv2xy(u2, v2, z) + + if abs(p1[0] - p2[0]) < w / 2: + pstart = np.ceil(min(p1[0], p2[0])) + pend = np.floor(max(p1[0], p2[0])) + else: + pstart = np.ceil(max(p1[0], p2[0])) + pend = np.floor(min(p1[0], p2[0]) + w) + coorxs = (np.arange(pstart, pend + 1) % w).astype(np.float64) + vx = x2 - x1 + vy = y2 - y1 + us = coorx2u(coorxs, w) + ps = (np.tan(us) * x1 - y1) / (vy - np.tan(us) * vx) + cs = np.sqrt((x1 + ps * vx)**2 + (y1 + ps * vy)**2) + vs = np.arctan2(z, cs) + coorys = v2coory(vs, h) + + return np.stack([coorxs, coorys], axis=-1) + + +def visualize_pano_stretch(stretched_img, stretched_cor, title): + thikness = 2 + color = (0, 255, 0) + for i in range(4): + xys = pano_connect_points( + stretched_cor[i * 2], stretched_cor[(i * 2 + 2) % 8], z=-50) + xys = xys.astype(int) + blue_split = np.where((xys[1:, 0] - xys[:-1, 0]) < 0)[0] + if len(blue_split) == 0: + cv2.polylines(stretched_img, [xys], False, color, 2) + else: + t = blue_split[0] + 1 + cv2.polylines(stretched_img, [xys[:t]], False, color, thikness) + cv2.polylines(stretched_img, [xys[t:]], False, color, thikness) + + for i in range(4): + xys = pano_connect_points( + stretched_cor[i * 2 + 1], stretched_cor[(i * 2 + 3) % 8], z=50) + xys = xys.astype(int) + blue_split = np.where((xys[1:, 0] - xys[:-1, 0]) < 0)[0] + if len(blue_split) == 0: + cv2.polylines(stretched_img, [xys], False, color, 2) + else: + t = blue_split[0] + 1 + cv2.polylines(stretched_img, [xys[:t]], False, color, thikness) + cv2.polylines(stretched_img, [xys[t:]], False, color, thikness) + + cv2.putText(stretched_img, title, (25, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, + (0, 0, 0), 2, cv2.LINE_AA) + + return stretched_img.astype(np.uint8) diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/post_proc.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/post_proc.py new file mode 100644 index 0000000000000000000000000000000000000000..823d5af5a880294dffb7da38fe509534f11a72b8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/misc/post_proc.py @@ -0,0 +1,479 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +from scipy.ndimage import map_coordinates +from scipy.spatial.distance import pdist, squareform +from sklearn.decomposition import PCA + +PI = float(np.pi) + + +def coorx2u(x, w=1024): + return ((x + 0.5) / w - 0.5) * 2 * np.pi + + +def coory2v(y, h=512): + return ((y + 0.5) / h - 0.5) * np.pi + + +def u2coorx(u, w=1024): + return (u / (2 * np.pi) + 0.5) * w - 0.5 + + +def v2coory(v, h=512): + return (v / np.pi + 0.5) * h - 0.5 + + +def uv2xy(u, v, z=-50): + c = z / np.tan(v) + x = c * np.cos(u) + y = c * np.sin(u) + return x, y + + +def pano_connect_points(p1, p2, z=-50, w=1024, h=512): + if p1[0] == p2[0]: + return np.array([p1, p2], np.float32) + + u1 = coorx2u(p1[0], w) + v1 = coory2v(p1[1], h) + u2 = coorx2u(p2[0], w) + v2 = coory2v(p2[1], h) + + x1, y1 = uv2xy(u1, v1, z) + x2, y2 = uv2xy(u2, v2, z) + + if abs(p1[0] - p2[0]) < w / 2: + pstart = np.ceil(min(p1[0], p2[0])) + pend = np.floor(max(p1[0], p2[0])) + else: + pstart = np.ceil(max(p1[0], p2[0])) + pend = np.floor(min(p1[0], p2[0]) + w) + coorxs = (np.arange(pstart, pend + 1) % w).astype(np.float64) + vx = x2 - x1 + vy = y2 - y1 + us = coorx2u(coorxs, w) + ps = (np.tan(us) * x1 - y1) / (vy - np.tan(us) * vx) + cs = np.sqrt((x1 + ps * vx)**2 + (y1 + ps * vy)**2) + vs = np.arctan2(z, cs) + coorys = v2coory(vs, h) + + return np.stack([coorxs, coorys], axis=-1) + + +def sort_xy_filter_unique(xs, ys, y_small_first=True): + xs, ys = np.array(xs), np.array(ys) + idx_sort = np.argsort(xs + ys / ys.max() * (int(y_small_first) * 2 - 1)) + xs, ys = xs[idx_sort], ys[idx_sort] + _, idx_unique = np.unique(xs, return_index=True) + xs, ys = xs[idx_unique], ys[idx_unique] + assert np.all(np.diff(xs) > 0) + return xs, ys + + +def cor_2_1d(cor, H, W): + bon_ceil_x, bon_ceil_y = [], [] + bon_floor_x, bon_floor_y = [], [] + n_cor = len(cor) + for i in range(n_cor // 2): + xys = pano_connect_points( + cor[i * 2], cor[(i * 2 + 2) % n_cor], z=-50, w=W, h=H) + bon_ceil_x.extend(xys[:, 0]) + bon_ceil_y.extend(xys[:, 1]) + for i in range(n_cor // 2): + xys = pano_connect_points( + cor[i * 2 + 1], cor[(i * 2 + 3) % n_cor], z=50, w=W, h=H) + bon_floor_x.extend(xys[:, 0]) + bon_floor_y.extend(xys[:, 1]) + bon_ceil_x, bon_ceil_y = sort_xy_filter_unique( + bon_ceil_x, bon_ceil_y, y_small_first=True) + bon_floor_x, bon_floor_y = sort_xy_filter_unique( + bon_floor_x, bon_floor_y, y_small_first=False) + bon = np.zeros((2, W)) + bon[0] = np.interp(np.arange(W), bon_ceil_x, bon_ceil_y, period=W) + bon[1] = np.interp(np.arange(W), bon_floor_x, bon_floor_y, period=W) + bon = ((bon + 0.5) / H - 0.5) * np.pi + return bon + + +def fuv2img(fuv, coorW=1024, floorW=1024, floorH=512): + floor_plane_x, floor_plane_y = np.meshgrid(range(floorW), range(floorH)) + floor_plane_x, floor_plane_y = -(floor_plane_y + - floorH / 2), floor_plane_x - floorW / 2 + floor_plane_coridx = \ + (np.arctan2(floor_plane_y, floor_plane_x) / (2 * PI) + 0.5) * coorW - 0.5 + floor_plane = map_coordinates( + fuv, floor_plane_coridx.reshape(1, -1), order=1, mode='wrap') + floor_plane = floor_plane.reshape(floorH, floorW) + return floor_plane + + +def np_coorx2u(coorx, coorW=1024): + return ((coorx + 0.5) / coorW - 0.5) * 2 * PI + + +def np_coory2v(coory, coorH=512): + return -((coory + 0.5) / coorH - 0.5) * PI + + +def np_coor2xy(coor, z=50, coorW=1024, coorH=512, floorW=1024, floorH=512): + coor = np.array(coor) + u = np_coorx2u(coor[:, 0], coorW) + v = np_coory2v(coor[:, 1], coorH) + c = z / np.tan(v) + x = c * np.sin(u) + floorW / 2 - 0.5 + y = -c * np.cos(u) + floorH / 2 - 0.5 + return np.hstack([x[:, None], y[:, None]]) + + +def np_x_u_solve_y(x, u, floorW=1024, floorH=512): + c = (x - floorW / 2 + 0.5) / np.sin(u) + return -c * np.cos(u) + floorH / 2 - 0.5 + + +def np_y_u_solve_x(y, u, floorW=1024, floorH=512): + c = -(y - floorH / 2 + 0.5) / np.cos(u) + return c * np.sin(u) + floorW / 2 - 0.5 + + +def np_xy2coor(xy, z=50, coorW=1024, coorH=512, floorW=1024, floorH=512): + x = xy[:, 0] - floorW / 2 + 0.5 + y = xy[:, 1] - floorH / 2 + 0.5 + + u = np.arctan2(x, -y) + v = np.arctan(z / np.sqrt(x**2 + y**2)) + + coorx = (u / (2 * PI) + 0.5) * coorW - 0.5 + coory = (-v / PI + 0.5) * coorH - 0.5 + + return np.hstack([coorx[:, None], coory[:, None]]) + + +def mean_percentile(vec, p1=25, p2=75): + vmin = np.percentile(vec, p1) + vmax = np.percentile(vec, p2) + return vec[(vmin <= vec) & (vec <= vmax)].mean() + + +def vote(vec, tol): + vec = np.sort(vec) + n = np.arange(len(vec))[::-1] + n = n[:, None] - n[None, :] + 1.0 + la = squareform(pdist(vec[:, None], 'minkowski', p=1) + 1e-9) + + invalid = (n < len(vec) * 0.4) | (la > tol) + if (~invalid).sum() == 0 or len(vec) < tol: + best_fit = np.median(vec) + p_score = 0 + else: + la[invalid] = 1e5 + n[invalid] = -1 + score = n + max_idx = score.argmax() + max_row = max_idx // len(vec) + max_col = max_idx % len(vec) + assert max_col > max_row + best_fit = np.median(vec) + p_score = (max_col - max_row + 1) / len(vec) + + l1_score = np.abs(vec - best_fit).mean() + + return best_fit, p_score, l1_score + + +def get_z1(coory0, coory1, z0=50, coorH=512): + v0 = np_coory2v(coory0, coorH) + v1 = np_coory2v(coory1, coorH) + c0 = z0 / np.tan(v0) + z1 = c0 * np.tan(v1) + return z1 + + +def np_refine_by_fix_z(coory0, coory1, z0=50, coorH=512): + v0 = np_coory2v(coory0, coorH) + v1 = np_coory2v(coory1, coorH) + + c0 = z0 / np.tan(v0) + z1 = c0 * np.tan(v1) + z1_mean = mean_percentile(z1) + v1_refine = np.arctan2(z1_mean, c0) + coory1_refine = (-v1_refine / PI + 0.5) * coorH - 0.5 + + return coory1_refine, z1_mean + + +def infer_coory(coory0, h, z0=50, coorH=512): + v0 = np_coory2v(coory0, coorH) + c0 = z0 / np.tan(v0) + z1 = z0 + h + v1 = np.arctan2(z1, c0) + return (-v1 / PI + 0.5) * coorH - 0.5 + + +def get_gpid(coorx, coorW): + gpid = np.zeros(coorW) + gpid[np.round(coorx).astype(int)] = 1 + gpid = np.cumsum(gpid).astype(int) + gpid[gpid == gpid[-1]] = 0 + return gpid + + +def get_gpid_idx(gpid, j): + idx = np.where(gpid == j)[0] + if idx[0] == 0 and idx[-1] != len(idx) - 1: + _shift = -np.where(idx != np.arange(len(idx)))[0][0] + idx = np.roll(idx, _shift) + return idx + + +def gpid_two_split(xy, tpid_a, tpid_b): + m = np.arange(len(xy)) + 1 + cum_a = np.cumsum(xy[:, tpid_a]) + cum_b = np.cumsum(xy[::-1, tpid_b]) + l1_a = cum_a / m - cum_a / (m * m) + l1_b = cum_b / m - cum_b / (m * m) + l1_b = l1_b[::-1] + + score = l1_a[:-1] + l1_b[1:] + best_split = score.argmax() + 1 + + va = xy[:best_split, tpid_a].mean() + vb = xy[best_split:, tpid_b].mean() + + return va, vb + + +def _get_rot_rad(px, py): + if px < 0: + px, py = -px, -py + rad = np.arctan2(py, px) * 180 / np.pi + if rad > 45: + return 90 - rad + if rad < -45: + return -90 - rad + return -rad + + +def get_rot_rad(init_coorx, + coory, + z=50, + coorW=1024, + coorH=512, + floorW=1024, + floorH=512, + tol=5): + gpid = get_gpid(init_coorx, coorW) + coor = np.hstack([np.arange(coorW)[:, None], coory[:, None]]) + xy = np_coor2xy(coor, z, coorW, coorH, floorW, floorH) + + rot_rad_suggestions = [] + for j in range(len(init_coorx)): + pca = PCA(n_components=1) + pca.fit(xy[gpid == j]) + rot_rad_suggestions.append(_get_rot_rad(*pca.components_[0])) + rot_rad_suggestions = np.sort(rot_rad_suggestions + [1e9]) + + rot_rad = np.mean(rot_rad_suggestions[:-1]) + best_rot_rad_sz = -1 + last_j = 0 + for j in range(1, len(rot_rad_suggestions)): + if rot_rad_suggestions[j] - rot_rad_suggestions[j - 1] > tol: + last_j = j + elif j - last_j > best_rot_rad_sz: + rot_rad = rot_rad_suggestions[last_j:j + 1].mean() + best_rot_rad_sz = j - last_j + + dx = int(round(rot_rad * 1024 / 360)) + return dx, rot_rad + + +def gen_ww_cuboid(xy, gpid): + assert len(np.unique(gpid)) == 4 + xy_cor = [ + { + 'type': 1, + 'val': np.median(xy[gpid == 0, 1]) + }, + { + 'type': 0, + 'val': np.median(xy[gpid == 1, 0]) + }, + { + 'type': 1, + 'val': np.median(xy[gpid == 2, 1]) + }, + { + 'type': 0, + 'val': np.median(xy[gpid == 3, 0]) + }, + ] + return xy_cor + + +def gen_ww_general(init_coorx, xy, gpid, tol): + xy_cor = [] + assert len(init_coorx) == len(np.unique(gpid)) + + for j in range(len(init_coorx)): + now_x = xy[gpid == j, 0] + now_y = xy[gpid == j, 1] + new_x, x_score, x_l1 = vote(now_x, tol) + new_y, y_score, y_l1 = vote(now_y, tol) + u0 = np_coorx2u(init_coorx[(j - 1 + len(init_coorx)) + % len(init_coorx)]) + u1 = np_coorx2u(init_coorx[j]) + if (x_score, -x_l1) > (y_score, -y_l1): + xy_cor.append({ + 'type': 0, + 'val': new_x, + 'score': x_score, + 'action': 'ori', + 'gpid': j, + 'u0': u0, + 'u1': u1, + 'tbd': True + }) + else: + xy_cor.append({ + 'type': 1, + 'val': new_y, + 'score': y_score, + 'action': 'ori', + 'gpid': j, + 'u0': u0, + 'u1': u1, + 'tbd': True + }) + + while True: + tbd = -1 + for i in range(len(xy_cor)): + if xy_cor[i]['tbd'] and ( + tbd == -1 or xy_cor[i]['score'] > xy_cor[tbd]['score']): + tbd = i + if tbd == -1: + break + + xy_cor[tbd]['tbd'] = False + p_idx = (tbd - 1 + len(xy_cor)) % len(xy_cor) + n_idx = (tbd + 1) % len(xy_cor) + + num_tbd_neighbor = xy_cor[p_idx]['tbd'] + xy_cor[n_idx]['tbd'] + + if num_tbd_neighbor == 2: + continue + + if num_tbd_neighbor == 1: + if (not xy_cor[p_idx]['tbd'] and xy_cor[p_idx]['type'] == xy_cor[tbd]['type']) or\ + (not xy_cor[n_idx]['tbd'] and xy_cor[n_idx]['type'] == xy_cor[tbd]['type']): + if xy_cor[tbd]['score'] >= -1: + xy_cor[tbd]['tbd'] = True + xy_cor[tbd]['score'] -= 100 + else: + if not xy_cor[p_idx]['tbd']: + insert_at = tbd + if xy_cor[p_idx]['type'] == 0: + new_val = np_x_u_solve_y(xy_cor[p_idx]['val'], + xy_cor[p_idx]['u1']) + new_type = 1 + else: + new_val = np_y_u_solve_x(xy_cor[p_idx]['val'], + xy_cor[p_idx]['u1']) + new_type = 0 + else: + insert_at = n_idx + if xy_cor[n_idx]['type'] == 0: + new_val = np_x_u_solve_y(xy_cor[n_idx]['val'], + xy_cor[n_idx]['u0']) + new_type = 1 + else: + new_val = np_y_u_solve_x(xy_cor[n_idx]['val'], + xy_cor[n_idx]['u0']) + new_type = 0 + new_add = { + 'type': new_type, + 'val': new_val, + 'score': 0, + 'action': 'forced infer', + 'gpid': -1, + 'u0': -1, + 'u1': -1, + 'tbd': False + } + xy_cor.insert(insert_at, new_add) + continue + + if xy_cor[p_idx]['type'] == xy_cor[n_idx]['type']: + if xy_cor[tbd]['type'] == xy_cor[p_idx]['type']: + xy_cor[tbd]['type'] = (xy_cor[tbd]['type'] + 1) % 2 + xy_cor[tbd]['action'] = 'forced change' + xy_cor[tbd]['val'] = xy[gpid == xy_cor[tbd]['gpid'], + xy_cor[tbd]['type']].mean() + else: + tp0 = xy_cor[n_idx]['type'] + tp1 = xy_cor[p_idx]['type'] + if xy_cor[p_idx]['type'] == 0: + val0 = np_x_u_solve_y(xy_cor[p_idx]['val'], + xy_cor[p_idx]['u1']) + val1 = np_y_u_solve_x(xy_cor[n_idx]['val'], + xy_cor[n_idx]['u0']) + else: + val0 = np_y_u_solve_x(xy_cor[p_idx]['val'], + xy_cor[p_idx]['u1']) + val1 = np_x_u_solve_y(xy_cor[n_idx]['val'], + xy_cor[n_idx]['u0']) + new_add = [ + { + 'type': tp0, + 'val': val0, + 'score': 0, + 'action': 'forced infer', + 'gpid': -1, + 'u0': -1, + 'u1': -1, + 'tbd': False + }, + { + 'type': tp1, + 'val': val1, + 'score': 0, + 'action': 'forced infer', + 'gpid': -1, + 'u0': -1, + 'u1': -1, + 'tbd': False + }, + ] + xy_cor = xy_cor[:tbd] + new_add + xy_cor[tbd + 1:] + + return xy_cor + + +def gen_ww(init_coorx, + coory, + z=50, + coorW=1024, + coorH=512, + floorW=1024, + floorH=512, + tol=3, + force_cuboid=True): + gpid = get_gpid(init_coorx, coorW) + coor = np.hstack([np.arange(coorW)[:, None], coory[:, None]]) + xy = np_coor2xy(coor, z, coorW, coorH, floorW, floorH) + + if force_cuboid: + xy_cor = gen_ww_cuboid(xy, gpid) + else: + xy_cor = gen_ww_general(init_coorx, xy, gpid, tol) + + cor = [] + for j in range(len(xy_cor)): + next_j = (j + 1) % len(xy_cor) + if xy_cor[j]['type'] == 1: + cor.append((xy_cor[next_j]['val'], xy_cor[j]['val'])) + else: + cor.append((xy_cor[j]['val'], xy_cor[next_j]['val'])) + cor = np_xy2coor(np.array(cor), z, coorW, coorH, floorW, floorH) + cor = np.roll(cor, -2 * cor[::2, 0].argmin(), axis=0) + + return cor, xy_cor diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/__init__.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c2ff23e2326221bc4fcb63de0a08e7f5e2bfd6b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .layout import LayoutEstimator diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/layout.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/layout.py new file mode 100644 index 0000000000000000000000000000000000000000..12ff3a345676206bf24b4273c4c2f95155ca032e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/modality/layout.py @@ -0,0 +1,147 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import numpy as np +import torch +import torch.nn as nn +from scipy.ndimage.filters import maximum_filter +from shapely.geometry import Polygon + +from ..misc import post_proc + + +class LayoutEstimator(nn.Module): + + def __init__(self, + emb_dim, + bon_weight=1., + cor_weight=1., + bon_loss='l1', + cor_loss='bce', + bon_scale=1., + init_weight=0.1, + dropout=0., + oneconv=True, + last_ks=1, + last_bias=True, + sigmod_normlize=False, + sample_weight=False, + H=512, + W=1024, + post_force_cuboid=False): + super(LayoutEstimator, self).__init__() + self.bon_loss = bon_loss + self.cor_loss = cor_loss + self.bon_scale = bon_scale + self.bon_weight = bon_weight + self.cor_weight = cor_weight + self.H = H + self.W = W + self.post_force_cuboid = post_force_cuboid + if oneconv: + self.pred_bon = nn.Conv1d( + emb_dim, 2, last_ks, padding=last_ks // 2, bias=last_bias) + self.pred_cor = nn.Conv1d( + emb_dim, 1, last_ks, padding=last_ks // 2, bias=last_bias) + if last_bias: + nn.init.constant_(self.pred_bon.bias[0], -0.478) + nn.init.constant_(self.pred_bon.bias[1], 0.425) + nn.init.constant_(self.pred_cor.bias, -1.) + else: + self.pred_bon = nn.Sequential( + nn.Conv1d(emb_dim, emb_dim, 3, padding=1, bias=False), + nn.BatchNorm1d(emb_dim), + nn.ReLU(inplace=True), + nn.Conv1d(emb_dim, 2, 1), + ) + self.pred_cor = nn.Sequential( + nn.Conv1d(emb_dim, emb_dim, 3, padding=1, bias=False), + nn.BatchNorm1d(emb_dim), + nn.ReLU(inplace=True), + nn.Conv1d(emb_dim, 1, 1), + ) + nn.init.constant_(self.pred_bon[-1].bias[0], -0.478) + nn.init.constant_(self.pred_bon[-1].bias[1], 0.425) + nn.init.constant_(self.pred_cor[-1].bias, -1.) + self.dropout = None + if dropout > 0: + self.dropout = nn.Dropout(dropout) + + self.sigmod_normlize = sigmod_normlize + + self.sample_weight = sample_weight + + def forward(self, x_emb): + x_emb = x_emb['1D'] + if self.dropout is not None: + x_emb = self.dropout(x_emb) + pred_bon = self.pred_bon(x_emb) + pred_cor = self.pred_cor(x_emb) + if self.sigmod_normlize: + bon = torch.sigmoid(pred_bon) + up = bon[:, 0:1, :] * -0.5 * math.pi + down = bon[:, 1:, :] * 0.5 * math.pi + pred_bon = torch.cat([up, down], dim=1) + return {'pred_bon': pred_bon, 'pred_cor': pred_cor} + + def infer(self, x_emb): + results = self(x_emb) + pred_bon, pred_cor = results['pred_bon'], results['pred_cor'] + pred_bon = pred_bon.clone() / self.bon_scale + H, W = self.H, self.W + + y_bon_ = (pred_bon[0].cpu().numpy() / np.pi + 0.5) * H - 0.5 + y_cor_ = pred_cor[0, 0].sigmoid().cpu().numpy() + z0 = 50 + _, z1 = post_proc.np_refine_by_fix_z(*y_bon_, z0) + + def find_N_peaks(signal, r, min_v, N): + max_v = maximum_filter(signal, size=r, mode='wrap') + pk_loc = np.where(max_v == signal)[0] + pk_loc = pk_loc[signal[pk_loc] > min_v] + if N is not None: + order = np.argsort(-signal[pk_loc]) + pk_loc = pk_loc[order[:N]] + pk_loc = pk_loc[np.argsort(pk_loc)] + return pk_loc, signal[pk_loc] + + min_v = 0 if self.post_force_cuboid else 0.05 + r = int(round(W * 0.05 / 2)) + N = 4 if self.post_force_cuboid else None + xs_ = find_N_peaks(y_cor_, r=r, min_v=min_v, N=N)[0] + + cor, xy_cor = post_proc.gen_ww( + xs_, + y_bon_[0], + z0, + tol=abs(0.16 * z1 / 1.6), + force_cuboid=self.post_force_cuboid) + if not self.post_force_cuboid: + xy2d = np.zeros((len(xy_cor), 2), np.float32) + for i in range(len(xy_cor)): + xy2d[i, xy_cor[i]['type']] = xy_cor[i]['val'] + xy2d[i, xy_cor[i - 1]['type']] = xy_cor[i - 1]['val'] + if len(xy2d) < 3 or not Polygon(xy2d).is_valid: + import sys + print( + 'Fail to generate valid general layout!! ' + 'Generate cuboid as fallback.', + file=sys.stderr) + xs_ = find_N_peaks(y_cor_, r=r, min_v=0, N=4)[0] + cor, xy_cor = post_proc.gen_ww( + xs_, + y_bon_[0], + z0, + tol=abs(0.16 * z1 / 1.6), + force_cuboid=True) + + cor = np.hstack( + [cor, post_proc.infer_coory(cor[:, 1], z1 - z0, z0)[:, None]]) + cor_id = np.zeros((len(cor) * 2, 2), np.float32) + for j in range(len(cor)): + cor_id[j * 2] = cor[j, 0], cor[j, 1] + cor_id[j * 2 + 1] = cor[j, 0], cor[j, 2] + y_pred_bon = post_proc.cor_2_1d(cor_id, 512, 1024) + y_pred_bon = ((y_pred_bon / np.pi + 0.5) * 512).round().astype(int) + return {'cor_id': cor_id, 'y_bon_': y_pred_bon, 'y_cor_': y_cor_} diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/panovit.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/panovit.py new file mode 100644 index 0000000000000000000000000000000000000000..23ba211d0dfba2bd0447debd323072feb2ef969c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/panovit.py @@ -0,0 +1,96 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from . import backbone, modality +from .utils import visualize_a_data + + +class PanoVIT(nn.Module): + + def __init__(self, + emb_dim=256, + input_hw=None, + input_norm='imagenet', + pretrain='', + backbone_config={'module': 'Resnet'}, + transformer_config={'module': 'ViT'}, + modalities_config={}): + super(PanoVIT, self).__init__() + self.input_hw = input_hw + if input_norm == 'imagenet': + self.register_buffer( + 'x_mean', + torch.FloatTensor( + np.array([0.485, 0.456, 0.406])[None, :, None, None])) + self.register_buffer( + 'x_std', + torch.FloatTensor( + np.array([0.229, 0.224, 0.225])[None, :, None, None])) + else: + raise NotImplementedError + + Encoder = getattr(backbone, backbone_config['module']) + Encoder_kwargs = backbone_config.get('kwargs', {}) + self.encoder = Encoder(**Encoder_kwargs) + + Transformer = getattr(backbone, transformer_config['module']) + Transformer_kwargs = transformer_config.get('kwargs', {}) + self.transformer = Transformer(**Transformer_kwargs) + self.transformer_config = transformer_config['module'] + self.transformer_Fourier = transformer_config['kwargs']['fourier'] + + self.modalities = nn.ModuleList([ + getattr(modality, key)(emb_dim, **config) + for key, config in modalities_config.items() + ]) + + def extract_feat(self, x): + img = x[:, 0:3, :, :] + if self.input_hw: + img = F.interpolate( + img, size=self.input_hw, mode='bilinear', align_corners=False) + + img = (img - self.x_mean) / self.x_std + if self.transformer_Fourier == 'fourier_res': + img = torch.cat((img, x[:, 3:, :, :]), dim=1) + res_f = self.encoder(img) + elif self.transformer_Fourier == 'fourier_trans': + res_f = self.encoder(img) + img = torch.cat((img, x[:, 3:, :, :]), dim=1) + else: + res_f = self.encoder(img) + + if self.transformer_config == 'ViTHorizonPryImage': + + feat = self.transformer(img, res_f) + else: + feat = self.transformer(x) + return feat + + def call_modality(self, method, *feed_args, **feed_kwargs): + output_dict = {} + for m in self.modalities: + curr_dict = getattr(m, method)(*feed_args, **feed_kwargs) + assert len(output_dict.keys() & curr_dict.keys() + ) == 0, 'Key collision for different modalities' + output_dict.update(curr_dict) + return output_dict + + def forward(self, x): + feat = self.extract_feat(x) + results = self.call_modality('forward', feat) + return torch.cat((results['pred_bon'], results['pred_cor']), dim=1) + + def infer(self, x): + feat = self.extract_feat(x) + result = self.call_modality('infer', feat) + result['image'] = x + return result + + def postprocess(self, image, y_bon, y_cor): + vis_layout = visualize_a_data(image, y_bon, y_cor) + return (vis_layout[:, :, (2, 1, 0)]) diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/utils.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..57753732ac868fd6474b75131c93e87af1bdead4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/networks/utils.py @@ -0,0 +1,110 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class StripPooling(nn.Module): + + def __init__(self, in_channels, input_size, pool_size, norm_layer): + super(StripPooling, self).__init__() + input_size = np.array(input_size) + output_size = np.array([pool_size[0], pool_size[0]]) + stride = np.floor((input_size / output_size)).astype(int) + kernel_size = (input_size - (output_size - 1) * stride).astype(int) + self.pool1 = nn.AvgPool2d( + kernel_size=(kernel_size[0], kernel_size[1]), + stride=(stride[0], stride[1]), + ceil_mode=False) + + output_size = np.array([pool_size[1], pool_size[1]]) + stride = np.floor((input_size / output_size)).astype(int) + kernel_size = (input_size - (output_size - 1) * stride).astype(int) + self.pool2 = nn.AvgPool2d( + kernel_size=(kernel_size[0], kernel_size[1]), + stride=(stride[0], stride[1]), + ceil_mode=False) + + output_size = np.array([1, input_size[1]]) + stride = np.floor((input_size / output_size)).astype(int) + kernel_size = (input_size - (output_size - 1) * stride).astype(int) + self.pool3 = nn.AvgPool2d( + kernel_size=(kernel_size[0], kernel_size[1]), + stride=(stride[0], stride[1]), + ceil_mode=False) + + output_size = np.array([input_size[0], 1]) + stride = np.floor((input_size / output_size)).astype(int) + kernel_size = (input_size - (output_size - 1) * stride).astype(int) + self.pool4 = nn.AvgPool2d( + kernel_size=(kernel_size[0], kernel_size[1]), + stride=(stride[0], stride[1]), + ceil_mode=False) + + inter_channels = in_channels // 4 + + self.conv1_1 = nn.Sequential( + nn.Conv2d(in_channels, inter_channels, 1, bias=False), + norm_layer(inter_channels), nn.ReLU(True)) + self.conv1_2 = nn.Sequential( + nn.Conv2d(in_channels, inter_channels, 1, bias=False), + norm_layer(inter_channels), nn.ReLU(True)) + self.conv2_0 = nn.Sequential( + nn.Conv2d(inter_channels, inter_channels, 3, 1, 1, bias=False), + norm_layer(inter_channels)) + self.conv2_1 = nn.Sequential( + nn.Conv2d(inter_channels, inter_channels, 3, 1, 1, bias=False), + norm_layer(inter_channels)) + self.conv2_2 = nn.Sequential( + nn.Conv2d(inter_channels, inter_channels, 3, 1, 1, bias=False), + norm_layer(inter_channels)) + self.conv2_3 = nn.Sequential( + nn.Conv2d( + inter_channels, inter_channels, (1, 3), 1, (0, 1), bias=False), + norm_layer(inter_channels)) + self.conv2_4 = nn.Sequential( + nn.Conv2d( + inter_channels, inter_channels, (3, 1), 1, (1, 0), bias=False), + norm_layer(inter_channels)) + self.conv2_5 = nn.Sequential( + nn.Conv2d(inter_channels, inter_channels, 3, 1, 1, bias=False), + norm_layer(inter_channels), nn.ReLU(True)) + self.conv2_6 = nn.Sequential( + nn.Conv2d(inter_channels, inter_channels, 3, 1, 1, bias=False), + norm_layer(inter_channels), nn.ReLU(True)) + self.conv3 = nn.Sequential( + nn.Conv2d(inter_channels * 2, in_channels, 1, bias=False), + norm_layer(in_channels)) + + def forward(self, x): + _, _, h, w = x.size() + x1 = self.conv1_1(x) + x2 = self.conv1_2(x) + x2_1 = self.conv2_0(x1) + x2_2 = F.interpolate(self.conv2_1(self.pool1(x1)), (h, w)) + x2_3 = F.interpolate(self.conv2_2(self.pool2(x1)), (h, w)) + x2_4 = F.interpolate(self.conv2_3(self.pool3(x2)), (h, w)) + x2_5 = F.interpolate(self.conv2_4(self.pool4(x2)), (h, w)) + x1 = self.conv2_5(F.relu_(x2_1 + x2_2 + x2_3)) + x2 = self.conv2_6(F.relu_(x2_5 + x2_4)) + out = self.conv3(torch.cat([x1, x2], dim=1)) + return F.relu_(x + out) + + +def visualize_a_data(x, y_bon, y_cor): + x = (x.cpu().numpy().transpose([1, 2, 0]) * 255).astype(np.uint8) + y_bon = y_bon.round().astype(int) + gt_cor = np.zeros((30, 1024, 3), np.uint8) + gt_cor[:] = y_cor[None, :, None] * 255 + img_pad = np.zeros((3, 1024, 3), np.uint8) + 255 + + img_bon = (x.copy()).astype(np.uint8) + img_bon[y_bon[0], np.arange(len(y_bon[0])), 1] = 255 + img_bon[y_bon[1], np.arange(len(y_bon[1])), 1] = 255 + img_bon[y_bon[0] - 1, np.arange(len(y_bon[0])), 1] = 255 + img_bon[y_bon[1] - 1, np.arange(len(y_bon[1])), 1] = 255 + img_bon[y_bon[0] + 1, np.arange(len(y_bon[0])), 1] = 255 + img_bon[y_bon[1] + 1, np.arange(len(y_bon[1])), 1] = 255 + return np.concatenate([gt_cor, img_pad, img_bon], 0) diff --git a/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/panovit.py b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/panovit.py new file mode 100644 index 0000000000000000000000000000000000000000..615ad5e48ecd7b331566e1a64c541a9fdbd67199 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/indoor_layout_estimation/panovit.py @@ -0,0 +1,57 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch +from yacs.config import CfgNode as CN + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.indoor_layout_estimation.networks.panovit import \ + PanoVIT +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.indoor_layout_estimation, + module_name=Models.panovit_layout_estimation) +class LayoutEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + config = CN() + config.model = CN() + config.model.kwargs = CN(new_allowed=True) + config.defrost() + config_path = osp.join(model_dir, ModelFile.YAML_FILE) + config.merge_from_file(config_path) + config.freeze() + # build model + self.model = PanoVIT(**config.model.kwargs) + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + if torch.cuda.is_available(): + state_dict = torch.load(model_path) + else: + state_dict = torch.load(model_path, map_location='cpu') + + self.model.load_state_dict(state_dict) + self.model.eval() + + def forward(self, Inputs): + return self.model.infer(Inputs['images']) + + def postprocess(self, Inputs): + image, y_bon, y_cor = Inputs['image'], Inputs['y_bon_'], Inputs[ + 'y_cor_'] + layout_image = self.model.postprocess(image[0, 0:3], y_bon, y_cor) + return layout_image + + def inference(self, data): + results = self.forward(data) + return results diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/__init__.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73f7bd03ba95735a45416bffdc2c39b4be8c8621 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .summarizer import ( + ClipItVideoSummarization, ) + +else: + _import_structure = { + 'summarizer': [ + 'ClipItVideoSummarization', + ] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/summarizer.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/summarizer.py new file mode 100644 index 0000000000000000000000000000000000000000..6e12ea2bfa7f148a991d391e38a34312866e9ac9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/summarizer.py @@ -0,0 +1,169 @@ +# Part of the implementation is borrowed and modified from BMT and video_features, +# publicly available at https://github.com/v-iashin/BMT +# and https://github.com/v-iashin/video_features + +import argparse +import os +import os.path as osp +from typing import Dict, Union + +import numpy as np +import torch +import torch.nn as nn +from bmt_clipit.sample.single_video_prediction import (caption_proposals, + generate_proposals, + load_cap_model, + load_prop_model) +from bmt_clipit.utilities.proposal_utils import non_max_suppresion +from torch.nn.parallel import DataParallel, DistributedDataParallel +from videofeatures_clipit.models.i3d.extract_i3d import ExtractI3D +from videofeatures_clipit.models.vggish.extract_vggish import ExtractVGGish +from videofeatures_clipit.utils.utils import (fix_tensorflow_gpu_allocation, + form_list_from_user_input) + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.language_guided_video_summarization.transformer import \ + Transformer +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def extract_text(args): + # Loading models and other essential stuff + cap_cfg, cap_model, train_dataset = load_cap_model( + args.pretrained_cap_model_path, args.device_id) + prop_cfg, prop_model = load_prop_model(args.device_id, + args.prop_generator_model_path, + args.pretrained_cap_model_path, + args.max_prop_per_vid) + # Proposal + proposals = generate_proposals(prop_model, args.features, + train_dataset.pad_idx, prop_cfg, + args.device_id, args.duration_in_secs) + # NMS if specified + if args.nms_tiou_thresh is not None: + proposals = non_max_suppresion(proposals.squeeze(), + args.nms_tiou_thresh) + proposals = proposals.unsqueeze(0) + # Captions for each proposal + captions = caption_proposals(cap_model, args.features, train_dataset, + cap_cfg, args.device_id, proposals, + args.duration_in_secs) + return captions + + +def extract_video_features(video_path, tmp_path, feature_type, i3d_flow_path, + i3d_rgb_path, kinetics_class_labels, pwc_path, + vggish_model_path, vggish_pca_path, extraction_fps, + device): + default_args = dict( + device=device, + extraction_fps=extraction_fps, + feature_type=feature_type, + file_with_video_paths=None, + i3d_flow_path=i3d_flow_path, + i3d_rgb_path=i3d_rgb_path, + keep_frames=False, + kinetics_class_labels=kinetics_class_labels, + min_side_size=256, + pwc_path=pwc_path, + show_kinetics_pred=False, + stack_size=64, + step_size=64, + tmp_path=tmp_path, + vggish_model_path=vggish_model_path, + vggish_pca_path=vggish_pca_path, + ) + args = argparse.Namespace(**default_args) + + if args.feature_type == 'i3d': + extractor = ExtractI3D(args) + elif args.feature_type == 'vggish': + extractor = ExtractVGGish(args) + + feats = extractor(video_path) + return feats + + +def video_features_to_txt(duration_in_secs, pretrained_cap_model_path, + prop_generator_model_path, features, device_id): + default_args = dict( + device_id=device_id, + duration_in_secs=duration_in_secs, + features=features, + pretrained_cap_model_path=pretrained_cap_model_path, + prop_generator_model_path=prop_generator_model_path, + max_prop_per_vid=100, + nms_tiou_thresh=0.4, + ) + args = argparse.Namespace(**default_args) + txt = extract_text(args) + return txt + + +@MODELS.register_module( + Tasks.language_guided_video_summarization, + module_name=Models.language_guided_video_summarization) +class ClipItVideoSummarization(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the video summarization model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.loss = nn.MSELoss() + self.model = Transformer() + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.model = self.model.to(self._device) + + self.model = self._load_pretrained(self.model, model_path) + + if self.training: + self.model.train() + else: + self.model.eval() + + def _train_forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + frame_features = input['frame_features'] + txt_features = input['txt_features'] + gtscore = input['gtscore'] + preds, attn_weights = self.model(frame_features, txt_features, + frame_features) + return {'loss': self.loss(preds, gtscore)} + + def _inference_forward(self, input: Dict[str, + Tensor]) -> Dict[str, Tensor]: + frame_features = input['frame_features'] + txt_features = input['txt_features'] + y, dec_output = self.model(frame_features, txt_features, + frame_features) + return {'scores': y} + + def forward(self, input: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Union[list, Tensor]]: results + """ + for key, value in input.items(): + input[key] = input[key].to(self._device) + if self.training: + return self._train_forward(input) + else: + return self._inference_forward(input) diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/__init__.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..68dccccfbaf47e537ac61abd3702fc7a150d1f1d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .models import ( + Transformer, ) + +else: + _import_structure = { + 'models': [ + 'Transformer', + ] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/layers.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..6782c20900ea73c5fcaf0a06c74b966c588cdf9a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/layers.py @@ -0,0 +1,48 @@ +# Part of the implementation is borrowed and modified from attention-is-all-you-need-pytorch, +# publicly available at https://github.com/jadore801120/attention-is-all-you-need-pytorch +import torch +import torch.nn as nn + +from .sub_layers import MultiHeadAttention, PositionwiseFeedForward + + +class EncoderLayer(nn.Module): + """Compose with two layers""" + + def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1): + super(EncoderLayer, self).__init__() + self.slf_attn = MultiHeadAttention( + n_head, d_model, d_k, d_v, dropout=dropout) + self.pos_ffn = PositionwiseFeedForward( + d_model, d_inner, dropout=dropout) + + def forward(self, enc_input, slf_attn_mask=None): + enc_output, enc_slf_attn = self.slf_attn( + enc_input, enc_input, enc_input, mask=slf_attn_mask) + enc_output = self.pos_ffn(enc_output) + return enc_output, enc_slf_attn + + +class DecoderLayer(nn.Module): + """Compose with three layers""" + + def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1): + super(DecoderLayer, self).__init__() + self.slf_attn = MultiHeadAttention( + n_head, d_model, d_k, d_v, dropout=dropout) + self.enc_attn = MultiHeadAttention( + n_head, d_model, d_k, d_v, dropout=dropout) + self.pos_ffn = PositionwiseFeedForward( + d_model, d_inner, dropout=dropout) + + def forward(self, + dec_input, + enc_output, + slf_attn_mask=None, + dec_enc_attn_mask=None): + dec_output, dec_slf_attn = self.slf_attn( + dec_input, dec_input, dec_input, mask=slf_attn_mask) + dec_output, dec_enc_attn = self.enc_attn( + dec_output, enc_output, enc_output, mask=dec_enc_attn_mask) + dec_output = self.pos_ffn(dec_output) + return dec_output, dec_slf_attn, dec_enc_attn diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/models.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/models.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ae34ee88d232308ec0cd8b95ed0822bb22cb01 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/models.py @@ -0,0 +1,229 @@ +# Part of the implementation is borrowed and modified from attention-is-all-you-need-pytorch, +# publicly available at https://github.com/jadore801120/attention-is-all-you-need-pytorch + +import numpy as np +import torch +import torch.nn as nn + +from .layers import DecoderLayer, EncoderLayer +from .sub_layers import MultiHeadAttention + + +class PositionalEncoding(nn.Module): + + def __init__(self, d_hid, n_position=200): + super(PositionalEncoding, self).__init__() + + # Not a parameter + self.register_buffer( + 'pos_table', self._get_sinusoid_encoding_table(n_position, d_hid)) + + def _get_sinusoid_encoding_table(self, n_position, d_hid): + """Sinusoid position encoding table""" + + # TODO: make it with torch instead of numpy + + def get_position_angle_vec(position): + return [ + position / np.power(10000, 2 * (hid_j // 2) / d_hid) + for hid_j in range(d_hid) + ] + + sinusoid_table = np.array( + [get_position_angle_vec(pos_i) for pos_i in range(n_position)]) + sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i + sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 + + return torch.FloatTensor(sinusoid_table).unsqueeze(0) + + def forward(self, x): + return x + self.pos_table[:, :x.size(1)].clone().detach() + + +class Encoder(nn.Module): + """A encoder model with self attention mechanism.""" + + def __init__(self, + d_word_vec=1024, + n_layers=6, + n_head=8, + d_k=64, + d_v=64, + d_model=512, + d_inner=2048, + dropout=0.1, + n_position=200): + + super().__init__() + + self.position_enc = PositionalEncoding( + d_word_vec, n_position=n_position) + self.dropout = nn.Dropout(p=dropout) + self.layer_stack = nn.ModuleList([ + EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout) + for _ in range(n_layers) + ]) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.d_model = d_model + + def forward(self, enc_output, return_attns=False): + + enc_slf_attn_list = [] + # -- Forward + enc_output = self.dropout(self.position_enc(enc_output)) + enc_output = self.layer_norm(enc_output) + + for enc_layer in self.layer_stack: + enc_output, enc_slf_attn = enc_layer(enc_output) + enc_slf_attn_list += [enc_slf_attn] if return_attns else [] + + if return_attns: + return enc_output, enc_slf_attn_list + return enc_output, + + +class Decoder(nn.Module): + """A decoder model with self attention mechanism.""" + + def __init__(self, + d_word_vec=1024, + n_layers=6, + n_head=8, + d_k=64, + d_v=64, + d_model=512, + d_inner=2048, + n_position=200, + dropout=0.1): + + super().__init__() + + self.position_enc = PositionalEncoding( + d_word_vec, n_position=n_position) + self.dropout = nn.Dropout(p=dropout) + self.layer_stack = nn.ModuleList([ + DecoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=dropout) + for _ in range(n_layers) + ]) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.d_model = d_model + + def forward(self, + dec_output, + enc_output, + src_mask=None, + trg_mask=None, + return_attns=False): + + dec_slf_attn_list, dec_enc_attn_list = [], [] + + # -- Forward + dec_output = self.dropout(self.position_enc(dec_output)) + dec_output = self.layer_norm(dec_output) + + for dec_layer in self.layer_stack: + dec_output, dec_slf_attn, dec_enc_attn = dec_layer( + dec_output, + enc_output, + slf_attn_mask=trg_mask, + dec_enc_attn_mask=src_mask) + dec_slf_attn_list += [dec_slf_attn] if return_attns else [] + dec_enc_attn_list += [dec_enc_attn] if return_attns else [] + + if return_attns: + return dec_output, dec_slf_attn_list, dec_enc_attn_list + return dec_output, + + +class Transformer(nn.Module): + """A sequence to sequence model with attention mechanism.""" + + def __init__(self, + num_sentence=7, + txt_atten_head=4, + d_frame_vec=512, + d_model=512, + d_inner=2048, + n_layers=6, + n_head=8, + d_k=256, + d_v=256, + dropout=0.1, + n_position=4000): + + super().__init__() + + self.d_model = d_model + + self.layer_norm_img_src = nn.LayerNorm(d_frame_vec, eps=1e-6) + self.layer_norm_img_trg = nn.LayerNorm(d_frame_vec, eps=1e-6) + self.layer_norm_txt = nn.LayerNorm( + num_sentence * d_frame_vec, eps=1e-6) + + self.linear_txt = nn.Linear( + in_features=num_sentence * d_frame_vec, out_features=d_model) + self.lg_attention = MultiHeadAttention( + n_head=txt_atten_head, d_model=d_model, d_k=d_k, d_v=d_v) + + self.encoder = Encoder( + n_position=n_position, + d_word_vec=d_frame_vec, + d_model=d_model, + d_inner=d_inner, + n_layers=n_layers, + n_head=n_head, + d_k=d_k, + d_v=d_v, + dropout=dropout) + + self.decoder = Decoder( + n_position=n_position, + d_word_vec=d_frame_vec, + d_model=d_model, + d_inner=d_inner, + n_layers=n_layers, + n_head=n_head, + d_k=d_k, + d_v=d_v, + dropout=dropout) + + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + assert d_model == d_frame_vec, 'the dimensions of all module outputs shall be the same.' + + self.linear_1 = nn.Linear(in_features=d_model, out_features=d_model) + self.linear_2 = nn.Linear( + in_features=self.linear_1.out_features, out_features=1) + + self.drop = nn.Dropout(p=0.5) + self.norm_y = nn.LayerNorm(normalized_shape=d_model, eps=1e-6) + self.norm_linear = nn.LayerNorm( + normalized_shape=self.linear_1.out_features, eps=1e-6) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, src_seq, src_txt, trg_seq): + + features_txt = self.linear_txt(src_txt) + atten_seq, txt_attn = self.lg_attention(src_seq, features_txt, + features_txt) + + enc_output, *_ = self.encoder(atten_seq) + dec_output, *_ = self.decoder(trg_seq, enc_output) + + y = self.drop(enc_output) + y = self.norm_y(y) + + # 2-layer NN (Regressor Network) + y = self.linear_1(y) + y = self.relu(y) + y = self.drop(y) + y = self.norm_linear(y) + + y = self.linear_2(y) + y = self.sigmoid(y) + y = y.view(1, -1) + + return y, dec_output diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/modules.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..03ef8eaf95cd2da4694e03b3cd214ea96bfbac57 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/modules.py @@ -0,0 +1,27 @@ +# Part of the implementation is borrowed and modified from attention-is-all-you-need-pytorch, +# publicly available at https://github.com/jadore801120/attention-is-all-you-need-pytorch + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ScaledDotProductAttention(nn.Module): + """Scaled Dot-Product Attention""" + + def __init__(self, temperature, attn_dropout=0.1): + super().__init__() + self.temperature = temperature + self.dropout = nn.Dropout(attn_dropout) + + def forward(self, q, k, v, mask=None): + + attn = torch.matmul(q / self.temperature, k.transpose(2, 3)) + + if mask is not None: + attn = attn.masked_fill(mask == 0, -1e9) + + attn = self.dropout(F.softmax(attn, dim=-1)) + output = torch.matmul(attn, v) + + return output, attn diff --git a/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/sub_layers.py b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/sub_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..42e10abb02a4e267cdf3f8b3aa624d559305e74b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/language_guided_video_summarization/transformer/sub_layers.py @@ -0,0 +1,83 @@ +# Part of the implementation is borrowed and modified from attention-is-all-you-need-pytorch, +# publicly available at https://github.com/jadore801120/attention-is-all-you-need-pytorch + +import numpy as np +import torch.nn as nn +import torch.nn.functional as F + +from .modules import ScaledDotProductAttention + + +class MultiHeadAttention(nn.Module): + """Multi-Head Attention module""" + + def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1): + super().__init__() + + self.n_head = n_head + self.d_k = d_k + self.d_v = d_v + + self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False) + self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False) + self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False) + self.fc = nn.Linear(n_head * d_v, d_model, bias=False) + + self.attention = ScaledDotProductAttention(temperature=d_k**0.5) + + self.dropout = nn.Dropout(dropout) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + + def forward(self, q, k, v, mask=None): + + d_k, d_v, n_head = self.d_k, self.d_v, self.n_head + sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1) + + residual = q + + # Pass through the pre-attention projection: b x lq x (n*dv) + # Separate different heads: b x lq x n x dv + q = self.w_qs(q).view(sz_b, len_q, n_head, d_k) + k = self.w_ks(k).view(sz_b, len_k, n_head, d_k) + v = self.w_vs(v).view(sz_b, len_v, n_head, d_v) + + # Transpose for attention dot product: b x n x lq x dv + q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) + + if mask is not None: + mask = mask.unsqueeze(1) # For head axis broadcasting. + + q, attn = self.attention(q, k, v, mask=mask) + + # Transpose to move the head dimension back: b x lq x n x dv + # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv) + q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1) + q = self.dropout(self.fc(q)) + q += residual + + q = self.layer_norm(q) + + return q, attn + + +class PositionwiseFeedForward(nn.Module): + """A two-feed-forward-layer module""" + + def __init__(self, d_in, d_hid, dropout=0.1): + super().__init__() + self.w_1 = nn.Linear(d_in, d_hid) # position-wise + self.w_2 = nn.Linear(d_hid, d_in) # position-wise + self.layer_norm = nn.LayerNorm(d_in, eps=1e-6) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + + residual = x + + x = self.w_2(F.relu(self.w_1(x))) + x = self.dropout(x) + x += residual + + x = self.layer_norm(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/__init__.py b/runtime/Lib/site_packages/models/cv/motion_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f8cbad7d5c3bd929d4571fd5348445726cc0cbf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model import create_model, load_model_wo_clip + from .modules.cfg_sampler import ClassifierFreeSampleModel +else: + _import_structure = { + 'model': ['create_model', 'load_model_wo_clip'], + 'modules.cfg_sampler': ['ClassifierFreeSampleModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/model.py b/runtime/Lib/site_packages/models/cv/motion_generation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..aa944adabebfd53c2f6dadc38bf3503356c00ac3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/model.py @@ -0,0 +1,65 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +from .modules import gaussian_diffusion as gd +from .modules.mdm import MDM +from .modules.respace import SpacedDiffusion, space_timesteps + + +def load_model_wo_clip(model, state_dict): + missing_keys, unexpected_keys = model.load_state_dict( + state_dict, strict=False) + assert len(unexpected_keys) == 0 + assert all([k.startswith('clip_model.') for k in missing_keys]) + + +def create_model(cfg): + model = MDM( + '', + njoints=263, + nfeats=1, + num_actions=1, + translation=True, + pose_rep='rot6d', + glob=True, + glob_rot=True, + latent_dim=512, + ff_size=1024, + smpl_data_path=cfg.smpl_data_path, + data_rep='hml_vec', + dataset='humanml', + clip_version='ViT-B/32', + **{ + 'cond_mode': 'text', + 'cond_mask_prob': 0.1, + 'action_emb': 'tensor' + }) + + predict_xstart = True # we always predict x_start (a.k.a. x0), that's our deal! + steps = cfg.sample_steps + scale_beta = 1. # no scaling + timestep_respacing = '' # can be used for ddim sampling, we don't use it. + learn_sigma = False + rescale_timesteps = False + + betas = gd.get_named_beta_schedule('cosine', steps, scale_beta) + loss_type = gd.LossType.MSE + + if not timestep_respacing: + timestep_respacing = [steps] + + diffusion = SpacedDiffusion( + use_timesteps=space_timesteps(steps, timestep_respacing), + betas=betas, + model_mean_type=(gd.ModelMeanType.EPSILON + if not predict_xstart else gd.ModelMeanType.START_X), + model_var_type=((gd.ModelVarType.FIXED_LARGE + if not True else gd.ModelVarType.FIXED_SMALL) + if not learn_sigma else gd.ModelVarType.LEARNED_RANGE), + loss_type=loss_type, + rescale_timesteps=rescale_timesteps, + lambda_vel=0.0, + lambda_rcxyz=0.0, + lambda_fc=0.0, + ) + return model, diffusion diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/__init__.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/cfg_sampler.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/cfg_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..cd07362d1508b2244be531f9f1bb9aeeedcbbae8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/cfg_sampler.py @@ -0,0 +1,33 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model +from copy import deepcopy + +import torch.nn as nn + + +# A wrapper model for Classifier-free guidance **SAMPLING** only +# https://arxiv.org/abs/2207.12598 +class ClassifierFreeSampleModel(nn.Module): + + def __init__(self, model): + super().__init__() + self.model = model # model is the actual model to run + + assert self.model.cond_mask_prob > 0 + + # pointers to inner model + self.rot2xyz = self.model.rot2xyz + self.translation = self.model.translation + self.njoints = self.model.njoints + self.nfeats = self.model.nfeats + self.data_rep = self.model.data_rep + self.cond_mode = self.model.cond_mode + + def forward(self, x, timesteps, y=None): + cond_mode = self.model.cond_mode + assert cond_mode in ['text', 'action'] + y_uncond = deepcopy(y) + y_uncond['uncond'] = True + out = self.model(x, timesteps, y) + out_uncond = self.model(x, timesteps, y_uncond) + return out_uncond + (y['scale'].view(-1, 1, 1, 1) * (out - out_uncond)) diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/gaussian_diffusion.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/gaussian_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..2d283642c6d9c3f077e7128928c71066384a4664 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/gaussian_diffusion.py @@ -0,0 +1,666 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +import enum +import math +from copy import deepcopy + +import numpy as np +import torch as th + + +def get_named_beta_schedule(schedule_name, + num_diffusion_timesteps, + scale_betas=1.): + """ + Get a pre-defined beta schedule for the given name. + + The beta schedule library consists of beta schedules which remain similar + in the limit of num_diffusion_timesteps. + Beta schedules may be added, but should not be removed or changed once + they are committed to maintain backwards compatibility. + """ + if schedule_name == 'linear': + # Linear schedule from Ho et al, extended to work for any number of + # diffusion steps. + scale = scale_betas * 1000 / num_diffusion_timesteps + beta_start = scale * 0.0001 + beta_end = scale * 0.02 + return np.linspace( + beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64) + elif schedule_name == 'cosine': + return betas_for_alpha_bar( + num_diffusion_timesteps, + lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2)**2, + ) + else: + raise NotImplementedError(f'unknown beta schedule: {schedule_name}') + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + + +class ModelMeanType(enum.Enum): + """ + Which type of output the model predicts. + """ + + PREVIOUS_X = enum.auto() # the model predicts x_{t-1} + START_X = enum.auto() # the model predicts x_0 + EPSILON = enum.auto() # the model predicts epsilon + + +class ModelVarType(enum.Enum): + """ + What is used as the model's output variance. + + The LEARNED_RANGE option has been added to allow the model to predict + values between FIXED_SMALL and FIXED_LARGE, making its job easier. + """ + + LEARNED = enum.auto() + FIXED_SMALL = enum.auto() + FIXED_LARGE = enum.auto() + LEARNED_RANGE = enum.auto() + + +class LossType(enum.Enum): + MSE = enum.auto() # use raw MSE loss (and KL when learning variances) + RESCALED_MSE = ( + enum.auto() + ) # use raw MSE loss (with RESCALED_KL when learning variances) + KL = enum.auto() # use the variational lower-bound + RESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB + + def is_vb(self): + return self == LossType.KL or self == LossType.RESCALED_KL + + +class GaussianDiffusion: + """ + Utilities for training and sampling diffusion models. + + Ported directly from here, and then adapted over time to further experimentation. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py#L42 + + :param betas: a 1-D numpy array of betas for each diffusion timestep, + starting at T and going to 1. + :param model_mean_type: a ModelMeanType determining what the model outputs. + :param model_var_type: a ModelVarType determining how variance is output. + :param loss_type: a LossType determining the loss function to use. + :param rescale_timesteps: if True, pass floating point timesteps into the + model so that they are always scaled like in the + original paper (0 to 1000). + """ + + def __init__( + self, + *, + betas, + model_mean_type, + model_var_type, + loss_type, + rescale_timesteps=False, + lambda_rcxyz=0., + lambda_vel=0., + lambda_pose=1., + lambda_orient=1., + lambda_loc=1., + data_rep='rot6d', + lambda_root_vel=0., + lambda_vel_rcxyz=0., + lambda_fc=0., + ): + self.model_mean_type = model_mean_type + self.model_var_type = model_var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + self.data_rep = data_rep + + if data_rep != 'rot_vel' and lambda_pose != 1.: + raise ValueError( + 'lambda_pose is relevant only when training on velocities!') + self.lambda_pose = lambda_pose + self.lambda_orient = lambda_orient + self.lambda_loc = lambda_loc + + self.lambda_rcxyz = lambda_rcxyz + self.lambda_vel = lambda_vel + self.lambda_root_vel = lambda_root_vel + self.lambda_vel_rcxyz = lambda_vel_rcxyz + self.lambda_fc = lambda_fc + + if self.lambda_rcxyz > 0. or self.lambda_vel > 0. or self.lambda_root_vel > 0. or \ + self.lambda_vel_rcxyz > 0. or self.lambda_fc > 0.: + assert self.loss_type == LossType.MSE, 'Geometric losses are supported by MSE loss type only!' + + # Use float64 for accuracy. + betas = np.array(betas, dtype=np.float64) + self.betas = betas + assert len(betas.shape) == 1, 'betas must be 1-D' + assert (betas > 0).all() and (betas <= 1).all() + + self.num_timesteps = int(betas.shape[0]) + + alphas = 1.0 - betas + self.alphas_cumprod = np.cumprod(alphas, axis=0) + self.alphas_cumprod_prev = np.append(1.0, self.alphas_cumprod[:-1]) + self.alphas_cumprod_next = np.append(self.alphas_cumprod[1:], 0.0) + assert self.alphas_cumprod_prev.shape == (self.num_timesteps, ) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.sqrt_alphas_cumprod = np.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = np.sqrt(1.0 - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = np.log(1.0 - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod + - 1) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + # log calculation clipped because the posterior variance is 0 at the + # beginning of the diffusion chain. + self.posterior_log_variance_clipped = np.log( + np.append(self.posterior_variance[1], self.posterior_variance[1:])) + self.posterior_mean_coef1 = betas * np.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = (1.0 - self.alphas_cumprod_prev) * np.sqrt( + alphas) / (1.0 - self.alphas_cumprod) + + self.l2_loss = lambda a, b: ( + a - b + )**2 # th.nn.MSELoss(reduction='none') # must be None for handling mask later on. + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start) + variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, + x_start.shape) + log_variance = _extract_into_tensor(self.log_one_minus_alphas_cumprod, + t, x_start.shape) + return mean, variance, log_variance + + def q_sample(self, x_start, t, noise=None): + """ + Diffuse the dataset for a given number of diffusion steps. + + In other words, sample from q(x_t | x_0). + + :param x_start: the initial dataset batch. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :param noise: if specified, the split-out normal noise. + :return: A noisy version of x_start. + """ + if noise is None: + noise = th.randn_like(x_start) + assert noise.shape == x_start.shape + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + _extract_into_tensor( + self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise) + + def q_posterior_mean_variance(self, x_start, x_t, t): + """ + Compute the mean and variance of the diffusion posterior: + + q(x_{t-1} | x_t, x_0) + + """ + assert x_start.shape == x_t.shape + posterior_mean = ( + _extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) + * x_start + + _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) + * x_t) + posterior_variance = _extract_into_tensor(self.posterior_variance, t, + x_t.shape) + posterior_log_variance_clipped = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape) + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def p_mean_variance(self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + model_kwargs=None): + """ + Apply the model to get p(x_{t-1} | x_t), as well as a prediction of + the initial x, x_0. + + :param model: the model, which takes a signal and a batch of timesteps + as input. + :param x: the [N x C x ...] tensor at time t. + :param t: a 1-D Tensor of timesteps. + :param clip_denoised: if True, clip the denoised signal into [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. Applies before + clip_denoised. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict with the following keys: + - 'mean': the model mean output. + - 'variance': the model variance output. + - 'log_variance': the log of 'variance'. + - 'pred_xstart': the prediction for x_0. + """ + if model_kwargs is None: + model_kwargs = {} + + B, C = x.shape[:2] + assert t.shape == (B, ) + model_output = model(x, self._scale_timesteps(t), **model_kwargs) + + if 'inpainting_mask' in model_kwargs['y'].keys( + ) and 'inpainted_motion' in model_kwargs['y'].keys(): + inpainting_mask, inpainted_motion = model_kwargs['y'][ + 'inpainting_mask'], model_kwargs['y']['inpainted_motion'] + assert self.model_mean_type == ModelMeanType.START_X, 'This feature supports only X_start pred for mow!' + assert model_output.shape == inpainting_mask.shape == inpainted_motion.shape + model_output = (model_output * ~inpainting_mask) + ( + inpainted_motion * inpainting_mask) + + if self.model_var_type in [ + ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE + ]: + assert model_output.shape == (B, C * 2, *x.shape[2:]) + model_output, model_var_values = th.split(model_output, C, dim=1) + if self.model_var_type == ModelVarType.LEARNED: + model_log_variance = model_var_values + model_variance = th.exp(model_log_variance) + else: + min_log = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x.shape) + max_log = _extract_into_tensor(np.log(self.betas), t, x.shape) + # The model_var_values is [-1, 1] for [min_var, max_var]. + frac = (model_var_values + 1) / 2 + model_log_variance = frac * max_log + (1 - frac) * min_log + model_variance = th.exp(model_log_variance) + else: + model_variance, model_log_variance = { + ModelVarType.FIXED_LARGE: ( + np.append(self.posterior_variance[1], self.betas[1:]), + np.log( + np.append(self.posterior_variance[1], self.betas[1:])), + ), + ModelVarType.FIXED_SMALL: ( + self.posterior_variance, + self.posterior_log_variance_clipped, + ), + }[self.model_var_type] + + model_variance = _extract_into_tensor(model_variance, t, x.shape) + model_log_variance = _extract_into_tensor(model_log_variance, t, + x.shape) + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + if self.model_mean_type == ModelMeanType.PREVIOUS_X: + pred_xstart = process_xstart( + self._predict_xstart_from_xprev( + x_t=x, t=t, xprev=model_output)) + model_mean = model_output + elif self.model_mean_type in [ + ModelMeanType.START_X, ModelMeanType.EPSILON + ]: # THIS IS US! + if self.model_mean_type == ModelMeanType.START_X: + pred_xstart = process_xstart(model_output) + else: + pred_xstart = process_xstart( + self._predict_xstart_from_eps( + x_t=x, t=t, eps=model_output)) + model_mean, _, _ = self.q_posterior_mean_variance( + x_start=pred_xstart, x_t=x, t=t) + else: + raise NotImplementedError(self.model_mean_type) + + return { + 'mean': model_mean, + 'variance': model_variance, + 'log_variance': model_log_variance, + 'pred_xstart': pred_xstart, + } + + def _predict_xstart_from_eps(self, x_t, t, eps): + assert x_t.shape == eps.shape + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, + x_t.shape) * eps) + + def _predict_xstart_from_xprev(self, x_t, t, xprev): + assert x_t.shape == xprev.shape + return ( # (xprev - coef2*x_t) / coef1 + _extract_into_tensor(1.0 / self.posterior_mean_coef1, t, x_t.shape) + * xprev - _extract_into_tensor( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + x_t.shape) * x_t) + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - pred_xstart) / _extract_into_tensor( + self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * (1000.0 / self.num_timesteps) + return t + + def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, self._scale_timesteps(t), **model_kwargs) + new_mean = ( + p_mean_var['mean'].float() + + p_mean_var['variance'] * gradient.float()) + return new_mean + + def condition_mean_with_grad(self, + cond_fn, + p_mean_var, + x, + t, + model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, t, p_mean_var, **model_kwargs) + new_mean = ( + p_mean_var['mean'].float() + + p_mean_var['variance'] * gradient.float()) + return new_mean + + def p_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + const_noise=False, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + noise = th.randn_like(x) + if const_noise: + noise = noise[[0]].repeat(x.shape[0], 1, 1, 1) + + nonzero_mask = ((t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out['mean'] = self.condition_mean( + cond_fn, out, x, t, model_kwargs=model_kwargs) + sample = out['mean'] + nonzero_mask * th.exp( + 0.5 * out['log_variance']) * noise + return {'sample': sample, 'pred_xstart': out['pred_xstart']} + + def p_sample_with_grad( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + with th.enable_grad(): + x = x.detach().requires_grad_() + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + noise = th.randn_like(x) + nonzero_mask = ((t != 0).float().view(-1, + *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out['mean'] = self.condition_mean_with_grad( + cond_fn, out, x, t, model_kwargs=model_kwargs) + sample = out['mean'] + nonzero_mask * th.exp( + 0.5 * out['log_variance']) * noise + return {'sample': sample, 'pred_xstart': out['pred_xstart'].detach()} + + def p_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + dump_steps=None, + const_noise=False, + ): + """ + Generate samples from the model. + + :param model: the model module. + :param shape: the shape of the samples, (N, C, H, W). + :param noise: if specified, the noise from the encoder to sample. + Should be of the same shape as `shape`. + :param clip_denoised: if True, clip x_start predictions to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param device: if specified, the device to create the samples on. + If not specified, use a model parameter's device. + :param progress: if True, show a tqdm progress bar. + :param const_noise: If True, will noise all samples with the same noise throughout sampling + :return: a non-differentiable batch of samples. + """ + final = None + if dump_steps is not None: + dump = [] + + for i, sample in enumerate( + self.p_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + skip_timesteps=skip_timesteps, + init_image=init_image, + randomize_class=randomize_class, + cond_fn_with_grad=cond_fn_with_grad, + const_noise=const_noise, + )): + if dump_steps is not None and i in dump_steps: + dump.append(deepcopy(sample['sample'])) + final = sample + if dump_steps is not None: + return dump + return final['sample'] + + def p_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + const_noise=False, + ): + """ + Generate samples from the model and yield intermediate samples from + each timestep of diffusion. + + Arguments are the same as p_sample_loop(). + Returns a generator over dicts, where each dict is the return value of + p_sample(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + + if skip_timesteps and init_image is None: + init_image = th.zeros_like(img) + + indices = list(range(self.num_timesteps - skip_timesteps))[::-1] + + if init_image is not None: + my_t = th.ones([shape[0]], device=device, + dtype=th.long) * indices[0] + img = self.q_sample(init_image, my_t, img) + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + if randomize_class and 'y' in model_kwargs: + model_kwargs['y'] = th.randint( + low=0, + high=model.num_classes, + size=model_kwargs['y'].shape, + device=model_kwargs['y'].device) + with th.no_grad(): + sample_fn = self.p_sample_with_grad if cond_fn_with_grad else self.p_sample + out = sample_fn( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + const_noise=const_noise, + ) + yield out + img = out['sample'] + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + """ + Extract values from a 1-D numpy array for a batch of indices. + + :param arr: the 1-D numpy array. + :param timesteps: a tensor of indices into the array to extract. + :param broadcast_shape: a larger shape of K dimensions with the batch + dimension equal to the length of timesteps. + :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims. + """ + res = th.from_numpy(arr).to(device=timesteps.device)[timesteps].float() + while len(res.shape) < len(broadcast_shape): + res = res[..., None] + return res.expand(broadcast_shape) diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/mdm.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/mdm.py new file mode 100644 index 0000000000000000000000000000000000000000..716acd8305a6d8589828453e0aaf745eefc2f0cb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/mdm.py @@ -0,0 +1,364 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +import clip +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .rotation2xyz import Rotation2xyz + + +class MDM(nn.Module): + + def __init__(self, + modeltype, + njoints, + nfeats, + num_actions, + translation, + pose_rep, + glob, + glob_rot, + latent_dim=256, + ff_size=1024, + num_layers=8, + num_heads=4, + dropout=0.1, + smpl_data_path=None, + ablation=None, + activation='gelu', + legacy=False, + data_rep='rot6d', + dataset='amass', + clip_dim=512, + arch='trans_enc', + emb_trans_dec=False, + clip_version=None, + **kargs): + super().__init__() + + self.legacy = legacy + self.modeltype = modeltype + self.njoints = njoints + self.nfeats = nfeats + self.num_actions = num_actions + self.data_rep = data_rep + self.dataset = dataset + + self.pose_rep = pose_rep + self.glob = glob + self.glob_rot = glob_rot + self.translation = translation + + self.latent_dim = latent_dim + + self.ff_size = ff_size + self.num_layers = num_layers + self.num_heads = num_heads + self.dropout = dropout + + self.ablation = ablation + self.activation = activation + self.clip_dim = clip_dim + self.action_emb = kargs.get('action_emb', None) + + self.input_feats = self.njoints * self.nfeats + + self.normalize_output = kargs.get('normalize_encoder_output', False) + + self.cond_mode = kargs.get('cond_mode', 'no_cond') + self.cond_mask_prob = kargs.get('cond_mask_prob', 0.) + self.arch = arch + self.gru_emb_dim = self.latent_dim if self.arch == 'gru' else 0 + self.input_process = InputProcess(self.data_rep, + self.input_feats + self.gru_emb_dim, + self.latent_dim) + + self.sequence_pos_encoder = PositionalEncoding(self.latent_dim, + self.dropout) + self.emb_trans_dec = emb_trans_dec + + if self.arch == 'trans_enc': + print('TRANS_ENC init') + seqTransEncoderLayer = nn.TransformerEncoderLayer( + d_model=self.latent_dim, + nhead=self.num_heads, + dim_feedforward=self.ff_size, + dropout=self.dropout, + activation=self.activation) + + self.seqTransEncoder = nn.TransformerEncoder( + seqTransEncoderLayer, num_layers=self.num_layers) + elif self.arch == 'trans_dec': + print('TRANS_DEC init') + seqTransDecoderLayer = nn.TransformerDecoderLayer( + d_model=self.latent_dim, + nhead=self.num_heads, + dim_feedforward=self.ff_size, + dropout=self.dropout, + activation=activation) + self.seqTransDecoder = nn.TransformerDecoder( + seqTransDecoderLayer, num_layers=self.num_layers) + elif self.arch == 'gru': + print('GRU init') + self.gru = nn.GRU( + self.latent_dim, + self.latent_dim, + num_layers=self.num_layers, + batch_first=True) + else: + raise ValueError( + 'Please choose correct architecture [trans_enc, trans_dec, gru]' + ) + + self.embed_timestep = TimestepEmbedder(self.latent_dim, + self.sequence_pos_encoder) + + if self.cond_mode != 'no_cond': + if 'text' in self.cond_mode: + self.embed_text = nn.Linear(self.clip_dim, self.latent_dim) + print('EMBED TEXT') + print('Loading CLIP...') + self.clip_version = clip_version + self.clip_model = self.load_and_freeze_clip(clip_version) + if 'action' in self.cond_mode: + self.embed_action = EmbedAction(self.num_actions, + self.latent_dim) + print('EMBED ACTION') + + self.output_process = OutputProcess(self.data_rep, self.input_feats, + self.latent_dim, self.njoints, + self.nfeats) + + self.rot2xyz = Rotation2xyz( + device='cpu', smpl_data_path=smpl_data_path, dataset=self.dataset) + + def parameters_wo_clip(self): + return [ + p for name, p in self.named_parameters() + if not name.startswith('clip_model.') + ] + + def load_and_freeze_clip(self, clip_version): + clip_model, clip_preprocess = clip.load( + clip_version, device='cpu', + jit=False) # Must set jit=False for training + # clip.model.convert_weights( + # clip_model) # Actually this line is unnecessary since clip by default already on float16 + + # Freeze CLIP weights + clip_model.eval() + for p in clip_model.parameters(): + p.requires_grad = False + + return clip_model + + def mask_cond(self, cond, force_mask=False): + bs, d = cond.shape + if force_mask: + return torch.zeros_like(cond) + elif self.training and self.cond_mask_prob > 0.: + mask = torch.bernoulli( + torch.ones(bs, device=cond.device) * self.cond_mask_prob).view( + bs, 1) # 1-> use null_cond, 0-> use real cond + return cond * (1. - mask) + else: + return cond + + def encode_text(self, raw_text): + device = next(self.parameters()).device + max_text_len = 20 if self.dataset in [ + 'humanml', 'kit' + ] else None # Specific hardcoding for humanml dataset + if max_text_len is not None: + default_context_length = 77 + context_length = max_text_len + 2 # start_token + 20 + end_token + assert context_length < default_context_length + texts = clip.tokenize( + raw_text, context_length=context_length, + truncate=True).to(device) + zero_pad = torch.zeros( + [texts.shape[0], default_context_length - context_length], + dtype=texts.dtype, + device=texts.device) + texts = torch.cat([texts, zero_pad], dim=1) + else: + texts = clip.tokenize(raw_text, truncate=True).to(device) + return self.clip_model.encode_text(texts).float() + + def forward(self, x, timesteps, y=None): + """ + x: [batch_size, njoints, nfeats, max_frames], denoted x_t in the paper + timesteps: [batch_size] (int) + """ + bs, njoints, nfeats, nframes = x.shape + emb = self.embed_timestep(timesteps) # [1, bs, d] + + force_mask = y.get('uncond', False) + if 'text' in self.cond_mode: + enc_text = self.encode_text(y['text']) + emb += self.embed_text( + self.mask_cond(enc_text, force_mask=force_mask)) + if 'action' in self.cond_mode: + action_emb = self.embed_action(y['action']) + emb += self.mask_cond(action_emb, force_mask=force_mask) + + if self.arch == 'gru': + x_reshaped = x.reshape(bs, njoints * nfeats, 1, nframes) + emb_gru = emb.repeat(nframes, 1, 1) # [#frames, bs, d] + emb_gru = emb_gru.permute(1, 2, 0) # [bs, d, #frames] + emb_gru = emb_gru.reshape(bs, self.latent_dim, 1, + nframes) # [bs, d, 1, #frames] + x = torch.cat((x_reshaped, emb_gru), + axis=1) # [bs, d+joints*feat, 1, #frames] + + x = self.input_process(x) + + if self.arch == 'trans_enc': + # adding the timestep embed + xseq = torch.cat((emb, x), axis=0) # [seqlen+1, bs, d] + xseq = self.sequence_pos_encoder(xseq) # [seqlen+1, bs, d] + output = self.seqTransEncoder(xseq)[ + 1:] # , src_key_padding_mask=~maskseq) # [seqlen, bs, d] + + elif self.arch == 'trans_dec': + if self.emb_trans_dec: + xseq = torch.cat((emb, x), axis=0) + else: + xseq = x + xseq = self.sequence_pos_encoder(xseq) # [seqlen+1, bs, d] + if self.emb_trans_dec: + output = self.seqTransDecoder( + tgt=xseq, memory=emb + )[1:] # [seqlen, bs, d] # FIXME - maybe add a causal mask + else: + output = self.seqTransDecoder(tgt=xseq, memory=emb) + elif self.arch == 'gru': + xseq = x + xseq = self.sequence_pos_encoder(xseq) # [seqlen, bs, d] + output, _ = self.gru(xseq) + + output = self.output_process(output) # [bs, njoints, nfeats, nframes] + return output + + def _apply(self, fn): + super()._apply(fn) + self.rot2xyz.smpl_model._apply(fn) + + def train(self, *args, **kwargs): + super().train(*args, **kwargs) + self.rot2xyz.smpl_model.train(*args, **kwargs) + + +class PositionalEncoding(nn.Module): + + def __init__(self, d_model, dropout=0.1, max_len=5000): + super(PositionalEncoding, self).__init__() + self.dropout = nn.Dropout(p=dropout) + + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, d_model, 2).float() * (-np.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0).transpose(0, 1) + + self.register_buffer('pe', pe) + + def forward(self, x): + # not used in the final model + x = x + self.pe[:x.shape[0], :] + return self.dropout(x) + + +class TimestepEmbedder(nn.Module): + + def __init__(self, latent_dim, sequence_pos_encoder): + super().__init__() + self.latent_dim = latent_dim + self.sequence_pos_encoder = sequence_pos_encoder + + time_embed_dim = self.latent_dim + self.time_embed = nn.Sequential( + nn.Linear(self.latent_dim, time_embed_dim), + nn.SiLU(), + nn.Linear(time_embed_dim, time_embed_dim), + ) + + def forward(self, timesteps): + return self.time_embed( + self.sequence_pos_encoder.pe[timesteps]).permute(1, 0, 2) + + +class InputProcess(nn.Module): + + def __init__(self, data_rep, input_feats, latent_dim): + super().__init__() + self.data_rep = data_rep + self.input_feats = input_feats + self.latent_dim = latent_dim + self.poseEmbedding = nn.Linear(self.input_feats, self.latent_dim) + if self.data_rep == 'rot_vel': + self.velEmbedding = nn.Linear(self.input_feats, self.latent_dim) + + def forward(self, x): + bs, njoints, nfeats, nframes = x.shape + x = x.permute((3, 0, 1, 2)).reshape(nframes, bs, njoints * nfeats) + + if self.data_rep in ['rot6d', 'xyz', 'hml_vec']: + x = self.poseEmbedding(x) # [seqlen, bs, d] + return x + elif self.data_rep == 'rot_vel': + first_pose = x[[0]] # [1, bs, 150] + first_pose = self.poseEmbedding(first_pose) # [1, bs, d] + vel = x[1:] # [seqlen-1, bs, 150] + vel = self.velEmbedding(vel) # [seqlen-1, bs, d] + return torch.cat((first_pose, vel), axis=0) # [seqlen, bs, d] + else: + raise ValueError + + +class OutputProcess(nn.Module): + + def __init__(self, data_rep, input_feats, latent_dim, njoints, nfeats): + super().__init__() + self.data_rep = data_rep + self.input_feats = input_feats + self.latent_dim = latent_dim + self.njoints = njoints + self.nfeats = nfeats + self.poseFinal = nn.Linear(self.latent_dim, self.input_feats) + if self.data_rep == 'rot_vel': + self.velFinal = nn.Linear(self.latent_dim, self.input_feats) + + def forward(self, output): + nframes, bs, d = output.shape + if self.data_rep in ['rot6d', 'xyz', 'hml_vec']: + output = self.poseFinal(output) # [seqlen, bs, 150] + elif self.data_rep == 'rot_vel': + first_pose = output[[0]] # [1, bs, d] + first_pose = self.poseFinal(first_pose) # [1, bs, 150] + vel = output[1:] # [seqlen-1, bs, d] + vel = self.velFinal(vel) # [seqlen-1, bs, 150] + output = torch.cat((first_pose, vel), axis=0) # [seqlen, bs, 150] + else: + raise ValueError + output = output.reshape(nframes, bs, self.njoints, self.nfeats) + output = output.permute(1, 2, 3, 0) # [bs, njoints, nfeats, nframes] + return output + + +class EmbedAction(nn.Module): + + def __init__(self, num_actions, latent_dim): + super().__init__() + self.action_embedding = nn.Parameter( + torch.randn(num_actions, latent_dim)) + + def forward(self, input): + idx = input[:, 0].to(torch.long) # an index array must be long + output = self.action_embedding[idx] + return output diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/respace.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/respace.py new file mode 100644 index 0000000000000000000000000000000000000000..45c37ee6e98d8430c91d4c919a57c726159ebcc1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/respace.py @@ -0,0 +1,132 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +import numpy as np +import torch as th + +from .gaussian_diffusion import GaussianDiffusion + + +def space_timesteps(num_timesteps, section_counts): + """ + Create a list of timesteps to use from an original diffusion process, + given the number of timesteps we want to take from equally-sized portions + of the original process. + + For example, if there's 300 timesteps and the section counts are [10,15,20] + then the first 100 timesteps are strided to be 10 timesteps, the second 100 + are strided to be 15 timesteps, and the final 100 are strided to be 20. + + If the stride is a string starting with "ddim", then the fixed striding + from the DDIM paper is used, and only one section is allowed. + + :param num_timesteps: the number of diffusion steps in the original + process to divide up. + :param section_counts: either a list of numbers, or a string containing + comma-separated numbers, indicating the step count + per section. As a special case, use "ddimN" where N + is a number of steps to use the striding from the + DDIM paper. + :return: a set of diffusion steps from the original process to use. + """ + if isinstance(section_counts, str): + if section_counts.startswith('ddim'): + desired_count = int(section_counts[len('ddim'):]) + for i in range(1, num_timesteps): + if len(range(0, num_timesteps, i)) == desired_count: + return set(range(0, num_timesteps, i)) + raise ValueError( + f'cannot create exactly {num_timesteps} steps with an integer stride' + ) + section_counts = [int(x) for x in section_counts.split(',')] + size_per = num_timesteps // len(section_counts) + extra = num_timesteps % len(section_counts) + start_idx = 0 + all_steps = [] + for i, section_count in enumerate(section_counts): + size = size_per + (1 if i < extra else 0) + if size < section_count: + raise ValueError( + f'cannot divide section of {size} steps into {section_count}') + if section_count <= 1: + frac_stride = 1 + else: + frac_stride = (size - 1) / (section_count - 1) + cur_idx = 0.0 + taken_steps = [] + for _ in range(section_count): + taken_steps.append(start_idx + round(cur_idx)) + cur_idx += frac_stride + all_steps += taken_steps + start_idx += size + return set(all_steps) + + +class SpacedDiffusion(GaussianDiffusion): + """ + A diffusion process which can skip steps in a base diffusion process. + + :param use_timesteps: a collection (sequence or set) of timesteps from the + original diffusion process to retain. + :param kwargs: the kwargs to create the base diffusion process. + """ + + def __init__(self, use_timesteps, **kwargs): + self.use_timesteps = set(use_timesteps) + self.timestep_map = [] + self.original_num_steps = len(kwargs['betas']) + + base_diffusion = GaussianDiffusion(**kwargs) # pylint: disable=missing-kwoa + last_alpha_cumprod = 1.0 + new_betas = [] + for i, alpha_cumprod in enumerate(base_diffusion.alphas_cumprod): + if i in self.use_timesteps: + new_betas.append(1 - alpha_cumprod / last_alpha_cumprod) + last_alpha_cumprod = alpha_cumprod + self.timestep_map.append(i) + kwargs['betas'] = np.array(new_betas) + super().__init__(**kwargs) + + def p_mean_variance(self, model, *args, **kwargs): # pylint: disable=signature-differs + return super().p_mean_variance( + self._wrap_model(model), *args, **kwargs) + + def training_losses(self, model, *args, **kwargs): # pylint: disable=signature-differs + return super().training_losses( + self._wrap_model(model), *args, **kwargs) + + def condition_mean(self, cond_fn, *args, **kwargs): + return super().condition_mean( + self._wrap_model(cond_fn), *args, **kwargs) + + def condition_score(self, cond_fn, *args, **kwargs): + return super().condition_score( + self._wrap_model(cond_fn), *args, **kwargs) + + def _wrap_model(self, model): + if isinstance(model, _WrappedModel): + return model + return _WrappedModel(model, self.timestep_map, self.rescale_timesteps, + self.original_num_steps) + + def _scale_timesteps(self, t): + # Scaling is done by the wrapped model. + return t + + +class _WrappedModel: + + def __init__(self, model, timestep_map, rescale_timesteps, + original_num_steps): + self.model = model + self.timestep_map = timestep_map + self.rescale_timesteps = rescale_timesteps + self.original_num_steps = original_num_steps + + def __call__(self, x, ts, **kwargs): + map_tensor = th.tensor( + self.timestep_map, device=ts.device, dtype=ts.dtype) + new_ts = map_tensor[ts] + if self.rescale_timesteps: + new_ts = new_ts.float() * (1000.0 / self.original_num_steps) + return self.model(x, new_ts, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/rotation2xyz.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/rotation2xyz.py new file mode 100644 index 0000000000000000000000000000000000000000..0b00015e66196057a43703cc5a394e96219b6785 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/rotation2xyz.py @@ -0,0 +1,112 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +import torch + +from modelscope.utils.cv.motion_utils import rotation_conversions as geometry +from .smpl import JOINTSTYPE_ROOT, SMPL + +JOINTSTYPES = ['a2m', 'a2mpl', 'smpl', 'vibe', 'vertices'] + + +class Rotation2xyz: + + def __init__(self, device, smpl_data_path, dataset='amass'): + self.device = device + self.dataset = dataset + self.smpl_model = SMPL(smpl_data_path).eval().to(device) + + def __call__(self, + x, + mask, + pose_rep, + translation, + glob, + jointstype, + vertstrans, + betas=None, + beta=0, + glob_rot=None, + get_rotations_back=False, + **kwargs): + if pose_rep == 'xyz': + return x + + if mask is None: + mask = torch.ones((x.shape[0], x.shape[-1]), + dtype=bool, + device=x.device) + + if not glob and glob_rot is None: + raise TypeError( + 'You must specify global rotation if glob is False') + + if jointstype not in JOINTSTYPES: + raise NotImplementedError('This jointstype is not implemented.') + + if translation: + x_translations = x[:, -1, :3] + x_rotations = x[:, :-1] + else: + x_rotations = x + + x_rotations = x_rotations.permute(0, 3, 1, 2) + nsamples, time, njoints, feats = x_rotations.shape + + # Compute rotations (convert only masked sequences output) + if pose_rep == 'rotvec': + rotations = geometry.axis_angle_to_matrix(x_rotations[mask]) + elif pose_rep == 'rotmat': + rotations = x_rotations[mask].view(-1, njoints, 3, 3) + elif pose_rep == 'rotquat': + rotations = geometry.quaternion_to_matrix(x_rotations[mask]) + elif pose_rep == 'rot6d': + rotations = geometry.rotation_6d_to_matrix(x_rotations[mask]) + else: + raise NotImplementedError('No geometry for this one.') + + if not glob: + global_orient = torch.tensor(glob_rot, device=x.device) + global_orient = geometry.axis_angle_to_matrix(global_orient).view( + 1, 1, 3, 3) + global_orient = global_orient.repeat(len(rotations), 1, 1, 1) + else: + global_orient = rotations[:, 0] + rotations = rotations[:, 1:] + + if betas is None: + betas = torch.zeros( + [rotations.shape[0], self.smpl_model.num_betas], + dtype=rotations.dtype, + device=rotations.device) + betas[:, 1] = beta + # import ipdb; ipdb.set_trace() + out = self.smpl_model( + body_pose=rotations, global_orient=global_orient, betas=betas) + + # get the desirable joints + joints = out[jointstype] + + x_xyz = torch.empty( + nsamples, time, joints.shape[1], 3, device=x.device, dtype=x.dtype) + x_xyz[~mask] = 0 + x_xyz[mask] = joints + + x_xyz = x_xyz.permute(0, 2, 3, 1).contiguous() + + # the first translation root at the origin on the prediction + if jointstype != 'vertices': + rootindex = JOINTSTYPE_ROOT[jointstype] + x_xyz = x_xyz - x_xyz[:, [rootindex], :, :] + + if translation and vertstrans: + # the first translation root at the origin + x_translations = x_translations - x_translations[:, :, [0]] + + # add the translation to all the joints + x_xyz = x_xyz + x_translations[:, None, :, :] + + if get_rotations_back: + return x_xyz, rotations, global_orient + else: + return x_xyz diff --git a/runtime/Lib/site_packages/models/cv/motion_generation/modules/smpl.py b/runtime/Lib/site_packages/models/cv/motion_generation/modules/smpl.py new file mode 100644 index 0000000000000000000000000000000000000000..60b027dee95a5055498eac2465ab6b5858f04db8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/motion_generation/modules/smpl.py @@ -0,0 +1,117 @@ +# This code is borrowed and modified from Human Motion Diffusion Model, +# made publicly available under MIT license at https://github.com/GuyTevet/motion-diffusion-model + +import contextlib +import os.path as osp + +import numpy as np +import torch +from smplx import SMPLLayer as _SMPLLayer +from smplx.lbs import vertices2joints + +action2motion_joints = [ + 8, 1, 2, 3, 4, 5, 6, 7, 0, 9, 10, 11, 12, 13, 14, 21, 24, 38 +] + +JOINTSTYPE_ROOT = { + 'a2m': 0, # action2motion + 'smpl': 0, + 'a2mpl': 0, # set(smpl, a2m) + 'vibe': 8 +} # 0 is the 8 position: OP MidHip below + +JOINT_MAP = { + 'OP Nose': 24, + 'OP Neck': 12, + 'OP RShoulder': 17, + 'OP RElbow': 19, + 'OP RWrist': 21, + 'OP LShoulder': 16, + 'OP LElbow': 18, + 'OP LWrist': 20, + 'OP MidHip': 0, + 'OP RHip': 2, + 'OP RKnee': 5, + 'OP RAnkle': 8, + 'OP LHip': 1, + 'OP LKnee': 4, + 'OP LAnkle': 7, + 'OP REye': 25, + 'OP LEye': 26, + 'OP REar': 27, + 'OP LEar': 28, + 'OP LBigToe': 29, + 'OP LSmallToe': 30, + 'OP LHeel': 31, + 'OP RBigToe': 32, + 'OP RSmallToe': 33, + 'OP RHeel': 34, + 'Right Ankle': 8, + 'Right Knee': 5, + 'Right Hip': 45, + 'Left Hip': 46, + 'Left Knee': 4, + 'Left Ankle': 7, + 'Right Wrist': 21, + 'Right Elbow': 19, + 'Right Shoulder': 17, + 'Left Shoulder': 16, + 'Left Elbow': 18, + 'Left Wrist': 20, + 'Neck (LSP)': 47, + 'Top of Head (LSP)': 48, + 'Pelvis (MPII)': 49, + 'Thorax (MPII)': 50, + 'Spine (H36M)': 51, + 'Jaw (H36M)': 52, + 'Head (H36M)': 53, + 'Nose': 24, + 'Left Eye': 26, + 'Right Eye': 25, + 'Left Ear': 28, + 'Right Ear': 27 +} + +JOINT_NAMES = list(JOINT_MAP.keys()) + + +class SMPL(_SMPLLayer): + """ Extension of the official SMPL implementation to support more joints """ + + def __init__(self, smpl_data_path, **kwargs): + kwargs['model_path'] = osp.join(smpl_data_path, 'SMPL_NEUTRAL.pkl') + + # remove the verbosity for the 10-shapes beta parameters + with contextlib.redirect_stdout(None): + super(SMPL, self).__init__(**kwargs) + + J_regressor_extra = np.load( + osp.join(smpl_data_path, 'J_regressor_extra.npy')) + self.register_buffer( + 'J_regressor_extra', + torch.tensor(J_regressor_extra, dtype=torch.float32)) + vibe_indexes = np.array([JOINT_MAP[i] for i in JOINT_NAMES]) + a2m_indexes = vibe_indexes[action2motion_joints] + smpl_indexes = np.arange(24) + a2mpl_indexes = np.unique(np.r_[smpl_indexes, a2m_indexes]) + + self.maps = { + 'vibe': vibe_indexes, + 'a2m': a2m_indexes, + 'smpl': smpl_indexes, + 'a2mpl': a2mpl_indexes + } + + def forward(self, *args, **kwargs): + smpl_output = super(SMPL, self).forward(*args, **kwargs) + + extra_joints = vertices2joints(self.J_regressor_extra, + smpl_output.vertices) + all_joints = torch.cat([smpl_output.joints, extra_joints], dim=1) + + output = {'vertices': smpl_output.vertices} + + for joinstype, indexes in self.maps.items(): + output[joinstype] = all_joints[:, indexes] + + return output diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..25dcda96432ae96483b16bcf98e1e33e6a488bf5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model import MovieSceneSegmentationModel + from .datasets import MovieSceneSegmentationDataset + +else: + _import_structure = { + 'model': ['MovieSceneSegmentationModel'], + 'datasets': ['MovieSceneSegmentationDataset'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/get_model.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/get_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5c66fc020e028816fd346c73b417f67c79a63932 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/get_model.py @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------------------ +# BaSSL +# Copyright (c) 2021 KakaoBrain. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# Github: https://github.com/kakaobrain/bassl +# ------------------------------------------------------------------------------------ + +from .utils.shot_encoder import resnet50 +from .utils.trn import TransformerCRN + + +def get_shot_encoder(cfg): + name = cfg['model']['shot_encoder']['name'] + shot_encoder_args = cfg['model']['shot_encoder'][name] + if name == 'resnet': + depth = shot_encoder_args['depth'] + if depth == 50: + shot_encoder = resnet50(**shot_encoder_args['params'], ) + else: + raise NotImplementedError + else: + raise NotImplementedError + + return shot_encoder + + +def get_contextual_relation_network(cfg): + crn = None + + if cfg['model']['contextual_relation_network']['enabled']: + name = cfg['model']['contextual_relation_network']['name'] + crn_args = cfg['model']['contextual_relation_network']['params'][name] + if name == 'trn': + sampling_name = cfg['model']['loss']['sampling_method']['name'] + crn_args['neighbor_size'] = ( + 2 * cfg['model']['loss']['sampling_method']['params'] + [sampling_name]['neighbor_size']) + crn = TransformerCRN(crn_args) + else: + raise NotImplementedError + + return crn + + +__all__ = ['get_shot_encoder', 'get_contextual_relation_network'] diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/model.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..818a38760ae1dc61db735e5760ede26f71bda28b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/model.py @@ -0,0 +1,229 @@ +# The implementation here is modified based on BaSSL, +# originally Apache 2.0 License and publicly available at https://github.com/kakaobrain/bassl + +import math +import os +import os.path as osp +from typing import Any, Dict + +import einops +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.transforms as TF +from PIL import Image +from shotdetect_scenedetect_lgss import shot_detector +from tqdm import tqdm + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .get_model import get_contextual_relation_network, get_shot_encoder +from .utils.save_op import get_pred_boundary, pred2scene, scene2video + +logger = get_logger() + + +@MODELS.register_module( + Tasks.movie_scene_segmentation, module_name=Models.resnet50_bert) +class MovieSceneSegmentationModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + params = torch.load(model_path, map_location='cpu') + + config_path = osp.join(model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(config_path) + + def load_param_with_prefix(prefix, model, src_params): + own_state = model.state_dict() + for name, param in own_state.items(): + src_name = prefix + '.' + name + own_state[name] = src_params[src_name] + + model.load_state_dict(own_state) + + self.shot_encoder = get_shot_encoder(self.cfg) + load_param_with_prefix('shot_encoder', self.shot_encoder, params) + self.crn = get_contextual_relation_network(self.cfg) + load_param_with_prefix('crn', self.crn, params) + + crn_name = self.cfg.model.contextual_relation_network.name + hdim = self.cfg.model.contextual_relation_network.params[crn_name][ + 'hidden_size'] + self.head_sbd = nn.Linear(hdim, 2) + load_param_with_prefix('head_sbd', self.head_sbd, params) + + self.shot_detector = shot_detector() + self.shot_detector.init(**self.cfg.preprocessor.shot_detect) + + self.test_transform = TF.Compose([ + TF.Resize(size=256, interpolation=Image.BICUBIC), + TF.CenterCrop(224), + TF.ToTensor(), + TF.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + sampling_method = self.cfg.dataset.sampling_method.name + self.neighbor_size = self.cfg.dataset.sampling_method.params[ + sampling_method].neighbor_size + + self.eps = 1e-5 + + def forward(self, inputs: Dict[str, Any]) -> Dict[str, torch.Tensor]: + data = inputs.pop('video') + labels = inputs['label'] + outputs = self.shared_step(data) + + loss = F.cross_entropy( + outputs.squeeze(), labels.squeeze(), reduction='none') + lpos = labels == 1 + lneg = labels == 0 + + pp, nn = 1, 1 + wp = (pp / float(pp + nn)) * lpos / (lpos.sum() + self.eps) + wn = (nn / float(pp + nn)) * lneg / (lneg.sum() + self.eps) + w = wp + wn + loss = (w * loss).sum() + + probs = torch.argmax(outputs, dim=1) + + re = dict(pred=probs, loss=loss) + return re + + def inference(self, batch): + logger.info('Begin scene detect ......') + bs = self.cfg.pipeline.batch_size_per_gpu + device = self.crn.attention_mask.device + + shot_timecode_lst = batch['shot_timecode_lst'] + shot_idx_lst = batch['shot_idx_lst'] + + shot_num = len(shot_timecode_lst) + cnt = math.ceil(shot_num / bs) + + infer_pred = [] + infer_result = {} + self.shot_detector.start() + + for i in tqdm(range(cnt)): + start = i * bs + end = (i + 1) * bs if (i + 1) * bs < shot_num else shot_num + + batch_shot_idx_lst = shot_idx_lst[start:end] + + shot_start_idx = batch_shot_idx_lst[0][0] + shot_end_idx = batch_shot_idx_lst[-1][-1] + batch_timecode_lst = { + i: shot_timecode_lst[i] + for i in range(shot_start_idx, shot_end_idx + 1) + } + batch_shot_keyf_lst = self.shot_detector.get_frame_img( + batch_timecode_lst, shot_start_idx, shot_num) + inputs = self.get_batch_input(batch_shot_keyf_lst, shot_start_idx, + batch_shot_idx_lst) + + input_ = torch.stack(inputs).to(device) + outputs = self.shared_step(input_) # shape [b,2] + prob = F.softmax(outputs, dim=1) + infer_pred.extend(prob[:, 1].cpu().detach().numpy()) + + infer_result.update({'pred': np.stack(infer_pred)}) + infer_result.update({'sid': np.arange(shot_num)}) + + assert len(infer_result['pred']) == shot_num + self.shot_detector.release() + return infer_result + + def shared_step(self, inputs): + with torch.no_grad(): + # infer shot encoder + shot_repr = self.extract_shot_representation(inputs) + assert len(shot_repr.shape) == 3 + + # infer CRN + _, pooled = self.crn(shot_repr, mask=None) + # infer boundary score + pred = self.head_sbd(pooled) + return pred + + def save_shot_feat(self, _repr): + feat = _repr.float().cpu().numpy() + pth = self.cfg.dataset.img_path + '/features' + os.makedirs(pth) + + for idx in range(_repr.shape[0]): + name = f'shot_{str(idx).zfill(4)}.npy' + name = osp.join(pth, name) + np.save(name, feat[idx]) + + def extract_shot_representation(self, + inputs: torch.Tensor) -> torch.Tensor: + """ inputs [b s k c h w] -> output [b d] """ + assert len(inputs.shape) == 6 # (B Shot Keyframe C H W) + b, s, k, c, h, w = inputs.shape + inputs = einops.rearrange(inputs, 'b s k c h w -> (b s) k c h w', s=s) + keyframe_repr = [self.shot_encoder(inputs[:, _k]) for _k in range(k)] + # [k (b s) d] -> [(b s) d] + shot_repr = torch.stack(keyframe_repr).mean(dim=0) + + shot_repr = einops.rearrange(shot_repr, '(b s) d -> b s d', s=s) + return shot_repr + + def postprocess(self, inputs: Dict[str, Any], **kwargs): + logger.info('Generate scene .......') + + pred_dict = inputs['feat'] + shot2keyf = inputs['shot2keyf'] + thres = self.cfg.pipeline.save_threshold + + anno_dict = get_pred_boundary(pred_dict, thres) + scene_dict_lst, scene_list, shot_num, shot_dict_lst = pred2scene( + shot2keyf, anno_dict) + if self.cfg.pipeline.save_split_scene: + re_dir = scene2video(inputs['input_video_pth'], scene_list, thres) + print(f'Split scene video saved to {re_dir}') + return len(scene_list), scene_dict_lst, shot_num, shot_dict_lst + + def get_batch_input(self, shot_keyf_lst, shot_start_idx, shot_idx_lst): + + single_shot_feat = [] + for idx, one_shot in enumerate(shot_keyf_lst): + one_shot = [ + self.test_transform(one_frame) for one_frame in one_shot + ] + one_shot = torch.stack(one_shot, dim=0) + single_shot_feat.append(one_shot) + + single_shot_feat = torch.stack(single_shot_feat, dim=0) + + shot_feat = [] + for idx, shot_idx in enumerate(shot_idx_lst): + shot_idx_ = shot_idx - shot_start_idx + _one_shot = single_shot_feat[shot_idx_] + shot_feat.append(_one_shot) + + return shot_feat + + def preprocess(self, inputs): + logger.info('Begin shot detect......') + shot_timecode_lst, anno, shot2keyf = self.shot_detector.shot_detect( + inputs, **self.cfg.preprocessor.shot_detect) + logger.info('Shot detect done!') + + shot_idx_lst = [] + for idx, one_shot in enumerate(anno): + shot_idx = int(one_shot['shot_id']) + np.arange( + -self.neighbor_size, self.neighbor_size + 1) + shot_idx = np.clip(shot_idx, 0, one_shot['num_shot'] - 1) + shot_idx_lst.append(shot_idx) + + return shot2keyf, anno, shot_timecode_lst, shot_idx_lst diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/__init__.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e5a929aad2081f24a4ce4ccef9e38f495cf09f2c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .save_op import get_pred_boundary, pred2scene, scene2video +from .shot_encoder import resnet50 +from .trn import TransformerCRN diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/head.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/head.py new file mode 100644 index 0000000000000000000000000000000000000000..6594af8f12a309786279ad9c48fa30e1a1efb0e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/head.py @@ -0,0 +1,25 @@ +# The implementation here is modified based on BaSSL, +# originally Apache 2.0 License and publicly available at https://github.com/kakaobrain/bassl + +import torch.nn as nn +import torch.nn.functional as F + + +class MlpHead(nn.Module): + + def __init__(self, input_dim=2048, hidden_dim=2048, output_dim=128): + super().__init__() + self.output_dim = output_dim + self.input_dim = input_dim + self.hidden_dim = hidden_dim + + self.model = nn.Sequential( + nn.Linear(self.input_dim, self.hidden_dim, bias=True), + nn.ReLU(), + nn.Linear(self.hidden_dim, self.output_dim, bias=True), + ) + + def forward(self, x): + # x shape: [b t d] where t means the number of views + x = self.model(x) + return F.normalize(x, dim=-1) diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/save_op.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/save_op.py new file mode 100644 index 0000000000000000000000000000000000000000..34bebce01002b32aade5001edeb578fb14d8ee9e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/save_op.py @@ -0,0 +1,128 @@ +# The implementation here is modified based on SceneSeg, +# originally Apache 2.0 License and publicly available at https://github.com/AnyiRao/SceneSeg +import os +import os.path as osp +import subprocess + +import cv2 +import numpy as np +from tqdm import tqdm + + +def get_pred_boundary(pred_dict, threshold=0.5): + pred = pred_dict['pred'].cpu().numpy() + sid = pred_dict['sid'].cpu().numpy().astype(np.int32) + tmp = (pred > threshold).astype(np.int32) + anno_dict = {} + for idx in range(len(tmp)): + anno_dict.update({str(sid[idx]).zfill(4): int(tmp[idx])}) + return anno_dict + + +def pred2scene(shot2keyf, anno_dict): + scene_list, pair_list = get_demo_scene_list(shot2keyf, anno_dict) + + scene_dict_lst = [] + shot_num = len(shot2keyf) + shot_dict_lst = [] + for item in shot2keyf: + tmp = item.split(' ') + shot_dict_lst.append({ + 'frame': [tmp[0], tmp[1]], + 'timestamps': [tmp[-2], tmp[-1]] + }) + assert len(scene_list) == len(pair_list) + for scene_ind, scene_item in enumerate(scene_list): + scene_dict_lst.append({ + 'shot': pair_list[scene_ind], + 'frame': scene_item[0], + 'timestamps': scene_item[1] + }) + + return scene_dict_lst, scene_list, shot_num, shot_dict_lst + + +def scene2video(source_movie_fn, scene_list, thres): + + vcap = cv2.VideoCapture(source_movie_fn) + fps = vcap.get(cv2.CAP_PROP_FPS) # video.fps + out_video_dir_fn = os.path.join(os.getcwd(), + f'pred_result/scene_video_{thres}') + os.makedirs(out_video_dir_fn, exist_ok=True) + + for scene_ind, scene_item in tqdm(enumerate(scene_list)): + scene = str(scene_ind).zfill(4) + start_frame = int(scene_item[0][0]) + end_frame = int(scene_item[0][1]) + start_time, end_time = start_frame / fps, end_frame / fps + duration_time = end_time - start_time + out_video_fn = os.path.join(out_video_dir_fn, + 'scene_{}.mp4'.format(scene)) + if os.path.exists(out_video_fn): + continue + call_list = ['ffmpeg'] + call_list += ['-v', 'quiet'] + call_list += [ + '-y', '-ss', + str(start_time), '-t', + str(duration_time), '-i', source_movie_fn + ] + call_list += ['-map_chapters', '-1'] + call_list += [out_video_fn] + subprocess.call(call_list) + return osp.join(os.getcwd(), 'pred_result') + + +def get_demo_scene_list(shot2keyf, anno_dict): + pair_list = get_pair_list(anno_dict) + + scene_list = [] + for pair in pair_list: + start_shot, end_shot = int(pair[0]), int(pair[-1]) + start_frame = shot2keyf[start_shot].split(' ')[0] + end_frame = shot2keyf[end_shot].split(' ')[1] + start_timestamp = shot2keyf[start_shot].split(' ')[-2] + end_timestamp = shot2keyf[end_shot].split(' ')[-1] + scene_list.append([[start_frame, end_frame], + [start_timestamp, end_timestamp]]) + return scene_list, pair_list + + +def get_pair_list(anno_dict): + sort_anno_dict_key = sorted(anno_dict.keys()) + tmp = 0 + tmp_list = [] + tmp_label_list = [] + anno_list = [] + anno_label_list = [] + for key in sort_anno_dict_key: + value = anno_dict.get(key) + tmp += value + tmp_list.append(key) + tmp_label_list.append(value) + if tmp == 1: + anno_list.append(tmp_list) + anno_label_list.append(tmp_label_list) + tmp = 0 + tmp_list = [] + tmp_label_list = [] + continue + if key == sort_anno_dict_key[-1]: + if len(tmp_list) > 0: + anno_list.append(tmp_list) + anno_label_list.append(tmp_label_list) + if len(anno_list) == 0: + return None + while [] in anno_list: + anno_list.remove([]) + tmp_anno_list = [anno_list[0]] + pair_list = [] + for ind in range(len(anno_list) - 1): + cont_count = int(anno_list[ind + 1][0]) - int(anno_list[ind][-1]) + if cont_count > 1: + pair_list.extend(tmp_anno_list) + tmp_anno_list = [anno_list[ind + 1]] + continue + tmp_anno_list.append(anno_list[ind + 1]) + pair_list.extend(tmp_anno_list) + return pair_list diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/shot_encoder.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/shot_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..11d20b13adae6a3d6396a0ec307afebe714bfcff --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/shot_encoder.py @@ -0,0 +1,329 @@ +# The implementation is adopted from torchvision + +from typing import Any, Callable, List, Optional, Type, Union + +import torch +import torch.nn as nn +from torch import Tensor + + +def conv3x3(in_planes: int, + out_planes: int, + stride: int = 1, + groups: int = 1, + dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation, + ) + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.0)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + in_channel_dim: int = 3, + zero_init_residual: bool = False, + use_last_block_grid: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.use_last_block_grid = use_last_block_grid + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + in_channel_dim, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias=False, + ) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, + 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, + 0) # type: ignore[arg-type] + + def _make_layer( + self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False, + ) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block( + self.inplanes, + planes, + stride, + downsample, + self.groups, + self.base_width, + previous_dilation, + norm_layer, + )) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer, + )) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor, grid: bool, level: List, both: bool, + grid_only: bool) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + + if grid: + x_grid = [] + + if 3 in level: + x_grid.append(x.detach().clone()) + if not both and len(level) == 1: + return x_grid + + x = self.layer4(x) + + if 4 in level: + x_grid.append(x.detach().clone()) + if not both and len(level) == 1: + return x_grid + + x = self.avgpool(x) + x = torch.flatten(x, 1) + + if not grid or len(level) == 0: + return x + + if grid_only: + return x_grid + + if both: + return x, x_grid + + return x + + def forward( + self, + x: Tensor, + grid: bool = False, + level: List = [], + both: bool = False, + grid_only: bool = False, + ) -> Tensor: + return self._forward_impl(x, grid, level, both, grid_only) + + +def resnet50(**kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + """ + return ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/trn.py b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/trn.py new file mode 100644 index 0000000000000000000000000000000000000000..769e9ee4dfe2a80cdaef93b5b229007bb955426f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/movie_scene_segmentation/utils/trn.py @@ -0,0 +1,132 @@ +# ------------------------------------------------------------------------------------ +# BaSSL +# Copyright (c) 2021 KakaoBrain. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# Github: https://github.com/kakaobrain/bassl +# ------------------------------------------------------------------------------------ + +import torch +import torch.nn as nn +from transformers.models.bert.modeling_bert import BertEncoder + + +class ShotEmbedding(nn.Module): + + def __init__(self, cfg): + super().__init__() + + nn_size = cfg.neighbor_size + 2 # +1 for center shot, +1 for cls + self.shot_embedding = nn.Linear(cfg.input_dim, cfg.hidden_size) + self.position_embedding = nn.Embedding(nn_size, cfg.hidden_size) + self.mask_embedding = nn.Embedding(2, cfg.input_dim, padding_idx=0) + + # tf naming convention for layer norm + self.LayerNorm = nn.LayerNorm(cfg.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(cfg.hidden_dropout_prob) + + self.register_buffer('pos_ids', + torch.arange(nn_size, dtype=torch.long)) + + def forward( + self, + shot_emb: torch.Tensor, + mask: torch.Tensor = None, + pos_ids: torch.Tensor = None, + ) -> torch.Tensor: + + assert len(shot_emb.size()) == 3 + + if pos_ids is None: + pos_ids = self.pos_ids + + # this for mask embedding (un-masked ones remain unchanged) + if mask is not None: + self.mask_embedding.weight.data[0, :].fill_(0) + mask_emb = self.mask_embedding(mask.long()) + shot_emb = (shot_emb * (1 - mask).float()[:, :, None]) + mask_emb + + # we set [CLS] token to averaged feature + cls_emb = shot_emb.mean(dim=1) + + # embedding shots + shot_emb = torch.cat([cls_emb[:, None, :], shot_emb], dim=1) + shot_emb = self.shot_embedding(shot_emb) + pos_emb = self.position_embedding(pos_ids) + embeddings = shot_emb + pos_emb[None, :] + embeddings = self.dropout(self.LayerNorm(embeddings)) + return embeddings + + +class TransformerCRN(nn.Module): + + def __init__(self, cfg): + super().__init__() + + self.pooling_method = cfg.pooling_method + self.shot_embedding = ShotEmbedding(cfg) + self.encoder = BertEncoder(cfg) + + nn_size = cfg.neighbor_size + 2 # +1 for center shot, +1 for cls + self.register_buffer( + 'attention_mask', + self._get_extended_attention_mask( + torch.ones((1, nn_size)).float()), + ) + + def forward( + self, + shot: torch.Tensor, + mask: torch.Tensor = None, + pos_ids: torch.Tensor = None, + pooling_method: str = None, + ): + if self.attention_mask.shape[1] != (shot.shape[1] + 1): + n_shot = shot.shape[1] + 1 # +1 for CLS token + attention_mask = self._get_extended_attention_mask( + torch.ones((1, n_shot), dtype=torch.float, device=shot.device)) + else: + attention_mask = self.attention_mask + + shot_emb = self.shot_embedding(shot, mask=mask, pos_ids=pos_ids) + encoded_emb = self.encoder( + shot_emb, attention_mask=attention_mask).last_hidden_state + + return encoded_emb, self.pooler( + encoded_emb, pooling_method=pooling_method) + + def pooler(self, sequence_output, pooling_method=None): + if pooling_method is None: + pooling_method = self.pooling_method + + if pooling_method == 'cls': + return sequence_output[:, 0, :] + elif pooling_method == 'avg': + return sequence_output[:, 1:].mean(dim=1) + elif pooling_method == 'max': + return sequence_output[:, 1:].max(dim=1)[0] + elif pooling_method == 'center': + cidx = sequence_output.shape[1] // 2 + return sequence_output[:, cidx, :] + else: + raise ValueError + + def _get_extended_attention_mask(self, attention_mask): + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + f'Wrong shape for attention_mask (shape {attention_mask.shape})' + ) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7e590e5e7805eac24e317f52aab47698182159 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .nerf_recon_4k import NeRFRecon4K + from .nerf_preprocess import NeRFReconPreprocessor + +else: + _import_structure = {'nerf_recon_4k': ['NeRFRecon4K']} + _import_structure = {'nerf_preprocess': ['NeRFReconPreprocessor']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_blender.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_blender.py new file mode 100644 index 0000000000000000000000000000000000000000..664a99ad3c59b376ef8fb8f14cee6163ebb7c6e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_blender.py @@ -0,0 +1,97 @@ +import os + +import cv2 +import imageio +import json +import numpy as np +import torch +import torch.nn.functional as F + + +def trans_t(t): + return torch.Tensor([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, t], + [0, 0, 0, 1]]).float() + + +def rot_phi(phi): + return torch.Tensor([[1, 0, 0, 0], [0, np.cos(phi), -np.sin(phi), 0], + [0, np.sin(phi), np.cos(phi), 0], [0, 0, 0, + 1]]).float() + + +def rot_theta(th): + return torch.Tensor([[np.cos(th), 0, -np.sin(th), 0], [0, 1, 0, 0], + [np.sin(th), 0, np.cos(th), 0], [0, 0, 0, + 1]]).float() + + +def pose_spherical(theta, phi, radius): + c2w = trans_t(radius) + c2w = rot_phi(phi / 180. * np.pi) @ c2w + c2w = rot_theta(theta / 180. * np.pi) @ c2w + c2w = torch.Tensor( + np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1] + ])) @ c2w + return c2w + + +def load_blender_data(basedir, half_res=False, testskip=1): + splits = ['train', 'val', 'test'] + metas = {} + for s in splits: + with open(os.path.join(basedir, 'transforms_{}.json'.format(s)), + 'r') as fp: + metas[s] = json.load(fp) + + all_imgs = [] + all_poses = [] + counts = [0] + for s in splits: + meta = metas[s] + imgs = [] + poses = [] + if s == 'train' or testskip == 0: + skip = 1 + elif s == 'val': + skip = 50 + else: + skip = testskip + + for frame in meta['frames'][::skip]: + fname = os.path.join(basedir, frame['file_path'] + '.png') + imgs.append(imageio.imread(fname)) + poses.append(np.array(frame['transform_matrix'])) + imgs = (np.array(imgs) / 255.).astype( + np.float32) # keep all 4 channels (RGBA) + poses = np.array(poses).astype(np.float32) + counts.append(counts[-1] + imgs.shape[0]) + all_imgs.append(imgs) + all_poses.append(poses) + + i_split = [np.arange(counts[i], counts[i + 1]) for i in range(3)] + + imgs = np.concatenate(all_imgs, 0) + poses = np.concatenate(all_poses, 0) + + H, W = imgs[0].shape[:2] + camera_angle_x = float(meta['camera_angle_x']) + focal = .5 * W / np.tan(.5 * camera_angle_x) + + render_poses = torch.stack([ + pose_spherical(angle, -30.0, 4.0) + for angle in np.linspace(-180, 180, 160 + 1)[:-1] + ], 0) + + if half_res: + H = H // 2 + W = W // 2 + focal = focal / 2. + + imgs_half_res = np.zeros((imgs.shape[0], H, W, 4)) + for i, img in enumerate(imgs): + imgs_half_res[i] = cv2.resize( + img, (W, H), interpolation=cv2.INTER_AREA) + imgs = imgs_half_res + # imgs = tf.image.resize_area(imgs, [400, 400]).numpy() + + return imgs, poses, render_poses, [H, W, focal], i_split diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_data.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_data.py new file mode 100644 index 0000000000000000000000000000000000000000..03de2783cfcf32a8a4c85c31d63f9edf5adbd415 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_data.py @@ -0,0 +1,143 @@ +import numpy as np + +from .load_blender import load_blender_data +from .load_llff import load_llff_data +from .load_tankstemple import load_tankstemple_data + + +def load_data(args): + + K, depths = None, None + near_clip = None + + if args.dataset_type == 'llff': + images, depths, poses, bds, render_poses, i_test, *srgt = load_llff_data( + args.datadir, + args.factor, + None, + None, + recenter=True, + bd_factor=0.75, + spherify=False, + load_depths=False, + load_SR=args.load_sr, + movie_render_kwargs=dict()) + hwf = poses[0, :3, -1] + poses = poses[:, :3, :4] + print('Loaded llff', images.shape, render_poses.shape, hwf, + args.datadir) + if not isinstance(i_test, list): + i_test = [i_test] + + llffhold = 8 + if llffhold > 0: + print('Auto LLFF holdout,', llffhold) + i_test = np.arange(images.shape[0])[::llffhold] + + i_val = [i_test[0]] + i_train = np.array([ + i for i in np.arange(int(images.shape[0])) + if (i not in i_test and i not in i_val) + ]) + + print('DEFINING BOUNDS') + if args.ndc: + near = 0. + far = 1. + else: + near_clip = max(np.ndarray.min(bds) * .9, 0) + _far = max(np.ndarray.max(bds) * 1., 0) + near = 0 + far = inward_nearfar_heuristic(poses[i_train, :3, 3])[1] + print('near_clip', near_clip) + print('original far', _far) + print('NEAR FAR', near, far) + + elif args.dataset_type == 'blender': + images, poses, render_poses, hwf, i_split = load_blender_data( + args.datadir, args.half_res, args.testskip) + print('Loaded blender', images.shape, render_poses.shape, hwf, + args.datadir) + i_train, i_val, i_test = i_split + + near, far = 2., 6. + + if images.shape[-1] == 4: + if args.white_bkgd: + images = images[..., :3] * images[..., -1:] + ( + 1. - images[..., -1:]) + else: + images = images[..., :3] * images[..., -1:] + + srgt = [images, 0] + + elif args.dataset_type == 'tankstemple': + images, poses, render_poses, hwf, K, i_split = load_tankstemple_data( + args.datadir, movie_render_kwargs=args.movie_render_kwargs) + print('Loaded tankstemple', images.shape, render_poses.shape, hwf, + args.datadir) + i_train, i_val, i_test = i_split + + near, far = inward_nearfar_heuristic(poses[i_train, :3, 3], ratio=0) + + if images.shape[-1] == 4: + if args.white_bkgd: + images = images[..., :3] * images[..., -1:] + ( + 1. - images[..., -1:]) + else: + images = images[..., :3] * images[..., -1:] + + else: + raise NotImplementedError( + f'Unknown dataset type {args.dataset_type} exiting') + + # Cast intrinsics to right types + H, W, focal = hwf + H, W = int(H), int(W) + hwf = [H, W, focal] + HW = np.array([im.shape[:2] for im in images]) + irregular_shape = (images.dtype is np.dtype('object')) + + if K is None: + K = np.array([[focal, 0, 0.5 * W], [0, focal, 0.5 * H], [0, 0, 1]]) + + if len(K.shape) == 2: + Ks = K[None].repeat(len(poses), axis=0) + else: + Ks = K + + render_poses = render_poses[..., :4] + + if args.load_sr: + srgt, w2c = srgt[0], srgt[1] + else: + srgt, w2c = 0, 0 + + data_dict = dict( + hwf=hwf, + HW=HW, + Ks=Ks, + near=near, + far=far, + near_clip=near_clip, + i_train=i_train, + i_val=i_val, + i_test=i_test, + poses=poses, + render_poses=render_poses, + images=images, + depths=depths, + white_bkgd=args.white_bkgd, + irregular_shape=irregular_shape, + srgt=srgt, + w2c=w2c) + return data_dict + + +def inward_nearfar_heuristic(cam_o, ratio=0.05): + dist = np.linalg.norm(cam_o[:, None] - cam_o, axis=-1) + far = dist.max() # could be too small to exist the scene bbox + # it is only used to determined scene bbox + # lib/dvgo use 1e9 as far + near = far * ratio + return near, far diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_llff.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_llff.py new file mode 100644 index 0000000000000000000000000000000000000000..9836da7140000780bd72b92a1e56d7a999005a06 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_llff.py @@ -0,0 +1,548 @@ +import os + +import imageio +import numpy as np +import scipy +import torch + + +# Slightly modified version of LLFF data loading code +# see https://github.com/Fyusion/LLFF for original +def imread(f): + if f.endswith('png'): + return imageio.imread(f, format='PNG-PIL', ignoregamma=True) + else: + return imageio.imread(f) + + +def depthread(path): + with open(path, 'rb') as fid: + width, height, channels = np.genfromtxt( + fid, delimiter='&', max_rows=1, usecols=(0, 1, 2), dtype=int) + fid.seek(0) + num_delimiter = 0 + byte = fid.read(1) + while True: + if byte == b'&': + num_delimiter += 1 + if num_delimiter >= 3: + break + byte = fid.read(1) + array = np.fromfile(fid, np.float32) + array = array.reshape((width, height, channels), order='F') + return np.transpose(array, (1, 0, 2)).squeeze() + + +def _minify(basedir, factors=[], resolutions=[]): + needtoload = False + for r in factors: + imgdir = os.path.join(basedir, 'images_{}'.format(r)) + if not os.path.exists(imgdir): + needtoload = True + for r in resolutions: + imgdir = os.path.join(basedir, 'images_{}x{}'.format(r[1], r[0])) + if not os.path.exists(imgdir): + needtoload = True + if not needtoload: + return + + from shutil import copy + from subprocess import check_output + + imgdir = os.path.join(basedir, 'images') + imgs = [os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir))] + imgs = [ + f for f in imgs + if any([f.endswith(ex) for ex in ['JPG', 'jpg', 'png', 'jpeg', 'PNG']]) + ] + imgdir_orig = imgdir + + wd = os.getcwd() + + for r in factors + resolutions: + if isinstance(r, int): + name = 'images_{}'.format(r) + resizearg = '{}%'.format(100. / r) + else: + name = 'images_{}x{}'.format(r[1], r[0]) + resizearg = '{}x{}'.format(r[1], r[0]) + imgdir = os.path.join(basedir, name) + if os.path.exists(imgdir): + continue + + print('Minifying', r, basedir) + + os.makedirs(imgdir) + check_output('cp {}/* {}'.format(imgdir_orig, imgdir), shell=True) + + ext = imgs[0].split('.')[-1] + args = ' '.join([ + 'mogrify', '-resize', resizearg, '-format', 'png', + '*.{}'.format(ext) + ]) + print(args) + os.chdir(imgdir) + check_output(args, shell=True) + os.chdir(wd) + + if ext != 'png': + check_output('rm {}/*.{}'.format(imgdir, ext), shell=True) + print('Removed duplicates') + print('Done') + + +def _load_data(basedir, + factor=None, + width=None, + height=None, + load_imgs=True, + load_depths=False, + load_SR=False): + + poses_arr = np.load(os.path.join(basedir, 'poses_bounds.npy')) + if poses_arr.shape[1] == 17: + poses = poses_arr[:, :-2].reshape([-1, 3, 5]).transpose([1, 2, 0]) + elif poses_arr.shape[1] == 14: + poses = poses_arr[:, :-2].reshape([-1, 3, 4]).transpose([1, 2, 0]) + else: + raise NotImplementedError + bds = poses_arr[:, -2:].transpose([1, 0]) + + img0 = [ + os.path.join(basedir, 'images', f) + for f in sorted(os.listdir(os.path.join(basedir, 'images'))) + if f.endswith('JPG') or f.endswith('jpg') or f.endswith('png') + ][0] + sh = imageio.imread(img0).shape + + sfx = '' + + if height is not None and width is not None: + _minify(basedir, resolutions=[[height, width]]) + sfx = '_{}x{}'.format(width, height) + elif factor is not None and factor != 1: + sfx = '_{}'.format(factor) + _minify(basedir, factors=[factor]) + factor = factor + elif height is not None: + factor = sh[0] / float(height) + width = int(sh[1] / factor) + _minify(basedir, resolutions=[[height, width]]) + sfx = '_{}x{}'.format(width, height) + elif width is not None: + factor = sh[1] / float(width) + height = int(sh[0] / factor) + _minify(basedir, resolutions=[[height, width]]) + sfx = '_{}x{}'.format(width, height) + else: + factor = 1 + + imgdir = os.path.join(basedir, 'images' + sfx) + print(f'Loading images from {imgdir}') + if not os.path.exists(imgdir): + print(imgdir, 'does not exist, returning') + return + + imgfiles = [ + os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir)) + if f.endswith('JPG') or f.endswith('jpg') or f.endswith('png') + ] + if poses.shape[-1] != len(imgfiles): + print() + print('Mismatch between imgs {} and poses {} !!!!'.format( + len(imgfiles), poses.shape[-1])) + names = set( + name[:-4] + for name in np.load(os.path.join(basedir, 'poses_names.npy'))) + assert len(names) == poses.shape[-1] + print('Below failed files are skip due to SfM failure:') + new_imgfiles = [] + for i in imgfiles: + fname = os.path.split(i)[1][:-4] + if fname in names: + new_imgfiles.append(i) + else: + print('==>', i) + imgfiles = new_imgfiles + + if len(imgfiles) < 3: + print('Too few images...') + import sys + sys.exit() + + sh = imageio.imread(imgfiles[0]).shape + if poses.shape[1] == 4: + poses = np.concatenate([poses, np.zeros_like(poses[:, [0]])], 1) + poses[2, 4, :] = np.load(os.path.join(basedir, 'hwf_cxcy.npy'))[2] + poses[:2, 4, :] = np.array(sh[:2]).reshape([2, 1]) + poses[2, 4, :] = poses[2, 4, :] * 1. / factor + + if not load_imgs: + return poses, bds + + imgs = [imread(f)[..., :3] / 255. for f in imgfiles] + imgs = np.stack(imgs, -1) + + if load_SR: + if load_SR == 16: + imgdir_sr = os.path.join(basedir, 'images_16') + elif load_SR == 8: + imgdir_sr = os.path.join(basedir, 'images_8') + elif load_SR == 4: + imgdir_sr = os.path.join(basedir, 'images_4') + elif load_SR == 2: + imgdir_sr = os.path.join(basedir, 'images_2') + elif load_SR == 1: + imgdir_sr = os.path.join(basedir, 'images') + imgfiles_sr = [ + os.path.join(imgdir_sr, f) for f in sorted(os.listdir(imgdir_sr)) + if f.endswith('JPG') or f.endswith('jpg') or f.endswith('png') + ] + imgs_sr = [imread(f)[..., :3] / 255. for f in imgfiles_sr] + imgs_sr = np.stack(imgs_sr, -1) + + print('Loaded image data', imgs.shape, poses[:, -1, 0]) + + if not load_depths and load_SR: + return poses, bds, imgs, imgs_sr + + if not load_depths: + return poses, bds, imgs + + depthdir = os.path.join(basedir, 'stereo', 'depth_maps') + assert os.path.exists(depthdir), f'Dir not found: {depthdir}' + + depthfiles = [ + os.path.join(depthdir, f) for f in sorted(os.listdir(depthdir)) + if f.endswith('.geometric.bin') + ] + assert poses.shape[-1] == len( + depthfiles), 'Mismatch between imgs {} and poses {} !!!!'.format( + len(depthfiles), poses.shape[-1]) + + depths = [depthread(f) for f in depthfiles] + depths = np.stack(depths, -1) + print('Loaded depth data', depths.shape) + return poses, bds, imgs, depths + + +def normalize(x): + return x / np.linalg.norm(x) + + +def viewmatrix(z, up, pos): + vec2 = normalize(z) + vec1_avg = up + vec0 = normalize(np.cross(vec1_avg, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.stack([vec0, vec1, vec2, pos], 1) + return m + + +def ptstocam(pts, c2w): + tt = np.matmul(c2w[:3, :3].T, (pts - c2w[:3, 3])[..., np.newaxis])[..., 0] + return tt + + +def poses_avg(poses): + + hwf = poses[0, :3, -1:] + + center = poses[:, :3, 3].mean(0) + vec2 = normalize(poses[:, :3, 2].sum(0)) + up = poses[:, :3, 1].sum(0) + c2w = np.concatenate([viewmatrix(vec2, up, center), hwf], 1) + + return c2w + + +def w2c_gen(poses): + final_pose = [] + for idx in range(len(poses)): + pose = poses[idx, ...] + z = normalize(pose[:3, 2]) + up = pose[:3, 1] + vec2 = normalize(z) + vec0 = normalize(np.cross(up, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.stack([vec0, vec1, vec2], 1) + mt = np.linalg.inv(m) + final_pose.append(mt) + final_pose = np.stack(final_pose, 0) + return final_pose + + +def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N): + render_poses = [] + rads = np.array(list(rads) + [1.]) + hwf = c2w[:, 4:5] + + # -np.sin(theta), -np.sin(theta*zrate)*zdelta + # 0, 0 + for theta in np.linspace(0., 2 * np.pi * rots, N + 1)[:-1]: + c = np.dot( + c2w[:3, :4], + np.array([ + np.cos(theta), -np.sin(theta), -np.sin(theta * zrate) * zdelta, + 1. + ]) * rads) + z = normalize(c - np.dot(c2w[:3, :4], np.array([0, 0, -focal, 1.]))) + render_poses.append(np.concatenate([viewmatrix(z, up, c), hwf], 1)) + return render_poses + + +def recenter_poses(poses): + + poses_ = poses + 0 + bottom = np.reshape([0, 0, 0, 1.], [1, 4]) + c2w = poses_avg(poses) + c2w = np.concatenate([c2w[:3, :4], bottom], -2) + bottom = np.tile(np.reshape(bottom, [1, 1, 4]), [poses.shape[0], 1, 1]) + poses = np.concatenate([poses[:, :3, :4], bottom], -2) + + poses = np.linalg.inv(c2w) @ poses + poses_[:, :3, :4] = poses[:, :3, :4] + poses = poses_ + return poses + + +def rerotate_poses(poses): + poses = np.copy(poses) + centroid = poses[:, :3, 3].mean(0) + + poses[:, :3, 3] = poses[:, :3, 3] - centroid + + # Find the minimum pca vector with minimum eigen value + x = poses[:, :, 3] + mu = x.mean(0) + cov = np.cov((x - mu).T) + ev, eig = np.linalg.eig(cov) + cams_up = eig[:, np.argmin(ev)] + if cams_up[1] < 0: + cams_up = -cams_up + + # Find rotation matrix that align cams_up with [0,1,0] + R = scipy.spatial.transform.Rotation.align_vectors( + [[0, 1, 0]], cams_up[None])[0].as_matrix() + + # Apply rotation and add back the centroid position + poses[:, :3, :3] = R @ poses[:, :3, :3] + poses[:, :3, [3]] = R @ poses[:, :3, [3]] + poses[:, :3, 3] = poses[:, :3, 3] + centroid + return poses + + +##################### + + +def spherify_poses(poses, bds, depths): + + def p34_to_44(p): + return np.concatenate([ + p, + np.tile( + np.reshape(np.eye(4)[-1, :], [1, 1, 4]), [p.shape[0], 1, 1]) + ], 1) + + rays_d = poses[:, :3, 2:3] + rays_o = poses[:, :3, 3:4] + + def min_line_dist(rays_o, rays_d): + A_i = np.eye(3) - rays_d * np.transpose(rays_d, [0, 2, 1]) + b_i = -A_i @ rays_o + pt_mindist = np.squeeze(-np.linalg.inv( + (np.transpose(A_i, [0, 2, 1]) @ A_i).mean(0)) @ (b_i).mean(0)) + return pt_mindist + + pt_mindist = min_line_dist(rays_o, rays_d) + + center = pt_mindist + up = (poses[:, :3, 3] - center).mean(0) + + vec0 = normalize(up) + vec1 = normalize(np.cross([.1, .2, .3], vec0)) + vec2 = normalize(np.cross(vec0, vec1)) + pos = center + c2w = np.stack([vec1, vec2, vec0, pos], 1) + + poses_reset = np.linalg.inv(p34_to_44(c2w[None])) @ p34_to_44( + poses[:, :3, :4]) + + radius = np.sqrt(np.mean(np.sum(np.square(poses_reset[:, :3, 3]), -1))) + + sc = 1. / radius + poses_reset[:, :3, 3] *= sc + bds *= sc + radius *= sc + depths *= sc + + poses_reset = np.concatenate([ + poses_reset[:, :3, :4], + np.broadcast_to(poses[0, :3, -1:], poses_reset[:, :3, -1:].shape) + ], -1) + + return poses_reset, radius, bds, depths + + +def load_llff_data(basedir, + factor=8, + width=None, + height=None, + recenter=True, + rerotate=True, + bd_factor=.75, + spherify=False, + path_zflat=False, + load_depths=False, + load_SR=False, + movie_render_kwargs={}): + + poses, bds, imgs, *depths = _load_data( + basedir, + factor=factor, + width=width, + height=height, + load_depths=load_depths, + load_SR=load_SR) # factor=8 downsamples original imgs by 8x + print('Loaded', basedir, bds.min(), bds.max()) + if load_depths: + depths = depths[0] + elif load_SR and not load_depths: + imgs_SRGT = depths[0] + depths = 0 + else: + depths = 0 + + # Correct rotation matrix ordering and move variable dim to axis 0 + poses = np.concatenate( + [poses[:, 1:2, :], -poses[:, 0:1, :], poses[:, 2:, :]], 1) + poses = np.moveaxis(poses, -1, 0).astype(np.float32) + imgs = np.moveaxis(imgs, -1, 0).astype(np.float32) + images = imgs + bds = np.moveaxis(bds, -1, 0).astype(np.float32) + + # Rescale if bd_factor is provided + if bds.min() < 0 and bd_factor is not None: + print('Found negative z values from SfM sparse points!?') + print('Please try bd_factor=None') + import sys + sys.exit() + sc = 1. if bd_factor is None else 1. / (bds.min() * bd_factor) + poses[:, :3, 3] *= sc + bds *= sc + depths *= sc + + if recenter: + poses = recenter_poses(poses) + + if spherify: + poses, radius, bds, depths = spherify_poses(poses, bds, depths) + if rerotate: + poses = rerotate_poses(poses) + + # generate spiral poses for rendering fly-through movie + centroid = poses[:, :3, 3].mean(0) + radcircle = movie_render_kwargs.get('scale_r', 1) * np.linalg.norm( + poses[:, :3, 3] - centroid, axis=-1).mean() + centroid[0] += movie_render_kwargs.get('shift_x', 0) + centroid[1] += movie_render_kwargs.get('shift_y', 0) + centroid[2] += movie_render_kwargs.get('shift_z', 0) + new_up_rad = movie_render_kwargs.get('pitch_deg', 0) * np.pi / 180 + target_y = radcircle * np.tan(new_up_rad) + + render_poses = [] + + for th in np.linspace(0., 2. * np.pi, 200): + camorigin = np.array( + [radcircle * np.cos(th), 0, radcircle * np.sin(th)]) + if movie_render_kwargs.get('flip_up', False): + up = np.array([0, 1., 0]) + else: + up = np.array([0, -1., 0]) + vec2 = normalize(camorigin) + vec0 = normalize(np.cross(vec2, up)) + vec1 = normalize(np.cross(vec2, vec0)) + pos = camorigin + centroid + # rotate to align with new pitch rotation + lookat = -vec2 + lookat[1] = target_y + lookat = normalize(lookat) + vec2 = -lookat + vec1 = normalize(np.cross(vec2, vec0)) + + p = np.stack([vec0, vec1, vec2, pos], 1) + + render_poses.append(p) + + render_poses = np.stack(render_poses, 0) + render_poses = np.concatenate([ + render_poses, + np.broadcast_to(poses[0, :3, -1:], render_poses[:, :3, -1:].shape) + ], -1) + + else: + + c2w = poses_avg(poses) + print('recentered', c2w.shape) + print(c2w[:3, :4]) + + # Get spiral + # Get average pose + up = normalize(poses[:, :3, 1].sum(0)) + + # Find a reasonable "focus depth" for this dataset + close_depth, inf_depth = bds.min() * .9, bds.max() * 5. + dt = .75 + mean_dz = 1. / (((1. - dt) / close_depth + dt / inf_depth)) + focal = mean_dz * movie_render_kwargs.get('scale_f', 1) + + # Get radii for spiral path + zdelta = movie_render_kwargs.get('zdelta', 0.5) + zrate = movie_render_kwargs.get('zrate', 1.0) + tt = poses[:, :3, 3] # ptstocam(poses[:3,3,:].T, c2w).T + rads = np.percentile(np.abs(tt), 90, 0) * movie_render_kwargs.get( + 'scale_r', 1) + c2w_path = c2w + N_views = 120 + N_rots = movie_render_kwargs.get('N_rots', 1) + if path_zflat: + # zloc = np.percentile(tt, 10, 0)[2] + zloc = -close_depth * .1 + c2w_path[:3, 3] = c2w_path[:3, 3] + zloc * c2w_path[:3, 2] + rads[2] = 0. + N_rots = 1 + N_views /= 2 + + # Generate poses for spiral path + render_poses = render_path_spiral( + c2w_path, + up, + rads, + focal, + zdelta, + zrate=zrate, + rots=N_rots, + N=N_views) + + render_poses = torch.Tensor(render_poses) + + # Because both world croodnate system and camera croodnate system are 3-d system, they can be transfer by a: + # 3x3 rotate matrix and 3x1 moving matrix + c2w = poses_avg(poses) + w2c = w2c_gen(poses) + print('Data:') + print(poses.shape, images.shape, bds.shape) + + dists = np.sum(np.square(c2w[:3, 3] - poses[:, :3, 3]), -1) + i_test = np.argmin(dists) + print('HOLDOUT view is', i_test) + + images = images.astype(np.float32) + poses = poses.astype(np.float32) + + if load_SR: + imgs_SRGT = np.moveaxis(imgs_SRGT, [-1, -2], [0, 1]).astype(np.float32) + else: + imgs_SRGT = None + + return images, depths, poses, bds, render_poses, i_test, imgs_SRGT, w2c diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_tankstemple.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_tankstemple.py new file mode 100644 index 0000000000000000000000000000000000000000..8dec4c79afbffab7d589f922f3e66fb90f27d691 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/load_tankstemple.py @@ -0,0 +1,75 @@ +import glob +import os + +import imageio +import numpy as np + + +def normalize(x): + return x / np.linalg.norm(x) + + +def load_tankstemple_data(basedir, movie_render_kwargs={}): + pose_paths = sorted(glob.glob(os.path.join(basedir, 'pose', '*txt'))) + rgb_paths = sorted(glob.glob(os.path.join(basedir, 'rgb', '*png'))) + + all_poses = [] + all_imgs = [] + i_split = [[], []] + for i, (pose_path, rgb_path) in enumerate(zip(pose_paths, rgb_paths)): + i_set = int(os.path.split(rgb_path)[-1][0]) + all_poses.append(np.loadtxt(pose_path).astype(np.float32)) + all_imgs.append((imageio.imread(rgb_path) / 255.).astype(np.float32)) + i_split[i_set].append(i) + + imgs = np.stack(all_imgs, 0) + poses = np.stack(all_poses, 0) + i_split.append(i_split[-1]) + + path_intrinsics = os.path.join(basedir, 'intrinsics.txt') + H, W = imgs[0].shape[:2] + K = np.loadtxt(path_intrinsics) + focal = float(K[0, 0]) + + # generate spiral poses for rendering fly-through movie + centroid = poses[:, :3, 3].mean(0) + radcircle = movie_render_kwargs.get('scale_r', 1.0) * np.linalg.norm( + poses[:, :3, 3] - centroid, axis=-1).mean() + centroid[0] += movie_render_kwargs.get('shift_x', 0) + centroid[1] += movie_render_kwargs.get('shift_y', 0) + centroid[2] += movie_render_kwargs.get('shift_z', 0) + new_up_rad = movie_render_kwargs.get('pitch_deg', 0) * np.pi / 180 + target_y = radcircle * np.tan(new_up_rad) + + render_poses = [] + + for th in np.linspace(0., 2. * np.pi, 200): + camorigin = np.array( + [radcircle * np.cos(th), 0, radcircle * np.sin(th)]) + if movie_render_kwargs.get('flip_up_vec', False): + up = np.array([0, -1., 0]) + else: + up = np.array([0, 1., 0]) + vec2 = normalize(camorigin) + vec0 = normalize(np.cross(vec2, up)) + vec1 = normalize(np.cross(vec2, vec0)) + pos = camorigin + centroid + # rotate to align with new pitch rotation + lookat = -vec2 + lookat[1] = target_y + lookat = normalize(lookat) + lookat *= -1 + vec2 = -lookat + vec1 = normalize(np.cross(vec2, vec0)) + + p = np.stack([vec0, vec1, vec2, pos], 1) + + render_poses.append(p) + + render_poses = np.stack(render_poses, 0) + render_poses = np.concatenate([ + render_poses, + np.broadcast_to(poses[0, :3, -1:], render_poses[:, :3, -1:].shape) + ], -1) + + return imgs, poses, render_poses, [H, W, focal], K, i_split diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/read_write_model.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/read_write_model.py new file mode 100644 index 0000000000000000000000000000000000000000..cec00a5284f0f1bca7c87436018f8450af0a49d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/dataloader/read_write_model.py @@ -0,0 +1,500 @@ +# Copyright (c) 2023, ETH Zurich and UNC Chapel Hill. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) + +import argparse +import collections +import os +import struct + +import numpy as np + +CameraModel = collections.namedtuple('CameraModel', + ['model_id', 'model_name', 'num_params']) +Camera = collections.namedtuple('Camera', + ['id', 'model', 'width', 'height', 'params']) +BaseImage = collections.namedtuple( + 'Image', ['id', 'qvec', 'tvec', 'camera_id', 'name', 'xys', 'point3D_ids']) +Point3D = collections.namedtuple( + 'Point3D', ['id', 'xyz', 'rgb', 'error', 'image_ids', 'point2D_idxs']) + + +class Image(BaseImage): + + def qvec2rotmat(self): + return qvec2rotmat(self.qvec) + + +CAMERA_MODELS = { + CameraModel(model_id=0, model_name='SIMPLE_PINHOLE', num_params=3), + CameraModel(model_id=1, model_name='PINHOLE', num_params=4), + CameraModel(model_id=2, model_name='SIMPLE_RADIAL', num_params=4), + CameraModel(model_id=3, model_name='RADIAL', num_params=5), + CameraModel(model_id=4, model_name='OPENCV', num_params=8), + CameraModel(model_id=5, model_name='OPENCV_FISHEYE', num_params=8), + CameraModel(model_id=6, model_name='FULL_OPENCV', num_params=12), + CameraModel(model_id=7, model_name='FOV', num_params=5), + CameraModel(model_id=8, model_name='SIMPLE_RADIAL_FISHEYE', num_params=4), + CameraModel(model_id=9, model_name='RADIAL_FISHEYE', num_params=5), + CameraModel(model_id=10, model_name='THIN_PRISM_FISHEYE', num_params=12) +} +CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) + for camera_model in CAMERA_MODELS]) +CAMERA_MODEL_NAMES = dict([(camera_model.model_name, camera_model) + for camera_model in CAMERA_MODELS]) + + +def read_next_bytes(fid, + num_bytes, + format_char_sequence, + endian_character='<'): + """Read and unpack the next bytes from a binary file. + :param fid: + :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + :param endian_character: Any of {@, =, <, >, !} + :return: Tuple of read and unpacked values. + """ + data = fid.read(num_bytes) + return struct.unpack(endian_character + format_char_sequence, data) + + +def write_next_bytes(fid, data, format_char_sequence, endian_character='<'): + """pack and write to a binary file. + :param fid: + :param data: data to send, if multiple elements are sent at the same time, + they should be encapsuled either in a list or a tuple + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + should be the same length as the data list or tuple + :param endian_character: Any of {@, =, <, >, !} + """ + if isinstance(data, (list, tuple)): + bytes = struct.pack(endian_character + format_char_sequence, *data) + else: + bytes = struct.pack(endian_character + format_char_sequence, data) + fid.write(bytes) + + +def read_cameras_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + cameras = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + camera_id = int(elems[0]) + model = elems[1] + width = int(elems[2]) + height = int(elems[3]) + params = np.array(tuple(map(float, elems[4:]))) + cameras[camera_id] = Camera( + id=camera_id, + model=model, + width=width, + height=height, + params=params) + return cameras + + +def read_cameras_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasBinary(const std::string& path) + void Reconstruction::ReadCamerasBinary(const std::string& path) + """ + cameras = {} + with open(path_to_model_file, 'rb') as fid: + num_cameras = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_cameras): + camera_properties = read_next_bytes( + fid, num_bytes=24, format_char_sequence='iiQQ') + camera_id = camera_properties[0] + model_id = camera_properties[1] + model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name + width = camera_properties[2] + height = camera_properties[3] + num_params = CAMERA_MODEL_IDS[model_id].num_params + params = read_next_bytes( + fid, + num_bytes=8 * num_params, + format_char_sequence='d' * num_params) + cameras[camera_id] = Camera( + id=camera_id, + model=model_name, + width=width, + height=height, + params=np.array(params)) + assert len(cameras) == num_cameras + return cameras + + +def write_cameras_text(cameras, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + HEADER = '# Camera list with one line of data per camera:\n' + \ + '# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n' + \ + '# Number of cameras: {}\n'.format(len(cameras)) + with open(path, 'w') as fid: + fid.write(HEADER) + for _, cam in cameras.items(): + to_write = [cam.id, cam.model, cam.width, cam.height, *cam.params] + line = ' '.join([str(elem) for elem in to_write]) + fid.write(line + '\n') + + +def write_cameras_binary(cameras, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasBinary(const std::string& path) + void Reconstruction::ReadCamerasBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(cameras), 'Q') + for _, cam in cameras.items(): + model_id = CAMERA_MODEL_NAMES[cam.model].model_id + camera_properties = [cam.id, model_id, cam.width, cam.height] + write_next_bytes(fid, camera_properties, 'iiQQ') + for p in cam.params: + write_next_bytes(fid, float(p), 'd') + return cameras + + +def read_images_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesText(const std::string& path) + void Reconstruction::WriteImagesText(const std::string& path) + """ + images = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + image_id = int(elems[0]) + qvec = np.array(tuple(map(float, elems[1:5]))) + tvec = np.array(tuple(map(float, elems[5:8]))) + camera_id = int(elems[8]) + image_name = elems[9] + elems = fid.readline().split() + xys = np.column_stack([ + tuple(map(float, elems[0::3])), + tuple(map(float, elems[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, elems[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def read_images_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesBinary(const std::string& path) + void Reconstruction::WriteImagesBinary(const std::string& path) + """ + images = {} + with open(path_to_model_file, 'rb') as fid: + num_reg_images = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_reg_images): + binary_image_properties = read_next_bytes( + fid, num_bytes=64, format_char_sequence='idddddddi') + image_id = binary_image_properties[0] + qvec = np.array(binary_image_properties[1:5]) + tvec = np.array(binary_image_properties[5:8]) + camera_id = binary_image_properties[8] + image_name = '' + current_char = read_next_bytes(fid, 1, 'c')[0] + while current_char != b'\x00': # look for the ASCII 0 entry + image_name += current_char.decode('utf-8') + current_char = read_next_bytes(fid, 1, 'c')[0] + num_points2D = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + x_y_id_s = read_next_bytes( + fid, + num_bytes=24 * num_points2D, + format_char_sequence='ddq' * num_points2D) + xys = np.column_stack([ + tuple(map(float, x_y_id_s[0::3])), + tuple(map(float, x_y_id_s[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def write_images_text(images, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesText(const std::string& path) + void Reconstruction::WriteImagesText(const std::string& path) + """ + if len(images) == 0: + mean_observations = 0 + else: + mean_observations = sum( + (len(img.point3D_ids) for _, img in images.items())) / len(images) + HEADER = '# Image list with two lines of data per image:\n' + \ + '# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n' + \ + '# POINTS2D[] as (X, Y, POINT3D_ID)\n' + \ + '# Number of images: {}, mean observations per image: {}\n'.format(len(images), mean_observations) + + with open(path, 'w') as fid: + fid.write(HEADER) + for _, img in images.items(): + image_header = [ + img.id, *img.qvec, *img.tvec, img.camera_id, img.name + ] + first_line = ' '.join(map(str, image_header)) + fid.write(first_line + '\n') + + points_strings = [] + for xy, point3D_id in zip(img.xys, img.point3D_ids): + points_strings.append(' '.join(map(str, [*xy, point3D_id]))) + fid.write(' '.join(points_strings) + '\n') + + +def write_images_binary(images, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesBinary(const std::string& path) + void Reconstruction::WriteImagesBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(images), 'Q') + for _, img in images.items(): + write_next_bytes(fid, img.id, 'i') + write_next_bytes(fid, img.qvec.tolist(), 'dddd') + write_next_bytes(fid, img.tvec.tolist(), 'ddd') + write_next_bytes(fid, img.camera_id, 'i') + for char in img.name: + write_next_bytes(fid, char.encode('utf-8'), 'c') + write_next_bytes(fid, b'\x00', 'c') + write_next_bytes(fid, len(img.point3D_ids), 'Q') + for xy, p3d_id in zip(img.xys, img.point3D_ids): + write_next_bytes(fid, [*xy, p3d_id], 'ddq') + + +def read_points3D_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DText(const std::string& path) + void Reconstruction::WritePoints3DText(const std::string& path) + """ + points3D = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + point3D_id = int(elems[0]) + xyz = np.array(tuple(map(float, elems[1:4]))) + rgb = np.array(tuple(map(int, elems[4:7]))) + error = float(elems[7]) + image_ids = np.array(tuple(map(int, elems[8::2]))) + point2D_idxs = np.array(tuple(map(int, elems[9::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def read_points3D_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DBinary(const std::string& path) + void Reconstruction::WritePoints3DBinary(const std::string& path) + """ + points3D = {} + with open(path_to_model_file, 'rb') as fid: + num_points = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_points): + binary_point_line_properties = read_next_bytes( + fid, num_bytes=43, format_char_sequence='QdddBBBd') + point3D_id = binary_point_line_properties[0] + xyz = np.array(binary_point_line_properties[1:4]) + rgb = np.array(binary_point_line_properties[4:7]) + error = np.array(binary_point_line_properties[7]) + track_length = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + track_elems = read_next_bytes( + fid, + num_bytes=8 * track_length, + format_char_sequence='ii' * track_length) + image_ids = np.array(tuple(map(int, track_elems[0::2]))) + point2D_idxs = np.array(tuple(map(int, track_elems[1::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def write_points3D_text(points3D, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DText(const std::string& path) + void Reconstruction::WritePoints3DText(const std::string& path) + """ + if len(points3D) == 0: + mean_track_length = 0 + else: + mean_track_length = sum( + (len(pt.image_ids) for _, pt in points3D.items())) / len(points3D) + HEADER = '# 3D point list with one line of data per point:\n' + \ + '# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)\n' + \ + '# Number of points: {}, mean track length: {}\n'.format(len(points3D), mean_track_length) + + with open(path, 'w') as fid: + fid.write(HEADER) + for _, pt in points3D.items(): + point_header = [pt.id, *pt.xyz, *pt.rgb, pt.error] + fid.write(' '.join(map(str, point_header)) + ' ') + track_strings = [] + for image_id, point2D in zip(pt.image_ids, pt.point2D_idxs): + track_strings.append(' '.join(map(str, [image_id, point2D]))) + fid.write(' '.join(track_strings) + '\n') + + +def write_points3D_binary(points3D, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DBinary(const std::string& path) + void Reconstruction::WritePoints3DBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(points3D), 'Q') + for _, pt in points3D.items(): + write_next_bytes(fid, pt.id, 'Q') + write_next_bytes(fid, pt.xyz.tolist(), 'ddd') + write_next_bytes(fid, pt.rgb.tolist(), 'BBB') + write_next_bytes(fid, pt.error, 'd') + track_length = pt.image_ids.shape[0] + write_next_bytes(fid, track_length, 'Q') + for image_id, point2D_id in zip(pt.image_ids, pt.point2D_idxs): + write_next_bytes(fid, [image_id, point2D_id], 'ii') + + +def detect_model_format(path, ext): + if os.path.isfile(os.path.join(path, 'cameras' + ext)) and \ + os.path.isfile(os.path.join(path, 'images' + ext)) and \ + os.path.isfile(os.path.join(path, 'points3D' + ext)): + print("Detected model format: '" + ext + "'") + return True + + return False + + +def read_model(path, ext=''): + # try to detect the extension automatically + if ext == '': + if detect_model_format(path, '.bin'): + ext = '.bin' + elif detect_model_format(path, '.txt'): + ext = '.txt' + else: + print("Provide model format: '.bin' or '.txt'") + return + + if ext == '.txt': + cameras = read_cameras_text(os.path.join(path, 'cameras' + ext)) + images = read_images_text(os.path.join(path, 'images' + ext)) + points3D = read_points3D_text(os.path.join(path, 'points3D') + ext) + else: + cameras = read_cameras_binary(os.path.join(path, 'cameras' + ext)) + images = read_images_binary(os.path.join(path, 'images' + ext)) + points3D = read_points3D_binary(os.path.join(path, 'points3D') + ext) + return cameras, images, points3D + + +def write_model(cameras, images, points3D, path, ext='.bin'): + if ext == '.txt': + write_cameras_text(cameras, os.path.join(path, 'cameras' + ext)) + write_images_text(images, os.path.join(path, 'images' + ext)) + write_points3D_text(points3D, os.path.join(path, 'points3D') + ext) + else: + write_cameras_binary(cameras, os.path.join(path, 'cameras' + ext)) + write_images_binary(images, os.path.join(path, 'images' + ext)) + write_points3D_binary(points3D, os.path.join(path, 'points3D') + ext) + return cameras, images, points3D + + +def qvec2rotmat(qvec): + array_10 = 1 - 2 * qvec[2]**2 - 2 * qvec[3]**2 + array_11 = 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3] + array_12 = 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2] + array_1 = [array_10, array_11, array_12] + array_20 = 2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3] + array_21 = 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2 + array_22 = 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1] + array_2 = [array_20, array_21, array_22] + array_30 = 2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2] + array_31 = 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1] + array_32 = 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2 + array_3 = [array_30, array_31, array_32] + + return np.array([array_1, array_2, array_3]) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_preprocess.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..8be3f42b4f143097e639555ad3c2b7e387a8ede9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_preprocess.py @@ -0,0 +1,200 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import glob +import os +import subprocess +from typing import Any, Dict, Union + +import cv2 +import numpy as np +import tensorflow as tf + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.utils.constant import Fields, ModeKeys +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.nerf_recon_acc_preprocessor) +class NeRFReconPreprocessor(Preprocessor): + + def __init__(self, + mode=ModeKeys.INFERENCE, + data_type='colmap', + use_mask=True, + match_type='exhaustive_matcher', + frame_count=60, + use_distortion=False, + *args, + **kwargs): + + super().__init__(mode) + + # set preprocessor info + self.data_type = data_type + self.use_mask = use_mask + + self.match_type = match_type + if match_type != 'exhaustive_matcher' and match_type != 'sequential_matcher': + raise Exception('matcher type {} is not valid'.format(match_type)) + self.frame_count = frame_count + self.use_distortion = use_distortion + + def __call__(self, data: Union[str, Dict], **kwargs) -> Dict[str, Any]: + + if self.data_type != 'blender' and self.data_type != 'colmap': + raise Exception('data type {} is not support currently'.format( + self.data_type)) + + data_dir = data['data_dir'] + os.makedirs(data_dir, exist_ok=True) + if self.data_type == 'blender': + transform_file = os.path.join(data_dir, 'transforms_train.json') + if not os.path.exists(transform_file): + raise Exception('Blender dataset is not found') + + if self.data_type == 'colmap': + video_path = data['video_input_path'] + if video_path != '': + self.split_frames(video_path, data_dir, self.frame_count) + self.gen_poses(data_dir, self.match_type, self.use_distortion) + files_needed = [ + '{}.bin'.format(f) for f in ['cameras', 'images', 'points3D'] + ] + if self.use_distortion: + colmap_dir = os.path.join(data_dir, 'preprocess/sparse') + files_had = os.listdir(colmap_dir) + else: + colmap_dir = os.path.join(data_dir, 'sparse/0') + files_had = os.listdir(colmap_dir) + if not all([f in files_had for f in files_needed]): + raise Exception('colmap run failed') + + data = {} + data['data_dir'] = data_dir + return data + + def split_frames(self, video_path, basedir, frame_count=60): + cap = cv2.VideoCapture(video_path) + fps = round(cap.get(cv2.CAP_PROP_FPS)) + frame_total = round(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + if not os.path.exists(os.path.join(basedir, 'images')): + logger.info('Need to run ffmpeg') + image_dir = os.path.join(basedir, 'images') + os.makedirs(image_dir, exist_ok=True) + fps = int(frame_count * fps / frame_total) + cmd = f"ffmpeg -i {video_path} -qscale:v 1 -qmin 1 -vf \"fps={fps}\" {image_dir}/%04d.png" + os.system(cmd) + logger.info('split frames done') + else: + logger.info('Don\'t need to run ffmpeg') + + def run_colmap(self, basedir, match_type, use_distortion): + logfile_name = os.path.join(basedir, 'colmap_output.txt') + logfile = open(logfile_name, 'w') + + feature_extractor_args = [ + 'colmap', 'feature_extractor', '--database_path', + os.path.join(basedir, 'database.db'), '--image_path', + os.path.join(basedir, 'images'), '--ImageReader.single_camera', '1' + ] + feat_output = ( + subprocess.check_output( + feature_extractor_args, universal_newlines=True)) + logfile.write(feat_output) + logger.info('Features extracted done') + + exhaustive_matcher_args = [ + 'colmap', + match_type, + '--database_path', + os.path.join(basedir, 'database.db'), + ] + + match_output = ( + subprocess.check_output( + exhaustive_matcher_args, universal_newlines=True)) + logfile.write(match_output) + logger.info('Features matched done') + + p = os.path.join(basedir, 'sparse') + if not os.path.exists(p): + os.makedirs(p) + + mapper_args = [ + 'colmap', + 'mapper', + '--database_path', + os.path.join(basedir, 'database.db'), + '--image_path', + os.path.join(basedir, 'images'), + '--output_path', + os.path.join( + basedir, 'sparse' + ), # --export_path changed to --output_path in colmap 3.6 + '--Mapper.num_threads', + '16', + '--Mapper.init_min_tri_angle', + '4', + '--Mapper.multiple_models', + '0', + '--Mapper.extract_colors', + '0', + ] + + map_output = ( + subprocess.check_output(mapper_args, universal_newlines=True)) + logfile.write(map_output) + logger.info('Sparse map created done.') + + bundle_adjuster_cmd = [ + 'colmap', + 'bundle_adjuster', + '--input_path', + os.path.join(basedir, 'sparse/0'), + '--output_path', + os.path.join(basedir, 'sparse/0'), + '--BundleAdjustment.refine_principal_point', + '1', + ] + map_output = ( + subprocess.check_output( + bundle_adjuster_cmd, universal_newlines=True)) + logfile.write(map_output) + logger.info('Refining intrinsics done.') + + if use_distortion: + os.makedirs(os.path.join(basedir, 'preprocess'), exist_ok=True) + distort_cmd = [ + 'colmap', 'image_undistorter', '--image_path', + os.path.join(basedir, 'images'), '--input_path', + os.path.join(basedir, 'sparse/0'), '--output_path', + os.path.join(basedir, 'preprocess'), '--output_type', 'COLMAP' + ] + map_output = ( + subprocess.check_output(distort_cmd, universal_newlines=True)) + logfile.write(map_output) + logger.info('Image distortion done.') + + logfile.close() + logger.info( + 'Finished running COLMAP, see {} for logs'.format(logfile_name)) + + def gen_poses(self, basedir, match_type, use_distortion): + files_needed = [ + '{}.bin'.format(f) for f in ['cameras', 'images', 'points3D'] + ] + if os.path.exists(os.path.join(basedir, 'sparse/0')): + files_had = os.listdir(os.path.join(basedir, 'sparse/0')) + else: + files_had = [] + if not all([f in files_had for f in files_needed]): + logger.info('Need to run COLMAP') + self.run_colmap(basedir, match_type, use_distortion) + else: + logger.info('Don\'t need to run COLMAP') diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_recon_4k.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_recon_4k.py new file mode 100644 index 0000000000000000000000000000000000000000..ca70facbe04d76f10e6daadc7a6527d499c834b3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/nerf_recon_4k.py @@ -0,0 +1,289 @@ +import argparse +import os +import random +import time + +import imageio +import mmcv +import numpy as np +import torch +from tqdm import tqdm, trange + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .dataloader.load_data import load_data +from .network.dvgo import DirectMPIGO, DirectVoxGO, SFTNet, get_rays_of_a_view + +logger = get_logger() + + +def to8b(x): + return (255 * np.clip(x, 0, 1)).astype(np.uint8) + + +__all__ = ['NeRFRecon4K'] + + +@MODELS.register_module(Tasks.nerf_recon_4k, module_name=Models.nerf_recon_4k) +class NeRFRecon4K(TorchModel): + + def __init__(self, model_dir, **kwargs): + super().__init__(model_dir, **kwargs) + + if not torch.cuda.is_available(): + raise Exception('GPU is required') + self.device = torch.device('cuda') + logger.info('model params:{}'.format(kwargs)) + self.data_type = kwargs['data_type'] + # self.use_mask = kwargs['use_mask'] + # self.num_samples_per_ray = kwargs['num_samples_per_ray'] + self.test_ray_chunk = kwargs['test_ray_chunk'] + # self.enc_ckpt_path = kwargs['enc_ckpt_path'] + # self.dec_ckpt_path = kwargs['dec_ckpt_path'] + + self.enc_ckpt_path = os.path.join(model_dir, 'fine_100000.tar') + if not os.path.exists(self.enc_ckpt_path): + raise Exception('encoder ckpt path not found') + # if self.dec_ckpt_path == '': + self.dec_ckpt_path = os.path.join(model_dir, 'sresrnet_100000.pth') + if not os.path.exists(self.dec_ckpt_path): + raise Exception('decoder ckpt path not found') + + self.ckpt_name = self.dec_ckpt_path.split('/')[-1][:-4] + self.ndc = True if self.data_type == 'llff' else False + self.sr_ratio = int(kwargs['factor'] / kwargs['load_sr']) + self.load_existed_model() + + self.test_tile = kwargs['test_tile'] + self.stepsize = kwargs['stepsize'] + + def load_existed_model(self): + if self.ndc: + model_class = DirectMPIGO + ckpt = torch.load(self.enc_ckpt_path, map_location='cpu') + else: + model_class = DirectVoxGO + ckpt = torch.load(self.enc_ckpt_path, map_location='cpu') + ckpt['model_kwargs']['mask_cache_path'] = self.enc_ckpt_path + self.encoder = model_class(**ckpt['model_kwargs']) + self.encoder.load_state_dict(ckpt['model_state_dict']) + self.encoder = self.encoder.to(self.device) + + self.decoder = SFTNet( + n_in_colors=3, + scale=self.sr_ratio, + num_feat=64, + num_block=5, + num_grow_ch=32, + num_cond=1, + dswise=False).to(self.device) + self.decoder.load_network( + load_path=self.dec_ckpt_path, device=self.device) + self.decoder.eval() + + def nerf_reconstruction(self, data_cfg, render_dir): + data_dict = load_everything(cfg_data=data_cfg) + + self.render_viewpoints_kwargs = { + 'render_kwargs': { + 'near': data_dict['near'], + 'far': data_dict['far'], + 'bg': 1 if data_dict['white_bkgd'] else 0, + 'stepsize': self.stepsize, + 'inverse_y': False, + 'flip_x': False, + 'flip_y': False, + 'render_depth': True, + }, + } + + os.makedirs(render_dir, exist_ok=True) + print('All results are dumped into', render_dir) + rgbs, depths, bgmaps, _, _, rgb_features = self.render_viewpoints( + render_poses=data_dict['poses'][data_dict['i_test']], + HW=data_dict['HW'][data_dict['i_test']], + Ks=data_dict['Ks'][data_dict['i_test']], + gt_imgs=[ + data_dict['images'][i].cpu().numpy() + for i in data_dict['i_test'] + ], + savedir=render_dir, + dump_images=False, + **self.render_viewpoints_kwargs) + + rgbsr = [] + for idx, rgbsave in enumerate(tqdm(rgb_features)): + rgbtest = torch.from_numpy(rgbsave).movedim(-1, 0).unsqueeze(0).to( + self.device) + # rgb = torch.from_numpy(rgbs[idx]).movedim(-1, 0).unsqueeze(0).to(self.device) + + input_cond = torch.from_numpy(depths).movedim(-1, 1) + input_cond = input_cond[idx, :, :, :].to(self.device) + + if self.test_tile: + rgb_srtest = self.decoder.tile_process( + rgbtest, input_cond, tile_size=self.test_tile) + else: + rgb_srtest = self.decoder(rgbtest, + input_cond).detach().to('cpu') + + rgb_srsave = rgb_srtest.squeeze().movedim(0, -1).detach().clamp( + 0, 1).numpy() + rgbsr.append(rgb_srsave) + print( + '''all inference process has done, saving images... because our images are + 4K (4032x3024), the saving process may be time-consuming.''') + rgbsr = np.array(rgbsr) + for i in trange(len(rgbsr)): + rgb8 = to8b(rgbsr[i]) + filename = os.path.join(render_dir, '{:03d}_dec.png'.format(i)) + imageio.imwrite(filename, rgb8) + + imageio.mimwrite( + os.path.join(render_dir, 'result_dec.mp4'), + to8b(rgbsr), + fps=25, + codec='libx264', + quality=8) + + @torch.no_grad() + def render_viewpoints(self, + render_poses, + HW, + Ks, + render_kwargs, + gt_imgs=None, + savedir=None, + dump_images=False, + render_factor=0, + eval_ssim=False, + eval_lpips_alex=False, + eval_lpips_vgg=False): + '''Render images for the given viewpoints; run evaluation if gt given. + ''' + assert len(render_poses) == len(HW) and len(HW) == len(Ks) + + if render_factor != 0: + HW = np.copy(HW) + Ks = np.copy(Ks) + HW = (HW / render_factor).astype(int) + Ks[:, :2, :3] /= render_factor + + rgbs = [] + rgb_features = [] + depths = [] + bgmaps = [] + psnrs = [] + viewdirs_all = [] + ssims = [] + lpips_alex = [] + lpips_vgg = [] + + for i, c2w in enumerate(tqdm(render_poses)): + + H, W = HW[i] + K = Ks[i] + c2w = torch.Tensor(c2w) + rays_o, rays_d, viewdirs = get_rays_of_a_view( + H, + W, + K, + c2w, + self.ndc, + inverse_y=False, + flip_x=False, + flip_y=False) + keys = ['rgb_marched', 'depth', 'alphainv_last', 'rgb_feature'] + rays_o = rays_o.flatten(0, -2).to('cuda') + rays_d = rays_d.flatten(0, -2).to('cuda') + viewdirs = viewdirs.flatten(0, -2).to('cuda') + time_rdstart = time.time() + render_result_chunks = [{ + k: v + for k, v in self.encoder(ro, rd, vd, **render_kwargs).items() + if k in keys + } for ro, rd, vd in zip( + rays_o.split(self.test_ray_chunk, 0), + rays_d.split(self.test_ray_chunk, 0), + viewdirs.split(self.test_ray_chunk, 0))] + render_result = { + k: + torch.cat([ret[k] + for ret in render_result_chunks]).reshape(H, W, -1) + for k in render_result_chunks[0].keys() + } + print(f'render time is: {time.time() - time_rdstart}') + rgb = render_result['rgb_marched'].clamp(0, 1).cpu().numpy() + rgb_feature = render_result['rgb_feature'].cpu().numpy() + depth = render_result['depth'].cpu().numpy() + bgmap = render_result['alphainv_last'].cpu().numpy() + + rgbs.append(rgb) + rgb_features.append(rgb_feature) + depths.append(depth) + bgmaps.append(bgmap) + viewdirs_all.append(viewdirs) + if i == 0: + print('Testing', rgb.shape) + + if gt_imgs is not None and render_factor == 0: + p = -10. * np.log10(np.mean(np.square(rgb - gt_imgs[i]))) + psnrs.append(p) + + if len(psnrs): + print('Testing psnr', np.mean(psnrs), '(avg)') + if eval_ssim: + print('Testing ssim', np.mean(ssims), '(avg)') + if eval_lpips_vgg: + print('Testing lpips (vgg)', np.mean(lpips_vgg), '(avg)') + if eval_lpips_alex: + print('Testing lpips (alex)', np.mean(lpips_alex), '(avg)') + + if savedir is not None and dump_images: + for i in trange(len(rgbs)): + rgb8 = to8b(rgbs[i]) + filename = os.path.join(savedir, '{:03d}_enc.png'.format(i)) + imageio.imwrite(filename, rgb8) + + rgbs = np.array(rgbs) + rgb_features = np.array(rgb_features) + depths = np.array(depths) + bgmaps = np.array(bgmaps) + + return rgbs, depths, bgmaps, psnrs, viewdirs_all, rgb_features + + +def load_everything(cfg_data): + '''Load images / poses / camera settings / data split. + ''' + cfg_data = mmcv.Config(cfg_data) + data_dict = load_data(cfg_data) + + # remove useless field + kept_keys = { + 'hwf', 'HW', 'Ks', 'near', 'far', 'near_clip', 'i_train', 'i_val', + 'i_test', 'irregular_shape', 'poses', 'render_poses', 'images', + 'white_bkgd' + } + # if cfg.data.load_sr: + kept_keys.add('srgt') + kept_keys.add('w2c') + data_dict['srgt'] = torch.FloatTensor(data_dict['srgt'], device='cpu') + data_dict['w2c'] = torch.FloatTensor(data_dict['w2c'], device='cpu') + for k in list(data_dict.keys()): + if k not in kept_keys: + data_dict.pop(k) + + # construct data tensor + if data_dict['irregular_shape']: + data_dict['images'] = [ + torch.FloatTensor(im, device='cpu') for im in data_dict['images'] + ] + else: + data_dict['images'] = torch.FloatTensor( + data_dict['images'], device='cpu') + data_dict['poses'] = torch.Tensor(data_dict['poses']) + return data_dict diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/dvgo.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/dvgo.py new file mode 100644 index 0000000000000000000000000000000000000000..d6cd86d067eae1f6e6a4b54492e4ac52f69bfa79 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/dvgo.py @@ -0,0 +1,1957 @@ +import functools +import math +import os +import time +from copy import deepcopy + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.cpp_extension import load +from torch_scatter import segment_coo + +parent_dir = os.path.dirname(os.path.abspath(__file__)) +parent_list = parent_dir.split('/') +del parent_list[-4:] +parent_dir = '/'.join(parent_list) +render_utils_cuda = load( + name='render_utils_cuda', + sources=[ + os.path.join(parent_dir, path) for path in + ['ops/4knerf/render_utils.cpp', 'ops/4knerf/render_utils_kernel.cu'] + ], + verbose=True) + + +def create_grid(type, **kwargs): + if type == 'DenseGrid': + return DenseGrid(**kwargs) + else: + raise NotImplementedError + + +class DenseGrid(nn.Module): + + def __init__(self, channels, world_size, xyz_min, xyz_max, **kwargs): + super(DenseGrid, self).__init__() + self.channels = channels + self.world_size = world_size + self.register_buffer('xyz_min', torch.Tensor(xyz_min)) + self.register_buffer('xyz_max', torch.Tensor(xyz_max)) + self.grid = nn.Parameter(torch.zeros([1, channels, *world_size])) + + def forward(self, xyz): + ''' + xyz: global coordinates to query + ''' + shape = xyz.shape[:-1] + xyz = xyz.reshape(1, 1, 1, -1, 3) + ind_norm = ((xyz - self.xyz_min) / (self.xyz_max - self.xyz_min)).flip( + (-1, )) * 2 - 1 + out = F.grid_sample( + self.grid, ind_norm, mode='bilinear', align_corners=True) + out = out.reshape(self.channels, -1).T.reshape(*shape, self.channels) + if self.channels == 1: + out = out.squeeze(-1) + return out + + def scale_volume_grid(self, new_world_size): + if self.channels == 0: + self.grid = nn.Parameter( + torch.zeros([1, self.channels, *new_world_size])) + else: + self.grid = nn.Parameter( + F.interpolate( + self.grid.data, + size=tuple(new_world_size), + mode='trilinear', + align_corners=True)) + + def get_dense_grid(self): + return self.grid + + @torch.no_grad() + def __isub__(self, val): + self.grid.data -= val + return self + + def extra_repr(self): + return f'channels={self.channels}, world_size={self.world_size.tolist()}' + + +''' Mask grid +It supports query for the known free space and unknown space. +''' + + +class MaskGrid(nn.Module): + + def __init__(self, + path=None, + mask_cache_thres=None, + mask=None, + xyz_min=None, + xyz_max=None): + super(MaskGrid, self).__init__() + if path is not None: + st = torch.load(path) + self.mask_cache_thres = mask_cache_thres + density = F.max_pool3d( + st['model_state_dict']['density.grid'], + kernel_size=3, + padding=1, + stride=1) + alpha = 1 - torch.exp( + -F.softplus(density + st['model_state_dict']['act_shift']) + * st['model_kwargs']['voxel_size_ratio']) + mask = (alpha >= self.mask_cache_thres).squeeze(0).squeeze(0) + xyz_min = torch.Tensor(st['model_kwargs']['xyz_min']) + xyz_max = torch.Tensor(st['model_kwargs']['xyz_max']) + else: + mask = mask.bool() + xyz_min = torch.Tensor(xyz_min) + xyz_max = torch.Tensor(xyz_max) + + self.register_buffer('mask', mask) + xyz_len = xyz_max - xyz_min + self.register_buffer('xyz2ijk_scale', + (torch.Tensor(list(mask.shape)) - 1) / xyz_len) + self.register_buffer('xyz2ijk_shift', -xyz_min * self.xyz2ijk_scale) + + @torch.no_grad() + def forward(self, xyz): + '''Skip know freespace + @xyz: [..., 3] the xyz in global coordinate. + ''' + shape = xyz.shape[:-1] + xyz = xyz.reshape(-1, 3) + mask = render_utils_cuda.maskcache_lookup(self.mask, xyz, + self.xyz2ijk_scale, + self.xyz2ijk_shift) + mask = mask.reshape(shape) + return mask + + def extra_repr(self): + return 'mask.shape=list(self.mask.shape)' + + +'''Model''' + + +class DirectVoxGO(torch.nn.Module): + + def __init__(self, + xyz_min, + xyz_max, + num_voxels=0, + num_voxels_base=0, + alpha_init=None, + mask_cache_path=None, + mask_cache_thres=1e-3, + mask_cache_world_size=None, + fast_color_thres=0, + density_type='DenseGrid', + k0_type='DenseGrid', + density_config={}, + k0_config={}, + rgbnet_dim=0, + rgbnet_direct=False, + rgbnet_full_implicit=False, + rgbnet_depth=3, + rgbnet_width=128, + viewbase_pe=4, + **kwargs): + super(DirectVoxGO, self).__init__() + self.register_buffer('xyz_min', torch.Tensor(xyz_min)) + self.register_buffer('xyz_max', torch.Tensor(xyz_max)) + self.fast_color_thres = fast_color_thres + + # determine based grid resolution + self.num_voxels_base = num_voxels_base + self.voxel_size_base = ((self.xyz_max - self.xyz_min).prod() + / self.num_voxels_base).pow(1 / 3) + + # determine the density bias shift + self.alpha_init = alpha_init + self.register_buffer( + 'act_shift', torch.FloatTensor([np.log(1 / (1 - alpha_init) - 1)])) + print('dvgo: set density bias shift to', self.act_shift) + + # determine init grid resolution + self._set_grid_resolution(num_voxels) + + # init density voxel grid + self.density_type = density_type + self.density_config = density_config + self.density = create_grid( + density_type, + channels=1, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.density_config) + + # init color representation + self.rgbnet_kwargs = { + 'rgbnet_dim': rgbnet_dim, + 'rgbnet_direct': rgbnet_direct, + 'rgbnet_full_implicit': rgbnet_full_implicit, + 'rgbnet_depth': rgbnet_depth, + 'rgbnet_width': rgbnet_width, + 'viewbase_pe': viewbase_pe, + } + self.k0_type = k0_type + self.k0_config = k0_config + self.rgbnet_full_implicit = rgbnet_full_implicit + self.dim_rend = 3 # kwargs['dim_rend'] + self.act_type = 'mlp' # kwargs['act_type'] + self.mode_type = 'mlp' + + if rgbnet_dim <= 0: + # color voxel grid (coarse stage) + self.k0_dim = 3 + self.k0 = create_grid( + k0_type, + channels=self.k0_dim, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.k0_config) + self.rgbnet = None + else: + # feature voxel grid + shallow MLP (fine stage) + if self.rgbnet_full_implicit: + self.k0_dim = 0 + else: + self.k0_dim = rgbnet_dim + self.k0 = create_grid( + k0_type, + channels=self.k0_dim, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.k0_config) + self.rgbnet_direct = rgbnet_direct + self.register_buffer( + 'viewfreq', + torch.FloatTensor([(2**i) for i in range(viewbase_pe)])) + dim0 = (3 + 3 * viewbase_pe * 2) + + if self.rgbnet_full_implicit: + pass + elif rgbnet_direct: + dim0 += self.k0_dim + else: + dim0 += self.k0_dim - 3 + + self.dim0 = dim0 + if self.dim_rend > 3: + self.rgbnet = nn.Sequential( + nn.Linear(self.dim0, rgbnet_width), + nn.LeakyReLU(), + nn.Linear(rgbnet_width, self.dim_rend), + nn.LeakyReLU(), + ) + self.rend_layer = nn.Sequential(nn.Linear(self.dim_rend, 3), ) + nn.init.constant_(self.rend_layer[-1].bias, 0) + else: + self.rgbnet = nn.Sequential( + nn.Linear(dim0, rgbnet_width), + nn.ReLU(inplace=True), + *[ + nn.Sequential( + nn.Linear(rgbnet_width, rgbnet_width), + nn.ReLU(inplace=True)) + for _ in range(rgbnet_depth - 2) + ], + nn.Linear(rgbnet_width, 3), + ) + nn.init.constant_(self.rgbnet[-1].bias, 0) + + print('dvgo: feature voxel grid', self.k0) + print('dvgo: mlp', self.rgbnet) + + # Using the coarse geometry if provided (used to determine known free space and unknown space) + # Re-implement as occupancy grid (2021/1/31) + self.mask_cache_path = mask_cache_path + self.mask_cache_thres = mask_cache_thres + if mask_cache_world_size is None: + mask_cache_world_size = self.world_size + if mask_cache_path is not None and mask_cache_path: + mask_cache = MaskGrid( + path=mask_cache_path, + mask_cache_thres=mask_cache_thres).to('cuda') + self_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + mask_cache_world_size[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + mask_cache_world_size[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + mask_cache_world_size[2]), + ), -1).to('cuda') + mask = mask_cache(self_grid_xyz) + else: + mask = torch.ones(list(mask_cache_world_size), dtype=torch.bool) + self.mask_cache = MaskGrid( + path=None, mask=mask, xyz_min=self.xyz_min, + xyz_max=self.xyz_max).to(self.xyz_min.device) + + def _set_grid_resolution(self, num_voxels): + # Determine grid resolution + self.num_voxels = num_voxels + self.voxel_size = ((self.xyz_max - self.xyz_min).prod() + / num_voxels).pow(1 / 3) + self.world_size = ((self.xyz_max - self.xyz_min) + / self.voxel_size).long() + self.max_world_size = self.world_size.max() + self.voxel_size_ratio = self.voxel_size / self.voxel_size_base + print('dvgo: voxel_size ', self.voxel_size) + print('dvgo: world_size ', self.world_size) + print('dvgo: voxel_size_base ', self.voxel_size_base) + print('dvgo: voxel_size_ratio', self.voxel_size_ratio) + + def get_kwargs(self): + return { + 'xyz_min': self.xyz_min.cpu().numpy(), + 'xyz_max': self.xyz_max.cpu().numpy(), + 'num_voxels': self.num_voxels, + 'num_voxels_base': self.num_voxels_base, + 'alpha_init': self.alpha_init, + 'voxel_size_ratio': self.voxel_size_ratio, + 'mask_cache_path': self.mask_cache_path, + 'mask_cache_thres': self.mask_cache_thres, + 'mask_cache_world_size': list(self.mask_cache.mask.shape), + 'fast_color_thres': self.fast_color_thres, + 'density_type': self.density_type, + 'k0_type': self.k0_type, + 'density_config': self.density_config, + 'k0_config': self.k0_config, + 'mode_type': self.mode_type, + 'act_type': self.act_type, + 'dim_rend': self.dim_rend, + **self.rgbnet_kwargs, + } + + @torch.no_grad() + def maskout_near_cam_vox(self, cam_o, near_clip): + # maskout grid points that between cameras and their near planes + self_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + self.world_size[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + self.world_size[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + self.world_size[2]), + ), -1) + nearest_dist = torch.stack([ + (self_grid_xyz.unsqueeze(-2) - co).pow(2).sum(-1).sqrt().amin(-1) + for co in cam_o.split(100) # for memory saving + ]).amin(0) + self.density.grid[nearest_dist[None, None] <= near_clip] = -100 + + @torch.no_grad() + def scale_volume_grid(self, num_voxels): + print('dvgo: scale_volume_grid start') + ori_world_size = self.world_size + self._set_grid_resolution(num_voxels) + print('dvgo: scale_volume_grid scale world_size from', + ori_world_size.tolist(), 'to', self.world_size.tolist()) + + self.density.scale_volume_grid(self.world_size) + self.k0.scale_volume_grid(self.world_size) + + if np.prod(self.world_size.tolist()) <= 256**3: + self_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + self.world_size[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + self.world_size[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + self.world_size[2]), + ), -1) + self_alpha = F.max_pool3d( + self.activate_density(self.density.get_dense_grid()), + kernel_size=3, + padding=1, + stride=1)[0, 0] + self.mask_cache = MaskGrid( + path=None, + mask=self.mask_cache(self_grid_xyz) + & (self_alpha > self.fast_color_thres), + xyz_min=self.xyz_min, + xyz_max=self.xyz_max) + + print('dvgo: scale_volume_grid finish') + + @torch.no_grad() + def update_occupancy_cache(self): + cache_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + self.mask_cache.mask.shape[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + self.mask_cache.mask.shape[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + self.mask_cache.mask.shape[2]), + ), -1) + cache_grid_density = self.density(cache_grid_xyz)[None, None] + cache_grid_alpha = self.activate_density(cache_grid_density) + cache_grid_alpha = F.max_pool3d( + cache_grid_alpha, kernel_size=3, padding=1, stride=1)[0, 0] + self.mask_cache.mask &= (cache_grid_alpha > self.fast_color_thres) + + def voxel_count_views(self, + rays_o_tr, + rays_d_tr, + imsz, + near, + far, + stepsize, + downrate=1, + irregular_shape=False): + print('dvgo: voxel_count_views start') + far = 1e9 # the given far can be too small while rays stop when hitting scene bbox + eps_time = time.time() + N_samples = int( + np.linalg.norm(np.array(self.world_size.cpu()) + 1) / stepsize) + 1 + rng = torch.arange(N_samples)[None].float() + count = torch.zeros_like(self.density.get_dense_grid()) + device = rng.device + for rays_o_, rays_d_ in zip( + rays_o_tr.split(imsz), rays_d_tr.split(imsz)): + ones = DenseGrid(1, self.world_size, self.xyz_min, self.xyz_max) + if irregular_shape: + rays_o_ = rays_o_.split(10000) + rays_d_ = rays_d_.split(10000) + else: + rays_o_ = rays_o_[::downrate, ::downrate].to(device).flatten( + 0, -2).split(10000) + rays_d_ = rays_d_[::downrate, ::downrate].to(device).flatten( + 0, -2).split(10000) + + for rays_o, rays_d in zip(rays_o_, rays_d_): + vec = torch.where(rays_d == 0, torch.full_like(rays_d, 1e-6), + rays_d) + rate_a = (self.xyz_max - rays_o) / vec + rate_b = (self.xyz_min - rays_o) / vec + t_min = torch.minimum(rate_a, rate_b).amax(-1).clamp( + min=near, max=far) + # t_max = torch.maximum(rate_a, rate_b).amin(-1).clamp( + # min=near, max=far) + step = stepsize * self.voxel_size * rng + interpx = ( + t_min[..., None] + + step / rays_d.norm(dim=-1, keepdim=True)) + rays_pts = rays_o[ + ..., None, :] + rays_d[..., None, :] * interpx[..., None] + ones(rays_pts).sum().backward() + with torch.no_grad(): + count += (ones.grid.grad > 1) + eps_time = time.time() - eps_time + print('dvgo: voxel_count_views finish (eps time:', eps_time, 'sec)') + return count + + def activate_density(self, density, interval=None): + interval = interval if interval is not None else self.voxel_size_ratio + shape = density.shape + return Raw2Alpha.apply(density.flatten(), self.act_shift, + interval).reshape(shape) + + def hit_coarse_geo(self, rays_o, rays_d, near, far, stepsize, + **render_kwargs): + '''Check whether the rays hit the solved coarse geometry or not''' + far = 1e9 # the given far can be too small while rays stop when hitting scene bbox + shape = rays_o.shape[:-1] + rays_o = rays_o.reshape(-1, 3).contiguous() + rays_d = rays_d.reshape(-1, 3).contiguous() + stepdist = stepsize * self.voxel_size + ray_pts, mask_outbbox, ray_id = render_utils_cuda.sample_pts_on_rays( + rays_o, rays_d, self.xyz_min, self.xyz_max, near, far, + stepdist)[:3] + mask_inbbox = ~mask_outbbox + hit = torch.zeros([len(rays_o)], dtype=torch.bool) + hit[ray_id[mask_inbbox][self.mask_cache(ray_pts[mask_inbbox])]] = 1 + return hit.reshape(shape) + + def sample_ray(self, rays_o, rays_d, near, far, stepsize, **render_kwargs): + '''Sample query points on rays. + All the output points are sorted from near to far. + Input: + rays_o, rayd_d: both in [N, 3] indicating ray configurations. + near, far: the near and far distance of the rays. + stepsize: the number of voxels of each sample step. + Output: + ray_pts: [M, 3] storing all the sampled points. + ray_id: [M] the index of the ray of each point. + step_id: [M] the i'th step on a ray of each point. + ''' + far = 1e9 # the given far can be too small while rays stop when hitting scene bbox + rays_o = rays_o.contiguous() + rays_d = rays_d.contiguous() + stepdist = stepsize * self.voxel_size + N_samples = int((self.max_world_size - 1) / stepsize) + 1 + + ray_pts, mask_outbbox, ray_id, step_id, N_steps, t_min, t_max = render_utils_cuda.sample_pts_on_rays( + rays_o, rays_d, self.xyz_min, self.xyz_max, near, far, stepdist) + mask_inbbox = ~mask_outbbox + + # mask_ori = torch.zeros((len(N_steps), N_steps.max())).bool() + # for idx, item in enumerate(N_steps): + # mask_ori[idx, :item] = mask_inbbox[ray_id == idx] + mask_ori = None + + ray_pts = ray_pts[mask_inbbox] + ray_id = ray_id[mask_inbbox] + step_id = step_id[mask_inbbox] + return ray_pts, ray_id, step_id, mask_ori, N_samples + + def forward(self, + rays_o, + rays_d, + viewdirs, + global_step=None, + **render_kwargs): + '''Volume rendering + @rays_o: [N, 3] the starting point of the N shooting rays. + @rays_d: [N, 3] the shooting direction of the N rays. + @viewdirs: [N, 3] viewing direction to compute positional embedding for MLP. + ''' + assert len(rays_o.shape) == 2 and rays_o.shape[ + -1] == 3, 'Only suuport point queries in [N, 3] format' + + ret_dict = {} + N = len(rays_o) + + # sample points on rays + ray_pts, ray_id, step_id, mask_inbbox, N_samples = self.sample_ray( + rays_o=rays_o, rays_d=rays_d, **render_kwargs) + interval = render_kwargs['stepsize'] * self.voxel_size_ratio + + # skip known free space + if self.mask_cache is not None: + mask1 = self.mask_cache(ray_pts) + ray_pts = ray_pts[mask1] + ray_id = ray_id[mask1] + step_id = step_id[mask1] + + # query for alpha w/ post-activation + density = self.density(ray_pts) + alpha = self.activate_density(density, interval) + if self.fast_color_thres > 0: + mask2 = (alpha > self.fast_color_thres) + ray_pts = ray_pts[mask2] + ray_id = ray_id[mask2] + step_id = step_id[mask2] + density = density[mask2] + alpha = alpha[mask2] + + # compute accumulated transmittance + weights, alphainv_last = Alphas2Weights.apply(alpha, ray_id, N) + if self.fast_color_thres > 0: + mask3 = (weights > self.fast_color_thres) + weights = weights[mask3] + alpha = alpha[mask3] + ray_pts = ray_pts[mask3] + ray_id = ray_id[mask3] + step_id = step_id[mask3] + + # query for color + if self.rgbnet_full_implicit: + pass + else: + k0 = self.k0(ray_pts) + + if self.rgbnet is None: + # no view-depend effect + rgb_raw = torch.sigmoid(k0) + else: + # view-dependent color emission + if self.rgbnet_direct: + k0_view = k0 + else: + k0_view = k0[:, 3:] + k0_diffuse = k0[:, :3] + viewdirs_emb = (viewdirs.unsqueeze(-1) * self.viewfreq).flatten(-2) + viewdirs_emb = torch.cat( + [viewdirs, viewdirs_emb.sin(), + viewdirs_emb.cos()], -1) + viewdirs_emb = viewdirs_emb.flatten(0, -2)[ray_id] + rgb_feat = torch.cat([k0_view, viewdirs_emb], -1) + + if self.mode_type == 'TRANS': + rgb_feat_pre3 = torch.zeros((mask3.shape[0], self.dim0)) + rgb_feat_pre3[mask3] = rgb_feat + rgb_feat_pre2 = torch.zeros((mask2.shape[0], self.dim0)) + rgb_feat_pre2[mask2] = rgb_feat_pre3 + rgb_feat_pre1 = torch.zeros((mask1.shape[0], self.dim0)) + rgb_feat_pre1[mask1] = rgb_feat_pre2 + rgb_feat_ori = torch.zeros( + (mask_inbbox.shape[0], mask_inbbox.shape[1], self.dim0)) + rgb_feat_ori[mask_inbbox] = rgb_feat_pre1 + # rgb_logit_tran = self.reformer(rgb_feat_ori) + rgb_logit = self.trans_nn(rgb_feat_ori) + rgb_logit = rgb_logit[mask_inbbox.view( + -1)][mask1][mask2][mask3] + elif self.mode_type == 'adain': + rgb_logit = self.adainet(rgb_feat, rgb_feat) + else: + rgb_logit = self.rgbnet(rgb_feat) + + if self.rgbnet_direct: + rgb_raw = torch.sigmoid(rgb_logit) + else: + rgb_raw = torch.sigmoid(rgb_logit + k0_diffuse) + + # Ray marching + rgb_feature = segment_coo( + src=(weights.unsqueeze(-1) * rgb_raw), + index=ray_id, + out=torch.zeros([N, self.dim_rend], device='cuda'), + reduce='sum') + if self.dim_rend > 3: + rgb_raw = torch.sigmoid(self.rend_layer(rgb_raw)) + rgb_marched = self.rend_layer(rgb_feature) + # rgb_marched = torch.sigmoid(rgb_marched) + else: + rgb_marched = rgb_feature + + rgb_marched += (alphainv_last.unsqueeze(-1) * render_kwargs['bg']) + s = (step_id + 0.5) / N_samples + ret_dict.update({ + 'alphainv_last': alphainv_last, + 'weights': weights, + 'rgb_marched': rgb_marched, + 'rgb_feature': rgb_feature, + 'raw_alpha': alpha, + 'raw_rgb': rgb_raw, + 'ray_id': ray_id, + }) + + if render_kwargs.get('render_depth', False): + with torch.no_grad(): + depth = segment_coo( + src=(weights * s), # step_id + index=ray_id, + out=torch.zeros([N], device='cuda'), + reduce='sum') + ret_dict.update({'depth': depth}) + + return ret_dict + + +class DirectMPIGO(torch.nn.Module): + + def __init__(self, + xyz_min, + xyz_max, + num_voxels=0, + mpi_depth=0, + mask_cache_path=None, + mask_cache_thres=1e-3, + mask_cache_world_size=None, + fast_color_thres=0, + density_type='DenseGrid', + k0_type='DenseGrid', + density_config={}, + k0_config={}, + rgbnet_dim=0, + rgbnet_depth=3, + rgbnet_width=128, + viewbase_pe=0, + spatial_pe=0, + **kwargs): + super(DirectMPIGO, self).__init__() + self.register_buffer('xyz_min', torch.Tensor(xyz_min)) + self.register_buffer('xyz_max', torch.Tensor(xyz_max)) + self.fast_color_thres = fast_color_thres + + # determine init grid resolution + self._set_grid_resolution(num_voxels, mpi_depth) + + # init density voxel grid + self.density_type = density_type + self.density_config = density_config + self.density = create_grid( + density_type, + channels=1, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.density_config) + + # init density bias so that the initial contribution (the alpha values) + # of each query points on a ray is equal + self.act_shift = DenseGrid( + channels=1, + world_size=[1, 1, mpi_depth], + xyz_min=xyz_min, + xyz_max=xyz_max) + self.act_shift.grid.requires_grad = False + with torch.no_grad(): + g = np.full([mpi_depth], 1. / mpi_depth - 1e-6) + p = [1 - g[0]] + for i in range(1, len(g)): + p.append((1 - g[:i + 1].sum()) / (1 - g[:i].sum())) + for i in range(len(p)): + self.act_shift.grid[..., i].fill_( + np.log(p[i]**(-1 / self.voxel_size_ratio) - 1)) + + # init color representation + # feature voxel grid + shallow MLP (fine stage) + self.rgbnet_kwargs = { + 'rgbnet_dim': rgbnet_dim, + 'rgbnet_depth': rgbnet_depth, + 'rgbnet_width': rgbnet_width, + 'viewbase_pe': viewbase_pe, + 'spatial_pe': spatial_pe, + } + self.k0_type = k0_type + self.k0_config = k0_config + if rgbnet_dim <= 0: + # color voxel grid (coarse stage) + self.k0_dim = 3 + self.k0 = create_grid( + k0_type, + channels=self.k0_dim, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.k0_config) + self.rgbnet = None + else: + self.k0_dim = rgbnet_dim + self.k0 = create_grid( + k0_type, + channels=self.k0_dim, + world_size=self.world_size, + xyz_min=self.xyz_min, + xyz_max=self.xyz_max, + config=self.k0_config) + self.register_buffer( + 'viewfreq', + torch.FloatTensor([(2**i) for i in range(viewbase_pe)])) + self.register_buffer( + 'posfreq', + torch.FloatTensor([(2**i) for i in range(spatial_pe)])) + self.dim0 = (3 + 3 * viewbase_pe * 2 + 3 + + 3 * spatial_pe * 2) + self.k0_dim + self.pe_dim = 3 + 3 * viewbase_pe * 2 + 3 + 3 * spatial_pe * 2 + + self.dim_rend = 3 # kwargs['dim_rend'] + self.act_type = kwargs['act_type'] + act = nn.ReLU(inplace=True) + + if self.dim_rend > 3: + self.rgbnet = nn.Sequential( + nn.Linear(self.dim0, rgbnet_width), + nn.LeakyReLU(), + nn.Linear(rgbnet_width, self.dim_rend), + nn.LeakyReLU(), + ) + self.rend_layer = nn.Sequential(nn.Linear(self.dim_rend, 3), ) + # self.rgbper_layer = nn.Sequential( + # nn.Linear(self.dim_rend, 3) + # ) + nn.init.constant_(self.rend_layer[-1].bias, 0) + else: + self.rgbnet = nn.Sequential( + nn.Linear(self.dim0, rgbnet_width), + act, + *[ + nn.Sequential( + nn.Linear(rgbnet_width, rgbnet_width), act) + for _ in range(rgbnet_depth - 2) + ], + nn.Linear(rgbnet_width, self.dim_rend), + ) + nn.init.constant_(self.rgbnet[-1].bias, 0) + + print('dmpigo: densitye grid', self.density) + print('dmpigo: feature grid', self.k0) + self.mode_type = kwargs['mode_type'] + print('dmpigo: mlp', self.rgbnet) + + # Using the coarse geometry if provided (used to determine known free space and unknown space) + # Re-implement as occupancy grid (2021/1/31) + self.mask_cache_path = mask_cache_path + self.mask_cache_thres = mask_cache_thres + if mask_cache_world_size is None: + mask_cache_world_size = self.world_size + if mask_cache_path is not None and mask_cache_path: + mask_cache = MaskGrid( + path=mask_cache_path, + mask_cache_thres=mask_cache_thres).to(self.xyz_min.device) + self_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + mask_cache_world_size[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + mask_cache_world_size[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + mask_cache_world_size[2]), + ), -1) + mask = mask_cache(self_grid_xyz) + else: + mask = torch.ones(list(mask_cache_world_size), dtype=torch.bool) + self.mask_cache = MaskGrid( + path=None, mask=mask, xyz_min=self.xyz_min, xyz_max=self.xyz_max) + + def _set_grid_resolution(self, num_voxels, mpi_depth): + # Determine grid resolution + self.num_voxels = num_voxels + self.mpi_depth = mpi_depth + r = num_voxels / self.mpi_depth + r = (r / (self.xyz_max - self.xyz_min)[:2].prod()).sqrt() + self.world_size = torch.zeros(3, dtype=torch.long) + self.world_size[:2] = (self.xyz_max - self.xyz_min)[:2] * r + self.world_size[2] = self.mpi_depth + self.voxel_size_ratio = 256. / mpi_depth + print('dmpigo: world_size ', self.world_size) + print('dmpigo: voxel_size_ratio', self.voxel_size_ratio) + + def get_kwargs(self): + return { + 'xyz_min': self.xyz_min.cpu().numpy(), + 'xyz_max': self.xyz_max.cpu().numpy(), + 'num_voxels': self.num_voxels, + 'mpi_depth': self.mpi_depth, + 'voxel_size_ratio': self.voxel_size_ratio, + 'mask_cache_path': self.mask_cache_path, + 'mask_cache_thres': self.mask_cache_thres, + 'mask_cache_world_size': list(self.mask_cache.mask.shape), + 'fast_color_thres': self.fast_color_thres, + 'density_type': self.density_type, + 'k0_type': self.k0_type, + 'density_config': self.density_config, + 'k0_config': self.k0_config, + 'mode_type': self.mode_type, + 'act_type': self.act_type, + 'dim_rend': self.dim_rend, + **self.rgbnet_kwargs, + } + + @torch.no_grad() + def scale_volume_grid(self, num_voxels, mpi_depth): + print('dmpigo: scale_volume_grid start') + ori_world_size = self.world_size + self._set_grid_resolution(num_voxels, mpi_depth) + print('dmpigo: scale_volume_grid scale world_size from', + ori_world_size.tolist(), 'to', self.world_size.tolist()) + + self.density.scale_volume_grid(self.world_size) + self.k0.scale_volume_grid(self.world_size) + + if np.prod(self.world_size.tolist()) <= 256**3: + self_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + self.world_size[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + self.world_size[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + self.world_size[2]), + ), -1) + dens = self.density.get_dense_grid() + self.act_shift.grid + self_alpha = F.max_pool3d( + self.activate_density(dens), + kernel_size=3, + padding=1, + stride=1)[0, 0] + self.mask_cache = MaskGrid( + path=None, + mask=self.mask_cache(self_grid_xyz) + & (self_alpha > self.fast_color_thres), + xyz_min=self.xyz_min, + xyz_max=self.xyz_max) + + print('dmpigo: scale_volume_grid finish') + + @torch.no_grad() + def update_occupancy_cache(self): + ori_p = self.mask_cache.mask.float().mean().item() + cache_grid_xyz = torch.stack( + torch.meshgrid( + torch.linspace(self.xyz_min[0], self.xyz_max[0], + self.mask_cache.mask.shape[0]), + torch.linspace(self.xyz_min[1], self.xyz_max[1], + self.mask_cache.mask.shape[1]), + torch.linspace(self.xyz_min[2], self.xyz_max[2], + self.mask_cache.mask.shape[2]), + ), -1) + cache_grid_density = self.density(cache_grid_xyz)[None, None] + cache_grid_alpha = self.activate_density(cache_grid_density) + cache_grid_alpha = F.max_pool3d( + cache_grid_alpha, kernel_size=3, padding=1, stride=1)[0, 0] + self.mask_cache.mask &= (cache_grid_alpha > self.fast_color_thres) + new_p = self.mask_cache.mask.float().mean().item() + print(f'dmpigo: update mask_cache {ori_p:.4f} => {new_p:.4f}') + + def update_occupancy_cache_lt_nviews(self, rays_o_tr, rays_d_tr, imsz, + render_kwargs, maskout_lt_nviews): + print('dmpigo: update mask_cache lt_nviews start') + eps_time = time.time() + count = torch.zeros_like(self.density.get_dense_grid()).long() + device = count.device + for rays_o_, rays_d_ in zip( + rays_o_tr.split(imsz), rays_d_tr.split(imsz)): + ones = DenseGrid(1, self.world_size, self.xyz_min, self.xyz_max) + for rays_o, rays_d in zip( + rays_o_.split(8192), rays_d_.split(8192)): + ray_pts, ray_id, step_id, N_samples, _ = self.sample_ray( + rays_o=rays_o.to(device), + rays_d=rays_d.to(device), + **render_kwargs) + ones(ray_pts).sum().backward() + count.data += (ones.grid.grad > 1) + ori_p = self.mask_cache.mask.float().mean().item() + self.mask_cache.mask &= (count >= maskout_lt_nviews)[0, 0] + new_p = self.mask_cache.mask.float().mean().item() + print(f'dmpigo: update mask_cache {ori_p:.4f} => {new_p:.4f}') + torch.cuda.empty_cache() + eps_time = time.time() - eps_time + print('dmpigo: update mask_cache lt_nviews finish (eps time:', + eps_time, 'sec)') + + def activate_density(self, density, interval=None): + interval = interval if interval is not None else self.voxel_size_ratio + shape = density.shape + return Raw2Alpha.apply(density.flatten(), 0, interval).reshape(shape) + + def sample_ray(self, rays_o, rays_d, near, far, stepsize, **render_kwargs): + '''Sample query points on rays. + All the output points are sorted from near to far. + Input: + rays_o, rayd_d: both in [N, 3] indicating ray configurations. + near, far: the near and far distance of the rays. + stepsize: the number of voxels of each sample step. + Output: + ray_pts: [M, 3] storing all the sampled points. + ray_id: [M] the index of the ray of each point. + step_id: [M] the i'th step on a ray of each point. + ''' + assert near == 0 and far == 1 + rays_o = rays_o.contiguous() + rays_d = rays_d.contiguous() + N_samples = int((self.mpi_depth - 1) / stepsize) + 1 + ray_pts, mask_outbbox = render_utils_cuda.sample_ndc_pts_on_rays( + rays_o, rays_d, self.xyz_min, self.xyz_max, N_samples) + mask_inbbox = ~mask_outbbox + + ray_pts = ray_pts.view(-1, 3) + ray_pts = ray_pts[mask_inbbox.view(-1)] + if mask_inbbox.all(): + ray_id, step_id = create_full_step_id(mask_inbbox.shape) + else: + ray_id = torch.arange(mask_inbbox.shape[0]).to('cuda').view( + -1, 1).expand_as(mask_inbbox)[mask_inbbox].to('cuda') + step_id = torch.arange(mask_inbbox.shape[1]).to('cuda').view( + 1, -1).expand_as(mask_inbbox)[mask_inbbox].to('cuda') + return ray_pts, ray_id, step_id, N_samples, mask_inbbox + + def forward(self, + rays_o, + rays_d, + viewdirs, + global_step=None, + **render_kwargs): + '''Volume rendering + @rays_o: [N, 3] the starting point of the N shooting rays. + @rays_d: [N, 3] the shooting direction of the N rays. + @viewdirs: [N, 3] viewing direction to compute positional embedding for MLP. + ''' + assert len(rays_o.shape) == 2 and rays_o.shape[ + -1] == 3, 'Only suuport point queries in [N, 3] format' + + ret_dict = {} + N = len(rays_o) + + # sample points on rays + ray_pts, ray_id, step_id, N_samples, mask_inbbox = self.sample_ray( + rays_o=rays_o, rays_d=rays_d, **render_kwargs) + ray_id = ray_id.to('cuda') + step_id = step_id.to('cuda') + interval = render_kwargs['stepsize'] * self.voxel_size_ratio + + # skip known free space + if self.mask_cache is not None: + mask1 = self.mask_cache(ray_pts) + ray_pts = ray_pts[mask1] + ray_id = ray_id[mask1] + step_id = step_id[mask1] + + # query for alpha w/ post-activation + density = self.density(ray_pts) + self.act_shift(ray_pts) + alpha = self.activate_density(density, interval) + if self.fast_color_thres > 0: + mask2 = (alpha > self.fast_color_thres) + ray_pts = ray_pts[mask2] + ray_id = ray_id[mask2] + step_id = step_id[mask2] + alpha = alpha[mask2] + + # compute accumulated transmittance + weights, alphainv_last = Alphas2Weights.apply(alpha, ray_id, N) + if self.fast_color_thres > 0: + mask3 = (weights > self.fast_color_thres) + ray_pts = ray_pts[mask3] + ray_id = ray_id[mask3] + step_id = step_id[mask3] + alpha = alpha[mask3] + weights = weights[mask3] + + # query for color + vox_emb = self.k0(ray_pts) + + pe_spa = ((ray_pts - self.xyz_min) / (self.xyz_max - self.xyz_min)) + pe_spa = pe_spa.flip((-1, )) * 2 - 1 + # B_gau = torch.normal(mean=0, std=1, size=(128, 3)).to(rays_o.device) * 10 + # pe_emb = input_mapping(pe_spa.detach().clone(), B_gau) + + if self.rgbnet is None: + # no view-depend effect + rgb_raw = torch.sigmoid(vox_emb) + else: + # view-dependent color emission + viewdirs_emb = (viewdirs.unsqueeze(-1) * self.viewfreq).flatten(-2) + viewdirs_emb = torch.cat( + [viewdirs, viewdirs_emb.sin(), + viewdirs_emb.cos()], -1) + viewdirs_emb = viewdirs_emb[ray_id] + pe_emb = (pe_spa.unsqueeze(-1) * self.posfreq).flatten(-2) + pe_emb = torch.cat([pe_spa, pe_emb.sin(), pe_emb.cos()], -1) + + rgb_feat = torch.cat([vox_emb, pe_emb, viewdirs_emb], -1) + rgb_logit = self.rgbnet(rgb_feat) + if self.dim_rend == 3: + rgb_raw = torch.sigmoid(rgb_logit) + else: + rgb_raw = torch.sigmoid(rgb_logit) + + # Ray marching + rgb_feature = segment_coo( + src=(weights.unsqueeze(-1) * rgb_raw), + index=ray_id, + out=torch.zeros([N, self.dim_rend], device='cuda'), + reduce='sum') + if self.dim_rend > 3: + rgb_raw = torch.sigmoid(self.rend_layer(rgb_raw)) + rgb_marched = self.rend_layer(rgb_feature) + # rgb_marched = torch.sigmoid(rgb_marched) + else: + rgb_marched = rgb_feature + + if render_kwargs.get('rand_bkgd', False) and global_step is not None: + rgb_marched = rgb_marched + ( + alphainv_last.unsqueeze(-1) * torch.rand_like(rgb_marched)) + else: + rgb_marched += (alphainv_last.unsqueeze(-1) * render_kwargs['bg']) + s = (step_id + 0.5) / N_samples + ret_dict.update({ + 'alphainv_last': alphainv_last, + 'weights': weights, + 'rgb_marched': rgb_marched, + 'rgb_feature': rgb_feature, + 'raw_alpha': alpha, + 'raw_rgb': rgb_raw, + 'ray_id': ray_id, + 'n_max': N_samples, + 's': s, + }) + # print('alphainv_last shape:', alphainv_last.shape) + # print('weights shape:', weights.shape) + # print('raw_alpha shape:', alpha.shape) + # print('raw_rgb shape:', rgb.shape) + # print('ray_id shape:', ray_id.shape) + # print('rgb_marched shape:', rgb_marched.shape) + # print('s shape:', s.shape, '\n') + + if render_kwargs.get('render_depth', False): + with torch.no_grad(): + depth = segment_coo( + src=(weights * s), + index=ray_id, + out=torch.zeros([N], device='cuda'), + reduce='sum') + ret_dict.update({'depth': depth}) + + return ret_dict + + +@functools.lru_cache(maxsize=128) +def create_full_step_id(shape): + ray_id = torch.arange(shape[0]).view(-1, 1).expand(shape).flatten() + step_id = torch.arange(shape[1]).view(1, -1).expand(shape).flatten() + return ray_id, step_id + + +''' Misc +''' + + +class Raw2Alpha(torch.autograd.Function): + + @staticmethod + def forward(ctx, density, shift, interval): + ''' + alpha = 1 - exp(-softplus(density + shift) * interval) + = 1 - exp(-log(1 + exp(density + shift)) * interval) + = 1 - exp(log(1 + exp(density + shift)) ^ (-interval)) + = 1 - (1 + exp(density + shift)) ^ (-interval) + ''' + exp, alpha = render_utils_cuda.raw2alpha(density, shift, interval) + if density.requires_grad: + ctx.save_for_backward(exp) + ctx.interval = interval + return alpha + + @staticmethod + @torch.autograd.function.once_differentiable + def backward(ctx, grad_back): + ''' + alpha' = interval * ((1 + exp(density + shift)) ^ (-interval-1)) * exp(density + shift)' + = interval * ((1 + exp(density + shift)) ^ (-interval-1)) * exp(density + shift) + ''' + exp = ctx.saved_tensors[0] + interval = ctx.interval + return render_utils_cuda.raw2alpha_backward(exp, + grad_back.contiguous(), + interval), None, None + + +class Raw2Alpha_nonuni(torch.autograd.Function): + + @staticmethod + def forward(ctx, density, shift, interval): + exp, alpha = render_utils_cuda.raw2alpha_nonuni( + density, shift, interval) + if density.requires_grad: + ctx.save_for_backward(exp) + ctx.interval = interval + return alpha + + @staticmethod + @torch.autograd.function.once_differentiable + def backward(ctx, grad_back): + exp = ctx.saved_tensors[0] + interval = ctx.interval + return render_utils_cuda.raw2alpha_nonuni_backward( + exp, grad_back.contiguous(), interval), None, None + + +class Alphas2Weights(torch.autograd.Function): + + @staticmethod + def forward(ctx, alpha, ray_id, N): + weights, T, alphainv_last, i_start, i_end = render_utils_cuda.alpha2weight( + alpha, ray_id, N) + if alpha.requires_grad: + ctx.save_for_backward(alpha, weights, T, alphainv_last, i_start, + i_end) + ctx.n_rays = N + return weights, alphainv_last + + @staticmethod + @torch.autograd.function.once_differentiable + def backward(ctx, grad_weights, grad_last): + alpha, weights, T, alphainv_last, i_start, i_end = ctx.saved_tensors + grad = render_utils_cuda.alpha2weight_backward(alpha, weights, T, + alphainv_last, i_start, + i_end, ctx.n_rays, + grad_weights, grad_last) + return grad, None, None + + +''' Ray and batch +''' + + +def get_rays(H, W, K, c2w, inverse_y, flip_x, flip_y, mode='center'): + i, j = torch.meshgrid( + torch.linspace(0, W - 1, W, device=c2w.device), + torch.linspace( + 0, H - 1, H, + device=c2w.device)) # pytorch's meshgrid has indexing='ij' + i = i.t().float() + j = j.t().float() + if mode == 'lefttop': + pass + elif mode == 'center': + i, j = i + 0.5, j + 0.5 + elif mode == 'random': + i = i + torch.rand_like(i) + j = j + torch.rand_like(j) + else: + raise NotImplementedError + + if flip_x: + i = i.flip((1, )) + if flip_y: + j = j.flip((0, )) + if inverse_y: + dirs = torch.stack([(i - K[0][2]) / K[0][0], (j - K[1][2]) / K[1][1], + torch.ones_like(i)], -1) + else: + dirs = torch.stack([(i - K[0][2]) / K[0][0], -(j - K[1][2]) / K[1][1], + -torch.ones_like(i)], -1) + # Rotate ray directions from camera frame to the world frame + rays_d = torch.sum( + dirs[..., np.newaxis, :] * c2w[:3, :3], + -1) # dot product, equals to: [c2w.dot(dir) for dir in dirs] + # Translate camera frame's origin to the world frame. It is the origin of all rays. + rays_o = c2w[:3, 3].expand(rays_d.shape) + return rays_o, rays_d + + +def get_rays_np(H, W, K, c2w): + i, j = np.meshgrid( + np.arange(W, dtype=np.float32), + np.arange(H, dtype=np.float32), + indexing='xy') + dirs = np.stack( + [(i - K[0][2]) / K[0][0], -(j - K[1][2]) / K[1][1], -np.ones_like(i)], + -1) + # Rotate ray directions from camera frame to the world frame + rays_d = np.sum( + dirs[..., np.newaxis, :] * c2w[:3, :3], + -1) # dot product, equals to: [c2w.dot(dir) for dir in dirs] + # Translate camera frame's origin to the world frame. It is the origin of all rays. + rays_o = np.broadcast_to(c2w[:3, 3], np.shape(rays_d)) + return rays_o, rays_d + + +def ndc_rays(H, W, focal, near, rays_o, rays_d): + # Shift ray origins to near plane + t = -(near + rays_o[..., 2]) / rays_d[..., 2] + rays_o = rays_o + t[..., None] * rays_d + + # Projection + o0 = -1. / (W / (2. * focal)) * rays_o[..., 0] / rays_o[..., 2] + o1 = -1. / (H / (2. * focal)) * rays_o[..., 1] / rays_o[..., 2] + o2 = 1. + 2. * near / rays_o[..., 2] + + d0 = -1. / (W / (2. * focal)) * ( + rays_d[..., 0] / rays_d[..., 2] - rays_o[..., 0] / rays_o[..., 2]) + d1 = -1. / (H / (2. * focal)) * ( + rays_d[..., 1] / rays_d[..., 2] - rays_o[..., 1] / rays_o[..., 2]) + d2 = -2. * near / rays_o[..., 2] + + rays_o = torch.stack([o0, o1, o2], -1) + rays_d = torch.stack([d0, d1, d2], -1) + + return rays_o, rays_d + + +def get_rays_of_a_view(H, + W, + K, + c2w, + ndc, + inverse_y, + flip_x, + flip_y, + mode='center'): + rays_o, rays_d = get_rays( + H, + W, + K, + c2w, + inverse_y=inverse_y, + flip_x=flip_x, + flip_y=flip_y, + mode=mode) + viewdirs = rays_d / rays_d.norm(dim=-1, keepdim=True) + if ndc: + rays_o, rays_d = ndc_rays(H, W, K[0][0], 1., rays_o, rays_d) + return rays_o, rays_d, viewdirs + + +@torch.no_grad() +def get_training_rays(rgb_tr, train_poses, HW, Ks, ndc, inverse_y, flip_x, + flip_y): + print('get_training_rays: start') + assert len(np.unique(HW, axis=0)) == 1 + assert len(np.unique(Ks.reshape(len(Ks), -1), axis=0)) == 1 + assert len(rgb_tr) == len(train_poses) and len(rgb_tr) == len(Ks) and len( + rgb_tr) == len(HW) + H, W = HW[0] + K = Ks[0] + eps_time = time.time() + rays_o_tr = torch.zeros([len(rgb_tr), H, W, 3], device=rgb_tr.device) + rays_d_tr = torch.zeros([len(rgb_tr), H, W, 3], device=rgb_tr.device) + viewdirs_tr = torch.zeros([len(rgb_tr), H, W, 3], device=rgb_tr.device) + imsz = [1] * len(rgb_tr) + for i, c2w in enumerate(train_poses): + rays_o, rays_d, viewdirs = get_rays_of_a_view( + H=H, + W=W, + K=K, + c2w=c2w, + ndc=ndc, + inverse_y=inverse_y, + flip_x=flip_x, + flip_y=flip_y) + rays_o_tr[i].copy_(rays_o.to(rgb_tr.device)) + rays_d_tr[i].copy_(rays_d.to(rgb_tr.device)) + viewdirs_tr[i].copy_(viewdirs.to(rgb_tr.device)) + del rays_o, rays_d, viewdirs + eps_time = time.time() - eps_time + print('get_training_rays: finish (eps time:', eps_time, 'sec)') + return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz + + +@torch.no_grad() +def get_training_rays_flatten(rgb_tr_ori, train_poses, HW, Ks, ndc, inverse_y, + flip_x, flip_y): + print('get_training_rays_flatten: start') + assert len(rgb_tr_ori) == len(train_poses) and len(rgb_tr_ori) == len( + Ks) and len(rgb_tr_ori) == len(HW) + eps_time = time.time() + DEVICE = rgb_tr_ori[0].device + N = sum(im.shape[0] * im.shape[1] for im in rgb_tr_ori) + rgb_tr = torch.zeros([N, 3], device=DEVICE) + rays_o_tr = torch.zeros_like(rgb_tr) + rays_d_tr = torch.zeros_like(rgb_tr) + viewdirs_tr = torch.zeros_like(rgb_tr) + imsz = [] + top = 0 + for c2w, img, (H, W), K in zip(train_poses, rgb_tr_ori, HW, Ks): + assert img.shape[:2] == (H, W) + rays_o, rays_d, viewdirs = get_rays_of_a_view( + H=H, + W=W, + K=K, + c2w=c2w, + ndc=ndc, + inverse_y=inverse_y, + flip_x=flip_x, + flip_y=flip_y) + n = H * W + rgb_tr[top:top + n].copy_(img.flatten(0, 1)) + rays_o_tr[top:top + n].copy_(rays_o.flatten(0, 1).to(DEVICE)) + rays_d_tr[top:top + n].copy_(rays_d.flatten(0, 1).to(DEVICE)) + viewdirs_tr[top:top + n].copy_(viewdirs.flatten(0, 1).to(DEVICE)) + imsz.append(n) + top += n + + assert top == N + eps_time = time.time() - eps_time + print('get_training_rays_flatten: finish (eps time:', eps_time, 'sec)') + return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz + + +@torch.no_grad() +def get_training_rays_in_maskcache_sampling(rgb_tr_ori, train_poses, HW, Ks, + ndc, inverse_y, flip_x, flip_y, + model, render_kwargs): + print('get_training_rays_in_maskcache_sampling: start') + assert len(rgb_tr_ori) == len(train_poses) and len(rgb_tr_ori) == len( + Ks) and len(rgb_tr_ori) == len(HW) + CHUNK = 64 + DEVICE = rgb_tr_ori[0].device + eps_time = time.time() + N = sum(im.shape[0] * im.shape[1] for im in rgb_tr_ori) + rgb_tr = torch.zeros([N, 3], device=DEVICE) + rays_o_tr = torch.zeros_like(rgb_tr) + rays_d_tr = torch.zeros_like(rgb_tr) + viewdirs_tr = torch.zeros_like(rgb_tr) + imsz = [] + top = 0 + for c2w, img, (H, W), K in zip(train_poses, rgb_tr_ori, HW, Ks): + assert img.shape[:2] == (H, W) + rays_o, rays_d, viewdirs = get_rays_of_a_view( + H=H, + W=W, + K=K, + c2w=c2w, + ndc=ndc, + inverse_y=inverse_y, + flip_x=flip_x, + flip_y=flip_y) + mask = torch.empty(img.shape[:2], device=DEVICE, dtype=torch.bool) + for i in range(0, img.shape[0], CHUNK): + mask[i:i + CHUNK] = model.hit_coarse_geo( + rays_o=rays_o[i:i + CHUNK], + rays_d=rays_d[i:i + CHUNK], + **render_kwargs).to(DEVICE) + n = mask.sum() + rgb_tr[top:top + n].copy_(img[mask]) + rays_o_tr[top:top + n].copy_(rays_o[mask].to(DEVICE)) + rays_d_tr[top:top + n].copy_(rays_d[mask].to(DEVICE)) + viewdirs_tr[top:top + n].copy_(viewdirs[mask].to(DEVICE)) + imsz.append(n) + top += n + + print('get_training_rays_in_maskcache_sampling: ratio', top / N) + rgb_tr = rgb_tr[:top] + rays_o_tr = rays_o_tr[:top] + rays_d_tr = rays_d_tr[:top] + viewdirs_tr = viewdirs_tr[:top] + eps_time = time.time() - eps_time + print('get_training_rays_in_maskcache_sampling: finish (eps time:', + eps_time, 'sec)') + return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz + + +@torch.no_grad() +def get_training_rays_in_maskcache_sampling_sr(rgb_tr_ori, train_poses, HW, Ks, + ndc, inverse_y, flip_x, flip_y, + model, render_kwargs, cfgs): + print('get_training_rays_in_maskcache_sampling: start') + assert len(rgb_tr_ori) == len(train_poses) and len(rgb_tr_ori) == len( + Ks) and len(rgb_tr_ori) == len(HW) + CHUNK = 64 + DEVICE = rgb_tr_ori[0].device + eps_time = time.time() + N = sum(im.shape[0] * im.shape[1] for im in rgb_tr_ori) + H, W = HW[0] + # rgb_tr = torch.zeros([N,3], device=DEVICE) + rgb_tr = torch.zeros([len(rgb_tr_ori), H, W, 3], device=rgb_tr_ori.device) + rays_o_tr = torch.zeros([len(rgb_tr_ori), H, W, 3], device=rgb_tr.device) + rays_d_tr = torch.zeros([len(rgb_tr_ori), H, W, 3], device=rgb_tr.device) + viewdirs_tr = torch.zeros([len(rgb_tr_ori), H, W, 3], device=rgb_tr.device) + + def mask_patch_generator(arr_all, index_all): + # arr_all_sr = arr_all + list_bp = list(range(len(arr_all))) + num_total = len(list_bp) + idx_im, top = torch.LongTensor(np.random.permutation(list_bp)), 0 + + while True: + if top >= num_total: + idx_im, top = torch.LongTensor( + np.random.permutation(list_bp)), 0 + bp_chioce = idx_im[top] + image_chioce = index_all[bp_chioce] + patch_chioce = arr_all[bp_chioce] + patch_4x_chioce = patch_chioce + # patch_chioce = arr_all[patch_ind] + # patch_4x_chioce = arr_all_sr[patch_ind] + top += 1 + pr, pc = patch_chioce.shape[0], patch_chioce.shape[1] + patch_chioce = patch_chioce.reshape(-1, 2) + patch_4x_chioce = patch_4x_chioce.reshape(-1, 2) + patch_chioce = np.moveaxis(patch_chioce, -1, 0) + patch_4x_chioce = np.moveaxis(patch_4x_chioce, -1, 0) + yield image_chioce, list(patch_chioce[0]), list( + patch_chioce[1]), list(patch_4x_chioce[0]), list( + patch_4x_chioce[1]), [pr, pc] + + imsz = [] + top = 0 + idx_b = 0 + patch_all = [] + index_all = [] + for c2w, img, (H, W), K in zip(train_poses, rgb_tr_ori, HW, Ks): + assert img.shape[:2] == (H, W) + rays_o, rays_d, viewdirs = get_rays_of_a_view( + H=H, + W=W, + K=K, + c2w=c2w, + ndc=ndc, + inverse_y=inverse_y, + flip_x=flip_x, + flip_y=flip_y) + mask = torch.empty(img.shape[:2], device=DEVICE, dtype=torch.bool) + for i in range(0, img.shape[0], CHUNK): + mask[i:i + CHUNK] = model.hit_coarse_geo( + rays_o=rays_o[i:i + CHUNK], + rays_d=rays_d[i:i + CHUNK], + **render_kwargs).to(DEVICE) + n = mask.sum() + arr_all = patch_gen(imsz=[H, W], num_im=100, BS=4096, sz_patch=CHUNK) + masks_arr = [mask[arr[:, :, 0], arr[:, :, 1]] for arr in arr_all] + for idx, patch_mask in enumerate(masks_arr): + if patch_mask.sum() > 2048: + index_all.append(idx_b) + patch_all.append(arr_all[idx]) + + rgb_tr[idx_b].copy_(img) + rays_o_tr[idx_b].copy_(rays_o.to(DEVICE)) + rays_d_tr[idx_b].copy_(rays_d.to(DEVICE)) + viewdirs_tr[idx_b].copy_(viewdirs.to(DEVICE)) + imsz.append(n) + top += n + idx_b += 1 + print(f'patches of image {idx_b} has generated.') + + patch_all = np.stack(patch_all, axis=0) + index_all = np.stack(index_all, axis=0) + print('get_training_rays_in_maskcache_sampling: ratio', top / N) + patch_generator = mask_patch_generator(patch_all, index_all) + eps_time = time.time() - eps_time + print('get_training_rays_in_maskcache_sampling: finish (eps time:', + eps_time, 'sec)') + return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz, patch_generator + + +def batch_indices_generator(N, BS): + # torch.randperm on cuda produce incorrect results in my machine + idx, top = torch.LongTensor(np.random.permutation(N)), 0 + while True: + if top + BS > N: + idx, top = torch.LongTensor(np.random.permutation(N)), 0 + yield idx[top:top + BS] + top += BS + + +def batch_images_generator(N, imsz, BS): + idx, top = range(imsz), 0 + n_im = 0 + while True: + if top + BS >= imsz: + yield idx[top:imsz], n_im, True + idx, top = range(imsz), 0 + n_im += 1 + if n_im >= N: + n_im = 0 + else: + yield idx[top:top + BS], n_im, False + top += BS + + +def simg_patch_indices_generator(imsz, BS): + BS = BS // 64 + H, W = imsz[0], imsz[1] + num_x, num_y = H // BS, W // BS + x = np.linspace(0, W - 1, W) + y = np.linspace(0, H - 1, H) + xx, yy = np.meshgrid(x, y) + arr_index = np.stack((yy, xx), axis=-1).astype(np.int64) + + slice_x = np.linspace(1, num_x, num_x).astype(np.int64) * BS + slcie_y = np.linspace(1, num_y, num_y).astype(np.int64) * BS + arr_yp = np.split(arr_index, slice_x, axis=0) + arr_yp_last = arr_yp.pop(-1) + arr_yp = np.stack(arr_yp, axis=-1) + arr_xyp = np.split(arr_yp, slcie_y, axis=1) + arr_yp_last = np.split(arr_yp_last, slcie_y, axis=1) + arr_xp_last = arr_xyp.pop(-1) + arr_xp_last = list(np.moveaxis(arr_xp_last, -1, 0)) + arr_xyp = np.concatenate(arr_xyp, axis=-1) + arr_xyp = list(np.moveaxis(arr_xyp, -1, 0)) + + arr_all = [] + arr_all.extend(arr_xyp) + arr_all.extend(arr_xp_last) + arr_all.extend(arr_yp_last) + num_p = len(arr_all) + idx, top = torch.LongTensor(np.random.permutation(num_p)), 0 + while True: + if top >= num_p: + idx, top = torch.LongTensor(np.random.permutation(num_p)), 0 + patch_chioce = arr_all[idx[top]] + top += 1 + patch_chioce = patch_chioce.reshape(-1, 2) + yield list(np.moveaxis(patch_chioce, -1, 0)) + + +def patch_gen(imsz, num_im, BS, sz_patch): + BS = BS // sz_patch + H, W = imsz[0], imsz[1] + num_x, num_y = H // BS, W // BS + x = np.linspace(0, W - 1, W) + y = np.linspace(0, H - 1, H) + xx, yy = np.meshgrid(x, y) + arr_index = np.stack((yy, xx), axis=-1).astype(np.int64) + + slice_x = np.linspace(1, num_x, num_x).astype(np.int64) * BS + slcie_y = np.linspace(1, num_y, num_y).astype(np.int64) * BS + arr_yp = np.split(arr_index, slice_x, axis=0) + arr_yp_last = arr_yp.pop(-1) + arr_yp = np.stack(arr_yp, axis=-1) + arr_xyp = np.split(arr_yp, slcie_y, axis=1) + arr_yp_last = np.split(arr_yp_last, slcie_y, axis=1) + arr_xp_last = arr_xyp.pop(-1) + arr_xp_last = list(np.moveaxis(arr_xp_last, -1, 0)) + arr_xyp = np.concatenate(arr_xyp, axis=-1) + arr_xyp = list(np.moveaxis(arr_xyp, -1, 0)) + + arr_all = [] + arr_all.extend(arr_xyp) + arr_all.extend(arr_xp_last) + arr_all.extend(arr_yp_last) + + return arr_all + + +def mimg_patch_indices_generator(imsz, num_im, BS, sz_patch, sr_ratio): + + arr_all = patch_gen(imsz, num_im, BS, sz_patch) + arr_all_sr = patch_gen(imsz * sr_ratio, num_im, BS * sr_ratio, sz_patch) + + num_p = len(arr_all) + list_p = np.ones(num_p) + list_b = [list_p * i for i in range(num_im)] + list_b = np.concatenate(list_b, axis=0) + list_p = np.array(range(num_p)) + list_p = np.tile(list_p, num_im) + list_bp = np.stack((list_b, list_p), axis=1) + num_total = num_p * num_im + idx_im, top = torch.LongTensor(np.random.permutation(list_bp)), 0 + + while True: + if top >= num_total: + idx_im, top = torch.LongTensor(np.random.permutation(list_bp)), 0 + bp_chioce = idx_im[top] + image_chioce, patch_ind = bp_chioce[0], bp_chioce[1] + patch_chioce = arr_all[patch_ind] + patch_4x_chioce = arr_all_sr[patch_ind] + top += 1 + pr, pc = patch_chioce.shape[0], patch_chioce.shape[1] + patch_chioce = patch_chioce.reshape(-1, 2) + patch_4x_chioce = patch_4x_chioce.reshape(-1, 2) + patch_chioce = np.moveaxis(patch_chioce, -1, 0) + patch_4x_chioce = np.moveaxis(patch_4x_chioce, -1, 0) + yield image_chioce, list(patch_chioce[0]), list(patch_chioce[1]), list( + patch_4x_chioce[0]), list(patch_4x_chioce[1]), [pr, pc] + + +def make_layer(basic_block, num_basic_block, **kwarg): + """Make layers by stacking the same blocks. + + Args: + basic_block (nn.module): nn.module class for basic block. + num_basic_block (int): number of blocks. + + Returns: + nn.Sequential: Stacked blocks in nn.Sequential. + """ + layers = [] + for _ in range(num_basic_block): + layers.append(basic_block(**kwarg)) + return nn.Sequential(*layers) + + +class SFTLayer(nn.Module): + + def __init__(self, num_feat=64, num_grow_ch=32): + super(SFTLayer, self).__init__() + self.SFT_scale_conv0 = nn.Conv2d(num_grow_ch, num_grow_ch, 1) + self.SFT_scale_conv1 = nn.Conv2d(num_grow_ch, num_feat, 1) + self.SFT_shift_conv0 = nn.Conv2d(num_grow_ch, num_grow_ch, 1) + self.SFT_shift_conv1 = nn.Conv2d(num_grow_ch, num_feat, 1) + + def forward(self, x, cond): + scale = self.SFT_scale_conv1( + F.leaky_relu(self.SFT_scale_conv0(cond), 0.2, inplace=True)) + shift = self.SFT_shift_conv1( + F.leaky_relu(self.SFT_shift_conv0(cond), 0.2, inplace=True)) + return x * (scale + 1) + shift + + +class ResidualDenseBlock_SFT(nn.Module): + """Residual Dense Block. + Used in RRDB block in ESRGAN. + Args: + num_feat (int): Channel number of intermediate features. + num_grow_ch (int): Channels for each growth. + """ + + def __init__(self, num_feat=64, num_grow_ch=32): + super(ResidualDenseBlock_SFT, self).__init__() + self.conv1 = nn.Conv2d(num_feat, num_grow_ch, 3, 1, 1) + self.conv2 = nn.Conv2d(num_feat + num_grow_ch, num_grow_ch, 3, 1, 1) + self.conv3 = nn.Conv2d(num_feat + 2 * num_grow_ch, num_grow_ch, 3, 1, + 1) + self.conv4 = nn.Conv2d(num_feat + 3 * num_grow_ch, num_grow_ch, 3, 1, + 1) + self.conv5 = nn.Conv2d(num_feat + 4 * num_grow_ch, num_feat, 3, 1, 1) + self.sft0 = SFTLayer(num_feat, num_grow_ch) + self.sft1 = SFTLayer(num_grow_ch, num_grow_ch) + + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def forward(self, x): + xc0 = self.sft0(x[0], x[1]) + x1 = self.lrelu(self.conv1(xc0)) + x2 = self.lrelu(self.conv2(torch.cat((xc0, x1), 1))) + x3 = self.lrelu(self.conv3(torch.cat((xc0, x1, x2), 1))) + x4 = self.lrelu(self.conv4(torch.cat((xc0, x1, x2, x3), 1))) + xc1 = self.sft1(x4, x[1]) + x5 = self.conv5(torch.cat((xc0, x1, x2, x3, xc1), 1)) + # Emperically, we use 0.2 to scale the residual for better performance + return (x5 * 0.2 + x[0], x[1]) + + +class RRDB_SFT(nn.Module): + """Residual in Residual Dense Block. + Used in RRDB-Net in ESRGAN. + Args: + num_feat (int): Channel number of intermediate features. + num_grow_ch (int): Channels for each growth. + """ + + def __init__(self, num_feat, num_grow_ch=32): + super(RRDB_SFT, self).__init__() + self.rdb1 = ResidualDenseBlock_SFT(num_feat, num_grow_ch) + self.rdb2 = ResidualDenseBlock_SFT(num_feat, num_grow_ch) + self.rdb3 = ResidualDenseBlock_SFT(num_feat, num_grow_ch) + self.sft0 = SFTLayer(num_feat, num_grow_ch) + + def forward(self, x): + out = self.rdb1(x) + out = self.rdb2(out) + out = self.rdb3(out) + out = self.sft0(out[0], x[1]) + # Emperically, we use 0.2 to scale the residual for better performance + return (out * 0.2 + x[0], x[1]) + + +class SFTNet(nn.Module): + """ + Args: + num_in_ch (int): Channel number of inputs. + num_out_ch (int): Channel number of outputs. + num_feat (int): Channel number of intermediate features. + Default: 64 + num_block (int): Block number in the trunk network. Defaults: 23 + num_grow_ch (int): Channels for each growth. Default: 32. + """ + + def __init__(self, + n_in_colors, + scale, + num_feat=64, + num_block=5, + num_grow_ch=32, + num_cond=1, + dswise=False): + super(SFTNet, self).__init__() + self.scale = scale + self.dswise = dswise + # if scale == 2: + # num_in_ch = num_in_ch * 4 + # elif scale == 1: + # num_in_ch = num_in_ch * 16 + if dswise: + self.conv_first = nn.Conv2d(n_in_colors, num_feat, 1) + else: + self.conv_first = nn.Conv2d(n_in_colors, num_feat, 3, 1, 1) + self.body = make_layer( + RRDB_SFT, num_block, num_feat=num_feat, num_grow_ch=num_grow_ch) + self.conv_body = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + if n_in_colors > 3: + self.conv_fea = nn.Conv2d(n_in_colors, num_feat, 3, 1, 1) + self.conv_prefea = nn.Conv2d(2 * num_feat, num_feat, 3, 1, 1) + + # upsample + if self.scale > 1: + self.conv_up1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + if self.scale == 4: + self.conv_up2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_hr = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_last = nn.Conv2d(num_feat, 3, 3, 1, 1) + + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + self.sftbody = SFTLayer(num_feat, num_grow_ch) + self.CondNet = nn.Sequential( + nn.Conv2d(num_cond, 64, 3, 1, 1), nn.LeakyReLU(0.2, True), + nn.Conv2d(64, 64, 1), nn.LeakyReLU(0.2, + True), nn.Conv2d(64, 64, 1), + nn.LeakyReLU(0.2, True), nn.Conv2d(64, 32, 1)) + + def forward(self, x, cond, fea=None): + if fea is None: + feat = self.conv_first(x) + else: + feat_rgb = self.conv_first(x) + # feat += torch.sigmoid(self.conv_fea(fea)) + feat = torch.cat((feat_rgb, fea), dim=1) + feat = self.conv_prefea(feat) + cond = self.CondNet(cond) + body_feat = self.body((feat, cond)) + body_feat = self.sftbody(body_feat[0], body_feat[1]) + body_feat = self.conv_body(body_feat) + body_feat += feat + # upsample + if self.scale > 1: + body_feat = self.lrelu( + self.conv_up1( + F.interpolate(body_feat, scale_factor=2, mode='nearest'))) + if self.scale == 4: + body_feat = self.lrelu( + self.conv_up2( + F.interpolate( + body_feat, scale_factor=2, mode='nearest'))) + out = self.conv_last(self.lrelu(self.conv_hr(body_feat))) + return out + + def tile_process(self, img, cond, tile_size, tile_pad=10): + """Modified from: https://github.com/ata4/esrgan-launcher + """ + batch, channel, height, width = img.shape + output_height = height * self.scale + output_width = width * self.scale + output_shape = (batch, channel, output_height, output_width) + cond = cond.unsqueeze(0) + + # start with black image + output = img.new_zeros(output_shape).to('cpu') + tiles_x = math.ceil(width / tile_size) + tiles_y = math.ceil(height / tile_size) + + # loop over all tiles + for y in range(tiles_y): + for x in range(tiles_x): + # extract tile from input image + ofs_x = x * tile_size + ofs_y = y * tile_size + # input tile area on total image + input_start_x = ofs_x + input_end_x = min(ofs_x + tile_size, width) + input_start_y = ofs_y + input_end_y = min(ofs_y + tile_size, height) + + # input tile area on total image with padding + input_start_x_pad = max(input_start_x - tile_pad, 0) + input_end_x_pad = min(input_end_x + tile_pad, width) + input_start_y_pad = max(input_start_y - tile_pad, 0) + input_end_y_pad = min(input_end_y + tile_pad, height) + + # input tile dimensions + input_tile_width = input_end_x - input_start_x + input_tile_height = input_end_y - input_start_y + tile_idx = y * tiles_x + x + 1 + input_tile = img[:, :, input_start_y_pad:input_end_y_pad, + input_start_x_pad:input_end_x_pad] + cond_tile = cond[:, :, input_start_y_pad:input_end_y_pad, + input_start_x_pad:input_end_x_pad] + # upscale tile + # try: + with torch.no_grad(): + out_t = self(input_tile, cond_tile) + # except Exception as error: + # print('Error', error) + print(f'\tTile {tile_idx}/{tiles_x * tiles_y}') + + # output tile area on total image + out_s_x = input_start_x * self.scale + out_e_x = input_end_x * self.scale + out_s_y = input_start_y * self.scale + out_e_y = input_end_y * self.scale + + # output tile area without padding + out_s_x_t = (input_start_x - input_start_x_pad) * self.scale + out_e_x_t = out_s_x_t + input_tile_width * self.scale + out_s_y_t = (input_start_y - input_start_y_pad) * self.scale + out_e_y_t = out_s_y_t + input_tile_height * self.scale + + # put tile into output image + output[:, :, out_s_y:out_e_y, + out_s_x:out_e_x] = out_t[:, :, out_s_y_t:out_e_y_t, + out_s_x_t:out_e_x_t] + return output.detach().to('cpu') + + def load_network(self, + load_path, + device, + strict=True, + param_key='params_ema'): + """Load network. + + Args: + load_path (str): The path of networks to be loaded. + net (nn.Module): Network. + strict (bool): Whether strictly loaded. + param_key (str): The parameter key of loaded network. If set to + None, use the root 'path'. + Default: 'params'. + """ + # net = self.get_bare_model(net) + load_net = torch.load(load_path, map_location=device) + if param_key is not None: + if param_key not in load_net and 'params' in load_net: + param_key = 'params' + print('Loading: params_ema does not exist, use params.') + load_net = load_net[param_key] + print( + f'Loading {self.__class__.__name__} model from {load_path}, with param key: [{param_key}].' + ) + # remove unnecessary 'module.' + for k, v in deepcopy(load_net).items(): + if k.startswith('module.'): + load_net[k[7:]] = v + load_net.pop(k) + self._print_different_keys_loading(load_net, strict) + self.load_state_dict(load_net, strict=strict) + + def _print_different_keys_loading(self, load_net, strict=True): + """Print keys with differnet name or different size when loading models. + + 1. Print keys with differnet names. + 2. If strict=False, print the same key but with different tensor size. + It also ignore these keys with different sizes (not load). + + Args: + crt_net (torch model): Current network. + load_net (dict): Loaded network. + strict (bool): Whether strictly loaded. Default: True. + """ + crt_net = self.state_dict() + crt_net_keys = set(crt_net.keys()) + load_net_keys = set(load_net.keys()) + + if crt_net_keys != load_net_keys: + print('Current net - loaded net:') + for v in sorted(list(crt_net_keys - load_net_keys)): + print(f' {v}') + print('Loaded net - current net:') + for v in sorted(list(load_net_keys - crt_net_keys)): + print(f' {v}') + + # check the size for the same keys + if not strict: + common_keys = crt_net_keys & load_net_keys + for k in common_keys: + if crt_net[k].size() != load_net[k].size(): + print(f'Size different, ignore [{k}]: crt_net: ' + f'{crt_net[k].shape}; load_net: {load_net[k].shape}') + load_net[k + '.ignore'] = load_net.pop(k) + + def save_network(self, + save_root, + net_label, + current_iter, + param_key='params'): + + if current_iter == -1: + current_iter = 'latest' + save_filename = f'{net_label}_{current_iter}.pth' + save_path = os.path.join(save_root, save_filename) + + net = self if isinstance(self, list) else [self] + param_key = param_key if isinstance(param_key, list) else [param_key] + assert len(net) == len( + param_key), 'The lengths of net and param_key should be the same.' + + save_dict = {} + for net_, param_key_ in zip(net, param_key): + state_dict = net_.state_dict() + for key, param in state_dict.items(): + if key.startswith('module.'): # remove unnecessary 'module.' + key = key[7:] + state_dict[key] = param.cpu() + save_dict[param_key_] = state_dict + + # avoid occasional writing errors + retry = 3 + while retry > 0: + try: + torch.save(save_dict, save_path) + except Exception as e: + print( + f'Save model error: {e}, remaining retry times: {retry - 1}' + ) + time.sleep(1) + else: + break + finally: + retry -= 1 + if retry == 0: + print(f'Still cannot save {save_path}. Just ignore it.') diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/utils.py b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aa8c3d6675748011a44c784fc2421e5c00555774 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_4k/network/utils.py @@ -0,0 +1,176 @@ +# The implementation is partly adopted from nerfacc, made publicly available under the MIT License +# at https://github.com/KAIR-BAIR/nerfacc/blob/master/examples/radiance_fields/ngp.py +import gc +from collections import defaultdict + +import mcubes +import numpy as np +import tinycudann as tcnn +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.cuda.amp import custom_bwd, custom_fwd + + +class PSNR(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, inputs, targets, valid_mask=None, reduction='mean'): + assert reduction in ['mean', 'none'] + value = (inputs - targets)**2 + if valid_mask is not None: + value = value[valid_mask] + if reduction == 'mean': + return -10 * torch.log10(torch.mean(value)) + elif reduction == 'none': + return -10 * torch.log10( + torch.mean(value, dim=tuple(range(value.ndim)[1:]))) + + +def extract_fields(bound_min, bound_max, resolution, query_func): + N = 64 + X = torch.linspace(bound_min[0], bound_max[0], resolution).split(N) + Y = torch.linspace(bound_min[1], bound_max[1], resolution).split(N) + Z = torch.linspace(bound_min[2], bound_max[2], resolution).split(N) + + u = np.zeros([resolution, resolution, resolution], dtype=np.float32) + with torch.no_grad(): + for xi, xs in enumerate(X): + for yi, ys in enumerate(Y): + for zi, zs in enumerate(Z): + xx, yy, zz = torch.meshgrid(xs, ys, zs) + xx = xx.reshape(-1, 1) + yy = yy.reshape(-1, 1) + zz = zz.reshape(-1, 1) + pts = torch.cat([xx, yy, zz], dim=-1).cuda() + val = query_func(pts).reshape( + len(xs), len(ys), len(zs)).detach().cpu().numpy() + u[xi * N:xi * N + len(xs), yi * N:yi * N + len(ys), + zi * N:zi * N + len(zs)] = val + return u + + +def extract_geometry(bound_min, bound_max, resolution, threshold, query_func): + u = extract_fields(bound_min, bound_max, resolution, query_func) + vertices, triangles = mcubes.marching_cubes(u, threshold) + b_max_np = bound_max.detach().cpu().numpy() + b_min_np = bound_min.detach().cpu().numpy() + + vertices = vertices / (resolution - 1.0) * ( + b_max_np - b_min_np)[None, :] + b_min_np[None, :] + return vertices, triangles + + +def chunk_batch(func, chunk_size, *args, **kwargs): + B = None + for arg in args: + if isinstance(arg, torch.Tensor): + B = arg.shape[0] + break + out = defaultdict(list) + out_type = None + for i in range(0, B, chunk_size): + out_chunk = func( + *[ + arg[i:i + chunk_size] if isinstance(arg, torch.Tensor) else arg + for arg in args + ], **kwargs) + if out_chunk is None: + continue + out_type = type(out_chunk) + if isinstance(out_chunk, torch.Tensor): + out_chunk = {0: out_chunk} + elif isinstance(out_chunk, tuple) or isinstance(out_chunk, list): + chunk_length = len(out_chunk) + out_chunk = {i: chunk for i, chunk in enumerate(out_chunk)} + elif isinstance(out_chunk, dict): + pass + else: + exit(1) + for k, v in out_chunk.items(): + out[k].append(v if torch.is_grad_enabled() else v.detach()) + + if out_type is None: + return + + out = {k: torch.cat(v, dim=0) for k, v in out.items()} + if out_type is torch.Tensor: + return out[0] + elif out_type in [tuple, list]: + return out_type([out[i] for i in range(chunk_length)]) + elif out_type is dict: + return out + + +def get_activation(name): + name = name.lower() + if name is None or name == 'none': + return nn.Identity() + elif name.startswith('scale'): + scale_factor = float(name[5:]) + return lambda x: x.clamp(0., scale_factor) / scale_factor + elif name.startswith('clamp'): + clamp_max = float(name[5:]) + return lambda x: x.clamp(0., clamp_max) + elif name.startswith('mul'): + mul_factor = float(name[3:]) + return lambda x: x * mul_factor + elif name == 'trunc_exp': + return trunc_exp + elif name.startswith('+') or name.startswith('-'): + return lambda x: x + float(name) + elif name.lower() == 'sigmoid': + return lambda x: torch.sigmoid(x) + elif name.lower() == 'tanh': + return lambda x: torch.tanh(x) + else: + return getattr(F, name) + + +class _TruncExp(Function): + # Implementation from torch-ngp: + # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, x): # pylint: disable=arguments-differ + ctx.save_for_backward(x) + return torch.exp(x) + + @staticmethod + @custom_bwd + def backward(ctx, g): + x = ctx.saved_tensors[0] + return g * torch.exp(torch.clamp(x, max=15)) + + +trunc_exp = _TruncExp.apply + + +def dot(x, y): + return torch.sum(x * y, -1, keepdim=True) + + +def reflect(x, n): + return 2 * dot(x, n) * n - x + + +def normalize(dat, inp_scale, tgt_scale): + if inp_scale is None: + inp_scale = [dat.min(), dat.max()] + dat = (dat - inp_scale[0]) / (inp_scale[1] - inp_scale[0]) + dat = dat * (tgt_scale[1] - tgt_scale[0]) + tgt_scale[0] + return dat + + +def cleanup(): + gc.collect() + torch.cuda.empty_cache() + tcnn.free_temporary_memory() + + +def update_module_step(m, epoch, global_step): + if hasattr(m, 'update_step'): + m.update_step(epoch, global_step) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95ff8357c2f1d61f72defee4d5b7d6388d192eec --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .nerf_recon_acc import NeRFReconAcc + from .nerf_preprocess import NeRFReconPreprocessor + +else: + _import_structure = {'nerf_recon_acc': ['NeRFReconAcc']} + _import_structure = {'nerf_preprocess': ['NeRFReconPreprocessor']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/nerf_dataset.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/nerf_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..8ff147747791388273fa5dae6c821b5d279f2c22 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/nerf_dataset.py @@ -0,0 +1,518 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os + +import json +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from PIL import Image +from torch.utils.data import Dataset + +from .read_write_model import (read_cameras_binary, read_images_binary, + read_points3D_binary) + + +def get_rays(directions, c2w, keepdim=False): + assert directions.shape[-1] == 3 + + if directions.ndim == 2: + assert c2w.ndim == 3 + rays_d = (directions[:, None, :] * c2w[:, :3, :3]).sum(-1) + rays_o = c2w[:, :, 3].expand(rays_d.shape) + elif directions.ndim == 3: + if c2w.ndim == 2: + rays_d = (directions[:, :, None, :] + * c2w[None, None, :3, :3]).sum(-1) + rays_o = c2w[None, None, :, 3].expand(rays_d.shape) + elif c2w.ndim == 3: + rays_d = (directions[None, :, :, None, :] + * c2w[:, None, None, :3, :3]).sum(-1) + rays_o = c2w[:, None, None, :, 3].expand(rays_d.shape) + + if not keepdim: + rays_o, rays_d = rays_o.reshape(-1, 3), rays_d.reshape(-1, 3) + + return rays_o, rays_d + + +def get_ray_directions(W, H, fx, fy, cx, cy, use_pixel_centers=True): + pixel_center = 0.5 if use_pixel_centers else 0 + i, j = np.meshgrid( + np.arange(W, dtype=np.float32) + pixel_center, + np.arange(H, dtype=np.float32) + pixel_center, + indexing='xy') + i, j = torch.from_numpy(i), torch.from_numpy(j) + + directions = torch.stack( + [(i - cx) / fx, -(j - cy) / fy, -torch.ones_like(i)], -1) # (H, W, 3) + + return directions + + +def get_center(pts): + center = pts.mean(0) + dis = (pts - center[None, :]).norm(p=2, dim=-1) + mean, std = dis.mean(), dis.std() + q25, q75 = torch.quantile(dis, 0.25), torch.quantile(dis, 0.75) + valid = (dis > mean - 1.5 * std) & (dis < mean + 1.5 * std) & ( + dis > mean - (q75 - q25) * 1.5) & ( + dis < mean + (q75 - q25) * 1.5) + pts = pts[valid] + center = pts.mean(0) + return center, pts + + +def normalize_poses(poses, pts): + center, pts = get_center(pts) + + z = F.normalize((poses[..., 3] - center).mean(0), dim=0) + y_ = torch.as_tensor([z[1], -z[0], 0.]) + x = F.normalize(y_.cross(z), dim=0) + y = z.cross(x) + + Rc = torch.stack([x, y, z], dim=1) + tc = center.reshape(3, 1) + + R, t = Rc.T, -Rc.T @ tc + + pose_last = torch.as_tensor([[[0., 0., 0., + 1.]]]).expand(poses.shape[0], -1, -1) + poses_homo = torch.cat([poses, pose_last], dim=1) + inv_trans = torch.cat( + [torch.cat([R, t], dim=1), + torch.as_tensor([[0., 0., 0., 1.]])], dim=0) + + poses_norm = (inv_trans @ poses_homo)[:, :3] # (N_images, 4, 4) + scale = poses_norm[..., 3].norm(p=2, dim=-1).min() + poses_norm[..., 3] /= scale + + pts = (inv_trans @ torch.cat([pts, torch.ones_like(pts[:, 0:1])], + dim=-1)[..., None])[:, :3, 0] + pts = pts / scale + + return poses_norm, pts + + +def create_spheric_poses(cameras, n_steps=120): + center = torch.as_tensor([0., 0., 0.], + dtype=cameras.dtype, + device=cameras.device) + mean_d = (cameras - center[None, :]).norm(p=2, dim=-1).mean() + mean_h = cameras[:, 2].mean() + r = (mean_d**2 - mean_h**2).sqrt() + up = torch.as_tensor([0., 0., 1.], + dtype=center.dtype, + device=center.device) + + all_c2w = [] + for theta in torch.linspace(0, 2 * math.pi, n_steps): + cam_pos = torch.stack([r * theta.cos(), r * theta.sin(), mean_h]) + h = F.normalize(center - cam_pos, p=2, dim=0) + s = F.normalize(h.cross(up), p=2, dim=0) + u = F.normalize(s.cross(h), p=2, dim=0) + concat = torch.stack([s, u, -h], dim=1) + c2w = torch.cat([concat, cam_pos[:, None]], axis=1) + all_c2w.append(c2w) + + all_c2w = torch.stack(all_c2w, dim=0) + + return all_c2w + + +def to4x4(pose): + constants = torch.zeros_like(pose[..., :1, :], device=pose.device) + constants[..., :, 3] = 1 + return torch.cat([pose, constants], dim=-2) + + +def get_spiral_path(cameras, + fx, + fy, + n_steps=120, + radius=0.1, + rots=2, + zrate=0.5): + up = cameras[0, :3, 2] + fx = torch.tensor(fx, dtype=torch.float32) + fy = torch.tensor(fy, dtype=torch.float32) + focal = torch.min(fx, fy) + target = torch.tensor( + [0, 0, -focal], + device=cameras.device) # camera looking in -z direction + rad = torch.tensor([radius] * 3, device=cameras.device) + c2w = cameras[0] + c2wh_global = to4x4(c2w) + + local_c2whs = [] + for theta in torch.linspace(0.0, 2.0 * torch.pi * rots, n_steps + 1)[:-1]: + theta_list = [ + torch.cos(theta), -torch.sin(theta), -torch.sin(theta * zrate) + ] + center = (torch.tensor(theta_list, device=cameras.device) * rad) + lookat = center - target + vec2 = F.normalize(lookat, p=2, dim=0) + vec1_avg = F.normalize(up, p=2, dim=0) + vec0 = F.normalize(torch.cross(vec1_avg, vec2), p=2, dim=0) + vec1 = F.normalize(torch.cross(vec2, vec0), p=2, dim=0) + c2w = torch.stack([vec0, vec1, vec2, center], 1) + c2wh = to4x4(c2w) + local_c2whs.append(c2wh) + + new_c2ws = [] + for local_c2wh in local_c2whs: + c2wh = torch.matmul(c2wh_global, local_c2wh) + new_c2ws.append(c2wh[:3, :4]) + new_c2ws = torch.stack(new_c2ws, dim=0) + return new_c2ws + + +class BlenderDataset(Dataset): + """Single subject data loader for training and evaluation.""" + + def __init__( + self, + root_fp, + split, + img_wh=(800, 800), + max_size=None, + num_rays=None, + color_bkgd_aug='white', + near=2.0, + far=6.0, + batch_over_images=True, + ): + super().__init__() + self.root_fp = root_fp + self.split = split + self.max_size = max_size + self.num_rays = num_rays + self.near = near + self.far = far + self.training = (num_rays is not None) and (split == 'train') + self.color_bkgd_aug = color_bkgd_aug + self.batch_over_images = batch_over_images + + with open( + os.path.join(self.root_fp, f'transforms_{self.split}.json'), + 'r') as f: + meta = json.load(f) + + if 'w' in meta and 'h' in meta: + W, H = int(meta['w']), int(meta['h']) + else: + W, H = img_wh + + self.w, self.h = W, H + self.image_wh = (self.w, self.h) + self.focal = 0.5 * self.w / math.tan(0.5 * meta['camera_angle_x']) + self.directions = get_ray_directions(self.w, self.h, self.focal, + self.focal, self.w // 2, + self.h // 2).cuda() + + self.all_c2w, self.all_images, self.all_fg_masks = [], [], [] + + for i, frame in enumerate(meta['frames']): + c2w = torch.from_numpy(np.array(frame['transform_matrix'])[:3, :4]) + self.all_c2w.append(c2w) + + img_path = os.path.join(self.root_fp, f"{frame['file_path']}.png") + img = Image.open(img_path) + img = TF.to_tensor(img).permute(1, 2, 0) # (4, h, w) => (h, w, 4) + + self.all_fg_masks.append(img[..., -1]) # (h, w) + self.all_images.append(img[..., :3]) + + self.all_c2w, self.all_images, self.all_fg_masks = \ + torch.stack(self.all_c2w, dim=0).float().cuda(), \ + torch.stack(self.all_images, dim=0).float().cuda(), \ + torch.stack(self.all_fg_masks, dim=0).float().cuda() + + def __len__(self): + return len(self.all_images) + + @torch.no_grad() + def __getitem__(self, index): + data = self.fetch_data(index) + return data + + def update_num_rays(self, num_rays): + self.num_rays = num_rays + + def fetch_data(self, index): + """Fetch the data (it maybe cached for multiple batches).""" + num_rays = self.num_rays + if self.training: + if self.batch_over_images: + index = torch.randint( + 0, + len(self.all_images), + size=(num_rays, ), + device=self.all_images.device) + + else: + index = torch.randint( + 0, + len(self.all_images), + size=(1, ), + device=self.all_images.device) + + x = torch.randint( + 0, self.w, size=(num_rays, ), device=self.all_images.device) + y = torch.randint( + 0, self.h, size=(num_rays, ), device=self.all_images.device) + c2w = self.all_c2w[index] + directions = self.directions[y, x] + rays_o, rays_d = get_rays(directions, c2w) + rgb = self.all_images[index, y, x].view(-1, + self.all_images.shape[-1]) + fg_mask = self.all_fg_masks[index, y, x].view(-1) + + else: + c2w = self.all_c2w[index] + directions = self.directions + rays_o, rays_d = get_rays(directions, c2w) + rgb = self.all_images[index].view(-1, self.all_images.shape[-1]) + fg_mask = self.all_fg_masks[index].view(-1) + + rays = torch.cat([rays_o, rays_d], dim=-1) + + if self.training: + if self.color_bkgd_aug == 'random': + color_bkgd = torch.rand(3, device=self.all_images.device) + elif self.color_bkgd_aug == 'white': + color_bkgd = torch.ones(3, device=self.all_images.device) + elif self.color_bkgd_aug == 'black': + color_bkgd = torch.zeros(3, device=self.all_images.device) + else: + # just use white during inference + color_bkgd = torch.ones(3, device=self.all_images.device) + + rgb = rgb * fg_mask[..., None] + color_bkgd * (1 - fg_mask[..., None]) + + return { + 'pixels': rgb, # [h*w, 4] or [num_rays, 4] + 'rays': rays, # [h*w, 6] or [num_rays, 6] + 'fg_mask': fg_mask, + 'image_wh': self.image_wh, + } + + +class ColmapDataset(Dataset): + """data loader for training and evaluation.""" + + def __init__( + self, + root_fp, + split, + img_wh, + max_size=1200, + num_rays=None, + use_mask=True, + color_bkgd_aug='random', + batch_over_images=True, + n_test_traj_steps=120, + ): + super().__init__() + if os.path.exists(os.path.join(root_fp, 'preprocess')): + self.root_fp = os.path.join(root_fp, 'preprocess') + self.distort = True + else: + self.root_fp = root_fp + self.distort = False + self.split = split + self.num_rays = num_rays + self.use_mask = use_mask + self.training = (num_rays is not None) and (split == 'train') + self.color_bkgd_aug = color_bkgd_aug + self.batch_over_images = batch_over_images + self.n_test_traj_steps = n_test_traj_steps + + if self.distort: + camdata = read_cameras_binary( + os.path.join(self.root_fp, 'sparse/cameras.bin')) + else: + camdata = read_cameras_binary( + os.path.join(self.root_fp, 'sparse/0/cameras.bin')) + H, W = int(camdata[1].height), int(camdata[1].width) + + if img_wh is not None: + w, h = img_wh + self.width, self.height = w, h + self.factor = w / W + else: + if H <= max_size and W <= max_size: + self.height = H + self.width = W + self.factor = 1 + else: + if H > W: + self.height = max_size + self.width = round(max_size * W / H) + self.factor = max_size / H + else: + self.width = max_size + self.height = round(max_size * H / W) + self.factor = max_size / W + self.image_wh = (self.width, self.height) + print('process image width and height: {}'.format(self.image_wh)) + + print(camdata[1].model) + if camdata[1].model == 'SIMPLE_RADIAL': + fx = fy = camdata[1].params[0] * self.factor + cx = camdata[1].params[1] * self.factor + cy = camdata[1].params[2] * self.factor + elif camdata[1].model in ['PINHOLE', 'OPENCV']: + fx = camdata[1].params[0] * self.factor + fy = camdata[1].params[1] * self.factor + cx = camdata[1].params[2] * self.factor + cy = camdata[1].params[3] * self.factor + else: + raise ValueError( + f'Please parse the intrinsics for camera model {camdata[1].model}!' + ) + + self.directions = get_ray_directions(self.width, self.height, fx, fy, + cx, cy).cuda() + if self.distort: + imdata = read_images_binary( + os.path.join(self.root_fp, 'sparse/images.bin')) + else: + imdata = read_images_binary( + os.path.join(self.root_fp, 'sparse/0/images.bin')) + + mask_dir = os.path.join(self.root_fp, 'masks') + self.use_mask = os.path.exists(mask_dir) and self.use_mask + + self.all_c2w, self.all_images, self.all_fg_masks = [], [], [] + for i, d in enumerate(imdata.values()): + R = d.qvec2rotmat() + t = d.tvec.reshape(3, 1) + c2w = torch.from_numpy(np.concatenate([R.T, -R.T @ t], + axis=1)).float() + c2w[:, 1:3] *= -1. + self.all_c2w.append(c2w) + if self.split in ['train', 'val']: + img_path = os.path.join(self.root_fp, 'images', d.name) + img = Image.open(img_path) + img = img.resize(self.image_wh, Image.BICUBIC) + img = TF.to_tensor(img).permute(1, 2, 0)[..., :3] + if self.use_mask: + mask_path = os.path.join(mask_dir, d.name) + mask = Image.open(mask_path).convert('L') + mask = mask.resize(self.image_wh, Image.BICUBIC) + mask = TF.to_tensor(mask)[0] + else: + mask = torch.ones_like(img[..., 0]) + self.all_fg_masks.append(mask) + self.all_images.append(img) + + self.all_c2w = torch.stack(self.all_c2w, dim=0) + + if self.distort: + pts3d = read_points3D_binary( + os.path.join(self.root_fp, 'sparse/points3D.bin')) + else: + pts3d = read_points3D_binary( + os.path.join(self.root_fp, 'sparse/0/points3D.bin')) + pts3d = torch.from_numpy(np.array([pts3d[k].xyz + for k in pts3d])).float() + + self.all_c2w, pts3d = normalize_poses(self.all_c2w, pts3d) + + if self.split == 'test': + # self.all_c2w = get_spiral_path( + # self.all_c2w, fx, fy, n_steps=self.n_test_traj_steps) + self.all_c2w = create_spheric_poses( + self.all_c2w[:, :, 3], n_steps=self.n_test_traj_steps) + self.all_images = torch.zeros( + (self.n_test_traj_steps, self.height, self.width, 3), + dtype=torch.float32) + self.all_fg_masks = torch.zeros( + (self.n_test_traj_steps, self.height, self.width), + dtype=torch.float32) + else: + self.all_images, self.all_fg_masks = torch.stack( + self.all_images, dim=0), torch.stack( + self.all_fg_masks, dim=0) + + self.all_c2w, self.all_images, self.all_fg_masks = \ + self.all_c2w.float().cuda(), \ + self.all_images.float().cuda(), \ + self.all_fg_masks.float().cuda() + + def __len__(self): + return len(self.all_images) + + @torch.no_grad() + def __getitem__(self, index): + data = self.fetch_data(index) + return data + + def update_num_rays(self, num_rays): + self.num_rays = num_rays + + def fetch_data(self, index): + """Fetch the data (it maybe cached for multiple batches).""" + num_rays = self.num_rays + if self.training: + if self.batch_over_images: + index = torch.randint( + 0, + len(self.all_images), + size=(num_rays, ), + device=self.all_images.device) + + else: + index = torch.randint( + 0, + len(self.all_images), + size=(1, ), + device=self.all_images.device) + + x = torch.randint( + 0, + self.width, + size=(num_rays, ), + device=self.all_images.device) + y = torch.randint( + 0, + self.height, + size=(num_rays, ), + device=self.all_images.device) + c2w = self.all_c2w[index] + directions = self.directions[y, x] + rays_o, rays_d = get_rays(directions, c2w) + rgb = self.all_images[index, y, x].view(-1, + self.all_images.shape[-1]) + fg_mask = self.all_fg_masks[index, y, x].view(-1) + + else: + c2w = self.all_c2w[index] + directions = self.directions + rays_o, rays_d = get_rays(directions, c2w) + rgb = self.all_images[index].view(-1, self.all_images.shape[-1]) + fg_mask = self.all_fg_masks[index].view(-1) + + rays = torch.cat([rays_o, F.normalize(rays_d, p=2, dim=-1)], dim=-1) + + if self.training: + if self.color_bkgd_aug == 'random': + color_bkgd = torch.rand(3, device=self.all_images.device) + elif self.color_bkgd_aug == 'white': + color_bkgd = torch.ones(3, device=self.all_images.device) + elif self.color_bkgd_aug == 'black': + color_bkgd = torch.zeros(3, device=self.all_images.device) + else: + # just use white during inference + color_bkgd = torch.ones(3, device=self.all_images.device) + + rgb = rgb * fg_mask[..., None] + color_bkgd * (1 - fg_mask[..., None]) + + return { + 'pixels': rgb, # [h*w, 4] or [num_rays, 4] + 'rays': rays, # [h*w, 6] or [num_rays, 6] + 'fg_mask': fg_mask, + 'image_wh': self.image_wh, + } diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/read_write_model.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/read_write_model.py new file mode 100644 index 0000000000000000000000000000000000000000..cec00a5284f0f1bca7c87436018f8450af0a49d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/dataloader/read_write_model.py @@ -0,0 +1,500 @@ +# Copyright (c) 2023, ETH Zurich and UNC Chapel Hill. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of +# its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) + +import argparse +import collections +import os +import struct + +import numpy as np + +CameraModel = collections.namedtuple('CameraModel', + ['model_id', 'model_name', 'num_params']) +Camera = collections.namedtuple('Camera', + ['id', 'model', 'width', 'height', 'params']) +BaseImage = collections.namedtuple( + 'Image', ['id', 'qvec', 'tvec', 'camera_id', 'name', 'xys', 'point3D_ids']) +Point3D = collections.namedtuple( + 'Point3D', ['id', 'xyz', 'rgb', 'error', 'image_ids', 'point2D_idxs']) + + +class Image(BaseImage): + + def qvec2rotmat(self): + return qvec2rotmat(self.qvec) + + +CAMERA_MODELS = { + CameraModel(model_id=0, model_name='SIMPLE_PINHOLE', num_params=3), + CameraModel(model_id=1, model_name='PINHOLE', num_params=4), + CameraModel(model_id=2, model_name='SIMPLE_RADIAL', num_params=4), + CameraModel(model_id=3, model_name='RADIAL', num_params=5), + CameraModel(model_id=4, model_name='OPENCV', num_params=8), + CameraModel(model_id=5, model_name='OPENCV_FISHEYE', num_params=8), + CameraModel(model_id=6, model_name='FULL_OPENCV', num_params=12), + CameraModel(model_id=7, model_name='FOV', num_params=5), + CameraModel(model_id=8, model_name='SIMPLE_RADIAL_FISHEYE', num_params=4), + CameraModel(model_id=9, model_name='RADIAL_FISHEYE', num_params=5), + CameraModel(model_id=10, model_name='THIN_PRISM_FISHEYE', num_params=12) +} +CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) + for camera_model in CAMERA_MODELS]) +CAMERA_MODEL_NAMES = dict([(camera_model.model_name, camera_model) + for camera_model in CAMERA_MODELS]) + + +def read_next_bytes(fid, + num_bytes, + format_char_sequence, + endian_character='<'): + """Read and unpack the next bytes from a binary file. + :param fid: + :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + :param endian_character: Any of {@, =, <, >, !} + :return: Tuple of read and unpacked values. + """ + data = fid.read(num_bytes) + return struct.unpack(endian_character + format_char_sequence, data) + + +def write_next_bytes(fid, data, format_char_sequence, endian_character='<'): + """pack and write to a binary file. + :param fid: + :param data: data to send, if multiple elements are sent at the same time, + they should be encapsuled either in a list or a tuple + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + should be the same length as the data list or tuple + :param endian_character: Any of {@, =, <, >, !} + """ + if isinstance(data, (list, tuple)): + bytes = struct.pack(endian_character + format_char_sequence, *data) + else: + bytes = struct.pack(endian_character + format_char_sequence, data) + fid.write(bytes) + + +def read_cameras_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + cameras = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + camera_id = int(elems[0]) + model = elems[1] + width = int(elems[2]) + height = int(elems[3]) + params = np.array(tuple(map(float, elems[4:]))) + cameras[camera_id] = Camera( + id=camera_id, + model=model, + width=width, + height=height, + params=params) + return cameras + + +def read_cameras_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasBinary(const std::string& path) + void Reconstruction::ReadCamerasBinary(const std::string& path) + """ + cameras = {} + with open(path_to_model_file, 'rb') as fid: + num_cameras = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_cameras): + camera_properties = read_next_bytes( + fid, num_bytes=24, format_char_sequence='iiQQ') + camera_id = camera_properties[0] + model_id = camera_properties[1] + model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name + width = camera_properties[2] + height = camera_properties[3] + num_params = CAMERA_MODEL_IDS[model_id].num_params + params = read_next_bytes( + fid, + num_bytes=8 * num_params, + format_char_sequence='d' * num_params) + cameras[camera_id] = Camera( + id=camera_id, + model=model_name, + width=width, + height=height, + params=np.array(params)) + assert len(cameras) == num_cameras + return cameras + + +def write_cameras_text(cameras, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + HEADER = '# Camera list with one line of data per camera:\n' + \ + '# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n' + \ + '# Number of cameras: {}\n'.format(len(cameras)) + with open(path, 'w') as fid: + fid.write(HEADER) + for _, cam in cameras.items(): + to_write = [cam.id, cam.model, cam.width, cam.height, *cam.params] + line = ' '.join([str(elem) for elem in to_write]) + fid.write(line + '\n') + + +def write_cameras_binary(cameras, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasBinary(const std::string& path) + void Reconstruction::ReadCamerasBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(cameras), 'Q') + for _, cam in cameras.items(): + model_id = CAMERA_MODEL_NAMES[cam.model].model_id + camera_properties = [cam.id, model_id, cam.width, cam.height] + write_next_bytes(fid, camera_properties, 'iiQQ') + for p in cam.params: + write_next_bytes(fid, float(p), 'd') + return cameras + + +def read_images_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesText(const std::string& path) + void Reconstruction::WriteImagesText(const std::string& path) + """ + images = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + image_id = int(elems[0]) + qvec = np.array(tuple(map(float, elems[1:5]))) + tvec = np.array(tuple(map(float, elems[5:8]))) + camera_id = int(elems[8]) + image_name = elems[9] + elems = fid.readline().split() + xys = np.column_stack([ + tuple(map(float, elems[0::3])), + tuple(map(float, elems[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, elems[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def read_images_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesBinary(const std::string& path) + void Reconstruction::WriteImagesBinary(const std::string& path) + """ + images = {} + with open(path_to_model_file, 'rb') as fid: + num_reg_images = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_reg_images): + binary_image_properties = read_next_bytes( + fid, num_bytes=64, format_char_sequence='idddddddi') + image_id = binary_image_properties[0] + qvec = np.array(binary_image_properties[1:5]) + tvec = np.array(binary_image_properties[5:8]) + camera_id = binary_image_properties[8] + image_name = '' + current_char = read_next_bytes(fid, 1, 'c')[0] + while current_char != b'\x00': # look for the ASCII 0 entry + image_name += current_char.decode('utf-8') + current_char = read_next_bytes(fid, 1, 'c')[0] + num_points2D = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + x_y_id_s = read_next_bytes( + fid, + num_bytes=24 * num_points2D, + format_char_sequence='ddq' * num_points2D) + xys = np.column_stack([ + tuple(map(float, x_y_id_s[0::3])), + tuple(map(float, x_y_id_s[1::3])) + ]) + point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) + images[image_id] = Image( + id=image_id, + qvec=qvec, + tvec=tvec, + camera_id=camera_id, + name=image_name, + xys=xys, + point3D_ids=point3D_ids) + return images + + +def write_images_text(images, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesText(const std::string& path) + void Reconstruction::WriteImagesText(const std::string& path) + """ + if len(images) == 0: + mean_observations = 0 + else: + mean_observations = sum( + (len(img.point3D_ids) for _, img in images.items())) / len(images) + HEADER = '# Image list with two lines of data per image:\n' + \ + '# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n' + \ + '# POINTS2D[] as (X, Y, POINT3D_ID)\n' + \ + '# Number of images: {}, mean observations per image: {}\n'.format(len(images), mean_observations) + + with open(path, 'w') as fid: + fid.write(HEADER) + for _, img in images.items(): + image_header = [ + img.id, *img.qvec, *img.tvec, img.camera_id, img.name + ] + first_line = ' '.join(map(str, image_header)) + fid.write(first_line + '\n') + + points_strings = [] + for xy, point3D_id in zip(img.xys, img.point3D_ids): + points_strings.append(' '.join(map(str, [*xy, point3D_id]))) + fid.write(' '.join(points_strings) + '\n') + + +def write_images_binary(images, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesBinary(const std::string& path) + void Reconstruction::WriteImagesBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(images), 'Q') + for _, img in images.items(): + write_next_bytes(fid, img.id, 'i') + write_next_bytes(fid, img.qvec.tolist(), 'dddd') + write_next_bytes(fid, img.tvec.tolist(), 'ddd') + write_next_bytes(fid, img.camera_id, 'i') + for char in img.name: + write_next_bytes(fid, char.encode('utf-8'), 'c') + write_next_bytes(fid, b'\x00', 'c') + write_next_bytes(fid, len(img.point3D_ids), 'Q') + for xy, p3d_id in zip(img.xys, img.point3D_ids): + write_next_bytes(fid, [*xy, p3d_id], 'ddq') + + +def read_points3D_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DText(const std::string& path) + void Reconstruction::WritePoints3DText(const std::string& path) + """ + points3D = {} + with open(path, 'r') as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != '#': + elems = line.split() + point3D_id = int(elems[0]) + xyz = np.array(tuple(map(float, elems[1:4]))) + rgb = np.array(tuple(map(int, elems[4:7]))) + error = float(elems[7]) + image_ids = np.array(tuple(map(int, elems[8::2]))) + point2D_idxs = np.array(tuple(map(int, elems[9::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def read_points3D_binary(path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DBinary(const std::string& path) + void Reconstruction::WritePoints3DBinary(const std::string& path) + """ + points3D = {} + with open(path_to_model_file, 'rb') as fid: + num_points = read_next_bytes(fid, 8, 'Q')[0] + for _ in range(num_points): + binary_point_line_properties = read_next_bytes( + fid, num_bytes=43, format_char_sequence='QdddBBBd') + point3D_id = binary_point_line_properties[0] + xyz = np.array(binary_point_line_properties[1:4]) + rgb = np.array(binary_point_line_properties[4:7]) + error = np.array(binary_point_line_properties[7]) + track_length = read_next_bytes( + fid, num_bytes=8, format_char_sequence='Q')[0] + track_elems = read_next_bytes( + fid, + num_bytes=8 * track_length, + format_char_sequence='ii' * track_length) + image_ids = np.array(tuple(map(int, track_elems[0::2]))) + point2D_idxs = np.array(tuple(map(int, track_elems[1::2]))) + points3D[point3D_id] = Point3D( + id=point3D_id, + xyz=xyz, + rgb=rgb, + error=error, + image_ids=image_ids, + point2D_idxs=point2D_idxs) + return points3D + + +def write_points3D_text(points3D, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DText(const std::string& path) + void Reconstruction::WritePoints3DText(const std::string& path) + """ + if len(points3D) == 0: + mean_track_length = 0 + else: + mean_track_length = sum( + (len(pt.image_ids) for _, pt in points3D.items())) / len(points3D) + HEADER = '# 3D point list with one line of data per point:\n' + \ + '# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)\n' + \ + '# Number of points: {}, mean track length: {}\n'.format(len(points3D), mean_track_length) + + with open(path, 'w') as fid: + fid.write(HEADER) + for _, pt in points3D.items(): + point_header = [pt.id, *pt.xyz, *pt.rgb, pt.error] + fid.write(' '.join(map(str, point_header)) + ' ') + track_strings = [] + for image_id, point2D in zip(pt.image_ids, pt.point2D_idxs): + track_strings.append(' '.join(map(str, [image_id, point2D]))) + fid.write(' '.join(track_strings) + '\n') + + +def write_points3D_binary(points3D, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DBinary(const std::string& path) + void Reconstruction::WritePoints3DBinary(const std::string& path) + """ + with open(path_to_model_file, 'wb') as fid: + write_next_bytes(fid, len(points3D), 'Q') + for _, pt in points3D.items(): + write_next_bytes(fid, pt.id, 'Q') + write_next_bytes(fid, pt.xyz.tolist(), 'ddd') + write_next_bytes(fid, pt.rgb.tolist(), 'BBB') + write_next_bytes(fid, pt.error, 'd') + track_length = pt.image_ids.shape[0] + write_next_bytes(fid, track_length, 'Q') + for image_id, point2D_id in zip(pt.image_ids, pt.point2D_idxs): + write_next_bytes(fid, [image_id, point2D_id], 'ii') + + +def detect_model_format(path, ext): + if os.path.isfile(os.path.join(path, 'cameras' + ext)) and \ + os.path.isfile(os.path.join(path, 'images' + ext)) and \ + os.path.isfile(os.path.join(path, 'points3D' + ext)): + print("Detected model format: '" + ext + "'") + return True + + return False + + +def read_model(path, ext=''): + # try to detect the extension automatically + if ext == '': + if detect_model_format(path, '.bin'): + ext = '.bin' + elif detect_model_format(path, '.txt'): + ext = '.txt' + else: + print("Provide model format: '.bin' or '.txt'") + return + + if ext == '.txt': + cameras = read_cameras_text(os.path.join(path, 'cameras' + ext)) + images = read_images_text(os.path.join(path, 'images' + ext)) + points3D = read_points3D_text(os.path.join(path, 'points3D') + ext) + else: + cameras = read_cameras_binary(os.path.join(path, 'cameras' + ext)) + images = read_images_binary(os.path.join(path, 'images' + ext)) + points3D = read_points3D_binary(os.path.join(path, 'points3D') + ext) + return cameras, images, points3D + + +def write_model(cameras, images, points3D, path, ext='.bin'): + if ext == '.txt': + write_cameras_text(cameras, os.path.join(path, 'cameras' + ext)) + write_images_text(images, os.path.join(path, 'images' + ext)) + write_points3D_text(points3D, os.path.join(path, 'points3D') + ext) + else: + write_cameras_binary(cameras, os.path.join(path, 'cameras' + ext)) + write_images_binary(images, os.path.join(path, 'images' + ext)) + write_points3D_binary(points3D, os.path.join(path, 'points3D') + ext) + return cameras, images, points3D + + +def qvec2rotmat(qvec): + array_10 = 1 - 2 * qvec[2]**2 - 2 * qvec[3]**2 + array_11 = 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3] + array_12 = 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2] + array_1 = [array_10, array_11, array_12] + array_20 = 2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3] + array_21 = 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2 + array_22 = 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1] + array_2 = [array_20, array_21, array_22] + array_30 = 2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2] + array_31 = 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1] + array_32 = 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2 + array_3 = [array_30, array_31, array_32] + + return np.array([array_1, array_2, array_3]) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_preprocess.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..8be3f42b4f143097e639555ad3c2b7e387a8ede9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_preprocess.py @@ -0,0 +1,200 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import glob +import os +import subprocess +from typing import Any, Dict, Union + +import cv2 +import numpy as np +import tensorflow as tf + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.utils.constant import Fields, ModeKeys +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.nerf_recon_acc_preprocessor) +class NeRFReconPreprocessor(Preprocessor): + + def __init__(self, + mode=ModeKeys.INFERENCE, + data_type='colmap', + use_mask=True, + match_type='exhaustive_matcher', + frame_count=60, + use_distortion=False, + *args, + **kwargs): + + super().__init__(mode) + + # set preprocessor info + self.data_type = data_type + self.use_mask = use_mask + + self.match_type = match_type + if match_type != 'exhaustive_matcher' and match_type != 'sequential_matcher': + raise Exception('matcher type {} is not valid'.format(match_type)) + self.frame_count = frame_count + self.use_distortion = use_distortion + + def __call__(self, data: Union[str, Dict], **kwargs) -> Dict[str, Any]: + + if self.data_type != 'blender' and self.data_type != 'colmap': + raise Exception('data type {} is not support currently'.format( + self.data_type)) + + data_dir = data['data_dir'] + os.makedirs(data_dir, exist_ok=True) + if self.data_type == 'blender': + transform_file = os.path.join(data_dir, 'transforms_train.json') + if not os.path.exists(transform_file): + raise Exception('Blender dataset is not found') + + if self.data_type == 'colmap': + video_path = data['video_input_path'] + if video_path != '': + self.split_frames(video_path, data_dir, self.frame_count) + self.gen_poses(data_dir, self.match_type, self.use_distortion) + files_needed = [ + '{}.bin'.format(f) for f in ['cameras', 'images', 'points3D'] + ] + if self.use_distortion: + colmap_dir = os.path.join(data_dir, 'preprocess/sparse') + files_had = os.listdir(colmap_dir) + else: + colmap_dir = os.path.join(data_dir, 'sparse/0') + files_had = os.listdir(colmap_dir) + if not all([f in files_had for f in files_needed]): + raise Exception('colmap run failed') + + data = {} + data['data_dir'] = data_dir + return data + + def split_frames(self, video_path, basedir, frame_count=60): + cap = cv2.VideoCapture(video_path) + fps = round(cap.get(cv2.CAP_PROP_FPS)) + frame_total = round(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + if not os.path.exists(os.path.join(basedir, 'images')): + logger.info('Need to run ffmpeg') + image_dir = os.path.join(basedir, 'images') + os.makedirs(image_dir, exist_ok=True) + fps = int(frame_count * fps / frame_total) + cmd = f"ffmpeg -i {video_path} -qscale:v 1 -qmin 1 -vf \"fps={fps}\" {image_dir}/%04d.png" + os.system(cmd) + logger.info('split frames done') + else: + logger.info('Don\'t need to run ffmpeg') + + def run_colmap(self, basedir, match_type, use_distortion): + logfile_name = os.path.join(basedir, 'colmap_output.txt') + logfile = open(logfile_name, 'w') + + feature_extractor_args = [ + 'colmap', 'feature_extractor', '--database_path', + os.path.join(basedir, 'database.db'), '--image_path', + os.path.join(basedir, 'images'), '--ImageReader.single_camera', '1' + ] + feat_output = ( + subprocess.check_output( + feature_extractor_args, universal_newlines=True)) + logfile.write(feat_output) + logger.info('Features extracted done') + + exhaustive_matcher_args = [ + 'colmap', + match_type, + '--database_path', + os.path.join(basedir, 'database.db'), + ] + + match_output = ( + subprocess.check_output( + exhaustive_matcher_args, universal_newlines=True)) + logfile.write(match_output) + logger.info('Features matched done') + + p = os.path.join(basedir, 'sparse') + if not os.path.exists(p): + os.makedirs(p) + + mapper_args = [ + 'colmap', + 'mapper', + '--database_path', + os.path.join(basedir, 'database.db'), + '--image_path', + os.path.join(basedir, 'images'), + '--output_path', + os.path.join( + basedir, 'sparse' + ), # --export_path changed to --output_path in colmap 3.6 + '--Mapper.num_threads', + '16', + '--Mapper.init_min_tri_angle', + '4', + '--Mapper.multiple_models', + '0', + '--Mapper.extract_colors', + '0', + ] + + map_output = ( + subprocess.check_output(mapper_args, universal_newlines=True)) + logfile.write(map_output) + logger.info('Sparse map created done.') + + bundle_adjuster_cmd = [ + 'colmap', + 'bundle_adjuster', + '--input_path', + os.path.join(basedir, 'sparse/0'), + '--output_path', + os.path.join(basedir, 'sparse/0'), + '--BundleAdjustment.refine_principal_point', + '1', + ] + map_output = ( + subprocess.check_output( + bundle_adjuster_cmd, universal_newlines=True)) + logfile.write(map_output) + logger.info('Refining intrinsics done.') + + if use_distortion: + os.makedirs(os.path.join(basedir, 'preprocess'), exist_ok=True) + distort_cmd = [ + 'colmap', 'image_undistorter', '--image_path', + os.path.join(basedir, 'images'), '--input_path', + os.path.join(basedir, 'sparse/0'), '--output_path', + os.path.join(basedir, 'preprocess'), '--output_type', 'COLMAP' + ] + map_output = ( + subprocess.check_output(distort_cmd, universal_newlines=True)) + logfile.write(map_output) + logger.info('Image distortion done.') + + logfile.close() + logger.info( + 'Finished running COLMAP, see {} for logs'.format(logfile_name)) + + def gen_poses(self, basedir, match_type, use_distortion): + files_needed = [ + '{}.bin'.format(f) for f in ['cameras', 'images', 'points3D'] + ] + if os.path.exists(os.path.join(basedir, 'sparse/0')): + files_had = os.listdir(os.path.join(basedir, 'sparse/0')) + else: + files_had = [] + if not all([f in files_had for f in files_needed]): + logger.info('Need to run COLMAP') + self.run_colmap(basedir, match_type, use_distortion) + else: + logger.info('Don\'t need to run COLMAP') diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_recon_acc.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_recon_acc.py new file mode 100644 index 0000000000000000000000000000000000000000..2d2fbd58d2d453c0b9ddbd5cd8fb380dda148e7d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/nerf_recon_acc.py @@ -0,0 +1,201 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import glob +import os +import time + +import cv2 +import numpy as np +import torch +import tqdm + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .dataloader.nerf_dataset import BlenderDataset, ColmapDataset +from .network.nerf import NeRFModel +from .network.utils import PSNR + +logger = get_logger() + +__all__ = ['NeRFReconAcc'] + + +@MODELS.register_module( + Tasks.nerf_recon_acc, module_name=Models.nerf_recon_acc) +class NeRFReconAcc(TorchModel): + + def __init__(self, model_dir, network_cfg, **kwargs): + """initialize the acceleration version of nerf reconstruction model for object. + NeRFReconAcc accelerate single object reconstruction time from ~10hours to ~10min. + + Args: + model_dir (str): the model path. + data_type (str): only support 'blender' or 'colmap' + use_mask (bool): whether use mask of objects, default True + num_samples_per_ray (int): sampling numbers for each ray, default 1024 + test_ray_chunk (int): chunk size for rendering, default 1024 + max_size (int): max size of (width, height) when training, default 800 + n_test_traj_steps (int): number of testing images, default 120 + log_every_n_steps (int): print log info every n steps, default 1000 + save_mesh (bool): whether to save the reconstructed mesh of object, default False + network_cfg (dict): args of network config + """ + super().__init__(model_dir, **kwargs) + + if not torch.cuda.is_available(): + raise Exception('GPU is required') + logger.info('model params:{}'.format(kwargs)) + self.data_type = kwargs['data_type'] + self.use_mask = kwargs['use_mask'] + self.num_samples_per_ray = kwargs['num_samples_per_ray'] + self.test_ray_chunk = kwargs['test_ray_chunk'] + self.save_mesh = kwargs['save_mesh'] + self.ckpt_path = kwargs['ckpt_path'] + + if self.ckpt_path == '': + self.ckpt_path = os.path.join(model_dir, 'model.ckpt') + if not os.path.exists(self.ckpt_path): + raise Exception('ckpt path not found') + + if self.data_type == 'blender': + self.img_wh = (800, 800) + network_cfg['radius'] = 1.5 + self.background = 'white' + network_cfg['background'] = 'white' + elif self.data_type == 'colmap': + self.img_wh = None + self.max_size = kwargs['max_size'] + self.n_test_traj_steps = kwargs['n_test_traj_steps'] + network_cfg['radius'] = 0.5 + if self.use_mask: + self.background = 'white' + network_cfg['background'] = 'white' + logger.info('run nerf with mask data') + else: + self.background = 'random' + network_cfg['background'] = 'random' + logger.info('run nerf without mask data') + logger.info(network_cfg) + + self.model = NeRFModel( + network_cfg, + num_samples_per_ray=self.num_samples_per_ray, + test_ray_chunk=self.test_ray_chunk).cuda() + + checkpoints = torch.load(self.ckpt_path) + self.model.load_state_dict(checkpoints['network_state_dict']) + self.model = self.model.cuda() + self.model.eval() + + self.criterions = PSNR() + + def nerf_reconstruction(self, data_dir, render_dir): + if self.data_type == 'blender': + self.test_dataset = BlenderDataset( + root_fp=data_dir, + split='test', + img_wh=self.img_wh, + ) + + elif self.data_type == 'colmap': + self.test_dataset = ColmapDataset( + root_fp=data_dir, + split='test', + img_wh=self.img_wh, + max_size=self.max_size, + n_test_traj_steps=self.n_test_traj_steps, + ) + + tic_start = time.time() + os.makedirs(render_dir, exist_ok=True) + logger.info('save render path: {}.'.format(render_dir)) + self.render_video(render_dir) + tic_end = time.time() + duration = tic_end - tic_start + logger.info('reconstruction done, cost time: {}'.format(duration)) + + def render_video(self, render_dir): + with torch.no_grad(): + psnr = 0 + for i in tqdm.tqdm(range(len(self.test_dataset))): + + data = self.test_dataset[i] + rays = data['rays'].cuda() + pixels = data['pixels'].cuda() + image_wh = data['image_wh'] + out = self.model.inference(rays) + + psnr += self.criterions(out['comp_rgb'], pixels) + + W, H = image_wh + img = out['comp_rgb'].view(H, W, 3) + save_img_dir = os.path.join(render_dir, 'render') + os.makedirs(save_img_dir, exist_ok=True) + save_img_path = os.path.join(save_img_dir, f'{i:d}.png') + self.save_image(save_img_path, img) + + save_video_path = os.path.join(render_dir, 'render.mp4') + self.save_video(save_video_path, save_img_dir) + logger.info('test psnr: {}'.format(psnr / len(self.test_dataset))) + logger.info('save render video done.') + + if self.save_mesh: + mesh = self.model.isosurface() + save_mesh_path = os.path.join(render_dir, 'render.obj') + self.save_obj(save_mesh_path, mesh['v_pos'], mesh['t_pos_idx']) + logger.info('save render mesh done.') + + def save_image(self, filename, img): + img = img.clip(0, 1).cpu().numpy() + img = (img * 255.).astype(np.uint8) + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + save_dir = os.path.dirname(filename) + os.makedirs(save_dir, exist_ok=True) + cv2.imwrite(filename, img) + + def save_video(self, filename, img_dir, fps=20): + img_paths = glob.glob('{}/*.png'.format(img_dir)) + img_paths = sorted( + img_paths, key=lambda f: int(os.path.basename(f)[:-4])) + imgs = [cv2.imread(f) for f in img_paths] + + H, W, _ = imgs[0].shape + writer = cv2.VideoWriter(filename, cv2.VideoWriter_fourcc(*'mp4v'), + fps, (W, H), True) + for img in imgs: + writer.write(img) + writer.release() + + def write_obj(self, filename, v_pos, t_pos_idx, v_tex, t_tex_idx): + with open(filename, 'w') as f: + for v in v_pos: + f.write('v {} {} {} \n'.format(v[0], v[1], v[2])) + + if v_tex is not None: + assert (len(t_pos_idx) == len(t_tex_idx)) + for v in v_tex: + f.write('vt {} {} \n'.format(v[0], 1.0 - v[1])) + + for i in range(len(t_pos_idx)): + f.write('f ') + for j in range(3): + f.write( + ' %s/%s' % + (str(t_pos_idx[i][j] + 1), + '' if v_tex is None else str(t_tex_idx[i][j] + 1))) + f.write('\n') + + def save_obj(self, filename, v_pos, t_pos_idx, v_tex=None, t_tex_idx=None): + + v_pos = v_pos.cpu().numpy() + t_pos_idx = t_pos_idx.cpu().numpy() + save_dir = os.path.dirname(filename) + os.makedirs(save_dir, exist_ok=True) + if v_tex is not None and t_tex_idx is not None: + v_tex = v_tex.cpu().numpy() + t_tex_idx = t_tex_idx.cpu().numpy() + self.write_obj(filename, v_pos, t_pos_idx, v_tex, t_tex_idx) + else: + self.write_obj(filename, v_pos, t_pos_idx, v_tex, t_tex_idx) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/nerf.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/nerf.py new file mode 100644 index 0000000000000000000000000000000000000000..972d2d85044c62c6eb52978cb8ab54024dfaae4e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/nerf.py @@ -0,0 +1,344 @@ +# The implementation is modified from nerfacc, made publicly available under the MIT License +# at https://github.com/KAIR-BAIR/nerfacc/blob/master/examples/radiance_fields/ngp.py +import numpy as np +import tinycudann as tcnn +import torch +import torch.nn as nn +from nerfacc import ContractionType, OccupancyGrid, ray_marching, rendering +from torch.autograd import Function +from torch.cuda.amp import custom_bwd, custom_fwd + +from .utils import chunk_batch, cleanup, get_activation, normalize + + +class VanillaMLP(nn.Module): + + def __init__(self, dim_in, dim_out, n_neurons, n_hidden_layers, + activation): + super().__init__() + self.layers = [ + self.make_linear(dim_in, n_neurons), + self.make_activation() + ] + for i in range(n_hidden_layers - 1): + self.layers += [ + self.make_linear(n_neurons, n_neurons), + self.make_activation() + ] + self.layers += [self.make_linear(n_neurons, dim_out)] + self.layers = nn.Sequential(*self.layers) + self.output_activation = get_activation(activation) + + def forward(self, x): + x = self.layers(x.float()) + x = self.output_activation(x) + return x + + def make_linear(self, dim_in, dim_out): + layer = nn.Linear(dim_in, dim_out, bias=True) + torch.nn.init.constant_(layer.bias, 0.0) + torch.nn.init.kaiming_uniform_(layer.weight, nonlinearity='relu') + return layer + + def make_activation(self): + return nn.ReLU(inplace=True) + + +class MarchingCubeHelper(nn.Module): + + def __init__(self, resolution, use_torch=True): + super().__init__() + self.resolution = resolution + self.use_torch = use_torch + self.points_range = (0, 1) + if self.use_torch: + import torchmcubes + self.mc_func = torchmcubes.marching_cubes + else: + import mcubes + self.mc_func = mcubes.marching_cubes + self.verts = None + + def grid_vertices(self): + if self.verts is None: + x, y, z = torch.linspace(*self.points_range, + self.resolution), torch.linspace( + *self.points_range, + self.resolution), torch.linspace( + *self.points_range, + self.resolution) + x, y, z = torch.meshgrid(x, y, z) + verts = torch.cat( + [x.reshape(-1, 1), + y.reshape(-1, 1), + z.reshape(-1, 1)], dim=-1).reshape(-1, 3) + self.verts = verts.cuda() + return self.verts + + def forward(self, level, threshold=0.): + level = level.float().view(self.resolution, self.resolution, + self.resolution) + if self.use_torch: + verts, faces = self.mc_func(level.cuda(), threshold) + verts, faces = verts.cpu(), faces.cpu().long() + else: + verts, faces = self.mc_func(-level.numpy(), + threshold) # transform to numpy + verts, faces = torch.from_numpy( + verts.astype(np.float32)), torch.from_numpy( + faces.astype(np.int64)) # transform back to pytorch + verts = verts / (self.resolution - 1.) + return {'v_pos': verts, 't_pos_idx': faces} + + +class _TruncExp(Function): # pylint: disable=abstract-method + # Implementation from torch-ngp: + # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, x): # pylint: disable=arguments-differ + ctx.save_for_backward(x) + return torch.exp(x) + + @staticmethod + @custom_bwd + def backward(ctx, g): # pylint: disable=arguments-differ + x = ctx.saved_tensors[0] + return g * torch.exp(x.clamp(-15, 15)) + + +trunc_exp = _TruncExp.apply + + +class VolumeDensity(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.radius = self.config.radius + self.n_input_dims = 3 + self.n_output_dims = self.config.geometry_feature_dim + 1 + point_encoding = tcnn.Encoding( + n_input_dims=self.n_input_dims, + encoding_config={ + 'otype': 'HashGrid', + 'n_levels': self.config.n_levels, + 'n_features_per_level': self.config.n_features_per_level, + 'log2_hashmap_size': self.config.log2_hashmap_size, + 'base_resolution': self.config.base_resolution, + 'per_level_scale': self.config.per_level_scale, + }, + ) + point_network = VanillaMLP(point_encoding.n_output_dims, + self.n_output_dims, 64, 1, 'none') + self.encoding_with_network = torch.nn.Sequential( + point_encoding, point_network) + + self.density_activation = trunc_exp + self.helper = MarchingCubeHelper( + self.config.isosurface_resolution, use_torch=False) + + def forward(self, points): + points = normalize(points, (-self.radius, self.radius), (0, 1)) + out = self.encoding_with_network(points.view( + -1, self.n_input_dims)).view(*points.shape[:-1], + self.n_output_dims).float() + density, feature = out[..., 0], out[..., 1:] + density = self.density_activation(density - 1) + return density, feature + + def forward_level(self, points): + points = normalize(points, (-self.radius, self.radius), (0, 1)) + out = self.encoding_with_network(points.view( + -1, self.n_input_dims)).view(*points.shape[:-1], + self.n_output_dims).float() + density, _ = out[..., 0], out[..., 1:] + density = self.density_activation(density - 1) + return -density + + def isosurface_(self, vmin, vmax): + grid_verts = self.helper.grid_vertices() + grid_verts_1 = normalize(grid_verts[..., 0], (0, 1), + (vmin[0], vmax[0])) + grid_verts_2 = normalize(grid_verts[..., 1], (0, 1), + (vmin[1], vmax[1])) + grid_verts_3 = normalize(grid_verts[..., 2], (0, 1), + (vmin[2], vmax[2])) + grid_verts = torch.stack([grid_verts_1, grid_verts_2, grid_verts_3], + dim=-1) + + def batch_func(x): + rv = self.forward_level(x).cpu() + cleanup() + return rv + + level = chunk_batch(batch_func, self.config.isosurface_chunk, + grid_verts) + mesh = self.helper(level, threshold=self.config.isosurface_threshold) + mesh_1 = normalize(mesh['v_pos'][..., 0], (0, 1), (vmin[0], vmax[0])) + mesh_2 = normalize(mesh['v_pos'][..., 1], (0, 1), (vmin[1], vmax[1])) + mesh_3 = normalize(mesh['v_pos'][..., 2], (0, 1), (vmin[2], vmax[2])) + mesh['v_pos'] = torch.stack([mesh_1, mesh_2, mesh_3], dim=-1) + return mesh + + @torch.no_grad() + def isosurface(self): + mesh_coarse = self.isosurface_( + (-self.radius, -self.radius, -self.radius), + (self.radius, self.radius, self.radius)) + vmin, vmax = mesh_coarse['v_pos'].amin( + dim=0), mesh_coarse['v_pos'].amax(dim=0) + vmin_ = (vmin - (vmax - vmin) * 0.1).clamp(-self.radius, self.radius) + vmax_ = (vmax + (vmax - vmin) * 0.1).clamp(-self.radius, self.radius) + mesh_fine = self.isosurface_(vmin_, vmax_) + return mesh_fine + + +class VolumeRadiance(nn.Module): + + def __init__(self, config): + super(VolumeRadiance, self).__init__() + self.config = config + self.n_dir_dims = 3 + self.n_output_dims = 3 + self.direction_encoding = tcnn.Encoding( + n_input_dims=self.n_dir_dims, + encoding_config={ + 'otype': + 'Composite', + 'nested': [ + { + 'n_dims_to_encode': self.n_dir_dims, + 'otype': 'SphericalHarmonics', + 'degree': self.config.degree, + }, + ], + }, + ) + self.n_input_dims = self.config.geometry_feature_dim + self.direction_encoding.n_output_dims + self.network = VanillaMLP(self.n_input_dims, self.n_output_dims, 64, 2, + 'sigmoid') + + def forward(self, features, dirs): + dirs = (dirs + 1.) / 2. # (-1, 1) => (0, 1) + dirs_embd = self.direction_encoding(dirs.view(-1, self.n_dir_dims)) + network_inp = torch.cat( + [dirs_embd, + features.view(-1, self.config.geometry_feature_dim)], + dim=-1) + color = self.network(network_inp).view(*features.shape[:-1], + self.n_output_dims).float() + return color + + +class NeRFModel(nn.Module): + + def __init__(self, network_cfg, **kwargs): + super().__init__() + self.config = network_cfg + self.num_samples_per_ray = kwargs['num_samples_per_ray'] + self.test_ray_chunk = kwargs['test_ray_chunk'] + self.background = self.config.background + self.geometry = VolumeDensity(self.config) + self.texture = VolumeRadiance(self.config) + radius_list = [ + -self.config.radius, -self.config.radius, -self.config.radius, + self.config.radius, self.config.radius, self.config.radius + ] + radius_tensor = torch.as_tensor(radius_list, dtype=torch.float32) + self.register_buffer('scene_aabb', radius_tensor) + self.occupancy_grid = OccupancyGrid( + roi_aabb=self.scene_aabb, + resolution=128, + contraction_type=ContractionType.AABB) + self.render_step_size = 1.732 * 2 * self.config.radius / self.num_samples_per_ray + + def update_step(self, global_step): + # progressive viewdir PE frequencies + + def occ_eval_fn(x): + density, _ = self.geometry(x) + # approximate for 1 - torch.exp(-density[...,None] * self.render_step_size) based on taylor series + return density[..., None] * self.render_step_size + + self.occupancy_grid.every_n_step( + step=global_step, occ_eval_fn=occ_eval_fn) + + def isosurface(self): + mesh = self.geometry.isosurface() + return mesh + + def forward(self, rays): + rays_o, rays_d = rays[:, 0:3], rays[:, 3:6] # both (N_rays, 3) + if self.training: + if self.background == 'random': + background_color = torch.rand( + 3, dtype=torch.float32, device=rays_o.device) + elif self.background == 'white': + background_color = torch.ones( + 3, dtype=torch.float32, device=rays_o.device) + elif self.background == 'black': + background_color = torch.zeros( + 3, dtype=torch.float32, device=rays_o.device) + else: + background_color = torch.ones( + 3, dtype=torch.float32, device=rays_o.device) + + def sigma_fn(t_starts, t_ends, ray_indices): + ray_indices = ray_indices.long() + t_origins = rays_o[ray_indices] + t_dirs = rays_d[ray_indices] + positions = t_origins + t_dirs * (t_starts + t_ends) / 2. + density, _ = self.geometry(positions) + + return density[..., None] + + def rgb_sigma_fn(t_starts, t_ends, ray_indices): + ray_indices = ray_indices.long() + t_origins = rays_o[ray_indices] + t_dirs = rays_d[ray_indices] + positions = t_origins + t_dirs * (t_starts + t_ends) / 2. + density, feature = self.geometry(positions) + rgb = self.texture(feature, t_dirs) + return rgb, density[..., None] + + with torch.no_grad(): + packed_info, t_starts, t_ends = ray_marching( + rays_o, + rays_d, + scene_aabb=self.scene_aabb, + grid=self.occupancy_grid, + sigma_fn=sigma_fn, + near_plane=None, + far_plane=None, + render_step_size=self.render_step_size, + stratified=self.training, + cone_angle=0.0, + alpha_thre=0.0) + rgb, opacity, depth = rendering( + packed_info, + t_starts, + t_ends, + rgb_sigma_fn=rgb_sigma_fn, + render_bkgd=background_color) + + opacity, depth = opacity.squeeze(-1), depth.squeeze(-1) + + return { + 'comp_rgb': + rgb, + 'opacity': + opacity, + 'depth': + depth, + 'rays_valid': + opacity > 0, + 'num_samples': + torch.as_tensor([len(t_starts)], + dtype=torch.int32, + device=rays.device) + } + + def inference(self, rays): + out = chunk_batch(self.forward, self.test_ray_chunk, rays) + return {**out} diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/segmenter.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/segmenter.py new file mode 100644 index 0000000000000000000000000000000000000000..e3d0ca8deeb7c2153be08bd8d340441961cc82c3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/segmenter.py @@ -0,0 +1,43 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import tensorflow as tf + + +class ObjectSegmenter(object): + """use ObjectSegmenter to segment object from input video. + + Args: + model_path (str): the segment model path. + """ + + def __init__(self, model_path): + super(ObjectSegmenter, self).__init__() + f = tf.gfile.FastGFile(model_path, 'rb') + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + persisted_graph = tf.import_graph_def(graph_def, name='') + + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.3 + self.sess = tf.InteractiveSession(graph=persisted_graph, config=config) + + self.image_node = self.sess.graph.get_tensor_by_name('input_image:0') + self.output_node = self.sess.graph.get_tensor_by_name('output_png:0') + self.logits_node = self.sess.graph.get_tensor_by_name('if_person:0') + + def image_preprocess(self, img): + if len(img.shape) == 2: + img = np.dstack((img, img, img)) + elif img.shape[2] == 4: + img = img[:, :, :3] + img = img[:, :, ::-1] + img = img.astype(float) + return img + + def run_mask(self, img): + image_feed = self.image_preprocess(img) + output_img_value, logits_value = self.sess.run( + [self.output_node, self.logits_node], + feed_dict={self.image_node: image_feed}) + mask = output_img_value[:, :, 3:] + return mask diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/utils.py b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aa8c3d6675748011a44c784fc2421e5c00555774 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_acc/network/utils.py @@ -0,0 +1,176 @@ +# The implementation is partly adopted from nerfacc, made publicly available under the MIT License +# at https://github.com/KAIR-BAIR/nerfacc/blob/master/examples/radiance_fields/ngp.py +import gc +from collections import defaultdict + +import mcubes +import numpy as np +import tinycudann as tcnn +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.cuda.amp import custom_bwd, custom_fwd + + +class PSNR(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, inputs, targets, valid_mask=None, reduction='mean'): + assert reduction in ['mean', 'none'] + value = (inputs - targets)**2 + if valid_mask is not None: + value = value[valid_mask] + if reduction == 'mean': + return -10 * torch.log10(torch.mean(value)) + elif reduction == 'none': + return -10 * torch.log10( + torch.mean(value, dim=tuple(range(value.ndim)[1:]))) + + +def extract_fields(bound_min, bound_max, resolution, query_func): + N = 64 + X = torch.linspace(bound_min[0], bound_max[0], resolution).split(N) + Y = torch.linspace(bound_min[1], bound_max[1], resolution).split(N) + Z = torch.linspace(bound_min[2], bound_max[2], resolution).split(N) + + u = np.zeros([resolution, resolution, resolution], dtype=np.float32) + with torch.no_grad(): + for xi, xs in enumerate(X): + for yi, ys in enumerate(Y): + for zi, zs in enumerate(Z): + xx, yy, zz = torch.meshgrid(xs, ys, zs) + xx = xx.reshape(-1, 1) + yy = yy.reshape(-1, 1) + zz = zz.reshape(-1, 1) + pts = torch.cat([xx, yy, zz], dim=-1).cuda() + val = query_func(pts).reshape( + len(xs), len(ys), len(zs)).detach().cpu().numpy() + u[xi * N:xi * N + len(xs), yi * N:yi * N + len(ys), + zi * N:zi * N + len(zs)] = val + return u + + +def extract_geometry(bound_min, bound_max, resolution, threshold, query_func): + u = extract_fields(bound_min, bound_max, resolution, query_func) + vertices, triangles = mcubes.marching_cubes(u, threshold) + b_max_np = bound_max.detach().cpu().numpy() + b_min_np = bound_min.detach().cpu().numpy() + + vertices = vertices / (resolution - 1.0) * ( + b_max_np - b_min_np)[None, :] + b_min_np[None, :] + return vertices, triangles + + +def chunk_batch(func, chunk_size, *args, **kwargs): + B = None + for arg in args: + if isinstance(arg, torch.Tensor): + B = arg.shape[0] + break + out = defaultdict(list) + out_type = None + for i in range(0, B, chunk_size): + out_chunk = func( + *[ + arg[i:i + chunk_size] if isinstance(arg, torch.Tensor) else arg + for arg in args + ], **kwargs) + if out_chunk is None: + continue + out_type = type(out_chunk) + if isinstance(out_chunk, torch.Tensor): + out_chunk = {0: out_chunk} + elif isinstance(out_chunk, tuple) or isinstance(out_chunk, list): + chunk_length = len(out_chunk) + out_chunk = {i: chunk for i, chunk in enumerate(out_chunk)} + elif isinstance(out_chunk, dict): + pass + else: + exit(1) + for k, v in out_chunk.items(): + out[k].append(v if torch.is_grad_enabled() else v.detach()) + + if out_type is None: + return + + out = {k: torch.cat(v, dim=0) for k, v in out.items()} + if out_type is torch.Tensor: + return out[0] + elif out_type in [tuple, list]: + return out_type([out[i] for i in range(chunk_length)]) + elif out_type is dict: + return out + + +def get_activation(name): + name = name.lower() + if name is None or name == 'none': + return nn.Identity() + elif name.startswith('scale'): + scale_factor = float(name[5:]) + return lambda x: x.clamp(0., scale_factor) / scale_factor + elif name.startswith('clamp'): + clamp_max = float(name[5:]) + return lambda x: x.clamp(0., clamp_max) + elif name.startswith('mul'): + mul_factor = float(name[3:]) + return lambda x: x * mul_factor + elif name == 'trunc_exp': + return trunc_exp + elif name.startswith('+') or name.startswith('-'): + return lambda x: x + float(name) + elif name.lower() == 'sigmoid': + return lambda x: torch.sigmoid(x) + elif name.lower() == 'tanh': + return lambda x: torch.tanh(x) + else: + return getattr(F, name) + + +class _TruncExp(Function): + # Implementation from torch-ngp: + # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, x): # pylint: disable=arguments-differ + ctx.save_for_backward(x) + return torch.exp(x) + + @staticmethod + @custom_bwd + def backward(ctx, g): + x = ctx.saved_tensors[0] + return g * torch.exp(torch.clamp(x, max=15)) + + +trunc_exp = _TruncExp.apply + + +def dot(x, y): + return torch.sum(x * y, -1, keepdim=True) + + +def reflect(x, n): + return 2 * dot(x, n) * n - x + + +def normalize(dat, inp_scale, tgt_scale): + if inp_scale is None: + inp_scale = [dat.min(), dat.max()] + dat = (dat - inp_scale[0]) / (inp_scale[1] - inp_scale[0]) + dat = dat * (tgt_scale[1] - tgt_scale[0]) + tgt_scale[0] + return dat + + +def cleanup(): + gc.collect() + torch.cuda.empty_cache() + tcnn.free_temporary_memory() + + +def update_module_step(m, epoch, global_step): + if hasattr(m, 'update_step'): + m.update_step(epoch, global_step) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..40bdbffbfb46a10a36328b6dda209aa0b0021113 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .nerf_recon_vq_compression import NeRFReconVQCompression + # from .nerf_preprocess import NeRFReconPreprocessor + +else: + _import_structure = { + 'nerf_recon_vq_compression': ['NeRFReconVQCompression'] + } + # _import_structure = {'nerf_preprocess': ['NeRFReconPreprocessor']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4e3a7cd63990ff4a6332e6e98291ed924c11c7fc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/__init__.py @@ -0,0 +1,12 @@ +from .blender import BlenderDataset +from .llff import LLFFDataset +from .nsvf import NSVF +from .ray_utils import get_rays, ndc_rays_blender +from .tankstemple import TanksTempleDataset + +dataset_dict = { + 'blender': BlenderDataset, + 'llff': LLFFDataset, + 'tankstemple': TanksTempleDataset, + 'nsvf': NSVF +} diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/blender.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/blender.py new file mode 100644 index 0000000000000000000000000000000000000000..7174514c4e98b577964bb4140eb12814adf1c11d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/blender.py @@ -0,0 +1,173 @@ +import os + +import cv2 +import json +import numpy as np +import torch +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms as T +from tqdm import tqdm + +from .ray_utils import * + + +def trans_t(t): + return torch.Tensor([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, t], + [0, 0, 0, 1]]).float() + + +def rot_phi(phi): + return torch.Tensor([[1, 0, 0, 0], [0, np.cos(phi), -np.sin(phi), 0], + [0, np.sin(phi), np.cos(phi), 0], [0, 0, 0, + 1]]).float() + + +def rot_theta(th): + return torch.Tensor([[np.cos(th), 0, -np.sin(th), 0], [0, 1, 0, 0], + [np.sin(th), 0, np.cos(th), 0], [0, 0, 0, + 1]]).float() + + +def pose_spherical(theta, phi, radius): + c2w = trans_t(radius) + c2w = rot_phi(phi / 180. * np.pi) @ c2w + c2w = rot_theta(theta / 180. * np.pi) @ c2w + c2w = torch.Tensor( + np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1] + ])) @ c2w + return c2w + + +class BlenderDataset(Dataset): + + def __init__(self, + datadir, + split='train', + downsample=1.0, + is_stack=False, + N_vis=-1): + + self.N_vis = N_vis + self.root_dir = datadir + self.split = split + self.is_stack = is_stack + self.img_wh = (int(800 / downsample), int(800 / downsample)) + self.define_transforms() + + self.scene_bbox = torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]]) + self.blender2opencv = np.array([[1, 0, 0, 0], [0, -1, 0, 0], + [0, 0, -1, 0], [0, 0, 0, 1]]) + self.read_meta() + self.define_proj_mat() + + self.white_bg = True + self.near_far = [2.0, 6.0] + + self.center = torch.mean(self.scene_bbox, axis=0).float().view(1, 1, 3) + self.radius = (self.scene_bbox[1] - self.center).float().view(1, 1, 3) + self.downsample = downsample + + def read_depth(self, filename): + depth = np.array(read_pfm(filename)[0], dtype=np.float32) # (800, 800) + return depth + + def read_meta(self): + + with open( + os.path.join(self.root_dir, f'transforms_{self.split}.json'), + 'r') as f: + self.meta = json.load(f) + + w, h = self.img_wh + self.focal = 0.5 * 800 / np.tan(0.5 * self.meta['camera_angle_x']) + self.focal *= self.img_wh[0] / 800 + + # ray directions for all pixels, same for all images (same H, W, focal) + self.directions = get_ray_directions( + h, w, [self.focal, self.focal]) # (h, w, 3) + self.directions = self.directions / torch.norm( + self.directions, dim=-1, keepdim=True) + self.intrinsics = torch.tensor([[self.focal, 0, w / 2], + [0, self.focal, h / 2], [0, 0, + 1]]).float() + + self.image_paths = [] + self.poses = [] + self.all_rays = [] + self.all_rgbs = [] + self.all_masks = [] + self.all_depth = [] + self.downsample = 1.0 + + img_eval_interval = 1 if self.N_vis < 0 else len( + self.meta['frames']) // self.N_vis + idxs = list(range(0, len(self.meta['frames']), img_eval_interval)) + for i in tqdm(idxs, desc=f'Loading data {self.split} ({len(idxs)})'): + + frame = self.meta['frames'][i] + pose = np.array(frame['transform_matrix']) @ self.blender2opencv + c2w = torch.FloatTensor(pose) + self.poses += [c2w] + + image_path = os.path.join(self.root_dir, + f"{frame['file_path']}.png") + self.image_paths += [image_path] + img = Image.open(image_path) + + if self.downsample != 1.0: + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (4, h, w) + img = img.view(4, -1).permute(1, 0) # (h*w, 4) RGBA + img = img[:, :3] * img[:, -1:] + (1 - img[:, -1:]) + self.all_rgbs += [img] + + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + self.all_rays += [torch.cat([rays_o, rays_d], 1)] # (h*w, 6) + + self.poses = torch.stack(self.poses) + if not self.is_stack: + self.all_rays = torch.cat(self.all_rays, 0) + self.all_rgbs = torch.cat(self.all_rgbs, 0) + + else: + self.all_rays = torch.stack(self.all_rays, 0) + self.all_rgbs = torch.stack(self.all_rgbs, + 0).reshape(-1, *self.img_wh[::-1], 3) + + def define_transforms(self): + self.transform = T.ToTensor() + + def define_proj_mat(self): + self.proj_mat = self.intrinsics.unsqueeze(0) @ torch.inverse( + self.poses)[:, :3] + + def world2ndc(self, points, lindisp=None): + device = points.device + return (points - self.center.to(device)) / self.radius.to(device) + + def __len__(self): + return len(self.all_rgbs) + + def __getitem__(self, idx): + + if self.split == 'train': # use data in the buffers + sample = {'rays': self.all_rays[idx], 'rgbs': self.all_rgbs[idx]} + + else: # create data for each image separately + + img = self.all_rgbs[idx] + rays = self.all_rays[idx] + mask = self.all_masks[idx] # for quantity evaluation + + sample = {'rays': rays, 'rgbs': img, 'mask': mask} + return sample + + def get_render_pose(self, N_cameras=120): + render_poses = torch.stack([ + pose_spherical(angle, -30.0, 4.0) + for angle in np.linspace(-180, 180, N_cameras + 1)[:-1] + ], 0) + blender2opencv = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], + [0, 0, 0, 1]]) + return render_poses @ torch.Tensor(blender2opencv).float() diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/llff.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/llff.py new file mode 100644 index 0000000000000000000000000000000000000000..5d25e358b50891a5e8470fa55047b7ebeba2a0a1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/llff.py @@ -0,0 +1,278 @@ +import glob +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms as T + +from .ray_utils import * + + +def normalize(v): + """Normalize a vector.""" + return v / np.linalg.norm(v) + + +def average_poses(poses): + """ + Calculate the average pose, which is then used to center all poses + using @center_poses. Its computation is as follows: + 1. Compute the center: the average of pose centers. + 2. Compute the z axis: the normalized average z axis. + 3. Compute axis y': the average y axis. + 4. Compute x' = y' cross product z, then normalize it as the x axis. + 5. Compute the y axis: z cross product x. + + Note that at step 3, we cannot directly use y' as y axis since it's + not necessarily orthogonal to z axis. We need to pass from x to y. + Inputs: + poses: (N_images, 3, 4) + Outputs: + pose_avg: (3, 4) the average pose + """ + # 1. Compute the center + center = poses[..., 3].mean(0) # (3) + + # 2. Compute the z axis + z = normalize(poses[..., 2].mean(0)) # (3) + + # 3. Compute axis y' (no need to normalize as it's not the final output) + y_ = poses[..., 1].mean(0) # (3) + + # 4. Compute the x axis + x = normalize(np.cross(z, y_)) # (3) + + # 5. Compute the y axis (as z and x are normalized, y is already of norm 1) + y = np.cross(x, z) # (3) + + pose_avg = np.stack([x, y, z, center], 1) # (3, 4) + + return pose_avg + + +def center_poses(poses, blender2opencv): + """ + Center the poses so that we can use NDC. + See https://github.com/bmild/nerf/issues/34 + Inputs: + poses: (N_images, 3, 4) + Outputs: + poses_centered: (N_images, 3, 4) the centered poses + pose_avg: (3, 4) the average pose + """ + poses = poses @ blender2opencv + pose_avg = average_poses(poses) # (3, 4) + pose_avg_homo = np.eye(4) + pose_avg_homo[: + 3] = pose_avg # convert to homogeneous coordinate for faster computation + pose_avg_homo = pose_avg_homo + # by simply adding 0, 0, 0, 1 as the last row + last_row = np.tile(np.array([0, 0, 0, 1]), + (len(poses), 1, 1)) # (N_images, 1, 4) + poses_homo = \ + np.concatenate([poses, last_row], 1) # (N_images, 4, 4) homogeneous coordinate + + poses_centered = np.linalg.inv( + pose_avg_homo) @ poses_homo # (N_images, 4, 4) + # poses_centered = poses_centered @ blender2opencv + poses_centered = poses_centered[:, :3] # (N_images, 3, 4) + + return poses_centered, pose_avg_homo + + +def viewmatrix(z, up, pos): + vec2 = normalize(z) + vec1_avg = up + vec0 = normalize(np.cross(vec1_avg, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.eye(4) + m[:3] = np.stack([-vec0, vec1, vec2, pos], 1) + return m + + +def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, N_rots=2, N=120): + render_poses = [] + rads = np.array(list(rads) + [1.]) + + for theta in np.linspace(0., 2. * np.pi * N_rots, N + 1)[:-1]: + c = np.dot( + c2w[:3, :4], + np.array( + [np.cos(theta), -np.sin(theta), -np.sin(theta * zrate), 1.]) + * rads) + z = normalize(c - np.dot(c2w[:3, :4], np.array([0, 0, -focal, 1.]))) + render_poses.append(viewmatrix(z, up, c)) + return render_poses + + +def get_spiral(c2ws_all, near_fars, rads_scale=1.0, N_views=120): + # center pose + c2w = average_poses(c2ws_all) + + # Get average pose + up = normalize(c2ws_all[:, :3, 1].sum(0)) + + # Find a reasonable "focus depth" for this dataset + dt = 0.75 + close_depth, inf_depth = near_fars.min() * 0.9, near_fars.max() * 5.0 + focal = 1.0 / (((1.0 - dt) / close_depth + dt / inf_depth)) + + # Get radii for spiral path + zdelta = near_fars.min() * .2 + tt = c2ws_all[:, :3, 3] + rads = np.percentile(np.abs(tt), 90, 0) * rads_scale + render_poses = render_path_spiral( + c2w, up, rads, focal, zdelta, zrate=.5, N=N_views) + return np.stack(render_poses) + + +class LLFFDataset(Dataset): + + def __init__(self, + datadir, + split='train', + downsample=4, + is_stack=False, + hold_every=8): + """ + spheric_poses: whether the images are taken in a spheric inward-facing manner + default: False (forward-facing) + val_num: number of val images (used for multigpu training, validate same image for all gpus) + """ + + self.root_dir = datadir + self.split = split + self.hold_every = hold_every + self.is_stack = is_stack + self.downsample = downsample + self.define_transforms() + + self.blender2opencv = np.eye(4) + # np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]) + self.read_meta() + self.white_bg = False + + # self.near_far = [np.min(self.near_fars[:,0]),np.max(self.near_fars[:,1])] + self.near_far = [0.0, 1.0] + self.scene_bbox = torch.tensor([[-1.5, -1.67, -1.0], [1.5, 1.67, 1.0]]) + # self.scene_bbox = torch.tensor([[-1.67, -1.5, -1.0], [1.67, 1.5, 1.0]]) + self.center = torch.mean(self.scene_bbox, dim=0).float().view(1, 1, 3) + self.invradius = 1.0 / (self.scene_bbox[1] - self.center).float().view( + 1, 1, 3) + + def read_meta(self): + + poses_bounds = np.load( + os.path.join(self.root_dir, 'poses_bounds.npy')) # (N_images, 17) + self.image_paths = sorted( + glob.glob(os.path.join(self.root_dir, 'images_4/*'))) + # load full resolution image then resize + if self.split in ['train', 'test']: + assert len(poses_bounds) == len(self.image_paths), \ + 'Mismatch between number of images and number of poses! Please rerun COLMAP!' + + poses = poses_bounds[:, :15].reshape(-1, 3, 5) # (N_images, 3, 5) + self.near_fars = poses_bounds[:, -2:] # (N_images, 2) + # hwf = poses[:, :, -1] + + # Step 1: rescale focal length according to training resolution + H, W, self.focal = poses[ + 0, :, -1] # original intrinsics, same for all images + self.img_wh = np.array( + [int(W / self.downsample), + int(H / self.downsample)]) + self.focal = [ + self.focal * self.img_wh[0] / W, self.focal * self.img_wh[1] / H + ] + + # Step 2: correct poses + # Original poses has rotation in form "down right back", change to "right up back" + # See https://github.com/bmild/nerf/issues/34 + poses = np.concatenate( + [poses[..., 1:2], -poses[..., :1], poses[..., 2:4]], -1) + # (N_images, 3, 4) exclude H, W, focal + self.poses, self.pose_avg = center_poses(poses, self.blender2opencv) + + # Step 3: correct scale so that the nearest depth is at a little more than 1.0 + # See https://github.com/bmild/nerf/issues/34 + near_original = self.near_fars.min() + scale_factor = near_original * 0.75 # 0.75 is the default parameter + # the nearest depth is at 1/0.75=1.33 + self.near_fars /= scale_factor + self.poses[..., 3] /= scale_factor + + # build rendering path + N_views = 120 + # N_rots = 2 + # tt = self.poses[:, :3, 3] # ptstocam(poses[:3,3,:].T, c2w).T + # up = normalize(self.poses[:, :3, 1].sum(0)) + # rads = np.percentile(np.abs(tt), 90, 0) + + self.render_path = get_spiral( + self.poses, self.near_fars, N_views=N_views) + + # distances_from_center = np.linalg.norm(self.poses[..., 3], axis=1) + # val_idx = np.argmin(distances_from_center) # choose val image as the closest to + # center image + + # ray directions for all pixels, same for all images (same H, W, focal) + W, H = self.img_wh + self.directions = get_ray_directions_blender(H, W, + self.focal) # (H, W, 3) + + # average_pose = average_poses(self.poses) + # dists = np.sum( + # np.square(average_pose[:3, 3] - self.poses[:, :3, 3]), -1) + i_test = np.arange(0, self.poses.shape[0], + self.hold_every) # [np.argmin(dists)] + img_list = i_test if self.split != 'train' else list( + set(np.arange(len(self.poses))) - set(i_test)) + + # use first N_images-1 to train, the LAST is val + self.all_rays = [] + self.all_rgbs = [] + for i in img_list: + image_path = self.image_paths[i] + c2w = torch.FloatTensor(self.poses[i]) + + img = Image.open(image_path).convert('RGB') + if self.downsample != 1.0: + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (3, h, w) + + img = img.view(3, -1).permute(1, 0) # (h*w, 3) RGB + self.all_rgbs += [img] + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + rays_o, rays_d = ndc_rays_blender(H, W, self.focal[0], 1.0, rays_o, + rays_d) + # viewdir = rays_d / torch.norm(rays_d, dim=-1, keepdim=True) + + self.all_rays += [torch.cat([rays_o, rays_d], 1)] # (h*w, 6) + + if not self.is_stack: + self.all_rays = torch.cat(self.all_rays, + 0) # (len(self.meta['frames])*h*w, 3) + self.all_rgbs = torch.cat(self.all_rgbs, + 0) # (len(self.meta['frames])*h*w,3) + else: + self.all_rays = torch.stack(self.all_rays, + 0) # (len(self.meta['frames]),h,w, 3) + self.all_rgbs = torch.stack(self.all_rgbs, 0).reshape( + -1, *self.img_wh[::-1], 3) # (len(self.meta['frames]),h,w,3) + + def define_transforms(self): + self.transform = T.ToTensor() + + def __len__(self): + return len(self.all_rgbs) + + def __getitem__(self, idx): + + sample = {'rays': self.all_rays[idx], 'rgbs': self.all_rgbs[idx]} + + return sample + + def get_render_pose(self, N_cameras=120): + return get_spiral(self.poses, self.near_fars, N_views=N_cameras) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/nsvf.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/nsvf.py new file mode 100644 index 0000000000000000000000000000000000000000..e3cdbafa82735436e9278da8963337a8b6c39e19 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/nsvf.py @@ -0,0 +1,182 @@ +import os + +import torch +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms as T +from tqdm import tqdm + +from .ray_utils import * + + +def trans_t(t): + return torch.Tensor([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, t], + [0, 0, 0, 1]]).float() + + +def rot_phi(phi): + return torch.Tensor([[1, 0, 0, 0], [0, np.cos(phi), -np.sin(phi), 0], + [0, np.sin(phi), np.cos(phi), 0], [0, 0, 0, + 1]]).float() + + +def rot_theta(th): + return torch.Tensor([[np.cos(th), 0, -np.sin(th), 0], [0, 1, 0, 0], + [np.sin(th), 0, np.cos(th), 0], [0, 0, 0, + 1]]).float() + + +def pose_spherical(theta, phi, radius): + c2w = trans_t(radius) + c2w = rot_phi(phi / 180. * np.pi) @ c2w + c2w = rot_theta(theta / 180. * np.pi) @ c2w + c2w = torch.Tensor( + np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1] + ])) @ c2w + return c2w + + +class NSVF(Dataset): + """NSVF Generic Dataset.""" + + def __init__(self, + datadir, + split='train', + downsample=1.0, + wh=[800, 800], + is_stack=False): + self.root_dir = datadir + self.split = split + self.is_stack = is_stack + self.downsample = downsample + self.img_wh = (int(wh[0] / downsample), int(wh[1] / downsample)) + self.define_transforms() + + self.white_bg = True + self.near_far = [0.5, 6.0] + self.scene_bbox = torch.from_numpy( + np.loadtxt(f'{self.root_dir}/bbox.txt')).float()[:6].view(2, 3) + self.blender2opencv = np.array([[1, 0, 0, 0], [0, -1, 0, 0], + [0, 0, -1, 0], [0, 0, 0, 1]]) + self.read_meta() + self.define_proj_mat() + + self.center = torch.mean(self.scene_bbox, axis=0).float().view(1, 1, 3) + self.radius = (self.scene_bbox[1] - self.center).float().view(1, 1, 3) + + def bbox2corners(self): + corners = self.scene_bbox.unsqueeze(0).repeat(4, 1, 1) + for i in range(3): + corners[i, [0, 1], i] = corners[i, [1, 0], i] + return corners.view(-1, 3) + + def read_meta(self): + with open(os.path.join(self.root_dir, 'intrinsics.txt')) as f: + focal = float(f.readline().split()[0]) + self.intrinsics = np.array([[focal, 0, 400.0], [0, focal, 400.0], + [0, 0, 1]]) + self.intrinsics[:2] *= (np.array(self.img_wh) + / np.array([800, 800])).reshape(2, 1) + + pose_files = sorted(os.listdir(os.path.join(self.root_dir, 'pose'))) + img_files = sorted(os.listdir(os.path.join(self.root_dir, 'rgb'))) + + if self.split == 'train': + pose_files = [x for x in pose_files if x.startswith('0_')] + img_files = [x for x in img_files if x.startswith('0_')] + elif self.split == 'val': + pose_files = [x for x in pose_files if x.startswith('1_')] + img_files = [x for x in img_files if x.startswith('1_')] + elif self.split == 'test': + test_pose_files = [x for x in pose_files if x.startswith('2_')] + test_img_files = [x for x in img_files if x.startswith('2_')] + if len(test_pose_files) == 0: + test_pose_files = [x for x in pose_files if x.startswith('1_')] + test_img_files = [x for x in img_files if x.startswith('1_')] + pose_files = test_pose_files + img_files = test_img_files + + # ray directions for all pixels, same for all images (same H, W, focal) + self.directions = get_ray_directions( + self.img_wh[1], + self.img_wh[0], [self.intrinsics[0, 0], self.intrinsics[1, 1]], + center=self.intrinsics[:2, 2]) # (h, w, 3) + self.directions = self.directions / torch.norm( + self.directions, dim=-1, keepdim=True) + + self.render_path = torch.stack([ + pose_spherical(angle, -30.0, 4.0) + for angle in np.linspace(-180, 180, 40 + 1)[:-1] + ], 0) + + self.poses = [] + self.all_rays = [] + self.all_rgbs = [] + + assert len(img_files) == len(pose_files) + for img_fname, pose_fname in tqdm( + zip(img_files, pose_files), + desc=f'Loading data {self.split} ({len(img_files)})'): + image_path = os.path.join(self.root_dir, 'rgb', img_fname) + img = Image.open(image_path) + if self.downsample != 1.0: + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (4, h, w) + img = img.view(img.shape[0], -1).permute(1, 0) # (h*w, 4) RGBA + if img.shape[-1] == 4: + img = img[:, :3] * img[:, -1:] + (1 - img[:, -1:] + ) # blend A to RGB + self.all_rgbs += [img] + + c2w = np.loadtxt(os.path.join(self.root_dir, 'pose', pose_fname)) + c2w = torch.FloatTensor(c2w) + self.poses.append(c2w) # C2W + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + self.all_rays += [torch.cat([rays_o, rays_d], 1)] # (h*w, 8) + + self.poses = torch.stack(self.poses) + if 'train' == self.split: + if self.is_stack: + self.all_rays = torch.stack(self.all_rays, + 0).reshape(-1, *self.img_wh[::-1], + 6) + self.all_rgbs = torch.stack(self.all_rgbs, + 0).reshape(-1, *self.img_wh[::-1], + 3) + else: + self.all_rays = torch.cat(self.all_rays, 0) + self.all_rgbs = torch.cat(self.all_rgbs, 0) + else: + self.all_rays = torch.stack(self.all_rays, 0) + self.all_rgbs = torch.stack(self.all_rgbs, + 0).reshape(-1, *self.img_wh[::-1], 3) + + def define_transforms(self): + self.transform = T.ToTensor() + + def define_proj_mat(self): + self.proj_mat = torch.from_numpy( + self.intrinsics[:3, :3]).unsqueeze(0).float() @ torch.inverse( + self.poses)[:, :3] + + def world2ndc(self, points): + device = points.device + return (points - self.center.to(device)) / self.radius.to(device) + + def __len__(self): + if self.split == 'train': + return len(self.all_rays) + return len(self.all_rgbs) + + def __getitem__(self, idx): + + if self.split == 'train': # use data in the buffers + sample = {'rays': self.all_rays[idx], 'rgbs': self.all_rgbs[idx]} + + else: # create data for each image separately + + img = self.all_rgbs[idx] + rays = self.all_rays[idx] + + sample = {'rays': rays, 'rgbs': img} + return sample diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/ray_utils.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/ray_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..39869fbc6efbf029988479d28410e626302ead77 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/ray_utils.py @@ -0,0 +1,295 @@ +import re + +import numpy as np +import torch +from kornia import create_meshgrid +from torch import searchsorted + +# from utils import index_point_feature + + +def depth2dist(z_vals, cos_angle): + # z_vals: [N_ray N_sample] + device = z_vals.device + dists = z_vals[..., 1:] - z_vals[..., :-1] + dists = torch.cat( + [dists, + torch.Tensor([1e10]).to(device).expand(dists[..., :1].shape)], + -1) # [N_rays, N_samples] + dists = dists * cos_angle.unsqueeze(-1) + return dists + + +def ndc2dist(ndc_pts, cos_angle): + dists = torch.norm(ndc_pts[:, 1:] - ndc_pts[:, :-1], dim=-1) + dists = torch.cat([dists, 1e10 * cos_angle.unsqueeze(-1)], + -1) # [N_rays, N_samples] + return dists + + +def get_ray_directions(H, W, focal, center=None): + """ + Get ray directions for all pixels in camera coordinate. + Reference: https://www.scratchapixel.com/lessons/3d-basic-rendering/ + ray-tracing-generating-camera-rays/standard-coordinate-systems + Inputs: + H, W, focal: image height, width and focal length + Outputs: + directions: (H, W, 3), the direction of the rays in camera coordinate + """ + grid = create_meshgrid(H, W, normalized_coordinates=False)[0] + 0.5 + + i, j = grid.unbind(-1) + # the direction here is without +0.5 pixel centering as calibration is not so accurate + # see https://github.com/bmild/nerf/issues/24 + cent = center if center is not None else [W / 2, H / 2] + directions = torch.stack([(i - cent[0]) / focal[0], + (j - cent[1]) / focal[1], + torch.ones_like(i)], -1) # (H, W, 3) + + return directions + + +def get_ray_directions_blender(H, W, focal, center=None): + """ + Get ray directions for all pixels in camera coordinate. + Reference: https://www.scratchapixel.com/lessons/3d-basic-rendering/ + ray-tracing-generating-camera-rays/standard-coordinate-systems + Inputs: + H, W, focal: image height, width and focal length + Outputs: + directions: (H, W, 3), the direction of the rays in camera coordinate + """ + grid = create_meshgrid(H, W, normalized_coordinates=False)[0] + 0.5 + i, j = grid.unbind(-1) + # the direction here is without +0.5 pixel centering as calibration is not so accurate + # see https://github.com/bmild/nerf/issues/24 + cent = center if center is not None else [W / 2, H / 2] + directions = torch.stack([(i - cent[0]) / focal[0], + -(j - cent[1]) / focal[1], -torch.ones_like(i)], + -1) # (H, W, 3) + + return directions + + +def get_rays(directions, c2w): + """ + Get ray origin and normalized directions in world coordinate for all pixels in one image. + Reference: https://www.scratchapixel.com/lessons/3d-basic-rendering/ + ray-tracing-generating-camera-rays/standard-coordinate-systems + Inputs: + directions: (H, W, 3) precomputed ray directions in camera coordinate + c2w: (3, 4) transformation matrix from camera coordinate to world coordinate + Outputs: + rays_o: (H*W, 3), the origin of the rays in world coordinate + rays_d: (H*W, 3), the normalized direction of the rays in world coordinate + """ + # Rotate ray directions from camera coordinate to the world coordinate + rays_d = directions @ c2w[:3, :3].T # (H, W, 3) + # rays_d = rays_d / torch.norm(rays_d, dim=-1, keepdim=True) + # The origin of all rays is the camera origin in world coordinate + rays_o = c2w[:3, 3].expand(rays_d.shape) # (H, W, 3) + + rays_d = rays_d.view(-1, 3) + rays_o = rays_o.view(-1, 3) + + return rays_o, rays_d + + +def ndc_rays_blender(H, W, focal, near, rays_o, rays_d): + # Shift ray origins to near plane + t = -(near + rays_o[..., 2]) / rays_d[..., 2] + rays_o = rays_o + t[..., None] * rays_d + + # Projection + o0 = -1. / (W / (2. * focal)) * rays_o[..., 0] / rays_o[..., 2] + o1 = -1. / (H / (2. * focal)) * rays_o[..., 1] / rays_o[..., 2] + o2 = 1. + 2. * near / rays_o[..., 2] + + d0 = -1. / (W / (2. * focal)) * ( + rays_d[..., 0] / rays_d[..., 2] - rays_o[..., 0] / rays_o[..., 2]) + d1 = -1. / (H / (2. * focal)) * ( + rays_d[..., 1] / rays_d[..., 2] - rays_o[..., 1] / rays_o[..., 2]) + d2 = -2. * near / rays_o[..., 2] + + rays_o = torch.stack([o0, o1, o2], -1) + rays_d = torch.stack([d0, d1, d2], -1) + + return rays_o, rays_d + + +def ndc_rays(H, W, focal, near, rays_o, rays_d): + # Shift ray origins to near plane + t = (near - rays_o[..., 2]) / rays_d[..., 2] + rays_o = rays_o + t[..., None] * rays_d + + # Projection + o0 = 1. / (W / (2. * focal)) * rays_o[..., 0] / rays_o[..., 2] + o1 = 1. / (H / (2. * focal)) * rays_o[..., 1] / rays_o[..., 2] + o2 = 1. - 2. * near / rays_o[..., 2] + + d0 = 1. / (W / (2. * focal)) * ( + rays_d[..., 0] / rays_d[..., 2] - rays_o[..., 0] / rays_o[..., 2]) + d1 = 1. / (H / (2. * focal)) * ( + rays_d[..., 1] / rays_d[..., 2] - rays_o[..., 1] / rays_o[..., 2]) + d2 = 2. * near / rays_o[..., 2] + + rays_o = torch.stack([o0, o1, o2], -1) + rays_d = torch.stack([d0, d1, d2], -1) + + return rays_o, rays_d + + +# Hierarchical sampling (section 5.2) +def sample_pdf(bins, weights, N_samples, det=False, pytest=False): + device = weights.device + # Get pdf + weights = weights + 1e-5 # prevent nans + pdf = weights / torch.sum(weights, -1, keepdim=True) + cdf = torch.cumsum(pdf, -1) + cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], + -1) # (batch, len(bins)) + + # Take uniform samples + if det: + u = torch.linspace(0., 1., steps=N_samples, device=device) + u = u.expand(list(cdf.shape[:-1]) + [N_samples]) + else: + u = torch.rand(list(cdf.shape[:-1]) + [N_samples], device=device) + + # Pytest, overwrite u with numpy's fixed random numbers + if pytest: + np.random.seed(0) + new_shape = list(cdf.shape[:-1]) + [N_samples] + if det: + u = np.linspace(0., 1., N_samples) + u = np.broadcast_to(u, new_shape) + else: + u = np.random.rand(*new_shape) + u = torch.Tensor(u) + + # Invert CDF + u = u.contiguous() + inds = searchsorted(cdf.detach(), u, right=True) + below = torch.max(torch.zeros_like(inds - 1), inds - 1) + above = torch.min((cdf.shape[-1] - 1) * torch.ones_like(inds), inds) + inds_g = torch.stack([below, above], -1) # (batch, N_samples, 2) + + matched_shape = [inds_g.shape[0], inds_g.shape[1], cdf.shape[-1]] + cdf_g = torch.gather(cdf.unsqueeze(1).expand(matched_shape), 2, inds_g) + bins_g = torch.gather(bins.unsqueeze(1).expand(matched_shape), 2, inds_g) + + denom = (cdf_g[..., 1] - cdf_g[..., 0]) + denom = torch.where(denom < 1e-5, torch.ones_like(denom), denom) + t = (u - cdf_g[..., 0]) / denom + samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0]) + + return samples + + +def dda(rays_o, rays_d, bbox_3D): + inv_ray_d = 1.0 / (rays_d + 1e-6) + t_min = (bbox_3D[:1] - rays_o) * inv_ray_d # N_rays 3 + t_max = (bbox_3D[1:] - rays_o) * inv_ray_d + t = torch.stack((t_min, t_max)) # 2 N_rays 3 + t_min = torch.max(torch.min(t, dim=0)[0], dim=-1, keepdim=True)[0] + t_max = torch.min(torch.max(t, dim=0)[0], dim=-1, keepdim=True)[0] + return t_min, t_max + + +def ray_marcher(rays, N_samples=64, lindisp=False, perturb=0, bbox_3D=None): + """ + sample points along the rays + Inputs: + rays: () + + Returns: + + """ + + # Decompose the inputs + N_rays = rays.shape[0] + rays_o, rays_d = rays[:, 0:3], rays[:, 3:6] # both (N_rays, 3) + near, far = rays[:, 6:7], rays[:, 7:8] # both (N_rays, 1) + + if bbox_3D is not None: + # cal aabb boundles + near, far = dda(rays_o, rays_d, bbox_3D) + + # Sample depth points + z_steps = torch.linspace( + 0, 1, N_samples, device=rays.device) # (N_samples) + if not lindisp: # use linear sampling in depth space + z_vals = near * (1 - z_steps) + far * z_steps + else: # use linear sampling in disparity space + z_vals = 1 / (1 / near * (1 - z_steps) + 1 / far * z_steps) + + z_vals = z_vals.expand(N_rays, N_samples) + + if perturb > 0: # perturb sampling depths (z_vals) + z_vals_mid = 0.5 * (z_vals[:, :-1] + z_vals[:, 1:] + ) # (N_rays, N_samples-1) interval mid points + # get intervals between samples + upper = torch.cat([z_vals_mid, z_vals[:, -1:]], -1) + lower = torch.cat([z_vals[:, :1], z_vals_mid], -1) + + perturb_rand = perturb * torch.rand(z_vals.shape, device=rays.device) + z_vals = lower + (upper - lower) * perturb_rand + + # (N_rays, N_samples, 3) + xyz_coarse_sampled = rays_o.unsqueeze( + 1) + rays_d.unsqueeze(1) * z_vals.unsqueeze(2) + + return xyz_coarse_sampled, rays_o, rays_d, z_vals + + +def read_pfm(filename): + file = open(filename, 'rb') + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().decode('utf-8').rstrip() + if header == 'PF': + color = True + elif header == 'Pf': + color = False + else: + raise Exception('Not a PFM file.') + + dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode('utf-8')) + if dim_match: + width, height = map(int, dim_match.groups()) + else: + raise Exception('Malformed PFM header.') + + scale = float(file.readline().rstrip()) + if scale < 0: # little-endian + endian = '<' + scale = -scale + else: + endian = '>' # big-endian + + data = np.fromfile(file, endian + 'f') + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + file.close() + return data, scale + + +def ndc_bbox(all_rays): + near_min = torch.min(all_rays[..., :3].view(-1, 3), dim=0)[0] + near_max = torch.max(all_rays[..., :3].view(-1, 3), dim=0)[0] + far_min = torch.min( + (all_rays[..., :3] + all_rays[..., 3:6]).view(-1, 3), dim=0)[0] + far_max = torch.max( + (all_rays[..., :3] + all_rays[..., 3:6]).view(-1, 3), dim=0)[0] + print( + f'===> ndc bbox near_min:{near_min} near_max:{near_max} far_min:{far_min} far_max:{far_max}' + ) + return torch.stack( + (torch.minimum(near_min, far_min), torch.maximum(near_max, far_max))) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/tankstemple.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/tankstemple.py new file mode 100644 index 0000000000000000000000000000000000000000..ba6aa717e7fe4d49dbab5464099b8e6066f4b6b8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/dataloader/tankstemple.py @@ -0,0 +1,249 @@ +import os + +import torch +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms as T +from tqdm import tqdm + +from .ray_utils import * + + +def circle(radius=3.5, h=0.0, axis='z', t0=0, r=1): + if axis == 'z': + return lambda t: [ + radius * np.cos(r * t + t0), radius * np.sin(r * t + t0), h + ] + elif axis == 'y': + return lambda t: [ + radius * np.cos(r * t + t0), h, radius * np.sin(r * t + t0) + ] + else: + return lambda t: [ + h, radius * np.cos(r * t + t0), radius * np.sin(r * t + t0) + ] + + +def cross(x, y, axis=0): + T = torch if isinstance(x, torch.Tensor) else np + return T.cross(x, y, axis) + + +def normalize(x, axis=-1, order=2): + if isinstance(x, torch.Tensor): + l2 = x.norm(p=order, dim=axis, keepdim=True) + return x / (l2 + 1e-8), l2 + + else: + l2 = np.linalg.norm(x, order, axis) + l2 = np.expand_dims(l2, axis) + l2[l2 == 0] = 1 + return x / l2, + + +def cat(x, axis=1): + if isinstance(x[0], torch.Tensor): + return torch.cat(x, dim=axis) + return np.concatenate(x, axis=axis) + + +def look_at_rotation(camera_position, + at=None, + up=None, + inverse=False, + cv=False): + """ + This function takes a vector 'camera_position' which specifies the location + of the camera in world coordinates and two vectors `at` and `up` which + indicate the position of the object and the up directions of the world + coordinate system respectively. The object is assumed to be centered at + the origin. + The output is a rotation matrix representing the transformation + from world coordinates -> view coordinates. + Input: + camera_position: 3 + at: 1 x 3 or N x 3 (0, 0, 0) in default + up: 1 x 3 or N x 3 (0, 1, 0) in default + """ + + if at is None: + at = torch.zeros_like(camera_position) + else: + at = torch.tensor(at).type_as(camera_position) + if up is None: + up = torch.zeros_like(camera_position) + up[2] = -1 + else: + up = torch.tensor(up).type_as(camera_position) + + z_axis = normalize(at - camera_position)[0] + x_axis = normalize(cross(up, z_axis))[0] + y_axis = normalize(cross(z_axis, x_axis))[0] + + R = cat([x_axis[:, None], y_axis[:, None], z_axis[:, None]], axis=1) + return R + + +def gen_path(pos_gen, at=(0, 0, 0), up=(0, -1, 0), frames=180): + c2ws = [] + for t in range(frames): + c2w = torch.eye(4) + cam_pos = torch.tensor(pos_gen(t * (360.0 / frames) / 180 * np.pi)) + cam_rot = look_at_rotation( + cam_pos, at=at, up=up, inverse=False, cv=True) + c2w[:3, 3], c2w[:3, :3] = cam_pos, cam_rot + c2ws.append(c2w) + return torch.stack(c2ws) + + +class TanksTempleDataset(Dataset): + """NSVF Generic Dataset.""" + + def __init__(self, + datadir, + split='train', + downsample=1.0, + wh=[1920, 1080], + is_stack=False): + self.root_dir = datadir + self.split = split + self.is_stack = is_stack + self.downsample = downsample + self.img_wh = (int(wh[0] / downsample), int(wh[1] / downsample)) + self.define_transforms() + + self.white_bg = True + self.near_far = [0.01, 6.0] + self.scene_bbox = torch.from_numpy( + np.loadtxt(f'{self.root_dir}/bbox.txt')).float()[:6].view(2, + 3) * 1.2 + + self.blender2opencv = np.array([[1, 0, 0, 0], [0, -1, 0, 0], + [0, 0, -1, 0], [0, 0, 0, 1]]) + self.read_meta() + self.define_proj_mat() + + self.center = torch.mean(self.scene_bbox, axis=0).float().view(1, 1, 3) + self.radius = (self.scene_bbox[1] - self.center).float().view(1, 1, 3) + + def bbox2corners(self): + corners = self.scene_bbox.unsqueeze(0).repeat(4, 1, 1) + for i in range(3): + corners[i, [0, 1], i] = corners[i, [1, 0], i] + return corners.view(-1, 3) + + def read_meta(self): + + self.intrinsics = np.loadtxt( + os.path.join(self.root_dir, 'intrinsics.txt')) + self.intrinsics[:2] *= (np.array(self.img_wh) + / np.array([1920, 1080])).reshape(2, 1) + pose_files = sorted(os.listdir(os.path.join(self.root_dir, 'pose'))) + img_files = sorted(os.listdir(os.path.join(self.root_dir, 'rgb'))) + + if self.split == 'train': + pose_files = [x for x in pose_files if x.startswith('0_')] + img_files = [x for x in img_files if x.startswith('0_')] + elif self.split == 'val': + pose_files = [x for x in pose_files if x.startswith('1_')] + img_files = [x for x in img_files if x.startswith('1_')] + elif self.split == 'test': + test_pose_files = [x for x in pose_files if x.startswith('2_')] + test_img_files = [x for x in img_files if x.startswith('2_')] + if len(test_pose_files) == 0: + test_pose_files = [x for x in pose_files if x.startswith('1_')] + test_img_files = [x for x in img_files if x.startswith('1_')] + pose_files = test_pose_files + img_files = test_img_files + + # ray directions for all pixels, same for all images (same H, W, focal) + self.directions = get_ray_directions( + self.img_wh[1], + self.img_wh[0], [self.intrinsics[0, 0], self.intrinsics[1, 1]], + center=self.intrinsics[:2, 2]) # (h, w, 3) + self.directions = self.directions / torch.norm( + self.directions, dim=-1, keepdim=True) + + self.poses = [] + self.all_rays = [] + self.all_rgbs = [] + + assert len(img_files) == len(pose_files) + for img_fname, pose_fname in tqdm( + zip(img_files, pose_files), + desc=f'Loading data {self.split} ({len(img_files)})'): + image_path = os.path.join(self.root_dir, 'rgb', img_fname) + img = Image.open(image_path) + if self.downsample != 1.0: + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (4, h, w) + img = img.view(img.shape[0], -1).permute(1, 0) # (h*w, 4) RGBA + if img.shape[-1] == 4: + img = img[:, :3] * img[:, -1:] + (1 - img[:, -1:] + ) # blend A to RGB + self.all_rgbs.append(img) + + c2w = np.loadtxt(os.path.join(self.root_dir, 'pose', + pose_fname)) # @ cam_trans + c2w = torch.FloatTensor(c2w) + self.poses.append(c2w) # C2W + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + self.all_rays += [torch.cat([rays_o, rays_d], 1)] # (h*w, 8) + + self.poses = torch.stack(self.poses) + + center = torch.mean(self.scene_bbox, dim=0) + radius = torch.norm(self.scene_bbox[1] - center) * 1.2 + up = torch.mean(self.poses[:, :3, 1], dim=0).tolist() + pos_gen = circle(radius=radius, h=-0.2 * up[1], axis='y') + self.render_path = gen_path(pos_gen, up=up, frames=200) + self.render_path[:, :3, 3] += center + + if 'train' == self.split: + if self.is_stack: + self.all_rays = torch.stack(self.all_rays, 0).reshape( + -1, *self.img_wh[::-1], + 6) # (len(self.meta['frames])*h*w, 3) + self.all_rgbs = torch.stack(self.all_rgbs, 0).reshape( + -1, *self.img_wh[::-1], + 3) # (len(self.meta['frames])*h*w, 3) + else: + self.all_rays = torch.cat( + self.all_rays, 0) # (len(self.meta['frames])*h*w, 3) + self.all_rgbs = torch.cat( + self.all_rgbs, 0) # (len(self.meta['frames])*h*w, 3) + else: + self.all_rays = torch.stack(self.all_rays, + 0) # (len(self.meta['frames]),h*w, 3) + self.all_rgbs = torch.stack(self.all_rgbs, 0).reshape( + -1, *self.img_wh[::-1], 3) # (len(self.meta['frames]),h,w,3) + + def define_transforms(self): + self.transform = T.ToTensor() + + def define_proj_mat(self): + self.proj_mat = torch.from_numpy( + self.intrinsics[:3, :3]).unsqueeze(0).float() @ torch.inverse( + self.poses)[:, :3] + + def world2ndc(self, points): + device = points.device + return (points - self.center.to(device)) / self.radius.to(device) + + def __len__(self): + if self.split == 'train': + return len(self.all_rays) + return len(self.all_rgbs) + + def __getitem__(self, idx): + + if self.split == 'train': # use data in the buffers + sample = {'rays': self.all_rays[idx], 'rgbs': self.all_rgbs[idx]} + + else: # create data for each image separately + + img = self.all_rgbs[idx] + rays = self.all_rays[idx] + + sample = {'rays': rays, 'rgbs': img} + return sample diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/nerf_recon_vq_compression.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/nerf_recon_vq_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..041a7af89d878f67d8f8369ba04f3b37ceb5fccb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/nerf_recon_vq_compression.py @@ -0,0 +1,116 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import glob +import os +import time +from functools import partial + +import cv2 +import numpy as np +import torch +import tqdm + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .dataloader import dataset_dict +from .network.tensoRF import TensorVM, raw2alpha +from .network.tensoRF_VQ import TensorVMSplitVQ +from .renderer import OctreeRender_trilinear_fast +from .renderer import evaluation as evaluation_render +from .renderer import render_path + +logger = get_logger() + +__all__ = ['NeRFReconVQCompression'] + + +@MODELS.register_module( + Tasks.nerf_recon_vq_compression, + module_name=Models.nerf_recon_vq_compression) +class NeRFReconVQCompression(TorchModel): + + def __init__(self, model_dir=None, **kwargs): + super().__init__(model_dir, **kwargs) + + if not torch.cuda.is_available(): + raise Exception('GPU is required') + + self.device = torch.device('cuda') + self.data_type = kwargs['dataset_name'] + self.data_dir = kwargs['data_dir'] + self.downsample = kwargs['downsample'] + self.ndc_ray = kwargs['ndc_ray'] + self.ckpt_path = os.path.join(model_dir, kwargs['ckpt_path']) + + if self.ckpt_path == '' or self.ckpt_path is None: + self.ckpt_path = os.path.join(model_dir, 'ficus_demo.pt') + if not os.path.exists(self.ckpt_path): + raise Exception('ckpt path not found') + + # load model + ckpt = torch.load(self.ckpt_path, map_location=self.device) + model_kwargs = ckpt['kwargs'] + model_kwargs['device'] = self.device + self.model = TensorVMSplitVQ(**model_kwargs) + self.model.extreme_load(ckpt) + + self.renderer = OctreeRender_trilinear_fast + + # load data + dataset = dataset_dict[self.data_type] + self.test_dataset = dataset( + self.data_dir, + split='test', + downsample=self.downsample, + is_stack=True) + + def evaluation(self, render_dir, N_vis=-1): + white_bg = self.test_dataset.white_bg + ndc_ray = self.ndc_ray + evaluation_test = partial( + evaluation_render, + test_dataset=self.test_dataset, + renderer=self.renderer, + white_bg=white_bg, + ndc_ray=ndc_ray, + device=self.device, + compute_extra_metrics=True, + im_save=True) + + logfolder = render_dir + os.makedirs(f'{logfolder}/evalution_test', exist_ok=True) + PSNRs = evaluation_test( + tensorf=self.model, + N_vis=N_vis, + savePath=f'{logfolder}/evalution_test') + logger.info( + f'VQRF-Evaluation: {self.ckpt_path} mean PSNR: {np.mean(PSNRs)}') + + def render_path(self, render_dir, N_vis=120): + white_bg = self.test_dataset.white_bg + ndc_ray = self.ndc_ray + + logfolder = render_dir + os.makedirs(f'{logfolder}/render_path', exist_ok=True) + + render_poses = self.get_render_pose(N_cameras=N_vis) + render_path( + self.test_dataset, + self.model, + render_poses, + self.renderer, + savePath=f'{logfolder}/render_path', + white_bg=white_bg, + ndc_ray=ndc_ray, + device=self.device) + logger.info( + f'VQRF-Render: {self.ckpt_path} render path video result saved in {logfolder}/render_path' + ) + + def get_render_pose(self, N_cameras=120): + if self.data_type == 'blender': + return self.test_dataset.get_render_pose(N_cameras=N_cameras) + elif self.data_type == 'llff': + return self.test_dataset.get_render_pose(N_cameras=N_cameras) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/__init__.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4bd3630b460daab7b69f45e9d1994b85026f7638 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/__init__.py @@ -0,0 +1,2 @@ +from .tensorBase import * +from .tensoRF import TensorVMSplit diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF.py new file mode 100644 index 0000000000000000000000000000000000000000..7ec0d867601e2346f36c00b7e3c88723fa582675 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF.py @@ -0,0 +1,579 @@ +from .tensorBase import * + + +class TensorVM(TensorBase): + + def __init__(self, aabb, gridSize, device, **kargs): + super(TensorVM, self).__init__(aabb, gridSize, device, **kargs) + + def init_svd_volume(self, res, device): + self.plane_coef = torch.nn.Parameter(0.1 * torch.randn( + (3, self.app_n_comp + self.density_n_comp, res, res), + device=device)) + self.line_coef = torch.nn.Parameter(0.1 * torch.randn( + (3, self.app_n_comp + self.density_n_comp, res, 1), device=device)) + self.basis_mat = torch.nn.Linear( + self.app_n_comp * 3, self.app_dim, bias=False, device=device) + + def get_optparam_groups(self, + lr_init_spatialxyz=0.02, + lr_init_network=0.001): + grad_vars = [{ + 'params': self.line_coef, + 'lr': lr_init_spatialxyz + }, { + 'params': self.plane_coef, + 'lr': lr_init_spatialxyz + }, { + 'params': self.basis_mat.parameters(), + 'lr': lr_init_network + }] + if isinstance(self.renderModule, torch.nn.Module): + grad_vars += [{ + 'params': self.renderModule.parameters(), + 'lr': lr_init_network + }] + return grad_vars + + def compute_features(self, xyz_sampled): + + coordinate_plane = torch.stack( + (xyz_sampled[..., self.matMode[0]], xyz_sampled[..., + self.matMode[1]], + xyz_sampled[..., self.matMode[2]])).detach() + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach() + + plane_feats = F.grid_sample( + self.plane_coef[:, -self.density_n_comp:], + coordinate_plane, + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_feats = F.grid_sample( + self.line_coef[:, -self.density_n_comp:], + coordinate_line, + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + + sigma_feature = torch.sum(plane_feats * line_feats, dim=0) + + plane_feats = F.grid_sample( + self.plane_coef[:, :self.app_n_comp], + coordinate_plane, + align_corners=True).view(3 * self.app_n_comp, -1) + line_feats = F.grid_sample( + self.line_coef[:, :self.app_n_comp], + coordinate_line, + align_corners=True).view(3 * self.app_n_comp, -1) + + app_features = self.basis_mat((plane_feats * line_feats).T) + + return sigma_feature, app_features + + def compute_densityfeature(self, xyz_sampled): + coordinate_plane = torch.stack( + (xyz_sampled[..., self.matMode[0]], xyz_sampled[..., + self.matMode[1]], + xyz_sampled[..., self.matMode[2]])).detach().view(3, -1, 1, 2) + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + + plane_feats = F.grid_sample( + self.plane_coef[:, -self.density_n_comp:], + coordinate_plane, + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_feats = F.grid_sample( + self.line_coef[:, -self.density_n_comp:], + coordinate_line, + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + + sigma_feature = torch.sum(plane_feats * line_feats, dim=0) + + return sigma_feature + + def compute_appfeature(self, xyz_sampled): + coordinate_plane = torch.stack( + (xyz_sampled[..., self.matMode[0]], xyz_sampled[..., + self.matMode[1]], + xyz_sampled[..., self.matMode[2]])).detach().view(3, -1, 1, 2) + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + + plane_feats = F.grid_sample( + self.plane_coef[:, :self.app_n_comp], + coordinate_plane, + align_corners=True).view(3 * self.app_n_comp, -1) + line_feats = F.grid_sample( + self.line_coef[:, :self.app_n_comp], + coordinate_line, + align_corners=True).view(3 * self.app_n_comp, -1) + + app_features = self.basis_mat((plane_feats * line_feats).T) + + return app_features + + def vectorDiffs(self, vector_comps): + total = 0 + + for idx in range(len(vector_comps)): + n_comp, n_size = vector_comps[idx].shape[:-1] + + dotp = torch.matmul( + vector_comps[idx].view(n_comp, n_size), + vector_comps[idx].view(n_comp, n_size).transpose(-1, -2)) + non_diagonal = dotp.view(-1)[1:].view(n_comp - 1, + n_comp + 1)[..., :-1] + total = total + torch.mean(torch.abs(non_diagonal)) + return total + + def vector_comp_diffs(self): + + return self.vectorDiffs( + self.line_coef[:, -self.density_n_comp:]) + self.vectorDiffs( + self.line_coef[:, :self.app_n_comp]) + + @torch.no_grad() + def up_sampling_VM(self, plane_coef, line_coef, res_target): + + for i in range(len(self.vecMode)): + vec_id = self.vecMode[i] + mat_id_0, mat_id_1 = self.matMode[i] + + plane_coef[i] = torch.nn.Parameter( + F.interpolate( + plane_coef[i].data, + size=(res_target[mat_id_1], res_target[mat_id_0]), + mode='bilinear', + align_corners=True)) + line_coef[i] = torch.nn.Parameter( + F.interpolate( + line_coef[i].data, + size=(res_target[vec_id], 1), + mode='bilinear', + align_corners=True)) + return plane_coef, line_coef + + @torch.no_grad() + def upsample_volume_grid(self, res_target): + # assuming xyz have the same scale + scale = res_target[0] / self.line_coef.shape[2] + plane_coef = F.interpolate( + self.plane_coef.detach().data, + scale_factor=scale, + mode='bilinear', + align_corners=True) + line_coef = F.interpolate( + self.line_coef.detach().data, + size=(res_target[0], 1), + mode='bilinear', + align_corners=True) + self.plane_coef, self.line_coef = torch.nn.Parameter( + plane_coef), torch.nn.Parameter(line_coef) + self.compute_stepSize(res_target) + print(f'upsamping to {res_target}') + + +class TensorVMSplit(TensorBase): + + def __init__(self, aabb, gridSize, device, **kargs): + super(TensorVMSplit, self).__init__(aabb, gridSize, device, **kargs) + + def init_svd_volume(self, res, device): + self.density_plane, self.density_line = self.init_one_svd( + self.density_n_comp, self.gridSize, 0.1, device) + self.app_plane, self.app_line = self.init_one_svd( + self.app_n_comp, self.gridSize, 0.1, device) + self.basis_mat = torch.nn.Linear( + sum(self.app_n_comp), self.app_dim, bias=False).to(device) + print(self.basis_mat) + + def init_one_svd(self, n_component, gridSize, scale, device): + plane_coef, line_coef = [], [] + for i in range(len(self.vecMode)): + vec_id = self.vecMode[i] + mat_id_0, mat_id_1 = self.matMode[i] + plane_coef.append( + torch.nn.Parameter(scale * torch.randn( + (1, n_component[i], gridSize[mat_id_1], + gridSize[mat_id_0])))) # + line_coef.append( + torch.nn.Parameter(scale * torch.randn( + (1, n_component[i], gridSize[vec_id], 1)))) + + return torch.nn.ParameterList(plane_coef).to( + device), torch.nn.ParameterList(line_coef).to(device) + + def get_optparam_groups(self, + lr_init_spatialxyz=0.02, + lr_init_network=0.001): + grad_vars = [{ + 'params': self.density_line, + 'lr': lr_init_spatialxyz + }, { + 'params': self.density_plane, + 'lr': lr_init_spatialxyz + }, { + 'params': self.app_line, + 'lr': lr_init_spatialxyz + }, { + 'params': self.app_plane, + 'lr': lr_init_spatialxyz + }, { + 'params': self.basis_mat.parameters(), + 'lr': lr_init_network + }] + if isinstance(self.renderModule, torch.nn.Module): + grad_vars += [{ + 'params': self.renderModule.parameters(), + 'lr': lr_init_network + }] + return grad_vars + + def vectorDiffs(self, vector_comps): + total = 0 + + for idx in range(len(vector_comps)): + n_comp, n_size = vector_comps[idx].shape[1:-1] + + dotp = torch.matmul( + vector_comps[idx].view(n_comp, n_size), + vector_comps[idx].view(n_comp, n_size).transpose(-1, -2)) + non_diagonal = dotp.view(-1)[1:].view(n_comp - 1, + n_comp + 1)[..., :-1] + total = total + torch.mean(torch.abs(non_diagonal)) + return total + + def vector_comp_diffs(self): + return self.vectorDiffs(self.density_line) + self.vectorDiffs( + self.app_line) + + def density_L1(self): + total = 0 + for idx in range(len(self.density_plane)): + total = total + torch.mean(torch.abs( + self.density_plane[idx])) + torch.mean( + torch.abs(self.density_line[idx])) + return total + + def TV_loss_density(self, reg): + total = 0 + for idx in range(len(self.density_plane)): + total = total + reg(self.density_plane[idx]) * 1e-2 + return total + + def TV_loss_app(self, reg): + total = 0 + for idx in range(len(self.app_plane)): + total = total + reg(self.app_plane[idx]) * 1e-2 + return total + + def compute_densityfeature(self, xyz_sampled): + + # plane + line basis + coordinate_plane = torch.stack( + (xyz_sampled[..., self.matMode[0]], xyz_sampled[..., + self.matMode[1]], + xyz_sampled[..., self.matMode[2]])).detach().view(3, -1, 1, 2) + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + + sigma_feature = torch.zeros((xyz_sampled.shape[0], ), + device=xyz_sampled.device) + for idx_plane in range(len(self.density_plane)): + plane_coef_point = F.grid_sample( + self.density_plane[idx_plane], + coordinate_plane[[idx_plane]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_coef_point = F.grid_sample( + self.density_line[idx_plane], + coordinate_line[[idx_plane]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + sigma_feature = sigma_feature + torch.sum( + plane_coef_point * line_coef_point, dim=0) + + return sigma_feature + + def compute_appfeature(self, xyz_sampled): + + # plane + line basis + coordinate_plane = torch.stack( + (xyz_sampled[..., self.matMode[0]], xyz_sampled[..., + self.matMode[1]], + xyz_sampled[..., self.matMode[2]])).detach().view(3, -1, 1, 2) + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + # import ipdb; ipdb.set_trace() + plane_coef_point, line_coef_point = [], [] + for idx_plane in range(len(self.app_plane)): + plane_coef_point.append( + F.grid_sample( + self.app_plane[idx_plane], + coordinate_plane[[idx_plane]], + align_corners=True).view(-1, *xyz_sampled.shape[:1])) + line_coef_point.append( + F.grid_sample( + self.app_line[idx_plane], + coordinate_line[[idx_plane]], + align_corners=True).view(-1, *xyz_sampled.shape[:1])) + plane_coef_point, line_coef_point = torch.cat( + plane_coef_point), torch.cat(line_coef_point) + + return self.basis_mat((plane_coef_point * line_coef_point).T) + + @torch.no_grad() + def up_sampling_VM(self, plane_coef, line_coef, res_target): + + for i in range(len(self.vecMode)): + vec_id = self.vecMode[i] + mat_id_0, mat_id_1 = self.matMode[i] + plane_coef[i] = torch.nn.Parameter( + F.interpolate( + plane_coef[i].data, + size=(res_target[mat_id_1], res_target[mat_id_0]), + mode='bilinear', + align_corners=True)) + line_coef[i] = torch.nn.Parameter( + F.interpolate( + line_coef[i].data, + size=(res_target[vec_id], 1), + mode='bilinear', + align_corners=True)) + + return plane_coef, line_coef + + @torch.no_grad() + def upsample_volume_grid(self, res_target): + self.app_plane, self.app_line = self.up_sampling_VM( + self.app_plane, self.app_line, res_target) + self.density_plane, self.density_line = self.up_sampling_VM( + self.density_plane, self.density_line, res_target) + + self.update_stepSize(res_target) + print(f'upsamping to {res_target}') + + @torch.no_grad() + def shrink(self, new_aabb): + print('====> shrinking ...') + xyz_min, xyz_max = new_aabb + t_l, b_r = (xyz_min - self.aabb[0]) / self.units, ( + xyz_max - self.aabb[0]) / self.units + # print(new_aabb, self.aabb) + # print(t_l, b_r,self.alphaMask.alpha_volume.shape) + t_l, b_r = torch.round( + torch.round(t_l)).long(), torch.round(b_r).long() + 1 + b_r = torch.stack([b_r, self.gridSize]).amin(0) + + for i in range(len(self.vecMode)): + mode0 = self.vecMode[i] + self.density_line[i] = torch.nn.Parameter( + self.density_line[i].data[..., t_l[mode0]:b_r[mode0], :]) + self.app_line[i] = torch.nn.Parameter( + self.app_line[i].data[..., t_l[mode0]:b_r[mode0], :]) + mode0, mode1 = self.matMode[i] + self.density_plane[i] = torch.nn.Parameter( + self.density_plane[i].data[..., t_l[mode1]:b_r[mode1], + t_l[mode0]:b_r[mode0]]) + self.app_plane[i] = torch.nn.Parameter( + self.app_plane[i].data[..., t_l[mode1]:b_r[mode1], + t_l[mode0]:b_r[mode0]]) + + if not torch.all(self.alphaMask.gridSize == self.gridSize): + t_l_r, b_r_r = t_l / (self.gridSize - 1), (b_r - 1) / ( + self.gridSize - 1) + correct_aabb = torch.zeros_like(new_aabb) + correct_aabb[0] = (1 - t_l_r) * self.aabb[0] + t_l_r * self.aabb[1] + correct_aabb[1] = (1 - b_r_r) * self.aabb[0] + b_r_r * self.aabb[1] + print('aabb', new_aabb, '\ncorrect aabb', correct_aabb) + new_aabb = correct_aabb + + newSize = b_r - t_l + self.aabb = new_aabb + self.update_stepSize((newSize[0], newSize[1], newSize[2])) + + +class TensorCP(TensorBase): + + def __init__(self, aabb, gridSize, device, **kargs): + super(TensorCP, self).__init__(aabb, gridSize, device, **kargs) + + def init_svd_volume(self, res, device): + self.density_line = self.init_one_svd(self.density_n_comp[0], + self.gridSize, 0.2, device) + self.app_line = self.init_one_svd(self.app_n_comp[0], self.gridSize, + 0.2, device) + self.basis_mat = torch.nn.Linear( + self.app_n_comp[0], self.app_dim, bias=False).to(device) + + def init_one_svd(self, n_component, gridSize, scale, device): + line_coef = [] + for i in range(len(self.vecMode)): + vec_id = self.vecMode[i] + line_coef.append( + torch.nn.Parameter(scale * torch.randn( + (1, n_component, gridSize[vec_id], 1)))) + return torch.nn.ParameterList(line_coef).to(device) + + def get_optparam_groups(self, + lr_init_spatialxyz=0.02, + lr_init_network=0.001): + grad_vars = [{ + 'params': self.density_line, + 'lr': lr_init_spatialxyz + }, { + 'params': self.app_line, + 'lr': lr_init_spatialxyz + }, { + 'params': self.basis_mat.parameters(), + 'lr': lr_init_network + }] + if isinstance(self.renderModule, torch.nn.Module): + grad_vars += [{ + 'params': self.renderModule.parameters(), + 'lr': lr_init_network + }] + return grad_vars + + def compute_densityfeature(self, xyz_sampled): + + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + + line_coef_point = F.grid_sample( + self.density_line[0], coordinate_line[[0]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_coef_point = line_coef_point * F.grid_sample( + self.density_line[1], coordinate_line[[1]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_coef_point = line_coef_point * F.grid_sample( + self.density_line[2], coordinate_line[[2]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + sigma_feature = torch.sum(line_coef_point, dim=0) + + return sigma_feature + + def compute_appfeature(self, xyz_sampled): + + coordinate_line = torch.stack( + (xyz_sampled[..., self.vecMode[0]], + xyz_sampled[..., self.vecMode[1]], xyz_sampled[..., + self.vecMode[2]])) + coordinate_line = torch.stack( + (torch.zeros_like(coordinate_line), coordinate_line), + dim=-1).detach().view(3, -1, 1, 2) + + line_coef_point = F.grid_sample( + self.app_line[0], coordinate_line[[0]], + align_corners=True).view(-1, *xyz_sampled.shape[:1]) + line_coef_point = line_coef_point * F.grid_sample( + self.app_line[1], coordinate_line[[1]], align_corners=True).view( + -1, *xyz_sampled.shape[:1]) + line_coef_point = line_coef_point * F.grid_sample( + self.app_line[2], coordinate_line[[2]], align_corners=True).view( + -1, *xyz_sampled.shape[:1]) + + return self.basis_mat(line_coef_point.T) + + @torch.no_grad() + def up_sampling_Vector(self, density_line_coef, app_line_coef, res_target): + + for i in range(len(self.vecMode)): + vec_id = self.vecMode[i] + density_line_coef[i] = torch.nn.Parameter( + F.interpolate( + density_line_coef[i].data, + size=(res_target[vec_id], 1), + mode='bilinear', + align_corners=True)) + app_line_coef[i] = torch.nn.Parameter( + F.interpolate( + app_line_coef[i].data, + size=(res_target[vec_id], 1), + mode='bilinear', + align_corners=True)) + + return density_line_coef, app_line_coef + + @torch.no_grad() + def upsample_volume_grid(self, res_target): + self.density_line, self.app_line = self.up_sampling_Vector( + self.density_line, self.app_line, res_target) + + self.update_stepSize(res_target) + print(f'upsamping to {res_target}') + + @torch.no_grad() + def shrink(self, new_aabb): + print('====> shrinking ...') + xyz_min, xyz_max = new_aabb + t_l, b_r = (xyz_min - self.aabb[0]) / self.units, ( + xyz_max - self.aabb[0]) / self.units + + t_l, b_r = torch.round( + torch.round(t_l)).long(), torch.round(b_r).long() + 1 + b_r = torch.stack([b_r, self.gridSize]).amin(0) + + for i in range(len(self.vecMode)): + mode0 = self.vecMode[i] + self.density_line[i] = torch.nn.Parameter( + self.density_line[i].data[..., t_l[mode0]:b_r[mode0], :]) + self.app_line[i] = torch.nn.Parameter( + self.app_line[i].data[..., t_l[mode0]:b_r[mode0], :]) + + if not torch.all(self.alphaMask.gridSize == self.gridSize): + t_l_r, b_r_r = t_l / (self.gridSize - 1), (b_r - 1) / ( + self.gridSize - 1) + correct_aabb = torch.zeros_like(new_aabb) + correct_aabb[0] = (1 - t_l_r) * self.aabb[0] + t_l_r * self.aabb[1] + correct_aabb[1] = (1 - b_r_r) * self.aabb[0] + b_r_r * self.aabb[1] + print('aabb', new_aabb, '\ncorrect aabb', correct_aabb) + new_aabb = correct_aabb + + newSize = b_r - t_l + self.aabb = new_aabb + self.update_stepSize((newSize[0], newSize[1], newSize[2])) + + def density_L1(self): + total = 0 + for idx in range(len(self.density_line)): + total = total + torch.mean(torch.abs(self.density_line[idx])) + return total + + def TV_loss_density(self, reg): + total = 0 + for idx in range(len(self.density_line)): + total = total + reg(self.density_line[idx]) * 1e-3 + return total + + def TV_loss_app(self, reg): + total = 0 + for idx in range(len(self.app_line)): + total = total + reg(self.app_line[idx]) * 1e-3 + return total diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF_VQ.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF_VQ.py new file mode 100644 index 0000000000000000000000000000000000000000..ee58de563b36e8aa3b16180475a95a35e457fb25 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensoRF_VQ.py @@ -0,0 +1,292 @@ +import os +import random +from typing import Callable, Iterator, List, Optional, Union + +import torch.nn as nn +from tqdm import tqdm + +from .tensorBase import * +from .tensoRF import TensorVMSplit +from .weighted_vq import VectorQuantize + + +class Timing: + """ + Timing environment + usage: + with Timing("message"): + your commands here + will print CUDA runtime in ms + """ + + def __init__(self, name, debug=False): + self.name = name + self.debug = debug + + def __enter__(self): + if not self.debug: + return + + self.start = torch.cuda.Event(enable_timing=True) + self.end = torch.cuda.Event(enable_timing=True) + self.start.record() + + def __exit__(self, type, value, traceback): + if not self.debug: + return + + self.end.record() + torch.cuda.synchronize() + print(self.name, 'elapsed', self.start.elapsed_time(self.end), 'ms') + + +def dec2bin(x, bits): + mask = 2**torch.arange(bits - 1, -1, -1).to(x.device, x.dtype) + return x.unsqueeze(-1).bitwise_and(mask).ne(0).float() + + +def bin2dec(b, bits): + mask = 2**torch.arange(bits - 1, -1, -1).to(b.device, b.dtype) + return torch.sum(mask * b, -1) + + +class TensorVMSplitVQ(TensorVMSplit): + + def __init__(self, aabb, gridSize, device, **kargs): + super(TensorVMSplitVQ, self).__init__(aabb, gridSize, device, **kargs) + self.codebook_size = kargs['codebook_size'] + print('codebook size: ' + str(self.codebook_size)) + self.use_cosine_sim = kargs['use_cosine_sim'] == 1 + self.codebook_dim = None if kargs['codebook_dim'] == 0 else kargs[ + 'codebook_dim'] + self.vq = nn.ModuleList([ + VectorQuantize( + dim=self.app_n_comp[0], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device), + VectorQuantize( + dim=self.app_n_comp[1], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device), + VectorQuantize( + dim=self.app_n_comp[2], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device) + ]) + self.den_vq = nn.ModuleList([ + VectorQuantize( + dim=self.density_n_comp[0], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device), + VectorQuantize( + dim=self.density_n_comp[1], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device), + VectorQuantize( + dim=self.density_n_comp[2], + codebook_size=self.codebook_size, # codebook size + decay=0.8, # specify number of quantizer + commitment_weight=1.0, + use_cosine_sim=self.use_cosine_sim, + codebook_dim=self.codebook_dim, + threshold_ema_dead_code=2.0, + ).to(self.device) + ]) + self.importance = kargs.get('importance', None) + self.plane_mask = kargs.get('plane_mask', None) + self.all_indices = kargs.get('all_indices', None) + + def extreme_load(self, ckpt): + if 'alphaMask.aabb' in ckpt.keys(): + length = np.prod(ckpt['alphaMask.shape']) + alpha_volume = torch.from_numpy( + np.unpackbits(ckpt['alphaMask.mask'])[:length].reshape( + ckpt['alphaMask.shape'])) + self.alphaMask = AlphaGridMask( + self.device, ckpt['alphaMask.aabb'].to(self.device), + alpha_volume.float().to(self.device)) + + # 1. load non-vq part + self.density_line.load_state_dict(ckpt['density_line']) + self.app_line.load_state_dict(ckpt['app_line']) + self.basis_mat.load_state_dict(ckpt['basis_mat']) + self.renderModule.load_state_dict(ckpt['mlp']) + + # 2. load vq part + # load vq_mask, keep_mask + self.plane_mask = [] + for i in range(3): + mask_shape = self.app_plane[i].shape[-2:] + vq_mask = np.unpackbits( + ckpt[f'vq_mask_{i}'], + count=np.prod(mask_shape)).reshape(mask_shape).astype(bool) + keep_mask = np.unpackbits( + ckpt[f'keep_mask_{i}'], + count=np.prod(mask_shape)).reshape(mask_shape).astype(bool) + self.plane_mask.append((vq_mask, keep_mask)) + + # recover app_plane, density_plane + import math + bits = int(math.log2(self.codebook_size)) + for idx_plane in range(3): + (vq_mask, keep_mask) = self.plane_mask[idx_plane] + # load appearance keep data from quantized data + int_repr = ckpt[f'quant_keep_data_{idx_plane}.int_repr'] + scale = ckpt[f'quant_keep_data_{idx_plane}.scale'] + zero_points = ckpt[f'quant_keep_data_{idx_plane}.zero_points'] + dequant = (int_repr - zero_points) * scale + keep_data = dequant.T.reshape( + *self.app_plane[idx_plane][:, :, keep_mask].shape) + self.app_plane[idx_plane].data[:, :, keep_mask] = keep_data + + # load appearance vq data from codebook + codebook = ckpt[f'codebook_{idx_plane}'].float() # + vq_count = int(vq_mask.sum()) + unpack1 = np.unpackbits( + ckpt[f'vq_indice_{idx_plane}'], count=vq_count * bits) + unpack2 = bin2dec( + torch.from_numpy(unpack1).reshape(vq_count, bits).long(), + bits=bits) + vq_data = codebook[0, unpack2, :] # N*len + vq_data = vq_data.T.reshape( + *(self.app_plane[idx_plane][:, :, vq_mask].shape)) + self.app_plane[idx_plane].data[:, :, vq_mask] = vq_data + + for idx_plane in range(3): + (vq_mask, keep_mask) = self.plane_mask[idx_plane] + # load density keep data from quantized data + int_repr = ckpt[f'quant_den_data_{idx_plane}.int_repr'] + scale = ckpt[f'quant_den_data_{idx_plane}.scale'] + zero_points = ckpt[f'quant_den_data_{idx_plane}.zero_points'] + dequant = (int_repr - zero_points) * scale + keep_data = dequant.T.reshape( + *self.density_plane[idx_plane][:, :, keep_mask].shape) + self.density_plane[idx_plane].data[:, :, keep_mask] = keep_data + + # load density vq data from codebook + codebook = ckpt[f'codebook_den_{idx_plane}'].float() # + vq_count = int(vq_mask.sum()) + unpack1 = np.unpackbits( + ckpt[f'den_vq_indice_{idx_plane}'], count=vq_count * bits) + unpack2 = bin2dec( + torch.from_numpy(unpack1).reshape(vq_count, bits).long(), + bits=bits) + vq_data = codebook[0, unpack2, :] # N*len + vq_data = vq_data.T.reshape( + *(self.density_plane[idx_plane][:, :, vq_mask].shape)) + self.density_plane[idx_plane].data[:, :, vq_mask] = vq_data + + def forward(self, + rays_chunk, + white_bg=True, + is_train=False, + ndc_ray=False, + N_samples=-1, + isvq=False): + # sample points + viewdirs = rays_chunk[:, 3:6] + if ndc_ray: + xyz_sampled, z_vals, ray_valid = self.sample_ray_ndc( + rays_chunk[:, :3], + viewdirs, + is_train=is_train, + N_samples=N_samples) + dists = torch.cat( + (z_vals[:, 1:] - z_vals[:, :-1], torch.zeros_like( + z_vals[:, :1])), + dim=-1) + rays_norm = torch.norm(viewdirs, dim=-1, keepdim=True) + dists = dists * rays_norm + viewdirs = viewdirs / rays_norm + else: + xyz_sampled, z_vals, ray_valid = self.sample_ray( + rays_chunk[:, :3], + viewdirs, + is_train=is_train, + N_samples=N_samples) + dists = torch.cat( + (z_vals[:, 1:] - z_vals[:, :-1], torch.zeros_like( + z_vals[:, :1])), + dim=-1) + viewdirs = viewdirs.view(-1, 1, 3).expand(xyz_sampled.shape) + + if self.alphaMask is not None: + alphas = self.alphaMask.sample_alpha(xyz_sampled[ray_valid]) + alpha_mask = alphas > 0 + ray_invalid = ~ray_valid + ray_invalid[ray_valid] |= (~alpha_mask) + ray_valid = ~ray_invalid + + sigma = torch.zeros(xyz_sampled.shape[:-1], device=xyz_sampled.device) + rgb = torch.zeros((*xyz_sampled.shape[:2], 3), + device=xyz_sampled.device) + + if ray_valid.any(): + xyz_sampled = self.normalize_coord(xyz_sampled) + sigma_feature = self.compute_densityfeature(xyz_sampled[ray_valid]) + + validsigma = self.feature2density(sigma_feature) + sigma[ray_valid] = validsigma + + alpha, weight, bg_weight = raw2alpha(sigma, + dists * self.distance_scale) + + app_mask = weight > self.rayMarch_weight_thres + + if app_mask.any(): + app_features = self.compute_appfeature(xyz_sampled[app_mask]) + valid_rgbs = self.renderModule(xyz_sampled[app_mask], + viewdirs[app_mask], app_features) + rgb[app_mask] = valid_rgbs + + acc_map = torch.sum(weight, -1) + rgb_map = torch.sum(weight[..., None] * rgb, -2) + + if white_bg or (is_train and torch.rand((1, )) < 0.5): + rgb_map = rgb_map + (1. - acc_map[..., None]) + + rgb_map = rgb_map.clamp(0, 1) + + with torch.no_grad(): + depth_map = torch.sum(weight * z_vals, -1) + depth_map = depth_map + (1. - acc_map) * rays_chunk[..., -1] + + return rgb_map, depth_map + + +def getsize(compressed_file, tag='MB'): + size = os.path.getsize(compressed_file) + if tag == 'B': + pass + elif tag == 'KB': + size = size / 1024 + elif tag == 'MB': + size = size / 1024 / 1024 + elif tag == 'GB': + size = size / 1024 / 1024 / 1024 + return f'{size} {tag}' diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensorBase.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensorBase.py new file mode 100644 index 0000000000000000000000000000000000000000..2e87227ad8bd7b35af7db7cffd20e75ddc1eca4f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/tensorBase.py @@ -0,0 +1,526 @@ +import time + +import numpy as np +import torch +import torch.nn +import torch.nn.functional as F + + +def positional_encoding(positions, freqs): + + freq_bands = (2**torch.arange(freqs).float()).to(positions.device) # (F,) + pts = (positions[..., None] * freq_bands).reshape( + positions.shape[:-1] + (freqs * positions.shape[-1], )) # (..., DF) + pts = torch.cat([torch.sin(pts), torch.cos(pts)], dim=-1) + return pts + + +def raw2alpha(sigma, dist): + # sigma, dist [N_rays, N_samples] + alpha = 1. - torch.exp(-sigma * dist) + + T = torch.cumprod( + torch.cat([ + torch.ones(alpha.shape[0], 1).to(alpha.device), 1. - alpha + 1e-10 + ], -1), -1) + + weights = alpha * T[:, :-1] # [N_rays, N_samples] + return alpha, weights, T[:, -1:] + + +def RGBRender(xyz_sampled, viewdirs, features): + + rgb = features + return rgb + + +class AlphaGridMask(torch.nn.Module): + + def __init__(self, device, aabb, alpha_volume): + super(AlphaGridMask, self).__init__() + self.device = device + + self.aabb = aabb.to(self.device) + self.aabbSize = self.aabb[1] - self.aabb[0] + self.invgridSize = 1.0 / self.aabbSize * 2 + self.alpha_volume = alpha_volume.view(1, 1, *alpha_volume.shape[-3:]) + self.gridSize = torch.LongTensor([ + alpha_volume.shape[-1], alpha_volume.shape[-2], + alpha_volume.shape[-3] + ]).to(self.device) + + def sample_alpha(self, xyz_sampled): + xyz_sampled = self.normalize_coord(xyz_sampled) + alpha_vals = F.grid_sample( + self.alpha_volume, + xyz_sampled.view(1, -1, 1, 1, 3), + align_corners=True).view(-1) + + return alpha_vals + + def normalize_coord(self, xyz_sampled): + return (xyz_sampled - self.aabb[0]) * self.invgridSize - 1 + + +class MLPRender_Fea(torch.nn.Module): + + def __init__(self, inChanel, viewpe=6, feape=6, featureC=128): + super(MLPRender_Fea, self).__init__() + + self.in_mlpC = 2 * viewpe * 3 + 2 * feape * inChanel + 3 + inChanel + self.viewpe = viewpe + self.feape = feape + layer1 = torch.nn.Linear(self.in_mlpC, featureC) + layer2 = torch.nn.Linear(featureC, featureC) + layer3 = torch.nn.Linear(featureC, 3) + + self.mlp = torch.nn.Sequential(layer1, + torch.nn.ReLU(inplace=True), layer2, + torch.nn.ReLU(inplace=True), layer3) + torch.nn.init.constant_(self.mlp[-1].bias, 0) + + def forward(self, pts, viewdirs, features): + indata = [features, viewdirs] + if self.feape > 0: + indata += [positional_encoding(features, self.feape)] + if self.viewpe > 0: + indata += [positional_encoding(viewdirs, self.viewpe)] + mlp_in = torch.cat(indata, dim=-1) + rgb = self.mlp(mlp_in) + rgb = torch.sigmoid(rgb) + + return rgb + + +class MLPRender_PE(torch.nn.Module): + + def __init__(self, inChanel, viewpe=6, pospe=6, featureC=128): + super(MLPRender_PE, self).__init__() + + self.in_mlpC = (3 + 2 * viewpe * 3) + (3 + 2 * pospe * 3) + inChanel # + self.viewpe = viewpe + self.pospe = pospe + layer1 = torch.nn.Linear(self.in_mlpC, featureC) + layer2 = torch.nn.Linear(featureC, featureC) + layer3 = torch.nn.Linear(featureC, 3) + + self.mlp = torch.nn.Sequential(layer1, + torch.nn.ReLU(inplace=True), layer2, + torch.nn.ReLU(inplace=True), layer3) + torch.nn.init.constant_(self.mlp[-1].bias, 0) + + def forward(self, pts, viewdirs, features): + indata = [features, viewdirs] + if self.pospe > 0: + indata += [positional_encoding(pts, self.pospe)] + if self.viewpe > 0: + indata += [positional_encoding(viewdirs, self.viewpe)] + mlp_in = torch.cat(indata, dim=-1) + rgb = self.mlp(mlp_in) + rgb = torch.sigmoid(rgb) + + return rgb + + +class MLPRender(torch.nn.Module): + + def __init__(self, inChanel, viewpe=6, featureC=128): + super(MLPRender, self).__init__() + + self.in_mlpC = (3 + 2 * viewpe * 3) + inChanel + self.viewpe = viewpe + + layer1 = torch.nn.Linear(self.in_mlpC, featureC) + layer2 = torch.nn.Linear(featureC, featureC) + layer3 = torch.nn.Linear(featureC, 3) + + self.mlp = torch.nn.Sequential(layer1, + torch.nn.ReLU(inplace=True), layer2, + torch.nn.ReLU(inplace=True), layer3) + torch.nn.init.constant_(self.mlp[-1].bias, 0) + + def forward(self, pts, viewdirs, features): + indata = [features, viewdirs] + if self.viewpe > 0: + indata += [positional_encoding(viewdirs, self.viewpe)] + mlp_in = torch.cat(indata, dim=-1) + rgb = self.mlp(mlp_in) + rgb = torch.sigmoid(rgb) + + return rgb + + +class TensorBase(torch.nn.Module): + + def __init__(self, + aabb, + gridSize, + device, + density_n_comp=8, + appearance_n_comp=24, + app_dim=27, + shadingMode='MLP_PE', + alphaMask=None, + near_far=[2.0, 6.0], + density_shift=-10, + alphaMask_thres=0.001, + distance_scale=25, + rayMarch_weight_thres=0.0001, + pos_pe=6, + view_pe=6, + fea_pe=6, + featureC=128, + step_ratio=2.0, + fea2denseAct='softplus', + **kargs): + super(TensorBase, self).__init__() + + self.density_n_comp = density_n_comp + self.app_n_comp = appearance_n_comp + self.app_dim = app_dim + self.aabb = aabb + self.alphaMask = alphaMask + self.device = device + + self.density_shift = density_shift + self.alphaMask_thres = alphaMask_thres + self.distance_scale = distance_scale + self.rayMarch_weight_thres = rayMarch_weight_thres + self.fea2denseAct = fea2denseAct + + self.near_far = near_far + self.step_ratio = step_ratio + + self.update_stepSize(gridSize) + + self.matMode = [[0, 1], [0, 2], [1, 2]] + self.vecMode = [2, 1, 0] + self.comp_w = [1, 1, 1] + + self.init_svd_volume(gridSize[0], device) + + self.shadingMode, self.pos_pe, self.view_pe = shadingMode, pos_pe, view_pe + self.fea_pe, self.featureC = fea_pe, featureC + self.init_render_func(shadingMode, pos_pe, view_pe, fea_pe, featureC, + device) + + def init_render_func(self, shadingMode, pos_pe, view_pe, fea_pe, featureC, + device): + if shadingMode == 'MLP_PE': + self.renderModule = MLPRender_PE(self.app_dim, view_pe, pos_pe, + featureC).to(device) + elif shadingMode == 'MLP_Fea': + self.renderModule = MLPRender_Fea(self.app_dim, view_pe, fea_pe, + featureC).to(device) + elif shadingMode == 'MLP': + self.renderModule = MLPRender(self.app_dim, view_pe, + featureC).to(device) + elif shadingMode == 'RGB': + assert self.app_dim == 3 + self.renderModule = RGBRender + else: + print('Unrecognized shading module') + exit() + print(self.renderModule) + + def update_stepSize(self, gridSize): + self.aabbSize = self.aabb[1] - self.aabb[0] + self.invaabbSize = 2.0 / self.aabbSize + self.gridSize = torch.LongTensor(gridSize).to(self.device) + self.units = self.aabbSize / (self.gridSize - 1) + self.stepSize = torch.mean(self.units) * self.step_ratio + self.aabbDiag = torch.sqrt(torch.sum(torch.square(self.aabbSize))) + self.nSamples = int((self.aabbDiag / self.stepSize).item()) + 1 + + def init_svd_volume(self, res, device): + pass + + def compute_features(self, xyz_sampled): + pass + + def compute_densityfeature(self, xyz_sampled): + pass + + def compute_appfeature(self, xyz_sampled): + pass + + def normalize_coord(self, xyz_sampled): + return (xyz_sampled - self.aabb[0]) * self.invaabbSize - 1 + + def get_optparam_groups(self, lr_init_spatial=0.02, lr_init_network=0.001): + pass + + def get_kwargs(self): + return { + 'aabb': self.aabb, + 'gridSize': self.gridSize.tolist(), + 'density_n_comp': self.density_n_comp, + 'appearance_n_comp': self.app_n_comp, + 'app_dim': self.app_dim, + 'density_shift': self.density_shift, + 'alphaMask_thres': self.alphaMask_thres, + 'distance_scale': self.distance_scale, + 'rayMarch_weight_thres': self.rayMarch_weight_thres, + 'fea2denseAct': self.fea2denseAct, + 'near_far': self.near_far, + 'step_ratio': self.step_ratio, + 'shadingMode': self.shadingMode, + 'pos_pe': self.pos_pe, + 'view_pe': self.view_pe, + 'fea_pe': self.fea_pe, + 'featureC': self.featureC + } + + def save(self, path): + kwargs = self.get_kwargs() + ckpt = {'kwargs': kwargs, 'state_dict': self.state_dict()} + if self.alphaMask is not None: + alpha_volume = self.alphaMask.alpha_volume.bool().cpu().numpy() + ckpt.update({'alphaMask.shape': alpha_volume.shape}) + ckpt.update( + {'alphaMask.mask': np.packbits(alpha_volume.reshape(-1))}) + ckpt.update({'alphaMask.aabb': self.alphaMask.aabb.cpu()}) + torch.save(ckpt, path) + + def load(self, ckpt): + if 'alphaMask.aabb' in ckpt.keys(): + length = np.prod(ckpt['alphaMask.shape']) + alpha_volume = torch.from_numpy( + np.unpackbits(ckpt['alphaMask.mask'])[:length].reshape( + ckpt['alphaMask.shape'])) + self.alphaMask = AlphaGridMask( + self.device, ckpt['alphaMask.aabb'].to(self.device), + alpha_volume.float().to(self.device)) + self.load_state_dict(ckpt['state_dict']) + + def sample_ray_ndc(self, rays_o, rays_d, is_train=True, N_samples=-1): + N_samples = N_samples if N_samples > 0 else self.nSamples + near, far = self.near_far + interpx = torch.linspace(near, far, N_samples).unsqueeze(0).to(rays_o) + if is_train: + interpx += torch.rand_like(interpx).to(rays_o) * ( + (far - near) / N_samples) + + rays_pts = rays_o[..., + None, :] + rays_d[..., None, :] * interpx[..., None] + mask_outbbox = ((self.aabb[0] > rays_pts) + | (rays_pts > self.aabb[1])).any(dim=-1) + return rays_pts, interpx, ~mask_outbbox + + def sample_ray(self, rays_o, rays_d, is_train=True, N_samples=-1): + N_samples = N_samples if N_samples > 0 else self.nSamples + stepsize = self.stepSize + near, far = self.near_far + vec = torch.where(rays_d == 0, torch.full_like(rays_d, 1e-6), rays_d) + rate_a = (self.aabb[1] - rays_o) / vec + rate_b = (self.aabb[0] - rays_o) / vec + t_min = torch.minimum(rate_a, rate_b).amax(-1).clamp(min=near, max=far) + + rng = torch.arange(N_samples)[None].float() + if is_train: + rng = rng.repeat(rays_d.shape[-2], 1) + rng += torch.rand_like(rng[:, [0]]) + step = stepsize * rng.to(rays_o.device) + interpx = (t_min[..., None] + step) + + rays_pts = rays_o[..., + None, :] + rays_d[..., None, :] * interpx[..., None] + mask_outbbox = ((self.aabb[0] > rays_pts) + | (rays_pts > self.aabb[1])).any(dim=-1) + + return rays_pts, interpx, ~mask_outbbox + + def shrink(self, new_aabb, voxel_size): + pass + + @torch.no_grad() + def getDenseAlpha(self, gridSize=None): + gridSize = self.gridSize if gridSize is None else gridSize + + samples = torch.stack( + torch.meshgrid( + torch.linspace(0, 1, gridSize[0]), + torch.linspace(0, 1, gridSize[1]), + torch.linspace(0, 1, gridSize[2]), + ), -1).to(self.device) + dense_xyz = self.aabb[0] * (1 - samples) + self.aabb[1] * samples + + alpha = torch.zeros_like(dense_xyz[..., 0]) + for i in range(gridSize[0]): + alpha[i] = self.compute_alpha(dense_xyz[i].view(-1, 3), + self.stepSize).view( + (gridSize[1], gridSize[2])) + return alpha, dense_xyz + + @torch.no_grad() + def updateAlphaMask(self, gridSize=(200, 200, 200)): + + alpha, dense_xyz = self.getDenseAlpha(gridSize) + dense_xyz = dense_xyz.transpose(0, 2).contiguous() + alpha = alpha.clamp(0, 1).transpose(0, 2).contiguous()[None, None] + total_voxels = gridSize[0] * gridSize[1] * gridSize[2] + + ks = 3 + alpha = F.max_pool3d( + alpha, kernel_size=ks, padding=ks // 2, + stride=1).view(gridSize[::-1]) + alpha[alpha >= self.alphaMask_thres] = 1 + alpha[alpha < self.alphaMask_thres] = 0 + + self.alphaMask = AlphaGridMask(self.device, self.aabb, alpha) + + valid_xyz = dense_xyz[alpha > 0.5] + + xyz_min = valid_xyz.amin(0) + xyz_max = valid_xyz.amax(0) + + new_aabb = torch.stack((xyz_min, xyz_max)) + + total = torch.sum(alpha) + print(f'bbox: {xyz_min, xyz_max} alpha rest %%%f' % + (total / total_voxels * 100)) + return new_aabb + + @torch.no_grad() + def filtering_rays(self, + all_rays, + all_rgbs, + N_samples=256, + chunk=10240 * 5, + bbox_only=False): + print('========> filtering rays ...') + tt = time.time() + + N = torch.tensor(all_rays.shape[:-1]).prod() + + mask_filtered = [] + idx_chunks = torch.split(torch.arange(N), chunk) + for idx_chunk in idx_chunks: + rays_chunk = all_rays[idx_chunk].to(self.device) + + rays_o, rays_d = rays_chunk[..., :3], rays_chunk[..., 3:6] + if bbox_only: + vec = torch.where(rays_d == 0, torch.full_like(rays_d, 1e-6), + rays_d) + rate_a = (self.aabb[1] - rays_o) / vec + rate_b = (self.aabb[0] - rays_o) / vec + t_min = torch.minimum(rate_a, rate_b).amax(-1) + t_max = torch.maximum(rate_a, rate_b).amin(-1) + mask_inbbox = t_max > t_min + + else: + xyz_sampled, _, _ = self.sample_ray( + rays_o, rays_d, N_samples=N_samples, is_train=False) + mask_inbbox = (self.alphaMask.sample_alpha(xyz_sampled).view( + xyz_sampled.shape[:-1]) > 0).any(-1) + + mask_filtered.append(mask_inbbox.cpu()) + + mask_filtered = torch.cat(mask_filtered).view(all_rgbs.shape[:-1]) + + print(f'Ray filtering done! takes {time.time()-tt} s.' + f' ray mask ratio: {torch.sum(mask_filtered) / N}') + return all_rays[mask_filtered], all_rgbs[mask_filtered] + + def feature2density(self, density_features): + if self.fea2denseAct == 'softplus': + return F.softplus(density_features + self.density_shift) + elif self.fea2denseAct == 'relu': + return F.relu(density_features) + + def compute_alpha(self, xyz_locs, length=1): + + if self.alphaMask is not None: + alphas = self.alphaMask.sample_alpha(xyz_locs) + alpha_mask = alphas > 0 + else: + alpha_mask = torch.ones_like(xyz_locs[:, 0], dtype=bool) + + sigma = torch.zeros(xyz_locs.shape[:-1], device=xyz_locs.device) + + if alpha_mask.any(): + xyz_sampled = self.normalize_coord(xyz_locs[alpha_mask]) + sigma_feature = self.compute_densityfeature(xyz_sampled) + validsigma = self.feature2density(sigma_feature) + sigma[alpha_mask] = validsigma + + alpha = 1 - torch.exp(-sigma * length).view(xyz_locs.shape[:-1]) + + return alpha + + def forward(self, + rays_chunk, + white_bg=True, + is_train=False, + ndc_ray=False, + N_samples=-1): + + # sample points + viewdirs = rays_chunk[:, 3:6] + if ndc_ray: + xyz_sampled, z_vals, ray_valid = self.sample_ray_ndc( + rays_chunk[:, :3], + viewdirs, + is_train=is_train, + N_samples=N_samples) + dists = torch.cat( + (z_vals[:, 1:] - z_vals[:, :-1], torch.zeros_like( + z_vals[:, :1])), + dim=-1) + rays_norm = torch.norm(viewdirs, dim=-1, keepdim=True) + dists = dists * rays_norm + viewdirs = viewdirs / rays_norm + else: + xyz_sampled, z_vals, ray_valid = self.sample_ray( + rays_chunk[:, :3], + viewdirs, + is_train=is_train, + N_samples=N_samples) + dists = torch.cat( + (z_vals[:, 1:] - z_vals[:, :-1], torch.zeros_like( + z_vals[:, :1])), + dim=-1) + viewdirs = viewdirs.view(-1, 1, 3).expand(xyz_sampled.shape) + + if self.alphaMask is not None: + alphas = self.alphaMask.sample_alpha(xyz_sampled[ray_valid]) + alpha_mask = alphas > 0 + ray_invalid = ~ray_valid + ray_invalid[ray_valid] |= (~alpha_mask) + ray_valid = ~ray_invalid + + sigma = torch.zeros(xyz_sampled.shape[:-1], device=xyz_sampled.device) + rgb = torch.zeros((*xyz_sampled.shape[:2], 3), + device=xyz_sampled.device) + + if ray_valid.any(): + xyz_sampled = self.normalize_coord(xyz_sampled) + sigma_feature = self.compute_densityfeature(xyz_sampled[ray_valid]) + + validsigma = self.feature2density(sigma_feature) + sigma[ray_valid] = validsigma + + alpha, weight, bg_weight = raw2alpha(sigma, + dists * self.distance_scale) + + app_mask = weight > self.rayMarch_weight_thres + + if app_mask.any(): + app_features = self.compute_appfeature(xyz_sampled[app_mask]) + valid_rgbs = self.renderModule(xyz_sampled[app_mask], + viewdirs[app_mask], app_features) + rgb[app_mask] = valid_rgbs + + acc_map = torch.sum(weight, -1) + rgb_map = torch.sum(weight[..., None] * rgb, -2) + + if white_bg or (is_train and torch.rand((1, )) < 0.5): + rgb_map = rgb_map + (1. - acc_map[..., None]) + + rgb_map = rgb_map.clamp(0, 1) + + with torch.no_grad(): + depth_map = torch.sum(weight * z_vals, -1) + depth_map = depth_map + (1. - acc_map) * rays_chunk[..., -1] + + return rgb_map, depth_map diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/weighted_vq.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/weighted_vq.py new file mode 100644 index 0000000000000000000000000000000000000000..eea10f189254780f7ae51a951c999103b56d79e7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/network/weighted_vq.py @@ -0,0 +1,504 @@ +from contextlib import contextmanager + +import torch +import torch.distributed as distributed +import torch.nn.functional as F +from einops import rearrange, repeat +from torch import einsum, nn +from torch.cuda.amp import autocast + + +def exists(val): + return val is not None + + +def default(val, d): + return val if exists(val) else d + + +def noop(*args, **kwargs): + pass + + +def l2norm(t): + return F.normalize(t, p=2, dim=-1) + + +def log(t, eps=1e-20): + return torch.log(t.clamp(min=eps)) + + +def uniform_init(*shape): + t = torch.empty(shape) + nn.init.kaiming_uniform_(t) + return t + + +def gumbel_noise(t): + noise = torch.zeros_like(t).uniform_(0, 1) + return -log(-log(noise)) + + +def gumbel_sample(t, temperature=1., dim=-1): + if temperature == 0: + return t.argmax(dim=dim) + + return ((t / temperature) + gumbel_noise(t)).argmax(dim=dim) + + +def ema_inplace(moving_avg, new, decay): + moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay)) + + +def laplace_smoothing(x, n_categories, eps=1e-5): + return (x + eps) / (x.sum() + n_categories * eps) + + +def sample_vectors(samples, num): + num_samples, device = samples.shape[0], samples.device + if num_samples >= num: + indices = torch.randperm(num_samples, device=device)[:num] + else: + indices = torch.randint(0, num_samples, (num, ), device=device) + + return samples[indices] + + +def batched_sample_vectors(samples, num): + return torch.stack( + [sample_vectors(sample, num) for sample in samples.unbind(dim=0)], + dim=0) + + +def pad_shape(shape, size, dim=0): + return [size if i == dim else s for i, s in enumerate(shape)] + + +def sample_multinomial(total_count, probs): + device = probs.device + probs = probs.cpu() + + total_count = probs.new_full((), total_count) + remainder = probs.new_ones(()) + sample = torch.empty_like(probs, dtype=torch.long) + + for i, p in enumerate(probs): + s = torch.binomial(total_count, p / remainder) + sample[i] = s + total_count -= s + remainder -= p + + return sample.to(device) + + +def all_gather_sizes(x, dim): + size = torch.tensor(x.shape[dim], dtype=torch.long, device=x.device) + all_sizes = [ + torch.empty_like(size) for _ in range(distributed.get_world_size()) + ] + distributed.all_gather(all_sizes, size) + + return torch.stack(all_sizes) + + +def all_gather_variably_sized(x, sizes, dim=0): + rank = distributed.get_rank() + all_x = [] + + for i, size in enumerate(sizes): + t = x if i == rank else x.new_empty(pad_shape(x.shape, size, dim)) + distributed.broadcast(t, src=i, async_op=True) + all_x.append(t) + + distributed.barrier() + return all_x + + +def sample_vectors_distributed(local_samples, num): + rank = distributed.get_rank() + all_num_samples = all_gather_sizes(local_samples, dim=0) + + if rank == 0: + samples_per_rank = sample_multinomial( + num, all_num_samples / all_num_samples.sum()) + else: + samples_per_rank = torch.empty_like(all_num_samples) + + distributed.broadcast(samples_per_rank, src=0) + samples_per_rank = samples_per_rank.tolist() + + local_samples = batched_sample_vectors(local_samples, + samples_per_rank[rank]) + all_samples = all_gather_variably_sized( + local_samples, samples_per_rank, dim=0) + return torch.cat(all_samples, dim=0) + + +def batched_bincount(x, *, minlength): + batch, dtype, device = x.shape[0], x.dtype, x.device + target = torch.zeros(batch, minlength, dtype=dtype, device=device) + values = torch.ones_like(x) + target.scatter_add_(-1, x, values) + return target + + +def kmeans(samples, + num_clusters, + num_iters=10, + use_cosine_sim=False, + sample_fn=batched_sample_vectors, + all_reduce_fn=noop): + num_codebooks, dim, dtype = samples.shape[0], samples.shape[ + -1], samples.dtype + + means = sample_fn(samples, num_clusters) + + for _ in range(num_iters): + if use_cosine_sim: + dists = samples @ rearrange(means, 'h n d -> h d n') + else: + dists = -torch.cdist(samples, means, p=2) + + buckets = torch.argmax(dists, dim=-1) + bins = batched_bincount(buckets, minlength=num_clusters) + all_reduce_fn(bins) + + zero_mask = bins == 0 + bins_min_clamped = bins.masked_fill(zero_mask, 1) + + new_means = buckets.new_zeros( + num_codebooks, num_clusters, dim, dtype=dtype) + + new_means.scatter_add_(1, repeat(buckets, 'h n -> h n d', d=dim), + samples) + new_means = new_means / rearrange(bins_min_clamped, '... -> ... 1') + all_reduce_fn(new_means) + + if use_cosine_sim: + new_means = l2norm(new_means) + + means = torch.where( + rearrange(zero_mask, '... -> ... 1'), means, new_means) + + return means, bins + + +def batched_embedding(indices, embeds): + batch, dim = indices.shape[1], embeds.shape[-1] + indices = repeat(indices, 'h b n -> h b n d', d=dim) + embeds = repeat(embeds, 'h c d -> h b c d', b=batch) + return embeds.gather(2, indices) + + +# regularization losses + + +def orthogonal_loss_fn(t): + # eq (2) from https://arxiv.org/abs/2112.00384 + h, n = t.shape[:2] + normed_codes = l2norm(t) + identity = repeat(torch.eye(n, device=t.device), 'i j -> h i j', h=h) + cosine_sim = einsum('h i d, h j d -> h i j', normed_codes, normed_codes) + return ((cosine_sim - identity)**2).sum() / (h * n**2) + + +# distance types + + +class EuclideanCodebook(nn.Module): + + def __init__(self, + dim, + codebook_size, + num_codebooks=1, + kmeans_init=False, + kmeans_iters=10, + decay=0.8, + eps=1e-5, + threshold_ema_dead_code=2, + use_ddp=False, + learnable_codebook=False, + sample_codebook_temp=0): + super().__init__() + self.decay = decay + init_fn = uniform_init if not kmeans_init else torch.zeros + embed = init_fn(num_codebooks, codebook_size, dim) + + self.codebook_size = codebook_size + self.num_codebooks = num_codebooks + + self.kmeans_iters = kmeans_iters + self.eps = eps + self.threshold_ema_dead_code = threshold_ema_dead_code + self.sample_codebook_temp = sample_codebook_temp + + self.sample_fn = sample_vectors_distributed if use_ddp else batched_sample_vectors + self.all_reduce_fn = distributed.all_reduce if use_ddp else noop + + self.register_buffer('initted', torch.Tensor([not kmeans_init])) + self.register_buffer('cluster_size', + torch.zeros(num_codebooks, codebook_size)) + self.register_buffer('embed_avg', embed.clone()) + + self.learnable_codebook = learnable_codebook + if learnable_codebook: + self.embed = nn.Parameter(embed) + else: + self.register_buffer('embed', embed) + + @torch.jit.ignore + def init_embed_(self, data): + if self.initted: + return + + embed, cluster_size = kmeans( + data, + self.codebook_size, + self.kmeans_iters, + sample_fn=self.sample_fn, + all_reduce_fn=self.all_reduce_fn) + + self.embed.data.copy_(embed) + self.embed_avg.data.copy_(embed.clone()) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(torch.Tensor([True])) + + def replace(self, batch_samples, batch_mask): + batch_samples = l2norm(batch_samples) + + for ind, (samples, mask) in enumerate( + zip(batch_samples.unbind(dim=0), batch_mask.unbind(dim=0))): + if not torch.any(mask): + continue + + sampled = self.sample_fn( + rearrange(samples, '... -> 1 ...'), + mask.sum().item()) + self.embed.data[ind][mask] = rearrange(sampled, '1 ... -> ...') + + def expire_codes_(self, batch_samples, verbose): + if self.threshold_ema_dead_code == 0: + return + + expired_codes = self.cluster_size < self.threshold_ema_dead_code + + if not torch.any(expired_codes): + return + if verbose: + print(f'expire code count: {expired_codes.sum()}') + batch_samples = rearrange(batch_samples, 'h ... d -> h (...) d') + self.replace(batch_samples, batch_mask=expired_codes) + + @autocast(enabled=False) + def forward(self, x, weight=None, verbose=False): + if weight is not None: + weight = weight * weight.numel() / weight.sum() + needs_codebook_dim = x.ndim < 4 + + x = x.float() + + if needs_codebook_dim: + x = rearrange(x, '... -> 1 ...') + + shape, dtype = x.shape, x.dtype + flatten = rearrange(x, 'h ... d -> h (...) d') + + self.init_embed_(flatten) + + embed = self.embed if not self.learnable_codebook else self.embed.detach( + ) + + dist = -torch.cdist(flatten, embed, p=2) + + embed_ind = gumbel_sample( + dist, dim=-1, temperature=self.sample_codebook_temp) + embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) + embed_ind = embed_ind.view(*shape[:-1]) + + quantize = batched_embedding(embed_ind, self.embed) + + if self.training: + + if weight is not None: + cluster_size = (embed_onehot * weight).sum(dim=1) + else: + cluster_size = embed_onehot.sum(dim=1) + self.all_reduce_fn(cluster_size) + ema_inplace(self.cluster_size, cluster_size, self.decay) + + if weight is not None: + + embed_sum = einsum('h n d, h n c -> h c d', flatten * weight, + embed_onehot) + else: + embed_sum = einsum('h n d, h n c -> h c d', flatten, + embed_onehot) + self.all_reduce_fn(embed_sum) + cluster_size = laplace_smoothing( + self.cluster_size, self.codebook_size, + self.eps) * self.cluster_size.sum() + + # embed_normalized = self.embed_avg / rearrange(cluster_size, '... -> ... 1') + # print("embed_normalized: ",embed_normalized, + # "\n embed_avg: ",self.embed_avg, + # "\n cluster_size: ", cluster_size) + # self.embed.data.copy_(embed_normalized) + # print("before ema: self.embed:", self.embed, "embed_sum: ", embed_sum) + ema_inplace(self.embed, + embed_sum / rearrange(cluster_size, '... -> ... 1'), + self.decay) + # print("after ema: self.embed:", self.embed, "embed_sum: ", embed_sum) + self.expire_codes_(x, verbose) + # print("after expire: self.embed:", self.embed, "embed_sum: ", embed_sum) + + if needs_codebook_dim: + quantize, embed_ind = map(lambda t: rearrange(t, '1 ... -> ...'), + (quantize, embed_ind)) + + return quantize, embed_ind + + +# main class + + +class VectorQuantize(nn.Module): + + def __init__(self, + dim, + codebook_size, + codebook_dim=None, + heads=1, + separate_codebook_per_head=False, + decay=0.8, + eps=1e-5, + kmeans_init=False, + kmeans_iters=10, + use_cosine_sim=False, + threshold_ema_dead_code=0, + channel_last=True, + accept_image_fmap=False, + commitment_weight=1., + orthogonal_reg_weight=0., + orthogonal_reg_active_codes_only=False, + orthogonal_reg_max_codes=None, + sample_codebook_temp=0., + sync_codebook=False): + super().__init__() + self.heads = heads + self.separate_codebook_per_head = separate_codebook_per_head + + codebook_dim = default(codebook_dim, dim) + codebook_input_dim = codebook_dim * heads + + requires_projection = codebook_input_dim != dim + self.project_in = nn.Linear( + dim, codebook_input_dim) if requires_projection else nn.Identity() + self.project_out = nn.Linear( + codebook_input_dim, dim) if requires_projection else nn.Identity() + + self.eps = eps + self.commitment_weight = commitment_weight + + has_codebook_orthogonal_loss = orthogonal_reg_weight > 0 + self.orthogonal_reg_weight = orthogonal_reg_weight + self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only + self.orthogonal_reg_max_codes = orthogonal_reg_max_codes + + codebook_class = EuclideanCodebook + + self._codebook = codebook_class( + dim=codebook_dim, + num_codebooks=heads if separate_codebook_per_head else 1, + codebook_size=codebook_size, + kmeans_init=kmeans_init, + kmeans_iters=kmeans_iters, + decay=decay, + eps=eps, + threshold_ema_dead_code=threshold_ema_dead_code, + use_ddp=sync_codebook, + learnable_codebook=has_codebook_orthogonal_loss, + sample_codebook_temp=sample_codebook_temp) + + self.codebook_size = codebook_size + + self.accept_image_fmap = accept_image_fmap + self.channel_last = channel_last + + @property + def codebook(self): + codebook = self._codebook.embed + if self.separate_codebook_per_head: + return codebook + + return rearrange(codebook, '1 ... -> ...') + + def forward(self, x, weight=None, verbose=False): + device, heads, is_multiheaded = x.device, self.heads, self.heads > 1 + + need_transpose = not self.channel_last and not self.accept_image_fmap + + if self.accept_image_fmap: + height, width = x.shape[-2:] + x = rearrange(x, 'b c h w -> b (h w) c') + + if need_transpose: + x = rearrange(x, 'b d n -> b n d') + + x = self.project_in(x) + + if is_multiheaded: + ein_rhs_eq = 'h b n d' if self.separate_codebook_per_head else '1 (b h) n d' + x = rearrange(x, f'b n (h d) -> {ein_rhs_eq}', h=heads) + + quantize, embed_ind = self._codebook(x, weight, verbose) + + if self.training: + quantize = x + (quantize - x).detach() + + loss = torch.tensor([0.], device=device, requires_grad=self.training) + + if self.training: + if self.commitment_weight > 0: + commit_loss = F.mse_loss(quantize.detach(), x) + loss = loss + commit_loss * self.commitment_weight + + if self.orthogonal_reg_weight > 0: + codebook = self._codebook.embed + + if self.orthogonal_reg_active_codes_only: + # only calculate orthogonal loss for the activated codes for this batch + unique_code_ids = torch.unique(embed_ind) + codebook = codebook[unique_code_ids] + + num_codes = codebook.shape[0] + if exists(self.orthogonal_reg_max_codes + ) and num_codes > self.orthogonal_reg_max_codes: + rand_ids = torch.randperm( + num_codes, + device=device)[:self.orthogonal_reg_max_codes] + codebook = codebook[rand_ids] + + orthogonal_reg_loss = orthogonal_loss_fn(codebook) + loss = loss + orthogonal_reg_loss * self.orthogonal_reg_weight + + if is_multiheaded: + if self.separate_codebook_per_head: + quantize = rearrange(quantize, 'h b n d -> b n (h d)', h=heads) + embed_ind = rearrange(embed_ind, 'h b n -> b n h', h=heads) + else: + quantize = rearrange( + quantize, '1 (b h) n d -> b n (h d)', h=heads) + embed_ind = rearrange(embed_ind, '1 (b h) n -> b n h', h=heads) + + quantize = self.project_out(quantize) + + if need_transpose: + quantize = rearrange(quantize, 'b n d -> b d n') + + if self.accept_image_fmap: + quantize = rearrange( + quantize, 'b (h w) c -> b c h w', h=height, w=width) + embed_ind = rearrange( + embed_ind, 'b (h w) ... -> b h w ...', h=height, w=width) + + return quantize, embed_ind, loss diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/renderer.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..03a96870fee1afc3785236352cdc085904478023 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/renderer.py @@ -0,0 +1,211 @@ +import os +import sys + +import imageio +import numpy as np +import torch +from tqdm.auto import tqdm + +from .dataloader.ray_utils import get_rays, ndc_rays_blender +from .network.tensoRF import (AlphaGridMask, TensorCP, TensorVM, TensorVMSplit, + raw2alpha) +from .network.tensoRF_VQ import TensorVMSplitVQ +from .utils import rgb_lpips, rgb_ssim, visualize_depth_numpy + + +def OctreeRender_trilinear_fast(rays, + tensorf, + chunk=4096, + N_samples=-1, + ndc_ray=False, + white_bg=True, + is_train=False, + device='cuda', + **kwargs): + + rgbs, depth_maps = [], [] + N_rays_all = rays.shape[0] + for chunk_idx in range(N_rays_all // chunk + int(N_rays_all % chunk > 0)): + rays_chunk = rays[chunk_idx * chunk:(chunk_idx + 1) * chunk].to(device) + + rgb_map, depth_map = tensorf( + rays_chunk, + is_train=is_train, + white_bg=white_bg, + ndc_ray=ndc_ray, + N_samples=N_samples, + **kwargs) + + rgbs.append(rgb_map) + depth_maps.append(depth_map) + + return torch.cat(rgbs), None, torch.cat(depth_maps), None, None + + +@torch.no_grad() +def evaluation(test_dataset, + tensorf, + renderer, + savePath=None, + N_vis=5, + prtx='', + N_samples=-1, + white_bg=False, + ndc_ray=False, + compute_extra_metrics=True, + device='cuda', + im_save=False): + if prtx is not None and len(prtx) > 0: + prtx = prtx + '_' + result_path = f'{savePath}/{prtx}res.txt' + PSNRs, rgb_maps, depth_maps = [], [], [] + ssims, l_alex, l_vgg = [], [], [] + if savePath is not None: + os.makedirs(savePath, exist_ok=True) + os.makedirs(savePath + '/rgbd', exist_ok=True) + + try: + tqdm._instances.clear() + except Exception: + pass + + near_far = test_dataset.near_far + img_eval_interval = 1 if N_vis < 0 else max( + test_dataset.all_rays.shape[0] // N_vis, 1) + # img_eval_interval = max(img_eval_interval, test_dataset.all_rays.shape[0]//49) + idxs = list(range(0, test_dataset.all_rays.shape[0], img_eval_interval)) + for idx, samples in tqdm( + enumerate(test_dataset.all_rays[0::img_eval_interval]), + file=sys.stdout): + + W, H = test_dataset.img_wh + rays = samples.view(-1, samples.shape[-1]) + + rgb_map, _, depth_map, _, _ = renderer( + rays, + tensorf, + chunk=4096, + N_samples=N_samples, + ndc_ray=ndc_ray, + white_bg=white_bg, + device=device) + rgb_map = rgb_map.clamp(0.0, 1.0) + + rgb_map, depth_map = rgb_map.reshape(H, W, 3).cpu(), depth_map.reshape( + H, W).cpu() + + depth_map, _ = visualize_depth_numpy(depth_map.numpy(), near_far) + if len(test_dataset.all_rgbs): + gt_rgb = test_dataset.all_rgbs[idxs[idx]].view(H, W, 3) + loss = torch.mean((rgb_map - gt_rgb)**2) + PSNRs.append(-10.0 * np.log(loss.item()) / np.log(10.0)) + + if compute_extra_metrics: + ssim = rgb_ssim(rgb_map, gt_rgb, 1) + l_a = rgb_lpips(gt_rgb.numpy(), rgb_map.numpy(), 'alex', + tensorf.device) + l_v = rgb_lpips(gt_rgb.numpy(), rgb_map.numpy(), 'vgg', + tensorf.device) + ssims.append(ssim) + l_alex.append(l_a) + l_vgg.append(l_v) + + rgb_map = (rgb_map.numpy() * 255).astype('uint8') + # rgb_map = np.concatenate((rgb_map, depth_map), axis=1) + rgb_maps.append(rgb_map) + depth_maps.append(depth_map) + if savePath is not None and im_save: + imageio.imwrite(f'{savePath}/{prtx}{idx:03d}.png', rgb_map) + rgb_map = np.concatenate((rgb_map, depth_map), axis=1) + imageio.imwrite(f'{savePath}/rgbd/{prtx}{idx:03d}.png', rgb_map) + if savePath is not None: + imageio.mimwrite( + f'{savePath}/{prtx}video.mp4', + np.stack(rgb_maps), + fps=30, + quality=10) + imageio.mimwrite( + f'{savePath}/{prtx}depthvideo.mp4', + np.stack(depth_maps), + fps=30, + quality=10) + + if PSNRs: + psnr = np.mean(np.asarray(PSNRs)) + if compute_extra_metrics: + ssim = np.mean(np.asarray(ssims)) + l_a = np.mean(np.asarray(l_alex)) + l_v = np.mean(np.asarray(l_vgg)) + if savePath is not None: + np.savetxt(result_path, np.asarray([psnr, ssim, l_a, l_v])) + else: + if savePath is not None: + np.savetxt(result_path, np.asarray([psnr])) + + return PSNRs + + +@torch.no_grad() +def render_path(test_dataset, + tensorf, + c2ws, + renderer, + savePath=None, + prtx='', + N_samples=-1, + white_bg=False, + ndc_ray=False, + device='cuda'): + rgb_maps, depth_maps = [], [] + os.makedirs(savePath, exist_ok=True) + os.makedirs(savePath + '/rgbd', exist_ok=True) + + try: + tqdm._instances.clear() + except Exception: + pass + + near_far = test_dataset.near_far + for idx, c2w in tqdm(enumerate(c2ws)): + + W, H = test_dataset.img_wh + + c2w = torch.FloatTensor(c2w) + rays_o, rays_d = get_rays(test_dataset.directions, + c2w) # both (h*w, 3) + if ndc_ray: + rays_o, rays_d = ndc_rays_blender(H, W, test_dataset.focal[0], 1.0, + rays_o, rays_d) + rays = torch.cat([rays_o, rays_d], 1) # (h*w, 6) + + rgb_map, _, depth_map, _, _ = renderer( + rays, + tensorf, + chunk=8192, + N_samples=N_samples, + ndc_ray=ndc_ray, + white_bg=white_bg, + device=device) + rgb_map = rgb_map.clamp(0.0, 1.0) + + rgb_map, depth_map = rgb_map.reshape(H, W, 3).cpu(), depth_map.reshape( + H, W).cpu() + + depth_map, _ = visualize_depth_numpy(depth_map.numpy(), near_far) + + rgb_map = (rgb_map.numpy() * 255).astype('uint8') + # rgb_map = np.concatenate((rgb_map, depth_map), axis=1) + rgb_maps.append(rgb_map) + depth_maps.append(depth_map) + if savePath is not None: + imageio.imwrite(f'{savePath}/{prtx}{idx:03d}.png', rgb_map) + rgb_map = np.concatenate((rgb_map, depth_map), axis=1) + imageio.imwrite(f'{savePath}/rgbd/{prtx}{idx:03d}.png', rgb_map) + + imageio.mimwrite( + f'{savePath}/{prtx}video.mp4', np.stack(rgb_maps), fps=30, quality=8) + imageio.mimwrite( + f'{savePath}/{prtx}depthvideo.mp4', + np.stack(depth_maps), + fps=30, + quality=8) diff --git a/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/utils.py b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2addb5f8beddd5f70eb980ae5eab0bda5e8fac68 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/nerf_recon_vq_compression/utils.py @@ -0,0 +1,269 @@ +import cv2 +import numpy as np +import plyfile +import scipy.signal +import skimage.measure +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.transforms as T +from PIL import Image + + +def mse2psnr(x): + return -10. * torch.log(x) / torch.log(torch.Tensor([10.])) + + +def visualize_depth_numpy(depth, minmax=None, cmap=cv2.COLORMAP_JET): + """ + depth: (H, W) + """ + + x = np.nan_to_num(depth) # change nan to 0 + if minmax is None: + mi = np.min(x[x > 0]) # get minimum positive depth (ignore background) + ma = np.max(x) + else: + mi, ma = minmax + + x = (x - mi) / (ma - mi + 1e-8) # normalize to 0~1 + x = (255 * x).astype(np.uint8) + x_ = cv2.applyColorMap(x, cmap) + return x_, [mi, ma] + + +def init_log(log, keys): + for key in keys: + log[key] = torch.tensor([0.0], dtype=float) + return log + + +def visualize_depth(depth, minmax=None, cmap=cv2.COLORMAP_JET): + """ + depth: (H, W) + """ + if type(depth) is not np.ndarray: + depth = depth.cpu().numpy() + + x = np.nan_to_num(depth) # change nan to 0 + if minmax is None: + mi = np.min(x[x > 0]) # get minimum positive depth (ignore background) + ma = np.max(x) + else: + mi, ma = minmax + + x = (x - mi) / (ma - mi + 1e-8) # normalize to 0~1 + x = (255 * x).astype(np.uint8) + x_ = Image.fromarray(cv2.applyColorMap(x, cmap)) + x_ = T.ToTensor()(x_) # (3, H, W) + return x_, [mi, ma] + + +def N_to_reso(n_voxels, bbox): + xyz_min, xyz_max = bbox + dim = len(xyz_min) + voxel_size = ((xyz_max - xyz_min).prod() / n_voxels).pow(1 / dim) + return ((xyz_max - xyz_min) / voxel_size).long().tolist() + + +def cal_n_samples(reso, step_ratio=0.5): + return int(np.linalg.norm(reso) / step_ratio) + + +__LPIPS__ = {} + + +def init_lpips(net_name, device): + assert net_name in ['alex', 'vgg'] + import lpips + print(f'init_lpips: lpips_{net_name}') + return lpips.LPIPS(net=net_name, version='0.1').eval().to(device) + + +def rgb_lpips(np_gt, np_im, net_name, device): + if net_name not in __LPIPS__: + __LPIPS__[net_name] = init_lpips(net_name, device) + gt = torch.from_numpy(np_gt).permute([2, 0, 1]).contiguous().to(device) + im = torch.from_numpy(np_im).permute([2, 0, 1]).contiguous().to(device) + return __LPIPS__[net_name](gt, im, normalize=True).item() + + +def findItem(items, target): + for one in items: + if one[:len(target)] == target: + return one + return None + + +''' Evaluation metrics (ssim, lpips) +''' + + +def rgb_ssim(img0, + img1, + max_val, + filter_size=11, + filter_sigma=1.5, + k1=0.01, + k2=0.03, + return_map=False): + # Modified from https://github.com/google/mipnerf/blob/16e73dfdb52044dcceb47cda5243a686391a6e0f/internal/math.py#L58 + assert len(img0.shape) == 3 + assert img0.shape[-1] == 3 + assert img0.shape == img1.shape + + # Construct a 1D Gaussian blur filter. + hw = filter_size // 2 + shift = (2 * hw - filter_size + 1) / 2 + f_i = ((np.arange(filter_size) - hw + shift) / filter_sigma)**2 + filt = np.exp(-0.5 * f_i) + filt /= np.sum(filt) + + # Blur in x and y (faster than the 2D convolution). + def convolve2d(z, f): + return scipy.signal.convolve2d(z, f, mode='valid') + + def filt_fn(z): + return np.stack([ + convolve2d(convolve2d(z[..., i], filt[:, None]), filt[None, :]) + for i in range(z.shape[-1]) + ], -1) + + mu0 = filt_fn(img0) + mu1 = filt_fn(img1) + mu00 = mu0 * mu0 + mu11 = mu1 * mu1 + mu01 = mu0 * mu1 + sigma00 = filt_fn(img0**2) - mu00 + sigma11 = filt_fn(img1**2) - mu11 + sigma01 = filt_fn(img0 * img1) - mu01 + + # Clip the variances and covariances to valid values. + # Variance must be non-negative: + sigma00 = np.maximum(0., sigma00) + sigma11 = np.maximum(0., sigma11) + sigma01 = np.sign(sigma01) * np.minimum( + np.sqrt(sigma00 * sigma11), np.abs(sigma01)) + c1 = (k1 * max_val)**2 + c2 = (k2 * max_val)**2 + numer = (2 * mu01 + c1) * (2 * sigma01 + c2) + denom = (mu00 + mu11 + c1) * (sigma00 + sigma11 + c2) + ssim_map = numer / denom + ssim = np.mean(ssim_map) + return ssim_map if return_map else ssim + + +class TVLoss(nn.Module): + + def __init__(self, TVLoss_weight=1): + super(TVLoss, self).__init__() + self.TVLoss_weight = TVLoss_weight + + def forward(self, x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + count_h = self._tensor_size(x[:, :, 1:, :]) + count_w = self._tensor_size(x[:, :, :, 1:]) + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, :h_x - 1, :]), 2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, :w_x - 1]), 2).sum() + return self.TVLoss_weight * 2 * (h_tv / count_h + + w_tv / count_w) / batch_size + + def _tensor_size(self, t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +def convert_sdf_samples_to_ply( + pytorch_3d_sdf_tensor, + ply_filename_out, + bbox, + level=0.5, + offset=None, + scale=None, +): + """ + Convert sdf samples to .ply + + :param pytorch_3d_sdf_tensor: a torch.FloatTensor of shape (n,n,n) + :voxel_grid_origin: a list of three floats: the bottom, left, down origin of the voxel grid + :voxel_size: float, the size of the voxels + :ply_filename_out: string, path of the filename to save to + + This function adapted from: https://github.com/RobotLocomotion/spartan + """ + + numpy_3d_sdf_tensor = pytorch_3d_sdf_tensor.numpy() + voxel_size = list( + (bbox[1] - bbox[0]) / np.array(pytorch_3d_sdf_tensor.shape)) + + verts, faces, normals, values = skimage.measure.marching_cubes( + numpy_3d_sdf_tensor, level=level, spacing=voxel_size) + faces = faces[..., ::-1] # inverse face orientation + + # transform from voxel coordinates to camera coordinates + # note x and y are flipped in the output of marching_cubes + mesh_points = np.zeros_like(verts) + mesh_points[:, 0] = bbox[0, 0] + verts[:, 0] + mesh_points[:, 1] = bbox[0, 1] + verts[:, 1] + mesh_points[:, 2] = bbox[0, 2] + verts[:, 2] + + # apply additional offset and scale + if scale is not None: + mesh_points = mesh_points / scale + if offset is not None: + mesh_points = mesh_points - offset + + # try writing to the ply file + + num_verts = verts.shape[0] + num_faces = faces.shape[0] + + verts_tuple = np.zeros((num_verts, ), + dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) + + for i in range(0, num_verts): + verts_tuple[i] = tuple(mesh_points[i, :]) + + faces_building = [] + for i in range(0, num_faces): + faces_building.append(((faces[i, :].tolist(), ))) + faces_tuple = np.array( + faces_building, dtype=[('vertex_indices', 'i4', (3, ))]) + + el_verts = plyfile.PlyElement.describe(verts_tuple, 'vertex') + el_faces = plyfile.PlyElement.describe(faces_tuple, 'face') + + ply_data = plyfile.PlyData([el_verts, el_faces]) + print('saving mesh to %s' % (ply_filename_out)) + ply_data.write(ply_filename_out) + + +class Timing: + """ + Timing environment + usage: + with Timing("message"): + your commands here + will print CUDA runtime in ms + """ + + def __init__(self, name, debug=False): + self.name = name + self.debug = debug + + def __enter__(self): + if not self.debug: + return + + self.start = torch.cuda.Event(enable_timing=True) + self.end = torch.cuda.Event(enable_timing=True) + self.start.record() + + def __exit__(self, type, value, traceback): + if not self.debug: + return + + self.end.record() + torch.cuda.synchronize() + print(self.name, 'elapsed', self.start.elapsed_time(self.end), 'ms') diff --git a/runtime/Lib/site_packages/models/cv/object_detection/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8728d2f1a653f37c039f4bdd84eae754e5c7718b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mmdet_model import DetectionModel + from .yolox_pai import YOLOX + from .dino import DINO + +else: + _import_structure = { + 'mmdet_model': ['DetectionModel'], + 'yolox_pai': ['YOLOX'], + 'dino': ['DINO'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_model.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_model.py new file mode 100644 index 0000000000000000000000000000000000000000..485d440a042fcc6450bc4dc87b3bf051161aa956 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_model.py @@ -0,0 +1,94 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .mmdet_ms.backbones import ViT +from .mmdet_ms.dense_heads import RPNNHead +from .mmdet_ms.necks import FPNF +from .mmdet_ms.roi_heads import FCNMaskNHead, Shared4Conv1FCBBoxNHead + + +@MODELS.register_module(Tasks.human_detection, module_name=Models.detection) +@MODELS.register_module( + Tasks.image_object_detection, module_name=Models.detection) +class DetectionModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + + from mmcv.runner import load_checkpoint + from mmdet.datasets import replace_ImageToTensor + from mmdet.datasets.pipelines import Compose + from mmdet.models import build_detector + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + config_path = osp.join(model_dir, 'mmcv_config.py') + config = Config.from_file(config_path) + config.model.pretrained = None + self.model = build_detector(config.model) + + checkpoint = load_checkpoint( + self.model, model_path, map_location='cpu') + self.class_names = checkpoint['meta']['CLASSES'] + config.test_pipeline[0].type = 'LoadImageFromWebcam' + self.transform_input = Compose( + replace_ImageToTensor(config.test_pipeline)) + self.model.cfg = config + self.model.eval() + self.score_thr = config.score_thr + + def inference(self, data): + """data is dict,contain img and img_metas,follow with mmdet.""" + + with torch.no_grad(): + results = self.model(return_loss=False, rescale=True, **data) + return results + + def preprocess(self, image): + """image is numpy return is dict contain img and img_metas,follow with mmdet.""" + + from mmcv.parallel import collate, scatter + data = dict(img=image) + data = self.transform_input(data) + data = collate([data], samples_per_gpu=1) + data['img_metas'] = [ + img_metas.data[0] for img_metas in data['img_metas'] + ] + data['img'] = [img.data[0] for img in data['img']] + + if next(self.model.parameters()).is_cuda: + data = scatter(data, [next(self.model.parameters()).device])[0] + + return data + + def postprocess(self, inputs): + + if isinstance(inputs[0], tuple): + bbox_result, _ = inputs[0] + else: + bbox_result, _ = inputs[0], None + labels = [ + np.full(bbox.shape[0], i, dtype=np.int32) + for i, bbox in enumerate(bbox_result) + ] + labels = np.concatenate(labels) + + bbox_result = np.vstack(bbox_result) + scores = bbox_result[:, -1] + inds = scores > self.score_thr + if np.sum(np.array(inds).astype('int')) == 0: + return None, None, None + bboxes = bbox_result[inds, :] + labels = labels[inds] + scores = np.around(bboxes[:, 4], 6) + bboxes = (bboxes[:, 0:4]).astype(int) + labels = [self.class_names[i_label] for i_label in labels] + return bboxes, scores, labels diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d5f40f7b6788bd24af8719483af5b13e4f4e45c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/__init__.py @@ -0,0 +1,6 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .backbones import ViT +from .dense_heads import AnchorNHead, RPNNHead +from .necks import FPNF +from .utils import ConvModule_Norm, load_checkpoint diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89a5302827ccb068306e04de15d03b8b643fa354 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/__init__.py @@ -0,0 +1,5 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .vit import ViT + +__all__ = ['ViT'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/vit.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..53bda3580f236b80cb1b3a20333cc7f0ec04e3b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/backbones/vit.py @@ -0,0 +1,626 @@ +# -------------------------------------------------------- +# BEIT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254) +# Github source: https://github.com/microsoft/unilm/tree/master/beit +# Copyright (c) 2021 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# By Hangbo Bao +# Based on timm, mmseg, setr, xcit and swin code bases +# https://github.com/rwightman/pytorch-image-models/tree/master/timm +# https://github.com/fudan-zvg/SETR +# https://github.com/facebookresearch/xcit/ +# https://github.com/microsoft/Swin-Transformer +# --------------------------------------------------------' +import math +from functools import partial + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from mmdet.models.builder import BACKBONES +from mmdet.utils import get_root_logger +from timm.models.layers import drop_path, to_2tuple, trunc_normal_ + +from ..utils import load_checkpoint + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self): + return 'p={}'.format(self.drop_prob) + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0., + window_size=None, + attn_head_dim=None): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim**-0.5 + self.qkv = nn.Linear(dim, all_head_dim * 3, bias=qkv_bias) + self.window_size = window_size + q_size = window_size[0] + rel_sp_dim = 2 * q_size - 1 + self.rel_pos_h = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, H, W, rel_pos_bias=None): + B, N, C = x.shape + qkv = self.qkv(x) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + attn = calc_rel_pos_spatial(attn, q, self.window_size, + self.window_size, self.rel_pos_h, + self.rel_pos_w) + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +def calc_rel_pos_spatial( + attn, + q, + q_shape, + k_shape, + rel_pos_h, + rel_pos_w, +): + """ + Spatial Relative Positional Embeddings. + """ + sp_idx = 0 + q_h, q_w = q_shape + k_h, k_w = k_shape + # Scale up rel pos if shapes for q and k are different. + q_h_ratio = max(k_h / q_h, 1.0) + k_h_ratio = max(q_h / k_h, 1.0) + dist_h = ( + torch.arange(q_h)[:, None] * q_h_ratio + - torch.arange(k_h)[None, :] * k_h_ratio) + dist_h += (k_h - 1) * k_h_ratio + q_w_ratio = max(k_w / q_w, 1.0) + k_w_ratio = max(q_w / k_w, 1.0) + dist_w = ( + torch.arange(q_w)[:, None] * q_w_ratio + - torch.arange(k_w)[None, :] * k_w_ratio) + dist_w += (k_w - 1) * k_w_ratio + Rh = rel_pos_h[dist_h.long()] + Rw = rel_pos_w[dist_w.long()] + B, n_head, q_N, dim = q.shape + r_q = q[:, :, sp_idx:].reshape(B, n_head, q_h, q_w, dim) + rel_h = torch.einsum('byhwc,hkc->byhwk', r_q, Rh) + rel_w = torch.einsum('byhwc,wkc->byhwk', r_q, Rw) + attn[:, :, sp_idx:, sp_idx:] = ( + attn[:, :, sp_idx:, sp_idx:].view(B, -1, q_h, q_w, k_h, k_w) + + rel_h[:, :, :, :, :, None] + rel_w[:, :, :, :, None, :]).view( + B, -1, q_h * q_w, k_h * k_w) + + return attn + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0., + attn_head_dim=None): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + q_size = window_size[0] + rel_sp_dim = 2 * q_size - 1 + self.rel_pos_h = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, H, W): + """ Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + x = x.reshape(B_, H, W, C) + pad_l = pad_t = 0 + pad_r = (self.window_size[1] + - W % self.window_size[1]) % self.window_size[1] + pad_b = (self.window_size[0] + - H % self.window_size[0]) % self.window_size[0] + + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + x = window_partition( + x, self.window_size[0]) # nW*B, window_size, window_size, C + x = x.view(-1, self.window_size[1] * self.window_size[0], + C) # nW*B, window_size*window_size, C + B_w = x.shape[0] + N_w = x.shape[1] + qkv = self.qkv(x).reshape(B_w, N_w, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + attn = calc_rel_pos_spatial(attn, q, self.window_size, + self.window_size, self.rel_pos_h, + self.rel_pos_w) + + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_w, N_w, C) + x = self.proj(x) + x = self.proj_drop(x) + + x = x.view(-1, self.window_size[1], self.window_size[0], C) + x = window_reverse(x, self.window_size[0], Hp, Wp) # B H' W' C + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B_, H * W, C) + + return x + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + init_values=None, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + window_size=None, + attn_head_dim=None, + window=False): + super().__init__() + self.norm1 = norm_layer(dim) + if not window: + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + window_size=window_size, + attn_head_dim=attn_head_dim) + else: + self.attn = WindowAttention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + window_size=window_size, + attn_head_dim=attn_head_dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + if init_values is not None: + self.gamma_1 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + self.gamma_2 = nn.Parameter( + init_values * torch.ones((dim)), requires_grad=True) + else: + self.gamma_1, self.gamma_2 = None, None + + def forward(self, x, H, W): + if self.gamma_1 is None: + x = x + self.drop_path(self.attn(self.norm1(x), H, W)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + x = x + self.drop_path( + self.gamma_1 * self.attn(self.norm1(x), H, W)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * ( + img_size[0] // patch_size[0]) + self.patch_shape = (img_size[0] // patch_size[0], + img_size[1] // patch_size[1]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x, **kwargs): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + # assert H == self.img_size[0] and W == self.img_size[1], \ + # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + Hp, Wp = x.shape[2], x.shape[3] + + x = x.flatten(2).transpose(1, 2) + return x, (Hp, Wp) + + +class HybridEmbed(nn.Module): + """ CNN Feature Map Embedding + Extract feature map from CNN, flatten, project to embedding dim. + """ + + def __init__(self, + backbone, + img_size=224, + feature_size=None, + in_chans=3, + embed_dim=768): + super().__init__() + assert isinstance(backbone, nn.Module) + img_size = to_2tuple(img_size) + self.img_size = img_size + self.backbone = backbone + if feature_size is None: + with torch.no_grad(): + # FIXME this is hacky, but most reliable way of determining the exact dim of the output feature + # map for all networks, the feature metadata has reliable channel and stride info, but using + # stride to calc feature dim requires info about padding of each stage that isn't captured. + training = backbone.training + if training: + backbone.eval() + o = self.backbone( + torch.zeros(1, in_chans, img_size[0], img_size[1]))[-1] + feature_size = o.shape[-2:] + feature_dim = o.shape[1] + backbone.train(training) + else: + feature_size = to_2tuple(feature_size) + feature_dim = self.backbone.feature_info.channels()[-1] + self.num_patches = feature_size[0] * feature_size[1] + self.proj = nn.Linear(feature_dim, embed_dim) + + def forward(self, x): + x = self.backbone(x)[-1] + x = x.flatten(2).transpose(1, 2) + x = self.proj(x) + return x + + +class Norm2d(nn.Module): + + def __init__(self, embed_dim): + super().__init__() + self.ln = nn.LayerNorm(embed_dim, eps=1e-6) + + def forward(self, x): + x = x.permute(0, 2, 3, 1) + x = self.ln(x) + x = x.permute(0, 3, 1, 2).contiguous() + return x + + +@BACKBONES.register_module() +class ViT(nn.Module): + """ Vision Transformer with support for patch or hybrid CNN input stage + """ + + def __init__(self, + img_size=224, + patch_size=16, + in_chans=3, + num_classes=80, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + hybrid_backbone=None, + norm_layer=None, + init_values=None, + use_checkpoint=False, + use_abs_pos_emb=False, + use_rel_pos_bias=False, + use_shared_rel_pos_bias=False, + out_indices=[11], + interval=3, + pretrained=None): + super().__init__() + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + + if hybrid_backbone is not None: + self.patch_embed = HybridEmbed( + hybrid_backbone, + img_size=img_size, + in_chans=in_chans, + embed_dim=embed_dim) + else: + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim) + + num_patches = self.patch_embed.num_patches + + self.out_indices = out_indices + + if use_abs_pos_emb: + self.pos_embed = nn.Parameter( + torch.zeros(1, num_patches, embed_dim)) + else: + self.pos_embed = None + + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.use_rel_pos_bias = use_rel_pos_bias + self.use_checkpoint = use_checkpoint + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + init_values=init_values, + window_size=(14, 14) if + ((i + 1) % interval != 0) else self.patch_embed.patch_shape, + window=((i + 1) % interval != 0)) for i in range(depth) + ]) + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=.02) + + self.norm = norm_layer(embed_dim) + + self.fpn1 = nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + Norm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ) + + self.fpn2 = nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, + stride=2), ) + + self.fpn3 = nn.Identity() + + self.fpn4 = nn.MaxPool2d(kernel_size=2, stride=2) + + self.apply(self._init_weights) + self.fix_init_weight() + self.pretrained = pretrained + + def fix_init_weight(self): + + def rescale(param, layer_id): + param.div_(math.sqrt(2.0 * layer_id)) + + for layer_id, layer in enumerate(self.blocks): + rescale(layer.attn.proj.weight.data, layer_id + 1) + rescale(layer.mlp.fc2.weight.data, layer_id + 1) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + pretrained = pretrained or self.pretrained + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if isinstance(pretrained, str): + self.apply(_init_weights) + logger = get_root_logger() + print(f'load from {pretrained}') + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def get_num_layers(self): + return len(self.blocks) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def forward_features(self, x): + B, C, H, W = x.shape + x, (Hp, Wp) = self.patch_embed(x) + batch_size, seq_len, _ = x.size() + + if self.pos_embed is not None: + x = x + self.pos_embed + x = self.pos_drop(x) + + features = [] + for i, blk in enumerate(self.blocks): + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x, Hp, Wp) + + x = self.norm(x) + xp = x.permute(0, 2, 1).reshape(B, -1, Hp, Wp) + + ops = [self.fpn1, self.fpn2, self.fpn3, self.fpn4] + for i in range(len(ops)): + features.append(ops[i](xp)) + + return tuple(features) + + def forward(self, x): + + x = self.forward_features(x) + + return x diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f65587b944d54858e633d4e410097fd1611262c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/__init__.py @@ -0,0 +1,6 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .anchor_head import AnchorNHead +from .rpn_head import RPNNHead + +__all__ = ['AnchorNHead', 'RPNNHead'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/anchor_head.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/anchor_head.py new file mode 100644 index 0000000000000000000000000000000000000000..dd489cb491245ab8641a6ce193d1994a64a8c9fa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/anchor_head.py @@ -0,0 +1,49 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from mmdet.models.builder import HEADS +from mmdet.models.dense_heads import AnchorHead + + +@HEADS.register_module() +class AnchorNHead(AnchorHead): + """Anchor-based head (RPN, RetinaNet, SSD, etc.). + + Args: + num_classes (int): Number of categories excluding the background + category. + in_channels (int): Number of channels in the input feature map. + feat_channels (int): Number of hidden channels. Used in child classes. + anchor_generator (dict): Config dict for anchor generator + bbox_coder (dict): Config of bounding box coder. + reg_decoded_bbox (bool): If true, the regression loss would be + applied directly on decoded bounding boxes, converting both + the predicted boxes and regression targets to absolute + coordinates format. Default False. It should be `True` when + using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. + loss_cls (dict): Config of classification loss. + loss_bbox (dict): Config of localization loss. + train_cfg (dict): Training config of anchor head. + test_cfg (dict): Testing config of anchor head. + init_cfg (dict or list[dict], optional): Initialization config dict. + """ # noqa: W605 + + def __init__(self, + num_classes, + in_channels, + feat_channels, + anchor_generator=None, + bbox_coder=None, + reg_decoded_bbox=False, + loss_cls=None, + loss_bbox=None, + train_cfg=None, + test_cfg=None, + norm_cfg=None, + init_cfg=None): + self.norm_cfg = norm_cfg + super(AnchorNHead, + self).__init__(num_classes, in_channels, feat_channels, + anchor_generator, bbox_coder, reg_decoded_bbox, + loss_cls, loss_bbox, train_cfg, test_cfg, + init_cfg) diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/rpn_head.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/rpn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..8de7c1a424fff23845cddef3729192407594745f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/dense_heads/rpn_head.py @@ -0,0 +1,269 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +import copy + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.ops import batched_nms +from mmdet.models.builder import HEADS + +from ..utils import ConvModule_Norm +from .anchor_head import AnchorNHead + + +@HEADS.register_module() +class RPNNHead(AnchorNHead): + """RPN head. + + Args: + in_channels (int): Number of channels in the input feature map. + init_cfg (dict or list[dict], optional): Initialization config dict. + num_convs (int): Number of convolution layers in the head. Default 1. + """ # noqa: W605 + + def __init__(self, + in_channels, + init_cfg=dict(type='Normal', layer='Conv2d', std=0.01), + num_convs=1, + **kwargs): + self.num_convs = num_convs + super(RPNNHead, self).__init__( + 1, in_channels, init_cfg=init_cfg, **kwargs) + + def _init_layers(self): + """Initialize layers of the head.""" + if self.num_convs > 1: + rpn_convs = [] + for i in range(self.num_convs): + if i == 0: + in_channels = self.in_channels + else: + in_channels = self.feat_channels + # use ``inplace=False`` to avoid error: one of the variables + # needed for gradient computation has been modified by an + # inplace operation. + rpn_convs.append( + ConvModule_Norm( + in_channels, + self.feat_channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + inplace=False)) + self.rpn_conv = nn.Sequential(*rpn_convs) + else: + self.rpn_conv = nn.Conv2d( + self.in_channels, self.feat_channels, 3, padding=1) + self.rpn_cls = nn.Conv2d(self.feat_channels, + self.num_base_priors * self.cls_out_channels, + 1) + self.rpn_reg = nn.Conv2d(self.feat_channels, self.num_base_priors * 4, + 1) + + def forward_single(self, x): + """Forward feature map of a single scale level.""" + x = self.rpn_conv(x) + x = F.relu(x, inplace=True) + rpn_cls_score = self.rpn_cls(x) + rpn_bbox_pred = self.rpn_reg(x) + return rpn_cls_score, rpn_bbox_pred + + def loss(self, + cls_scores, + bbox_preds, + gt_bboxes, + img_metas, + gt_bboxes_ignore=None): + """Compute losses of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_anchors * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_anchors * 4, H, W) + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + losses = super(RPNNHead, self).loss( + cls_scores, + bbox_preds, + gt_bboxes, + None, + img_metas, + gt_bboxes_ignore=gt_bboxes_ignore) + return dict( + loss_rpn_cls=losses['loss_cls'], loss_rpn_bbox=losses['loss_bbox']) + + def _get_bboxes_single(self, + cls_score_list, + bbox_pred_list, + score_factor_list, + mlvl_anchors, + img_meta, + cfg, + rescale=False, + with_nms=True, + **kwargs): + """Transform outputs of a single image into bbox predictions. + + Args: + cls_score_list (list[Tensor]): Box scores from all scale + levels of a single image, each item has shape + (num_anchors * num_classes, H, W). + bbox_pred_list (list[Tensor]): Box energies / deltas from + all scale levels of a single image, each item has + shape (num_anchors * 4, H, W). + score_factor_list (list[Tensor]): Score factor from all scale + levels of a single image. RPN head does not need this value. + mlvl_anchors (list[Tensor]): Anchors of all scale level + each item has shape (num_anchors, 4). + img_meta (dict): Image meta info. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + Tensor: Labeled boxes in shape (n, 5), where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. + """ + cfg = self.test_cfg if cfg is None else cfg + cfg = copy.deepcopy(cfg) + img_shape = img_meta['img_shape'] + + # bboxes from different level should be independent during NMS, + # level_ids are used as labels for batched NMS to separate them + level_ids = [] + mlvl_scores = [] + mlvl_bbox_preds = [] + mlvl_valid_anchors = [] + nms_pre = cfg.get('nms_pre', -1) + for level_idx in range(len(cls_score_list)): + rpn_cls_score = cls_score_list[level_idx] + rpn_bbox_pred = bbox_pred_list[level_idx] + assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] + rpn_cls_score = rpn_cls_score.permute(1, 2, 0) + if self.use_sigmoid_cls: + rpn_cls_score = rpn_cls_score.reshape(-1) + scores = rpn_cls_score.sigmoid() + else: + rpn_cls_score = rpn_cls_score.reshape(-1, 2) + # We set FG labels to [0, num_class-1] and BG label to + # num_class in RPN head since mmdet v2.5, which is unified to + # be consistent with other head since mmdet v2.0. In mmdet v2.0 + # to v2.4 we keep BG label as 0 and FG label as 1 in rpn head. + scores = rpn_cls_score.softmax(dim=1)[:, 0] + rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, 4) + + anchors = mlvl_anchors[level_idx] + if 0 < nms_pre < scores.shape[0]: + # sort is faster than topk + # _, topk_inds = scores.topk(cfg.nms_pre) + ranked_scores, rank_inds = scores.sort(descending=True) + topk_inds = rank_inds[:nms_pre] + scores = ranked_scores[:nms_pre] + rpn_bbox_pred = rpn_bbox_pred[topk_inds, :] + anchors = anchors[topk_inds, :] + + mlvl_scores.append(scores) + mlvl_bbox_preds.append(rpn_bbox_pred) + mlvl_valid_anchors.append(anchors) + level_ids.append( + scores.new_full((scores.size(0), ), + level_idx, + dtype=torch.long)) + + return self._bbox_post_process(mlvl_scores, mlvl_bbox_preds, + mlvl_valid_anchors, level_ids, cfg, + img_shape) + + def _bbox_post_process(self, mlvl_scores, mlvl_bboxes, mlvl_valid_anchors, + level_ids, cfg, img_shape, **kwargs): + """bbox post-processing method. + + The boxes would be rescaled to the original image scale and do + the nms operation. Usually with_nms is False is used for aug test. + + Args: + mlvl_scores (list[Tensor]): Box scores from all scale + levels of a single image, each item has shape + (num_bboxes, num_class). + mlvl_bboxes (list[Tensor]): Decoded bboxes from all scale + levels of a single image, each item has shape (num_bboxes, 4). + mlvl_valid_anchors (list[Tensor]): Anchors of all scale level + each item has shape (num_bboxes, 4). + level_ids (list[Tensor]): Indexes from all scale levels of a + single image, each item has shape (num_bboxes, ). + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + img_shape (tuple(int)): Shape of current image. + + Returns: + Tensor: Labeled boxes in shape (n, 5), where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. + """ + scores = torch.cat(mlvl_scores) + anchors = torch.cat(mlvl_valid_anchors) + rpn_bbox_pred = torch.cat(mlvl_bboxes) + proposals = self.bbox_coder.decode( + anchors, rpn_bbox_pred, max_shape=img_shape) + ids = torch.cat(level_ids) + + if cfg.min_bbox_size >= 0: + w = proposals[:, 2] - proposals[:, 0] + h = proposals[:, 3] - proposals[:, 1] + valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) + if not valid_mask.all(): + proposals = proposals[valid_mask] + scores = scores[valid_mask] + ids = ids[valid_mask] + + if proposals.numel() > 0: + dets, _ = batched_nms(proposals, scores, ids, cfg.nms) + else: + return proposals.new_zeros(0, 5) + + return dets[:cfg.max_per_img] + + def onnx_export(self, x, img_metas): + """Test without augmentation. + + Args: + x (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + img_metas (list[dict]): Meta info of each image. + Returns: + Tensor: dets of shape [N, num_det, 5]. + """ + cls_scores, bbox_preds = self(x) + + assert len(cls_scores) == len(bbox_preds) + + batch_bboxes, batch_scores = super(RPNNHead, self).onnx_export( + cls_scores, bbox_preds, img_metas=img_metas, with_nms=False) + # Use ONNX::NonMaxSuppression in deployment + from mmdet.core.export import add_dummy_nms_for_onnx + cfg = copy.deepcopy(self.test_cfg) + score_threshold = cfg.nms.get('score_thr', 0.0) + nms_pre = cfg.get('deploy_nms_pre', -1) + # Different from the normal forward doing NMS level by level, + # we do NMS across all levels when exporting ONNX. + dets, _ = add_dummy_nms_for_onnx(batch_bboxes, batch_scores, + cfg.max_per_img, + cfg.nms.iou_threshold, + score_threshold, nms_pre, + cfg.max_per_img) + return dets diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..244ecbf0efa756c057916ff99587805c02a2ec11 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/__init__.py @@ -0,0 +1,5 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .fpn import FPNF + +__all__ = ['FPNF'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/fpn.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..5a6a82229993733ad372bc1e4cba46e966997854 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/necks/fpn.py @@ -0,0 +1,208 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +import torch.nn as nn +import torch.nn.functional as F +from mmcv.runner import BaseModule, auto_fp16 +from mmdet.models.builder import NECKS + +from ..utils import ConvModule_Norm + + +@NECKS.register_module() +class FPNF(BaseModule): + r"""Feature Pyramid Network. + + This is an implementation of paper `Feature Pyramid Networks for Object + Detection `_. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, it is equivalent to `add_extra_convs='on_input'`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + init_cfg (dict or list[dict], optional): Initialization config dict. + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + use_residual=True, + upsample_cfg=dict(mode='nearest'), + init_cfg=dict( + type='Xavier', layer='Conv2d', distribution='uniform')): + super(FPNF, self).__init__(init_cfg) + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + self.use_residual = use_residual + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + self.add_extra_convs = 'on_input' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule_Norm( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule_Norm( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule_Norm( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + @auto_fp16() + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + if self.use_residual: + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + + return tuple(outs) diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60bf212d93664dbe0d12f2166416b945090f5b84 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/__init__.py @@ -0,0 +1,10 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .bbox_heads import (ConvFCBBoxNHead, Shared2FCBBoxNHead, + Shared4Conv1FCBBoxNHead) +from .mask_heads import FCNMaskNHead + +__all__ = [ + 'ConvFCBBoxNHead', 'Shared2FCBBoxNHead', 'Shared4Conv1FCBBoxNHead', + 'FCNMaskNHead' +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e0b83ae667dbd2ce2097453cb88bfa65ea33233 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/__init__.py @@ -0,0 +1,6 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .convfc_bbox_head import (ConvFCBBoxNHead, Shared2FCBBoxNHead, + Shared4Conv1FCBBoxNHead) + +__all__ = ['ConvFCBBoxNHead', 'Shared2FCBBoxNHead', 'Shared4Conv1FCBBoxNHead'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/convfc_bbox_head.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/convfc_bbox_head.py new file mode 100644 index 0000000000000000000000000000000000000000..f0c8c3988f61d9790bba8346a1a0dbcb73c5a867 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/bbox_heads/convfc_bbox_head.py @@ -0,0 +1,230 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +import torch.nn as nn +from mmdet.models.builder import HEADS +from mmdet.models.roi_heads.bbox_heads.bbox_head import BBoxHead +from mmdet.models.utils import build_linear_layer + +from ...utils import ConvModule_Norm + + +@HEADS.register_module() +class ConvFCBBoxNHead(BBoxHead): + r"""More general bbox head, with shared conv and fc layers and two optional + separated branches. + + .. code-block:: none + + /-> cls convs -> cls fcs -> cls + shared convs -> shared fcs + \-> reg convs -> reg fcs -> reg + """ # noqa: W605 + + def __init__(self, + num_shared_convs=0, + num_shared_fcs=0, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + conv_out_channels=256, + fc_out_channels=1024, + conv_cfg=None, + norm_cfg=None, + init_cfg=None, + *args, + **kwargs): + super(ConvFCBBoxNHead, self).__init__( + *args, init_cfg=init_cfg, **kwargs) + assert (num_shared_convs + num_shared_fcs + num_cls_convs + num_cls_fcs + + num_reg_convs + num_reg_fcs > 0) + if num_cls_convs > 0 or num_reg_convs > 0: + assert num_shared_fcs == 0 + if not self.with_cls: + assert num_cls_convs == 0 and num_cls_fcs == 0 + if not self.with_reg: + assert num_reg_convs == 0 and num_reg_fcs == 0 + self.num_shared_convs = num_shared_convs + self.num_shared_fcs = num_shared_fcs + self.num_cls_convs = num_cls_convs + self.num_cls_fcs = num_cls_fcs + self.num_reg_convs = num_reg_convs + self.num_reg_fcs = num_reg_fcs + self.conv_out_channels = conv_out_channels + self.fc_out_channels = fc_out_channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + + # add shared convs and fcs + self.shared_convs, self.shared_fcs, last_layer_dim = \ + self._add_conv_fc_branch( + self.num_shared_convs, self.num_shared_fcs, self.in_channels, + True) + self.shared_out_channels = last_layer_dim + + # add cls specific branch + self.cls_convs, self.cls_fcs, self.cls_last_dim = \ + self._add_conv_fc_branch( + self.num_cls_convs, self.num_cls_fcs, self.shared_out_channels) + + # add reg specific branch + self.reg_convs, self.reg_fcs, self.reg_last_dim = \ + self._add_conv_fc_branch( + self.num_reg_convs, self.num_reg_fcs, self.shared_out_channels) + + if self.num_shared_fcs == 0 and not self.with_avg_pool: + if self.num_cls_fcs == 0: + self.cls_last_dim *= self.roi_feat_area + if self.num_reg_fcs == 0: + self.reg_last_dim *= self.roi_feat_area + + self.relu = nn.ReLU(inplace=True) + # reconstruct fc_cls and fc_reg since input channels are changed + if self.with_cls: + if self.custom_cls_channels: + cls_channels = self.loss_cls.get_cls_channels(self.num_classes) + else: + cls_channels = self.num_classes + 1 + self.fc_cls = build_linear_layer( + self.cls_predictor_cfg, + in_features=self.cls_last_dim, + out_features=cls_channels) + if self.with_reg: + out_dim_reg = (4 if self.reg_class_agnostic else 4 + * self.num_classes) + self.fc_reg = build_linear_layer( + self.reg_predictor_cfg, + in_features=self.reg_last_dim, + out_features=out_dim_reg) + + if init_cfg is None: + # when init_cfg is None, + # It has been set to + # [[dict(type='Normal', std=0.01, override=dict(name='fc_cls'))], + # [dict(type='Normal', std=0.001, override=dict(name='fc_reg'))] + # after `super(ConvFCBBoxHead, self).__init__()` + # we only need to append additional configuration + # for `shared_fcs`, `cls_fcs` and `reg_fcs` + self.init_cfg += [ + dict( + type='Xavier', + override=[ + dict(name='shared_fcs'), + dict(name='cls_fcs'), + dict(name='reg_fcs') + ]) + ] + + def _add_conv_fc_branch(self, + num_branch_convs, + num_branch_fcs, + in_channels, + is_shared=False): + """Add shared or separable branch. + + convs -> avg pool (optional) -> fcs + """ + last_layer_dim = in_channels + # add branch specific conv layers + branch_convs = nn.ModuleList() + if num_branch_convs > 0: + for i in range(num_branch_convs): + conv_in_channels = ( + last_layer_dim if i == 0 else self.conv_out_channels) + branch_convs.append( + ConvModule_Norm( + conv_in_channels, + self.conv_out_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg)) + last_layer_dim = self.conv_out_channels + # add branch specific fc layers + branch_fcs = nn.ModuleList() + if num_branch_fcs > 0: + # for shared branch, only consider self.with_avg_pool + # for separated branches, also consider self.num_shared_fcs + if (is_shared + or self.num_shared_fcs == 0) and not self.with_avg_pool: + last_layer_dim *= self.roi_feat_area + for i in range(num_branch_fcs): + fc_in_channels = ( + last_layer_dim if i == 0 else self.fc_out_channels) + branch_fcs.append( + nn.Linear(fc_in_channels, self.fc_out_channels)) + last_layer_dim = self.fc_out_channels + return branch_convs, branch_fcs, last_layer_dim + + def forward(self, x): + # shared part + if self.num_shared_convs > 0: + for conv in self.shared_convs: + x = conv(x) + + if self.num_shared_fcs > 0: + if self.with_avg_pool: + x = self.avg_pool(x) + + x = x.flatten(1) + + for fc in self.shared_fcs: + x = self.relu(fc(x)) + # separate branches + x_cls = x + x_reg = x + + for conv in self.cls_convs: + x_cls = conv(x_cls) + if x_cls.dim() > 2: + if self.with_avg_pool: + x_cls = self.avg_pool(x_cls) + x_cls = x_cls.flatten(1) + for fc in self.cls_fcs: + x_cls = self.relu(fc(x_cls)) + + for conv in self.reg_convs: + x_reg = conv(x_reg) + if x_reg.dim() > 2: + if self.with_avg_pool: + x_reg = self.avg_pool(x_reg) + x_reg = x_reg.flatten(1) + for fc in self.reg_fcs: + x_reg = self.relu(fc(x_reg)) + + cls_score = self.fc_cls(x_cls) if self.with_cls else None + bbox_pred = self.fc_reg(x_reg) if self.with_reg else None + return cls_score, bbox_pred + + +@HEADS.register_module() +class Shared2FCBBoxNHead(ConvFCBBoxNHead): + + def __init__(self, fc_out_channels=1024, *args, **kwargs): + super(Shared2FCBBoxNHead, self).__init__( + num_shared_convs=0, + num_shared_fcs=2, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + fc_out_channels=fc_out_channels, + *args, + **kwargs) + + +@HEADS.register_module() +class Shared4Conv1FCBBoxNHead(ConvFCBBoxNHead): + + def __init__(self, fc_out_channels=1024, *args, **kwargs): + super(Shared4Conv1FCBBoxNHead, self).__init__( + num_shared_convs=4, + num_shared_fcs=1, + num_cls_convs=0, + num_cls_fcs=0, + num_reg_convs=0, + num_reg_fcs=0, + fc_out_channels=fc_out_channels, + *args, + **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b3f46ba8184bb47c272971b3f23b95eaf28e2526 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/__init__.py @@ -0,0 +1,5 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .fcn_mask_head import FCNMaskNHead + +__all__ = ['FCNMaskNHead'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/fcn_mask_head.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/fcn_mask_head.py new file mode 100644 index 0000000000000000000000000000000000000000..ed90249f377540605bd660283b247a4e49c0888f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/roi_heads/mask_heads/fcn_mask_head.py @@ -0,0 +1,415 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from warnings import warn + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule, build_conv_layer, build_upsample_layer +from mmcv.ops.carafe import CARAFEPack +from mmcv.runner import BaseModule, ModuleList, auto_fp16, force_fp32 +from mmdet.core import mask_target +from mmdet.models.builder import HEADS, build_loss +from torch.nn.modules.utils import _pair + +from ...utils import ConvModule_Norm + +BYTES_PER_FLOAT = 4 +# TODO: This memory limit may be too much or too little. It would be better to +# determine it based on available resources. +GPU_MEM_LIMIT = 1024**3 # 1 GB memory limit + + +@HEADS.register_module() +class FCNMaskNHead(BaseModule): + + def __init__(self, + num_convs=4, + roi_feat_size=14, + in_channels=256, + conv_kernel_size=3, + conv_out_channels=256, + num_classes=80, + class_agnostic=False, + upsample_cfg=dict(type='deconv', scale_factor=2), + conv_cfg=None, + norm_cfg=None, + predictor_cfg=dict(type='Conv'), + loss_mask=dict( + type='CrossEntropyLoss', use_mask=True, loss_weight=1.0), + init_cfg=None): + assert init_cfg is None, 'To prevent abnormal initialization ' \ + 'behavior, init_cfg is not allowed to be set' + super(FCNMaskNHead, self).__init__(init_cfg) + self.upsample_cfg = upsample_cfg.copy() + if self.upsample_cfg['type'] not in [ + None, 'deconv', 'nearest', 'bilinear', 'carafe' + ]: + raise ValueError( + f'Invalid upsample method {self.upsample_cfg["type"]}, ' + 'accepted methods are "deconv", "nearest", "bilinear", ' + '"carafe"') + self.num_convs = num_convs + # WARN: roi_feat_size is reserved and not used + self.roi_feat_size = _pair(roi_feat_size) + self.in_channels = in_channels + self.conv_kernel_size = conv_kernel_size + self.conv_out_channels = conv_out_channels + self.upsample_method = self.upsample_cfg.get('type') + self.scale_factor = self.upsample_cfg.pop('scale_factor', None) + self.num_classes = num_classes + self.class_agnostic = class_agnostic + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.predictor_cfg = predictor_cfg + self.fp16_enabled = False + self.loss_mask = build_loss(loss_mask) + + self.convs = ModuleList() + for i in range(self.num_convs): + in_channels = ( + self.in_channels if i == 0 else self.conv_out_channels) + padding = (self.conv_kernel_size - 1) // 2 + self.convs.append( + ConvModule_Norm( + in_channels, + self.conv_out_channels, + self.conv_kernel_size, + padding=padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg)) + upsample_in_channels = ( + self.conv_out_channels if self.num_convs > 0 else in_channels) + upsample_cfg_ = self.upsample_cfg.copy() + if self.upsample_method is None: + self.upsample = None + elif self.upsample_method == 'deconv': + upsample_cfg_.update( + in_channels=upsample_in_channels, + out_channels=self.conv_out_channels, + kernel_size=self.scale_factor, + stride=self.scale_factor) + self.upsample = build_upsample_layer(upsample_cfg_) + elif self.upsample_method == 'carafe': + upsample_cfg_.update( + channels=upsample_in_channels, scale_factor=self.scale_factor) + self.upsample = build_upsample_layer(upsample_cfg_) + else: + # suppress warnings + align_corners = (None + if self.upsample_method == 'nearest' else False) + upsample_cfg_.update( + scale_factor=self.scale_factor, + mode=self.upsample_method, + align_corners=align_corners) + self.upsample = build_upsample_layer(upsample_cfg_) + + out_channels = 1 if self.class_agnostic else self.num_classes + logits_in_channel = ( + self.conv_out_channels + if self.upsample_method == 'deconv' else upsample_in_channels) + self.conv_logits = build_conv_layer(self.predictor_cfg, + logits_in_channel, out_channels, 1) + self.relu = nn.ReLU(inplace=True) + self.debug_imgs = None + + def init_weights(self): + super(FCNMaskNHead, self).init_weights() + for m in [self.upsample, self.conv_logits]: + if m is None: + continue + elif isinstance(m, CARAFEPack): + m.init_weights() + elif hasattr(m, 'weight') and hasattr(m, 'bias'): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + nn.init.constant_(m.bias, 0) + + @auto_fp16() + def forward(self, x): + for conv in self.convs: + x = conv(x) + if self.upsample is not None: + x = self.upsample(x) + if self.upsample_method == 'deconv': + x = self.relu(x) + mask_pred = self.conv_logits(x) + return mask_pred + + def get_targets(self, sampling_results, gt_masks, rcnn_train_cfg): + pos_proposals = [res.pos_bboxes for res in sampling_results] + pos_assigned_gt_inds = [ + res.pos_assigned_gt_inds for res in sampling_results + ] + mask_targets = mask_target(pos_proposals, pos_assigned_gt_inds, + gt_masks, rcnn_train_cfg) + return mask_targets + + @force_fp32(apply_to=('mask_pred', )) + def loss(self, mask_pred, mask_targets, labels): + """ + Example: + >>> from mmdet.models.roi_heads.mask_heads.fcn_mask_head import * # NOQA + >>> N = 7 # N = number of extracted ROIs + >>> C, H, W = 11, 32, 32 + >>> # Create example instance of FCN Mask Head. + >>> # There are lots of variations depending on the configuration + >>> self = FCNMaskHead(num_classes=C, num_convs=1) + >>> inputs = torch.rand(N, self.in_channels, H, W) + >>> mask_pred = self.forward(inputs) + >>> sf = self.scale_factor + >>> labels = torch.randint(0, C, size=(N,)) + >>> # With the default properties the mask targets should indicate + >>> # a (potentially soft) single-class label + >>> mask_targets = torch.rand(N, H * sf, W * sf) + >>> loss = self.loss(mask_pred, mask_targets, labels) + >>> print('loss = {!r}'.format(loss)) + """ + loss = dict() + if mask_pred.size(0) == 0: + loss_mask = mask_pred.sum() + else: + if self.class_agnostic: + loss_mask = self.loss_mask(mask_pred, mask_targets, + torch.zeros_like(labels)) + else: + loss_mask = self.loss_mask(mask_pred, mask_targets, labels) + loss['loss_mask'] = loss_mask + return loss + + def get_seg_masks(self, mask_pred, det_bboxes, det_labels, rcnn_test_cfg, + ori_shape, scale_factor, rescale): + """Get segmentation masks from mask_pred and bboxes. + + Args: + mask_pred (Tensor or ndarray): shape (n, #class, h, w). + For single-scale testing, mask_pred is the direct output of + model, whose type is Tensor, while for multi-scale testing, + it will be converted to numpy array outside of this method. + det_bboxes (Tensor): shape (n, 4/5) + det_labels (Tensor): shape (n, ) + rcnn_test_cfg (dict): rcnn testing config + ori_shape (Tuple): original image height and width, shape (2,) + scale_factor(ndarray | Tensor): If ``rescale is True``, box + coordinates are divided by this scale factor to fit + ``ori_shape``. + rescale (bool): If True, the resulting masks will be rescaled to + ``ori_shape``. + + Returns: + list[list]: encoded masks. The c-th item in the outer list + corresponds to the c-th class. Given the c-th outer list, the + i-th item in that inner list is the mask for the i-th box with + class label c. + + Example: + >>> import mmcv + >>> from mmdet.models.roi_heads.mask_heads.fcn_mask_head import * # NOQA + >>> N = 7 # N = number of extracted ROIs + >>> C, H, W = 11, 32, 32 + >>> # Create example instance of FCN Mask Head. + >>> self = FCNMaskHead(num_classes=C, num_convs=0) + >>> inputs = torch.rand(N, self.in_channels, H, W) + >>> mask_pred = self.forward(inputs) + >>> # Each input is associated with some bounding box + >>> det_bboxes = torch.Tensor([[1, 1, 42, 42 ]] * N) + >>> det_labels = torch.randint(0, C, size=(N,)) + >>> rcnn_test_cfg = mmcv.Config({'mask_thr_binary': 0, }) + >>> ori_shape = (H * 4, W * 4) + >>> scale_factor = torch.FloatTensor((1, 1)) + >>> rescale = False + >>> # Encoded masks are a list for each category. + >>> encoded_masks = self.get_seg_masks( + >>> mask_pred, det_bboxes, det_labels, rcnn_test_cfg, ori_shape, + >>> scale_factor, rescale + >>> ) + >>> assert len(encoded_masks) == C + >>> assert sum(list(map(len, encoded_masks))) == N + """ + if isinstance(mask_pred, torch.Tensor): + mask_pred = mask_pred.sigmoid() + else: + # In AugTest, has been activated before + mask_pred = det_bboxes.new_tensor(mask_pred) + + device = mask_pred.device + cls_segms = [[] for _ in range(self.num_classes) + ] # BG is not included in num_classes + bboxes = det_bboxes[:, :4] + labels = det_labels + + # In most cases, scale_factor should have been + # converted to Tensor when rescale the bbox + if not isinstance(scale_factor, torch.Tensor): + if isinstance(scale_factor, float): + scale_factor = np.array([scale_factor] * 4) + warn('Scale_factor should be a Tensor or ndarray ' + 'with shape (4,), float would be deprecated. ') + assert isinstance(scale_factor, np.ndarray) + scale_factor = torch.Tensor(scale_factor) + + if rescale: + img_h, img_w = ori_shape[:2] + bboxes = bboxes / scale_factor.to(bboxes) + else: + w_scale, h_scale = scale_factor[0], scale_factor[1] + img_h = np.round(ori_shape[0] * h_scale.item()).astype(np.int32) + img_w = np.round(ori_shape[1] * w_scale.item()).astype(np.int32) + + N = len(mask_pred) + # The actual implementation split the input into chunks, + # and paste them chunk by chunk. + if device.type == 'cpu': + # CPU is most efficient when they are pasted one by one with + # skip_empty=True, so that it performs minimal number of + # operations. + num_chunks = N + else: + # GPU benefits from parallelism for larger chunks, + # but may have memory issue + # the types of img_w and img_h are np.int32, + # when the image resolution is large, + # the calculation of num_chunks will overflow. + # so we need to change the types of img_w and img_h to int. + # See https://github.com/open-mmlab/mmdetection/pull/5191 + num_chunks = int( + np.ceil(N * int(img_h) * int(img_w) * BYTES_PER_FLOAT + / GPU_MEM_LIMIT)) + # assert (num_chunks <= N), 'Default GPU_MEM_LIMIT is too small; try increasing it' + assert num_chunks <= N, 'Default GPU_MEM_LIMIT is too small; try increasing it' + chunks = torch.chunk(torch.arange(N, device=device), num_chunks) + + threshold = rcnn_test_cfg.mask_thr_binary + im_mask = torch.zeros( + N, + img_h, + img_w, + device=device, + dtype=torch.bool if threshold >= 0 else torch.uint8) + + if not self.class_agnostic: + mask_pred = mask_pred[range(N), labels][:, None] + + for inds in chunks: + masks_chunk, spatial_inds = _do_paste_mask( + mask_pred[inds], + bboxes[inds], + img_h, + img_w, + skip_empty=device.type == 'cpu') + + if threshold >= 0: + masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool) + else: + # for visualization and debugging + masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8) + + im_mask[(inds, ) + spatial_inds] = masks_chunk + + for i in range(N): + cls_segms[labels[i]].append(im_mask[i].detach().cpu().numpy()) + return cls_segms + + def onnx_export(self, mask_pred, det_bboxes, det_labels, rcnn_test_cfg, + ori_shape, **kwargs): + """Get segmentation masks from mask_pred and bboxes. + + Args: + mask_pred (Tensor): shape (n, #class, h, w). + det_bboxes (Tensor): shape (n, 4/5) + det_labels (Tensor): shape (n, ) + rcnn_test_cfg (dict): rcnn testing config + ori_shape (Tuple): original image height and width, shape (2,) + + Returns: + Tensor: a mask of shape (N, img_h, img_w). + """ + + mask_pred = mask_pred.sigmoid() + bboxes = det_bboxes[:, :4] + labels = det_labels + # No need to consider rescale and scale_factor while exporting to ONNX + img_h, img_w = ori_shape[:2] + threshold = rcnn_test_cfg.mask_thr_binary + if not self.class_agnostic: + box_inds = torch.arange(mask_pred.shape[0]) + mask_pred = mask_pred[box_inds, labels][:, None] + masks, _ = _do_paste_mask( + mask_pred, bboxes, img_h, img_w, skip_empty=False) + if threshold >= 0: + # should convert to float to avoid problems in TRT + masks = (masks >= threshold).to(dtype=torch.float) + return masks + + +def _do_paste_mask(masks, boxes, img_h, img_w, skip_empty=True): + """Paste instance masks according to boxes. + + This implementation is modified from + https://github.com/facebookresearch/detectron2/ + + Args: + masks (Tensor): N, 1, H, W + boxes (Tensor): N, 4 + img_h (int): Height of the image to be pasted. + img_w (int): Width of the image to be pasted. + skip_empty (bool): Only paste masks within the region that + tightly bound all boxes, and returns the results this region only. + An important optimization for CPU. + + Returns: + tuple: (Tensor, tuple). The first item is mask tensor, the second one + is the slice object. + If skip_empty == False, the whole image will be pasted. It will + return a mask of shape (N, img_h, img_w) and an empty tuple. + If skip_empty == True, only area around the mask will be pasted. + A mask of shape (N, h', w') and its start and end coordinates + in the original image will be returned. + """ + # On GPU, paste all masks together (up to chunk size) + # by using the entire image to sample the masks + # Compared to pasting them one by one, + # this has more operations but is faster on COCO-scale dataset. + device = masks.device + if skip_empty: + x0_int, y0_int = torch.clamp( + boxes.min(dim=0).values.floor()[:2] - 1, + min=0).to(dtype=torch.int32) + x1_int = torch.clamp( + boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) + y1_int = torch.clamp( + boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) + else: + x0_int, y0_int = 0, 0 + x1_int, y1_int = img_w, img_h + x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 + + N = masks.shape[0] + + img_y = torch.arange(y0_int, y1_int, device=device).to(torch.float32) + 0.5 + img_x = torch.arange(x0_int, x1_int, device=device).to(torch.float32) + 0.5 + img_y = (img_y - y0) / (y1 - y0) * 2 - 1 + img_x = (img_x - x0) / (x1 - x0) * 2 - 1 + # img_x, img_y have shapes (N, w), (N, h) + # IsInf op is not supported with ONNX<=1.7.0 + if not torch.onnx.is_in_onnx_export(): + if torch.isinf(img_x).any(): + inds = torch.where(torch.isinf(img_x)) + img_x[inds] = 0 + if torch.isinf(img_y).any(): + inds = torch.where(torch.isinf(img_y)) + img_y[inds] = 0 + + gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) + gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) + grid = torch.stack([gx, gy], dim=3) + + img_masks = F.grid_sample( + masks.to(dtype=torch.float32), grid, align_corners=False) + + if skip_empty: + return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) + else: + return img_masks[:, 0], () diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b24d5a3dd945aec721d7ff1aafd0d2ffece8d85b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/__init__.py @@ -0,0 +1,6 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from .checkpoint import load_checkpoint +from .convModule_norm import ConvModule_Norm + +__all__ = ['load_checkpoint', 'ConvModule_Norm'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/checkpoint.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..be153caaa62c7cd5dd013e9b6fd0a8defc777f67 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/checkpoint.py @@ -0,0 +1,559 @@ +# Copyright (c) Open-MMLab. All rights reserved. +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +import io +import os +import os.path as osp +import pkgutil +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import mmcv +import torch +import torchvision +from mmcv.fileio import FileClient +from mmcv.fileio import load as load_file +from mmcv.parallel import is_module_wrapper +from mmcv.runner import get_dist_info +from torch.nn import functional as F +from torch.optim import Optimizer +from torch.utils import model_zoo + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + print('finish load') + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def load_pavimodel_dist(model_path, map_location=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load( + downloaded_file, map_location=map_location) + return checkpoint + + +def load_fileclient_dist(filename, backend, map_location): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + allowed_backends = ['ceph'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + if rank == 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + return new_checkpoint + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('torchvision://'): + model_urls = get_torchvision_models() + model_name = filename[14:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('open-mmlab://'): + model_urls = get_external_models() + model_name = filename[13:] + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'open-mmlab://{model_name} is deprecated in favor ' + f'of open-mmlab://{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_url_dist(model_url) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + elif filename.startswith('mmcls://'): + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_url_dist(model_urls[model_name]) + checkpoint = _process_mmcls_checkpoint(checkpoint) + elif filename.startswith(('http://', 'https://')): + checkpoint = load_url_dist(filename) + elif filename.startswith('pavi://'): + model_path = filename[7:] + checkpoint = load_pavimodel_dist(model_path, map_location=map_location) + elif filename.startswith('s3://'): + checkpoint = load_fileclient_dist( + filename, backend='ceph', map_location=map_location) + else: + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, + filename, + map_location='cpu', + strict=False, + logger=None, + load_ema=True): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if load_ema and 'state_dict_ema' in checkpoint: + state_dict = checkpoint['state_dict_ema'] + # logger.info(f'loading from state_dict_ema') + logger.info('loading from state_dict_ema') + elif 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + # logger.info(f'loading from state_dict') + logger.info('loading from state_dict') + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + # logger.info(f'loading from model') + logger.info('loading from model') + print('loading from model') + else: + state_dict = checkpoint + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = { + k.replace('encoder.', ''): v + for k, v in state_dict.items() if k.startswith('encoder.') + } + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H * W: + logger.warning('Error in loading absolute_pos_embed, pass') + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view( + N2, H, W, C2).permute(0, 3, 1, 2) + + all_keys = list(state_dict.keys()) + for key in all_keys: + if 'relative_position_index' in key: + state_dict.pop(key) + + if 'relative_position_bias_table' in key: + state_dict.pop(key) + + if '.q_bias' in key: + q_bias = state_dict[key] + v_bias = state_dict[key.replace('q_bias', 'v_bias')] + qkv_bias = torch.cat([q_bias, torch.zeros_like(q_bias), v_bias], 0) + state_dict[key.replace('q_bias', 'qkv.bias')] = qkv_bias + + if '.v.bias' in key: + continue + + all_keys = list(state_dict.keys()) + new_state_dict = {} + for key in all_keys: + if 'qkv.bias' in key: + value = state_dict[key] + dim = value.shape[0] + selected_dim = (dim * 2) // 3 + new_state_dict[key.replace( + 'qkv.bias', 'pos_bias')] = state_dict[key][:selected_dim] + + # interpolate position bias table if needed + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + if table_key not in model.state_dict().keys(): + logger.warning( + 'relative_position_bias_table exits in pretrained model but not in current one, pass' + ) + continue + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + logger.warning(f'Error in loading {table_key}, pass') + else: + if L1 != L2: + S1 = int(L1**0.5) + S2 = int(L2**0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), + mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view( + nH2, L2).permute(1, 0) + rank, _ = get_dist_info() + if 'pos_embed' in state_dict: + pos_embed_checkpoint = state_dict['pos_embed'] + embedding_size = pos_embed_checkpoint.shape[-1] + H, W = model.patch_embed.patch_shape + num_patches = model.patch_embed.num_patches + num_extra_tokens = 1 + # height (== width) for the checkpoint position embedding + orig_size = int( + (pos_embed_checkpoint.shape[-2] - num_extra_tokens)**0.5) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + if rank == 0: + print('Position interpolate from %dx%d to %dx%d' % + (orig_size, orig_size, H, W)) + # extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, + embedding_size).permute( + 0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(H, W), mode='bicubic', align_corners=False) + new_pos_embed = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + # new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + state_dict['pos_embed'] = new_pos_embed + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() diff --git a/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/convModule_norm.py b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/convModule_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..f0e9a7680b86430766dae013931aa4b5453392cc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection/mmdet_ms/utils/convModule_norm.py @@ -0,0 +1,30 @@ +# Implementation in this file is modified based on ViTAE-Transformer +# Originally Apache 2.0 License and publicly available at https://github.com/ViTAE-Transformer/ViTDet +from mmcv.cnn import ConvModule + + +class ConvModule_Norm(ConvModule): + + def __init__(self, in_channels, out_channels, kernel, **kwargs): + super().__init__(in_channels, out_channels, kernel, **kwargs) + + self.normType = kwargs.get('norm_cfg', {'type': ''}) + if self.normType is not None: + self.normType = self.normType['type'] + + def forward(self, x, activate=True, norm=True): + for layer in self.order: + if layer == 'conv': + if self.with_explicit_padding: + x = self.padding_layer(x) + x = self.conv(x) + elif layer == 'norm' and norm and self.with_norm: + if 'LN' in self.normType: + x = x.permute(0, 2, 3, 1) + x = self.norm(x) + x = x.permute(0, 3, 1, 2).contiguous() + else: + x = self.norm(x) + elif layer == 'act' and activate and self.with_activation: + x = self.activate(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2477a104ecf0c5c7a1b2e0bf1c7ecc688528bb91 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .depe import ScrfdDetect +else: + _import_structure = { + 'depe': ['DepeDetect'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1576aa9f87ce7c6d07da83b7fb8311939681025f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .depe_detect import DepeDetect diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/depe_detect.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/depe_detect.py new file mode 100644 index 0000000000000000000000000000000000000000..a6112a166ebbe355f0891b142cedc19e91415d0a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/depe_detect.py @@ -0,0 +1,69 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import os.path as osp +from typing import Any, Dict, List, Union + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['DepeDetect'] + + +@MODELS.register_module(Tasks.object_detection_3d, module_name=Models.depe) +class DepeDetect(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """DEPE is a simple and pure DETR-like 3D detector with transformers, + for more information please refer to: + https://www.modelscope.cn/models/damo/cv_object-detection-3d_depe/summary + + initialize the 3d object detection model from the `model_dir` path. + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + from mmcv.runner import load_checkpoint + import modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin + from modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin.models.detectors import Petr3D + + # build model and load checkpoint + config_path = osp.join(model_dir, ModelFile.CONFIGURATION) + config = Config.from_file(config_path) + detector = Petr3D(**config.model.network_param) + model_file = kwargs.get('model_file', ModelFile.TORCH_MODEL_BIN_FILE) + ckpt_path = osp.join(model_dir, model_file) + logger.info(f'loading model from {ckpt_path}') + load_checkpoint(detector, ckpt_path, map_location='cpu') + detector.eval() + self.detector = detector + logger.info('load model done') + + def forward(self, img: Union[torch.Tensor, List[torch.Tensor]], + img_metas: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Args: + img (`torch.Tensor`): batched image tensor or list of batched image tensor, + shape of each tensor is [B, N, C, H, W], N is 12 for 6 views from current + and history frame. + img_metas (` List[Dict[str, Any]`): image meta info. + Return: + result (`List[Dict[str, Any]]`): list of detection results. + """ + + if isinstance(img, torch.Tensor): + img = [img] + img_metas = [img_metas] + + result = self.detector( + return_loss=False, rescale=True, img=img, img_metas=img_metas) + assert result is not None + return result diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..af0a990a7471243c1acb603a31c33b7b9876c297 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/__init__.py @@ -0,0 +1,13 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin +""" +from .core.bbox.assigners import HungarianAssigner3D +from .core.bbox.coders import NMSFreeCoder +from .core.bbox.match_costs import BBox3DL1Cost +from .datasets import CustomNuScenesDataset +from .datasets.pipelines import NormalizeMultiviewImage, PadMultiViewImage +from .models.backbones import VoVNet +from .models.dense_heads import PETRv2DEDNHead +from .models.detectors import Petr3D +from .models.necks import CPFPN diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..166ad20d707de08e291b0a747a37ce3bcb85e44c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/assigners +""" +from .hungarian_assigner_3d import HungarianAssigner3D + +__all__ = ['HungarianAssigner3D'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_3d.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_3d.py new file mode 100644 index 0000000000000000000000000000000000000000..3bae8ddd46ed7fbb5d99d9a75d65c01fad74dce6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_3d.py @@ -0,0 +1,139 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/assigners +""" +import torch +from mmdet.core.bbox.assigners import AssignResult, BaseAssigner +from mmdet.core.bbox.builder import BBOX_ASSIGNERS +from mmdet.core.bbox.match_costs import build_match_cost +from mmdet.models.utils.transformer import inverse_sigmoid + +from modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin.core.bbox.util import \ + normalize_bbox + +try: + from scipy.optimize import linear_sum_assignment +except ImportError: + linear_sum_assignment = None + + +@BBOX_ASSIGNERS.register_module() +class HungarianAssigner3D(BaseAssigner): + """Computes one-to-one matching between predictions and ground truth. + This class computes an assignment between the targets and the predictions + based on the costs. The costs are weighted sum of three components: + classification cost, regression L1 cost and regression iou cost. The + targets don't include the no_object, so generally there are more + predictions than targets. After the one-to-one matching, the un-matched + are treated as backgrounds. Thus each query prediction will be assigned + with `0` or a positive integer indicating the ground truth index: + - 0: negative sample, no assigned gt + - positive integer: positive sample, index (1-based) of assigned gt + Args: + cls_weight (int | float, optional): The scale factor for classification + cost. Default 1.0. + bbox_weight (int | float, optional): The scale factor for regression + L1 cost. Default 1.0. + iou_weight (int | float, optional): The scale factor for regression + iou cost. Default 1.0. + iou_calculator (dict | optional): The config for the iou calculation. + Default type `BboxOverlaps2D`. + iou_mode (str | optional): "iou" (intersection over union), "iof" + (intersection over foreground), or "giou" (generalized + intersection over union). Default "giou". + """ + + def __init__(self, + cls_cost=dict(type='ClassificationCost', weight=1.), + reg_cost=dict(type='BBoxL1Cost', weight=1.0), + iou_cost=dict(type='IoUCost', weight=0.0), + pc_range=None): + self.cls_cost = build_match_cost(cls_cost) + self.reg_cost = build_match_cost(reg_cost) + self.iou_cost = build_match_cost(iou_cost) + self.pc_range = pc_range + + def assign(self, + bbox_pred, + cls_pred, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + eps=1e-7): + """Computes one-to-one matching based on the weighted costs. + This method assign each query prediction to a ground truth or + background. The `assigned_gt_inds` with -1 means don't care, + 0 means negative sample, and positive number is the index (1-based) + of assigned gt. + The assignment is done in the following steps, the order matters. + 1. assign every prediction to -1 + 2. compute the weighted costs + 3. do Hungarian matching on CPU based on the costs + 4. assign all to 0 (background) first, then for each matched pair + between predictions and gts, treat this prediction as foreground + and assign the corresponding gt index (plus 1) to it. + Args: + bbox_pred (Tensor): Predicted boxes with normalized coordinates + (cx, cy, w, h), which are all in range [0, 1]. Shape + [num_query, 4]. + cls_pred (Tensor): Predicted classification logits, shape + [num_query, num_class]. + gt_bboxes (Tensor): Ground truth boxes with unnormalized + coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. + gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`. Default None. + eps (int | float, optional): A value added to the denominator for + numerical stability. Default 1e-7. + Returns: + :obj:`AssignResult`: The assigned result. + """ + assert gt_bboxes_ignore is None, \ + 'Only case when gt_bboxes_ignore is None is supported.' + num_gts, num_bboxes = gt_bboxes.size(0), bbox_pred.size(0) + + # 1. assign -1 by default + assigned_gt_inds = bbox_pred.new_full((num_bboxes, ), + -1, + dtype=torch.long) + assigned_labels = bbox_pred.new_full((num_bboxes, ), + -1, + dtype=torch.long) + if num_gts == 0 or num_bboxes == 0: + # No ground truth or boxes, return empty assignment + if num_gts == 0: + # No ground truth, assign all to background + assigned_gt_inds[:] = 0 + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) + + # 2. compute the weighted costs + # classification and bboxcost. + cls_cost = self.cls_cost(cls_pred, gt_labels) + # regression L1 cost + normalized_gt_bboxes = normalize_bbox(gt_bboxes, self.pc_range) + reg_cost = self.reg_cost(bbox_pred[:, :8], normalized_gt_bboxes[:, :8]) + + # weighted sum of above two costs + cost = cls_cost + reg_cost + + # 3. do Hungarian matching on CPU using linear_sum_assignment + cost = cost.detach().cpu() + if linear_sum_assignment is None: + raise ImportError('Please run "pip install scipy" ' + 'to install scipy first.') + cost = torch.nan_to_num(cost, nan=100.0, posinf=100.0, neginf=-100.0) + matched_row_inds, matched_col_inds = linear_sum_assignment(cost) + matched_row_inds = torch.from_numpy(matched_row_inds).to( + bbox_pred.device) + matched_col_inds = torch.from_numpy(matched_col_inds).to( + bbox_pred.device) + + # 4. assign backgrounds and foregrounds + # assign all indices to backgrounds first + assigned_gt_inds[:] = 0 + # assign foregrounds based on matching results + assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 + assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] + return AssignResult( + num_gts, assigned_gt_inds, None, labels=assigned_labels) diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..036b17f0c4b27925fb5ff92389db2745206c012b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/__init__.py @@ -0,0 +1,9 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/coders +""" +from .nms_free_coder import NMSFreeCoder + +__all__ = [ + 'NMSFreeCoder', +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/nms_free_coder.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/nms_free_coder.py new file mode 100644 index 0000000000000000000000000000000000000000..5388e7cd9174487861ed2e88070891389bcc77e8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/coders/nms_free_coder.py @@ -0,0 +1,118 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/coders +""" +import torch +import torch.nn.functional as F +from mmdet.core.bbox import BaseBBoxCoder +from mmdet.core.bbox.builder import BBOX_CODERS + +from modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin.core.bbox.util import \ + denormalize_bbox + + +@BBOX_CODERS.register_module() +class NMSFreeCoder(BaseBBoxCoder): + """Bbox coder for NMS-free detector. + Args: + pc_range (list[float]): Range of point cloud. + post_center_range (list[float]): Limit of the center. + Default: None. + max_num (int): Max number to be kept. Default: 100. + score_threshold (float): Threshold to filter boxes based on score. + Default: None. + code_size (int): Code size of bboxes. Default: 9 + """ + + def __init__(self, + pc_range, + voxel_size=None, + post_center_range=None, + max_num=100, + score_threshold=None, + num_classes=10): + + self.pc_range = pc_range + self.voxel_size = voxel_size + self.post_center_range = post_center_range + self.max_num = max_num + self.score_threshold = score_threshold + self.num_classes = num_classes + + def encode(self): + pass + + def decode_single(self, cls_scores, bbox_preds): + """Decode bboxes. + Args: + cls_scores (Tensor): Outputs from the classification head, \ + shape [num_query, cls_out_channels]. Note \ + cls_out_channels should includes background. + bbox_preds (Tensor): Outputs from the regression \ + head with normalized coordinate format (cx, cy, w, l, cz, h, rot_sine, rot_cosine, vx, vy). \ + Shape [num_query, 9]. + Returns: + list[dict]: Decoded boxes. + """ + max_num = self.max_num + + cls_scores = cls_scores.sigmoid() + scores, indexs = cls_scores.view(-1).topk(max_num) + labels = indexs % self.num_classes + bbox_index = torch.div(indexs, self.num_classes, rounding_mode='trunc') + bbox_preds = bbox_preds[bbox_index] + + final_box_preds = denormalize_bbox(bbox_preds, self.pc_range) + final_scores = scores + final_preds = labels + + # use score threshold + if self.score_threshold is not None: + thresh_mask = final_scores > self.score_threshold + if self.post_center_range is not None: + self.post_center_range = torch.tensor( + self.post_center_range, device=scores.device) + tmp = final_box_preds[..., :3] >= self.post_center_range[:3] + mask = (tmp).all(1) + tmp = final_box_preds[..., :3] <= self.post_center_range[3:] + mask &= (tmp).all(1) + + if self.score_threshold: + mask &= thresh_mask + + boxes3d = final_box_preds[mask] + scores = final_scores[mask] + labels = final_preds[mask] + predictions_dict = { + 'bboxes': boxes3d, + 'scores': scores, + 'labels': labels + } + + else: + raise NotImplementedError( + 'Need to reorganize output as a batch, only ' + 'support post_center_range is not None for now!') + return predictions_dict + + def decode(self, preds_dicts): + """Decode bboxes. + Args: + all_cls_scores (Tensor): Outputs from the classification head, \ + shape [nb_dec, bs, num_query, cls_out_channels]. Note \ + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression \ + head with normalized coordinate format (cx, cy, w, l, cz, h, rot_sine, rot_cosine, vx, vy). \ + Shape [nb_dec, bs, num_query, 9]. + Returns: + list[dict]: Decoded boxes. + """ + all_cls_scores = preds_dicts['all_cls_scores'][-1] + all_bbox_preds = preds_dicts['all_bbox_preds'][-1] + + batch_size = all_cls_scores.size()[0] + predictions_list = [] + for i in range(batch_size): + predictions_list.append( + self.decode_single(all_cls_scores[i], all_bbox_preds[i])) + return predictions_list diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..daf3de04ef7f26dff25ca148c80579fbd979e8b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/match_costs +""" +from .match_cost import BBox3DL1Cost + +__all__ = ['BBox3DL1Cost'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/match_cost.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/match_cost.py new file mode 100644 index 0000000000000000000000000000000000000000..000354fa1e831af20160a678ae1358552f858c8f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/match_costs/match_cost.py @@ -0,0 +1,31 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox/match_costs +""" +import torch +from mmdet.core.bbox.match_costs.builder import MATCH_COST + + +@MATCH_COST.register_module() +class BBox3DL1Cost(object): + """BBox3DL1Cost. + Args: + weight (int | float, optional): loss_weight + """ + + def __init__(self, weight=1.): + self.weight = weight + + def __call__(self, bbox_pred, gt_bboxes): + """ + Args: + bbox_pred (Tensor): Predicted boxes with normalized coordinates + (cx, cy, w, h), which are all in range [0, 1]. Shape + [num_query, 4]. + gt_bboxes (Tensor): Ground truth boxes with normalized + coordinates (x1, y1, x2, y2). Shape [num_gt, 4]. + Returns: + torch.Tensor: bbox_cost value with weight + """ + bbox_cost = torch.cdist(bbox_pred, gt_bboxes, p=1) + return bbox_cost * self.weight diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/util.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/util.py new file mode 100644 index 0000000000000000000000000000000000000000..52c1bf049d1f119636ab59edad2d1f3971f81821 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/core/bbox/util.py @@ -0,0 +1,67 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/core/bbox +""" +import mmdet3d +import numpy as np +import torch + + +def normalize_bbox(bboxes, pc_range): + cx = bboxes[..., 0:1] + cy = bboxes[..., 1:2] + cz = bboxes[..., 2:3] + w = bboxes[..., 3:4].log() + _l = bboxes[..., 4:5].log() + h = bboxes[..., 5:6].log() + + rot = bboxes[..., 6:7] + if bboxes.size(-1) > 7: + vx = bboxes[..., 7:8] + vy = bboxes[..., 8:9] + normalized_bboxes = torch.cat( + (cx, cy, w, _l, cz, h, rot.sin(), rot.cos(), vx, vy), dim=-1) + else: + normalized_bboxes = torch.cat( + (cx, cy, w, _l, cz, h, rot.sin(), rot.cos()), dim=-1) + return normalized_bboxes + + +def denormalize_bbox(normalized_bboxes, pc_range): + nan = normalized_bboxes.isnan().sum() + if nan > 0: + print(f'found {nan} nan!') + normalized_bboxes = torch.nan_to_num(normalized_bboxes, 0.1) + # rotation + rot_sine = normalized_bboxes[..., 6:7] + rot_cosine = normalized_bboxes[..., 7:8] + rot = torch.atan2(rot_sine, rot_cosine) + + # center in the bev + cx = normalized_bboxes[..., 0:1] + cy = normalized_bboxes[..., 1:2] + cz = normalized_bboxes[..., 4:5] + + # size + w = normalized_bboxes[..., 2:3] + _l = normalized_bboxes[..., 3:4] + h = normalized_bboxes[..., 5:6] + + w = w.exp() + _l = _l.exp() + h = h.exp() + # check mmdet3d version + if int(mmdet3d.__version__.split('.')[0]) > 0: + tmp = w + w = _l + _l = tmp + rot = -rot - 0.5 * np.pi + if normalized_bboxes.size(-1) > 8: + # velocity + vx = normalized_bboxes[:, 8:9] + vy = normalized_bboxes[:, 9:10] + denormalized_bboxes = torch.cat([cx, cy, cz, w, _l, h, rot, vx, vy], + dim=-1) + else: + denormalized_bboxes = torch.cat([cx, cy, cz, w, _l, h, rot], dim=-1) + return denormalized_bboxes diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d679075658951df86ac59787c379ad4f860c51 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/__init__.py @@ -0,0 +1,9 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/datasets +""" +from .nuscenes_dataset import CustomNuScenesDataset + +__all__ = [ + 'CustomNuScenesDataset', +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/nuscenes_dataset.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/nuscenes_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5069c8e255ca64ba64b6ee973533b19b66375360 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/nuscenes_dataset.py @@ -0,0 +1,77 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/datasets +""" +import numpy as np +from mmdet3d.datasets import NuScenesDataset +from mmdet.datasets import DATASETS + + +@DATASETS.register_module() +class CustomNuScenesDataset(NuScenesDataset): + r"""NuScenes Dataset. + This datset only add camera intrinsics and extrinsics to the results. + """ + + def __init__(self, idx_range=None, **kwargs): + self.idx_range = idx_range + super().__init__(**kwargs) + if idx_range is not None: + assert isinstance(idx_range, (tuple, list)) + assert len(idx_range) == 2 + assert idx_range[0] < idx_range[1] + assert idx_range[1] <= len( + self.data_infos + ), f'the idx_range {idx_range} exceeds total number of dataset:{len(self.data_infos)}' + self.data_infos = self.data_infos[idx_range[0]:idx_range[1]] + + def get_data_info(self, index): + info = self.data_infos[index] + # standard protocal modified from SECOND.Pytorch + input_dict = dict( + sample_idx=info['token'], + pts_filename=[info['lidar_path']], + sweeps=info['sweeps'], + timestamp=info['timestamp'] / 1e6, + ) + + if self.modality['use_camera']: + image_paths = [] + lidar2img_rts = [] + intrinsics = [] + extrinsics = [] + extrinsics_sweep = [] + img_timestamp = [] + for cam_type, cam_info in info['cams'].items(): + img_timestamp.append(cam_info['timestamp'] / 1e6) + image_paths.append(cam_info['data_path']) + # obtain lidar to image transformation matrix + lidar2cam_r = np.linalg.inv(cam_info['sensor2lidar_rotation']) + lidar2cam_t = cam_info[ + 'sensor2lidar_translation'] @ lidar2cam_r.T + lidar2cam_rt = np.eye(4) + lidar2cam_rt[:3, :3] = lidar2cam_r.T + lidar2cam_rt[3, :3] = -lidar2cam_t + intrinsic = cam_info['cam_intrinsic'] + viewpad = np.eye(4) + viewpad[:intrinsic.shape[0], :intrinsic.shape[1]] = intrinsic + lidar2img_rt = (viewpad @ lidar2cam_rt.T) + intrinsics.append(viewpad) + extrinsics.append(lidar2cam_rt) + lidar2img_rts.append(lidar2img_rt) + extrinsics_sweep.append(None) # placeholder for sweeps + + input_dict.update( + dict( + img_timestamp=img_timestamp, + img_filename=image_paths, + lidar2img=lidar2img_rts, + intrinsics=intrinsics, + extrinsics=extrinsics, + extrinsics_sweep=extrinsics_sweep, + )) + + if not self.test_mode: + annos = self.get_ann_info(index) + input_dict['ann_info'] = annos + return input_dict diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e86a9ed40509cded4fce01927977bfcb5708b268 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/__init__.py @@ -0,0 +1,12 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/datasets/pipelines +""" +from .loading import LoadMultiViewImageFromMultiSweepsFiles +from .transform_3d import (NormalizeMultiviewImage, PadMultiViewImage, + ResizeCropFlipImage) + +__all__ = [ + 'PadMultiViewImage', 'NormalizeMultiviewImage', + 'LoadMultiViewImageFromMultiSweepsFiles', 'ResizeCropFlipImage' +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/loading.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..9443828c7e4902ae34d5f5dd42aa074dbbc566a2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/loading.py @@ -0,0 +1,177 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/datasets/pipelines +""" +import mmcv +import numpy as np +from mmdet.datasets.builder import PIPELINES + + +@PIPELINES.register_module() +class LoadMultiViewImageFromMultiSweepsFiles(object): + """Load multi channel images from a list of separate channel files. + Expects results['img_filename'] to be a list of filenames. + Args: + to_float32 (bool): Whether to convert the img to float32. + Defaults to False. + color_type (str): Color type of the file. Defaults to 'unchanged'. + """ + + def __init__( + self, + sweeps_num=5, + to_float32=False, + file_client_args=dict(backend='disk'), + pad_empty_sweeps=False, + sweep_range=[3, 27], + sweeps_id=None, + color_type='unchanged', + sensors=[ + 'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_FRONT_LEFT', 'CAM_BACK', + 'CAM_BACK_LEFT', 'CAM_BACK_RIGHT' + ], + test_mode=True, + prob=1.0, + ): + + self.sweeps_num = sweeps_num + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + self.pad_empty_sweeps = pad_empty_sweeps + self.sensors = sensors + self.test_mode = test_mode + self.sweeps_id = sweeps_id + self.sweep_range = sweep_range + self.prob = prob + if self.sweeps_id: + assert len(self.sweeps_id) == self.sweeps_num + + def __call__(self, results): + """Call function to load multi-view image from files. + Args: + results (dict): Result dict containing multi-view image filenames. + Returns: + dict: The result dict containing the multi-view image data. \ + Added keys and values are described below. + - filename (str): Multi-view image filenames. + - img (np.ndarray): Multi-view image arrays. + - img_shape (tuple[int]): Shape of multi-view image arrays. + - ori_shape (tuple[int]): Shape of original image arrays. + - pad_shape (tuple[int]): Shape of padded image arrays. + - scale_factor (float): Scale factor. + - img_norm_cfg (dict): Normalization configuration of images. + """ + sweep_imgs_list = [] + timestamp_imgs_list = [] + imgs = results['img'] + img_timestamp = results['img_timestamp'] + lidar_timestamp = results['timestamp'] + img_timestamp = [ + lidar_timestamp - timestamp for timestamp in img_timestamp + ] + sweep_imgs_list.extend(imgs) + timestamp_imgs_list.extend(img_timestamp) + nums = len(imgs) + if self.pad_empty_sweeps and len(results['sweeps']) == 0: + for i in range(self.sweeps_num): + sweep_imgs_list.extend(imgs) + results['pts_filename'] += [results['pts_filename'][0]] + mean_time = (self.sweep_range[0] + + self.sweep_range[1]) / 2.0 * 0.083 + timestamp_imgs_list.extend( + [time + mean_time for time in img_timestamp]) + for j in range(nums): + results['filename'].append(results['filename'][j]) + results['lidar2img'].append( + np.copy(results['lidar2img'][j])) + results['intrinsics'].append( + np.copy(results['intrinsics'][j])) + results['extrinsics'].append( + np.copy(results['extrinsics'][j])) + else: + if self.sweeps_id: + choices = self.sweeps_id + elif len(results['sweeps']) <= self.sweeps_num: + bin_size = len(results['sweeps']) / (self.sweeps_num + 1) + choices = [ + int(np.floor((i + 1) * bin_size)) + for i in range(self.sweeps_num) + ] + elif self.test_mode: + if self.sweep_range[1] <= len(results['sweeps']): + sweep_range = list( + range(self.sweep_range[0], self.sweep_range[1])) + elif self.sweep_range[0] >= len(results['sweeps']): + sweep_range = list(range(0, len(results['sweeps']))) + else: + sweep_range = list( + range(self.sweep_range[0], len(results['sweeps']))) + if len(sweep_range) <= self.sweeps_num: + sweep_range = list(range(0, len(results['sweeps']))) + bin_size = len(sweep_range) / (self.sweeps_num + 1) + choices = [ + sweep_range[0] + int(np.floor((i + 1) * bin_size)) + for i in range(self.sweeps_num) + ] + else: + if np.random.random() < self.prob: + if self.sweep_range[1] <= len(results['sweeps']): + sweep_range = list( + range(self.sweep_range[0], self.sweep_range[1])) + elif self.sweep_range[0] >= len(results['sweeps']): + sweep_range = list(range(0, len(results['sweeps']))) + else: + sweep_range = list( + range(self.sweep_range[0], len(results['sweeps']))) + if len(sweep_range) <= self.sweeps_num: + sweep_range = list( + range(0, len(results['sweeps']))) + choices = np.random.choice( + sweep_range, self.sweeps_num, replace=False) + else: + bin_size = len(results['sweeps']) / (self.sweeps_num + 1) + choices = [ + int(np.floor((i + 1) * bin_size)) + for i in range(self.sweeps_num) + ] + choices = sorted(choices) + for idx in choices: + sweep_idx = min(idx, len(results['sweeps']) - 1) + sweep = results['sweeps'][sweep_idx] + if 'lidar_path' in sweep: + results['pts_filename'] += [sweep['lidar_path']] + if len(sweep.keys()) < len(self.sensors): + sweep = results['sweeps'][sweep_idx - 1] + results['filename'].extend( + [sweep[sensor]['data_path'] for sensor in self.sensors]) + tmp = [ + mmcv.imread(sweep[sensor]['data_path'], self.color_type) + for sensor in self.sensors + ] + img = np.stack(tmp, axis=-1) + if self.to_float32: + img = img.astype(np.float32) + img = [img[..., i] for i in range(img.shape[-1])] + sweep_imgs_list.extend(img) + sweep_ts = [ + lidar_timestamp - sweep[sensor]['timestamp'] / 1e6 + for sensor in self.sensors + ] + timestamp_imgs_list.extend(sweep_ts) + for sensor in self.sensors: + results['lidar2img'].append(sweep[sensor]['lidar2img']) + results['intrinsics'].append(sweep[sensor]['intrinsics']) + results['extrinsics'].append(sweep[sensor]['extrinsics']) + results['img'] = sweep_imgs_list + results['timestamp'] = timestamp_imgs_list + + return results + + def __repr__(self): + """str: Return a string that describes the module.""" + repr_str = self.__class__.__name__ + repr_str += f'(to_float32={self.to_float32}, ' + repr_str += f"color_type='{self.color_type}')" + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/transform_3d.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/transform_3d.py new file mode 100644 index 0000000000000000000000000000000000000000..7b912318b0fff107cd066e9563b0b15fa41b3d0b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/datasets/pipelines/transform_3d.py @@ -0,0 +1,234 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/datasets/pipelines +""" +import copy + +import mmcv +import mmdet3d +import numpy as np +import torch +from mmdet.datasets.builder import PIPELINES +from numpy import random +from PIL import Image + + +@PIPELINES.register_module() +class PadMultiViewImage(object): + """Pad the multi-view image. + There are two padding modes: (1) pad to a fixed size and (2) pad to the + minimum size that is divisible by some number. + Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor", + Args: + size (tuple, optional): Fixed padding size. + size_divisor (int, optional): The divisor of padded size. + pad_val (float, optional): Padding value, 0 by default. + """ + + def __init__(self, size=None, size_divisor=None, pad_val=0): + self.size = size + self.size_divisor = size_divisor + self.pad_val = pad_val + # only one of size and size_divisor should be valid + assert size is not None or size_divisor is not None + assert size is None or size_divisor is None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + if self.size is not None: + padded_img = [ + mmcv.impad(img, shape=self.size, pad_val=self.pad_val) + for img in results['img'] + ] + elif self.size_divisor is not None: + padded_img = [ + mmcv.impad_to_multiple( + img, self.size_divisor, pad_val=self.pad_val) + for img in results['img'] + ] + results['img_shape'] = [img.shape for img in results['img']] + results['img'] = padded_img + results['pad_shape'] = [img.shape for img in padded_img] + results['pad_fixed_size'] = self.size + results['pad_size_divisor'] = self.size_divisor + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Updated result dict. + """ + self._pad_img(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, ' + repr_str += f'size_divisor={self.size_divisor}, ' + repr_str += f'pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class NormalizeMultiviewImage(object): + """Normalize the image. + Added key is "img_norm_cfg". + Args: + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB, + default is true. + """ + + def __init__(self, mean, std, to_rgb=True): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) + self.to_rgb = to_rgb + + def __call__(self, results): + """Call function to normalize images. + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Normalized results, 'img_norm_cfg' key is added into + result dict. + """ + results['img'] = [ + mmcv.imnormalize(img, self.mean, self.std, self.to_rgb) + for img in results['img'] + ] + results['img_norm_cfg'] = dict( + mean=self.mean, std=self.std, to_rgb=self.to_rgb) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, std={self.std}, to_rgb={self.to_rgb})' + return repr_str + + +@PIPELINES.register_module() +class ResizeCropFlipImage(object): + """Random resize, Crop and flip the image + Args: + size (tuple, optional): Fixed padding size. + """ + + def __init__(self, data_aug_conf=None, training=True, diff_aug=False): + self.data_aug_conf = data_aug_conf + self.training = training + self.diff_aug = diff_aug # whether use different aug in all views + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Updated result dict. + """ + + imgs = results['img'] + N = len(imgs) + new_imgs = [] + results['intrin_ori'] = [] + results['extrin_ori'] = [] + results['ida_mat'] = [] + results['bda_mat'] = [np.eye(4)] * len( + results['lidar2img']) # no bda when test + if not self.diff_aug: + resize, resize_dims, crop, flip, rotate = self._sample_augmentation( + ) + for i in range(N): + img = Image.fromarray(np.uint8(imgs[i])) + # augmentation (resize, crop, horizontal flip, rotate) + if self.diff_aug: + resize, resize_dims, crop, flip, rotate = self._sample_augmentation( + ) # different view use different aug (BEV Det) + img, ida_mat = self._img_transform( + img, + resize=resize, + resize_dims=resize_dims, + crop=crop, + flip=flip, + rotate=rotate, + ) + new_imgs.append(np.array(img).astype(np.float32)) + # --save intrin&extrin for depthnet input_mlp-- + results['intrin_ori'] += [results['intrinsics'][i][:3, :3].copy()] + # results['extrin_ori'] += [results['extrinsics'][i % 6].T.copy()] # all sweeps use keyframe's lidar2cam + results['extrin_ori'] += [results['extrinsics'][i].T.copy()] + results['ida_mat'] += [ida_mat.cpu().numpy()] + # --save intrin&extrin for depthnet input_mlp-- + results['intrinsics'][ + i][:3, :3] = ida_mat @ results['intrinsics'][i][:3, :3] + results['img'] = new_imgs + results['lidar2img'] = [ + results['intrinsics'][i] @ results['extrinsics'][i].T + for i in range(len(results['extrinsics'])) + ] + + return results + + def _get_rot(self, h): + + return torch.Tensor([ + [np.cos(h), np.sin(h)], + [-np.sin(h), np.cos(h)], + ]) + + def _img_transform(self, img, resize, resize_dims, crop, flip, rotate): + ida_rot = torch.eye(2) + ida_tran = torch.zeros(2) + # adjust image + img = img.resize(resize_dims) + img = img.crop(crop) + if flip: + img = img.transpose(method=Image.FLIP_LEFT_RIGHT) + img = img.rotate(rotate) + + # post-homography transformation + ida_rot *= resize + ida_tran -= torch.Tensor(crop[:2]) + if flip: + A = torch.Tensor([[-1, 0], [0, 1]]) + b = torch.Tensor([crop[2] - crop[0], 0]) + ida_rot = A.matmul(ida_rot) + ida_tran = A.matmul(ida_tran) + b + A = self._get_rot(rotate / 180 * np.pi) + b = torch.Tensor([crop[2] - crop[0], crop[3] - crop[1]]) / 2 + b = A.matmul(-b) + b + ida_rot = A.matmul(ida_rot) + ida_tran = A.matmul(ida_tran) + b + ida_mat = torch.eye(3) + ida_mat[:2, :2] = ida_rot + ida_mat[:2, 2] = ida_tran + return img, ida_mat + + def _sample_augmentation(self): + H, W = self.data_aug_conf['H'], self.data_aug_conf['W'] + fH, fW = self.data_aug_conf['final_dim'] + if self.training: + resize = np.random.uniform(*self.data_aug_conf['resize_lim']) + resize_dims = (int(W * resize), int(H * resize)) + newW, newH = resize_dims + crop_h = int( + (1 - np.random.uniform(*self.data_aug_conf['bot_pct_lim'])) + * newH) - fH + crop_w = int(np.random.uniform(0, max(0, newW - fW))) + crop = (crop_w, crop_h, crop_w + fW, crop_h + fH) + flip = False + if self.data_aug_conf['rand_flip'] and np.random.choice([0, 1]): + flip = True + rotate = np.random.uniform(*self.data_aug_conf['rot_lim']) + else: + resize = max(fH / H, fW / W) + resize_dims = (int(W * resize), int(H * resize)) + newW, newH = resize_dims + crop_h = int( + (1 - np.mean(self.data_aug_conf['bot_pct_lim'])) * newH) - fH + crop_w = int(max(0, newW - fW) / 2) + crop = (crop_w, crop_h, crop_w + fW, crop_h + fH) + flip = False + rotate = 0 + return resize, resize_dims, crop, flip, rotate diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..089ceb5474b019ceac00a6bc60771974caf8551a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/backbones +""" +from .vovnet import VoVNet + +__all__ = ['VoVNet'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/vovnet.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/vovnet.py new file mode 100644 index 0000000000000000000000000000000000000000..9107f9bc4839665115798a5b0feaf5fce41fa2bb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/backbones/vovnet.py @@ -0,0 +1,441 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/backbones +""" +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.runner import BaseModule +from mmdet.models.builder import BACKBONES +from torch.nn.modules.batchnorm import _BatchNorm + +VoVNet19_slim_dw_eSE = { + 'stem': [64, 64, 64], + 'stage_conv_ch': [64, 80, 96, 112], + 'stage_out_ch': [112, 256, 384, 512], + 'layer_per_block': 3, + 'block_per_stage': [1, 1, 1, 1], + 'eSE': True, + 'dw': True +} + +VoVNet19_dw_eSE = { + 'stem': [64, 64, 64], + 'stage_conv_ch': [128, 160, 192, 224], + 'stage_out_ch': [256, 512, 768, 1024], + 'layer_per_block': 3, + 'block_per_stage': [1, 1, 1, 1], + 'eSE': True, + 'dw': True +} + +VoVNet19_slim_eSE = { + 'stem': [64, 64, 128], + 'stage_conv_ch': [64, 80, 96, 112], + 'stage_out_ch': [112, 256, 384, 512], + 'layer_per_block': 3, + 'block_per_stage': [1, 1, 1, 1], + 'eSE': True, + 'dw': False +} + +VoVNet19_eSE = { + 'stem': [64, 64, 128], + 'stage_conv_ch': [128, 160, 192, 224], + 'stage_out_ch': [256, 512, 768, 1024], + 'layer_per_block': 3, + 'block_per_stage': [1, 1, 1, 1], + 'eSE': True, + 'dw': False +} + +VoVNet39_eSE = { + 'stem': [64, 64, 128], + 'stage_conv_ch': [128, 160, 192, 224], + 'stage_out_ch': [256, 512, 768, 1024], + 'layer_per_block': 5, + 'block_per_stage': [1, 1, 2, 2], + 'eSE': True, + 'dw': False +} + +VoVNet57_eSE = { + 'stem': [64, 64, 128], + 'stage_conv_ch': [128, 160, 192, 224], + 'stage_out_ch': [256, 512, 768, 1024], + 'layer_per_block': 5, + 'block_per_stage': [1, 1, 4, 3], + 'eSE': True, + 'dw': False +} + +VoVNet99_eSE = { + 'stem': [64, 64, 128], + 'stage_conv_ch': [128, 160, 192, 224], + 'stage_out_ch': [256, 512, 768, 1024], + 'layer_per_block': 5, + 'block_per_stage': [1, 3, 9, 3], + 'eSE': True, + 'dw': False +} + +_STAGE_SPECS = { + 'V-19-slim-dw-eSE': VoVNet19_slim_dw_eSE, + 'V-19-dw-eSE': VoVNet19_dw_eSE, + 'V-19-slim-eSE': VoVNet19_slim_eSE, + 'V-19-eSE': VoVNet19_eSE, + 'V-39-eSE': VoVNet39_eSE, + 'V-57-eSE': VoVNet57_eSE, + 'V-99-eSE': VoVNet99_eSE, +} + + +def dw_conv3x3(in_channels, + out_channels, + module_name, + postfix, + stride=1, + kernel_size=3, + padding=1): + """3x3 convolution with padding""" + return [ + ('{}_{}/dw_conv3x3'.format(module_name, postfix), + nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=out_channels, + bias=False)), + ('{}_{}/pw_conv1x1'.format(module_name, postfix), + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + groups=1, + bias=False)), + ('{}_{}/pw_norm'.format(module_name, + postfix), nn.BatchNorm2d(out_channels)), + ('{}_{}/pw_relu'.format(module_name, postfix), nn.ReLU(inplace=True)), + ] + + +def conv3x3(in_channels, + out_channels, + module_name, + postfix, + stride=1, + groups=1, + kernel_size=3, + padding=1): + """3x3 convolution with padding""" + return [ + ( + f'{module_name}_{postfix}/conv', + nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False, + ), + ), + (f'{module_name}_{postfix}/norm', nn.BatchNorm2d(out_channels)), + (f'{module_name}_{postfix}/relu', nn.ReLU(inplace=True)), + ] + + +def conv1x1(in_channels, + out_channels, + module_name, + postfix, + stride=1, + groups=1, + kernel_size=1, + padding=0): + """1x1 convolution with padding""" + return [ + ( + f'{module_name}_{postfix}/conv', + nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False, + ), + ), + (f'{module_name}_{postfix}/norm', nn.BatchNorm2d(out_channels)), + (f'{module_name}_{postfix}/relu', nn.ReLU(inplace=True)), + ] + + +class Hsigmoid(nn.Module): + + def __init__(self, inplace=True): + super(Hsigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + return F.relu6(x + 3.0, inplace=self.inplace) / 6.0 + + +class eSEModule(nn.Module): + + def __init__(self, channel, reduction=4): + super(eSEModule, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Conv2d(channel, channel, kernel_size=1, padding=0) + self.hsigmoid = Hsigmoid() + + def forward(self, x): + input = x + x = self.avg_pool(x) + x = self.fc(x) + x = self.hsigmoid(x) + return input * x + + +class _OSA_module(nn.Module): + + def __init__(self, + in_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name, + SE=False, + identity=False, + depthwise=False): + + super(_OSA_module, self).__init__() + + self.identity = identity + self.depthwise = depthwise + self.isReduced = False + self.layers = nn.ModuleList() + in_channel = in_ch + if self.depthwise and in_channel != stage_ch: + self.isReduced = True + self.conv_reduction = nn.Sequential( + OrderedDict( + conv1x1(in_channel, stage_ch, + '{}_reduction'.format(module_name), '0'))) + for i in range(layer_per_block): + if self.depthwise: + self.layers.append( + nn.Sequential( + OrderedDict( + dw_conv3x3(stage_ch, stage_ch, module_name, i)))) + else: + self.layers.append( + nn.Sequential( + OrderedDict( + conv3x3(in_channel, stage_ch, module_name, i)))) + in_channel = stage_ch + + # feature aggregation + in_channel = in_ch + layer_per_block * stage_ch + self.concat = nn.Sequential( + OrderedDict(conv1x1(in_channel, concat_ch, module_name, 'concat'))) + + self.ese = eSEModule(concat_ch) + + def forward(self, x): + + identity_feat = x + + output = [] + output.append(x) + if self.depthwise and self.isReduced: + x = self.conv_reduction(x) + for layer in self.layers: + x = layer(x) + output.append(x) + + x = torch.cat(output, dim=1) + xt = self.concat(x) + + xt = self.ese(xt) + + if self.identity: + xt = xt + identity_feat + + return xt + + +class _OSA_stage(nn.Sequential): + + def __init__(self, + in_ch, + stage_ch, + concat_ch, + block_per_stage, + layer_per_block, + stage_num, + SE=False, + depthwise=False): + + super(_OSA_stage, self).__init__() + + if not stage_num == 2: + self.add_module( + 'Pooling', + nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True)) + + if block_per_stage != 1: + SE = False + module_name = f'OSA{stage_num}_1' + self.add_module( + module_name, + _OSA_module( + in_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name, + SE, + depthwise=depthwise)) + for i in range(block_per_stage - 1): + if i != block_per_stage - 2: # last block + SE = False + module_name = f'OSA{stage_num}_{i + 2}' + self.add_module( + module_name, + _OSA_module( + concat_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name, + SE, + identity=True, + depthwise=depthwise), + ) + + +@BACKBONES.register_module() +class VoVNet(BaseModule): + + def __init__(self, + spec_name, + input_ch=3, + out_features=None, + frozen_stages=-1, + norm_eval=True, + pretrained=None, + init_cfg=None): + """ + Args: + input_ch(int) : the number of input channel + out_features (list[str]): name of the layers whose outputs should + be returned in forward. Can be anything in "stem", "stage2" ... + """ + super(VoVNet, self).__init__(init_cfg) + self.frozen_stages = frozen_stages + self.norm_eval = norm_eval + + if isinstance(pretrained, str): + warnings.warn('DeprecationWarning: pretrained is deprecated, ' + 'please use "init_cfg" instead') + self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) + stage_specs = _STAGE_SPECS[spec_name] + + stem_ch = stage_specs['stem'] + config_stage_ch = stage_specs['stage_conv_ch'] + config_concat_ch = stage_specs['stage_out_ch'] + block_per_stage = stage_specs['block_per_stage'] + layer_per_block = stage_specs['layer_per_block'] + SE = stage_specs['eSE'] + depthwise = stage_specs['dw'] + + self._out_features = out_features + + # Stem module + conv_type = dw_conv3x3 if depthwise else conv3x3 + stem = conv3x3(input_ch, stem_ch[0], 'stem', '1', 2) + stem += conv_type(stem_ch[0], stem_ch[1], 'stem', '2', 1) + stem += conv_type(stem_ch[1], stem_ch[2], 'stem', '3', 2) + self.add_module('stem', nn.Sequential((OrderedDict(stem)))) + current_stirde = 4 + self._out_feature_strides = { + 'stem': current_stirde, + 'stage2': current_stirde + } + self._out_feature_channels = {'stem': stem_ch[2]} + + stem_out_ch = [stem_ch[2]] + in_ch_list = stem_out_ch + config_concat_ch[:-1] + # OSA stages + self.stage_names = [] + for i in range(4): # num_stages + name = 'stage%d' % (i + 2) # stage 2 ... stage 5 + self.stage_names.append(name) + self.add_module( + name, + _OSA_stage( + in_ch_list[i], + config_stage_ch[i], + config_concat_ch[i], + block_per_stage[i], + layer_per_block, + i + 2, + SE, + depthwise, + ), + ) + + self._out_feature_channels[name] = config_concat_ch[i] + if not i == 0: + self._out_feature_strides[name] = current_stirde = int( + current_stirde * 2) + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + + def forward(self, x): + outputs = {} + x = self.stem(x) + if 'stem' in self._out_features: + outputs['stem'] = x + for name in self.stage_names: + x = getattr(self, name)(x) + if name in self._out_features: + outputs[name] = x + + return outputs + + def _freeze_stages(self): + if self.frozen_stages >= 0: + m = getattr(self, 'stem') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'stage{i+1}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(VoVNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f8fb9cb120d6585223be571c3668b09769e2b1d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/dense_heads +""" +from .petrv2_dednhead import PETRv2DEDNHead + +__all__ = ['PETRv2DEDNHead'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/depth_net.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/depth_net.py new file mode 100644 index 0000000000000000000000000000000000000000..ff56e6a372bbf29fe9bca4f1f838cd504f086457 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/depth_net.py @@ -0,0 +1,234 @@ +""" +The implementation here is modified based on BEVDepth, originally MIT license and publicly available at +https://github.com/Megvii-BaseDetection/BEVDepth/blob/main/bevdepth/layers/backbones/base_lss_fpn.py +""" +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import build_conv_layer +from mmdet.models.backbones.resnet import BasicBlock + + +class _ASPPModule(nn.Module): + + def __init__(self, inplanes, planes, kernel_size, padding, dilation, + BatchNorm): + super(_ASPPModule, self).__init__() + self.atrous_conv = nn.Conv2d( + inplanes, + planes, + kernel_size=kernel_size, + stride=1, + padding=padding, + dilation=dilation, + bias=False) + self.bn = BatchNorm(planes) + self.relu = nn.ReLU() + + self._init_weight() + + def forward(self, x): + x = self.atrous_conv(x) + x = self.bn(x) + + return self.relu(x) + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +class ASPP(nn.Module): + + def __init__(self, inplanes, mid_channels=256, BatchNorm=nn.BatchNorm2d): + super(ASPP, self).__init__() + + dilations = [1, 6, 12, 18] + + self.aspp1 = _ASPPModule( + inplanes, + mid_channels, + 1, + padding=0, + dilation=dilations[0], + BatchNorm=BatchNorm) + self.aspp2 = _ASPPModule( + inplanes, + mid_channels, + 3, + padding=dilations[1], + dilation=dilations[1], + BatchNorm=BatchNorm) + self.aspp3 = _ASPPModule( + inplanes, + mid_channels, + 3, + padding=dilations[2], + dilation=dilations[2], + BatchNorm=BatchNorm) + self.aspp4 = _ASPPModule( + inplanes, + mid_channels, + 3, + padding=dilations[3], + dilation=dilations[3], + BatchNorm=BatchNorm) + + self.global_avg_pool = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(inplanes, mid_channels, 1, stride=1, bias=False), + BatchNorm(mid_channels), + nn.ReLU(), + ) + self.conv1 = nn.Conv2d( + int(mid_channels * 5), mid_channels, 1, bias=False) + self.bn1 = BatchNorm(mid_channels) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(0.5) + self._init_weight() + + def forward(self, x): + x1 = self.aspp1(x) + x2 = self.aspp2(x) + x3 = self.aspp3(x) + x4 = self.aspp4(x) + x5 = self.global_avg_pool(x) + x5 = F.interpolate( + x5, size=x4.size()[2:], mode='bilinear', align_corners=True) + x = torch.cat((x1, x2, x3, x4, x5), dim=1) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + return self.dropout(x) + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.ReLU, + drop=0.0): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.drop1 = nn.Dropout(drop) + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop2 = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.fc2(x) + x = self.drop2(x) + return x + + +class SELayer(nn.Module): + + def __init__(self, channels, act_layer=nn.ReLU, gate_layer=nn.Sigmoid): + super().__init__() + self.conv_reduce = nn.Conv2d(channels, channels, 1, bias=True) + self.act1 = act_layer() + self.conv_expand = nn.Conv2d(channels, channels, 1, bias=True) + self.gate = gate_layer() + + def forward(self, x, x_se): + x_se = self.conv_reduce(x_se) + x_se = self.act1(x_se) + x_se = self.conv_expand(x_se) + return x * self.gate(x_se) + + +class DepthNet(nn.Module): + + def __init__(self, + in_channels, + mid_channels, + context_channels, + depth_channels, + use_dcn=True, + use_aspp=True, + use_mlp=False): + super(DepthNet, self).__init__() + self.use_mlp = use_mlp + if use_mlp: + self.reduce_conv = nn.Sequential( + nn.Conv2d( + in_channels, + mid_channels, + kernel_size=3, + stride=1, + padding=1), + nn.BatchNorm2d(mid_channels), + nn.ReLU(inplace=True), + ) + self.bn = nn.BatchNorm1d(12) + self.depth_mlp = Mlp(12, mid_channels, mid_channels) + self.depth_se = SELayer(mid_channels) # NOTE: add camera-aware + self.context_mlp = Mlp(12, mid_channels, mid_channels) + self.context_se = SELayer(mid_channels) # NOTE: add camera-aware + + self.context_conv = nn.Conv2d( + mid_channels, context_channels, kernel_size=1, stride=1, padding=0) + depth_conv_list = [ + BasicBlock(mid_channels, mid_channels), + BasicBlock(mid_channels, mid_channels), + BasicBlock(mid_channels, mid_channels), + ] + if use_aspp: + depth_conv_list.append(ASPP(mid_channels, mid_channels)) + if use_dcn: + depth_conv_list.append( + build_conv_layer( + cfg=dict( + type='DCN', + in_channels=mid_channels, + out_channels=mid_channels, + kernel_size=3, + padding=1, + groups=4, + im2col_step=128, + ))) + depth_conv_list.append( + nn.Conv2d( + mid_channels, + depth_channels, + kernel_size=1, + stride=1, + padding=0)) + self.depth_conv = nn.Sequential(*depth_conv_list) + + def forward(self, x, mlp_input): + if self.use_mlp: + mlp_input = self.bn(mlp_input.reshape(-1, mlp_input.shape[-1])) + x = self.reduce_conv(x) + context_se = self.context_mlp(mlp_input)[..., None, None] + context = self.context_se(x, context_se) + context = self.context_conv(context) + depth_se = self.depth_mlp(mlp_input)[..., None, None] + depth = self.depth_se(x, depth_se) + depth = self.depth_conv(depth) + else: + context = self.context_conv(x) + depth = self.depth_conv(x) + + return torch.cat([depth, context], dim=1) diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/petrv2_dednhead.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/petrv2_dednhead.py new file mode 100644 index 0000000000000000000000000000000000000000..657aadab63d2b9c6eefaecfa1cac07bd904dcb7f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/dense_heads/petrv2_dednhead.py @@ -0,0 +1,1307 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/dense_heads +""" +import copy +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import Conv2d, Linear, bias_init_with_prob +from mmcv.cnn.bricks.transformer import FFN, build_positional_encoding +from mmcv.runner import force_fp32 +from mmdet3d.core.bbox.coders import build_bbox_coder +from mmdet.core import (bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh, + build_assigner, build_sampler, multi_apply, + reduce_mean) +from mmdet.models import HEADS, build_loss +from mmdet.models.dense_heads.anchor_free_head import AnchorFreeHead +from mmdet.models.utils import NormedLinear, build_transformer +from mmdet.models.utils.transformer import inverse_sigmoid +from torch.cuda.amp.autocast_mode import autocast + +from modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin.core.bbox.util import \ + normalize_bbox +from .depth_net import DepthNet + + +def pos2posemb3d(pos, num_pos_feats=128, temperature=10000): + scale = 2 * math.pi + pos = pos * scale + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos.device) + dim_t = temperature**(2 * torch.div(dim_t, 2, rounding_mode='floor') + / num_pos_feats) + pos_x = pos[..., 0, None] / dim_t + pos_y = pos[..., 1, None] / dim_t + pos_z = pos[..., 2, None] / dim_t + pos_x = torch.stack((pos_x[..., 0::2].sin(), pos_x[..., 1::2].cos()), + dim=-1).flatten(-2) + pos_y = torch.stack((pos_y[..., 0::2].sin(), pos_y[..., 1::2].cos()), + dim=-1).flatten(-2) + pos_z = torch.stack((pos_z[..., 0::2].sin(), pos_z[..., 1::2].cos()), + dim=-1).flatten(-2) + posemb = torch.cat((pos_y, pos_x, pos_z), dim=-1) + return posemb + + +class SELayer(nn.Module): + + def __init__(self, + se_channels, + x_channels, + act_layer=nn.ReLU, + gate_layer=nn.Sigmoid): + super().__init__() + self.conv_reduce = nn.Conv2d(se_channels, x_channels, 1, bias=True) + self.act1 = act_layer() + self.conv_expand = nn.Conv2d(x_channels, x_channels, 1, bias=True) + self.gate = gate_layer() + + def forward(self, x, x_se): + x_se = self.conv_reduce(x_se) + x_se = self.act1(x_se) + x_se = self.conv_expand(x_se) + return x * self.gate(x_se) + + +class RegLayer(nn.Module): + + def __init__( + self, + embed_dims=256, + shared_reg_fcs=2, + group_reg_dims=(2, 2, 2, 2, 2), # xy, wl, zh, rot, velo + act_layer=nn.ReLU, + drop=0.0): + super().__init__() + + reg_branch = [] + for _ in range(shared_reg_fcs): + reg_branch.append(Linear(embed_dims, embed_dims)) + reg_branch.append(act_layer()) + reg_branch.append(nn.Dropout(drop)) + self.reg_branch = nn.Sequential(*reg_branch) + + self.task_heads = nn.ModuleList() + for reg_dim in group_reg_dims: + task_head = nn.Sequential( + Linear(embed_dims, embed_dims), act_layer(), + Linear(embed_dims, reg_dim)) + self.task_heads.append(task_head) + + def forward(self, x): + reg_feat = self.reg_branch(x) + outs = [] + for task_head in self.task_heads: + out = task_head(reg_feat.clone()) + outs.append(out) + outs = torch.cat(outs, -1) + return outs + + +@HEADS.register_module() +class PETRv2DEDNHead(AnchorFreeHead): + """Implements the DETR transformer head. + See `paper: End-to-End Object Detection with Transformers + `_ for details. + Args: + num_classes (int): Number of categories excluding the background. + in_channels (int): Number of channels in the input feature map. + num_query (int): Number of query in Transformer. + num_reg_fcs (int, optional): Number of fully-connected layers used in + `FFN`, which is then used for the regression head. Default 2. + transformer (obj:`mmcv.ConfigDict`|dict): Config for transformer. + Default: None. + sync_cls_avg_factor (bool): Whether to sync the avg_factor of + all ranks. Default to False. + positional_encoding (obj:`mmcv.ConfigDict`|dict): + Config for position encoding. + loss_cls (obj:`mmcv.ConfigDict`|dict): Config of the + classification loss. Default `CrossEntropyLoss`. + loss_bbox (obj:`mmcv.ConfigDict`|dict): Config of the + regression loss. Default `L1Loss`. + loss_iou (obj:`mmcv.ConfigDict`|dict): Config of the + regression iou loss. Default `GIoULoss`. + tran_cfg (obj:`mmcv.ConfigDict`|dict): Training config of + transformer head. + test_cfg (obj:`mmcv.ConfigDict`|dict): Testing config of + transformer head. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + + def __init__( + self, + num_classes, + in_channels, + num_query=100, + num_reg_fcs=2, + transformer=None, + sync_cls_avg_factor=False, + positional_encoding=dict( + type='SinePositionalEncoding', num_feats=128, normalize=True), + code_weights=None, + bbox_coder=None, + loss_cls=dict( + type='CrossEntropyLoss', + bg_cls_weight=0.1, + use_sigmoid=False, + loss_weight=1.0, + class_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=5.0), + loss_iou=dict(type='GIoULoss', loss_weight=2.0), + train_cfg=dict( + assigner=dict( + type='HungarianAssigner', + cls_cost=dict(type='ClassificationCost', weight=1.), + reg_cost=dict(type='BBoxL1Cost', weight=5.0), + iou_cost=dict(type='IoUCost', iou_mode='giou', + weight=2.0))), + test_cfg=dict(max_per_img=100), + with_position=True, + with_multiview=False, + depth_step=0.8, + depth_num=64, + LID=False, + depth_start=1, + position_level=0, + depth_level=0, + position_range=[-65, -65, -8.0, 65, 65, 8.0], + group_reg_dims=(2, 2, 2, 2, 2), # xy, wl, zh, rot, velo + scalar=5, + noise_scale=0.4, + noise_trans=0.0, + dn_weight=1.0, + split=0.5, + init_cfg=None, + normedlinear=False, + with_fpe=False, + with_time=False, + with_multi=False, + **kwargs): + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since it brings inconvenience when the initialization of + # `AnchorFreeHead` is called. + if 'code_size' in kwargs: + self.code_size = kwargs['code_size'] + else: + self.code_size = 10 + if code_weights is not None: + self.code_weights = code_weights + else: + self.code_weights = [ + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2 + ] + self.code_weights = self.code_weights[:self.code_size] + self.bg_cls_weight = 0 + self.sync_cls_avg_factor = sync_cls_avg_factor + class_weight = loss_cls.get('class_weight', None) + if class_weight is not None and (self.__class__ is PETRv2DEDNHead): + assert isinstance(class_weight, float), 'Expected ' \ + 'class_weight to have type float. Found ' \ + f'{type(class_weight)}.' + # NOTE following the official DETR rep0, bg_cls_weight means + # relative classification weight of the no-object class. + bg_cls_weight = loss_cls.get('bg_cls_weight', class_weight) + assert isinstance(bg_cls_weight, float), 'Expected ' \ + 'bg_cls_weight to have type float. Found ' \ + f'{type(bg_cls_weight)}.' + class_weight = torch.ones(num_classes + 1) * class_weight + # set background class as the last indice + class_weight[num_classes] = bg_cls_weight + loss_cls.update({'class_weight': class_weight}) + if 'bg_cls_weight' in loss_cls: + loss_cls.pop('bg_cls_weight') + self.bg_cls_weight = bg_cls_weight + + if train_cfg: + assert 'assigner' in train_cfg, 'assigner should be provided '\ + 'when train_cfg is set.' + assigner = train_cfg['assigner'] + self.assigner = build_assigner(assigner) + # DETR sampling=False, so use PseudoSampler + sampler_cfg = dict(type='PseudoSampler') + self.sampler = build_sampler(sampler_cfg, context=self) + + self.num_query = num_query + self.num_classes = num_classes + self.in_channels = in_channels + self.num_reg_fcs = num_reg_fcs + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.fp16_enabled = False + self.embed_dims = 256 + self.depth_step = depth_step + self.depth_num = depth_num + self.position_range = position_range + self.LID = LID + self.with_depthnet = kwargs.get('with_depthnet', False) + self.with_fde = kwargs.get('with_fde', False) + self.depth_pe = kwargs.get('depth_pe', False) + self.sweep_lidar = kwargs.get('sweep_lidar', False) + self.depth_LID = kwargs.get('depth_LID', False) + self.depth_LID_num = kwargs.get('depth_LID_num', 64) + self.depth_thresh = kwargs.get('depth_thresh', 0.0) + if not self.with_depthnet: + self.with_fde = False + self.depth_pe = False + self.sweep_lidar = False + self.depth_start = depth_start + self.de_intv = 0.5 + if self.with_depthnet: # get num of depth bin for depthnet + if self.depth_LID: + self.D = self.depth_LID_num + else: + coords_d = np.arange(self.depth_start, self.position_range[3], + self.de_intv) + self.D = len(coords_d) + self.position_dim = 3 * self.depth_num + self.position_level = position_level + self.depth_level = depth_level + self.with_position = with_position + self.with_multiview = with_multiview + self.scalar = scalar + self.bbox_noise_scale = noise_scale + self.bbox_noise_trans = noise_trans + self.dn_weight = dn_weight + self.split = split + assert 'num_feats' in positional_encoding + num_feats = positional_encoding['num_feats'] + assert num_feats * 2 == self.embed_dims, 'embed_dims should' \ + f' be exactly 2 times of num_feats. Found {self.embed_dims}' \ + f' and {num_feats}.' + self.act_cfg = transformer.get('act_cfg', + dict(type='ReLU', inplace=True)) + self.num_pred = 6 + self.normedlinear = normedlinear + self.with_fpe = with_fpe + self.with_time = with_time + self.with_multi = with_multi + self.group_reg_dims = group_reg_dims + super(PETRv2DEDNHead, self).__init__( + num_classes, in_channels, init_cfg=init_cfg) + + self.loss_cls = build_loss(loss_cls) + self.loss_bbox = build_loss(loss_bbox) + self.loss_iou = build_loss(loss_iou) + + if self.loss_cls.use_sigmoid: + self.cls_out_channels = num_classes + else: + self.cls_out_channels = num_classes + 1 + self.positional_encoding = build_positional_encoding( + positional_encoding) + self.transformer = build_transformer(transformer) + self.code_weights = nn.Parameter( + torch.tensor(self.code_weights, requires_grad=False), + requires_grad=False) + self.bbox_coder = build_bbox_coder(bbox_coder) + self.pc_range = self.bbox_coder.pc_range + if 'depthnet_cfg' in kwargs: + depthnet_cfg = kwargs['depthnet_cfg'] + else: + depthnet_cfg = {} + self.depthnet_cfg = depthnet_cfg + self.loss_depth_weight = kwargs.get('loss_depth_weight', 3.0) + if self.with_depthnet: + self.depth_net = DepthNet(self.in_channels, self.in_channels, + self.in_channels, self.D, **depthnet_cfg) + self._init_layers() + + def _init_layers(self): + """Initialize layers of the transformer head.""" + if not self.with_depthnet: + self.input_proj = Conv2d( + self.in_channels, self.embed_dims, kernel_size=1) + + cls_branch = [] + for _ in range(self.num_reg_fcs): + cls_branch.append(Linear(self.embed_dims, self.embed_dims)) + cls_branch.append(nn.LayerNorm(self.embed_dims)) + cls_branch.append(nn.ReLU(inplace=True)) + if self.normedlinear: + cls_branch.append( + NormedLinear(self.embed_dims, self.cls_out_channels)) + else: + cls_branch.append(Linear(self.embed_dims, self.cls_out_channels)) + fc_cls = nn.Sequential(*cls_branch) + + if self.with_multi: + reg_branch = RegLayer(self.embed_dims, self.num_reg_fcs, + self.group_reg_dims) + else: + reg_branch = [] + for _ in range(self.num_reg_fcs): + reg_branch.append(Linear(self.embed_dims, self.embed_dims)) + reg_branch.append(nn.ReLU()) + reg_branch.append(Linear(self.embed_dims, self.code_size)) + reg_branch = nn.Sequential(*reg_branch) + + self.cls_branches = nn.ModuleList( + [copy.deepcopy(fc_cls) for _ in range(self.num_pred)]) + self.reg_branches = nn.ModuleList( + [copy.deepcopy(reg_branch) for _ in range(self.num_pred)]) + + if self.with_multiview: + self.adapt_pos3d = nn.Sequential( + nn.Conv2d( + self.embed_dims * 3 // 2, + self.embed_dims * 4, + kernel_size=1, + stride=1, + padding=0), + nn.ReLU(), + nn.Conv2d( + self.embed_dims * 4, + self.embed_dims, + kernel_size=1, + stride=1, + padding=0), + ) + else: + self.adapt_pos3d = nn.Sequential( + nn.Conv2d( + self.embed_dims, + self.embed_dims, + kernel_size=1, + stride=1, + padding=0), + nn.ReLU(), + nn.Conv2d( + self.embed_dims, + self.embed_dims, + kernel_size=1, + stride=1, + padding=0), + ) + + if self.with_position: + chan_in = self.position_dim + chan_mid = self.embed_dims * 4 + self.position_encoder = nn.Sequential( + nn.Conv2d( + chan_in, chan_mid, kernel_size=1, stride=1, padding=0), + nn.ReLU(), + nn.Conv2d( + chan_mid, + self.embed_dims, + kernel_size=1, + stride=1, + padding=0), + ) + + if self.depth_pe: + self.depth_encoder = nn.Conv2d( + self.D, self.embed_dims, kernel_size=1, stride=1, padding=0) + + self.reference_points = nn.Embedding(self.num_query, 3) + self.query_embedding = nn.Sequential( + nn.Linear(self.embed_dims * 3 // 2, self.embed_dims), + nn.ReLU(), + nn.Linear(self.embed_dims, self.embed_dims), + ) + if self.with_fpe: + self.fpe = SELayer(self.embed_dims, self.embed_dims) + if self.with_fde: + self.fde = SELayer(self.embed_dims, self.D) + + def init_weights(self): + """Initialize weights of the transformer head.""" + # The initialization for transformer is important + self.transformer.init_weights() + nn.init.uniform_(self.reference_points.weight.data, 0, 1) + if self.loss_cls.use_sigmoid: + bias_init = bias_init_with_prob(0.01) + for m in self.cls_branches: + nn.init.constant_(m[-1].bias, bias_init) + + def get_mlp_input(self, img_metas): + num_view = 6 + num_img = len(img_metas[0]['lidar2img']) + lidar2img = torch.tensor( + [img_meta['lidar2img'][:num_view] for img_meta in img_metas]) + lidar2img = lidar2img.repeat(1, num_img // num_view, 1, 1) + lidar2img = lidar2img[..., :3, :] + B, N, _, _, = lidar2img.shape + mlp_input = lidar2img.reshape(B, N, -1) + return mlp_input + + def position_embeding(self, img_feats, img_metas, masks=None): + eps = 1e-5 + pad_h, pad_w, _ = img_metas[0]['pad_shape'][0] + B, N, C, H, W = img_feats[self.position_level].shape + coords_h = torch.arange( + H, device=img_feats[0].device).float() * pad_h / H + coords_w = torch.arange( + W, device=img_feats[0].device).float() * pad_w / W + # move to center + coords_h += pad_h / H / 2.0 + coords_w += pad_w / W / 2.0 + + if self.LID: + index = torch.arange( + start=0, + end=self.depth_num, + step=1, + device=img_feats[0].device).float() + index_1 = index + 1 + bin_size = (self.position_range[3] - self.depth_start) / ( + self.depth_num * (1 + self.depth_num)) + coords_d = self.depth_start + bin_size * index * index_1 + else: + index = torch.arange( + start=0, + end=self.depth_num, + step=1, + device=img_feats[0].device).float() + bin_size = (self.position_range[3] + - self.depth_start) / self.depth_num + coords_d = self.depth_start + bin_size * index + self.coords_d = coords_d + + D = coords_d.shape[0] + coords = torch.stack(torch.meshgrid([coords_w, coords_h, coords_d + ])).permute(1, 2, 3, + 0) # W, H, D, 3 + coords = torch.cat((coords, torch.ones_like(coords[..., :1])), -1) + coords[..., :2] = coords[..., :2] * torch.maximum( + coords[..., 2:3], + torch.ones_like(coords[..., 2:3]) * eps) + + img2lidars = [] + for img_meta in img_metas: + img2lidar = [] + for i in range(len(img_meta['lidar2img'])): + img2lidar.append(np.linalg.inv(img_meta['lidar2img'][i])) + img2lidars.append(np.asarray(img2lidar)) + img2lidars = np.asarray(img2lidars) + img2lidars = coords.new_tensor(img2lidars) # (B, N, 4, 4) + + coords = coords.view(1, 1, W, H, D, 4, 1).repeat(B, N, 1, 1, 1, 1, 1) + img2lidars = img2lidars.view(B, N, 1, 1, 1, 4, + 4).repeat(1, 1, W, H, D, 1, 1) + coords3d = torch.matmul(img2lidars, coords).squeeze(-1)[..., :3] + coords3d[..., 0:1] = (coords3d[..., 0:1] - self.position_range[0]) / ( + self.position_range[3] - self.position_range[0]) + coords3d[..., 1:2] = (coords3d[..., 1:2] - self.position_range[1]) / ( + self.position_range[4] - self.position_range[1]) + coords3d[..., 2:3] = (coords3d[..., 2:3] - self.position_range[2]) / ( + self.position_range[5] - self.position_range[2]) + coords_mask = (coords3d > 1.0) | (coords3d < 0.0) + coords_mask = coords_mask.flatten(-2).sum(-1) > (D * 0.5) + coords_mask = masks | coords_mask.permute(0, 1, 3, 2) + coords3d = coords3d.permute(0, 1, 4, 5, 3, + 2).contiguous().view(B * N, -1, H, W) + coords3d = inverse_sigmoid(coords3d) + coords_position_embeding = self.position_encoder(coords3d) + return coords_position_embeding.view(B, N, self.embed_dims, H, + W), coords_mask + + def prepare_for_dn(self, batch_size, reference_points, img_metas): + if self.training: + targets = [ + torch.cat((img_meta['gt_bboxes_3d']._data.gravity_center, + img_meta['gt_bboxes_3d']._data.tensor[:, 3:]), + dim=1) for img_meta in img_metas + ] + labels = [img_meta['gt_labels_3d']._data for img_meta in img_metas] + known = [(torch.ones_like(t)).cuda() for t in labels] + know_idx = known + unmask_bbox = unmask_label = torch.cat(known) + known_num = [t.size(0) for t in targets] + labels = torch.cat([t for t in labels]) + boxes = torch.cat([t for t in targets]) + batch_idx = torch.cat( + [torch.full((t.size(0), ), i) for i, t in enumerate(targets)]) + + known_indice = torch.nonzero(unmask_label + unmask_bbox) + known_indice = known_indice.view(-1) + # add noise + known_indice = known_indice.repeat(self.scalar, 1).view(-1) + known_labels = labels.repeat(self.scalar, 1).view(-1).long().to( + reference_points.device) + known_bid = batch_idx.repeat(self.scalar, 1).view(-1) + known_bboxs = boxes.repeat(self.scalar, + 1).to(reference_points.device) + known_bbox_center = known_bboxs[:, :3].clone() + known_bbox_scale = known_bboxs[:, 3:6].clone() + + if self.bbox_noise_scale > 0: + diff = known_bbox_scale / 2 + self.bbox_noise_trans + rand_prob = torch.rand_like( + known_bbox_center) * 2 - 1.0 # (-1, 1) + known_bbox_center += torch.mul(rand_prob, + diff) * self.bbox_noise_scale + known_bbox_center[ + ..., + 0:1] = (known_bbox_center[..., 0:1] - self.pc_range[0]) / ( + self.pc_range[3] - self.pc_range[0]) + known_bbox_center[ + ..., + 1:2] = (known_bbox_center[..., 1:2] - self.pc_range[1]) / ( + self.pc_range[4] - self.pc_range[1]) + known_bbox_center[ + ..., + 2:3] = (known_bbox_center[..., 2:3] - self.pc_range[2]) / ( + self.pc_range[5] - self.pc_range[2]) + known_bbox_center = known_bbox_center.clamp(min=0.0, max=1.0) + mask = torch.norm(rand_prob, 2, 1) > self.split + known_labels[mask] = self.num_classes + + single_pad = int(max(known_num)) + pad_size = int(single_pad * self.scalar) + padding_bbox = torch.zeros(pad_size, 3).to(reference_points.device) + padded_reference_points = torch.cat( + [padding_bbox, reference_points], + dim=0).unsqueeze(0).repeat(batch_size, 1, 1) + + if len(known_num): + map_known_indice = torch.cat([ + torch.tensor(range(num)) for num in known_num + ]) # [1,2, 1,2,3] + map_known_indice = torch.cat([ + map_known_indice + single_pad * i + for i in range(self.scalar) + ]).long() + if len(known_bid): + padded_reference_points[( + known_bid.long(), + map_known_indice)] = known_bbox_center.to( + reference_points.device) + + tgt_size = pad_size + self.num_query + attn_mask = torch.ones(tgt_size, tgt_size).to( + reference_points.device) < 0 + # match query cannot see the reconstruct + attn_mask[pad_size:, :pad_size] = True + # reconstruct cannot see each other + for i in range(self.scalar): + if i == 0: + attn_mask[single_pad * i:single_pad * (i + 1), + single_pad * (i + 1):pad_size] = True + if i == self.scalar - 1: + attn_mask[single_pad * i:single_pad * (i + 1), :single_pad + * i] = True + else: + attn_mask[single_pad * i:single_pad * (i + 1), + single_pad * (i + 1):pad_size] = True + attn_mask[single_pad * i:single_pad * (i + 1), :single_pad + * i] = True + + mask_dict = { + 'known_indice': torch.as_tensor(known_indice).long(), + 'batch_idx': torch.as_tensor(batch_idx).long(), + 'map_known_indice': torch.as_tensor(map_known_indice).long(), + 'known_lbs_bboxes': (known_labels, known_bboxs), + 'know_idx': know_idx, + 'pad_size': pad_size + } + else: + padded_reference_points = reference_points.unsqueeze(0).repeat( + batch_size, 1, 1) + attn_mask = None + mask_dict = None + + return padded_reference_points, attn_mask, mask_dict + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """load checkpoints.""" + # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, + # since `AnchorFreeHead._load_from_state_dict` should not be + # called here. Invoking the default `Module._load_from_state_dict` + # is enough. + + # Names of some parameters in has been changed. + version = local_metadata.get('version', None) + if (version is None + or version < 2) and self.__class__ is PETRv2DEDNHead: + convert_dict = { + '.self_attn.': '.attentions.0.', + '.multihead_attn.': '.attentions.1.', + '.decoder.norm.': '.decoder.post_norm.' + } + state_dict_keys = list(state_dict.keys()) + for k in state_dict_keys: + for ori_key, convert_key in convert_dict.items(): + if ori_key in k: + convert_key = k.replace(ori_key, convert_key) + state_dict[convert_key] = state_dict[k] + del state_dict[k] + + super(AnchorFreeHead, + self)._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, + unexpected_keys, error_msgs) + + def forward(self, mlvl_feats, img_metas): + """Forward function. + Args: + mlvl_feats (tuple[Tensor]): Features from the upstream + network, each is a 5D-tensor with shape + (B, N, C, H, W). + Returns: + all_cls_scores (Tensor): Outputs from the classification head, \ + shape [nb_dec, bs, num_query, cls_out_channels]. Note \ + cls_out_channels should includes background. + all_bbox_preds (Tensor): Sigmoid outputs from the regression \ + head with normalized coordinate format (cx, cy, w, l, cz, h, theta, vx, vy). \ + Shape [nb_dec, bs, num_query, 9]. + """ + + x = mlvl_feats[self.position_level] + batch_size, num_cams = x.size(0), x.size(1) + + input_img_h, input_img_w, _ = img_metas[0]['pad_shape'][0] + masks = x.new_ones((batch_size, num_cams, input_img_h, input_img_w)) + for img_id in range(batch_size): + for cam_id in range(num_cams): + img_h, img_w, _ = img_metas[img_id]['img_shape'][cam_id] + masks[img_id, cam_id, :img_h, :img_w] = 0 + + if not self.with_depthnet: + feat = self.input_proj(x.flatten(0, 1)) + feat = feat.view(batch_size, num_cams, *feat.shape[-3:]) + depth = None + depth_all = None + depth_enc = None + else: # with depth net + feat = x + use_mlp = self.depthnet_cfg['use_mlp'] + if use_mlp: + mlp_input_all = self.get_mlp_input(img_metas).to( + x.device).float() + x_de = mlvl_feats[self.depth_level] + self.depth_downsample = int(input_img_w / x_de.shape[-1]) + num_views = 6 + depth_list = [] + feat_list = [] + keyFrame = True + for frame in range(num_cams // num_views): + input_x = x_de[:, frame * num_views:(frame + 1) * num_views, + ...] + mlp_input = mlp_input_all[:, frame * num_views:(frame + 1) + * num_views, + ...] if use_mlp else None + if keyFrame: + output_x = self.depth_net(input_x.flatten(0, 1), mlp_input) + output_x = output_x.view(batch_size, num_views, + *output_x.shape[-3:]) + depth_digit = output_x[:, :, :self.D, ...] + depth = depth_digit.softmax(dim=2) + feat = output_x[:, :, self.D:self.D + self.in_channels, + ...] + else: + with torch.no_grad(): + output_x = self.depth_net( + input_x.flatten(0, 1), mlp_input) + output_x = output_x.view(batch_size, num_views, + *output_x.shape[-3:]) + depth_digit = output_x[:, :, :self.D, ...] + depth = depth_digit.softmax(dim=2) + feat = output_x[:, :, self.D:self.D + self.in_channels, + ...] + if not self.sweep_lidar: + keyFrame = False # if not use sweep lidar, not take sweep as keyframe and no grad. + depth_list += [depth] + feat_list += [feat] + depth = torch.cat(depth_list, dim=1) + feat = torch.cat(feat_list, dim=1) + depth_all = depth + if self.with_fde: + depth_all = self.fde(depth_all, + feat.flatten(0, 1)).view(depth_all.size()) + # filter depth with low prob + if self.depth_thresh > 0.0: + d_mask = depth_all.max(dim=2, keepdim=True).values + d_mask = torch.where(d_mask > self.depth_thresh, + torch.ones_like(d_mask), + torch.zeros_like(d_mask)) + d_mask = d_mask.repeat(1, 1, depth_all.shape[2], 1, 1) + depth_all = torch.mul(depth_all, d_mask) + # resize depth map + de_d, de_h, de_w = depth_all.shape[-3:] + ft_h, ft_w = feat.shape[-2:] + if de_h != ft_h or de_w != ft_w: # make depth and feat same size + depth_all = F.interpolate(depth_all, size=(de_d, ft_h, ft_w)) + if self.depth_pe: + depth_enc = self.depth_encoder(depth_all.flatten(0, 1)) + + # interpolate masks to have the same spatial shape with x + masks = F.interpolate(masks, size=x.shape[-2:]).to(torch.bool) + + if self.with_position: + coords_position_embeding, _ = self.position_embeding( + mlvl_feats, img_metas, masks) + if self.with_fpe: + coords_position_embeding = self.fpe( + coords_position_embeding.flatten(0, 1), + feat.flatten(0, 1)).view(feat.size()) + + pos_embed = coords_position_embeding + if self.depth_pe: + pos_embed += depth_enc.view(feat.size()) + + if self.with_multiview: + sin_embed = self.positional_encoding(masks) + sin_embed = self.adapt_pos3d(sin_embed.flatten(0, 1)).view( + feat.size()) + pos_embed = pos_embed + sin_embed + else: + pos_embeds = [] + for i in range(num_cams): + xy_embed = self.positional_encoding(masks[:, i, :, :]) + pos_embeds.append(xy_embed.unsqueeze(1)) + sin_embed = torch.cat(pos_embeds, 1) + sin_embed = self.adapt_pos3d(sin_embed.flatten(0, 1)).view( + feat.size()) + pos_embed = pos_embed + sin_embed + else: + if self.with_multiview: + pos_embed = self.positional_encoding(masks) + pos_embed = self.adapt_pos3d(pos_embed.flatten(0, 1)).view( + feat.size()) + else: + pos_embeds = [] + for i in range(num_cams): + pos_embed = self.positional_encoding(masks[:, i, :, :]) + pos_embeds.append(pos_embed.unsqueeze(1)) + pos_embed = torch.cat(pos_embeds, 1) + + reference_points = self.reference_points.weight + reference_points, attn_mask, mask_dict = self.prepare_for_dn( + batch_size, reference_points, img_metas) + query_embeds = self.query_embedding(pos2posemb3d(reference_points)) + outs_dec, _ = self.transformer(feat, masks, query_embeds, pos_embed, + attn_mask, self.reg_branches) + + if self.with_time: + time_stamps = [] + for img_meta in img_metas: + time_stamps.append(np.asarray(img_meta['timestamp'][:12])) + time_stamp = x.new_tensor(np.array(time_stamps)) + time_stamp = time_stamp.view(batch_size, -1, 6) + mean_time_stamp = (time_stamp[:, 1, :] + - time_stamp[:, 0, :]).mean(-1) + + outputs_classes = [] + outputs_coords = [] + for lvl in range(outs_dec.shape[0]): + reference = inverse_sigmoid(reference_points.clone()) + assert reference.shape[-1] == 3 + outputs_class = self.cls_branches[lvl](outs_dec[lvl]) + tmp = self.reg_branches[lvl](outs_dec[lvl]) + + tmp[..., 0:2] += reference[..., 0:2] + tmp[..., 0:2] = tmp[..., 0:2].sigmoid() + tmp[..., 4:5] += reference[..., 2:3] + tmp[..., 4:5] = tmp[..., 4:5].sigmoid() + + if self.with_time: + tmp[..., 8:] = tmp[..., 8:] / mean_time_stamp[:, None, None] + + outputs_coord = tmp + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + all_cls_scores = torch.stack(outputs_classes) + all_bbox_preds = torch.stack(outputs_coords) + + tmp = all_bbox_preds[..., 0:1] * (self.pc_range[3] - self.pc_range[0]) + all_bbox_preds[..., 0:1] = (tmp + self.pc_range[0]) + tmp = all_bbox_preds[..., 1:2] * (self.pc_range[4] - self.pc_range[1]) + all_bbox_preds[..., 1:2] = (tmp + self.pc_range[1]) + tmp = all_bbox_preds[..., 4:5] * (self.pc_range[5] - self.pc_range[2]) + all_bbox_preds[..., 4:5] = (tmp + self.pc_range[2]) + + if mask_dict and mask_dict['pad_size'] > 0: + output_known_class = all_cls_scores[:, :, : + mask_dict['pad_size'], :] + output_known_coord = all_bbox_preds[:, :, : + mask_dict['pad_size'], :] + outputs_class = all_cls_scores[:, :, mask_dict['pad_size']:, :] + outputs_coord = all_bbox_preds[:, :, mask_dict['pad_size']:, :] + mask_dict['output_known_lbs_bboxes'] = (output_known_class, + output_known_coord) + outs = { + 'all_cls_scores': outputs_class, + 'all_bbox_preds': outputs_coord, + 'enc_cls_scores': None, + 'enc_bbox_preds': None, + 'dn_mask_dict': mask_dict, + 'depth_pred': depth, + } + else: + outs = { + 'all_cls_scores': all_cls_scores, + 'all_bbox_preds': all_bbox_preds, + 'enc_cls_scores': None, + 'enc_bbox_preds': None, + 'dn_mask_dict': None, + 'depth_pred': depth, + } + return outs + + def get_downsampled_gt_depth(self, gt_depths): + """ + Input: + gt_depths: [B, N, H, W] + Output: + gt_depths: [B*N*h*w, d] + """ + downsample = self.depth_downsample + depth_config = [self.depth_start, self.position_range[3], self.de_intv] + B, N, H, W = gt_depths.shape + gt_depths = gt_depths.view(B * N, H // downsample, downsample, + W // downsample, downsample, 1) + gt_depths = gt_depths.permute(0, 1, 3, 5, 2, 4).contiguous() + gt_depths = gt_depths.view(-1, downsample * downsample) + gt_depths_tmp = torch.where(gt_depths == 0.0, + 1e5 * torch.ones_like(gt_depths), + gt_depths) + gt_depths = torch.min(gt_depths_tmp, dim=-1).values + gt_depths = gt_depths.view(B * N, H // downsample, W // downsample) + + if self.depth_LID: # use 64 LID + bin_size = (self.position_range[3] - self.depth_start) / ( + self.D * (1 + self.D)) + t = (gt_depths - self.depth_start) / bin_size + gt_depths = (torch.sqrt(1 + 4 * t) + - 1) / 2.0 + 1 # (-b+sqrt(b^2-4ac))/2a + else: # use 121 UD + tmp = gt_depths - (depth_config[0] - depth_config[2]) + gt_depths = tmp / depth_config[2] + gt_depths = torch.where((gt_depths < self.D + 1) & (gt_depths >= 0.0), + gt_depths, torch.zeros_like(gt_depths)) + gt_depths = F.one_hot( + gt_depths.long(), num_classes=self.D + 1).view(-1, self.D + 1)[:, + 1:] + return gt_depths.float() + + @force_fp32() + def get_depth_loss(self, gt_depth, pred_depth): + depth_labels = self.get_downsampled_gt_depth(gt_depth) + depth_preds = pred_depth.permute(0, 2, 3, + 1).contiguous().view(-1, self.D) + fg_mask = torch.max(depth_labels, dim=1).values > 0.0 + depth_labels = depth_labels[fg_mask] + depth_preds = depth_preds[fg_mask] + with autocast(enabled=False): + depth_loss = F.binary_cross_entropy( + depth_preds, + depth_labels, + reduction='none', + ).sum() / max(1.0, fg_mask.sum()) + return self.loss_depth_weight * depth_loss + + def prepare_for_loss(self, mask_dict): + """ + prepare dn components to calculate loss + Args: + mask_dict: a dict that contains dn information + """ + output_known_class, output_known_coord = mask_dict[ + 'output_known_lbs_bboxes'] + known_labels, known_bboxs = mask_dict['known_lbs_bboxes'] + map_known_indice = mask_dict['map_known_indice'].long() + known_indice = mask_dict['known_indice'].long() + batch_idx = mask_dict['batch_idx'].long() + bid = batch_idx[known_indice] + if len(output_known_class) > 0: + output_known_class = output_known_class.permute( + 1, 2, 0, 3)[(bid, map_known_indice)].permute(1, 0, 2) + output_known_coord = output_known_coord.permute( + 1, 2, 0, 3)[(bid, map_known_indice)].permute(1, 0, 2) + num_tgt = known_indice.numel() + return known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt + + def _get_target_single(self, + cls_score, + bbox_pred, + gt_labels, + gt_bboxes, + gt_bboxes_ignore=None): + """"Compute regression and classification targets for one image. + Outputs from a single decoder layer of a single feature level are used. + Args: + cls_score (Tensor): Box score logits from a single decoder layer + for one image. Shape [num_query, cls_out_channels]. + bbox_pred (Tensor): Sigmoid outputs from a single decoder layer + for one image, with normalized coordinate (cx, cy, w, h) and + shape [num_query, 4]. + gt_bboxes (Tensor): Ground truth bboxes for one image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (Tensor): Ground truth class indices for one image + with shape (num_gts, ). + gt_bboxes_ignore (Tensor, optional): Bounding boxes + which can be ignored. Default None. + Returns: + tuple[Tensor]: a tuple containing the following for one image. + - labels (Tensor): Labels of each image. + - label_weights (Tensor]): Label weights of each image. + - bbox_targets (Tensor): BBox targets of each image. + - bbox_weights (Tensor): BBox weights of each image. + - pos_inds (Tensor): Sampled positive indices for each image. + - neg_inds (Tensor): Sampled negative indices for each image. + """ + + num_bboxes = bbox_pred.size(0) + # assigner and sampler + assign_result = self.assigner.assign(bbox_pred, cls_score, gt_bboxes, + gt_labels, gt_bboxes_ignore) + sampling_result = self.sampler.sample(assign_result, bbox_pred, + gt_bboxes) + pos_inds = sampling_result.pos_inds + neg_inds = sampling_result.neg_inds + + # label targets + labels = gt_bboxes.new_full((num_bboxes, ), + self.num_classes, + dtype=torch.long) + labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] + label_weights = gt_bboxes.new_ones(num_bboxes) + + # bbox targets + code_size = gt_bboxes.size(1) + bbox_targets = torch.zeros_like(bbox_pred)[..., :code_size] + bbox_weights = torch.zeros_like(bbox_pred) + bbox_weights[pos_inds] = 1.0 + + # DETR + if sampling_result.pos_gt_bboxes.shape[1] == 4: + bbox_targets[pos_inds] = sampling_result.pos_gt_bboxes.reshape( + sampling_result.pos_gt_bboxes.shape[0], self.code_size - 1) + else: + bbox_targets[pos_inds] = sampling_result.pos_gt_bboxes + + return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, + neg_inds) + + def get_targets(self, + cls_scores_list, + bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + gt_bboxes_ignore_list=None): + """"Compute regression and classification targets for a batch image. + Outputs from a single decoder layer of a single feature level are used. + Args: + cls_scores_list (list[Tensor]): Box score logits from a single + decoder layer for each image with shape [num_query, + cls_out_channels]. + bbox_preds_list (list[Tensor]): Sigmoid outputs from a single + decoder layer for each image, with normalized coordinate + (cx, cy, w, h) and shape [num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + Returns: + tuple: a tuple containing the following targets. + - labels_list (list[Tensor]): Labels for all images. + - label_weights_list (list[Tensor]): Label weights for all \ + images. + - bbox_targets_list (list[Tensor]): BBox targets for all \ + images. + - bbox_weights_list (list[Tensor]): BBox weights for all \ + images. + - num_total_pos (int): Number of positive samples in all \ + images. + - num_total_neg (int): Number of negative samples in all \ + images. + """ + assert gt_bboxes_ignore_list is None, \ + 'Only supports for gt_bboxes_ignore setting to None.' + num_imgs = len(cls_scores_list) + gt_bboxes_ignore_list = [ + gt_bboxes_ignore_list for _ in range(num_imgs) + ] + + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + pos_inds_list, + neg_inds_list) = multi_apply(self._get_target_single, cls_scores_list, + bbox_preds_list, gt_labels_list, + gt_bboxes_list, gt_bboxes_ignore_list) + num_total_pos = sum((inds.numel() for inds in pos_inds_list)) + num_total_neg = sum((inds.numel() for inds in neg_inds_list)) + return (labels_list, label_weights_list, bbox_targets_list, + bbox_weights_list, num_total_pos, num_total_neg) + + def loss_single(self, + cls_scores, + bbox_preds, + gt_bboxes_list, + gt_labels_list, + gt_bboxes_ignore_list=None): + """"Loss function for outputs from a single decoder layer of a single + feature level. + Args: + cls_scores (Tensor): Box score logits from a single decoder layer + for all images. Shape [bs, num_query, cls_out_channels]. + bbox_preds (Tensor): Sigmoid outputs from a single decoder layer + for all images, with normalized coordinate (cx, cy, w, h) and + shape [bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + Returns: + dict[str, Tensor]: A dictionary of loss components for outputs from + a single decoder layer. + """ + num_imgs = cls_scores.size(0) + cls_scores_list = [cls_scores[i] for i in range(num_imgs)] + bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] + cls_reg_targets = self.get_targets(cls_scores_list, bbox_preds_list, + gt_bboxes_list, gt_labels_list, + gt_bboxes_ignore_list) + (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, + num_total_pos, num_total_neg) = cls_reg_targets + labels = torch.cat(labels_list, 0) + label_weights = torch.cat(label_weights_list, 0) + bbox_targets = torch.cat(bbox_targets_list, 0) + bbox_weights = torch.cat(bbox_weights_list, 0) + + # classification loss + cls_scores = cls_scores.reshape(-1, self.cls_out_channels) + # construct weighted avg_factor to match with the official DETR repo + cls_avg_factor = num_total_pos * 1.0 + \ + num_total_neg * self.bg_cls_weight + if self.sync_cls_avg_factor: + cls_avg_factor = reduce_mean( + cls_scores.new_tensor([cls_avg_factor])) + + cls_avg_factor = max(cls_avg_factor, 1) + loss_cls = self.loss_cls( + cls_scores, labels, label_weights, avg_factor=cls_avg_factor) + + # Compute the average number of gt boxes accross all gpus, for + # normalization purposes + num_total_pos = loss_cls.new_tensor([num_total_pos]) + num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() + + # regression L1 loss + bbox_preds = bbox_preds.reshape(-1, bbox_preds.size(-1)) + normalized_bbox_targets = normalize_bbox(bbox_targets, self.pc_range) + isnotnan = torch.isfinite(normalized_bbox_targets).all(dim=-1) + bbox_weights = bbox_weights * self.code_weights + + loss_bbox = self.loss_bbox( + bbox_preds[isnotnan, :10], + normalized_bbox_targets[isnotnan, :10], + bbox_weights[isnotnan, :10], + avg_factor=num_total_pos) + + loss_cls = torch.nan_to_num(loss_cls) + loss_bbox = torch.nan_to_num(loss_bbox) + return loss_cls, loss_bbox + + def dn_loss_single(self, + cls_scores, + bbox_preds, + known_bboxs, + known_labels, + num_total_pos=None): + """"Loss function for outputs from a single decoder layer of a single + feature level. + Args: + cls_scores (Tensor): Box score logits from a single decoder layer + for all images. Shape [bs, num_query, cls_out_channels]. + bbox_preds (Tensor): Sigmoid outputs from a single decoder layer + for all images, with normalized coordinate (cx, cy, w, h) and + shape [bs, num_query, 4]. + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + gt_bboxes_ignore_list (list[Tensor], optional): Bounding + boxes which can be ignored for each image. Default None. + Returns: + dict[str, Tensor]: A dictionary of loss components for outputs from + a single decoder layer. + """ + # classification loss + cls_scores = cls_scores.reshape(-1, self.cls_out_channels) + # construct weighted avg_factor to match with the official DETR repo + cls_avg_factor = num_total_pos * 3.14159 / 6 * self.split * self.split * self.split # positive rate + if self.sync_cls_avg_factor: + cls_avg_factor = reduce_mean( + cls_scores.new_tensor([cls_avg_factor])) + bbox_weights = torch.ones_like(bbox_preds) + label_weights = torch.ones_like(known_labels) + cls_avg_factor = max(cls_avg_factor, 1) + loss_cls = self.loss_cls( + cls_scores, + known_labels.long(), + label_weights, + avg_factor=cls_avg_factor) + + # Compute the average number of gt boxes accross all gpus, for + # normalization purposes + num_total_pos = loss_cls.new_tensor([num_total_pos]) + num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() + + # regression L1 loss + bbox_preds = bbox_preds.reshape(-1, bbox_preds.size(-1)) + normalized_bbox_targets = normalize_bbox(known_bboxs, self.pc_range) + isnotnan = torch.isfinite(normalized_bbox_targets).all(dim=-1) + bbox_weights = bbox_weights * self.code_weights + bbox_weights[:, 6: + 8] = 0 # dn alaways reduce the mAOE, which is useless when training for a long time. + loss_bbox = self.loss_bbox( + bbox_preds[isnotnan, :10], + normalized_bbox_targets[isnotnan, :10], + bbox_weights[isnotnan, :10], + avg_factor=num_total_pos) + + loss_cls = torch.nan_to_num(loss_cls) + loss_bbox = torch.nan_to_num(loss_bbox) + + return self.dn_weight * loss_cls, self.dn_weight * loss_bbox + + @force_fp32(apply_to=('preds_dicts')) + def loss(self, + gt_bboxes_list, + gt_labels_list, + preds_dicts, + gt_depth, + gt_bboxes_ignore=None): + """"Loss function. + Args: + gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image + with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels_list (list[Tensor]): Ground truth class indices for each + image with shape (num_gts, ). + preds_dicts: + all_cls_scores (Tensor): Classification score of all + decoder layers, has shape + [nb_dec, bs, num_query, cls_out_channels]. + all_bbox_preds (Tensor): Sigmoid regression + outputs of all decode layers. Each is a 4D-tensor with + normalized coordinate format (cx, cy, w, h) and shape + [nb_dec, bs, num_query, 4]. + enc_cls_scores (Tensor): Classification scores of + points on encode feature map , has shape + (N, h*w, num_classes). Only be passed when as_two_stage is + True, otherwise is None. + enc_bbox_preds (Tensor): Regression results of each points + on the encode feature map, has shape (N, h*w, 4). Only be + passed when as_two_stage is True, otherwise is None. + gt_bboxes_ignore (list[Tensor], optional): Bounding boxes + which can be ignored for each image. Default None. + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert gt_bboxes_ignore is None, \ + f'{self.__class__.__name__} only supports ' \ + f'for gt_bboxes_ignore setting to None.' + + loss_dict = dict() + # depth loss + if self.with_depthnet: + depth_pred = preds_dicts['depth_pred'] + if not self.sweep_lidar: + depth_pred = depth_pred[:, :6, ...] + gt_depth = gt_depth[:, :6, ...] + B, N, C, H, W = depth_pred.shape + depth_pred = depth_pred.view(B * N, C, H, W) + loss_depth = self.get_depth_loss( + gt_depth.to(depth_pred.device), depth_pred) + loss_dict['loss_depth'] = loss_depth + + all_cls_scores = preds_dicts['all_cls_scores'] + all_bbox_preds = preds_dicts['all_bbox_preds'] + enc_cls_scores = preds_dicts['enc_cls_scores'] + enc_bbox_preds = preds_dicts['enc_bbox_preds'] + + num_dec_layers = len(all_cls_scores) + device = gt_labels_list[0].device + gt_bboxes_list = [ + torch.cat((gt_bboxes.gravity_center, gt_bboxes.tensor[:, 3:]), + dim=1).to(device) for gt_bboxes in gt_bboxes_list + ] + + all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] + all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] + all_gt_bboxes_ignore_list = [ + gt_bboxes_ignore for _ in range(num_dec_layers) + ] + + losses_cls, losses_bbox = multi_apply(self.loss_single, all_cls_scores, + all_bbox_preds, + all_gt_bboxes_list, + all_gt_labels_list, + all_gt_bboxes_ignore_list) + + # loss of proposal generated from encode feature map. + if enc_cls_scores is not None: + binary_labels_list = [ + torch.zeros_like(gt_labels_list[i]) + for i in range(len(all_gt_labels_list)) + ] + enc_loss_cls, enc_losses_bbox = \ + self.loss_single(enc_cls_scores, enc_bbox_preds, + gt_bboxes_list, binary_labels_list, gt_bboxes_ignore) + loss_dict['enc_loss_cls'] = enc_loss_cls + loss_dict['enc_loss_bbox'] = enc_losses_bbox + + if preds_dicts['dn_mask_dict'] is not None: + known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt = self.prepare_for_loss( + preds_dicts['dn_mask_dict']) + all_known_bboxs_list = [known_bboxs for _ in range(num_dec_layers)] + all_known_labels_list = [ + known_labels for _ in range(num_dec_layers) + ] + all_num_tgts_list = [num_tgt for _ in range(num_dec_layers)] + dn_losses_cls, dn_losses_bbox = multi_apply( + self.dn_loss_single, output_known_class, output_known_coord, + all_known_bboxs_list, all_known_labels_list, all_num_tgts_list) + loss_dict['dn_loss_cls'] = dn_losses_cls[-1] + loss_dict['dn_loss_bbox'] = dn_losses_bbox[-1] + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i in zip(dn_losses_cls[:-1], + dn_losses_bbox[:-1]): + loss_dict[f'd{num_dec_layer}.dn_loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.dn_loss_bbox'] = loss_bbox_i + num_dec_layer += 1 + + # loss from the last decoder layer + loss_dict['loss_cls'] = losses_cls[-1] + loss_dict['loss_bbox'] = losses_bbox[-1] + + # loss from other decoder layers + num_dec_layer = 0 + for loss_cls_i, loss_bbox_i in zip(losses_cls[:-1], losses_bbox[:-1]): + loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i + loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i + num_dec_layer += 1 + return loss_dict + + @force_fp32(apply_to=('preds_dicts')) + def get_bboxes(self, preds_dicts, img_metas, rescale=False): + """Generate bboxes from bbox head predictions. + Args: + preds_dicts (tuple[list[dict]]): Prediction results. + img_metas (list[dict]): Point cloud and image's meta info. + Returns: + list[dict]: Decoded bbox, scores and labels after nms. + """ + preds_dicts = self.bbox_coder.decode(preds_dicts) + num_samples = len(preds_dicts) + + ret_list = [] + for i in range(num_samples): + preds = preds_dicts[i] + bboxes = preds['bboxes'] + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 5] * 0.5 + bboxes = img_metas[i]['box_type_3d'](bboxes, bboxes.size(-1)) + scores = preds['scores'] + labels = preds['labels'] + ret_list.append([bboxes, scores, labels]) + return ret_list diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b72289ea4a7a0a7325ce172df49f928668cecaf3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/__init__.py @@ -0,0 +1,7 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/detectors +""" +from .petr3d import Petr3D + +__all__ = ['Petr3D'] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/petr3d.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/petr3d.py new file mode 100644 index 0000000000000000000000000000000000000000..a997ebac0085753698eeb09c12fd392456370760 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/detectors/petr3d.py @@ -0,0 +1,225 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/detectors +""" +import numpy as np +import torch +from mmcv.parallel.data_container import DataContainer as DC +from mmcv.runner import auto_fp16, force_fp32 +from mmdet3d.core import bbox3d2result +from mmdet3d.models.detectors.mvx_two_stage import MVXTwoStageDetector +from mmdet.models import DETECTORS + +import modelscope.models.cv.object_detection_3d.depe.mmdet3d_plugin.models.utils + + +@DETECTORS.register_module() +class Petr3D(MVXTwoStageDetector): + """Petr3D.""" + + def __init__(self, + use_grid_mask=False, + pts_voxel_layer=None, + pts_voxel_encoder=None, + pts_middle_encoder=None, + pts_fusion_layer=None, + img_backbone=None, + pts_backbone=None, + img_neck=None, + pts_neck=None, + pts_bbox_head=None, + img_roi_head=None, + img_rpn_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(Petr3D, + self).__init__(pts_voxel_layer, pts_voxel_encoder, + pts_middle_encoder, pts_fusion_layer, + img_backbone, pts_backbone, img_neck, pts_neck, + pts_bbox_head, img_roi_head, img_rpn_head, + train_cfg, test_cfg, pretrained) + + def extract_img_feat(self, img, img_metas): + """Extract features of images.""" + if isinstance(img, list): + img = torch.stack(img, dim=0) + + B = img.size(0) + if img is not None: + input_shape = img.shape[-2:] + # update real input shape of each single img + for img_meta in img_metas: + img_meta.update(input_shape=input_shape) + if img.dim() == 5: + if img.size(0) == 1 and img.size(1) != 1: + img.squeeze_() + else: + B, N, C, H, W = img.size() + img = img.view(B * N, C, H, W) + img_feats = self.img_backbone(img) + if isinstance(img_feats, dict): + img_feats = list(img_feats.values()) + else: + return None + if self.with_img_neck: + img_feats = self.img_neck(img_feats) + img_feats_reshaped = [] + for img_feat in img_feats: + BN, C, H, W = img_feat.size() + img_feats_reshaped.append(img_feat.view(B, int(BN / B), C, H, W)) + return img_feats_reshaped + + @auto_fp16(apply_to=('img'), out_fp32=True) + def extract_feat(self, img, img_metas): + """Extract features from images and points.""" + img_feats = self.extract_img_feat(img, img_metas) + return img_feats + + def forward_pts_train(self, + pts_feats, + gt_bboxes_3d, + gt_labels_3d, + img_metas, + gt_bboxes_ignore=None): + """Forward function for point cloud branch. + Args: + pts_feats (list[torch.Tensor]): Features of point cloud branch + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth + boxes for each sample. + gt_labels_3d (list[torch.Tensor]): Ground truth labels for + boxes of each sampole + img_metas (list[dict]): Meta information of samples. + gt_bboxes_ignore (list[torch.Tensor], optional): Ground truth + boxes to be ignored. Defaults to None. + Returns: + dict: Losses of each branch. + """ + outs = self.pts_bbox_head(pts_feats, img_metas) + gt_depth = torch.stack( + [img_meta['gt_depth'] for img_meta in img_metas]) + loss_inputs = [gt_bboxes_3d, gt_labels_3d, outs, gt_depth] + losses = self.pts_bbox_head.loss(*loss_inputs) + + return losses + + @force_fp32(apply_to=('img', 'points')) + def forward(self, return_loss=True, **kwargs): + """Calls either forward_train or forward_test depending on whether + return_loss=True. + Note this setting will change the expected inputs. When + `return_loss=True`, img and img_metas are single-nested (i.e. + torch.Tensor and list[dict]), and when `resturn_loss=False`, img and + img_metas should be double nested (i.e. list[torch.Tensor], + list[list[dict]]), with the outer list indicating test time + augmentations. + """ + if return_loss: + return self.forward_train(**kwargs) + else: + return self.forward_test(**kwargs) + + def forward_train(self, + points=None, + img_metas=None, + gt_bboxes_3d=None, + gt_labels_3d=None, + gt_labels=None, + gt_bboxes=None, + img=None, + proposals=None, + gt_bboxes_ignore=None, + img_depth=None, + img_mask=None): + """Forward training function. + Args: + points (list[torch.Tensor], optional): Points of each sample. + Defaults to None. + img_metas (list[dict], optional): Meta information of each sample. + Defaults to None. + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`], optional): + Ground truth 3D boxes. Defaults to None. + gt_labels_3d (list[torch.Tensor], optional): Ground truth labels + of 3D boxes. Defaults to None. + gt_labels (list[torch.Tensor], optional): Ground truth labels + of 2D boxes in images. Defaults to None. + gt_bboxes (list[torch.Tensor], optional): Ground truth 2D boxes in + images. Defaults to None. + img (torch.Tensor optional): Images of each sample with shape + (N, C, H, W). Defaults to None. + proposals ([list[torch.Tensor], optional): Predicted proposals + used for training Fast RCNN. Defaults to None. + gt_bboxes_ignore (list[torch.Tensor], optional): Ground truth + 2D boxes in images to be ignored. Defaults to None. + Returns: + dict: Losses of different branches. + """ + + img_feats = self.extract_feat(img=img, img_metas=img_metas) + + losses = dict() + losses_pts = self.forward_pts_train(img_feats, gt_bboxes_3d, + gt_labels_3d, img_metas, + gt_bboxes_ignore) + losses.update(losses_pts) + return losses + + def forward_test(self, img_metas, img=None, **kwargs): + for var, name in [(img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError('{} must be a list, but got {}'.format( + name, type(var))) + img = [img] if img is None else img + return self.simple_test(img_metas[0], img[0], **kwargs) + + def simple_test_pts(self, x, img_metas, rescale=False): + """Test function of point cloud branch.""" + outs = self.pts_bbox_head(x, img_metas) + bbox_list = self.pts_bbox_head.get_bboxes( + outs, img_metas, rescale=rescale) + bbox_results = [ + bbox3d2result(bboxes, scores, labels) + for bboxes, scores, labels in bbox_list + ] + return bbox_results + + def simple_test(self, img_metas, img=None, rescale=False): + """Test function without augmentaiton.""" + if not torch.cuda.is_available() and img is not None: + if isinstance(img, torch.Tensor): + img = img[0] + elif isinstance(img, DC) and isinstance(img_metas, DC): + img = img.data[0] + img_metas = img_metas.data[0] + img_feats = self.extract_feat(img=img, img_metas=img_metas) + bbox_list = [dict() for i in range(len(img_metas))] + bbox_pts = self.simple_test_pts(img_feats, img_metas, rescale=rescale) + for result_dict, pts_bbox in zip(bbox_list, bbox_pts): + result_dict['pts_bbox'] = pts_bbox + return bbox_list + + def aug_test_pts(self, feats, img_metas, rescale=False): + feats_list = [] + for j in range(len(feats[0])): + feats_list_level = [] + for i in range(len(feats)): + feats_list_level.append(feats[i][j]) + feats_list.append(torch.stack(feats_list_level, -1).mean(-1)) + outs = self.pts_bbox_head(feats_list, img_metas) + bbox_list = self.pts_bbox_head.get_bboxes( + outs, img_metas, rescale=rescale) + bbox_results = [ + bbox3d2result(bboxes, scores, labels) + for bboxes, scores, labels in bbox_list + ] + return bbox_results + + def aug_test(self, img_metas, imgs=None, rescale=False): + """Test function with augmentaiton.""" + img_feats = self.extract_feats(img_metas, imgs) + img_metas = img_metas[0] + bbox_list = [dict() for i in range(len(img_metas))] + bbox_pts = self.aug_test_pts(img_feats, img_metas, rescale) + for result_dict, pts_bbox in zip(bbox_list, bbox_pts): + result_dict['pts_bbox'] = pts_bbox + return bbox_list diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..871b3eae831e31c786f701fbe58de6504827662d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/__init__.py @@ -0,0 +1,9 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/necks +""" +from .cp_fpn import CPFPN + +__all__ = [ + 'CPFPN', +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/cp_fpn.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/cp_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..badcbd5e1ec58f6395710665b102cd19725299bf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/necks/cp_fpn.py @@ -0,0 +1,207 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/necks +""" +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule +from mmcv.runner import BaseModule, auto_fp16 +from mmdet.models import NECKS + + +# This FPN remove the unused parameters which can used with checkpoint (with_cp = True in Backbone) +@NECKS.register_module() +class CPFPN(BaseModule): + r"""Feature Pyramid Network. + + This is an implementation of paper `Feature Pyramid Networks for Object + Detection `_. + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, it is equivalent to `add_extra_convs='on_input'`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + init_cfg (dict or list[dict], optional): Initialization config dict. + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest'), + init_cfg=dict( + type='Xavier', layer='Conv2d', distribution='uniform')): + super(CPFPN, self).__init__(init_cfg) + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + self.add_extra_convs = 'on_input' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + self.lateral_convs.append(l_conv) + if i == 0: + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + @auto_fp16() + def forward(self, inputs): + """Forward function.""" + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) if i == 0 else laterals[i] + for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/__init__.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ae88d6d8209e81b58d4041d19d67d3dc06c4768c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/__init__.py @@ -0,0 +1,12 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/utils +""" +from .petr_transformer import (PETRDNTransformer, PETRMultiheadAttention, + PETRTransformerDecoder, PETRTransformerEncoder) +from .positional_encoding import SinePositionalEncoding3D + +__all__ = [ + 'SinePositionalEncoding3D', 'PETRDNTransformer', 'PETRMultiheadAttention', + 'PETRTransformerEncoder', 'PETRTransformerDecoder' +] diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/petr_transformer.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/petr_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..7cc0242b0f70acdb19fb9817eac118eb1493eaf1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/petr_transformer.py @@ -0,0 +1,453 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/utils +""" +import copy +import math +import warnings +from typing import Sequence + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from mmcv.cnn import (build_activation_layer, build_conv_layer, + build_norm_layer, xavier_init) +from mmcv.cnn.bricks.drop import build_dropout +from mmcv.cnn.bricks.registry import (ATTENTION, TRANSFORMER_LAYER, + TRANSFORMER_LAYER_SEQUENCE) +from mmcv.cnn.bricks.transformer import (BaseTransformerLayer, + TransformerLayerSequence, + build_transformer_layer_sequence) +from mmcv.runner.base_module import BaseModule +from mmcv.utils import (ConfigDict, build_from_cfg, deprecated_api_warning, + to_2tuple) +from mmdet.models.utils.builder import TRANSFORMER + + +@TRANSFORMER.register_module() +class PETRDNTransformer(BaseModule): + """Implements the DETR transformer. + Following the official DETR implementation, this module copy-paste + from torch.nn.Transformer with modifications: + * positional encodings are passed in MultiheadAttention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers + See `paper: End-to-End Object Detection with Transformers + `_ for details. + Args: + encoder (`mmcv.ConfigDict` | Dict): Config of + TransformerEncoder. Defaults to None. + decoder ((`mmcv.ConfigDict` | Dict)): Config of + TransformerDecoder. Defaults to None + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Defaults to None. + """ + + def __init__(self, encoder=None, decoder=None, init_cfg=None, cross=False): + super(PETRDNTransformer, self).__init__(init_cfg=init_cfg) + if encoder is not None: + self.encoder = build_transformer_layer_sequence(encoder) + else: + self.encoder = None + self.decoder = build_transformer_layer_sequence(decoder) + self.embed_dims = self.decoder.embed_dims + self.cross = cross + + def init_weights(self): + # follow the official DETR to init parameters + for m in self.modules(): + if hasattr(m, 'weight') and m.weight.dim() > 1: + xavier_init(m, distribution='uniform') + self._is_init = True + + def forward(self, + x, + mask, + query_embed, + pos_embed, + attn_masks=None, + reg_branch=None): + """Forward function for `Transformer`. + Args: + x (Tensor): Input query with shape [bs, c, h, w] where + c = embed_dims. + mask (Tensor): The key_padding_mask used for encoder and decoder, + with shape [bs, h, w]. + query_embed (Tensor): The query embedding for decoder, with shape + [num_query, c]. + pos_embed (Tensor): The positional encoding for encoder and + decoder, with the same shape as `x`. + Returns: + tuple[Tensor]: results of decoder containing the following tensor. + - out_dec: Output from decoder. If return_intermediate_dec \ + is True output has shape [num_dec_layers, bs, + num_query, embed_dims], else has shape [1, bs, \ + num_query, embed_dims]. + - memory: Output results from encoder, with shape \ + [bs, embed_dims, h, w]. + """ + bs, n, c, h, w = x.shape + memory = x.permute(1, 3, 4, 0, + 2).reshape(-1, bs, + c) # [bs, n, c, h, w] -> [n*h*w, bs, c] + pos_embed = pos_embed.permute(1, 3, 4, 0, 2).reshape( + -1, bs, c) # [bs, n, c, h, w] -> [n*h*w, bs, c] + query_embed = query_embed.transpose( + 0, 1) # [num_query, dim] -> [num_query, bs, dim] + mask = mask.view(bs, -1) # [bs, n, h, w] -> [bs, n*h*w] + target = torch.zeros_like(query_embed) + # out_dec: [num_layers, num_query, bs, dim] + out_dec = self.decoder( + query=target, + key=memory, + value=memory, + key_pos=pos_embed, + query_pos=query_embed, + key_padding_mask=mask, + attn_masks=[attn_masks, None], + reg_branch=reg_branch, + ) + out_dec = out_dec.transpose(1, 2) + memory = memory.reshape(n, h, w, bs, c).permute(3, 0, 4, 1, 2) + return out_dec, memory + + +@TRANSFORMER_LAYER.register_module() +class PETRTransformerDecoderLayer(BaseTransformerLayer): + """Implements decoder layer in DETR transformer. + Args: + attn_cfgs (list[`mmcv.ConfigDict`] | list[dict] | dict )): + Configs for self_attention or cross_attention, the order + should be consistent with it in `operation_order`. If it is + a dict, it would be expand to the number of attention in + `operation_order`. + feedforward_channels (int): The hidden dimension for FFNs. + ffn_dropout (float): Probability of an element to be zeroed + in ffn. Default 0.0. + operation_order (tuple[str]): The execution order of operation + in transformer. Such as ('self_attn', 'norm', 'ffn', 'norm'). + Default:None + act_cfg (dict): The activation config for FFNs. Default: `LN` + norm_cfg (dict): Config dict for normalization layer. + Default: `LN`. + ffn_num_fcs (int): The number of fully-connected layers in FFNs. + Default:2. + """ + + def __init__(self, + attn_cfgs, + feedforward_channels, + embed_dims=256, + ffn_dropout=0.0, + operation_order=None, + act_cfg=dict(type='ReLU', inplace=True), + norm_cfg=dict(type='LN'), + ffn_num_fcs=2, + with_cp=True, + **kwargs): + ffn_cfgs = dict( + embed_dims=embed_dims, + feedforward_channels=feedforward_channels, + ffn_dropout=ffn_dropout, + ffn_num_fcs=ffn_num_fcs) + kwargs['ffn_cfgs'] = ffn_cfgs + super(PETRTransformerDecoderLayer, self).__init__( + attn_cfgs=attn_cfgs, + operation_order=operation_order, + act_cfg=act_cfg, + norm_cfg=norm_cfg, + **kwargs) + assert len(operation_order) == 6 + assert set(operation_order) == set( + ['self_attn', 'norm', 'cross_attn', 'ffn']) + self.use_checkpoint = with_cp + + def _forward( + self, + query, + key=None, + value=None, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + ): + """Forward function for `TransformerCoder`. + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + x = super(PETRTransformerDecoderLayer, self). \ + forward(query, + key=key, + value=value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + ) + return x + + def forward(self, + query, + key=None, + value=None, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `TransformerCoder`. + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + + if self.use_checkpoint and self.training: + x = cp.checkpoint( + self._forward, + query, + key, + value, + query_pos, + key_pos, + attn_masks, + query_key_padding_mask, + key_padding_mask, + ) + else: + x = self._forward( + query, + key=key, + value=value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask) + return x + + +@ATTENTION.register_module() +class PETRMultiheadAttention(BaseModule): + """A wrapper for ``torch.nn.MultiheadAttention``. + This module implements MultiheadAttention with identity connection, + and positional encoding is also passed as input. + Args: + embed_dims (int): The embedding dimension. + num_heads (int): Parallel attention heads. + attn_drop (float): A Dropout layer on attn_output_weights. + Default: 0.0. + proj_drop (float): A Dropout layer after `nn.MultiheadAttention`. + Default: 0.0. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + batch_first (bool): When it is True, Key, Query and Value are shape of + (batch, n, embed_dim), otherwise (n, batch, embed_dim). + Default to False. + """ + + def __init__(self, + embed_dims, + num_heads, + attn_drop=0., + proj_drop=0., + dropout_layer=dict(type='Dropout', drop_prob=0.), + init_cfg=None, + batch_first=False, + **kwargs): + super(PETRMultiheadAttention, self).__init__(init_cfg) + if 'dropout' in kwargs: + warnings.warn( + 'The arguments `dropout` in MultiheadAttention ' + 'has been deprecated, now you can separately ' + 'set `attn_drop`(float), proj_drop(float), ' + 'and `dropout_layer`(dict) ', DeprecationWarning) + attn_drop = kwargs['dropout'] + dropout_layer['drop_prob'] = kwargs.pop('dropout') + + self.embed_dims = embed_dims + self.num_heads = num_heads + self.batch_first = batch_first + + self.attn = nn.MultiheadAttention(embed_dims, num_heads, attn_drop, + **kwargs) + + self.proj_drop = nn.Dropout(proj_drop) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else nn.Identity() + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='MultiheadAttention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_pos=None, + attn_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `MultiheadAttention`. + **kwargs allow passing a more general data flow when combining + with other operations in `transformerlayer`. + Args: + query (Tensor): The input query with shape [num_queries, bs, + embed_dims] if self.batch_first is False, else + [bs, num_queries embed_dims]. + key (Tensor): The key tensor with shape [num_keys, bs, + embed_dims] if self.batch_first is False, else + [bs, num_keys, embed_dims] . + If None, the ``query`` will be used. Defaults to None. + value (Tensor): The value tensor with same shape as `key`. + Same in `nn.MultiheadAttention.forward`. Defaults to None. + If None, the `key` will be used. + identity (Tensor): This tensor, with the same shape as x, + will be used for the identity link. + If None, `x` will be used. Defaults to None. + query_pos (Tensor): The positional encoding for query, with + the same shape as `x`. If not None, it will + be added to `x` before forward function. Defaults to None. + key_pos (Tensor): The positional encoding for `key`, with the + same shape as `key`. Defaults to None. If not None, it will + be added to `key` before forward function. If None, and + `query_pos` has the same shape as `key`, then `query_pos` + will be used for `key_pos`. Defaults to None. + attn_mask (Tensor): ByteTensor mask with shape [num_queries, + num_keys]. Same in `nn.MultiheadAttention.forward`. + Defaults to None. + key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. + Defaults to None. + Returns: + Tensor: forwarded results with shape + [num_queries, bs, embed_dims] + if self.batch_first is False, else + [bs, num_queries embed_dims]. + """ + + if key is None: + key = query + if value is None: + value = key + if identity is None: + identity = query + if key_pos is None: + if query_pos is not None: + # use query_pos if key_pos is not available + if query_pos.shape == key.shape: + key_pos = query_pos + else: + warnings.warn(f'position encoding of key is' + f'missing in {self.__class__.__name__}.') + if query_pos is not None: + query = query + query_pos + if key_pos is not None: + key = key + key_pos + + # Because the dataflow('key', 'query', 'value') of + # ``torch.nn.MultiheadAttention`` is (num_query, batch, + # embed_dims), We should adjust the shape of dataflow from + # batch_first (batch, num_query, embed_dims) to num_query_first + # (num_query ,batch, embed_dims), and recover ``attn_output`` + # from num_query_first to batch_first. + if self.batch_first: + query = query.transpose(0, 1) + key = key.transpose(0, 1) + value = value.transpose(0, 1) + + out = self.attn( + query=query, + key=key, + value=value, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask)[0] + + if self.batch_first: + out = out.transpose(0, 1) + + return identity + self.dropout_layer(self.proj_drop(out)) + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class PETRTransformerEncoder(TransformerLayerSequence): + """TransformerEncoder of DETR. + Args: + post_norm_cfg (dict): Config of last normalization layer. Default: + `LN`. Only used when `self.pre_norm` is `True` + """ + + def __init__(self, *args, post_norm_cfg=dict(type='LN'), **kwargs): + super(PETRTransformerEncoder, self).__init__(*args, **kwargs) + if post_norm_cfg is not None: + self.post_norm = build_norm_layer( + post_norm_cfg, self.embed_dims)[1] if self.pre_norm else None + else: + assert not self.pre_norm, f'Use prenorm in ' \ + f'{self.__class__.__name__},' \ + f'Please specify post_norm_cfg' + self.post_norm = None + + def forward(self, *args, **kwargs): + """Forward function for `TransformerCoder`. + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + x = super(PETRTransformerEncoder, self).forward(*args, **kwargs) + if self.post_norm is not None: + x = self.post_norm(x) + return x + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class PETRTransformerDecoder(TransformerLayerSequence): + """Implements the decoder in DETR transformer. + Args: + return_intermediate (bool): Whether to return intermediate outputs. + post_norm_cfg (dict): Config of last normalization layer. Default: + `LN`. + """ + + def __init__(self, + *args, + post_norm_cfg=dict(type='LN'), + return_intermediate=False, + **kwargs): + + super(PETRTransformerDecoder, self).__init__(*args, **kwargs) + self.return_intermediate = return_intermediate + if post_norm_cfg is not None: + self.post_norm = build_norm_layer(post_norm_cfg, + self.embed_dims)[1] + else: + self.post_norm = None + + def forward(self, query, *args, **kwargs): + """Forward function for `TransformerDecoder`. + Args: + query (Tensor): Input query with shape + `(num_query, bs, embed_dims)`. + Returns: + Tensor: Results with shape [1, num_query, bs, embed_dims] when + return_intermediate is `False`, otherwise it has shape + [num_layers, num_query, bs, embed_dims]. + """ + if not self.return_intermediate: + x = super().forward(query, *args, **kwargs) + if self.post_norm: + x = self.post_norm(x)[None] + return x + + intermediate = [] + for layer in self.layers: + query = layer(query, *args, **kwargs) + if self.return_intermediate: + if self.post_norm is not None: + intermediate.append(self.post_norm(query)) + else: + intermediate.append(query) + return torch.stack(intermediate) diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/positional_encoding.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/positional_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..096cd2c6698f3ee101d80cbc4aedc21d25d5e773 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/mmdet3d_plugin/models/utils/positional_encoding.py @@ -0,0 +1,120 @@ +""" +The implementation here is modified based on PETR, originally Apache-2.0 license and publicly available at +https://github.com/megvii-research/PETR/blob/main/projects/mmdet3d_plugin/models/utils +""" +import math + +import torch +import torch.nn as nn +from mmcv.cnn.bricks.transformer import POSITIONAL_ENCODING +from mmcv.runner import BaseModule + + +@POSITIONAL_ENCODING.register_module() +class SinePositionalEncoding3D(BaseModule): + """Position encoding with sine and cosine functions. + See `End-to-End Object Detection with Transformers + `_ for details. + Args: + num_feats (int): The feature dimension for each position + along x-axis or y-axis. Note the final returned dimension + for each position is 2 times of this value. + temperature (int, optional): The temperature used for scaling + the position embedding. Defaults to 10000. + normalize (bool, optional): Whether to normalize the position + embedding. Defaults to False. + scale (float, optional): A scale factor that scales the position + embedding. The scale will be used only when `normalize` is True. + Defaults to 2*pi. + eps (float, optional): A value added to the denominator for + numerical stability. Defaults to 1e-6. + offset (float): offset add to embed when do the normalization. + Defaults to 0. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + + def __init__(self, + num_feats, + temperature=10000, + normalize=False, + scale=2 * math.pi, + eps=1e-6, + offset=0., + init_cfg=None, + skip_n=False): + super(SinePositionalEncoding3D, self).__init__(init_cfg) + if normalize: + assert isinstance(scale, (float, int)), 'when normalize is set,' \ + 'scale should be provided and in float or int type, ' \ + f'found {type(scale)}' + self.num_feats = num_feats + self.temperature = temperature + self.normalize = normalize + self.scale = scale + self.eps = eps + self.offset = offset + self.skip_n = skip_n + + def forward(self, mask): + """Forward function for `SinePositionalEncoding`. + Args: + mask (Tensor): ByteTensor mask. Non-zero values representing + ignored positions, while zero values means valid positions + for this image. Shape [bs, h, w]. + Returns: + pos (Tensor): Returned position embedding with shape + [bs, num_feats*2, h, w]. + """ + # For convenience of exporting to ONNX, it's required to convert + # `masks` from bool to int. + mask = mask.to(torch.int) + not_mask = 1 - mask # logical_not + if not self.skip_n: + n_embed = not_mask.cumsum(1, dtype=torch.float32) + y_embed = not_mask.cumsum(2, dtype=torch.float32) + x_embed = not_mask.cumsum(3, dtype=torch.float32) + if self.normalize: + if not self.skip_n: + n_embed = (n_embed + self.offset) / \ + (n_embed[:, -1:, :, :] + self.eps) * self.scale + y_embed = (y_embed + self.offset) / \ + (y_embed[:, :, -1:, :] + self.eps) * self.scale + x_embed = (x_embed + self.offset) / \ + (x_embed[:, :, :, -1:] + self.eps) * self.scale + dim_t = torch.arange( + self.num_feats, dtype=torch.float32, device=mask.device) + dim_t = self.temperature**( + 2 * torch.div(dim_t, 2, rounding_mode='floor') / self.num_feats) + if not self.skip_n: + pos_n = n_embed[:, :, :, :, None] / dim_t + pos_x = x_embed[:, :, :, :, None] / dim_t + pos_y = y_embed[:, :, :, :, None] / dim_t + # use `view` instead of `flatten` for dynamically exporting to ONNX + B, N, H, W = mask.size() + if not self.skip_n: + pos_n = torch.stack( + (pos_n[:, :, :, :, 0::2].sin(), pos_n[:, :, :, :, 1::2].cos()), + dim=4).view(B, N, H, W, -1) + pos_x = torch.stack( + (pos_x[:, :, :, :, 0::2].sin(), pos_x[:, :, :, :, 1::2].cos()), + dim=4).view(B, N, H, W, -1) + pos_y = torch.stack( + (pos_y[:, :, :, :, 0::2].sin(), pos_y[:, :, :, :, 1::2].cos()), + dim=4).view(B, N, H, W, -1) + if self.skip_n: + pos = torch.cat((pos_y, pos_x), dim=4).permute(0, 1, 4, 2, 3) + else: + pos = torch.cat((pos_n, pos_y, pos_x), + dim=4).permute(0, 1, 4, 2, 3) + return pos + + def __repr__(self): + """str: a string that describes the module""" + repr_str = self.__class__.__name__ + repr_str += f'(num_feats={self.num_feats}, ' + repr_str += f'temperature={self.temperature}, ' + repr_str += f'normalize={self.normalize}, ' + repr_str += f'scale={self.scale}, ' + repr_str += f'eps={self.eps})' + return repr_str diff --git a/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/result_vis.py b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/result_vis.py new file mode 100644 index 0000000000000000000000000000000000000000..efaef0b501e4ecf846c5fcb650614c3cacd7c2bf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/object_detection_3d/depe/result_vis.py @@ -0,0 +1,261 @@ +""" +The implementation here is modified based on BEVDet, originally Apache-2.0 license and publicly available at +https://github.com/HuangJunJie2017/BEVDet/blob/dev2.0/tools/analysis_tools/vis.py +""" +import argparse +import os +import pickle + +import cv2 +import json +import numpy as np +from mmdet3d.core.bbox.structures.lidar_box3d import LiDARInstance3DBoxes as LB +from pyquaternion.quaternion import Quaternion + + +def check_point_in_img(points, height, width): + valid = np.logical_and(points[:, 0] >= 0, points[:, 1] >= 0) + valid = np.logical_and( + valid, np.logical_and(points[:, 0] < width, points[:, 1] < height)) + return valid + + +def depth2color(depth): + gray = max(0, min((depth + 2.5) / 3.0, 1.0)) + max_lumi = 200 + colors = np.array( + [[max_lumi, 0, max_lumi], [max_lumi, 0, 0], [max_lumi, max_lumi, 0], + [0, max_lumi, 0], [0, max_lumi, max_lumi], [0, 0, max_lumi]], + dtype=np.float32) + if gray == 1: + return tuple(colors[-1].tolist()) + num_rank = len(colors) - 1 + rank = np.floor(gray * num_rank).astype(int) + diff = (gray - rank / num_rank) * num_rank + tmp = colors[rank + 1] - colors[rank] + return tuple((colors[rank] + tmp * diff).tolist()) + + +def lidar2img(points_lidar, camrera_info): + points_lidar_homogeneous = \ + np.concatenate([points_lidar, + np.ones((points_lidar.shape[0], 1), + dtype=points_lidar.dtype)], axis=1) + camera2lidar = np.eye(4, dtype=np.float32) + camera2lidar[:3, :3] = camrera_info['sensor2lidar_rotation'] + camera2lidar[:3, 3] = camrera_info['sensor2lidar_translation'] + lidar2camera = np.linalg.inv(camera2lidar) + points_camera_homogeneous = points_lidar_homogeneous @ lidar2camera.T + points_camera = points_camera_homogeneous[:, :3] + valid = np.ones((points_camera.shape[0]), dtype=bool) + valid = np.logical_and(points_camera[:, -1] > 0.5, valid) + points_camera = points_camera / points_camera[:, 2:3] + camera2img = camrera_info['cam_intrinsic'] + points_img = points_camera @ camera2img.T + points_img = points_img[:, :2] + return points_img, valid + + +def get_lidar2global(infos): + lidar2ego = np.eye(4, dtype=np.float32) + lidar2ego[:3, :3] = Quaternion(infos['lidar2ego_rotation']).rotation_matrix + lidar2ego[:3, 3] = infos['lidar2ego_translation'] + ego2global = np.eye(4, dtype=np.float32) + ego2global[:3, :3] = Quaternion( + infos['ego2global_rotation']).rotation_matrix + ego2global[:3, 3] = infos['ego2global_translation'] + return ego2global @ lidar2ego + + +def plot_result(res_path, + vis_thred=0.3, + version='val', + draw_gt=True, + save_format='image'): + img_list = [] + # fixed parameters + root_path = '/data/Dataset/nuScenes' + show_range = 50 # Range of visualization in BEV + canva_size = 1000 # Size of canva in pixel + vis_frames = 500 # Max number of frames for visualization + scale_factor = 2 # Trade-off between image-view and bev in size of the visualized canvas + fps = 5 # Frame rate of video + vis_dir = './video_result' # Video output path + color_map = {0: (255, 255, 0), 1: (0, 255, 255)} + + # load predicted results + res = json.load(open(res_path, 'r')) + # load dataset information + info_path = os.path.join(root_path, + f'mmdet3d_nuscenes_30f_infos_{version}.pkl') + with open(info_path, 'rb') as f: + dataset = pickle.load(f) + # prepare save path and medium + if save_format == 'video' and not os.path.exists(vis_dir): + os.makedirs(vis_dir) + fourcc = cv2.VideoWriter_fourcc(*'MP4V') + vout = cv2.VideoWriter( + os.path.join(vis_dir, 'vis.mp4'), fourcc, fps, + (int(1600 / scale_factor * 3), + int(900 / scale_factor * 2 + canva_size))) + + draw_boxes_indexes_bev = [(0, 1), (1, 2), (2, 3), (3, 0)] + draw_boxes_indexes_img_view = [(0, 1), (1, 2), (2, 3), (3, 0), (4, 5), + (5, 6), (6, 7), (7, 4), (0, 4), (1, 5), + (2, 6), (3, 7)] + views = [ + 'CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_BACK_LEFT', + 'CAM_BACK', 'CAM_BACK_RIGHT' + ] + dataset_dict = {} + for sample in dataset['infos']: + if sample['token'] not in dataset_dict: + dataset_dict[sample['token']] = sample + for cnt, rst_token in enumerate(res['results']): + if cnt >= vis_frames: + break + # collect instances + pred_res = res['results'][rst_token] + infos = dataset_dict[rst_token] + pred_boxes = [ + pred_res[rid]['translation'] + pred_res[rid]['size'] + [ + Quaternion(pred_res[rid]['rotation']).yaw_pitch_roll[0] + + np.pi / 2 + ] for rid in range(len(pred_res)) + ] + if len(pred_boxes) == 0: + corners_lidar = np.zeros((0, 3), dtype=np.float32) + else: + pred_boxes = np.array(pred_boxes, dtype=np.float32) + boxes = LB(pred_boxes, origin=(0.5, 0.5, 0.5)) + corners_global = boxes.corners.numpy().reshape(-1, 3) + corners_global = np.concatenate( + [corners_global, + np.ones([corners_global.shape[0], 1])], + axis=1) + l2g = get_lidar2global(infos) + corners_lidar = corners_global @ np.linalg.inv(l2g).T + corners_lidar = corners_lidar[:, :3] + pred_flag = np.ones((corners_lidar.shape[0] // 8, ), dtype=bool) + scores = [ + pred_res[rid]['detection_score'] for rid in range(len(pred_res)) + ] + if draw_gt: + gt_boxes = infos['gt_boxes'] + gt_boxes[:, -1] = gt_boxes[:, -1] + np.pi / 2 + width = gt_boxes[:, 4].copy() + gt_boxes[:, 4] = gt_boxes[:, 3] + gt_boxes[:, 3] = width + corners_lidar_gt = \ + LB(infos['gt_boxes'], + origin=(0.5, 0.5, 0.5)).corners.numpy().reshape(-1, 3) + corners_lidar = np.concatenate([corners_lidar, corners_lidar_gt], + axis=0) + gt_flag = np.ones((corners_lidar_gt.shape[0] // 8), dtype=bool) + pred_flag = np.concatenate( + [pred_flag, np.logical_not(gt_flag)], axis=0) + scores = scores + [0 for _ in range(infos['gt_boxes'].shape[0])] + scores = np.array(scores, dtype=np.float32) + sort_ids = np.argsort(scores) + + # image view + imgs = [] + for view in views: + img = cv2.imread(infos['cams'][view]['data_path']) + # draw instances + corners_img, valid = lidar2img(corners_lidar, infos['cams'][view]) + valid = np.logical_and( + valid, + check_point_in_img(corners_img, img.shape[0], img.shape[1])) + valid = valid.reshape( + -1, 8) # valid means: d>0 and visible in current view + corners_img = corners_img.reshape(-1, 8, 2).astype(int) + for aid in range(valid.shape[0]): + if scores[aid] < vis_thred and pred_flag[aid]: + continue + for index in draw_boxes_indexes_img_view: + if valid[aid, index[0]] and valid[aid, index[1]]: + cv2.line( + img, + corners_img[aid, index[0]], + corners_img[aid, index[1]], + color=color_map[int(pred_flag[aid])], + thickness=scale_factor) + imgs.append(img) + + # bird-eye-view + canvas = np.zeros((int(canva_size), int(canva_size), 3), + dtype=np.uint8) + # draw lidar points + lidar_points = np.fromfile(infos['lidar_path'], dtype=np.float32) + lidar_points = lidar_points.reshape(-1, 5)[:, :3] + lidar_points[:, 1] = -lidar_points[:, 1] + lidar_points[:, :2] = \ + (lidar_points[:, :2] + show_range) / show_range / 2.0 * canva_size + for p in lidar_points: + if check_point_in_img( + p.reshape(1, 3), canvas.shape[1], canvas.shape[0])[0]: + color = depth2color(p[2]) + cv2.circle( + canvas, (int(p[0]), int(p[1])), + radius=0, + color=color, + thickness=1) + + # draw instances + corners_lidar = corners_lidar.reshape(-1, 8, 3) + corners_lidar[:, :, 1] = -corners_lidar[:, :, 1] + bottom_corners_bev = corners_lidar[:, [0, 3, 7, 4], :2] + bottom_corners_bev = \ + (bottom_corners_bev + show_range) / show_range / 2.0 * canva_size + bottom_corners_bev = np.round(bottom_corners_bev).astype(np.int32) + center_bev = corners_lidar[:, [0, 3, 7, 4], :2].mean(axis=1) + head_bev = corners_lidar[:, [0, 4], :2].mean(axis=1) + canter_canvas = \ + (center_bev + show_range) / show_range / 2.0 * canva_size + center_canvas = canter_canvas.astype(np.int32) + head_canvas = (head_bev + show_range) / show_range / 2.0 * canva_size + head_canvas = head_canvas.astype(np.int32) + + for rid in sort_ids: + score = scores[rid] + if score < vis_thred and pred_flag[rid]: + continue + score = min(score * 2.0, 1.0) if pred_flag[rid] else 1.0 + color = color_map[int(pred_flag[rid])] + for index in draw_boxes_indexes_bev: + cv2.line( + canvas, + bottom_corners_bev[rid, index[0]], + bottom_corners_bev[rid, index[1]], + [color[0] * score, color[1] * score, color[2] * score], + thickness=1) + cv2.line( + canvas, + center_canvas[rid], + head_canvas[rid], + [color[0] * score, color[1] * score, color[2] * score], + 1, + lineType=8) + + # fuse image-view and bev + img = np.zeros((900 * 2 + canva_size * scale_factor, 1600 * 3, 3), + dtype=np.uint8) + img[:900, :, :] = np.concatenate(imgs[:3], axis=1) + img_back = np.concatenate( + [imgs[3][:, ::-1, :], imgs[4][:, ::-1, :], imgs[5][:, ::-1, :]], + axis=1) + img[900 + canva_size * scale_factor:, :, :] = img_back + img = cv2.resize(img, (int(1600 / scale_factor * 3), + int(900 / scale_factor * 2 + canva_size))) + w_begin = int((1600 * 3 / scale_factor - canva_size) // 2) + img[int(900 / scale_factor):int(900 / scale_factor) + canva_size, + w_begin:w_begin + canva_size, :] = canvas + + if save_format == 'image': + img_list += [img] + elif save_format == 'video': + vout.write(img) + if save_format == 'video': + vout.release() + return img_list diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e9671fc7584ba6d23b83ea103d56c0cbc7c3715 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import OCRDetection + +else: + _import_structure = { + 'model': ['OCRDetection'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/model.py b/runtime/Lib/site_packages/models/cv/ocr_detection/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b148e9a16ba4616335f82d4c2a857c3d10cd05dd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/model.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict + +import numpy as np +import torch +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .modules.dbnet import DBModel, DBNasModel, VLPTModel +from .utils import boxes_from_bitmap, polygons_from_bitmap + +LOGGER = get_logger() + + +@MODELS.register_module(Tasks.ocr_detection, module_name=Models.ocr_detection) +class OCRDetection(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """initialize the ocr recognition model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, **kwargs) + + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + cfgs = Config.from_file( + os.path.join(model_dir, ModelFile.CONFIGURATION)) + self.thresh = cfgs.model.inference_kwargs.thresh + self.return_polygon = cfgs.model.inference_kwargs.return_polygon + self.backbone = cfgs.model.backbone + self.detector = None + self.onnx_export = False + if self.backbone == 'resnet50': + self.detector = VLPTModel() + elif self.backbone == 'resnet18': + self.detector = DBModel() + elif self.backbone == 'proxylessnas': + self.detector = DBNasModel() + else: + raise TypeError( + f'detector backbone should be either resnet18, resnet50, but got {cfgs.model.backbone}' + ) + if model_path != '': + self.detector.load_state_dict( + torch.load(model_path, map_location='cpu'), strict=False) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """ + Args: + img (`torch.Tensor`): image tensor, + shape of each tensor is [3, H, W]. + + Return: + results (`torch.Tensor`): bitmap tensor, + shape of each tensor is [1, H, W]. + org_shape (`List`): image original shape, + value is [height, width]. + """ + if type(input) is dict: + pred = self.detector(input['img']) + else: + # for onnx convert + input = {'img': input, 'org_shape': [800, 800]} + pred = self.detector(input['img']) + return {'results': pred, 'org_shape': input['org_shape']} + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + pred = inputs['results'][0] + + if self.onnx_export: + return pred + + height, width = inputs['org_shape'] + segmentation = pred > self.thresh + if self.return_polygon: + boxes, scores = polygons_from_bitmap(pred, segmentation, width, + height) + else: + boxes, scores = boxes_from_bitmap(pred, segmentation, width, + height) + result = {'det_polygons': np.array(boxes)} + return result diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/dbnet.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/dbnet.py new file mode 100644 index 0000000000000000000000000000000000000000..2745dbf4ec9857d0c4f80da3ffdca5884bc687f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/dbnet.py @@ -0,0 +1,769 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from ViLT, +# made publicly available under the Apache License 2.0 at https://github.com/dandelin/ViLT. +# ------------------------------------------------------------------------------ +import math +import os +import sys +from collections import OrderedDict + +import torch +import torch.nn as nn + +from .proxyless import CompactDetBackbone + +BatchNorm2d = nn.BatchNorm2d + + +def constant_init(module, constant, bias=0): + nn.init.constant_(module.weight, constant) + if hasattr(module, 'bias'): + nn.init.constant_(module.bias, bias) + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + + +class DwPwConv(nn.Module): + + def __init__(self, + in_planes, + out_planes, + kernel_size, + stride=1, + padding=1, + bias=False): + super(DwPwConv, self).__init__() + self.depthwise = nn.Conv2d( + in_planes, + in_planes, + kernel_size, + stride, + padding, + groups=in_planes, + bias=bias) + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + + self.pointwise = nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=1, + padding=0, + groups=1, + bias=bias) + + def forward(self, x): + out = self.depthwise(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.pointwise(out) + + return out + + +class DwPwConvTranspose(nn.Module): + + def __init__(self, in_planes, out_planes, kernel_size, stride): + super(DwPwConvTranspose, self).__init__() + self.depthwise = nn.ConvTranspose2d( + in_planes, in_planes, kernel_size, stride, groups=in_planes) + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + + self.pointwise = nn.Conv2d( + in_planes, + out_planes, + kernel_size=1, + stride=1, + padding=0, + groups=1) + + def forward(self, x): + out = self.depthwise(x) + out = self.bn1(out) + out = self.relu1(out) + + out = self.pointwise(out) + + return out + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, dcn=None): + super(BasicBlock, self).__init__() + self.with_dcn = dcn is not None + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = dcn.get('fallback_on_stride', False) + self.with_modulated_dcn = dcn.get('modulated', False) + # self.conv2 = conv3x3(planes, planes) + if not self.with_dcn or fallback_on_stride: + self.conv2 = nn.Conv2d( + planes, planes, kernel_size=3, padding=1, bias=False) + else: + deformable_groups = dcn.get('deformable_groups', 1) + if not self.with_modulated_dcn: + from assets.ops.dcn import DeformConv + conv_op = DeformConv + offset_channels = 18 + else: + from assets.ops.dcn import ModulatedDeformConv + conv_op = ModulatedDeformConv + offset_channels = 27 + self.conv2_offset = nn.Conv2d( + planes, + deformable_groups * offset_channels, + kernel_size=3, + padding=1) + self.conv2 = conv_op( + planes, + planes, + kernel_size=3, + padding=1, + deformable_groups=deformable_groups, + bias=False) + self.bn2 = BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + # out = self.conv2(out) + if not self.with_dcn: + out = self.conv2(out) + elif self.with_modulated_dcn: + offset_mask = self.conv2_offset(out) + offset = offset_mask[:, :18, :, :] + mask = offset_mask[:, -9:, :, :].sigmoid() + out = self.conv2(out, offset, mask) + else: + offset = self.conv2_offset(out) + out = self.conv2(out, offset) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, dcn=None): + super(Bottleneck, self).__init__() + self.with_dcn = dcn is not None + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = BatchNorm2d(planes) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = dcn.get('fallback_on_stride', False) + self.with_modulated_dcn = dcn.get('modulated', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + else: + deformable_groups = dcn.get('deformable_groups', 1) + if not self.with_modulated_dcn: + from assets.ops.dcn import DeformConv + conv_op = DeformConv + offset_channels = 18 + else: + from assets.ops.dcn import ModulatedDeformConv + conv_op = ModulatedDeformConv + offset_channels = 27 + self.conv2_offset = nn.Conv2d( + planes, + deformable_groups * offset_channels, + kernel_size=3, + padding=1) + self.conv2 = conv_op( + planes, + planes, + kernel_size=3, + padding=1, + stride=stride, + deformable_groups=deformable_groups, + bias=False) + self.bn2 = BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dcn = dcn + self.with_dcn = dcn is not None + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + # out = self.conv2(out) + if not self.with_dcn: + out = self.conv2(out) + elif self.with_modulated_dcn: + offset_mask = self.conv2_offset(out) + offset = offset_mask[:, :18, :, :] + mask = offset_mask[:, -9:, :, :].sigmoid() + out = self.conv2(out, offset, mask) + else: + offset = self.conv2_offset(out) + out = self.conv2(out, offset) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, + block, + layers, + num_classes=1000, + dcn=None, + stage_with_dcn=(False, False, False, False)): + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + self.inplanes = 64 + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, 128, layers[1], stride=2, dcn=dcn) + self.layer3 = self._make_layer( + block, 256, layers[2], stride=2, dcn=dcn) + self.layer4 = self._make_layer( + block, 512, layers[3], stride=2, dcn=dcn) + # self.avgpool = nn.AvgPool2d(7, stride=1) + # self.fc = nn.Linear(512 * block.expansion, num_classes) + + # self.smooth = nn.Conv2d(2048, 256, kernel_size=1, stride=1, padding=1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) or isinstance(m, BasicBlock): + if hasattr(m, 'conv2_offset'): + constant_init(m.conv2_offset, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dcn=None): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, dcn=dcn)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes, dcn=dcn)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x2 = self.layer1(x) + x3 = self.layer2(x2) + x4 = self.layer3(x3) + x5 = self.layer4(x4) + + return x2, x3, x4, x5 + + +class LightSegDetector(nn.Module): + + def __init__(self, + in_channels=[64, 128, 256, 512], + inner_channels=256, + k=10, + bias=False, + adaptive=False, + smooth=False, + serial=False, + dw_kernel_size=3, + dw_padding=1, + *args, + **kwargs): + ''' + bias: Whether conv layers have bias or not. + adaptive: Whether to use adaptive threshold training or not. + smooth: If true, use bilinear instead of deconv. + serial: If true, thresh prediction will combine segmentation result as input. + ''' + super(LightSegDetector, self).__init__() + self.k = k + self.serial = serial + self.inner_channels = inner_channels + self.bias = bias + self.dw_kernel_size = dw_kernel_size + self.dw_padding = dw_padding + + self.up5 = nn.Upsample(scale_factor=8, mode='nearest') + self.up4 = nn.Upsample(scale_factor=4, mode='nearest') + self.up3 = nn.Upsample(scale_factor=2, mode='nearest') + + self.in5 = nn.Conv2d(in_channels[-1], inner_channels, 1, bias=bias) + self.in4 = nn.Conv2d(in_channels[-2], inner_channels, 1, bias=bias) + self.in3 = nn.Conv2d(in_channels[-3], inner_channels, 1, bias=bias) + self.in2 = nn.Conv2d(in_channels[-4], inner_channels, 1, bias=bias) + + # DwPM + self.binarize = nn.Sequential( + DwPwConv( + inner_channels, + inner_channels // 4, + kernel_size=self.dw_kernel_size, + padding=self.dw_padding, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + DwPwConvTranspose(inner_channels // 4, inner_channels // 4, 2, 2), + BatchNorm2d(inner_channels // 4), nn.ReLU(inplace=True), + DwPwConvTranspose(inner_channels // 4, 1, 2, 2), nn.Sigmoid()) + + self.adaptive = adaptive + if adaptive: + self.thresh = self._init_thresh( + inner_channels, serial=serial, smooth=smooth, bias=bias) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): + nn.init.kaiming_normal_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.) + m.bias.data.fill_(1e-4) + + def _init_thresh(self, + inner_channels, + serial=False, + smooth=False, + bias=False): + in_channels = inner_channels + if serial: + in_channels += 1 + self.thresh = nn.Sequential( + nn.Conv2d( + inner_channels, + inner_channels // 4, + self.dw_kernel_size, + padding=self.dw_padding, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, + inner_channels // 4, + smooth=smooth, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, 1, smooth=smooth, bias=bias), + nn.Sigmoid()) + + return self.thresh + + def _init_upsample(self, + in_channels, + out_channels, + smooth=False, + bias=False): + if smooth: + inter_out_channels = out_channels + if out_channels == 1: + inter_out_channels = in_channels + module_list = [ + nn.Upsample(scale_factor=2, mode='nearest'), + nn.Conv2d(in_channels, inter_out_channels, 3, 1, 1, bias=bias) + ] + if out_channels == 1: + module_list.append( + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=1, + bias=True)) + + return nn.Sequential(module_list) + else: + return nn.ConvTranspose2d(in_channels, out_channels, 2, 2) + + def forward(self, features, gt=None, masks=None, training=False): + c2, c3, c4, c5 = features + p5 = self.up5(self.in5(c5)) + p4 = self.up4(self.in4(c4)) + p3 = self.up3(self.in3(c3)) + p2 = self.in2(c2) + + fuse = p5 + p4 + p3 + p2 + + # this is the pred module, not binarization module; + # We do not correct the name due to the trained model. + binary = self.binarize(fuse) + if self.training: + result = OrderedDict(binary=binary) + else: + return binary + if self.adaptive and self.training: + if self.serial: + fuse = torch.cat( + (fuse, nn.functional.interpolate(binary, fuse.shape[2:])), + 1) + thresh = self.thresh(fuse) + thresh_binary = self.step_function(binary, thresh) + result.update(thresh=thresh, thresh_binary=thresh_binary) + return result + + def step_function(self, x, y): + return torch.reciprocal(1 + torch.exp(-self.k * (x - y))) + + +class SegDetector(nn.Module): + + def __init__(self, + in_channels=[64, 128, 256, 512], + inner_channels=256, + k=10, + bias=False, + adaptive=False, + smooth=False, + serial=False, + *args, + **kwargs): + ''' + bias: Whether conv layers have bias or not. + adaptive: Whether to use adaptive threshold training or not. + smooth: If true, use bilinear instead of deconv. + serial: If true, thresh prediction will combine segmentation result as input. + ''' + super(SegDetector, self).__init__() + self.k = k + self.serial = serial + self.up5 = nn.Upsample(scale_factor=2, mode='nearest') + self.up4 = nn.Upsample(scale_factor=2, mode='nearest') + self.up3 = nn.Upsample(scale_factor=2, mode='nearest') + + self.in5 = nn.Conv2d(in_channels[-1], inner_channels, 1, bias=bias) + self.in4 = nn.Conv2d(in_channels[-2], inner_channels, 1, bias=bias) + self.in3 = nn.Conv2d(in_channels[-3], inner_channels, 1, bias=bias) + self.in2 = nn.Conv2d(in_channels[-4], inner_channels, 1, bias=bias) + + self.out5 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=bias), + nn.Upsample(scale_factor=8, mode='nearest')) + self.out4 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=bias), + nn.Upsample(scale_factor=4, mode='nearest')) + self.out3 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=bias), + nn.Upsample(scale_factor=2, mode='nearest')) + self.out2 = nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=bias) + + self.binarize = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=bias), + BatchNorm2d(inner_channels // 4), nn.ReLU(inplace=True), + nn.ConvTranspose2d(inner_channels // 4, inner_channels // 4, 2, 2), + BatchNorm2d(inner_channels // 4), nn.ReLU(inplace=True), + nn.ConvTranspose2d(inner_channels // 4, 1, 2, 2), nn.Sigmoid()) + self.binarize.apply(self.weights_init) + + self.adaptive = adaptive + if adaptive: + self.thresh = self._init_thresh( + inner_channels, serial=serial, smooth=smooth, bias=bias) + self.thresh.apply(self.weights_init) + + self.in5.apply(self.weights_init) + self.in4.apply(self.weights_init) + self.in3.apply(self.weights_init) + self.in2.apply(self.weights_init) + self.out5.apply(self.weights_init) + self.out4.apply(self.weights_init) + self.out3.apply(self.weights_init) + self.out2.apply(self.weights_init) + + def weights_init(self, m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + nn.init.kaiming_normal_(m.weight.data) + elif classname.find('BatchNorm') != -1: + m.weight.data.fill_(1.) + m.bias.data.fill_(1e-4) + + def _init_thresh(self, + inner_channels, + serial=False, + smooth=False, + bias=False): + in_channels = inner_channels + if serial: + in_channels += 1 + self.thresh = nn.Sequential( + nn.Conv2d( + in_channels, inner_channels // 4, 3, padding=1, bias=bias), + BatchNorm2d(inner_channels // 4), nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, + inner_channels // 4, + smooth=smooth, + bias=bias), BatchNorm2d(inner_channels // 4), + nn.ReLU(inplace=True), + self._init_upsample( + inner_channels // 4, 1, smooth=smooth, bias=bias), + nn.Sigmoid()) + return self.thresh + + def _init_upsample(self, + in_channels, + out_channels, + smooth=False, + bias=False): + if smooth: + inter_out_channels = out_channels + if out_channels == 1: + inter_out_channels = in_channels + module_list = [ + nn.Upsample(scale_factor=2, mode='nearest'), + nn.Conv2d(in_channels, inter_out_channels, 3, 1, 1, bias=bias) + ] + if out_channels == 1: + module_list.append( + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=1, + bias=True)) + + return nn.Sequential(module_list) + else: + return nn.ConvTranspose2d(in_channels, out_channels, 2, 2) + + def forward(self, features, gt=None, masks=None, training=False): + c2, c3, c4, c5 = features + in5 = self.in5(c5) + in4 = self.in4(c4) + in3 = self.in3(c3) + in2 = self.in2(c2) + + out4 = self.up5(in5) + in4 # 1/16 + out3 = self.up4(out4) + in3 # 1/8 + out2 = self.up3(out3) + in2 # 1/4 + + p5 = self.out5(in5) + p4 = self.out4(out4) + p3 = self.out3(out3) + p2 = self.out2(out2) + + fuse = torch.cat((p5, p4, p3, p2), 1) + # this is the pred module, not binarization module; + # We do not correct the name due to the trained model. + binary = self.binarize(fuse) + if self.training: + result = OrderedDict(binary=binary) + else: + return binary + if self.adaptive and self.training: + if self.serial: + fuse = torch.cat( + (fuse, nn.functional.interpolate(binary, fuse.shape[2:])), + 1) + thresh = self.thresh(fuse) + thresh_binary = self.step_function(binary, thresh) + result.update(thresh=thresh, thresh_binary=thresh_binary) + return result + + def step_function(self, x, y): + return torch.reciprocal(1 + torch.exp(-self.k * (x - y))) + + +class BasicModel(nn.Module): + + def __init__(self, *args, **kwargs): + nn.Module.__init__(self) + + self.backbone = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + self.decoder = SegDetector( + in_channels=[64, 128, 256, 512], adaptive=True, k=50, **kwargs) + + def forward(self, data, *args, **kwargs): + return self.decoder(self.backbone(data), *args, **kwargs) + + +def parallelize(model, distributed, local_rank): + if distributed: + return nn.parallel.DistributedDataParallel( + model, + device_ids=[local_rank], + output_device=[local_rank], + find_unused_parameters=True) + else: + return nn.DataParallel(model) + + +class VLPTModel(nn.Module): + + def __init__(self, *args, **kwargs): + """ + VLPT-STD pretrained DBNet-resnet50 model, + paper reference: https://arxiv.org/pdf/2204.13867.pdf + """ + super(VLPTModel, self).__init__() + self.backbone = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) + self.decoder = SegDetector( + in_channels=[256, 512, 1024, 2048], adaptive=True, k=50, **kwargs) + + def forward(self, x): + return self.decoder(self.backbone(x)) + + +class DBNasModel(nn.Module): + + def __init__(self, *args, **kwargs): + """ + DB-NAS model + """ + super(DBNasModel, self).__init__() + self.backbone = CompactDetBackbone( + width_stages=[32, 64, 96, 128], input_channel=32, **kwargs) + self.decoder = LightSegDetector( + in_channels=[32, 64, 96, 128], + adaptive=True, + k=50, + inner_channels=64, + dw_kernel_size=5, + dw_padding=2, + **kwargs) + + def forward(self, x): + return self.decoder(self.backbone(x)) + + +class DBModel(nn.Module): + + def __init__(self, *args, **kwargs): + """ + DBNet-resnet18 model without deformable conv, + paper reference: https://arxiv.org/pdf/1911.08947.pdf + """ + super(DBModel, self).__init__() + self.backbone = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + self.decoder = SegDetector( + in_channels=[64, 128, 256, 512], adaptive=True, k=50, **kwargs) + + def forward(self, x): + return self.decoder(self.backbone(x)) + + +class DBModel_v2(nn.Module): + + def __init__(self, + device, + distributed: bool = False, + local_rank: int = 0, + *args, + **kwargs): + """ + DBNet-resnet18 model without deformable conv, + paper reference: https://arxiv.org/pdf/1911.08947.pdf + """ + super(DBModel_v2, self).__init__() + from .seg_detector_loss import L1BalanceCELoss + + self.model = BasicModel(*args, **kwargs) + self.model = parallelize(self.model, distributed, local_rank) + self.criterion = L1BalanceCELoss() + self.criterion = parallelize(self.criterion, distributed, local_rank) + self.device = device + self.to(self.device) + + def forward(self, batch, training=False): + if isinstance(batch, dict): + data = batch['image'].to(self.device) + else: + data = batch.to(self.device) + data = data.float() + pred = self.model(data, training=self.training) + + if self.training: + for key, value in batch.items(): + if value is not None: + if hasattr(value, 'to'): + batch[key] = value.to(self.device) + loss_with_metrics = self.criterion(pred, batch) + loss, metrics = loss_with_metrics + return loss, pred, metrics + return pred diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/layers.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..515c0bc6afa3378d614f0ba437daa16376613eb5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/layers.py @@ -0,0 +1,906 @@ +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def get_same_padding(kernel_size): + if isinstance(kernel_size, tuple): + assert len(kernel_size) == 2, 'invalid kernel size: %s' % kernel_size + p1 = get_same_padding(kernel_size[0]) + p2 = get_same_padding(kernel_size[1]) + return p1, p2 + assert isinstance(kernel_size, + int), 'kernel size should be either `int` or `tuple`' + assert kernel_size % 2 > 0, 'kernel size should be odd number' + return kernel_size // 2 + + +def count_conv_flop(layer, x): + out_h = int(x.size(2) / layer.stride[0]) + out_w = int(x.size(3) / layer.stride[1]) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[ + 0] * layer.kernel_size[1] * out_h * out_w / layer.groups + return delta_ops + + +def set_layer_from_config(layer_config): + if layer_config is None: + return None + name2layer = { + ZeroLayer.__name__: ZeroLayer, + MBInvertedConvLayer.__name__: MBInvertedConvLayer, + IdentityLayer.__name__: IdentityLayer, + } + layer_name = layer_config.pop('name') + layer = name2layer[layer_name] + return layer.build_from_config(layer_config) + + +class MobileInvertedResidualBlock(nn.Module): + + def __init__(self, mobile_inverted_conv, shortcut): + super(MobileInvertedResidualBlock, self).__init__() + self.mobile_inverted_conv = mobile_inverted_conv + self.shortcut = shortcut + + def forward(self, x): + if self.mobile_inverted_conv.is_zero_layer(): + res = x + elif self.shortcut is None or self.shortcut.is_zero_layer(): + res = self.mobile_inverted_conv(x) + else: + conv_x = self.mobile_inverted_conv(x) + skip_x = self.shortcut(x) + res = skip_x + conv_x + return res + + @property + def module_str(self): + return '(%s, %s)' % (self.mobile_inverted_conv.module_str, + self.shortcut.module_str + if self.shortcut is not None else None) + + @property + def config(self): + return { + 'name': + MobileInvertedResidualBlock.__name__, + 'mobile_inverted_conv': + self.mobile_inverted_conv.config, + 'shortcut': + self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + mobile_inverted_conv = set_layer_from_config( + config['mobile_inverted_conv']) + shortcut = set_layer_from_config(config['shortcut']) + return MobileInvertedResidualBlock(mobile_inverted_conv, shortcut) + + def get_flops(self, x): + flops1, _ = self.mobile_inverted_conv.get_flops(x) + if self.shortcut: + flops2, _ = self.shortcut.get_flops(x) + else: + flops2 = 0 + return flops1 + flops2, self.forward(x) + + +class MBInvertedConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + pad = get_same_padding(self.kernel_size) + self.depth_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%dx%d_MBConv%d' % (self.kernel_size, self.kernel_size, + self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += count_conv_flop(self.depth_conv.conv, x) + x = self.depth_conv(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class IdentityLayer(nn.Module): + + def __init__(self): + super(IdentityLayer, self).__init__() + + def forward(self, x): + return x + + @property + def module_str(self): + return 'Identity' + + @property + def config(self): + return { + 'name': IdentityLayer.__name__, + } + + @staticmethod + def build_from_config(config): + return IdentityLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + return 0, self.forward(x) + + +class ZeroLayer(nn.Module): + + def __init__(self, stride): + super(ZeroLayer, self).__init__() + self.stride = stride + + def forward(self, x): + n, c, h, w = x.shape + h //= self.stride[0] + w //= self.stride[1] + device = x.device + padding = torch.zeros(n, c, h, w, device=device, requires_grad=False) + return padding + + @property + def module_str(self): + return 'Zero' + + @property + def config(self): + return {'name': ZeroLayer.__name__, 'stride': self.stride} + + @staticmethod + def build_from_config(config): + return ZeroLayer(**config) + + @staticmethod + def is_zero_layer(): + return True + + def get_flops(self, x): + return 0, self.forward(x) + + +def split_layer(total_channels, num_groups): + split = [ + int(np.ceil(total_channels / num_groups)) for _ in range(num_groups) + ] + split[num_groups - 1] += total_channels - sum(split) + return split + + +class MBInvertedMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedMixConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.split_in_channels = split_layer(feature_dim, self.n_chunks) + + self.mix_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + split_in_channels_ = self.split_in_channels[idx] + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ('act', nn.PReLU()), + ]))) + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + x = torch.cat([layer(s) for layer, s in zip(self.mix_conv, split)], + dim=1) + + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_MixConv%d' % (str(self.mix_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + out = [] + for layer, s in zip(self.mix_conv, split): + out.append(layer(s)) + total_flops += count_conv_flop(layer.conv, s) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class LinearMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + mid_channels=None): + super(LinearMixConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.mid_channels = mid_channels + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.mix_conv = nn.ModuleList() + + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ('act', nn.ReLU6(inplace=True)), + ]))) + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels * self.n_chunks, + out_channels, + 1, + 1, + 0, + bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = torch.cat([layer(x) for layer in self.mix_conv], dim=1) + + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_LinearMixConv' % (str(self.mix_conv_size)) + + @property + def config(self): + return { + 'name': LinearMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return LinearMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + out = [] + for layer in self.mix_conv: + out.append(layer(x)) + total_flops += count_conv_flop(layer.conv, x) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class SELayer(nn.Module): + ''' + ''' + + def __init__(self, input_channels: int, squeeze_factor: int = 4): + super(SELayer, self).__init__() + self.input_channels = input_channels + self.squeeze_factor = squeeze_factor + self.squeeze_channels = input_channels // squeeze_factor + self.fc1 = nn.Conv2d(self.input_channels, self.squeeze_channels, 1) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Conv2d(self.squeeze_channels, self.input_channels, 1) + + def _scale(self, input): + scale = F.adaptive_avg_pool2d(input, 1) + scale = self.fc1(scale) + scale = self.relu(scale) + scale = self.fc2(scale) + return torch.sigmoid(scale) + + def forward(self, input): + scale = self._scale(input) + return scale * input + + @property + def module_str(self): + return 'SE_%d' % (self.squeeze_factor) + + @property + def config(self): + return { + 'name': SELayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'squeeze_factor': self.squeeze_factor, + } + + @staticmethod + def build_from_config(config): + return SELayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + total_flops = 0 + total_flops += self.input_channels * self.squeeze_channels * 2 + b, c, h, w = x.shape + total_flops += c * h * w + return total_flops, x + + +class MHSA(nn.Module): + + def __init__(self, n_dims, width=14, height=14, heads=4): + super(MHSA, self).__init__() + self.heads = heads + + self.query = nn.Conv2d(n_dims, n_dims, kernel_size=1) + self.key = nn.Conv2d(n_dims, n_dims, kernel_size=1) + self.value = nn.Conv2d(n_dims, n_dims, kernel_size=1) + + self.width = width + self.height = height + + self.rel_h = nn.Parameter( + torch.randn([1, heads, n_dims // heads, 1, height]), + requires_grad=True) + self.rel_w = nn.Parameter( + torch.randn([1, heads, n_dims // heads, width, 1]), + requires_grad=True) + + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x): + + n_batch, C, width, height = x.size() + q = self.query(x).view(n_batch, self.heads, C // self.heads, -1) + k = self.key(x).view(n_batch, self.heads, C // self.heads, -1) + v = self.value(x).view(n_batch, self.heads, C // self.heads, -1) + + content_content = torch.matmul(q.permute(0, 1, 3, 2), k) + + content_position = (self.rel_h + self.rel_w).view( + 1, self.heads, C // self.heads, -1).permute(0, 1, 3, 2) + content_position = torch.matmul(content_position, q) + energy = content_content + content_position + attention = self.softmax(energy) + + out = torch.matmul(v, attention.permute(0, 1, 3, 2)) + out = out.view(n_batch, C, width, height) + + return out + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + n_batch, C, width, height = x.size() + + total_flops = 0 + total_flops += count_conv_flop(self.query, x) * 3 + + # content_content + total_flops += (width * height) * C * (width * height) + + # content_position + total_flops += (width * height) * C + total_flops += (width * height) * C * (width * height) + + # attention + total_flops += (width * height) * C * (width * height) + + return total_flops, x + + +class MBInvertedMHSALayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + expand_ratio=6, + width=1, + height=175, + mid_channels=None): + super(MBInvertedMHSALayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + # ('act', nn.PReLU()), + ('act', nn.ReLU6(inplace=True)), + ])) + self.mhsa = MHSA(feature_dim, width, height) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.mhsa(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return 'MSHA%d' % (self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMHSALayer.__name__, + 'in_channels': self.in_channels, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMHSALayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += self.mhsa.get_flops(x)[0] + x = self.mhsa(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class MBInvertedRepConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + rep_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedRepConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.rep_conv_size = rep_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.rep_conv_size = rep_conv_size + self.n_chunks = len(rep_conv_size) + + self.rep_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.rep_conv_size[idx] + pad = get_same_padding(kernel_size) + self.rep_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ]))) + + self.rep_conv_deploy = None + self.act = nn.PReLU() + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + self.deploy = False + + def forward(self, x): + x = self.inverted_bottleneck(x) + if not self.deploy: + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + else: + x = self.rep_conv_deploy(x) + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_RepConv%d' % (str(self.rep_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'rep_conv_size': self.rep_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def switch_to_deploy(self): + self.deploy = True + feature_dim = self.rep_conv[0].conv.in_channels + stride = self.rep_conv[0].conv.stride + kernel_size = max(self.rep_conv_size) + pad = get_same_padding(kernel_size) + self.rep_conv_deploy = nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=True) + + kernel, bias = self.get_equivalent_kernel_bias() + + self.rep_conv_deploy.weight.data = kernel + self.rep_conv_deploy.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rep_conv') + + def get_equivalent_kernel_bias(self): + max_kernel_size = max(self.rep_conv_size) + + if max_kernel_size == 5: + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + if 5 in self.rep_conv_size: + kernel5x5, bias5x5 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(5)]) + + else: + kernel5x5 = 0 + bias5x5 = 0 + + return kernel5x5 + self._pad_1x1_to_5x5_tensor( + kernel1x1) + self._pad_3x3_to_5x5_tensor( + kernel3x3), bias5x5 + bias3x3 + bias1x1 + else: + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + return kernel3x3 + self._pad_1x1_to_3x3_tensor( + kernel1x1), bias3x3 + bias1x1 + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _pad_1x1_to_5x5_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [2, 2, 2, 2]) + + def _pad_3x3_to_5x5_tensor(self, kernel3x3): + if kernel3x3 is None: + return 0 + else: + return torch.nn.functional.pad(kernel3x3, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + + return kernel * t, beta - running_mean * gamma / std + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + + total_flops += count_conv_flop(self.rep_conv[-1].conv, x) + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/mix_ops.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/mix_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..5c6dd4a51f473e55c003610329044ad869744c58 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/mix_ops.py @@ -0,0 +1,689 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from ProxylessNAS, +# made publicly available under the Apache License 2.0 at https://github.com/mit-han-lab/proxylessnas. +# ------------------------------------------------------------------------------ + +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from .layers import (IdentityLayer, LinearMixConvLayer, MBInvertedConvLayer, + MBInvertedMHSALayer, MBInvertedMixConvLayer, + MBInvertedRepConvLayer, SELayer, ZeroLayer) + + +def detach_variable(inputs): + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + + +def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + +def conv_func_by_name(name): + name2ops = { + 'Identity': + lambda in_C, out_C, S: IdentityLayer(in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S: ZeroLayer(stride=S), + } + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3], S), + '35_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [3, 5], S), + '135_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3, 5], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S: SELayer(in_C, 8), + ####################################################################################### + 'MHSA1': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 1, width, height), + 'MHSA2': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 2, width, height), + 'MHSA3': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 3, width, height), + 'MHSA4': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 4, width, height), + 'MHSA5': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 5, width, height), + 'MHSA6': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 6, width, height), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + }) + return name2ops[name] + + +def build_candidate_ops(candidate_ops, + in_channels, + out_channels, + stride, + ops_order, + spatio_size=None): + if candidate_ops is None: + raise ValueError('please specify a candidate set') + + name2ops = { + 'Identity': + lambda in_C, out_C, S: IdentityLayer(in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S: ZeroLayer(stride=S), + } + # add MBConv layers + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 3, S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 5, S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S: MBInvertedConvLayer(in_C, out_C, 7, S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_MixConv2': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_MixConv3': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_MixConv4': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_MixConv5': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_MixConv6': + lambda in_C, out_C, S: MBInvertedMixConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3], S), + '35_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [3, 5], S), + '135_LinMixConv': + lambda in_C, out_C, S: LinearMixConvLayer(in_C, out_C, [1, 3, 5], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S: SELayer(in_C, 8), + ####################################################################################### + 'MHSA1': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 1, width, height), + 'MHSA2': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 2, width, height), + 'MHSA3': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 3, width, height), + 'MHSA4': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 4, width, height), + 'MHSA5': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 5, width, height), + 'MHSA6': + lambda in_C, out_C, width, height: MBInvertedMHSALayer( + in_C, out_C, 6, width, height), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 1 + ), + '13_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 2 + ), + '13_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 3 + ), + '13_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 4 + ), + '13_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 5 + ), + '13_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3], S, 6 + ), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 1 + ), + '35_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 2 + ), + '35_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 3 + ), + '35_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 4 + ), + '35_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 5 + ), + '35_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [3, 5], S, 6 + ), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 1), + '135_RepConv2': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 2), + '135_RepConv3': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 3), + '135_RepConv4': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 4), + '135_RepConv5': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 5), + '135_RepConv6': + lambda in_C, out_C, S: MBInvertedRepConvLayer(in_C, out_C, [1, 3, 5], + S, 6), + }) + + out = [] + for name in candidate_ops: + + if 'MHSA' in name: + out.append(name2ops[name](in_channels, out_channels, + spatio_size[0], spatio_size[1])) + else: + out.append(name2ops[name](in_channels, out_channels, stride)) + return out + + +class MixedEdge(nn.Module): + MODE = None # full, two, None, full_v2 + + def __init__(self, candidate_ops): + super(MixedEdge, self).__init__() + + self.candidate_ops = nn.ModuleList(candidate_ops) + self.AP_path_alpha = Parameter(torch.Tensor( + self.n_choices)) # architecture parameters + self.AP_path_wb = Parameter(torch.Tensor( + self.n_choices)) # binary gates + + self.active_index = [0] + self.inactive_index = None + + self.log_prob = None + self.current_prob_over_ops = None + + @property + def n_choices(self): + return len(self.candidate_ops) + + @property + def probs_over_ops(self): + probs = F.softmax(self.AP_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + @property + def chosen_op(self): + index, _ = self.chosen_index + return self.candidate_ops[index] + + @property + def random_op(self): + index = np.random.choice([_i for _i in range(self.n_choices)], 1)[0] + return self.candidate_ops[index] + + def entropy(self, eps=1e-8): + probs = self.probs_over_ops + log_probs = torch.log(probs + eps) + entropy = -torch.sum(torch.mul(probs, log_probs)) + return entropy + + def is_zero_layer(self): + return self.active_op.is_zero_layer() + + @property + def active_op(self): + """ assume only one path is active """ + return self.candidate_ops[self.active_index[0]] + + def set_chosen_op_active(self): + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def forward(self, x): + if MixedEdge.MODE == 'full' or MixedEdge.MODE == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.AP_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.AP_path_wb[_i] * oi.detach() + elif MixedEdge.MODE == 'full_v2': + + def run_function(candidate_ops, active_id): + + def forward(_x): + return candidate_ops[active_id](_x) + + return forward + + def backward_function(candidate_ops, active_id, binary_gates): + + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + + return backward + + output = ArchGradientFunction.apply( + x, self.AP_path_wb, + run_function(self.candidate_ops, self.active_index[0]), + backward_function(self.candidate_ops, self.active_index[0], + self.AP_path_wb)) + else: + output = self.active_op(x) + return output + + @property + def module_str(self): + chosen_index, probs = self.chosen_index + return 'Mix(%s, %.3f)' % (self.candidate_ops[chosen_index].module_str, + probs) + + @property + def config(self): + raise ValueError('not needed') + + @staticmethod + def build_from_config(config): + raise ValueError('not needed') + + def get_flops(self, x): + """ Only active paths taken into consideration when calculating FLOPs """ + flops = 0 + for i in self.active_index: + delta_flop, _ = self.candidate_ops[i].get_flops(x) + flops += delta_flop + return flops, self.forward(x) + + def binarize(self): + """ prepare: active_index, inactive_index, AP_path_wb, log_prob (optional), current_prob_over_ops (optional) """ + self.log_prob = None + # reset binary gates + self.AP_path_wb.data.zero_() + # binarize according to probs + probs = self.probs_over_ops + if MixedEdge.MODE == 'two': + # sample two ops according to `probs` + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax( + torch.stack([self.AP_path_alpha[idx] for idx in sample_op]), + dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # chose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1 - c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.AP_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs.data, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, self.n_choices)] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + # set binary gate + self.AP_path_wb.data[sample] = 1.0 + # avoid over-regularization + for _i in range(self.n_choices): + for name, param in self.candidate_ops[_i].named_parameters(): + param.grad = None + + def set_arch_param_grad(self): + binary_grads = self.AP_path_wb.grad.data + if self.active_op.is_zero_layer(): + self.AP_path_alpha.grad = None + return + if self.AP_path_alpha.grad is None: + self.AP_path_alpha.grad = torch.zeros_like(self.AP_path_alpha.data) + if MixedEdge.MODE == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax( + torch.stack([self.AP_path_alpha[idx] for idx in involved_idx]), + dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.AP_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, + self.AP_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, + self.AP_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.AP_path_alpha.grad.data[ + i] += binary_grads[j] * probs[j] * ( + delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [ + idx for idx, _ in (self.active_index + self.inactive_index) + ] + old_alphas = [ + alpha for _, alpha in (self.active_index + self.inactive_index) + ] + new_alphas = [self.AP_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) + / sum([math.exp(alpha) for alpha in old_alphas])) + + for idx in involved_idx: + self.AP_path_alpha.data[idx] -= offset + + +class ArchGradientFunction(torch.autograd.Function): + + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad( + output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, + grad_output.data) + + return grad_x[0], binary_grads, None, None diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/proxyless.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/proxyless.py new file mode 100644 index 0000000000000000000000000000000000000000..68458bc3ce1869c0e25b43285db5372dfcde5052 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/proxyless.py @@ -0,0 +1,178 @@ +import re +import sys + +import numpy as np +import torch +import torch.nn as nn + +from .layers import (IdentityLayer, MBInvertedConvLayer, + MobileInvertedResidualBlock, ZeroLayer) +from .mix_ops import MixedEdge, build_candidate_ops, conv_func_by_name + + +class NasRecBackbone(nn.Module): + + def __init__(self, first_conv, blocks): + super(NasRecBackbone, self).__init__() + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + self.output_idx = [5, 11, 17, 23] + + def forward(self, x): + x = self.first_conv(x) + idx = 0 + out = [] + for block in self.blocks: + x = block(x) + if (idx + 1) % (int(len(self.blocks) / 4)) == 0: + out.append(x) + idx += 1 + return out[0], out[1], out[2], out[3] + + def get_bn_param(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + return { + 'momentum': m.momentum, + 'eps': m.eps, + } + return None + + @property + def config(self): + return { + 'name': NasRecBackbone.__name__, + 'bn': self.get_bn_param(), + 'first_conv': 'conv_in3_out32_k3_s2_p1', + 'blocks': [block.config for block in self.blocks] + } + + def set_bn_param(self, momentum, eps): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + m.momentum = momentum + m.eps = eps + return + + @staticmethod + def build_from_config(config): + first_conv_config = config['first_conv'] + match_obj = re.match(r'conv_in(\d+)_out(\d+)_k(\d+)_s(\d+)_p(\d+)', + first_conv_config) + in_channel = int(match_obj.group(1)) + out_channel = int(match_obj.group(2)) + kernel_size = int(match_obj.group(3)) + stride = int(match_obj.group(4)) + padding = int(match_obj.group(5)) + first_conv = nn.Sequential( + nn.Conv2d( + in_channel, + out_channel, + kernel_size, + stride, + padding, + bias=False), + nn.BatchNorm2d(out_channel), + nn.ReLU(inplace=True), + ) + blocks = [] + for block_config in config['blocks']: + blocks.append( + MobileInvertedResidualBlock.build_from_config(block_config)) + net = NasRecBackbone(first_conv, blocks) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-3) + return net + + +class CompactDetBackbone(NasRecBackbone): + ''' + proxyless nas backbone, 5M. + ''' + + def __init__(self, + width_stages, + input_channel=None, + bn_param=(0.1, 1e-3), + **kwargs): + + if input_channel is None: + input_channel = width_stages[0] + first_conv = nn.Sequential( + nn.Conv2d( + 3, + input_channel, + kernel_size=(3, 3), + stride=2, + padding=1, + bias=False), nn.BatchNorm2d(input_channel), + nn.ReLU(inplace=True)) + + conv_candidates = [ + '5x5_MBConv2', '5x5_MBConv4', '3x3_MBConv2', '3x3_MBConv4', + '13_MixConv2', '13_MixConv4', '35_MixConv2', '35_MixConv4', + '135_MixConv2', '135_MixConv4', '13_LinMixConv', '35_LinMixConv', + '135_LinMixConv', '13_RepConv2', '13_RepConv4', '35_RepConv2', + '35_RepConv4', '135_RepConv2', '135_RepConv4', 'Zero' + ] + + se_candidates = ['SE_2', 'SE_4', 'SE_8', 'Zero'] + + conv_op_ids = [ + 15, 17, 17, 17, 17, 0, 16, 16, 18, 18, 16, 2, 16, 18, 16, 18, 18, + 2, 1, 18, 18, 18, 16, 2 + ] + + n_cell_stages = [5, 5, 5, 5] + + stride_stages = [(2, 2), (2, 2), (2, 2), (2, 2)] + + if se_candidates: + assert len(conv_op_ids) == sum(n_cell_stages) + 4 + else: + assert len(conv_op_ids) == sum(n_cell_stages) + + blocks = [] + + for width, n_cell, s in zip(width_stages, n_cell_stages, + stride_stages): + for i in range(n_cell): + if i == 0: + stride = s + else: + stride = (1, 1) + block_i = len(blocks) + conv_op = conv_func_by_name( + conv_candidates[conv_op_ids[block_i]])(input_channel, + width, stride) + + if stride == (1, 1) and input_channel == width: + shortcut = IdentityLayer() + else: + shortcut = None + inverted_residual_block = MobileInvertedResidualBlock( + conv_op, shortcut) + blocks.append(inverted_residual_block) + input_channel = width + + if se_candidates is not None: + block_i = len(blocks) + se_op = conv_func_by_name(se_candidates[conv_op_ids[block_i]])( + input_channel, width, stride) + shortcut = IdentityLayer() + inverted_residual_block = MobileInvertedResidualBlock( + se_op, shortcut) + blocks.append(inverted_residual_block) + + self.out_channel = input_channel + + super(CompactDetBackbone, self).__init__(first_conv, blocks) + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/modules/seg_detector_loss.py b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/seg_detector_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..38446e3525345a64c28e09d12deadf042e64ce42 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/modules/seg_detector_loss.py @@ -0,0 +1,257 @@ +# ------------------------------------------------------------------------------ +# The implementation is adopted from DBNet, +# made publicly available under the Apache License 2.0 at https://github.com/MhLiao/DB. +# ------------------------------------------------------------------------------ +import sys + +import torch +import torch.nn as nn + + +class SegDetectorLossBuilder(): + ''' + Build loss functions for SegDetector. + Details about the built functions: + Input: + pred: A dict which contains predictions. + thresh: The threshold prediction + binary: The text segmentation prediction. + thresh_binary: Value produced by `step_function(binary - thresh)`. + batch: + gt: Text regions bitmap gt. + mask: Ignore mask, + pexels where value is 1 indicates no contribution to loss. + thresh_mask: Mask indicates regions cared by thresh supervision. + thresh_map: Threshold gt. + Return: + (loss, metrics). + loss: A scalar loss value. + metrics: A dict contraining partial loss values. + ''' + + def __init__(self, loss_class, *args, **kwargs): + self.loss_class = loss_class + self.loss_args = args + self.loss_kwargs = kwargs + + def build(self): + return getattr(sys.modules[__name__], + self.loss_class)(*self.loss_args, **self.loss_kwargs) + + +def _neg_loss(pred, gt): + ''' Modified focal loss. Exactly the same as CornerNet. + Runs faster and costs a little bit more memory + Arguments: + pred (batch x c x h x w) + gt_regr (batch x c x h x w) + ''' + pos_inds = gt.eq(1).float() + neg_inds = gt.lt(1).float() + + neg_weights = torch.pow(1 - gt, 4) + + loss = 0 + + pos_loss = torch.log(pred) * torch.pow(1 - pred, 2) * pos_inds + neg_loss = torch.log(1 - pred) * torch.pow(pred, + 2) * neg_weights * neg_inds + + num_pos = pos_inds.float().sum() + pos_loss = pos_loss.sum() + neg_loss = neg_loss.sum() + + if num_pos == 0: + loss = loss - neg_loss + else: + loss = loss - (pos_loss + neg_loss) / num_pos + + b = pred.shape[0] + loss = loss / b + if loss > 10: + print('Loss', loss) + loss /= 1000 + print('HM Loss > 10\n') + else: + loss + + return loss + + +class FocalLoss(nn.Module): + '''nn.Module warpper for focal loss''' + + def __init__(self): + super(FocalLoss, self).__init__() + self.neg_loss = _neg_loss + + def forward(self, out, target): + return self.neg_loss(out, target) + + +class DiceLoss(nn.Module): + ''' + Loss function from https://arxiv.org/abs/1707.03237, + where iou computation is introduced heatmap manner to measure the + diversity bwtween tow heatmaps. + ''' + + def __init__(self, eps=1e-6): + super(DiceLoss, self).__init__() + self.eps = eps + + def forward(self, pred: torch.Tensor, gt, mask, weights=None): + ''' + pred: one or two heatmaps of shape (N, 1, H, W), + the losses of tow heatmaps are added together. + gt: (N, 1, H, W) + mask: (N, H, W) + ''' + assert pred.dim() == 4, pred.dim() + return self._compute(pred, gt, mask, weights) + + def _compute(self, pred, gt, mask, weights): + if pred.dim() == 4: + pred = pred[:, 0, :, :] + gt = gt[:, 0, :, :] + assert pred.shape == gt.shape + assert pred.shape == mask.shape + if weights is not None: + assert weights.shape == mask.shape + mask = weights * mask + + intersection = (pred * gt * mask).sum() + union = (pred * mask).sum() + (gt * mask).sum() + self.eps + loss = 1 - 2.0 * intersection / union + assert loss <= 1 + return loss + + +class MaskL1Loss(nn.Module): + + def __init__(self): + super(MaskL1Loss, self).__init__() + + def forward(self, pred: torch.Tensor, gt, mask): + mask_sum = mask.sum() + if mask_sum.item() == 0: + return mask_sum, dict(l1_loss=mask_sum) + else: + loss = (torch.abs(pred[:, 0] - gt) * mask).sum() / mask_sum + return loss, dict(l1_loss=loss) + + +class MaskL2Loss(nn.Module): + + def __init__(self): + super(MaskL2Loss, self).__init__() + + def forward(self, pred: torch.Tensor, gt, mask): + mask_sum = mask.sum() + if mask_sum.item() == 0: + return mask_sum, dict(l1_loss=mask_sum) + else: + loss = (((pred[:, 0] - gt)**2) * mask).sum() / mask_sum + return loss, dict(l1_loss=loss) + + +class BalanceCrossEntropyLoss(nn.Module): + ''' + Balanced cross entropy loss. + Shape: + - Input: :math:`(N, 1, H, W)` + - GT: :math:`(N, 1, H, W)`, same shape as the input + - Mask: :math:`(N, H, W)`, same spatial shape as the input + - Output: scalar. + + Examples:: + + >>> m = nn.Sigmoid() + >>> loss = nn.BCELoss() + >>> input = torch.randn(3, requires_grad=True) + >>> target = torch.empty(3).random_(2) + >>> output = loss(m(input), target) + >>> output.backward() + ''' + + def __init__(self, negative_ratio=3.0, eps=1e-6): + super(BalanceCrossEntropyLoss, self).__init__() + self.negative_ratio = negative_ratio + self.eps = eps + + def forward(self, + pred: torch.Tensor, + gt: torch.Tensor, + mask: torch.Tensor, + return_origin=False): + ''' + Args: + pred: shape :math:`(N, 1, H, W)`, the prediction of network + gt: shape :math:`(N, 1, H, W)`, the target + mask: shape :math:`(N, H, W)`, the mask indicates positive regions + ''' + positive = (gt * mask).byte() + negative = ((1 - gt) * mask).byte() + positive_count = int(positive.float().sum()) + negative_count = min( + int(negative.float().sum()), + int(positive_count * self.negative_ratio)) + loss = nn.functional.binary_cross_entropy( + pred, gt, reduction='none')[:, 0, :, :] + positive_loss = loss * positive.float() + negative_loss = loss * negative.float() + negative_loss, _ = torch.topk(negative_loss.view(-1), negative_count) + + balance_loss = (positive_loss.sum() + negative_loss.sum()) /\ + (positive_count + negative_count + self.eps) + + if return_origin: + return balance_loss, loss + return balance_loss + + +class L1BalanceCELoss(nn.Module): + ''' + Balanced CrossEntropy Loss on `binary`, + MaskL1Loss on `thresh`, + DiceLoss on `thresh_binary`. + Note: The meaning of inputs can be figured out in `SegDetectorLossBuilder`. + ''' + + def __init__(self, eps=1e-6, l1_scale=10, bce_scale=5, hm_scale=10): + super(L1BalanceCELoss, self).__init__() + self.dice_loss = DiceLoss(eps=eps) + self.l1_loss = MaskL1Loss() + self.bce_loss = BalanceCrossEntropyLoss() + + self.l2_loss = MaskL2Loss() + self.hm_loss = FocalLoss() + + self.l1_scale = l1_scale + self.bce_scale = bce_scale + self.hm_scale = hm_scale + + def forward(self, pred, batch): + + bce_loss = self.bce_loss(pred['binary'], batch['gt'], batch['mask']) + metrics = dict(bce_loss=bce_loss) + if 'thresh' in pred: + l1_loss, l1_metric = self.l1_loss(pred['thresh'], + batch['thresh_map'], + batch['thresh_mask']) + dice_loss = self.dice_loss(pred['thresh_binary'], batch['gt'], + batch['mask']) + metrics['thresh_loss'] = dice_loss + loss = dice_loss + self.l1_scale * l1_loss + bce_loss * self.bce_scale + metrics.update(**l1_metric) + else: + loss = bce_loss + + if 'hm' in pred: + hm_loss, _ = self.l2_loss(pred['hm'], batch['heatmap'], + batch['mask']) + + metrics['hm_loss'] = hm_loss + loss = loss + self.hm_scale * hm_loss + + return loss, metrics diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/preprocessor.py b/runtime/Lib/site_packages/models/cv/ocr_detection/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..d3009af859771e61d8fb7eaf0a890cd296204013 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/preprocessor.py @@ -0,0 +1,71 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os +from typing import Any, Dict + +import cv2 +import numpy as np +import PIL +import torch + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor, load_image +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.utils.config import Config +from modelscope.utils.constant import Fields, ModeKeys, ModelFile + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.ocr_detection) +class OCRDetectionPreprocessor(Preprocessor): + + def __init__(self, model_dir: str, mode: str = ModeKeys.INFERENCE): + """The base constructor for all ocr recognition preprocessors. + + Args: + model_dir (str): model directory to initialize some resource + mode: The mode for the preprocessor. + """ + super().__init__(mode) + cfgs = Config.from_file( + os.path.join(model_dir, ModelFile.CONFIGURATION)) + self.image_short_side = cfgs.model.inference_kwargs.image_short_side + + def __call__(self, inputs): + """process the raw input data + Args: + inputs: + - A string containing an HTTP link pointing to an image + - A string containing a local path to an image + - An image loaded in PIL(PIL.Image.Image) or opencv(np.ndarray) directly, 3 channels RGB + Returns: + outputs: the preprocessed image + """ + if isinstance(inputs, str): + img = np.array(load_image(inputs)) + elif isinstance(inputs, PIL.Image.Image): + img = np.array(inputs) + elif isinstance(inputs, np.ndarray): + img = inputs + else: + raise TypeError( + f'inputs should be either str, PIL.Image, np.array, but got {type(inputs)}' + ) + + img = img[:, :, ::-1] + height, width, _ = img.shape + if height < width: + new_height = self.image_short_side + new_width = int(math.ceil(new_height / height * width / 32) * 32) + else: + new_width = self.image_short_side + new_height = int(math.ceil(new_width / width * height / 32) * 32) + resized_img = cv2.resize(img, (new_width, new_height)) + resized_img = resized_img - np.array([123.68, 116.78, 103.94], + dtype=np.float32) + resized_img /= 255. + resized_img = torch.from_numpy(resized_img).permute( + 2, 0, 1).float().unsqueeze(0) + + result = {'img': resized_img, 'org_shape': [height, width]} + return result diff --git a/runtime/Lib/site_packages/models/cv/ocr_detection/utils.py b/runtime/Lib/site_packages/models/cv/ocr_detection/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..489b3587726a84b3f7060492d83acc76788451e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_detection/utils.py @@ -0,0 +1,258 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import cv2 +import numpy as np +import pyclipper +from shapely.geometry import Polygon + + +def rboxes_to_polygons(rboxes): + """ + Convert rboxes to polygons + ARGS + `rboxes`: [n, 5] + RETURN + `polygons`: [n, 8] + """ + + theta = rboxes[:, 4:5] + cxcy = rboxes[:, :2] + half_w = rboxes[:, 2:3] / 2. + half_h = rboxes[:, 3:4] / 2. + v1 = np.hstack([np.cos(theta) * half_w, np.sin(theta) * half_w]) + v2 = np.hstack([-np.sin(theta) * half_h, np.cos(theta) * half_h]) + p1 = cxcy - v1 - v2 + p2 = cxcy + v1 - v2 + p3 = cxcy + v1 + v2 + p4 = cxcy - v1 + v2 + polygons = np.hstack([p1, p2, p3, p4]) + return polygons + + +def cal_width(box): + pd1 = point_dist(box[0], box[1], box[2], box[3]) + pd2 = point_dist(box[4], box[5], box[6], box[7]) + return (pd1 + pd2) / 2 + + +def point_dist(x1, y1, x2, y2): + return np.sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)) + + +def draw_polygons(img, polygons): + for p in polygons.tolist(): + p = [int(o) for o in p] + cv2.line(img, (p[0], p[1]), (p[2], p[3]), (0, 255, 0), 1) + cv2.line(img, (p[2], p[3]), (p[4], p[5]), (0, 255, 0), 1) + cv2.line(img, (p[4], p[5]), (p[6], p[7]), (0, 255, 0), 1) + cv2.line(img, (p[6], p[7]), (p[0], p[1]), (0, 255, 0), 1) + return img + + +def nms_python(boxes): + boxes = sorted(boxes, key=lambda x: -x[8]) + nms_flag = [True] * len(boxes) + for i, a in enumerate(boxes): + if not nms_flag[i]: + continue + else: + for j, b in enumerate(boxes): + if not j > i: + continue + if not nms_flag[j]: + continue + score_a = a[8] + score_b = b[8] + rbox_a = polygon2rbox(a[:8]) + rbox_b = polygon2rbox(b[:8]) + if point_in_rbox(rbox_a[:2], rbox_b) or point_in_rbox( + rbox_b[:2], rbox_a): + if score_a > score_b: + nms_flag[j] = False + boxes_nms = [] + for i, box in enumerate(boxes): + if nms_flag[i]: + boxes_nms.append(box) + return boxes_nms + + +def point_in_rbox(c, rbox): + cx0, cy0 = c[0], c[1] + cx1, cy1 = rbox[0], rbox[1] + w, h = rbox[2], rbox[3] + theta = rbox[4] + dist_x = np.abs((cx1 - cx0) * np.cos(theta) + (cy1 - cy0) * np.sin(theta)) + dist_y = np.abs(-(cx1 - cx0) * np.sin(theta) + (cy1 - cy0) * np.cos(theta)) + return ((dist_x < w / 2.0) and (dist_y < h / 2.0)) + + +def polygon2rbox(polygon): + x1, x2, x3, x4 = polygon[0], polygon[2], polygon[4], polygon[6] + y1, y2, y3, y4 = polygon[1], polygon[3], polygon[5], polygon[7] + c_x = (x1 + x2 + x3 + x4) / 4 + c_y = (y1 + y2 + y3 + y4) / 4 + w1 = point_dist(x1, y1, x2, y2) + w2 = point_dist(x3, y3, x4, y4) + h1 = point_line_dist(c_x, c_y, x1, y1, x2, y2) + h2 = point_line_dist(c_x, c_y, x3, y3, x4, y4) + h = h1 + h2 + w = (w1 + w2) / 2 + theta1 = np.arctan2(y2 - y1, x2 - x1) + theta2 = np.arctan2(y3 - y4, x3 - x4) + theta = (theta1 + theta2) / 2.0 + return [c_x, c_y, w, h, theta] + + +def point_line_dist(px, py, x1, y1, x2, y2): + eps = 1e-6 + dx = x2 - x1 + dy = y2 - y1 + div = np.sqrt(dx * dx + dy * dy) + eps + dist = np.abs(px * dy - py * dx + x2 * y1 - y2 * x1) / div + return dist + + +# Part of the implementation is borrowed and modified from DB, +# publicly available at https://github.com/MhLiao/DB. +def polygons_from_bitmap(pred, _bitmap, dest_width, dest_height): + """ + _bitmap: single map with shape (1, H, W), + whose values are binarized as {0, 1} + """ + + assert _bitmap.size(0) == 1 + bitmap = _bitmap.cpu().numpy()[0] + pred = pred.cpu().detach().numpy()[0] + height, width = bitmap.shape + boxes = [] + scores = [] + + contours, _ = cv2.findContours((bitmap * 255).astype(np.uint8), + cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + + for contour in contours[:100]: + epsilon = 0.01 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) + points = approx.reshape((-1, 2)) + if points.shape[0] < 4: + continue + + score = box_score_fast(pred, points.reshape(-1, 2)) + if 0.7 > score: + continue + + if points.shape[0] > 2: + box = unclip(points, unclip_ratio=2.0) + if len(box) > 1: + continue + else: + continue + box = box.reshape(-1, 2) + _, sside = get_mini_boxes(box.reshape((-1, 1, 2))) + if sside < 3 + 2: + continue + + if not isinstance(dest_width, int): + dest_width = dest_width.item() + dest_height = dest_height.item() + + box[:, 0] = np.clip( + np.round(box[:, 0] / width * dest_width), 0, dest_width) + box[:, 1] = np.clip( + np.round(box[:, 1] / height * dest_height), 0, dest_height) + boxes.append(box.tolist()) + scores.append(score) + return boxes, scores + + +def boxes_from_bitmap(pred, _bitmap, dest_width, dest_height, is_numpy=False): + """ + _bitmap: single map with shape (1, H, W), + whose values are binarized as {0, 1} + """ + if is_numpy: + bitmap = _bitmap[0] + pred = pred[0] + else: + bitmap = _bitmap.cpu().numpy()[0] + pred = pred.cpu().detach().numpy()[0] + height, width = bitmap.shape + boxes = [] + scores = [] + + contours, _ = cv2.findContours((bitmap * 255).astype(np.uint8), + cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + + for contour in contours[:1000]: + points, sside = get_mini_boxes(contour) + if sside < 3: + continue + points = np.array(points) + + score = box_score_fast(pred, points.reshape(-1, 2)) + if 0.3 > score: + continue + + box = unclip(points, unclip_ratio=1.5).reshape(-1, 1, 2) + box, sside = get_mini_boxes(box) + + if sside < 3 + 2: + continue + + box = np.array(box).astype(np.int32) + if not isinstance(dest_width, int): + dest_width = dest_width.item() + dest_height = dest_height.item() + + box[:, 0] = np.clip( + np.round(box[:, 0] / width * dest_width), 0, dest_width) + box[:, 1] = np.clip( + np.round(box[:, 1] / height * dest_height), 0, dest_height) + boxes.append(box.reshape(-1).tolist()) + scores.append(score) + return boxes, scores + + +def box_score_fast(bitmap, _box): + h, w = bitmap.shape[:2] + box = _box.copy() + xmin = np.clip(np.floor(box[:, 0].min()).astype(np.int32), 0, w - 1) + xmax = np.clip(np.ceil(box[:, 0].max()).astype(np.int32), 0, w - 1) + ymin = np.clip(np.floor(box[:, 1].min()).astype(np.int32), 0, h - 1) + ymax = np.clip(np.ceil(box[:, 1].max()).astype(np.int32), 0, h - 1) + + mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8) + box[:, 0] = box[:, 0] - xmin + box[:, 1] = box[:, 1] - ymin + cv2.fillPoly(mask, box.reshape(1, -1, 2).astype(np.int32), 1) + return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0] + + +def unclip(box, unclip_ratio=1.5): + poly = Polygon(box) + distance = poly.area * unclip_ratio / poly.length + offset = pyclipper.PyclipperOffset() + offset.AddPath(box, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON) + expanded = np.array(offset.Execute(distance)) + return expanded + + +def get_mini_boxes(contour): + bounding_box = cv2.minAreaRect(contour) + points = sorted(list(cv2.boxPoints(bounding_box)), key=lambda x: x[0]) + + index_1, index_2, index_3, index_4 = 0, 1, 2, 3 + if points[1][1] > points[0][1]: + index_1 = 0 + index_4 = 1 + else: + index_1 = 1 + index_4 = 0 + if points[3][1] > points[2][1]: + index_2 = 2 + index_3 = 3 + else: + index_2 = 3 + index_3 = 2 + + box = [points[index_1], points[index_2], points[index_3], points[index_4]] + return box, min(bounding_box[1]) diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..39c09384a179af1599f89c847ad2d07278c8a281 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import OCRRecognition + +else: + _import_structure = { + 'model': ['OCRRecognition'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/model.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/model.py new file mode 100644 index 0000000000000000000000000000000000000000..4c5aa362dbe9409faccf0e775c18352236190a49 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/model.py @@ -0,0 +1,168 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import torch +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .modules.ConvNextViT.main_model import ConvNextViT +from .modules.CRNN.main_model import CRNN +from .modules.LightweightEdge.main_model import LightweightEdge + +LOGGER = get_logger() + + +def flatten_label(target): + label_flatten = [] + label_length = [] + label_dict = [] + for i in range(0, target.size()[0]): + cur_label = target[i].tolist() + temp_label = cur_label[:cur_label.index(0)] + label_flatten += temp_label + label_dict.append(temp_label) + label_length.append(len(temp_label)) + label_flatten = torch.LongTensor(label_flatten) + label_length = torch.IntTensor(label_length) + return (label_dict, label_length, label_flatten) + + +class cha_encdec(): + + def __init__(self, charMapping, case_sensitive=True): + self.case_sensitive = case_sensitive + self.text_seq_len = 160 + self.charMapping = charMapping + + def encode(self, label_batch): + max_len = max([len(s) for s in label_batch]) + out = torch.zeros(len(label_batch), max_len + 1).long() + for i in range(0, len(label_batch)): + if not self.case_sensitive: + cur_encoded = torch.tensor([ + self.charMapping[char.lower()] - 1 if char.lower() + in self.charMapping else len(self.charMapping) + for char in label_batch[i] + ]) + 1 + else: + cur_encoded = torch.tensor([ + self.charMapping[char] + - 1 if char in self.charMapping else len(self.charMapping) + for char in label_batch[i] + ]) + 1 + out[i][0:len(cur_encoded)] = cur_encoded + out = torch.cat( + (out, torch.zeros( + (out.size(0), self.text_seq_len - out.size(1))).type_as(out)), + dim=1) + label_dict, label_length, label_flatten = flatten_label(out) + return label_dict, label_length, label_flatten + + +@MODELS.register_module( + Tasks.ocr_recognition, module_name=Models.ocr_recognition) +class OCRRecognition(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """initialize the ocr recognition model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, **kwargs) + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + cfgs = Config.from_file( + os.path.join(model_dir, ModelFile.CONFIGURATION)) + self.do_chunking = cfgs.model.inference_kwargs.do_chunking + self.target_height = cfgs.model.inference_kwargs.img_height + self.target_width = cfgs.model.inference_kwargs.img_width + self.recognizer = None + if cfgs.model.recognizer == 'ConvNextViT': + self.recognizer = ConvNextViT() + elif cfgs.model.recognizer == 'CRNN': + self.recognizer = CRNN() + elif cfgs.model.recognizer == 'LightweightEdge': + self.recognizer = LightweightEdge() + else: + raise TypeError( + f'recognizer should be either ConvNextViT, CRNN, but got {cfgs.model.recognizer}' + ) + if model_path != '': + params_pretrained = torch.load(model_path, map_location='cpu') + model_dict = self.recognizer.state_dict() + # remove prefix for finetuned models + check_point = { + k.replace('recognizer.', '').replace('module.', ''): v + for k, v in params_pretrained.items() + } + model_dict.update(check_point) + self.recognizer.load_state_dict(model_dict) + + dict_path = os.path.join(model_dir, ModelFile.VOCAB_FILE) + self.labelMapping = dict() + self.charMapping = dict() + with open(dict_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + cnt = 1 + # ConvNextViT and LightweightEdge model start from index=2 + if cfgs.model.recognizer == 'ConvNextViT' or cfgs.model.recognizer == 'LightweightEdge': + cnt += 1 + for line in lines: + line = line.strip('\n') + self.labelMapping[cnt] = line + self.charMapping[line] = cnt + cnt += 1 + + self.encdec = cha_encdec(self.charMapping) + self.criterion_CTC = torch.nn.CTCLoss(zero_infinity=True) + + def forward(self, inputs): + """ + Args: + img (`torch.Tensor`): batched image tensor, + shape of each tensor is [N, 1, H, W]. + + Return: + `probs [T, N, Classes] of the sequence feature` + """ + return self.recognizer(inputs) + + def do_step(self, batch): + inputs = batch['images'] + labels = batch['labels'] + bs = inputs.shape[0] + if self.do_chunking: + inputs = inputs.view(bs * 3, 3, self.target_height, 300) + else: + inputs = inputs.view(bs, 3, self.target_height, self.target_width) + output = self(inputs) + probs = output['probs'].permute(1, 0, 2) + _, label_length, label_flatten = self.encdec.encode(labels) + probs_sizes = torch.IntTensor([probs.size(0)] * probs.size(1)) + loss = self.criterion_CTC( + probs.log_softmax(2), label_flatten, probs_sizes, label_length) + output = dict(loss=loss, preds=output['preds']) + return output + + def postprocess(self, inputs): + outprobs = inputs + outprobs = F.softmax(outprobs, dim=-1) + preds = torch.argmax(outprobs, -1) + batchSize, length = preds.shape + final_str_list = [] + for i in range(batchSize): + pred_idx = preds[i].cpu().data.tolist() + last_p = 0 + str_pred = [] + for p in pred_idx: + if p != last_p and p != 0: + str_pred.append(self.labelMapping[p]) + last_p = p + final_str = ''.join(str_pred) + final_str_list.append(final_str) + return {'preds': final_str_list, 'probs': inputs} diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/CRNN/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/CRNN/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/CRNN/main_model.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/CRNN/main_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f2ce8c21f27db87a08979ca3938da4d26e8f4c97 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/CRNN/main_model.py @@ -0,0 +1,105 @@ +# Part of the implementation is borrowed and modified from CRNN, +# publicly available at https://github.com/meijieru/crnn.pytorch +# paper linking at https://arxiv.org/pdf/1507.05717.pdf +import torch +import torch.nn as nn + + +class BidirectionalLSTM(nn.Module): + + def __init__(self, nIn, nHidden, nOut): + super(BidirectionalLSTM, self).__init__() + self.rnn = nn.LSTM(nIn, nHidden, bidirectional=True) + self.embedding = nn.Linear(nHidden * 2, nOut) + + def forward(self, input): + recurrent, _ = self.rnn(input) + T, b, h = recurrent.size() + t_rec = recurrent.view(T * b, h) + + output = self.embedding(t_rec) # [T * b, nOut] + output = output.view(T, b, -1) + return output + + +class CRNN(nn.Module): + + def __init__(self): + super(CRNN, self).__init__() + self.conv0 = nn.Sequential( + nn.Conv2d( + 1, 64, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True), + ) + self.p0 = nn.MaxPool2d( + kernel_size=(2, 2), stride=(2, 2), padding=(0, 0)) + self.conv1 = nn.Sequential( + nn.Conv2d( + 64, 128, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + ) + self.p1 = nn.MaxPool2d( + kernel_size=(2, 2), stride=(2, 2), padding=(0, 0)) + self.conv2 = nn.Sequential( + nn.Conv2d( + 128, 256, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(256), + nn.ReLU(inplace=True), + nn.Conv2d( + 256, 256, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(256), + nn.ReLU(inplace=True), + ) + self.p2 = nn.MaxPool2d( + kernel_size=(2, 1), stride=(2, 1), padding=(0, 0)) + self.conv3 = nn.Sequential( + nn.Conv2d( + 256, 512, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(512), + nn.ReLU(inplace=True), + nn.Conv2d( + 512, 512, kernel_size=(3, 3), padding=(1, 1), stride=(1, 1)), + nn.BatchNorm2d(512), + nn.ReLU(inplace=True), + ) + self.p3 = nn.MaxPool2d( + kernel_size=(2, 1), stride=(2, 1), padding=(0, 0)) + self.conv4 = nn.Sequential( + nn.Conv2d( + 512, 512, kernel_size=(2, 1), padding=(0, 0), stride=(2, 1)), + nn.BatchNorm2d(512), + nn.ReLU(inplace=True), + ) + + self.rnn = nn.Sequential( + BidirectionalLSTM(512, 256, 256), BidirectionalLSTM(256, 256, 512)) + + self.cls = nn.Linear(512, 7644, bias=False) + + def forward(self, input): + # RGB2GRAY + input = input[:, 0: + 1, :, :] * 0.2989 + input[:, 1: + 2, :, :] * 0.5870 + input[:, 2: + 3, :, :] * 0.1140 + feats = self.conv0(input) + feats = self.p0(feats) + feats = self.conv1(feats) + feats = self.p1(feats) + feats = self.conv2(feats) + feats = self.p2(feats) + feats = self.conv3(feats) + feats = self.p3(feats) + convfeats = self.conv4(feats) + + b, c, h, w = convfeats.size() + assert h == 1, 'the height of conv must be 1' + convfeats = convfeats.squeeze(2) + convfeats = convfeats.permute(2, 0, 1) # [w, b, c] + + rnnfeats = self.rnn(convfeats) + output = self.cls(rnnfeats) + output = output.permute(1, 0, 2) # [b, w, c] + return output diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/convnext.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/convnext.py new file mode 100644 index 0000000000000000000000000000000000000000..c0e30616cd8313eebfae55fcb8ba97ccff81eb14 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/convnext.py @@ -0,0 +1,163 @@ +# Part of the implementation is borrowed and modified from ConvNext, +# publicly available at https://github.com/facebookresearch/ConvNeXt +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .timm_tinyc import DropPath + + +class Block(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + self.dwconv = nn.Conv2d( + dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv + self.norm = LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, + 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) + + x = input + self.drop_path(x) + return x + + +class ConvNeXt(nn.Module): + r""" ConvNeXt + A PyTorch impl of : `A ConvNet for the 2020s` - + https://arxiv.org/pdf/2201.03545.pdf + + Args: + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3] + dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768] + drop_path_rate (float): Stochastic depth rate. Default: 0. + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1. + """ + + def __init__( + self, + in_chans=1, + num_classes=1000, + depths=[3, 3, 9, 3], + dims=[96, 192, 384, 768], + drop_path_rate=0., + layer_scale_init_value=1e-6, + head_init_scale=1., + ): + super().__init__() + + self.downsample_layers = nn.ModuleList( + ) # stem and 3 intermediate downsampling conv layers + stem = nn.Sequential( + nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4), + LayerNorm(dims[0], eps=1e-6, data_format='channels_first')) + self.downsample_layers.append(stem) + for i in range(3): + downsample_layer = nn.Sequential( + LayerNorm(dims[i], eps=1e-6, data_format='channels_first'), + nn.Conv2d( + dims[i], dims[i + 1], kernel_size=(2, 1), stride=(2, 1)), + ) + self.downsample_layers.append(downsample_layer) + + self.stages = nn.ModuleList( + ) # 4 feature resolution stages, each consisting of multiple residual blocks + dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + cur = 0 + for i in range(4): + stage = nn.Sequential(*[ + Block( + dim=dims[i], + drop_path=dp_rates[cur + j], + layer_scale_init_value=layer_scale_init_value) + for j in range(depths[i]) + ]) + self.stages.append(stage) + cur += depths[i] + + def _init_weights(self, m): + if isinstance(m, (nn.Conv2d, nn.Linear)): + trunc_normal_(m.weight, std=.02) + nn.init.constant_(m.bias, 0) + + def forward_features(self, x): + for i in range(4): + x = self.downsample_layers[i](x.contiguous()) + x = self.stages[i](x.contiguous()) + return x # global average pooling, (N, C, H, W) -> (N, C) + + def forward(self, x): + x = self.forward_features(x.contiguous()) + + return x.contiguous() + + +class LayerNorm(nn.Module): + r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with + shape (batch_size, height, width, channels) while channels_first corresponds to inputs + with shape (batch_size, channels, height, width). + """ + + def __init__(self, + normalized_shape, + eps=1e-6, + data_format='channels_last'): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ['channels_last', 'channels_first']: + raise NotImplementedError + self.normalized_shape = (normalized_shape, ) + + def forward(self, x): + if self.data_format == 'channels_last': + return F.layer_norm(x, self.normalized_shape, self.weight, + self.bias, self.eps) + elif self.data_format == 'channels_first': + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + + +def convnext_tiny(): + model = ConvNeXt(depths=[3, 3, 8, 3], dims=[96, 192, 256, 512]) + return model diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py new file mode 100644 index 0000000000000000000000000000000000000000..365d4005d8994303be18cb87131685805256813e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/main_model.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from .convnext import convnext_tiny +from .vitstr import vitstr_tiny + + +class ConvNextViT(nn.Module): + + def __init__(self): + super(ConvNextViT, self).__init__() + self.cnn_model = convnext_tiny() + self.vitstr = vitstr_tiny(num_tokens=7644) + + def forward(self, input): + # RGB2GRAY + input = input[:, 0: + 1, :, :] * 0.2989 + input[:, 1: + 2, :, :] * 0.5870 + input[:, 2: + 3, :, :] * 0.1140 + features = self.cnn_model(input) + output = self.vitstr(features) + return output diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/timm_tinyc.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/timm_tinyc.py new file mode 100644 index 0000000000000000000000000000000000000000..555b1e422aa98a8b1886e2d947180c8b7f26b90e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/timm_tinyc.py @@ -0,0 +1,332 @@ +# Part of the implementation is borrowed and modified from timm, +# publicly available at https://github.com/rwightman/pytorch-image-models +import collections.abc +import logging +import math +from collections import OrderedDict +from copy import deepcopy +from functools import partial +from itertools import repeat + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def _ntuple(n): + + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + + def __init__(self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True): + super().__init__() + img_size = (1, 75) + to_2tuple = _ntuple(2) + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], + img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + B, C, H, W = x.shape + assert H == self.img_size[0] and W == self.img_size[1], \ + f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + x = x.permute(0, 1, 3, 2) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + attn_drop=0.1, + proj_drop=0.1): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + + def __init__(self, + dim, + num_heads, + mlp_ratio=4., + qkv_bias=False, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + attn_drop=attn_drop, + proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + img_size=224, + patch_size=16, + in_chans=3, + num_classes=1000, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4., + qkv_bias=True, + representation_size=None, + distilled=False, + drop_rate=0.1, + attn_drop_rate=0.1, + drop_path_rate=0., + embed_layer=PatchEmbed, + norm_layer=None, + act_layer=None, + weight_init=''): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set + distilled (bool): model includes a distillation token and head as in DeiT models + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + embed_layer (nn.Module): patch embedding layer + norm_layer: (nn.Module): normalization layer + weight_init: (str): weight init scheme + """ + super().__init__() + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 2 if distilled else 1 + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + act_layer = act_layer or nn.GELU + + self.patch_embed = embed_layer( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.dist_token = nn.Parameter(torch.zeros( + 1, 1, embed_dim)) if distilled else None + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer) for i in range(depth) + ]) + self.norm = norm_layer(embed_dim) + + # Representation layer + if representation_size and not distilled: + self.num_features = representation_size + self.pre_logits = nn.Sequential( + OrderedDict([('fc', nn.Linear(embed_dim, representation_size)), + ('act', nn.Tanh())])) + else: + self.pre_logits = nn.Identity() + + # Classifier head(s) + self.head = nn.Linear( + self.num_features, + num_classes) if num_classes > 0 else nn.Identity() + self.head_dist = None + if distilled: + self.head_dist = nn.Linear( + self.embed_dim, + self.num_classes) if num_classes > 0 else nn.Identity() + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear( + self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + if self.num_tokens == 2: + self.head_dist = nn.Linear( + self.embed_dim, + self.num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + x = self.patch_embed(x) + cls_token = self.cls_token.expand( + x.shape[0], -1, -1) # stole cls_tokens impl from Phil Wang, thanks + if self.dist_token is None: + x = torch.cat((cls_token, x), dim=1) + else: + x = torch.cat( + (cls_token, self.dist_token.expand(x.shape[0], -1, -1), x), + dim=1) + x = self.pos_drop(x + self.pos_embed) + x = self.blocks(x) + x = self.norm(x) + if self.dist_token is None: + return self.pre_logits(x[:, 0]) + else: + return x[:, 0], x[:, 1] + + def forward(self, x): + x = self.forward_features(x) + if self.head_dist is not None: + x, x_dist = self.head(x[0]), self.head_dist( + x[1]) # x must be a tuple + if self.training and not torch.jit.is_scripting(): + # during inference, return the average of both classifier predictions + return x, x_dist + else: + return (x + x_dist) / 2 + else: + x = self.head(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/vitstr.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/vitstr.py new file mode 100644 index 0000000000000000000000000000000000000000..c9fa06935c65647a6789fc0f7034a339705603e8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/ConvNextViT/vitstr.py @@ -0,0 +1,65 @@ +# Part of the implementation is borrowed and modified from ViTSTR, +# publicly available at https://github.com/roatienza/deep-text-recognition-benchmark +from __future__ import absolute_import, division, print_function +import logging +from copy import deepcopy +from functools import partial + +import torch +import torch.nn as nn +import torch.utils.model_zoo as model_zoo + +from .timm_tinyc import VisionTransformer + + +class ViTSTR(VisionTransformer): + ''' + ViTSTR is basically a ViT that uses DeiT weights. + Modified head to support a sequence of characters prediction for STR. + ''' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def reset_classifier(self, num_classes): + self.num_classes = num_classes + self.head = nn.Linear( + self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + x = self.patch_embed(x) + + x = x + self.pos_embed + x = self.pos_drop(x) + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + return x + + def forward(self, x): + x = self.forward_features(x) + ap = x.view(x.shape[0] // 3, 3, 75, x.shape[2]) + features_1d_concat = torch.ones(x.shape[0] // 3, 201, + x.shape[2]).type_as(x) + features_1d_concat[:, :69, :] = ap[:, 0, :69, :] + features_1d_concat[:, 69:69 + 63, :] = ap[:, 1, 6:-6, :] + features_1d_concat[:, 69 + 63:, :] = ap[:, 2, 6:, :] + x = features_1d_concat + b, s, e = x.size() + x = x.reshape(b * s, e) + x = self.head(x).view(b, s, self.num_classes) + return x + + +def vitstr_tiny(num_tokens): + vitstr = ViTSTR( + patch_size=1, + in_chans=512, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4, + qkv_bias=True) + vitstr.reset_classifier(num_classes=num_tokens) + return vitstr diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5159a355f8c331b94e596cd32b82d3af8c54ff --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/main_model.py @@ -0,0 +1,36 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from collections import OrderedDict + +import torch +import torch.nn as nn + +from .nas_block import plnas_linear_mix_se + + +class LightweightEdge(nn.Module): + """ + 基于混合rep block的nas模型 + Args: + input (tensor): batch of input images + """ + + def __init__(self): + super(LightweightEdge, self).__init__() + self.our_nas_model = plnas_linear_mix_se(1, 128) + self.embed_dim = 128 + self.head = nn.Linear(self.embed_dim, 7644) + + def forward(self, input): + # RGB2GRAY + input = input[:, 0: + 1, :, :] * 0.2989 + input[:, 1: + 2, :, :] * 0.5870 + input[:, 2: + 3, :, :] * 0.1140 + x = self.our_nas_model(input) + x = torch.squeeze(x, 2) + x = torch.transpose(x, 1, 2) + b, s, e = x.size() + x = x.reshape(b * s, e) + prediction = self.head(x).view(b, s, -1) + return prediction diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c8115b7c1316ed4d1f64342d8f8454e0e53929b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/__init__.py @@ -0,0 +1 @@ +from .proxyless import plnas_linear_mix_se diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..23802f6d12b8267db25c24d72a4bad5fa9fe3c70 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/layers.py @@ -0,0 +1,790 @@ +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def get_same_padding(kernel_size): + + if isinstance(kernel_size, tuple): + assert len(kernel_size) == 2, 'invalid kernel size: %s' % kernel_size + p1 = get_same_padding(kernel_size[0]) + p2 = get_same_padding(kernel_size[1]) + return p1, p2 + assert isinstance(kernel_size, + int), 'kernel size should be either `int` or `tuple`' + assert kernel_size % 2 > 0, 'kernel size should be odd number' + return kernel_size // 2 + + +def count_conv_flop(layer, x): + out_h = int(x.size(2) / layer.stride[0]) + out_w = int(x.size(3) / layer.stride[1]) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[ + 0] * layer.kernel_size[1] * out_h * out_w / layer.groups + return delta_ops + + +def set_layer_from_config(layer_config): + if layer_config is None: + return None + name2layer = { + ZeroLayer.__name__: ZeroLayer, + MBInvertedConvLayer.__name__: MBInvertedConvLayer, + IdentityLayer.__name__: IdentityLayer, + } + layer_name = layer_config.pop('name') + layer = name2layer[layer_name] + return layer.build_from_config(layer_config) + + +class MobileInvertedResidualBlock(nn.Module): + + def __init__(self, mobile_inverted_conv, shortcut): + super(MobileInvertedResidualBlock, self).__init__() + self.mobile_inverted_conv = mobile_inverted_conv + self.shortcut = shortcut + + def forward(self, x): + if self.mobile_inverted_conv.is_zero_layer(): + res = x + elif self.shortcut is None or self.shortcut.is_zero_layer(): + res = self.mobile_inverted_conv(x) + else: + conv_x = self.mobile_inverted_conv(x) + skip_x = self.shortcut(x) + res = skip_x + conv_x + return res + + @property + def module_str(self): + return '(%s, %s)' % (self.mobile_inverted_conv.module_str, + self.shortcut.module_str + if self.shortcut is not None else None) + + @property + def config(self): + return { + 'name': + MobileInvertedResidualBlock.__name__, + 'mobile_inverted_conv': + self.mobile_inverted_conv.config, + 'shortcut': + self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + mobile_inverted_conv = set_layer_from_config( + config['mobile_inverted_conv']) + shortcut = set_layer_from_config(config['shortcut']) + return MobileInvertedResidualBlock(mobile_inverted_conv, shortcut) + + def get_flops(self, x): + flops1, _ = self.mobile_inverted_conv.get_flops(x) + if self.shortcut: + flops2, _ = self.shortcut.get_flops(x) + else: + flops2 = 0 + return flops1 + flops2, self.forward(x) + + +class MBInvertedConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + pad = get_same_padding(self.kernel_size) + self.depth_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%dx%d_MBConv%d' % (self.kernel_size[0], self.kernel_size[1], + self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + total_flops += count_conv_flop(self.depth_conv.conv, x) + x = self.depth_conv(x) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class IdentityLayer(nn.Module): + + def __init__(self): + super(IdentityLayer, self).__init__() + + def forward(self, x): + return x + + @property + def module_str(self): + return 'Identity' + + @property + def config(self): + return { + 'name': IdentityLayer.__name__, + } + + @staticmethod + def build_from_config(config): + return IdentityLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + return 0, self.forward(x) + + +class ZeroLayer(nn.Module): + + def __init__(self, stride): + super(ZeroLayer, self).__init__() + self.stride = stride + + def forward(self, x): + n, c, h, w = x.shape + h //= self.stride[0] + w //= self.stride[1] + device = x.device + padding = torch.zeros(n, c, h, w, device=device, requires_grad=False) + return padding + + @property + def module_str(self): + return 'Zero' + + @property + def config(self): + return {'name': ZeroLayer.__name__, 'stride': self.stride} + + @staticmethod + def build_from_config(config): + return ZeroLayer(**config) + + @staticmethod + def is_zero_layer(): + return True + + def get_flops(self, x): + return 0, self.forward(x) + + +def split_layer(total_channels, num_groups): + split = [ + int(np.ceil(total_channels / num_groups)) for _ in range(num_groups) + ] + split[num_groups - 1] += total_channels - sum(split) + return split + + +class MBInvertedMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedMixConvLayer, self).__init__() + self.mix_type = 0 + + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.split_in_channels = split_layer(feature_dim, self.n_chunks) + + self.mix_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + split_in_channels_ = self.split_in_channels[idx] + if self.mix_type == 0: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ('act', nn.PReLU()), + ]))) + else: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + split_in_channels_, + split_in_channels_, + kernel_size, + stride, + pad, + groups=split_in_channels_, + bias=False)), + ('bn', nn.BatchNorm2d(split_in_channels_)), + ]))) + if self.mix_type != 0: + self.act = nn.PReLU() + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + x = torch.cat([layer(s) for layer, s in zip(self.mix_conv, split)], + dim=1) + + if self.mix_type != 0: + x = self.act(x) + x = self.point_conv(x) + + return x + + @property + def module_str(self): + return '%s_MixConv%d' % (str(self.mix_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + # if self.inverted_bottleneck: + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + split = torch.split(x, self.split_in_channels, dim=1) + out = [] + # import pdb;pdb.set_trace() + for layer, s in zip(self.mix_conv, split): + out.append(layer(s)) + total_flops += count_conv_flop(layer.conv, s) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class LinearMixConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + mix_conv_size=[1, 3, 5], + stride=(1, 1), + mid_channels=None): + super(LinearMixConvLayer, self).__init__() + self.mix_type = 0 + + self.in_channels = in_channels + self.out_channels = out_channels + self.mix_conv_size = mix_conv_size + self.stride = stride + self.mid_channels = mid_channels + + self.mix_conv_size = mix_conv_size + self.n_chunks = len(mix_conv_size) + self.mix_conv = nn.ModuleList() + + for idx in range(self.n_chunks): + kernel_size = self.mix_conv_size[idx] + pad = get_same_padding(kernel_size) + if self.mix_type == 0: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ('act', nn.PReLU()), + ]))) + else: + self.mix_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels, + in_channels, + kernel_size, + stride, + pad, + groups=in_channels, + bias=False)), + ('bn', nn.BatchNorm2d(in_channels)), + ]))) + + if self.mix_conv != 0: + self.act = nn.PReLU() + + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + in_channels * self.n_chunks, + out_channels, + 1, + 1, + 0, + bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + x = torch.cat([layer(x) for layer in self.mix_conv], dim=1) + if self.mix_conv != 0: + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_LinearMixConv' % (str(self.mix_conv_size)) + + @property + def config(self): + return { + 'name': LinearMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'mix_conv_size': self.mix_conv_size, + 'stride': self.stride, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return LinearMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + out = [] + for layer in self.mix_conv: + out.append(layer(x)) + total_flops += count_conv_flop(layer.conv, x) + x = torch.cat(out, dim=1) + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x + + +class SELayer(nn.Module): + ''' + ''' + + def __init__(self, input_channels: int, squeeze_factor: int = 4): + super(SELayer, self).__init__() + self.input_channels = input_channels + self.squeeze_factor = squeeze_factor + self.squeeze_channels = input_channels // squeeze_factor + self.fc1 = nn.Conv2d( + self.input_channels, self.squeeze_channels, 1, bias=True) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Conv2d( + self.squeeze_channels, self.input_channels, 1, bias=True) + + def _scale(self, input): + scale = F.adaptive_avg_pool2d(input, 1) + scale = self.fc1(scale) + scale = self.relu(scale) + scale = self.fc2(scale) + return torch.sigmoid(scale) + + def forward(self, input): + scale = self._scale(input) + return scale * input + + @property + def module_str(self): + return 'SE_%d' % (self.squeeze_factor) + + @property + def config(self): + return { + 'name': SELayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'squeeze_factor': self.squeeze_factor, + } + + @staticmethod + def build_from_config(config): + return SELayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def get_flops(self, x): + ''' + count se flops, only compute the fc layers' calculation + ''' + total_flops = 0 + total_flops += self.input_channels * self.squeeze_channels * 2 + b, c, h, w = x.shape + total_flops += c * h * w + return total_flops, x + + +class MBInvertedRepConvLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + rep_conv_size=[1, 3, 5], + stride=(1, 1), + expand_ratio=6, + mid_channels=None): + super(MBInvertedRepConvLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.rep_conv_size = rep_conv_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + + feature_dim = round( + self.in_channels + * self.expand_ratio) if mid_channels is None else mid_channels + + self.inverted_bottleneck = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', nn.PReLU()), + ])) + + self.rep_conv_size = rep_conv_size + self.n_chunks = len(rep_conv_size) + + self.rep_conv = nn.ModuleList() + for idx in range(self.n_chunks): + kernel_size = self.rep_conv_size[idx] + pad = get_same_padding(kernel_size) + self.rep_conv.append( + nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ]))) + + self.rep_conv_deploy = None + self.act = nn.PReLU() + self.point_conv = nn.Sequential( + OrderedDict([ + ('conv', + nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + self.deploy = False + + def forward(self, x): + # if self.inverted_bottleneck: + + x = self.inverted_bottleneck(x) + + if not self.deploy: + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + else: + x = self.rep_conv_deploy(x) + x = self.act(x) + x = self.point_conv(x) + return x + + @property + def module_str(self): + return '%s_RepConv%d' % (str(self.rep_conv_size), self.expand_ratio) + + @property + def config(self): + return { + 'name': MBInvertedMixConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'rep_conv_size': self.rep_conv_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + } + + @staticmethod + def build_from_config(config): + return MBInvertedMixConvLayer(**config) + + @staticmethod + def is_zero_layer(): + return False + + def switch_to_deploy(self): + self.deploy = True + feature_dim = self.rep_conv[0].conv.in_channels + stride = self.rep_conv[0].conv.stride + kernel_size = max(self.rep_conv_size) + pad = get_same_padding(kernel_size) + self.rep_conv_deploy = nn.Conv2d( + feature_dim, + feature_dim, + kernel_size, + stride, + pad, + groups=feature_dim, + bias=True) + + kernel, bias = self.get_equivalent_kernel_bias() + self.rep_conv_deploy.weight.data = kernel + self.rep_conv_deploy.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rep_conv') + + def get_equivalent_kernel_bias(self): + import pdb + pdb.set_trace() + if 1 in self.rep_conv_size: + kernel1x1, bias1x1 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(1)]) + else: + kernel1x1 = None + bias1x1 = 0 + + if 3 in self.rep_conv_size: + kernel3x3, bias3x3 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(3)]) + else: + kernel3x3 = None + bias3x3 = 0 + + if 5 in self.rep_conv_size: + kernel5x5, bias5x5 = self._fuse_bn_tensor( + self.rep_conv[self.rep_conv_size.index(5)]) + + else: + kernel5x5 = 0 + bias5x5 = 0 + + return kernel5x5 + self._pad_1x1_to_5x5_tensor( + kernel1x1) + self._pad_3x3_to_5x5_tensor( + kernel3x3), bias5x5 + bias3x3 + bias1x1 + + def _pad_1x1_to_5x5_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [2, 2, 2, 2]) + + def _pad_3x3_to_5x5_tensor(self, kernel3x3): + if kernel3x3 is None: + return 0 + else: + return torch.nn.functional.pad(kernel3x3, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + + return kernel * t, beta - running_mean * gamma / std + + def get_flops(self, x): + '''count conv flops, skip BN and other small flops + ''' + total_flops = 0 + total_flops += count_conv_flop(self.inverted_bottleneck.conv, x) + x = self.inverted_bottleneck(x) + + total_flops += count_conv_flop(self.rep_conv[-1].conv, x) + out = [] + for layer in self.rep_conv: + out.append(layer(x)) + x = out[0] + for out_ in out[1:]: + x += out_ + total_flops += count_conv_flop(self.point_conv.conv, x) + x = self.point_conv(x) + return total_flops, x diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..effed1a6671fba133780a4c628fc28616ab8b913 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/mix_ops.py @@ -0,0 +1,226 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from .layers import (LinearMixConvLayer, MBInvertedConvLayer, + MBInvertedMixConvLayer, MBInvertedRepConvLayer, SELayer, + ZeroLayer) + + +def detach_variable(inputs): + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + + +def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + +def conv_func_by_name(name): + + name2ops = { + 'Identity': + lambda in_C, out_C, S, height: IdentityLayer( + in_C, out_C, ops_order=ops_order), + 'Zero': + lambda in_C, out_C, S, height: ZeroLayer(stride=S), + } + name2ops.update({ + '3x3_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 1), + '3x3_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 2), + '3x3_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 3), + '3x3_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 4), + '3x3_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 5), + '3x3_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 3), 3), S, 6), + ####################################################################################### + '5x5_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 1), + '5x5_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 2), + '5x5_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 3), + '5x5_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 4), + '5x5_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 5), + '5x5_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 5), 5), S, 6), + ####################################################################################### + '7x7_MBConv1': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 1), + '7x7_MBConv2': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 2), + '7x7_MBConv3': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 3), + '7x7_MBConv4': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 4), + '7x7_MBConv5': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 5), + '7x7_MBConv6': + lambda in_C, out_C, S, height: MBInvertedConvLayer( + in_C, out_C, (min(height, 7), 7), S, 6), + ####################################################################################### + '13_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 1), + '13_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 2), + '13_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 3), + '13_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 4), + '13_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 5), + '13_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 6), + ####################################################################################### + '35_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 1), + '35_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 2), + '35_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 3), + '35_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 4), + '35_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 5), + '35_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '135_MixConv1': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 1), + '135_MixConv2': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 2), + '135_MixConv3': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 3), + '135_MixConv4': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 4), + '135_MixConv5': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 5), + '135_MixConv6': + lambda in_C, out_C, S, height: MBInvertedMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '13_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S), + '35_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S), + '135_LinMixConv': + lambda in_C, out_C, S, height: LinearMixConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S), + ####################################################################################### + 'SE_2': + lambda in_C, out_C, S, height: SELayer(in_C, 2), + 'SE_4': + lambda in_C, out_C, S, height: SELayer(in_C, 4), + 'SE_8': + lambda in_C, out_C, S, height: SELayer(in_C, 8), + ####################################################################################### + '13_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 1), + '13_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 2), + '13_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 3), + '13_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 4), + '13_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 5), + '13_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3)], S, 6), + ####################################################################################### + '35_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 1), + '35_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 2), + '35_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 3), + '35_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 4), + '35_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 5), + '35_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [(min(height, 3), 3), (min(height, 5), 5)], S, 6), + ####################################################################################### + '135_RepConv1': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 1), + '135_RepConv2': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 2), + '135_RepConv3': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 3), + '135_RepConv4': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 4), + '135_RepConv5': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 5), + '135_RepConv6': + lambda in_C, out_C, S, height: MBInvertedRepConvLayer( + in_C, out_C, [1, (min(height, 3), 3), (min(height, 5), 5)], S, 6), + }) + return name2ops[name] diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py new file mode 100644 index 0000000000000000000000000000000000000000..c438c6e18e0edbdc3445c6ab3fb19e48efd0b9ec --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/LightweightEdge/nas_block/proxyless.py @@ -0,0 +1,138 @@ +# Part of the implementation is borrowed and modified from ProxylessNAS, +# publicly available at https://github.com/mit-han-lab/proxylessnas +# paper linking at https://arxiv.org/pdf/1812.00332.pdf +import re +import sys +from queue import Queue + +import numpy as np +import torch +import torch.nn as nn + +from .layers import IdentityLayer, MobileInvertedResidualBlock +from .mix_ops import conv_func_by_name + + +class NasRecBackbone(nn.Module): + + def __init__(self, first_conv, blocks): + super(NasRecBackbone, self).__init__() + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + + def forward(self, x): + x = self.first_conv(x) + for block in self.blocks: + x = block(x) + return x + + def get_flops(self, x): + expected_flops = 0 + # first conv + flop = count_conv_flop(self.first_conv[0], x) + x = self.first_conv(x) + expected_flops += flop + # blocks + for mb_conv in self.blocks: + assert isinstance(mb_conv, MobileInvertedResidualBlock) + if mb_conv.shortcut is None: + shortcut_flop = 0 + else: + shortcut_flop, _ = mb_conv.shortcut.get_flops(x) + expected_flops += shortcut_flop + expected_flops += mb_conv.get_flops(x)[0] + + x = mb_conv(x) + return expected_flops + + +class CompactRecBackboneMixSE(NasRecBackbone): + + def __init__(self, first_stride, input_channel, stride_stages, + n_cell_stages, width_stages, conv_op_ids, conv_candidates, + se_candidates): + input_block_channel = 24 + first_conv = nn.Sequential( + nn.Conv2d( + input_channel, + input_block_channel, + kernel_size=(3, 3), + stride=first_stride, + padding=1, + bias=False), nn.BatchNorm2d(input_block_channel), nn.PReLU()) + + assert len(conv_op_ids) - 4 == sum(n_cell_stages) + blocks = [] + img_height = 16 + height_flag = 0 + for width, n_cell, s in zip(width_stages, n_cell_stages, + stride_stages): + for i in range(n_cell): + if i == 1: + img_height = int(img_height / 2) + + if img_height % 2 == 0: + height_flag = 1 + else: + height_flag = 0 + + if i == 0: + stride = s + else: + stride = (1, 1) + block_i = len(blocks) + conv_op = conv_func_by_name( + conv_candidates[conv_op_ids[block_i]])( + input_block_channel, width, stride, + img_height + height_flag) + if stride == (1, 1) and input_block_channel == width: + shortcut = IdentityLayer() + else: + shortcut = None + inverted_residual_block = MobileInvertedResidualBlock( + conv_op, shortcut) + blocks.append(inverted_residual_block) + input_block_channel = width + + block_i = len(blocks) + se_op = conv_func_by_name(se_candidates[conv_op_ids[block_i]])( + input_block_channel, width, stride, img_height) + + inverted_residual_block = MobileInvertedResidualBlock(se_op, None) + blocks.append(inverted_residual_block) + self.out_channel = input_block_channel + + super(CompactRecBackboneMixSE, self).__init__(first_conv, blocks) + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + +def plnas_linear_mix_se(input_channel, output_channel): + conv_candidates = [ + '5x5_MBConv2', '5x5_MBConv4', '5x5_MBConv6', '3x3_MBConv2', + '3x3_MBConv4', '3x3_MBConv6', '13_MixConv2', '13_MixConv4', + '13_MixConv6', '35_MixConv2', '35_MixConv4', '35_MixConv6', + '135_MixConv2', '135_MixConv4', '135_MixConv6', '13_LinMixConv', + '35_LinMixConv', '135_LinMixConv', '13_RepConv2', '13_RepConv4', + '13_RepConv6', '35_RepConv2', '35_RepConv4', '35_RepConv6', + '135_RepConv2', '135_RepConv4', '135_RepConv6', 'Zero' + ] + se_candidates = ['SE_2', 'SE_4', 'SE_8', 'Zero'] + + stride_stages = [(2, 2), (2, 1), (2, 1), (2, 1)] + n_cell_stages = [5, 5, 5, 5] + width_stages = [32, 64, 96, 128] + conv_op_ids = [ + 2, 23, 24, 26, 2, 2, 11, 27, 27, 27, 27, 2, 0, 2, 16, 10, 27, 2, 2, 2, + 22, 10, 27, 3 + ] + net = CompactRecBackboneMixSE(2, input_channel, stride_stages, + n_cell_stages, width_stages, conv_op_ids, + conv_candidates, se_candidates) + + return net diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/__init__.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/ocr_recognition/preprocessor.py b/runtime/Lib/site_packages/models/cv/ocr_recognition/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..b111637f982ce9f57b597caa26f3993f08ee3a3d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/ocr_recognition/preprocessor.py @@ -0,0 +1,94 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import cv2 +import numpy as np +import PIL +import torch + +from modelscope.metainfo import Preprocessors +from modelscope.preprocessors import Preprocessor, load_image +from modelscope.preprocessors.builder import PREPROCESSORS +from modelscope.utils.config import Config +from modelscope.utils.constant import Fields, ModeKeys, ModelFile + + +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.ocr_recognition) +class OCRRecognitionPreprocessor(Preprocessor): + + def __init__(self, model_dir: str, mode: str = ModeKeys.INFERENCE): + """The base constructor for all ocr recognition preprocessors. + + Args: + model_dir (str): model directory to initialize some resource + mode: The mode for the preprocessor. + """ + super().__init__(mode) + cfgs = Config.from_file( + os.path.join(model_dir, ModelFile.CONFIGURATION)) + self.do_chunking = cfgs.model.inference_kwargs.do_chunking + self.target_height = cfgs.model.inference_kwargs.img_height + self.target_width = cfgs.model.inference_kwargs.img_width + + def keepratio_resize(self, img): + cur_ratio = img.shape[1] / float(img.shape[0]) + mask_height = self.target_height + mask_width = self.target_width + if cur_ratio > float(self.target_width) / self.target_height: + cur_target_height = self.target_height + cur_target_width = self.target_width + else: + cur_target_height = self.target_height + cur_target_width = int(self.target_height * cur_ratio) + img = cv2.resize(img, (cur_target_width, cur_target_height)) + mask = np.zeros([mask_height, mask_width, 3]).astype(np.uint8) + mask[:img.shape[0], :img.shape[1], :] = img + img = mask + return img + + def __call__(self, inputs): + """process the raw input data + Args: + inputs: + - A string containing an HTTP link pointing to an image + - A string containing a local path to an image + - An image loaded in PIL or opencv directly + Returns: + outputs: the preprocessed image + """ + if not isinstance(inputs, list): + inputs = [inputs] + data_batch = [] + for item in inputs: + if isinstance(item, str): + img = np.array(load_image(item).convert('RGB')) + elif isinstance(item, PIL.Image.Image): + img = np.array(item.convert('RGB')) + elif isinstance(item, np.ndarray): + if len(item.shape) == 2: + img = cv2.cvtColor(item, cv2.COLOR_GRAY2RGB) + else: + img = item + else: + raise TypeError( + f'inputs should be either (a list of) str, PIL.Image, np.array, but got {type(item)}' + ) + img = self.keepratio_resize(img) + img = torch.FloatTensor(img) + if self.do_chunking: + chunk_img = [] + for i in range(3): + left = (300 - 48) * i + chunk_img.append(img[:, left:left + 300, :]) + merge_img = torch.cat(chunk_img, 0) + data = merge_img.view(3, self.target_height, 300, 3) / 255. + data = data.permute(0, 3, 1, 2) + else: + data = img.view(1, self.target_height, self.target_width, + 3) / 255. + data = data.permute(0, 3, 1, 2) + data_batch.append(data) + data_batch = torch.cat(data_batch, 0) + return {'image': data_batch} diff --git a/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/__init__.py b/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba7711cc79058fcca6cef221570bc4cd1ee95eab --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .vild import OpenVocabularyDetectionViLD + +else: + _import_structure = { + 'vild': ['OpenVocabularyDetectionViLD'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/vild.py b/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/vild.py new file mode 100644 index 0000000000000000000000000000000000000000..2aea0593136487a597b76dd1c486a0307241e0d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/open_vocabulary_detection_vild/vild.py @@ -0,0 +1,389 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import os +from typing import Any, Dict, Union + +import clip +import numpy as np +import tensorflow.compat.v1 as tf +import torch.cuda +from scipy.special import softmax + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_model import Model +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module( + Tasks.open_vocabulary_detection, + module_name=Models.open_vocabulary_detection_vild) +class OpenVocabularyDetectionViLD(Model): + """ + Vild: Open-Vocabulary Detection via Vision and Language Knowledge Distillation + https://arxiv.org/abs/2104.13921 + """ + + def __init__(self, model_dir, *args, **kwargs): + self.model_dir = model_dir + device_name = kwargs.get('device', 'gpu') + self._device_name = device_name + + model_path = os.path.join(model_dir, ModelFile.TF_GRAPH_FILE) + # model_path = os.path.join(model_dir, 'test_out.pb') + graph = tf.Graph() + with graph.as_default(): + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.2 + compute_graph = tf.Graph() + compute_graph.as_default() + sess = tf.Session(config=config) + + with tf.gfile.GFile(model_path, 'rb') as fid: + graph_def = tf.GraphDef() + graph_def.ParseFromString(fid.read()) + tf.import_graph_def(graph_def, name='') + self.sess = sess + + # + # clip.available_models() + self.clip, self.clip_preprocess = clip.load( + 'ViT-B/32', device='cuda:0') + + self.prompt_engineering = True + self.this_is = True + self.temperature = 100.0 + self.use_softmax = False + self.out_name = [ + 'RoiBoxes:0', 'RoiScores:0', '2ndStageBoxes:0', + '2ndStageScoresUnused:0', 'BoxOutputs:0', 'MaskOutputs:0', + 'VisualFeatOutputs:0', 'ImageInfo:0' + ] + + def __call__(self, *args, **kwargs) -> Dict[str, Any]: + return self.postprocess(self.forward(*args, **kwargs)) + + def forward(self, img: np.array, category_names: str, + **kwargs) -> Dict[str, Any]: + """ + Run the forward pass for a model. + + Returns: + Dict[str, Any]: output from the model forward pass + """ + (roi_boxes, roi_scores, detection_boxes, scores_unused, box_outputs, + detection_masks, visual_features, image_info) = self.sess.run( + self.out_name, feed_dict={'Placeholder:0': img}) + return_dict = { + 'roi_boxes': roi_boxes, + 'roi_scores': roi_scores, + 'detection_boxes': detection_boxes, + 'scores_unused': scores_unused, + 'box_outputs': box_outputs, + 'detection_masks': detection_masks, + 'visual_features': visual_features, + 'image_info': image_info, + 'category_names': category_names + } + return return_dict + + def postprocess(self, inputs: Dict[str, Any], **kwargs) -> Dict[str, Any]: + """ Model specific postprocess and convert model output to + standard model outputs. + + Args: + inputs: input data + + Return: + dict of results: a dict containing outputs of model, each + output should have the standard output name. + """ + max_boxes_to_return = 25 + nms_threshold = 0.6 + min_rpn_score_thresh = 0.9 + min_box_area = 220 + + roi_boxes = inputs['roi_boxes'] + roi_scores = inputs['roi_scores'] + detection_boxes = inputs['detection_boxes'] + scores_unused = inputs['scores_unused'] + box_outputs = inputs['box_outputs'] + detection_masks = inputs['detection_masks'] + visual_features = inputs['visual_features'] + image_info = inputs['image_info'] + category_names = inputs['category_names'] + + ################################################################# + # Preprocessing categories and get params + category_names = [x.strip() for x in category_names.split(';')] + category_names = ['background'] + category_names + categories = [{ + 'name': item, + 'id': idx + 1, + } for idx, item in enumerate(category_names)] + # category_indices = {cat['id']: cat for cat in categories} + + ################################################################# + # Obtain results and read image + + roi_boxes = np.squeeze(roi_boxes, axis=0) # squeeze + # no need to clip the boxes, already done + roi_scores = np.squeeze(roi_scores, axis=0) + + detection_boxes = np.squeeze(detection_boxes, axis=(0, 2)) + scores_unused = np.squeeze(scores_unused, axis=0) + box_outputs = np.squeeze(box_outputs, axis=0) + detection_masks = np.squeeze(detection_masks, axis=0) + visual_features = np.squeeze(visual_features, axis=0) + + # obtain image info + image_info = np.squeeze(image_info, axis=0) + image_scale = np.tile(image_info[2:3, :], (1, 2)) + + # rescale + rescaled_detection_boxes = detection_boxes / image_scale + + ################################################################# + # Filter boxes + + # Apply non-maximum suppression to detected boxes with nms threshold. + nmsed_indices = nms(detection_boxes, roi_scores, thresh=nms_threshold) + + # Compute RPN box size. + box_sizes = (rescaled_detection_boxes[:, 2] + - rescaled_detection_boxes[:, 0]) * ( + rescaled_detection_boxes[:, 3] + - rescaled_detection_boxes[:, 1]) + + # Filter out invalid rois (nmsed rois) + valid_indices = np.where( + np.logical_and( + np.isin(np.arange(len(roi_scores), dtype=int), nmsed_indices), + np.logical_and( + np.logical_not(np.all(roi_boxes == 0., axis=-1)), + np.logical_and(roi_scores >= min_rpn_score_thresh, + box_sizes > min_box_area))))[0] + # print('number of valid indices', len(valid_indices)) + + # detection_roi_scores = roi_scores[valid_indices][:max_boxes_to_return, + # ...] + detection_boxes = detection_boxes[valid_indices][:max_boxes_to_return, + ...] + detection_masks = detection_masks[valid_indices][:max_boxes_to_return, + ...] + detection_visual_feat = visual_features[ + valid_indices][:max_boxes_to_return, ...] + rescaled_detection_boxes = rescaled_detection_boxes[ + valid_indices][:max_boxes_to_return, ...] + + ################################################################# + # Compute text embeddings and detection scores, and rank results + text_features = self._build_text_embedings(categories) + + raw_scores = detection_visual_feat.dot(text_features.T) + if self.use_softmax: + scores_all = softmax(self.temperature * raw_scores, axis=-1) + else: + scores_all = raw_scores + + # Results are ranked by scores + indices = np.argsort(-np.max(scores_all, axis=1)) + # indices_fg = np.array( + # [i for i in indices if np.argmax(scores_all[i]) != 0]) + + ################################################################# + # Plot detected boxes on the input image. + ymin, xmin, ymax, xmax = np.split(rescaled_detection_boxes, 4, axis=-1) + processed_boxes = np.concatenate([xmin, ymin, xmax, ymax], axis=-1) + + n_boxes = processed_boxes.shape[0] + # print(rescaled_detection_boxes) + + categories = [] + bboxes = [] + scores = [] + labels = [] + + for anno_idx in indices[0:int(n_boxes)]: + anno_bbox = processed_boxes[anno_idx] + anno_scores = scores_all[anno_idx] + + if np.argmax(anno_scores) == 0: + continue + bboxes.append(anno_bbox) + scores.append(anno_scores[1:]) + categories.append(category_names[1:]) + labels.append(np.argmax(anno_scores) - 1) + bboxes = np.vstack(bboxes) + scores = np.vstack(scores) + + return scores, categories, bboxes + + def _build_text_embedings(self, categories): + + def processed_name(name, rm_dot=False): + # _ for lvis + # / for obj365 + res = name.replace('_', ' ').replace('/', ' or ').lower() + if rm_dot: + res = res.rstrip('.') + return res + + def article(name): + return 'an' if name[0] in 'aeiou' else 'a' + + templates = multiple_templates + + run_on_gpu = torch.cuda.is_available() + + with torch.no_grad(): + all_text_embeddings = [] + # print('Building text embeddings...') + for category in categories: + texts = [ + template.format( + processed_name(category['name'], rm_dot=True), + article=article(category['name'])) + for template in templates + ] + if self.this_is: + texts = [ + 'This is ' + text if text.startswith('a') + or text.startswith('the') else text for text in texts + ] + # tokenize + texts = clip.tokenize(texts) + if run_on_gpu: + texts = texts.cuda() + # embed with text encoder + text_embeddings = self.clip.encode_text(texts) + text_embeddings /= text_embeddings.norm(dim=-1, keepdim=True) + text_embedding = text_embeddings.mean(dim=0) + text_embedding /= text_embedding.norm() + all_text_embeddings.append(text_embedding) + all_text_embeddings = torch.stack(all_text_embeddings, dim=1) + if run_on_gpu: + all_text_embeddings = all_text_embeddings.cuda() + return all_text_embeddings.cpu().numpy().T + + +multiple_templates = [ + 'There is {article} {} in the scene.', + 'There is the {} in the scene.', + 'a photo of {article} {} in the scene.', + 'a photo of the {} in the scene.', + 'a photo of one {} in the scene.', + 'itap of {article} {}.', + 'itap of my {}.', # itap: I took a picture of + 'itap of the {}.', + 'a photo of {article} {}.', + 'a photo of my {}.', + 'a photo of the {}.', + 'a photo of one {}.', + 'a photo of many {}.', + 'a good photo of {article} {}.', + 'a good photo of the {}.', + 'a bad photo of {article} {}.', + 'a bad photo of the {}.', + 'a photo of a nice {}.', + 'a photo of the nice {}.', + 'a photo of a cool {}.', + 'a photo of the cool {}.', + 'a photo of a weird {}.', + 'a photo of the weird {}.', + 'a photo of a small {}.', + 'a photo of the small {}.', + 'a photo of a large {}.', + 'a photo of the large {}.', + 'a photo of a clean {}.', + 'a photo of the clean {}.', + 'a photo of a dirty {}.', + 'a photo of the dirty {}.', + 'a bright photo of {article} {}.', + 'a bright photo of the {}.', + 'a dark photo of {article} {}.', + 'a dark photo of the {}.', + 'a photo of a hard to see {}.', + 'a photo of the hard to see {}.', + 'a low resolution photo of {article} {}.', + 'a low resolution photo of the {}.', + 'a cropped photo of {article} {}.', + 'a cropped photo of the {}.', + 'a close-up photo of {article} {}.', + 'a close-up photo of the {}.', + 'a jpeg corrupted photo of {article} {}.', + 'a jpeg corrupted photo of the {}.', + 'a blurry photo of {article} {}.', + 'a blurry photo of the {}.', + 'a pixelated photo of {article} {}.', + 'a pixelated photo of the {}.', + 'a black and white photo of the {}.', + 'a black and white photo of {article} {}.', + 'a plastic {}.', + 'the plastic {}.', + 'a toy {}.', + 'the toy {}.', + 'a plushie {}.', + 'the plushie {}.', + 'a cartoon {}.', + 'the cartoon {}.', + 'an embroidered {}.', + 'the embroidered {}.', + 'a painting of the {}.', + 'a painting of a {}.', +] + + +def nms(dets, scores, thresh, max_dets=1000): + """Non-maximum suppression. + Args: + dets: [N, 4] + scores: [N,] + thresh: iou threshold. Float + max_dets: int. + """ + y1 = dets[:, 0] + x1 = dets[:, 1] + y2 = dets[:, 2] + x2 = dets[:, 3] + + areas = (x2 - x1) * (y2 - y1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0 and len(keep) < max_dets: + i = order[0] + keep.append(i) + + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1) + h = np.maximum(0.0, yy2 - yy1) + intersection = w * h + overlap = intersection / ( + areas[i] + areas[order[1:]] - intersection + 1e-12) + + inds = np.where(overlap <= thresh)[0] + order = order[inds + 1] + return keep diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b20f6209cf3d1dcfb5a86c738f5d330a0ac81dee --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .unifuse_model import PanoramaDepthEstimation + +else: + _import_structure = { + 'unifuse_model': ['PanoramaDepthEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/__init__.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f0fbd21b9ed5aa6f3413a447351b79ba15dd6f2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .equi import Equi +from .unifuse import UniFuse diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/equi.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/equi.py new file mode 100644 index 0000000000000000000000000000000000000000..3b8bb3d31b9ef342501131bb680bc43d813fb38c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/equi.py @@ -0,0 +1,133 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from __future__ import absolute_import, division, print_function +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn + +from .layers import Conv3x3, ConvBlock, upsample +from .mobilenet import mobilenet_v2 +from .resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + + +class Equi(nn.Module): + """ Model: Resnet based Encoder + Decoder + """ + + def __init__(self, + num_layers, + equi_h, + equi_w, + pretrained=False, + max_depth=10.0, + **kwargs): + super(Equi, self).__init__() + + self.num_layers = num_layers + self.equi_h = equi_h + self.equi_w = equi_w + self.cube_h = equi_h // 2 + + # encoder + encoder = { + 2: mobilenet_v2, + 18: resnet18, + 34: resnet34, + 50: resnet50, + 101: resnet101, + 152: resnet152 + } + + if num_layers not in encoder: + raise ValueError( + '{} is not a valid number of resnet layers'.format(num_layers)) + self.equi_encoder = encoder[num_layers](pretrained) + + self.num_ch_enc = np.array([64, 64, 128, 256, 512]) + if num_layers > 34: + self.num_ch_enc[1:] *= 4 + if num_layers < 18: + self.num_ch_enc = np.array([16, 24, 32, 96, 320]) + + # decoder + self.num_ch_dec = np.array([16, 32, 64, 128, 256]) + self.equi_dec_convs = OrderedDict() + + self.equi_dec_convs['upconv_5'] = ConvBlock(self.num_ch_enc[4], + self.num_ch_dec[4]) + + self.equi_dec_convs['deconv_4'] = ConvBlock( + self.num_ch_dec[4] + self.num_ch_enc[3], self.num_ch_dec[4]) + self.equi_dec_convs['upconv_4'] = ConvBlock(self.num_ch_dec[4], + self.num_ch_dec[3]) + + self.equi_dec_convs['deconv_3'] = ConvBlock( + self.num_ch_dec[3] + self.num_ch_enc[2], self.num_ch_dec[3]) + self.equi_dec_convs['upconv_3'] = ConvBlock(self.num_ch_dec[3], + self.num_ch_dec[2]) + + self.equi_dec_convs['deconv_2'] = ConvBlock( + self.num_ch_dec[2] + self.num_ch_enc[1], self.num_ch_dec[2]) + self.equi_dec_convs['upconv_2'] = ConvBlock(self.num_ch_dec[2], + self.num_ch_dec[1]) + + self.equi_dec_convs['deconv_1'] = ConvBlock( + self.num_ch_dec[1] + self.num_ch_enc[0], self.num_ch_dec[1]) + self.equi_dec_convs['upconv_1'] = ConvBlock(self.num_ch_dec[1], + self.num_ch_dec[0]) + + self.equi_dec_convs['deconv_0'] = ConvBlock(self.num_ch_dec[0], + self.num_ch_dec[0]) + self.equi_dec_convs['depthconv_0'] = Conv3x3(self.num_ch_dec[0], 1) + + self.equi_decoder = nn.ModuleList(list(self.equi_dec_convs.values())) + + self.sigmoid = nn.Sigmoid() + self.max_depth = nn.Parameter( + torch.tensor(max_depth), requires_grad=False) + + def forward(self, input_equi_image, input_cube_image): + + # euqi image encoding + if self.num_layers < 18: + equi_enc_feat0, equi_enc_feat1, equi_enc_feat2, equi_enc_feat3, equi_enc_feat4 \ + = self.equi_encoder(input_equi_image) + else: + x = self.equi_encoder.conv1(input_equi_image) + x = self.equi_encoder.relu(self.equi_encoder.bn1(x)) + equi_enc_feat0 = x + + x = self.equi_encoder.maxpool(x) + equi_enc_feat1 = self.equi_encoder.layer1(x) + equi_enc_feat2 = self.equi_encoder.layer2(equi_enc_feat1) + equi_enc_feat3 = self.equi_encoder.layer3(equi_enc_feat2) + equi_enc_feat4 = self.equi_encoder.layer4(equi_enc_feat3) + + # euqi image decoding + outputs = {} + + equi_x = equi_enc_feat4 + equi_x = upsample(self.equi_dec_convs['upconv_5'](equi_x)) + + equi_x = torch.cat([equi_x, equi_enc_feat3], 1) + equi_x = self.equi_dec_convs['deconv_4'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_4'](equi_x)) + + equi_x = torch.cat([equi_x, equi_enc_feat2], 1) + equi_x = self.equi_dec_convs['deconv_3'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_3'](equi_x)) + + equi_x = torch.cat([equi_x, equi_enc_feat1], 1) + equi_x = self.equi_dec_convs['deconv_2'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_2'](equi_x)) + + equi_x = torch.cat([equi_x, equi_enc_feat0], 1) + equi_x = self.equi_dec_convs['deconv_1'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_1'](equi_x)) + + equi_x = self.equi_dec_convs['deconv_0'](equi_x) + equi_depth = self.equi_dec_convs['depthconv_0'](equi_x) + outputs['pred_depth'] = self.max_depth * self.sigmoid(equi_depth) + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/layers.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..52fb3d39df7a41b3c695e615e566c9b82032589f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/layers.py @@ -0,0 +1,235 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Conv3x3(nn.Module): + """Layer to pad and convolve input + """ + + def __init__(self, in_channels, out_channels, bias=True): + super(Conv3x3, self).__init__() + + self.pad = nn.ZeroPad2d(1) + self.conv = nn.Conv2d( + int(in_channels), int(out_channels), 3, bias=bias) + + def forward(self, x): + out = self.pad(x) + out = self.conv(out) + return out + + +class ConvBlock(nn.Module): + """Layer to perform a convolution followed by ELU + """ + + def __init__(self, in_channels, out_channels, bias=True): + super(ConvBlock, self).__init__() + + self.conv = Conv3x3(in_channels, out_channels, bias) + self.nonlin = nn.ELU(inplace=True) + + def forward(self, x): + out = self.conv(x) + out = self.nonlin(out) + return out + + +def upsample(x): + """Upsample input tensor by a factor of 2 + """ + return F.interpolate(x, scale_factor=2, mode='nearest') + + +# Based on https://github.com/sunset1995/py360convert +class Cube2Equirec(nn.Module): + + def __init__(self, face_w, equ_h, equ_w): + super(Cube2Equirec, self).__init__() + ''' + face_w: int, the length of each face of the cubemap + equ_h: int, height of the equirectangular image + equ_w: int, width of the equirectangular image + ''' + + self.face_w = face_w + self.equ_h = equ_h + self.equ_w = equ_w + + # Get face id to each pixel: 0F 1R 2B 3L 4U 5D + self._equirect_facetype() + self._equirect_faceuv() + + def _equirect_facetype(self): + ''' + 0F 1R 2B 3L 4U 5D + ''' + tp = np.roll( + np.arange(4).repeat(self.equ_w // 4)[None, :].repeat( + self.equ_h, 0), 3 * self.equ_w // 8, 1) + + # Prepare ceil mask + mask = np.zeros((self.equ_h, self.equ_w // 4), bool) + idx = np.linspace(-np.pi, np.pi, self.equ_w // 4) / 4 + idx = self.equ_h // 2 - np.round( + np.arctan(np.cos(idx)) * self.equ_h / np.pi).astype(int) + for i, j in enumerate(idx): + mask[:j, i] = 1 + mask = np.roll(np.concatenate([mask] * 4, 1), 3 * self.equ_w // 8, 1) + + tp[mask] = 4 + tp[np.flip(mask, 0)] = 5 + + self.tp = tp + self.mask = mask + + def _equirect_faceuv(self): + + lon = ( + (np.linspace(0, self.equ_w - 1, num=self.equ_w, dtype=np.float32) + + 0.5) / self.equ_w - 0.5) * 2 * np.pi + lat = -( + (np.linspace(0, self.equ_h - 1, num=self.equ_h, dtype=np.float32) + + 0.5) / self.equ_h - 0.5) * np.pi + + lon, lat = np.meshgrid(lon, lat) + + coor_u = np.zeros((self.equ_h, self.equ_w), dtype=np.float32) + coor_v = np.zeros((self.equ_h, self.equ_w), dtype=np.float32) + + for i in range(4): + mask = (self.tp == i) + coor_u[mask] = 0.5 * np.tan(lon[mask] - np.pi * i / 2) + coor_v[mask] = -0.5 * np.tan( + lat[mask]) / np.cos(lon[mask] - np.pi * i / 2) + + mask = (self.tp == 4) + c = 0.5 * np.tan(np.pi / 2 - lat[mask]) + coor_u[mask] = c * np.sin(lon[mask]) + coor_v[mask] = c * np.cos(lon[mask]) + + mask = (self.tp == 5) + c = 0.5 * np.tan(np.pi / 2 - np.abs(lat[mask])) + coor_u[mask] = c * np.sin(lon[mask]) + coor_v[mask] = -c * np.cos(lon[mask]) + + # Final renormalize + coor_u = (np.clip(coor_u, -0.5, 0.5)) * 2 + coor_v = (np.clip(coor_v, -0.5, 0.5)) * 2 + + # Convert to torch tensor + self.tp = torch.from_numpy(self.tp.astype(np.float32) / 2.5 - 1) + self.coor_u = torch.from_numpy(coor_u) + self.coor_v = torch.from_numpy(coor_v) + + sample_grid = torch.stack([self.coor_u, self.coor_v, self.tp], + dim=-1).view(1, 1, self.equ_h, self.equ_w, 3) + self.sample_grid = nn.Parameter(sample_grid, requires_grad=False) + + def forward(self, cube_feat): + + bs, ch, h, w = cube_feat.shape + assert h == self.face_w and w // 6 == self.face_w + + cube_feat = cube_feat.view(bs, ch, 1, h, w) + cube_feat = torch.cat( + torch.split(cube_feat, self.face_w, dim=-1), dim=2) + + cube_feat = cube_feat.view([bs, ch, 6, self.face_w, self.face_w]) + sample_grid = torch.cat(bs * [self.sample_grid], dim=0) + equi_feat = F.grid_sample( + cube_feat, sample_grid, padding_mode='border', align_corners=True) + + return equi_feat.squeeze(2) + + +class Concat(nn.Module): + + def __init__(self, channels, **kwargs): + super(Concat, self).__init__() + self.conv = nn.Conv2d(channels * 2, channels, 1, bias=False) + self.relu = nn.ReLU(inplace=True) + + def forward(self, equi_feat, c2e_feat): + + x = torch.cat([equi_feat, c2e_feat], 1) + x = self.relu(self.conv(x)) + return x + + +# Based on https://github.com/Yeh-yu-hsuan/BiFuse/blob/master/models/FCRN.py +class BiProj(nn.Module): + + def __init__(self, channels, **kwargs): + super(BiProj, self).__init__() + + self.conv_c2e = nn.Sequential( + nn.Conv2d(channels, channels, kernel_size=3, padding=1), + nn.ReLU(inplace=True)) + self.conv_e2c = nn.Sequential( + nn.Conv2d(channels, channels, kernel_size=3, padding=1), + nn.ReLU(inplace=True)) + self.conv_mask = nn.Sequential( + nn.Conv2d(channels * 2, 1, kernel_size=1, padding=0), nn.Sigmoid()) + + def forward(self, equi_feat, c2e_feat): + aaa = self.conv_e2c(equi_feat) + tmp_equi = self.conv_c2e(c2e_feat) + mask_equi = self.conv_mask(torch.cat([aaa, tmp_equi], dim=1)) + tmp_equi = tmp_equi.clone() * mask_equi + return equi_feat + tmp_equi + + +# from https://github.com/moskomule/senet.pytorch/blob/master/senet/se_module.py +class SELayer(nn.Module): + + def __init__(self, channel, reduction=16): + super(SELayer, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), nn.Sigmoid()) + + def forward(self, x): + b, c, _, _ = x.size() + y = self.avg_pool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + return x * y.expand_as(x) + + +class CEELayer(nn.Module): + + def __init__(self, channels, SE=True): + super(CEELayer, self).__init__() + + self.res_conv1 = nn.Conv2d( + channels * 2, channels, kernel_size=1, padding=0, bias=False) + self.res_bn1 = nn.BatchNorm2d(channels) + + self.res_conv2 = nn.Conv2d( + channels, channels, kernel_size=3, padding=1, bias=False) + self.res_bn2 = nn.BatchNorm2d(channels) + self.relu = nn.ReLU(inplace=True) + + self.SE = SE + if self.SE: + self.selayer = SELayer(channels * 2) + + self.conv = nn.Conv2d(channels * 2, channels, 1, bias=False) + + def forward(self, equi_feat, c2e_feat): + + x = torch.cat([equi_feat, c2e_feat], 1) + x = self.relu(self.res_bn1(self.res_conv1(x))) + shortcut = self.res_bn2(self.res_conv2(x)) + + x = c2e_feat + shortcut + x = torch.cat([equi_feat, x], 1) + if self.SE: + x = self.selayer(x) + x = self.relu(self.conv(x)) + return x diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/mobilenet.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/mobilenet.py new file mode 100644 index 0000000000000000000000000000000000000000..d49b9a058e1a8025e9464b695b800cc06d906a6e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/mobilenet.py @@ -0,0 +1,238 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Modified from https://github.com/pytorch/vision/blob/master/torchvision/models/mobilenet.py +from torch import nn + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + + def __init__(self, + in_planes, + out_planes, + kernel_size=3, + stride=1, + groups=1, + norm_layer=None): + padding = (kernel_size - 1) // 2 + if norm_layer is None: + norm_layer = nn.BatchNorm2d + super(ConvBNReLU, self).__init__( + nn.Conv2d( + in_planes, + out_planes, + kernel_size, + stride, + padding, + groups=groups, + bias=False), norm_layer(out_planes), nn.ReLU6(inplace=True)) + + +class InvertedResidual(nn.Module): + + def __init__(self, inp, oup, stride, expand_ratio, norm_layer=None): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append( + ConvBNReLU( + inp, hidden_dim, kernel_size=1, norm_layer=norm_layer)) + layers.extend([ + # dw + ConvBNReLU( + hidden_dim, + hidden_dim, + stride=stride, + groups=hidden_dim, + norm_layer=norm_layer), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + norm_layer(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + + def __init__(self, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None, + norm_layer=None): + """ + MobileNet V2 main class + + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + norm_layer: Module specifying the normalization layer to use + + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + input_channel = 32 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len( + inverted_residual_setting[0]) != 4: + raise ValueError('inverted_residual_setting should be non-empty ' + 'or a 4-element list, got {}'.format( + inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, + round_nearest) + features = [ + ConvBNReLU(3, input_channel, stride=2, norm_layer=norm_layer) + ] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append( + block( + input_channel, + output_channel, + stride, + expand_ratio=t, + norm_layer=norm_layer)) + input_channel = output_channel + # building last several layers + # features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1, norm_layer=norm_layer)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + """ + # remove fcn as we don't need it for a depth prediction task + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + """ + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + # Cannot use "squeeze" as batch-size can be 1 => must use reshape with x.shape[0] + + st = 0 + for i in range(2): + x = self.features[st + i](x) + st = st + 2 + feat0 = x + + for i in range(2): + x = self.features[st + i](x) + st = st + 2 + feat1 = x + + for i in range(3): + x = self.features[st + i](x) + st = st + 3 + feat2 = x + + for i in range(7): + x = self.features[st + i](x) + st = st + 7 + feat3 = x + + for i in range(4): + x = self.features[st + i](x) + feat4 = x + + return feat0, feat1, feat2, feat3, feat4 + + def forward(self, x): + return self._forward_impl(x) + + +def mobilenet_v2(pretrained=False, progress=True, **kwargs): + """ + Constructs a MobileNetV2 architecture from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet (deprecated) + progress (bool): If True, displays a progress bar of the download to stderr + """ + model = MobileNetV2(**kwargs) + + return model diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/resnet.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e6b1d3f744d74fe85cbcf991607ab3128a85fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/resnet.py @@ -0,0 +1,424 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Modified from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py +import torch +import torch.nn as nn + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + + +def conv3x3(in_planes, out_planes, padding, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=padding, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + + self.conv1 = conv3x3(inplanes, planes, 1, stride) + + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + + self.conv2 = conv3x3(planes, planes, 1) + + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(identity) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition" https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + + self.conv2 = conv3x3(width, width, 1, stride, groups, dilation) + + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, + block, + layers, + num_input_images=1, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None): + super(ResNet, self).__init__() + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + self.conv1 = nn.Conv2d( + 3 * num_input_images, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias=False, + padding_mode='zeros') + + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + """ + # remove fcn as we don't need it for a depth prediction task + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + """ + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + """ + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.fc(x) + """ + + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _resnet(arch, + block, + layers, + pretrained, + progress, + num_input_images=1, + **kwargs): + model = ResNet(block, layers, num_input_images=num_input_images, **kwargs) + + return model + + +def resnet18(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet (deprecated) + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/unifuse.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/unifuse.py new file mode 100644 index 0000000000000000000000000000000000000000..5715c89bc428b01e7b52c3ae055ea03fd16e9f77 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/unifuse.py @@ -0,0 +1,219 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from __future__ import absolute_import, division, print_function +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn + +from .layers import (BiProj, CEELayer, Concat, Conv3x3, ConvBlock, + Cube2Equirec, upsample) +from .mobilenet import mobilenet_v2 +from .resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + + +class UniFuse(nn.Module): + """ UniFuse Model: Resnet based Euqi Encoder and Cube Encoder + Euqi Decoder + """ + + def __init__(self, + num_layers, + equi_h, + equi_w, + pretrained=False, + max_depth=10.0, + fusion_type='cee', + se_in_fusion=True): + super(UniFuse, self).__init__() + + self.num_layers = num_layers + self.equi_h = equi_h + self.equi_w = equi_w + self.cube_h = equi_h // 2 + + self.fusion_type = fusion_type + self.se_in_fusion = se_in_fusion + + # encoder + encoder = { + 2: mobilenet_v2, + 18: resnet18, + 34: resnet34, + 50: resnet50, + 101: resnet101, + 152: resnet152 + } + + if num_layers not in encoder: + raise ValueError( + '{} is not a valid number of resnet layers'.format(num_layers)) + self.equi_encoder = encoder[num_layers](pretrained) + self.cube_encoder = encoder[num_layers](pretrained) + + self.num_ch_enc = np.array([64, 64, 128, 256, 512]) + if num_layers > 34: + self.num_ch_enc[1:] *= 4 + + if num_layers < 18: + self.num_ch_enc = np.array([16, 24, 32, 96, 320]) + + # decoder + self.num_ch_dec = np.array([16, 32, 64, 128, 256]) + self.equi_dec_convs = OrderedDict() + self.c2e = {} + + Fusion_dict = {'cat': Concat, 'biproj': BiProj, 'cee': CEELayer} + FusionLayer = Fusion_dict[self.fusion_type] + + self.c2e['5'] = Cube2Equirec(self.cube_h // 32, self.equi_h // 32, + self.equi_w // 32) + + self.equi_dec_convs['fusion_5'] = FusionLayer( + self.num_ch_enc[4], SE=self.se_in_fusion) + self.equi_dec_convs['upconv_5'] = ConvBlock(self.num_ch_enc[4], + self.num_ch_dec[4]) + + self.c2e['4'] = Cube2Equirec(self.cube_h // 16, self.equi_h // 16, + self.equi_w // 16) + self.equi_dec_convs['fusion_4'] = FusionLayer( + self.num_ch_enc[3], SE=self.se_in_fusion) + self.equi_dec_convs['deconv_4'] = ConvBlock( + self.num_ch_dec[4] + self.num_ch_enc[3], self.num_ch_dec[4]) + self.equi_dec_convs['upconv_4'] = ConvBlock(self.num_ch_dec[4], + self.num_ch_dec[3]) + + self.c2e['3'] = Cube2Equirec(self.cube_h // 8, self.equi_h // 8, + self.equi_w // 8) + self.equi_dec_convs['fusion_3'] = FusionLayer( + self.num_ch_enc[2], SE=self.se_in_fusion) + self.equi_dec_convs['deconv_3'] = ConvBlock( + self.num_ch_dec[3] + self.num_ch_enc[2], self.num_ch_dec[3]) + self.equi_dec_convs['upconv_3'] = ConvBlock(self.num_ch_dec[3], + self.num_ch_dec[2]) + + self.c2e['2'] = Cube2Equirec(self.cube_h // 4, self.equi_h // 4, + self.equi_w // 4) + self.equi_dec_convs['fusion_2'] = FusionLayer( + self.num_ch_enc[1], SE=self.se_in_fusion) + self.equi_dec_convs['deconv_2'] = ConvBlock( + self.num_ch_dec[2] + self.num_ch_enc[1], self.num_ch_dec[2]) + self.equi_dec_convs['upconv_2'] = ConvBlock(self.num_ch_dec[2], + self.num_ch_dec[1]) + + self.c2e['1'] = Cube2Equirec(self.cube_h // 2, self.equi_h // 2, + self.equi_w // 2) + self.equi_dec_convs['fusion_1'] = FusionLayer( + self.num_ch_enc[0], SE=self.se_in_fusion) + self.equi_dec_convs['deconv_1'] = ConvBlock( + self.num_ch_dec[1] + self.num_ch_enc[0], self.num_ch_dec[1]) + self.equi_dec_convs['upconv_1'] = ConvBlock(self.num_ch_dec[1], + self.num_ch_dec[0]) + + self.equi_dec_convs['deconv_0'] = ConvBlock(self.num_ch_dec[0], + self.num_ch_dec[0]) + + self.equi_dec_convs['depthconv_0'] = Conv3x3(self.num_ch_dec[0], 1) + + self.equi_decoder = nn.ModuleList(list(self.equi_dec_convs.values())) + self.projectors = nn.ModuleList(list(self.c2e.values())) + + self.sigmoid = nn.Sigmoid() + + self.max_depth = nn.Parameter( + torch.tensor(max_depth), requires_grad=False) + + def forward(self, input_equi_image, input_cube_image): + + # euqi image encoding + + if self.num_layers < 18: + equi_enc_feat0, equi_enc_feat1, equi_enc_feat2, equi_enc_feat3, equi_enc_feat4 \ + = self.equi_encoder(input_equi_image) + else: + x = self.equi_encoder.conv1(input_equi_image) + x = self.equi_encoder.relu(self.equi_encoder.bn1(x)) + equi_enc_feat0 = x + + x = self.equi_encoder.maxpool(x) + equi_enc_feat1 = self.equi_encoder.layer1(x) + equi_enc_feat2 = self.equi_encoder.layer2(equi_enc_feat1) + equi_enc_feat3 = self.equi_encoder.layer3(equi_enc_feat2) + equi_enc_feat4 = self.equi_encoder.layer4(equi_enc_feat3) + + # cube image encoding + cube_inputs = torch.cat( + torch.split(input_cube_image, self.cube_h, dim=-1), dim=0) + + if self.num_layers < 18: + cube_enc_feat0, cube_enc_feat1, cube_enc_feat2, cube_enc_feat3, cube_enc_feat4 \ + = self.cube_encoder(cube_inputs) + else: + + x = self.cube_encoder.conv1(cube_inputs) + x = self.cube_encoder.relu(self.cube_encoder.bn1(x)) + cube_enc_feat0 = x + + x = self.cube_encoder.maxpool(x) + + cube_enc_feat1 = self.cube_encoder.layer1(x) + cube_enc_feat2 = self.cube_encoder.layer2(cube_enc_feat1) + cube_enc_feat3 = self.cube_encoder.layer3(cube_enc_feat2) + cube_enc_feat4 = self.cube_encoder.layer4(cube_enc_feat3) + + # euqi image decoding fused with cubemap features + outputs = {} + + cube_enc_feat4 = torch.cat( + torch.split(cube_enc_feat4, input_equi_image.shape[0], dim=0), + dim=-1) + c2e_enc_feat4 = self.c2e['5'](cube_enc_feat4) + fused_feat4 = self.equi_dec_convs['fusion_5'](equi_enc_feat4, + c2e_enc_feat4) + equi_x = upsample(self.equi_dec_convs['upconv_5'](fused_feat4)) + + cube_enc_feat3 = torch.cat( + torch.split(cube_enc_feat3, input_equi_image.shape[0], dim=0), + dim=-1) + c2e_enc_feat3 = self.c2e['4'](cube_enc_feat3) + fused_feat3 = self.equi_dec_convs['fusion_4'](equi_enc_feat3, + c2e_enc_feat3) + equi_x = torch.cat([equi_x, fused_feat3], 1) + equi_x = self.equi_dec_convs['deconv_4'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_4'](equi_x)) + + cube_enc_feat2 = torch.cat( + torch.split(cube_enc_feat2, input_equi_image.shape[0], dim=0), + dim=-1) + c2e_enc_feat2 = self.c2e['3'](cube_enc_feat2) + fused_feat2 = self.equi_dec_convs['fusion_3'](equi_enc_feat2, + c2e_enc_feat2) + equi_x = torch.cat([equi_x, fused_feat2], 1) + equi_x = self.equi_dec_convs['deconv_3'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_3'](equi_x)) + + cube_enc_feat1 = torch.cat( + torch.split(cube_enc_feat1, input_equi_image.shape[0], dim=0), + dim=-1) + c2e_enc_feat1 = self.c2e['2'](cube_enc_feat1) + fused_feat1 = self.equi_dec_convs['fusion_2'](equi_enc_feat1, + c2e_enc_feat1) + equi_x = torch.cat([equi_x, fused_feat1], 1) + equi_x = self.equi_dec_convs['deconv_2'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_2'](equi_x)) + + cube_enc_feat0 = torch.cat( + torch.split(cube_enc_feat0, input_equi_image.shape[0], dim=0), + dim=-1) + c2e_enc_feat0 = self.c2e['1'](cube_enc_feat0) + fused_feat0 = self.equi_dec_convs['fusion_1'](equi_enc_feat0, + c2e_enc_feat0) + equi_x = torch.cat([equi_x, fused_feat0], 1) + equi_x = self.equi_dec_convs['deconv_1'](equi_x) + equi_x = upsample(self.equi_dec_convs['upconv_1'](equi_x)) + + equi_x = self.equi_dec_convs['deconv_0'](equi_x) + + equi_depth = self.equi_dec_convs['depthconv_0'](equi_x) + outputs['pred_depth'] = self.max_depth * self.sigmoid(equi_depth) + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/util.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/util.py new file mode 100644 index 0000000000000000000000000000000000000000..546ccc928f3bd8f2c5ed9aa54bde668a831c821b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/networks/util.py @@ -0,0 +1,111 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import cv2 +import numpy as np +from scipy.ndimage import map_coordinates + + +class Equirec2Cube: + + def __init__(self, equ_h, equ_w, face_w): + ''' + equ_h: int, height of the equirectangular image + equ_w: int, width of the equirectangular image + face_w: int, the length of each face of the cubemap + ''' + + self.equ_h = equ_h + self.equ_w = equ_w + self.face_w = face_w + + self._xyzcube() + self._xyz2coor() + + # For convert R-distance to Z-depth for CubeMaps + cosmap = 1 / np.sqrt((2 * self.grid[..., 0])**2 + + (2 * self.grid[..., 1])**2 + 1) + self.cosmaps = np.concatenate(6 * [cosmap], axis=1)[..., np.newaxis] + + def _xyzcube(self): + ''' + Compute the xyz cordinates of the unit cube in [F R B L U D] format. + ''' + self.xyz = np.zeros((self.face_w, self.face_w * 6, 3), np.float32) + rng = np.linspace(-0.5, 0.5, num=self.face_w, dtype=np.float32) + self.grid = np.stack(np.meshgrid(rng, -rng), -1) + + # Front face (z = 0.5) + self.xyz[:, 0 * self.face_w:1 * self.face_w, [0, 1]] = self.grid + self.xyz[:, 0 * self.face_w:1 * self.face_w, 2] = 0.5 + + # Right face (x = 0.5) + self.xyz[:, 1 * self.face_w:2 * self.face_w, + [2, 1]] = self.grid[:, ::-1] + self.xyz[:, 1 * self.face_w:2 * self.face_w, 0] = 0.5 + + # Back face (z = -0.5) + self.xyz[:, 2 * self.face_w:3 * self.face_w, + [0, 1]] = self.grid[:, ::-1] + self.xyz[:, 2 * self.face_w:3 * self.face_w, 2] = -0.5 + + # Left face (x = -0.5) + self.xyz[:, 3 * self.face_w:4 * self.face_w, [2, 1]] = self.grid + self.xyz[:, 3 * self.face_w:4 * self.face_w, 0] = -0.5 + + # Up face (y = 0.5) + self.xyz[:, 4 * self.face_w:5 * self.face_w, + [0, 2]] = self.grid[::-1, :] + self.xyz[:, 4 * self.face_w:5 * self.face_w, 1] = 0.5 + + # Down face (y = -0.5) + self.xyz[:, 5 * self.face_w:6 * self.face_w, [0, 2]] = self.grid + self.xyz[:, 5 * self.face_w:6 * self.face_w, 1] = -0.5 + + def _xyz2coor(self): + + # x, y, z to longitude and latitude + x, y, z = np.split(self.xyz, 3, axis=-1) + lon = np.arctan2(x, z) + c = np.sqrt(x**2 + z**2) + lat = np.arctan2(y, c) + + # longitude and latitude to equirectangular coordinate + self.coor_x = (lon / (2 * np.pi) + 0.5) * self.equ_w - 0.5 + self.coor_y = (-lat / np.pi + 0.5) * self.equ_h - 0.5 + + def sample_equirec(self, e_img, order=0): + pad_u = np.roll(e_img[[0]], self.equ_w // 2, 1) + pad_d = np.roll(e_img[[-1]], self.equ_w // 2, 1) + e_img = np.concatenate([e_img, pad_d, pad_u], 0) + + return map_coordinates( + e_img, [self.coor_y, self.coor_x], order=order, mode='wrap')[..., + 0] + + def run(self, equ_img, equ_dep=None): + + h, w = equ_img.shape[:2] + if h != self.equ_h or w != self.equ_w: + equ_img = cv2.resize(equ_img, (self.equ_w, self.equ_h)) + if equ_dep is not None: + equ_dep = cv2.resize( + equ_dep, (self.equ_w, self.equ_h), + interpolation=cv2.INTER_NEAREST) + + cube_img = np.stack([ + self.sample_equirec(equ_img[..., i], order=1) + for i in range(equ_img.shape[2]) + ], + axis=-1) # noqa + + if equ_dep is not None: + cube_dep = np.stack([ + self.sample_equirec(equ_dep[..., i], order=0) + for i in range(equ_dep.shape[2]) + ], + axis=-1) # noqa + cube_dep = cube_dep * self.cosmaps + + if equ_dep is not None: + return cube_img, cube_dep + else: + return cube_img diff --git a/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/unifuse_model.py b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/unifuse_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5fe33e522fb531c1b34e8dd6609ef059de07accc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/panorama_depth_estimation/unifuse_model.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch +from torchvision import transforms + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.panorama_depth_estimation.networks import (Equi, + UniFuse) +from modelscope.models.cv.panorama_depth_estimation.networks.util import \ + Equirec2Cube +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module( + Tasks.panorama_depth_estimation, + module_name=Models.unifuse_depth_estimation) +class PanoramaDepthEstimation(TorchModel): + """ + UniFuse: Unidirectional Fusion for 360 Panorama Depth Estimation + https://arxiv.org/abs/2102.03550 + """ + + def __init__(self, model_dir: str, **kwargs): + """ + Args: + model_dir: the path of the pretrained model file + """ + super().__init__(model_dir, **kwargs) + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model {model_path}') + model_dict = torch.load(model_path, map_location=torch.device('cpu')) + Net_dict = {'UniFuse': UniFuse, 'Equi': Equi} + Net = Net_dict[model_dict['net']] + self.w = model_dict['width'] + self.h = model_dict['height'] + self.max_depth_meters = 10.0 + self.e2c = Equirec2Cube(self.h, self.w, self.h // 2) + self.to_tensor = transforms.ToTensor() + self.normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + # build model + self.model = Net( + model_dict['layers'], + model_dict['height'], + model_dict['width'], + max_depth=self.max_depth_meters, + fusion_type=model_dict['fusion'], + se_in_fusion=model_dict['se_in_fusion']) + + # load state dict + self.model.to(self.device) + model_state_dict = self.model.state_dict() + self.model.load_state_dict( + {k: v + for k, v in model_dict.items() if k in model_state_dict}) + self.model.eval() + + logger.info(f'model init done! Device:{self.device}') + + def forward(self, Inputs): + """ + Args: + Inputs: model inputs containning equirectangular panorama images and the corresponding cubmap images + The torch size of Inputs['rgb'] should be [n, 3, 512, 1024] + The torch size of Inputs['cube_rgb'] should be [n, 3, 256, 1536] + Returns: + Unifuse model outputs containing the predicted equirectangular depth images in metric + """ + equi_inputs = Inputs['rgb'].to(self.device) + cube_inputs = Inputs['cube_rgb'].to(self.device) + return self.model(equi_inputs, cube_inputs) + + def postprocess(self, Inputs): + depth_result = Inputs['pred_depth'][0] + results = {OutputKeys.DEPTHS: depth_result} + return results diff --git a/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a97b5c4e98163d24d1bcdfc4e3e9fac49ffb5ffd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model import PedestrainAttribute + +else: + _import_structure = { + 'model': ['PedestrainAttribute'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/model.py b/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/model.py new file mode 100644 index 0000000000000000000000000000000000000000..3d9a6c3fa63d2eabd55eb76e6c82d45785f7db38 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pedestrian_attribute_recognition/model.py @@ -0,0 +1,107 @@ +# The implementation is based on +# "Improving Pedestrian Attribute Recognition With Weakly-Supervised Multi-Scale Attribute-Specific Localization", +# ICCV 2019, Seoul, paper available at https://arxiv.org/abs/1910.04562 +# Poster available at https://chufengt.github.io/publication/pedestrian-attribute/iccv_poster_id2029.pdf + +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import init +from torchvision import models + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks + + +def gem(x, p=3, eps=1e-6): + return F.adaptive_avg_pool2d(F.relu(x + eps).pow(p), (1, 1)).pow(1. / p) + + +class GeM(nn.Module): + + def __init__(self, p=3, eps=1e-6): + super(GeM, self).__init__() + self.p = nn.Parameter(torch.ones(1) * p) + self.eps = eps + + def forward(self, x): + return gem(x, p=self.p, eps=self.eps) + + def __repr__(self): + return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format( + self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')' + + +def weights_init_kaiming(m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif classname.find('Linear') != -1: + init.kaiming_normal_(m.weight.data, a=0, mode='fan_out') + init.constant_(m.bias.data, 0.0) + elif classname.find('BatchNorm1d') != -1: + init.normal_(m.weight.data, 1.0, 0.02) + init.constant_(m.bias.data, 0.0) + + +def weights_init_classifier(m): + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + init.normal_(m.weight.data, std=0.001) + init.constant_(m.bias.data, 0.0) + if classname.find('Conv') != -1: + init.normal_(m.weight.data, std=0.001) + init.constant_(m.bias.data, 0.0) + + +@MODELS.register_module( + Tasks.pedestrian_attribute_recognition, + module_name=Models.pedestrian_attribute_recognition) +class PedestrainAttribute(TorchModel): + """Pedestrain Attribute Recognition model. + """ + + def __init__(self, num_classes=51, **kwargs): + """initialize the pedestrain attribute recognition model. + + Args: + num_classes (int): the number of attributes. + """ + super(PedestrainAttribute, self).__init__(**kwargs) + model_ft = models.resnet50(pretrained=False) + model_ft.avgpool = GeM() + model_ft.fc = nn.Sequential() + self.model = model_ft + self.fc = nn.Sequential(nn.Linear(2048, 512), nn.BatchNorm1d(512)) + self.fc.apply(weights_init_kaiming) + self.classifier = nn.Linear(512, num_classes) + self.classifier.apply(weights_init_classifier) + + def forward(self, x): + """ + Args: + x (`torch.Tensor`): batched image tensor, + shape of each tensor is [N, 3, H, W]. + + Return: + `labels [N, num_classes] of the attributes` + """ + + x = self.model.conv1(x) + x = self.model.bn1(x) + x = self.model.relu(x) + x = self.model.maxpool(x) + x = self.model.layer1(x) + x = self.model.layer2(x) + x = self.model.layer3(x) + x = self.model.layer4(x) + x = self.model.avgpool(x) + x = x.flatten(1) + x = self.fc(x) + x = self.classifier(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b9953f61496221e564c99f7b71b3dd86ee5ec625 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .rcp_model import SceneFlowEstimation + +else: + _import_structure = { + 'rcp_model': ['SceneFlowEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/common.py b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/common.py new file mode 100644 index 0000000000000000000000000000000000000000..ee44b51f695868edb192d09facfe673c05d70ba5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/common.py @@ -0,0 +1,446 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from . import pointnet2_utils as pointutils + +RADIUS = 2.5 + + +def index_points_group(points, knn_idx): + """ + Input: + points: input points data, [B, N, C] + knn_idx: sample index data, [B, N, K] + Return: + new_points:, indexed points data, [B, N, K, C] + """ + points_flipped = points.permute(0, 2, 1).contiguous() + new_points = pointutils.grouping_operation(points_flipped, + knn_idx.int()).permute( + 0, 2, 3, 1) + + return new_points + + +def curvature(pc, nsample=10, radius=RADIUS): + # pc: B 3 N + assert pc.shape[1] == 3 + pc = pc.permute(0, 2, 1) + + dist, kidx = pointutils.knn(nsample, pc.contiguous(), + pc.contiguous()) # (B, N, 10) + + if radius is not None: + tmp_idx = kidx[:, :, 0].unsqueeze(2).repeat(1, 1, + nsample).to(kidx.device) + kidx[dist > radius] = tmp_idx[dist > radius] + + grouped_pc = index_points_group(pc, kidx) # B N 10 3 + pc_curvature = torch.sum(grouped_pc - pc.unsqueeze(2), dim=2) / 9.0 + return pc_curvature # B N 3 + + +class PointNetSetAbstractionRatio(nn.Module): + + def __init__(self, + ratio, + radius, + nsample, + in_channel, + mlp, + group_all, + return_fps=False, + use_xyz=True, + use_act=True, + act=F.relu, + mean_aggr=False, + use_instance_norm=False): + super(PointNetSetAbstractionRatio, self).__init__() + self.ratio = ratio + self.radius = radius + self.nsample = nsample + self.group_all = group_all + self.use_xyz = use_xyz + self.use_act = use_act + self.mean_aggr = mean_aggr + self.act = act + self.mlp_convs = nn.ModuleList() + self.mlp_bns = nn.ModuleList() + last_channel = (in_channel + 3) if use_xyz else in_channel + for out_channel in mlp: + self.mlp_convs.append( + nn.Conv2d(last_channel, out_channel, 1, bias=False)) + if use_instance_norm: + self.mlp_bns.append( + nn.InstanceNorm2d(out_channel, affine=True)) + else: + self.mlp_bns.append(nn.BatchNorm2d(out_channel)) + + last_channel = out_channel + + if group_all: + self.queryandgroup = pointutils.GroupAll(self.use_xyz) + else: + self.queryandgroup = pointutils.QueryAndGroup( + radius, nsample, self.use_xyz) + self.return_fps = return_fps + + def forward(self, xyz, points, fps_idx=None): + """ + Input: + xyz: input points position data, [B, C, N] + points: input points data, [B, D, N] + Return: + new_xyz: sampled points position data, [B, C, S] + new_points: sample points feature data, [B, D', S] + """ + B, C, N = xyz.shape + npoint = int(N * self.ratio) + + xyz = xyz.contiguous() + xyz_t = xyz.permute(0, 2, 1).contiguous() + + if (self.group_all is False) and (npoint != -1): + if fps_idx is None: + fps_idx = pointutils.furthest_point_sample(xyz_t, + npoint) # [B, N] + new_xyz = pointutils.gather_operation(xyz, fps_idx) # [B, C, N] + else: + new_xyz = xyz + new_points, _ = self.queryandgroup(xyz_t, + new_xyz.transpose(2, + 1).contiguous(), + points) # [B, 3+C, N, S] + + # new_xyz: sampled points position data, [B, C, npoint] + # new_points: sampled points data, [B, C+D, npoint, nsample] + for i, conv in enumerate(self.mlp_convs): + if self.use_act: + bn = self.mlp_bns[i] + new_points = self.act(bn(conv(new_points))) + else: + new_points = conv(new_points) + + if self.mean_aggr: + new_points = torch.mean(new_points, -1) + else: + new_points = torch.max(new_points, -1)[0] + + if self.return_fps: + return new_xyz, new_points, fps_idx + else: + return new_xyz, new_points + + +class PointNetSetAbstraction(nn.Module): + + def __init__(self, + npoint, + radius, + nsample, + in_channel, + mlp, + group_all, + return_fps=False, + use_xyz=True, + use_act=True, + act=F.relu, + mean_aggr=False, + use_instance_norm=False): + super(PointNetSetAbstraction, self).__init__() + self.npoint = npoint + self.radius = radius + self.nsample = nsample + self.group_all = group_all + self.use_xyz = use_xyz + self.use_act = use_act + self.mean_aggr = mean_aggr + self.act = act + self.mlp_convs = nn.ModuleList() + self.mlp_bns = nn.ModuleList() + last_channel = (in_channel + 3) if use_xyz else in_channel + for out_channel in mlp: + self.mlp_convs.append( + nn.Conv2d(last_channel, out_channel, 1, bias=False)) + if use_instance_norm: + self.mlp_bns.append( + nn.InstanceNorm2d(out_channel, affine=True)) + else: + self.mlp_bns.append(nn.BatchNorm2d(out_channel)) + + last_channel = out_channel + + if group_all: + self.queryandgroup = pointutils.GroupAll(self.use_xyz) + else: + self.queryandgroup = pointutils.QueryAndGroup( + radius, nsample, self.use_xyz) + self.return_fps = return_fps + + def forward(self, xyz, points, fps_idx=None): + """ + Input: + xyz: input points position data, [B, C, N] + points: input points data, [B, D, N] + Return: + new_xyz: sampled points position data, [B, S, C] + new_points: sample points feature data, [B, S, D'] + """ + # device = xyz.device + B, C, N = xyz.shape + xyz = xyz.contiguous() + xyz_t = xyz.permute(0, 2, 1).contiguous() + + if (self.group_all is False) and (self.npoint != -1): + if fps_idx is None: + fps_idx = pointutils.furthest_point_sample( + xyz_t, self.npoint) # [B, N] + new_xyz = pointutils.gather_operation(xyz, fps_idx) # [B, C, N] + else: + new_xyz = xyz + new_points, _ = self.queryandgroup(xyz_t, + new_xyz.transpose(2, + 1).contiguous(), + points) # [B, 3+C, N, S] + + # new_xyz: sampled points position data, [B, C, npoint] + # new_points: sampled points data, [B, C+D, npoint, nsample] + for i, conv in enumerate(self.mlp_convs): + if self.use_act: + bn = self.mlp_bns[i] + new_points = self.act(bn(conv(new_points))) + else: + new_points = conv(new_points) + + if self.mean_aggr: + new_points = torch.mean(new_points, -1) + else: + new_points = torch.max(new_points, -1)[0] + + if self.return_fps: + return new_xyz, new_points, fps_idx + else: + return new_xyz, new_points + + +class PointNetFeaturePropogation(nn.Module): + + def __init__(self, in_channel, mlp, learn_mask=False, nsample=3): + super(PointNetFeaturePropogation, self).__init__() + self.mlp_convs = nn.ModuleList() + self.mlp_bns = nn.ModuleList() + self.apply_mlp = mlp is not None + last_channel = in_channel + self.nsample = nsample + if self.apply_mlp: + for out_channel in mlp: + self.mlp_convs.append(nn.Conv1d(last_channel, out_channel, 1)) + self.mlp_bns.append(nn.BatchNorm1d(out_channel)) + last_channel = out_channel + + if learn_mask: + self.queryandgroup = pointutils.QueryAndGroup( + None, 9, use_xyz=True) + last_channel = (128 + 3) + for out_channel in [32, 1]: + self.mlp_convs.append( + nn.Conv2d(last_channel, out_channel, 1, bias=False)) + self.mlp_bns.append(nn.BatchNorm2d(out_channel)) + last_channel = out_channel + + def forward(self, pos1, pos2, feature1, feature2, hidden=None): + """ + Input: + pos1: input points position data, [B, C, N] + pos2: sampled input points position data, [B, C, S] + feature1: input points data, [B, D, N] + feature2: input points data, [B, D, S] + Return: + feat_new: upsampled points data, [B, D', N] + """ + pos1_t = pos1.permute(0, 2, 1).contiguous() + pos2_t = pos2.permute(0, 2, 1).contiguous() + B, C, N = pos1.shape + + if hidden is None: + if self.nsample == 3: + dists, idx = pointutils.three_nn(pos1_t, pos2_t) + else: + dists, idx = pointutils.knn(self.nsample, pos1_t, pos2_t) + dists[dists < 1e-10] = 1e-10 + weight = 1.0 / dists + weight = weight / torch.sum(weight, -1, keepdim=True) # [B,N,3] + interpolated_feat = torch.sum( + pointutils.grouping_operation(feature2, idx) + * weight.view(B, 1, N, self.nsample), + dim=-1) # [B,C,N,3] + else: + dist, idx = pointutils.knn(9, pos1_t, pos2_t) + + new_feat, _ = self.queryandgroup(pos2_t, pos1_t, + hidden) # [B, 3+C, N, 9] + + for i, conv in enumerate(self.mlp_convs): + new_feat = conv(new_feat) + weight = torch.softmax(new_feat, dim=-1) # [B, 1, N, 9] + interpolated_feat = torch.sum( + pointutils.grouping_operation(feature2, idx) * weight, + dim=-1) # [B, C, N] + + if feature1 is not None: + feat_new = torch.cat([interpolated_feat, feature1], 1) + else: + feat_new = interpolated_feat + + if self.apply_mlp: + for i, conv in enumerate(self.mlp_convs): + bn = self.mlp_bns[i] + feat_new = F.relu(bn(conv(feat_new))) + return feat_new + + +class Sinkhorn(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, corr, epsilon, gamma, max_iter): + # Early return if no iteration + if max_iter == 0: + return corr + + # Init. of Sinkhorn algorithm + power = gamma / (gamma + epsilon) + a = ( + torch.ones((corr.shape[0], corr.shape[1], 1), + device=corr.device, + dtype=corr.dtype) / corr.shape[1]) + prob1 = ( + torch.ones((corr.shape[0], corr.shape[1], 1), + device=corr.device, + dtype=corr.dtype) / corr.shape[1]) + prob2 = ( + torch.ones((corr.shape[0], corr.shape[2], 1), + device=corr.device, + dtype=corr.dtype) / corr.shape[2]) + + # Sinkhorn algorithm + for _ in range(max_iter): + # Update b + KTa = torch.bmm(corr.transpose(1, 2), a) + b = torch.pow(prob2 / (KTa + 1e-8), power) + # Update a + Kb = torch.bmm(corr, b) + a = torch.pow(prob1 / (Kb + 1e-8), power) + + # Transportation map + T = torch.mul(torch.mul(a, corr), b.transpose(1, 2)) + + return T + + +class PointWiseOptimLayer(nn.Module): + + def __init__(self, nsample, radius, in_channel, mlp, use_curvature=True): + super().__init__() + self.nsample = nsample + self.radius = radius + self.use_curvature = use_curvature + + self.pos_embed = nn.Sequential( + nn.Conv1d(3, 32, 1), nn.ReLU(inplace=True), nn.Conv1d(32, 64, 1)) + + self.qk_net = nn.Sequential( + nn.Conv1d(in_channel + 64, in_channel + 64, 1), + nn.ReLU(inplace=True), + nn.Conv1d(in_channel + 64, in_channel + 64, 1)) + if self.use_curvature: + self.curvate_net = nn.Sequential( + nn.Conv1d(3, 32, 1), nn.ReLU(inplace=True), + nn.Conv1d(32, 32, 1)) + self.mlp_conv = nn.Conv1d( + in_channel + 64 + 32, mlp[-1], 1, bias=True) + else: + self.mlp_conv = nn.Conv1d(in_channel + 64, mlp[-1], 1, bias=True) + + def forward(self, + pos1, + pos2, + feature1, + feature2, + nsample, + radius=None, + pos1_raw=None, + return_score=False): + """ + Input: + pos1: (batch_size, 3, npoint) + pos2: (batch_size, 3, npoint) + feature1: (batch_size, channel, npoint) + feature2: (batch_size, channel, npoint) + Output: + pos1: (batch_size, 3, npoint) + cost: (batch_size, channel, npoint) + """ + pos1_t = pos1.permute(0, 2, 1).contiguous() + pos2_t = pos2.permute(0, 2, 1).contiguous() + self.nsample = nsample + self.radius = radius + + dist, idx = pointutils.knn(self.nsample, pos1_t, pos2_t) # [B, N, K] + if self.radius is not None: + tmp_idx = idx[:, :, + 0].unsqueeze(2).repeat(1, 1, + self.nsample).to(idx.device) + idx[dist > self.radius] = tmp_idx[dist > self.radius] + + pos1_embed_norm = self.pos_embed(pos1) + pos2_embed_norm = self.pos_embed(pos2) # [B, C1, N] + + feat1_w_pos = torch.cat([feature1, pos1_embed_norm], dim=1) + feat2_w_pos = torch.cat([feature2, pos2_embed_norm], + dim=1) # [B, C1+C2, N] + + feat1_w_pos = self.qk_net(feat1_w_pos) + feat2_w_pos = self.qk_net(feat2_w_pos) # [B, C1+C2, N] + + feat2_grouped = pointutils.grouping_operation(feat2_w_pos, + idx) # [B, C1+C2, N, S] + + score = torch.softmax( + feat1_w_pos.unsqueeze(-1) * feat2_grouped * 1. + / math.sqrt(feat1_w_pos.shape[1]), + dim=-1) # [B, C1+C2, N, S] + cost = (score * (feat1_w_pos.unsqueeze(-1) - feat2_grouped)**2).sum( + dim=-1) # [B, C1+C2, N] + + if self.use_curvature: + curvate1_raw = curvature(pos1_raw).permute(0, 2, 1) # [B, 3, N] + curvate1 = curvature(pos1).permute(0, 2, 1) # [B, 3, N] + curvate_cost = self.curvate_net(curvate1_raw) - self.curvate_net( + curvate1) + curvate_cost = curvate_cost**2 + cost = self.mlp_conv(torch.cat([cost, curvate_cost], + dim=1)) # [B, C, N] + else: + cost = self.mlp_conv(cost) # [B, C, N] + + if return_score: + pos2_grouped = pointutils.grouping_operation(pos2, + idx) # [B, 3, N, S] + # [B, N, K] + index = (dist > self.radius).sum( + dim=2, keepdim=True).float() > (dist.shape[2] - 0.1 + ) # [B, N, 1] + index = index.unsqueeze(1).repeat(1, score.shape[1], 1, + dist.shape[2]) # [B, N, K] + score_tmp = score.clone() + score_tmp[index] = 0.0 + score = score_tmp + return pos1, cost, score, pos2_grouped + else: + return pos1, cost diff --git a/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/pointnet2_utils.py b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/pointnet2_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddc66229f6294501b2c494f4bd08df388b8e0f11 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/pointnet2_utils.py @@ -0,0 +1,360 @@ +# The implementation is adopt from PointNet2, open-sourced under MIT license, +# made publicy available at https://github.com/sshaoshuai/Pointnet2.PyTorch +from typing import Tuple + +import pointnet2_cuda as pointnet2 +import torch +import torch.nn as nn +from torch.autograd import Function, Variable + + +class FurthestPointSampling(Function): + + @staticmethod + def forward(ctx, xyz: torch.Tensor, npoint: int) -> torch.Tensor: + """ + Uses iterative furthest point sampling to select a set of npoint features that have the largest + minimum distance + :param ctx: + :param xyz: (B, N, 3) where N > npoint + :param npoint: int, number of features in the sampled set + :return: + output: (B, npoint) tensor containing the set + """ + assert xyz.is_contiguous() + + B, N, _ = xyz.size() + output = torch.cuda.IntTensor(B, npoint) + temp = torch.cuda.FloatTensor(B, N).fill_(1e10) + + pointnet2.furthest_point_sampling_wrapper(B, N, npoint, xyz, temp, + output) + return output + + @staticmethod + def backward(xyz, a=None): + return None, None + + +furthest_point_sample = FurthestPointSampling.apply + + +class GatherOperation(Function): + + @staticmethod + def forward(ctx, features: torch.Tensor, + idx: torch.Tensor) -> torch.Tensor: + """ + :param ctx: + :param features: (B, C, N) + :param idx: (B, npoint) index tensor of the features to gather + :return: + output: (B, C, npoint) + """ + assert features.is_contiguous() + assert idx.is_contiguous() + + B, npoint = idx.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, npoint) + + pointnet2.gather_points_wrapper(B, C, N, npoint, features, idx, output) + + ctx.for_backwards = (idx, C, N) + return output + + @staticmethod + def backward(ctx, grad_out): + idx, C, N = ctx.for_backwards + B, npoint = idx.size() + + grad_features = Variable(torch.cuda.FloatTensor(B, C, N).zero_()) + grad_out_data = grad_out.data.contiguous() + pointnet2.gather_points_grad_wrapper(B, C, N, npoint, grad_out_data, + idx, grad_features.data) + return grad_features, None + + +gather_operation = GatherOperation.apply + + +class KNN(Function): + + @staticmethod + def forward(ctx, k: int, unknown: torch.Tensor, + known: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Find the three nearest neighbors of unknown in known + :param ctx: + :param unknown: (B, N, 3) + :param known: (B, M, 3) + :return: + dist: (B, N, k) l2 distance to the three nearest neighbors + idx: (B, N, k) index of 3 nearest neighbors + """ + assert unknown.is_contiguous() + assert known.is_contiguous() + + B, N, _ = unknown.size() + m = known.size(1) + dist2 = torch.cuda.FloatTensor(B, N, k) + idx = torch.cuda.IntTensor(B, N, k) + + pointnet2.knn_wrapper(B, N, m, k, unknown, known, dist2, idx) + return torch.sqrt(dist2), idx + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None, None + + +knn = KNN.apply + + +class ThreeNN(Function): + + @staticmethod + def forward(ctx, unknown: torch.Tensor, + known: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Find the three nearest neighbors of unknown in known + :param ctx: + :param unknown: (B, N, 3) + :param known: (B, M, 3) + :return: + dist: (B, N, 3) l2 distance to the three nearest neighbors + idx: (B, N, 3) index of 3 nearest neighbors + """ + assert unknown.is_contiguous() + assert known.is_contiguous() + + B, N, _ = unknown.size() + m = known.size(1) + dist2 = torch.cuda.FloatTensor(B, N, 3) + idx = torch.cuda.IntTensor(B, N, 3) + + pointnet2.three_nn_wrapper(B, N, m, unknown, known, dist2, idx) + return torch.sqrt(dist2), idx + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None + + +three_nn = ThreeNN.apply + + +class ThreeInterpolate(Function): + + @staticmethod + def forward(ctx, features: torch.Tensor, idx: torch.Tensor, + weight: torch.Tensor) -> torch.Tensor: + """ + Performs weight linear interpolation on 3 features + :param ctx: + :param features: (B, C, M) Features descriptors to be interpolated from + :param idx: (B, n, 3) three nearest neighbors of the target features in features + :param weight: (B, n, 3) weights + :return: + output: (B, C, N) tensor of the interpolated features + """ + assert features.is_contiguous() + assert idx.is_contiguous() + assert weight.is_contiguous() + + B, c, m = features.size() + n = idx.size(1) + ctx.three_interpolate_for_backward = (idx, weight, m) + output = torch.cuda.FloatTensor(B, c, n) + + pointnet2.three_interpolate_wrapper(B, c, m, n, features, idx, weight, + output) + return output + + @staticmethod + def backward( + ctx, grad_out: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + :param ctx: + :param grad_out: (B, C, N) tensor with gradients of outputs + :return: + grad_features: (B, C, M) tensor with gradients of features + None: + None: + """ + idx, weight, m = ctx.three_interpolate_for_backward + B, c, n = grad_out.size() + + grad_features = Variable(torch.cuda.FloatTensor(B, c, m).zero_()) + grad_out_data = grad_out.data.contiguous() + + pointnet2.three_interpolate_grad_wrapper(B, c, n, m, grad_out_data, + idx, weight, + grad_features.data) + return grad_features, None, None + + +three_interpolate = ThreeInterpolate.apply + + +class GroupingOperation(Function): + + @staticmethod + def forward(ctx, features: torch.Tensor, + idx: torch.Tensor) -> torch.Tensor: + """ + :param ctx: + :param features: (B, C, N) tensor of features to group + :param idx: (B, npoint, nsample) tensor containing the indicies of features to group with + :return: + output: (B, C, npoint, nsample) tensor + """ + assert features.is_contiguous() + assert idx.is_contiguous() + idx = idx.int() + B, nfeatures, nsample = idx.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, nfeatures, nsample) + + pointnet2.group_points_wrapper(B, C, N, nfeatures, nsample, features, + idx, output) + + ctx.for_backwards = (idx, N) + return output + + @staticmethod + def backward(ctx, + grad_out: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + :param ctx: + :param grad_out: (B, C, npoint, nsample) tensor of the gradients of the output from forward + :return: + grad_features: (B, C, N) gradient of the features + """ + idx, N = ctx.for_backwards + + B, C, npoint, nsample = grad_out.size() + grad_features = Variable(torch.cuda.FloatTensor(B, C, N).zero_()) + + grad_out_data = grad_out.data.contiguous() + pointnet2.group_points_grad_wrapper(B, C, N, npoint, nsample, + grad_out_data, idx, + grad_features.data) + return grad_features, None + + +grouping_operation = GroupingOperation.apply + + +class BallQuery(Function): + + @staticmethod + def forward(ctx, radius: float, nsample: int, xyz: torch.Tensor, + new_xyz: torch.Tensor) -> torch.Tensor: + """ + :param ctx: + :param radius: float, radius of the balls + :param nsample: int, maximum number of features in the balls + :param xyz: (B, N, 3) xyz coordinates of the features + :param new_xyz: (B, npoint, 3) centers of the ball query + :return: + idx: (B, npoint, nsample) tensor with the indicies of the features that form the query balls + """ + assert new_xyz.is_contiguous() + assert xyz.is_contiguous() + + B, N, _ = xyz.size() + npoint = new_xyz.size(1) + idx = torch.cuda.IntTensor(B, npoint, nsample).zero_() + + pointnet2.ball_query_wrapper(B, N, npoint, radius, nsample, new_xyz, + xyz, idx) + return idx + + @staticmethod + def backward(ctx, a=None): + return None, None, None, None + + +ball_query = BallQuery.apply + + +class QueryAndGroup(nn.Module): + + def __init__(self, radius: float, nsample: int, use_xyz: bool = True): + """ + :param radius: float, radius of ball + :param nsample: int, maximum number of features to gather in the ball + :param use_xyz: + """ + super().__init__() + self.radius, self.nsample, self.use_xyz = radius, nsample, use_xyz + + def forward(self, + xyz: torch.Tensor, + new_xyz: torch.Tensor, + features: torch.Tensor = None) -> Tuple[torch.Tensor]: + """ + :param xyz: (B, N, 3) xyz coordinates of the features + :param new_xyz: (B, npoint, 3) centroids + :param features: (B, C, N) descriptors of the features + :return: + new_features: (B, 3 + C, npoint, nsample) + """ + # idx = ball_query(self.radius, self.nsample, xyz, new_xyz) + B, N, C = new_xyz.shape + dist, idx = knn(self.nsample, new_xyz, xyz) + if self.radius is not None: + tmp_idx = idx[:, :, + 0].unsqueeze(2).repeat(1, 1, + self.nsample).to(idx.device) + idx[dist > self.radius] = tmp_idx[dist > self.radius] + xyz_trans = xyz.transpose(1, 2).contiguous() + grouped_xyz = grouping_operation(xyz_trans, + idx) # (B, 3, npoint, nsample) + grouped_xyz -= new_xyz.transpose(1, 2).unsqueeze(-1) + + if features is not None: + grouped_features = grouping_operation(features, idx) + if self.use_xyz: + new_features = torch.cat([grouped_xyz, grouped_features], + dim=1) # (B, C + 3, npoint, nsample) + else: + new_features = grouped_features + else: + assert self.use_xyz, 'Cannot have not features and not use xyz as a feature!' + new_features = grouped_xyz + + return new_features, grouped_xyz + + +class GroupAll(nn.Module): + + def __init__(self, use_xyz: bool = True): + super().__init__() + self.use_xyz = use_xyz + + def forward(self, + xyz: torch.Tensor, + new_xyz: torch.Tensor, + features: torch.Tensor = None): + """ + :param xyz: (B, N, 3) xyz coordinates of the features + :param new_xyz: ignored + :param features: (B, C, N) descriptors of the features + :return: + new_features: (B, C + 3, 1, N) + """ + grouped_xyz = xyz.transpose(1, 2).unsqueeze(2) + if features is not None: + grouped_features = features.unsqueeze(2) + if self.use_xyz: + new_features = torch.cat([grouped_xyz, grouped_features], + dim=1) # (B, 3 + C, 1, N) + else: + new_features = grouped_features + else: + new_features = grouped_xyz + + return new_features, grouped_xyz diff --git a/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/rcp_model.py b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/rcp_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8d2e5b2e51d1550bfcfc5572e12c5a5058d65251 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/rcp_model.py @@ -0,0 +1,64 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .sf_rcp import SF_RCP + +logger = get_logger() + + +@MODELS.register_module( + Tasks.pointcloud_sceneflow_estimation, + module_name=Models.rcp_sceneflow_estimation) +class SceneFlowEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + assert torch.cuda.is_available( + ), 'current model only support run in gpu' + + # build model + self.model = SF_RCP( + npoint=8192, + use_instance_norm=False, + model_name='SF_RCP', + use_insrance_norm=False, + use_curvature=True) + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + logger.info(f'load ckpt from:{model_path}') + + checkpoint = torch.load(model_path, map_location='cpu') + + self.model.load_state_dict({k: v for k, v in checkpoint.items()}) + self.model.cuda() + self.model.eval() + + def forward(self, Inputs): + + return self.model(Inputs['pcd1'], Inputs['pcd2'], Inputs['pcd1'], + Inputs['pcd2'])[-1] + + def postprocess(self, Inputs): + output = Inputs['output'] + + results = {OutputKeys.OUTPUT: output.detach().cpu().numpy()[0]} + + return results + + def inference(self, data): + results = self.forward(data) + + return results diff --git a/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/sf_rcp.py b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/sf_rcp.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ec6057e1947a23ce090aa9d9437840500568a9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/pointcloud_sceneflow_estimation/sf_rcp.py @@ -0,0 +1,523 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .common import (PointNetFeaturePropogation, PointNetSetAbstraction, + PointWiseOptimLayer, Sinkhorn) + + +class FeatureMatching(nn.Module): + + def __init__(self, npoint, use_instance_norm, supporth_th, feature_norm, + max_iter): + super(FeatureMatching, self).__init__() + self.support_th = supporth_th**2 # 10m + self.feature_norm = feature_norm + self.max_iter = max_iter + # Mass regularisation + self.gamma = torch.nn.Parameter(torch.zeros(1)) + # Entropic regularisation + self.epsilon = torch.nn.Parameter(torch.zeros(1)) + self.sinkhorn = Sinkhorn() + + self.extract_glob = FeatureExtractionGlobal(npoint, use_instance_norm) + + # upsample flow + self.fp0 = PointNetFeaturePropogation(in_channel=3, mlp=[]) + self.sa1 = PointNetSetAbstraction( + npoint=int(npoint / 16), + radius=None, + nsample=16, + in_channel=3, + mlp=[32, 32, 64], + group_all=False, + use_instance_norm=use_instance_norm) + self.fp1 = PointNetFeaturePropogation(in_channel=64, mlp=[]) + self.sa2 = PointNetSetAbstraction( + npoint=int(npoint / 8), + radius=None, + nsample=16, + in_channel=64, + mlp=[64, 64, 128], + group_all=False, + use_instance_norm=use_instance_norm) + self.fp2 = PointNetFeaturePropogation(in_channel=128, mlp=[]) + + self.flow_regressor = FlowRegressor(npoint, use_instance_norm) + self.flow_up_sample = PointNetFeaturePropogation(in_channel=3, mlp=[]) + + def upsample_flow(self, pc1_l, pc1_l_glob, flow_inp): + """ + flow_inp: [B, N, 3] + return: [B, 3, N] + """ + + flow_inp = flow_inp.permute(0, 2, 1).contiguous() # [B, 3, N] + + flow_feat = self.fp0(pc1_l_glob['s16'], pc1_l_glob['s32'], None, + flow_inp) + _, corr_feats_l2 = self.sa1(pc1_l_glob['s16'], flow_feat) + + flow_feat = self.fp1(pc1_l_glob['s8'], pc1_l_glob['s16'], None, + corr_feats_l2) + _, flow_feat = self.sa2(pc1_l_glob['s8'], flow_feat) + + flow_feat = self.fp2(pc1_l['s4'], pc1_l_glob['s8'], None, flow_feat) + + flow, flow_lr = self.flow_regressor(pc1_l, flow_feat) + + flow_up = self.flow_up_sample(pc1_l['s1'], pc1_l_glob['s32'], None, + flow_inp) + flow_lr_up = self.flow_up_sample(pc1_l['s4'], pc1_l_glob['s32'], None, + flow_inp) + + flow, flow_lr = flow + flow_up, flow_lr + flow_lr_up + + return flow, flow_lr + + def calc_feats_corr(self, pcloud1, pcloud2, feature1, feature2, norm): + """ + pcloud1, pcloud2: [B, N, 3] + feature1, feature2: [B, N, C] + """ + if norm: + feature1 = feature1 / torch.sqrt( + torch.sum(feature1**2, -1, keepdim=True) + 1e-6) + feature2 = feature2 / torch.sqrt( + torch.sum(feature2**2, -1, keepdim=True) + 1e-6) + corr_mat = torch.bmm(feature1, + feature2.transpose(1, 2)) # [B, N1, N2] + else: + corr_mat = torch.bmm(feature1, feature2.transpose( + 1, 2)) / feature1.shape[2]**.5 # [B, N1, N2] + + if self.support_th is not None: + distance_matrix = torch.sum( + pcloud1**2, -1, keepdim=True) # [B, N1, 1] + distance_matrix = distance_matrix + torch.sum( + pcloud2**2, -1, keepdim=True).transpose(1, 2) # [B, N1, N2] + distance_matrix = distance_matrix - 2 * torch.bmm( + pcloud1, pcloud2.transpose(1, 2)) # [B, N1, N2] + support = (distance_matrix < self.support_th) # [B, N1, N2] + support = support.float() + else: + support = torch.ones_like(corr_mat) + return corr_mat, support + + def calc_corr_mat(self, pcloud1, pcloud2, feature1, feature2): + """ + pcloud1, pcloud2: [B, N, 3] + feature1, feature2: [B, N, C] + corr_mat: [B, N1, N2] + """ + epsilon = torch.exp(self.epsilon) + 0.03 + corr_mat, support = self.calc_feats_corr( + pcloud1, pcloud2, feature1, feature2, norm=self.feature_norm) + C = 1.0 - corr_mat + corr_mat = torch.exp(-C / epsilon) * support + return corr_mat + + def get_flow_init(self, pcloud1, pcloud2, feats1, feats2): + """ + pcloud1, pcloud2: [B, 3, N] + feats1, feats2: [B, C, N] + """ + + corr_mat = self.calc_corr_mat( + pcloud1.permute(0, 2, 1), pcloud2.permute(0, 2, 1), + feats1.permute(0, 2, 1), feats2.permute(0, 2, 1)) + + corr_mat = self.sinkhorn(corr_mat, + torch.exp(self.epsilon) + 0.03, self.gamma, + self.max_iter) + + row_sum = corr_mat.sum(-1, keepdim=True) # [B, N1, 1] + flow_init = (corr_mat @ pcloud2.permute(0, 2, 1).contiguous()) / ( + row_sum + 1e-6) - pcloud1.permute(0, 2, + 1).contiguous() # [B, N1, 3] + + return flow_init + + def forward(self, pc1_l, pc2_l, feats1, feats2): + """ + pc1_l, pc2_l: dict([B, 3, N]) + feats1, feats2: [B, C, N] + """ + pc1_l_glob, feats1_glob = self.extract_glob(pc1_l['s4'], feats1) + pc2_l_glob, feats2_glob = self.extract_glob(pc2_l['s4'], feats2) + + flow_init_s32 = self.get_flow_init(pc1_l_glob['s32'], + pc2_l_glob['s32'], feats1_glob, + feats2_glob) + + flow_init, flow_init_s4 = self.upsample_flow(pc1_l, pc1_l_glob, + flow_init_s32) + + return flow_init, flow_init_s4 + + +class FlowRegressor(nn.Module): + + def __init__(self, npoint, use_instance_norm, input_dim=128, nsample=32): + super(FlowRegressor, self).__init__() + self.sa1 = PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=nsample, + in_channel=input_dim, + mlp=[input_dim, input_dim], + group_all=False, + use_instance_norm=use_instance_norm) + self.sa2 = PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=nsample, + in_channel=input_dim, + mlp=[input_dim, input_dim], + group_all=False, + use_instance_norm=use_instance_norm) + + self.fc = nn.Sequential( + nn.Linear(input_dim, input_dim), nn.ReLU(inplace=True), + nn.Linear(input_dim, 3)) + + self.up_sample = PointNetFeaturePropogation(in_channel=3, mlp=[]) + + def forward(self, pc1_l, feats): + """ + pc1_l: dict([B, 3, N]) + feats: [B, C, N] + return: [B, 3, N] + """ + _, x = self.sa1(pc1_l['s4'], feats) + _, x = self.sa2(pc1_l['s4'], x) + x = x.permute(0, 2, 1).contiguous() # [B, N, C] + x = self.fc(x) + flow_lr = x.permute(0, 2, 1).contiguous() # [B, 3, N] + + flow = self.up_sample(pc1_l['s1'], pc1_l['s4'], None, + flow_lr) # [B, 3, N] + + return flow, flow_lr + + +class FeatureExtractionGlobal(nn.Module): + + def __init__(self, npoint, use_instance_norm): + super(FeatureExtractionGlobal, self).__init__() + self.sa1 = PointNetSetAbstraction( + npoint=int(npoint / 8), + radius=None, + nsample=32, + in_channel=64, + mlp=[128, 128, 128], + group_all=False, + use_instance_norm=use_instance_norm) + self.sa2 = PointNetSetAbstraction( + npoint=int(npoint / 16), + radius=None, + nsample=24, + in_channel=128, + mlp=[128, 128, 128], + group_all=False, + use_instance_norm=use_instance_norm) + self.sa3 = PointNetSetAbstraction( + npoint=int(npoint / 32), + radius=None, + nsample=16, + in_channel=128, + mlp=[256, 256, 256], + group_all=False, + use_instance_norm=use_instance_norm) + + def forward(self, pc, feature): + pc_l1, feat_l1 = self.sa1(pc, feature) + pc_l2, feat_l2 = self.sa2(pc_l1, feat_l1) + pc_l3, feat_l3 = self.sa3(pc_l2, feat_l2) + + pc_l = dict(s8=pc_l1, s16=pc_l2, s32=pc_l3) + return pc_l, feat_l3 + + +class FeatureExtraction(nn.Module): + + def __init__(self, npoint, use_instance_norm): + super(FeatureExtraction, self).__init__() + self.sa1 = PointNetSetAbstraction( + npoint=int(npoint / 2), + radius=None, + nsample=32, + in_channel=3, + mlp=[32, 32, 32], + group_all=False, + return_fps=True, + use_instance_norm=use_instance_norm) + self.sa2 = PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=32, + in_channel=32, + mlp=[64, 64, 64], + group_all=False, + return_fps=True, + use_instance_norm=use_instance_norm) + + def forward(self, pc, feature, fps_idx=None): + """ + pc: [B, 3, N] + feature: [B, 3, N] + """ + fps_idx1 = fps_idx['s2'] if fps_idx is not None else None + pc_l1, feat_l1, fps_idx1 = self.sa1(pc, feature, fps_idx=fps_idx1) + fps_idx2 = fps_idx['s4'] if fps_idx is not None else None + pc_l2, feat_l2, fps_idx2 = self.sa2(pc_l1, feat_l1, fps_idx=fps_idx2) + pc_l = dict(s1=pc, s2=pc_l1, s4=pc_l2) + fps_idx = dict(s2=fps_idx1, s4=fps_idx2) + return pc_l, feat_l2, fps_idx + + +class HiddenInitNet(nn.Module): + + def __init__(self, npoint, use_instance_norm): + super(HiddenInitNet, self).__init__() + self.sa1 = PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=8, + in_channel=64, + mlp=[128, 128, 128], + group_all=False, + use_instance_norm=use_instance_norm) + self.sa2 = PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=8, + in_channel=128, + mlp=[128], + group_all=False, + use_act=False, + use_instance_norm=use_instance_norm) + + def forward(self, pc, feature): + _, feat_l1 = self.sa1(pc, feature) + _, feat_l2 = self.sa2(pc, feat_l1) + + h_init = torch.tanh(feat_l2) + return h_init + + +class GRUReg(nn.Module): + + def __init__(self, npoint, hidden_dim, input_dim, use_instance_norm): + super().__init__() + in_ch = hidden_dim + input_dim + + self.flow_proj = nn.ModuleList([ + PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=16, + in_channel=3, + mlp=[32, 32, 32], + group_all=False, + use_instance_norm=use_instance_norm), + PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=8, + in_channel=32, + mlp=[16, 16, 16], + group_all=False, + use_instance_norm=use_instance_norm) + ]) + + self.hidden_init_net = HiddenInitNet(npoint, use_instance_norm) + + self.gru_layers = nn.ModuleList([ + PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=4, + in_channel=in_ch, + mlp=[hidden_dim], + group_all=False, + use_act=False, + use_instance_norm=use_instance_norm), + PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=4, + in_channel=in_ch, + mlp=[hidden_dim], + group_all=False, + use_act=False, + use_instance_norm=use_instance_norm), + PointNetSetAbstraction( + npoint=int(npoint / 4), + radius=None, + nsample=4, + in_channel=in_ch, + mlp=[hidden_dim], + group_all=False, + use_act=False, + use_instance_norm=use_instance_norm) + ]) + + def gru(self, h, gru_inp, pc): + hx = torch.cat([h, gru_inp], dim=1) + z = torch.sigmoid(self.gru_layers[0](pc, hx)[1]) + r = torch.sigmoid(self.gru_layers[1](pc, hx)[1]) + q = torch.tanh(self.gru_layers[2](pc, torch.cat([r * h, gru_inp], + dim=1))[1]) + h = (1 - z) * h + z * q + return h + + def get_gru_input(self, feats1_new, cost, flow, pc): + flow_feats = flow + for flow_conv in self.flow_proj: + _, flow_feats = flow_conv(pc, flow_feats) + + gru_inp = torch.cat([feats1_new, cost, flow_feats, flow], + dim=1) # [64, 128, 16, 3] + + return gru_inp + + def forward(self, h, feats1_new, cost, flow_lr, pc1_l): + gru_inp = self.get_gru_input(feats1_new, cost, flow_lr, pc=pc1_l['s4']) + + h = self.gru(h, gru_inp, pc1_l['s4']) + return h + + +class SF_RCP(nn.Module): + + def __init__(self, npoint=8192, use_instance_norm=False, **kwargs): + super().__init__() + self.radius = kwargs.get('radius', 3.5) + self.nsample = kwargs.get('nsample', 6) + self.radius_min = kwargs.get('radius_min', 3.5) + self.nsample_min = kwargs.get('nsample_min', 6) + self.use_curvature = kwargs.get('use_curvature', True) + self.flow_ratio = kwargs.get('flow_ratio', 0.1) + self.init_max_iter = kwargs.get('init_max_iter', 0) + self.init_feature_norm = kwargs.get('init_feature_norm', True) + self.support_th = kwargs.get('support_th', 10) + + self.feature_extraction = FeatureExtraction(npoint, use_instance_norm) + self.feature_matching = FeatureMatching( + npoint, + use_instance_norm, + supporth_th=self.support_th, + feature_norm=self.init_feature_norm, + max_iter=self.init_max_iter) + + self.pointwise_optim_layer = PointWiseOptimLayer( + nsample=self.nsample, + radius=self.radius, + in_channel=64, + mlp=[128, 128, 128], + use_curvature=self.use_curvature) + + self.gru = GRUReg( + npoint, + hidden_dim=128, + input_dim=128 + 64 + 16 + 3, + use_instance_norm=use_instance_norm) + + self.flow_regressor = FlowRegressor(npoint, use_instance_norm) + + def initialization(self, pc1_l, pc2_l, feats1, feats2): + """ + pc1: [B, 3, N] + pc2: [B, 3, N] + feature1: [B, 3, N] + feature2: [B, 3, N] + """ + flow, flow_lr = self.feature_matching(pc1_l, pc2_l, feats1, feats2) + + return flow, flow_lr + + def pointwise_optimization(self, pc1_l_new, pc2_l, feats1_new, feats2, + pc1_l, flow_lr, iter): + _, cost, score, pos2_grouped = self.pointwise_optim_layer( + pc1_l_new['s4'], + pc2_l['s4'], + feats1_new, + feats2, + nsample=max(self.nsample_min, self.nsample // (2**iter)), + radius=max(self.radius_min, self.radius / (2**iter)), + pos1_raw=pc1_l['s4'], + return_score=True) + + # pc1_new_l_loc: [B, 3, N, S] + # pos2_grouped: [B, C, N, S] + delta_flow_tmp = ((pos2_grouped - pc1_l_new['s4'].unsqueeze(-1)) + * score.mean(dim=1, keepdim=True)).sum( + dim=-1) # [B, 3, N] + flow_lr = flow_lr + self.flow_ratio * delta_flow_tmp + + return flow_lr, cost + + def update_pos(self, pc, pc_lr, flow, flow_lr): + pc = pc + flow + pc_lr = pc_lr + flow_lr + return pc, pc_lr + + def forward(self, pc1, pc2, feature1, feature2, iters=1): + """ + pc1: [B, N, 3] + pc2: [B, N, 3] + feature1: [B, N, 3] + feature2: [B, N, 3] + """ + # prepare + flow_predictions = [] + pc1 = pc1.permute(0, 2, 1).contiguous() # B 3 N + pc2 = pc2.permute(0, 2, 1).contiguous() # B 3 N + feature1 = feature1.permute(0, 2, 1).contiguous() # B 3 N + feature2 = feature2.permute(0, 2, 1).contiguous() # B 3 N + + # feature extraction + pc1_l, feats1, fps_idx1 = self.feature_extraction(pc1, feature1) + pc2_l, feats2, _ = self.feature_extraction(pc2, feature2) + + # initialization, flow_lr_init(flow_low_resolution) + flow_init, flow_lr_init = self.initialization(pc1_l, pc2_l, feats1, + feats2) + flow_predictions.append(flow_init.permute(0, 2, 1)) + + # gru init hidden state + h = self.gru.hidden_init_net(pc1_l['s4'], feats1) + + # update position + pc1_lr_raw = pc1_l['s4'] + pc1_new, pc1_lr_new = self.update_pos(pc1, pc1_lr_raw, flow_init, + flow_lr_init) + + # iterative optim + for iter in range(iters - 1): + pc1_new = pc1_new.detach() + pc1_lr_new = pc1_lr_new.detach() + flow_lr = pc1_lr_new - pc1_lr_raw + + pc1_l_new, feats1_new, _ = self.feature_extraction( + pc1_new, pc1_new, fps_idx1) + + # pointwise optimization to get udpated flow_lr and cost + flow_lr_update, cost = self.pointwise_optimization( + pc1_l_new, pc2_l, feats1_new, feats2, pc1_l, flow_lr, iter) + flow_lr = flow_lr_update + + # gru regularization + h = self.gru(h, feats1_new, cost, flow_lr, pc1_l) + # pred flow_lr + delta_flow, delta_flow_lr = self.flow_regressor(pc1_l, h) + + pc1_new, pc1_lr_new = self.update_pos(pc1_new, pc1_lr_new, + delta_flow, delta_flow_lr) + + flow = pc1_new - pc1 + flow_predictions.append(flow.permute(0, 2, 1)) + + return flow_predictions diff --git a/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/__init__.py b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2cbc90998d7c61eeeda840a1c0a2bc219eec100d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .item_model import ProductRetrievalEmbedding + +else: + _import_structure = { + 'item_model': ['ProductRetrievalEmbedding'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_detection.py b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..b5024368a985fc7dd63f5b2fbadfde26872f68bd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_detection.py @@ -0,0 +1,524 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import cv2 +import numpy as np + + +class YOLOXONNX(object): + """ + Product detection model with onnx inference + """ + + def __init__(self, onnx_path, multi_detect=False): + """Create product detection model + Args: + onnx_path: onnx model path for product detection + multi_detect: detection parameter, should be set as False + + """ + self.input_reso = 416 + self.iou_thr = 0.45 + self.score_thr = 0.3 + self.img_shape = tuple([self.input_reso, self.input_reso, 3]) + self.num_classes = 13 + self.onnx_path = onnx_path + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + options.inter_op_num_threads = 1 + self.ort_session = ort.InferenceSession( + self.onnx_path, + sess_options=options, + providers=['CUDAExecutionProvider', 'CPUExecutionProvider']) + self.with_p6 = False + self.multi_detect = multi_detect + + def format_judge(self, img): + m_min_width = 100 + m_min_height = 100 + + height, width, c = img.shape + + if width * height > 1024 * 1024: + if height > width: + long_side = height + short_side = width + long_ratio = float(long_side) / 1024.0 + short_ratio = float(short_side) / float(m_min_width) + else: + long_side = width + short_side = height + long_ratio = float(long_side) / 1024.0 + short_ratio = float(short_side) / float(m_min_height) + + if long_side == height: + if long_ratio < short_ratio: + height_new = 1024 + width_new = (int)((1024 * width) / height) + + img_res = cv2.resize(img, (width_new, height_new), + cv2.INTER_LINEAR) + else: + height_new = (int)((m_min_width * height) / width) + width_new = m_min_width + + img_res = cv2.resize(img, (width_new, height_new), + cv2.INTER_LINEAR) + + elif long_side == width: + if long_ratio < short_ratio: + height_new = (int)((1024 * height) / width) + width_new = 1024 + + img_res = cv2.resize(img, (width_new, height_new), + cv2.INTER_LINEAR) + else: + width_new = (int)((m_min_height * width) / height) + height_new = m_min_height + + img_res = cv2.resize(img, (width_new, height_new), + cv2.INTER_LINEAR) + else: + img_res = img + + return img_res + + def preprocess(self, image, input_size, swap=(2, 0, 1)): + """ + Args: + image, cv2 image with BGR format + input_size, model input size + """ + if len(image.shape) == 3: + padded_img = np.ones((input_size[0], input_size[1], 3)) * 114.0 + else: + padded_img = np.ones(input_size) * 114.0 + img = np.array(image) + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.float32) + padded_img[:int(img.shape[0] * r), :int(img.shape[1] + * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + def cal_iou(self, val1, val2): + x11, y11, x12, y12 = val1 + x21, y21, x22, y22 = val2 + + leftX = max(x11, x21) + topY = max(y11, y21) + rightX = min(x12, x22) + bottomY = min(y12, y22) + if rightX < leftX or bottomY < topY: + return 0 + area = float((rightX - leftX) * (bottomY - topY)) + barea = (x12 - x11) * (y12 - y11) + (x22 - x21) * (y22 - y21) - area + if barea <= 0: + return 0 + return area / barea + + def nms(self, boxes, scores, nms_thr): + """ + Single class NMS implemented in Numpy. + """ + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + + def multiclass_nms(self, boxes, scores, nms_thr, score_thr): + """ + Multiclass NMS implemented in Numpy + """ + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = self.nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate([ + valid_boxes[keep], valid_scores[keep, None], cls_inds + ], 1) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + + def postprocess(self, outputs, img_size, p6=False): + grids = [] + expanded_strides = [] + + if not p6: + strides = [8, 16, 32] + else: + strides = [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs + + def get_new_box_order(self, bboxes, labels, img_h, img_w): + """ + refine bbox score + """ + bboxes = np.hstack((bboxes, np.zeros((bboxes.shape[0], 1)))) + scores = bboxes[:, 4] + order = scores.argsort()[::-1] + bboxes_temp = bboxes[order] + labels_temp = labels[order] + bboxes = np.empty((0, 6)) + # import pdb;pdb.set_trace() + bboxes = np.vstack((bboxes, bboxes_temp[0].tolist())) + labels = np.empty((0, )) + + labels = np.hstack((labels, [labels_temp[0]])) + for i in range(1, bboxes_temp.shape[0]): + iou_max = 0 + for j in range(bboxes.shape[0]): + iou_temp = self.cal_iou(bboxes_temp[i][:4], bboxes[j][:4]) + if (iou_temp > iou_max): + iou_max = iou_temp + if (iou_max < 0.45): + bboxes = np.vstack((bboxes, bboxes_temp[i].tolist())) + labels = np.hstack((labels, [labels_temp[i]])) + + num_03 = scores > 0.3 + num_03 = num_03.sum() + num_out = max(num_03, 1) + bboxes = bboxes[:num_out, :] + labels = labels[:num_out] + + return bboxes, labels + + def forward(self, img_input, cid='0', sub_class=False): + """ + forward for product detection + """ + input_shape = self.img_shape + + img, ratio = self.preprocess(img_input, input_shape) + img_h, img_w = img_input.shape[:2] + + ort_inputs = { + self.ort_session.get_inputs()[0].name: img[None, :, :, :] + } + + output = self.ort_session.run(None, ort_inputs) + + predictions = self.postprocess(output[0], input_shape, self.with_p6)[0] + + boxes = predictions[:, :4] + scores = predictions[:, 4:5] * predictions[:, 5:] + + boxes_xyxy = np.ones_like(boxes) + boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2. + boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2. + boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2. + boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2. + boxes_xyxy /= ratio + dets = self.multiclass_nms( + boxes_xyxy, scores, nms_thr=0.45, score_thr=0.1) + + if dets is None: + top1_bbox_str = str(0) + ',' + str(img_w) + ',' + str( + 0) + ',' + str(img_h) + crop_img = img_input.copy() + coord = top1_bbox_str + else: + bboxes = dets[:, :5] + labels = dets[:, 5] + + if not self.multi_detect: + cid = int(cid) + if (not sub_class): + if cid > -1: + if cid == 0: # cloth + cid_ind1 = np.where(labels < 3) + cid_ind2 = np.where(labels == 9) + cid_ind = np.hstack((cid_ind1[0], cid_ind2[0])) + scores = bboxes[cid_ind, -1] # 0, 1, 2, 9 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 3: # bag + cid_ind = np.where(labels == 3) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 4: # shoe + cid_ind = np.where(labels == 4) + scores = bboxes[cid_ind, -1] # 4 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + else: # other + cid_ind5 = np.where(labels == 5) + cid_ind6 = np.where(labels == 6) + cid_ind7 = np.where(labels == 7) + cid_ind8 = np.where(labels == 8) + cid_ind10 = np.where(labels == 10) + cid_ind11 = np.where(labels == 11) + cid_ind12 = np.where(labels == 12) + cid_ind = np.hstack( + (cid_ind5[0], cid_ind6[0], cid_ind7[0], + cid_ind8[0], cid_ind10[0], cid_ind11[0], + cid_ind12[0])) + scores = bboxes[cid_ind, -1] # 5,6,7,8,10,11,12 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + else: + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + else: + if cid > -1: + if cid == 0: # upper + cid_ind = np.where(labels == 0) + + scores = bboxes[cid_ind, -1] # 0, 1, 2, 9 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 1: # skirt + cid_ind = np.where(labels == 1) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 2: # lower + cid_ind = np.where(labels == 2) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 3: # bag + cid_ind = np.where(labels == 3) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 4: # shoe + cid_ind = np.where(labels == 4) + scores = bboxes[cid_ind, -1] # 4 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 5: # access + cid_ind = np.where(labels == 5) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 7: # beauty + cid_ind = np.where(labels == 6) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 9: # furniture + cid_ind = np.where(labels == 8) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + elif cid == 21: # underwear + cid_ind = np.where(labels == 9) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + elif cid == 22: # digital + cid_ind = np.where(labels == 11) + scores = bboxes[cid_ind, -1] # 3 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + else: # other + cid_ind5 = np.where(labels == 7) # bottle + cid_ind6 = np.where(labels == 10) # toy + cid_ind7 = np.where(labels == 12) # toy + cid_ind = np.hstack( + (cid_ind5[0], cid_ind6[0], cid_ind7[0])) + scores = bboxes[cid_ind, -1] # 5,6,7 + + if scores.size > 0: + + bboxes = bboxes[cid_ind] + labels = labels[cid_ind] + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + + else: + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + else: + bboxes, labels = self.get_new_box_order( + bboxes, labels, img_h, img_w) + top1_bbox = bboxes[0].astype(np.int32) + top1_bbox[0] = min(max(0, top1_bbox[0]), img_input.shape[1] - 1) + top1_bbox[1] = min(max(0, top1_bbox[1]), img_input.shape[0] - 1) + top1_bbox[2] = max(min(img_input.shape[1] - 1, top1_bbox[2]), 0) + top1_bbox[3] = max(min(img_input.shape[0] - 1, top1_bbox[3]), 0) + if not self.multi_detect: + + top1_bbox_str = str(top1_bbox[0]) + ',' + str( + top1_bbox[2]) + ',' + str(top1_bbox[1]) + ',' + str( + top1_bbox[3]) # x1, x2, y1, y2 + crop_img = img_input[top1_bbox[1]:top1_bbox[3], + top1_bbox[0]:top1_bbox[2], :] + coord = top1_bbox_str + coord = '' + for i in range(0, len(bboxes)): + top_bbox = bboxes[i].astype(np.int32) + top_bbox[0] = min( + max(0, top_bbox[0]), img_input.shape[1] - 1) + top_bbox[1] = min( + max(0, top_bbox[1]), img_input.shape[0] - 1) + top_bbox[2] = max( + min(img_input.shape[1] - 1, top_bbox[2]), 0) + top_bbox[3] = max( + min(img_input.shape[0] - 1, top_bbox[3]), 0) + coord = coord + str(top_bbox[0]) + ',' + str( + top_bbox[2]) + ',' + str(top_bbox[1]) + ',' + str( + top_bbox[3]) + ',' + str(bboxes[i][4]) + ',' + str( + bboxes[i][5]) + ';' + + else: + coord = '' + for i in range(0, len(bboxes)): + top_bbox = bboxes[i].astype(np.int32) + top_bbox[0] = min( + max(0, top_bbox[0]), img_input.shape[1] - 1) + top_bbox[1] = min( + max(0, top_bbox[1]), img_input.shape[0] - 1) + top_bbox[2] = max( + min(img_input.shape[1] - 1, top_bbox[2]), 0) + top_bbox[3] = max( + min(img_input.shape[0] - 1, top_bbox[3]), 0) + coord = coord + str(top_bbox[0]) + ',' + str( + top_bbox[2]) + ',' + str(top_bbox[1]) + ',' + str( + top_bbox[3]) + ',' + str(bboxes[i][4]) + ',' + str( + bboxes[i][5]) + ';' # x1, x2, y1, y2, conf + crop_img = img_input[top1_bbox[1]:top1_bbox[3], + top1_bbox[0]:top1_bbox[2], :] + + crop_img = cv2.resize(crop_img, (224, 224)) + + return coord, crop_img # return top1 image and coord diff --git a/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_embedding.py b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..ea9ec846c6b7eb4a87801a282a87776abbd7362c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_embedding.py @@ -0,0 +1,154 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import cv2 +import numpy as np +import torch.nn as nn +import torch.nn.functional as F + + +def gn_init(m, zero_init=False): + assert isinstance(m, nn.GroupNorm) + m.weight.data.fill_(0. if zero_init else 1.) + m.bias.data.zero_() + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + """Bottleneck for resnet-style networks + Args: + inplanes: input channel number + planes: output channel number + """ + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.GroupNorm(32, planes) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = nn.GroupNorm(32, planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.GroupNorm(32, planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + gn_init(self.bn1) + gn_init(self.bn2) + gn_init(self.bn3, zero_init=True) + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + """ + resnet-style network with group normalization + """ + + def __init__(self, block, layers, num_classes=1000): + self.inplanes = 64 + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.GroupNorm(32, 64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=1) + + self.gap = nn.AvgPool2d((14, 14)) + self.reduce_conv = nn.Conv2d(2048, 512, kernel_size=1) + + gn_init(self.bn1) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.AvgPool2d(stride, stride), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=1, + bias=False), + nn.GroupNorm(32, planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.gap(x) + x = self.reduce_conv(x) # 512 + + x = x.view(x.size(0), -1) # 512 + return F.normalize(x, p=2, dim=1) + + +def preprocess(img): + """ + preprocess the image with cv2-bgr style to tensor + """ + mean = np.array([0.485, 0.456, 0.406]) + std = np.array([0.229, 0.224, 0.225]) + + img_size = 224 + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img_new = cv2.resize( + img, (img_size, img_size), interpolation=cv2.INTER_LINEAR) + content = np.array(img_new).astype(np.float32) + content = (content / 255.0 - mean) / std + # transpose + img_new = content.transpose(2, 0, 1) + img_new = img_new[np.newaxis, :, :, :] + return img_new + + +def resnet50_embed(): + """ + create resnet50 network with group normalization + """ + net = ResNet(Bottleneck, [3, 4, 6, 3]) + return net diff --git a/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_model.py b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_model.py new file mode 100644 index 0000000000000000000000000000000000000000..3964efbea3c423b7d4159d24fb5ecefec755d9e4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_retrieval_embedding/item_model.py @@ -0,0 +1,116 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os.path as osp +from typing import Any, Dict + +import numpy as np +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.product_retrieval_embedding.item_detection import \ + YOLOXONNX +from modelscope.models.cv.product_retrieval_embedding.item_embedding import ( + preprocess, resnet50_embed) +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ProductRetrievalEmbedding'] + + +@MODELS.register_module( + Tasks.product_retrieval_embedding, + module_name=Models.product_retrieval_embedding) +class ProductRetrievalEmbedding(TorchModel): + + def __init__(self, model_dir, device='cpu', **kwargs): + super().__init__(model_dir=model_dir, device=device, **kwargs) + + def filter_param(src_params, own_state): + copied_keys = [] + for name, param in src_params.items(): + if 'module.' == name[0:7]: + name = name[7:] + if '.module.' not in list(own_state.keys())[0]: + name = name.replace('.module.', '.') + if (name in own_state) and (own_state[name].shape + == param.shape): + own_state[name].copy_(param) + copied_keys.append(name) + + def load_pretrained(model, src_params): + if 'state_dict' in src_params: + src_params = src_params['state_dict'] + own_state = model.state_dict() + filter_param(src_params, own_state) + model.load_state_dict(own_state) + + self.device = create_device( + device) # device.type == "cpu" or device.type == "cuda" + self.use_gpu = self.device.type == 'cuda' + + # config the model path + self.local_model_dir = model_dir + + # init feat model + self.preprocess_for_embed = preprocess # input is cv2 bgr format + model_feat = resnet50_embed() + src_params = torch.load( + osp.join(self.local_model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + 'cpu') + load_pretrained(model_feat, src_params) + if self.use_gpu: + model_feat.to(self.device) + logger.info('Use GPU: {}'.format(self.device)) + else: + logger.info('Use CPU for inference') + + self.model_feat = model_feat + + # init det model + self.model_det = YOLOXONNX( + onnx_path=osp.join(self.local_model_dir, 'onnx_detection.onnx'), + multi_detect=False) + logger.info('load model done') + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """ + detection and feature extraction for input product image + """ + # input should be cv2 bgr format + assert 'img' in input.keys() + + def set_phase(model, is_train): + if is_train: + model.train() + else: + model.eval() + + is_train = False + set_phase(self.model_feat, is_train) + img = input['img'] # for detection + cid = '3' # preprocess detection category bag + # transform img(tensor) to numpy array with bgr + if isinstance(img, torch.Tensor): + img = img.data.cpu().numpy() + res, crop_img = self.model_det.forward(img, + cid) # detect with bag category + crop_img = self.preprocess_for_embed(crop_img) # feat preprocess + input_tensor = torch.from_numpy(crop_img.astype(np.float32)) + device = next(self.model_feat.parameters()).device + use_gpu = device.type == 'cuda' + with torch.no_grad(): + if use_gpu: + input_tensor = input_tensor.to(device) + out_embedding = self.model_feat(input_tensor) + out_embedding = out_embedding.cpu().numpy()[ + 0, :] # feature array with 512 elements + + output = {OutputKeys.IMG_EMBEDDING: None} + output[OutputKeys.IMG_EMBEDDING] = out_embedding + return output diff --git a/runtime/Lib/site_packages/models/cv/product_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/product_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e87c8db140ca4375f873cbdc960dff1c98404f57 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_segmentation/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .seg_infer import F3NetProductSegmentation + +else: + _import_structure = {'seg_infer': ['F3NetProductSegmentation']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/product_segmentation/net.py b/runtime/Lib/site_packages/models/cv/product_segmentation/net.py new file mode 100644 index 0000000000000000000000000000000000000000..12bc3057a2f2f17721c8efe793e8228a5796d43f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_segmentation/net.py @@ -0,0 +1,197 @@ +# The implementation here is modified based on F3Net, +# originally Apache 2.0 License and publicly available at https://github.com/weijun88/F3Net + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Bottleneck(nn.Module): + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + dilation=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=(3 * dilation - 1) // 2, + bias=False, + dilation=dilation) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.downsample = downsample + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x)), inplace=True) + out = F.relu(self.bn2(self.conv2(out)), inplace=True) + out = self.bn3(self.conv3(out)) + if self.downsample is not None: + x = self.downsample(x) + return F.relu(out + x, inplace=True) + + +class ResNet(nn.Module): + + def __init__(self): + super(ResNet, self).__init__() + self.inplanes = 64 + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self.make_layer(64, 3, stride=1, dilation=1) + self.layer2 = self.make_layer(128, 4, stride=2, dilation=1) + self.layer3 = self.make_layer(256, 6, stride=2, dilation=1) + self.layer4 = self.make_layer(512, 3, stride=2, dilation=1) + + def make_layer(self, planes, blocks, stride, dilation): + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * 4, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(planes * 4)) + layers = [ + Bottleneck( + self.inplanes, planes, stride, downsample, dilation=dilation) + ] + self.inplanes = planes * 4 + for _ in range(1, blocks): + layers.append(Bottleneck(self.inplanes, planes, dilation=dilation)) + return nn.Sequential(*layers) + + def forward(self, x): + x = x.reshape(1, 3, 448, 448) + out1 = F.relu(self.bn1(self.conv1(x)), inplace=True) + out1 = F.max_pool2d(out1, kernel_size=3, stride=2, padding=1) + out2 = self.layer1(out1) + out3 = self.layer2(out2) + out4 = self.layer3(out3) + out5 = self.layer4(out4) + return out2, out3, out4, out5 + + +class CFM(nn.Module): + + def __init__(self): + super(CFM, self).__init__() + self.conv1h = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn1h = nn.BatchNorm2d(64) + self.conv2h = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn2h = nn.BatchNorm2d(64) + self.conv3h = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn3h = nn.BatchNorm2d(64) + self.conv4h = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn4h = nn.BatchNorm2d(64) + + self.conv1v = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn1v = nn.BatchNorm2d(64) + self.conv2v = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn2v = nn.BatchNorm2d(64) + self.conv3v = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn3v = nn.BatchNorm2d(64) + self.conv4v = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) + self.bn4v = nn.BatchNorm2d(64) + + def forward(self, left, down): + if down.size()[2:] != left.size()[2:]: + down = F.interpolate(down, size=left.size()[2:], mode='bilinear') + out1h = F.relu(self.bn1h(self.conv1h(left)), inplace=True) + out2h = F.relu(self.bn2h(self.conv2h(out1h)), inplace=True) + out1v = F.relu(self.bn1v(self.conv1v(down)), inplace=True) + out2v = F.relu(self.bn2v(self.conv2v(out1v)), inplace=True) + fuse = out2h * out2v + out3h = F.relu(self.bn3h(self.conv3h(fuse)), inplace=True) + out1h + out4h = F.relu(self.bn4h(self.conv4h(out3h)), inplace=True) + out3v = F.relu(self.bn3v(self.conv3v(fuse)), inplace=True) + out1v + out4v = F.relu(self.bn4v(self.conv4v(out3v)), inplace=True) + return out4h, out4v + + +class Decoder(nn.Module): + + def __init__(self): + super(Decoder, self).__init__() + self.cfm45 = CFM() + self.cfm34 = CFM() + self.cfm23 = CFM() + + def forward(self, out2h, out3h, out4h, out5v, fback=None): + if fback is not None: + refine5 = F.interpolate( + fback, size=out5v.size()[2:], mode='bilinear') + refine4 = F.interpolate( + fback, size=out4h.size()[2:], mode='bilinear') + refine3 = F.interpolate( + fback, size=out3h.size()[2:], mode='bilinear') + refine2 = F.interpolate( + fback, size=out2h.size()[2:], mode='bilinear') + out5v = out5v + refine5 + out4h, out4v = self.cfm45(out4h + refine4, out5v) + out3h, out3v = self.cfm34(out3h + refine3, out4v) + out2h, pred = self.cfm23(out2h + refine2, out3v) + else: + out4h, out4v = self.cfm45(out4h, out5v) + out3h, out3v = self.cfm34(out3h, out4v) + out2h, pred = self.cfm23(out2h, out3v) + return out2h, out3h, out4h, out5v, pred + + +class F3Net(nn.Module): + + def __init__(self): + super(F3Net, self).__init__() + self.bkbone = ResNet() + self.squeeze5 = nn.Sequential( + nn.Conv2d(2048, 64, 1), nn.BatchNorm2d(64), nn.ReLU(inplace=True)) + self.squeeze4 = nn.Sequential( + nn.Conv2d(1024, 64, 1), nn.BatchNorm2d(64), nn.ReLU(inplace=True)) + self.squeeze3 = nn.Sequential( + nn.Conv2d(512, 64, 1), nn.BatchNorm2d(64), nn.ReLU(inplace=True)) + self.squeeze2 = nn.Sequential( + nn.Conv2d(256, 64, 1), nn.BatchNorm2d(64), nn.ReLU(inplace=True)) + + self.decoder1 = Decoder() + self.decoder2 = Decoder() + self.linearp1 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + self.linearp2 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + + self.linearr2 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + self.linearr3 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + self.linearr4 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + self.linearr5 = nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1) + + def forward(self, x, shape=None): + x = x.reshape(1, 3, 448, 448) + out2h, out3h, out4h, out5v = self.bkbone(x) + out2h, out3h, out4h, out5v = self.squeeze2(out2h), self.squeeze3( + out3h), self.squeeze4(out4h), self.squeeze5(out5v) + out2h, out3h, out4h, out5v, pred1 = self.decoder1( + out2h, out3h, out4h, out5v) + out2h, out3h, out4h, out5v, pred2 = self.decoder2( + out2h, out3h, out4h, out5v, pred1) + + shape = x.size()[2:] if shape is None else shape + pred1 = F.interpolate( + self.linearp1(pred1), size=shape, mode='bilinear') + pred2 = F.interpolate( + self.linearp2(pred2), size=shape, mode='bilinear') + + out2h = F.interpolate( + self.linearr2(out2h), size=shape, mode='bilinear') + out3h = F.interpolate( + self.linearr3(out3h), size=shape, mode='bilinear') + out4h = F.interpolate( + self.linearr4(out4h), size=shape, mode='bilinear') + out5h = F.interpolate( + self.linearr5(out5v), size=shape, mode='bilinear') + return pred1, pred2, out2h, out3h, out4h, out5h diff --git a/runtime/Lib/site_packages/models/cv/product_segmentation/seg_infer.py b/runtime/Lib/site_packages/models/cv/product_segmentation/seg_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..8814d6190bb6f39eac85ceb93a9b53eaa1824df3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/product_segmentation/seg_infer.py @@ -0,0 +1,76 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import cv2 +import numpy as np +import torch +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .net import F3Net + +logger = get_logger() + + +def load_state_dict(model_dir, device): + _dict = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location=device) + state_dict = {} + for k, v in _dict.items(): + if k.startswith('module'): + k = k[7:] + state_dict[k] = v + return state_dict + + +@MODELS.register_module( + Tasks.product_segmentation, module_name=Models.product_segmentation) +class F3NetForProductSegmentation(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + self.model = F3Net() + if torch.cuda.is_available(): + self.device = 'cuda' + logger.info('Use GPU') + else: + self.device = 'cpu' + logger.info('Use CPU') + + self.params = load_state_dict(model_dir, self.device) + self.model.load_state_dict(self.params) + self.model.to(self.device) + self.model.eval() + self.model.to(self.device) + + def forward(self, x): + pred_result = self.model(x) + return pred_result + + +mean, std = np.array([[[124.55, 118.90, + 102.94]]]), np.array([[[56.77, 55.97, 57.50]]]) + + +def inference(model, device, img): + img = img.cpu().numpy() + img = (img - mean) / std + img = cv2.resize(img, dsize=(448, 448), interpolation=cv2.INTER_LINEAR) + img = torch.from_numpy(img) + img = img.permute(2, 0, 1) + img = img.to(device).float() + outputs = model(img) + out = outputs[0] + pred = (torch.sigmoid(out[0, 0]) * 255).cpu().numpy() + pred[pred < 20] = 0 + pred = pred[:, :, np.newaxis] + pred = np.round(pred) + logger.info('Inference Done') + return pred diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c97bd7b238136a9a4769879e4a9e03341327ccd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .model import ReferringVideoObjectSegmentation + +else: + _import_structure = { + 'model': ['ReferringVideoObjectSegmentation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/model.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..239d0ea54663c6fa51ac81c6a1c2d0d4da59d945 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/model.py @@ -0,0 +1,149 @@ +# Part of the implementation is borrowed and modified from MTTR, +# publicly available at https://github.com/mttr2021/MTTR + +import os.path as osp +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .utils import (MTTR, A2DSentencesPostProcess, HungarianMatcher, + ReferYoutubeVOSPostProcess, SetCriterion, + flatten_temporal_batch_dims, + nested_tensor_from_videos_list) + +logger = get_logger() + + +@MODELS.register_module( + Tasks.referring_video_object_segmentation, + module_name=Models.referring_video_object_segmentation) +class ReferringVideoObjectSegmentation(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + + config_path = osp.join(model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(config_path) + transformer_cfg_dir = osp.join(model_dir, 'transformer_cfg_dir') + + self.model = MTTR( + transformer_cfg_dir=transformer_cfg_dir, **self.cfg.model) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + params_dict = torch.load(model_path, map_location='cpu') + if 'model_state_dict' in params_dict.keys(): + params_dict = params_dict['model_state_dict'] + compatible_position_ids( + params_dict, 'transformer.text_encoder.embeddings.position_ids') + self.model.load_state_dict(params_dict, strict=True) + + self.set_postprocessor(self.cfg.pipeline.dataset_name) + self.set_criterion() + + def set_device(self, device, name): + self.device = device + self._device_name = name + + def set_postprocessor(self, dataset_name): + if 'a2d_sentences' in dataset_name or 'jhmdb_sentences' in dataset_name: + self.postprocessor = A2DSentencesPostProcess() # fine-tune + elif 'ref_youtube_vos' in dataset_name: + self.postprocessor = ReferYoutubeVOSPostProcess() # inference + else: + assert False, f'postprocessing for dataset: {dataset_name} is not supported' + + def forward(self, inputs: Dict[str, Any]): + samples = inputs['samples'] + targets = inputs['targets'] + text_queries = inputs['text_queries'] + + valid_indices = torch.tensor( + [i for i, t in enumerate(targets) if None not in t]) + targets = [targets[i] for i in valid_indices.tolist()] + if self._device_name == 'gpu': + samples = samples.to(self.device) + valid_indices = valid_indices.to(self.device) + if isinstance(text_queries, tuple): + text_queries = list(text_queries) + + outputs = self.model(samples, valid_indices, text_queries) + losses = -1 + if self.training: + loss_dict = self.criterion(outputs, targets) + weight_dict = self.criterion.weight_dict + losses = sum(loss_dict[k] * weight_dict[k] + for k in loss_dict.keys() if k in weight_dict) + + predictions = [] + if not self.training: + outputs.pop('aux_outputs', None) + outputs, targets = flatten_temporal_batch_dims(outputs, targets) + processed_outputs = self.postprocessor( + outputs, + resized_padded_sample_size=samples.tensors.shape[-2:], + resized_sample_sizes=[t['size'] for t in targets], + orig_sample_sizes=[t['orig_size'] for t in targets]) + image_ids = [t['image_id'] for t in targets] + predictions = [] + for p, image_id in zip(processed_outputs, image_ids): + for s, m in zip(p['scores'], p['rle_masks']): + predictions.append({ + 'image_id': image_id, + 'category_id': + 1, # dummy label, as categories are not predicted in ref-vos + 'segmentation': m, + 'score': s.item() + }) + + re = dict(pred=predictions, loss=losses) + return re + + def inference(self, **kwargs): + window = kwargs['window'] + text_query = kwargs['text_query'] + video_metadata = kwargs['metadata'] + + window = nested_tensor_from_videos_list([window]) + valid_indices = torch.arange(len(window.tensors)) + if self._device_name == 'gpu': + valid_indices = valid_indices.cuda() + outputs = self.model(window, valid_indices, [text_query]) + window_masks = self.postprocessor( + outputs, [video_metadata], + window.tensors.shape[-2:])[0]['pred_masks'] + return window_masks + + def postprocess(self, inputs: Dict[str, Any], **kwargs): + return inputs + + def set_criterion(self): + matcher = HungarianMatcher( + cost_is_referred=self.cfg.matcher.set_cost_is_referred, + cost_dice=self.cfg.matcher.set_cost_dice) + weight_dict = { + 'loss_is_referred': self.cfg.loss.is_referred_loss_coef, + 'loss_dice': self.cfg.loss.dice_loss_coef, + 'loss_sigmoid_focal': self.cfg.loss.sigmoid_focal_loss_coef + } + + if self.cfg.loss.aux_loss: + aux_weight_dict = {} + for i in range(self.cfg.model.num_decoder_layers - 1): + aux_weight_dict.update( + {k + f'_{i}': v + for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + self.criterion = SetCriterion( + matcher=matcher, + weight_dict=weight_dict, + eos_coef=self.cfg.loss.eos_coef) diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/__init__.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fbb75b006720d2ae782b567028e15e75ba1a5bf8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .criterion import SetCriterion, flatten_temporal_batch_dims +from .matcher import HungarianMatcher +from .misc import interpolate, nested_tensor_from_videos_list +from .mttr import MTTR +from .postprocessing import A2DSentencesPostProcess, ReferYoutubeVOSPostProcess diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/backbone.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..afa384c195ae199fe499b1163c68970f772f9feb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/backbone.py @@ -0,0 +1,198 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR + +import torch +import torch.nn.functional as F +import torchvision +from einops import rearrange +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter + +from .misc import NestedTensor, is_main_process +from .swin_transformer import SwinTransformer3D + + +class VideoSwinTransformerBackbone(nn.Module): + """ + A wrapper which allows using Video-Swin Transformer as a temporal encoder for MTTR. + Check out video-swin's original paper at: https://arxiv.org/abs/2106.13230 for more info about this architecture. + Only the 'tiny' version of video swin was tested and is currently supported in our project. + Additionally, we slightly modify video-swin to make it output per-frame embeddings as required by MTTR (check our + paper's supplementary for more details), and completely discard of its 4th block. + """ + + def __init__(self, backbone_pretrained, backbone_pretrained_path, + train_backbone, running_mode, **kwargs): + super(VideoSwinTransformerBackbone, self).__init__() + # patch_size is (1, 4, 4) instead of the original (2, 4, 4). + # this prevents swinT's original temporal downsampling so we can get per-frame features. + swin_backbone = SwinTransformer3D( + patch_size=(1, 4, 4), + embed_dim=96, + depths=(2, 2, 6, 2), + num_heads=(3, 6, 12, 24), + window_size=(8, 7, 7), + drop_path_rate=0.1, + patch_norm=True) + if backbone_pretrained and running_mode == 'train': + state_dict = torch.load(backbone_pretrained_path)['state_dict'] + # extract swinT's kinetics-400 pretrained weights and ignore the rest (prediction head etc.) + state_dict = { + k[9:]: v + for k, v in state_dict.items() if 'backbone.' in k + } + + # sum over the patch embedding weight temporal dim [96, 3, 2, 4, 4] --> [96, 3, 1, 4, 4] + patch_embed_weight = state_dict['patch_embed.proj.weight'] + patch_embed_weight = patch_embed_weight.sum(dim=2, keepdims=True) + state_dict['patch_embed.proj.weight'] = patch_embed_weight + swin_backbone.load_state_dict(state_dict) + + self.patch_embed = swin_backbone.patch_embed + self.pos_drop = swin_backbone.pos_drop + self.layers = swin_backbone.layers[:-1] + self.downsamples = nn.ModuleList() + for layer in self.layers: + self.downsamples.append(layer.downsample) + layer.downsample = None + self.downsamples[ + -1] = None # downsampling after the last layer is not necessary + + self.layer_output_channels = [ + swin_backbone.embed_dim * 2**i for i in range(len(self.layers)) + ] + self.train_backbone = train_backbone + if not train_backbone: + for parameter in self.parameters(): + parameter.requires_grad_(False) + + def forward(self, samples: NestedTensor): + vid_frames = rearrange(samples.tensors, 't b c h w -> b c t h w') + + vid_embeds = self.patch_embed(vid_frames) + vid_embeds = self.pos_drop(vid_embeds) + layer_outputs = [] # layer outputs before downsampling + for layer, downsample in zip(self.layers, self.downsamples): + vid_embeds = layer(vid_embeds.contiguous()) + layer_outputs.append(vid_embeds) + if downsample: + vid_embeds = rearrange(vid_embeds, 'b c t h w -> b t h w c') + vid_embeds = downsample(vid_embeds) + vid_embeds = rearrange(vid_embeds, 'b t h w c -> b c t h w') + layer_outputs = [ + rearrange(o, 'b c t h w -> t b c h w') for o in layer_outputs + ] + + outputs = [] + orig_pad_mask = samples.mask + for l_out in layer_outputs: + pad_mask = F.interpolate( + orig_pad_mask.float(), size=l_out.shape[-2:]).to(torch.bool) + outputs.append(NestedTensor(l_out, pad_mask)) + return outputs + + def num_parameters(self): + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + +class FrozenBatchNorm2d(torch.nn.Module): + """ + Modified from DETR https://github.com/facebookresearch/detr + BatchNorm2d where the batch statistics and the affine parameters are fixed. + Copy-paste from torchvision.misc.ops with added eps before rqsrt, + without which any other models than torchvision.models.resnet[18,34,50,101] + produce nans. + """ + + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer('weight', torch.ones(n)) + self.register_buffer('bias', torch.zeros(n)) + self.register_buffer('running_mean', torch.zeros(n)) + self.register_buffer('running_var', torch.ones(n)) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + num_batches_tracked_key = prefix + 'num_batches_tracked' + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super(FrozenBatchNorm2d, + self)._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, + unexpected_keys, error_msgs) + + def forward(self, x): + # move reshapes to the beginning + # to make it fuser-friendly + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + rv = self.running_var.reshape(1, -1, 1, 1) + rm = self.running_mean.reshape(1, -1, 1, 1) + eps = 1e-5 + scale = w * (rv + eps).rsqrt() + bias = b - rm * scale + return x * scale + bias + + +class ResNetBackbone(nn.Module): + """ + Modified from DETR https://github.com/facebookresearch/detr + ResNet backbone with frozen BatchNorm. + """ + + def __init__(self, + backbone_name: str = 'resnet50', + train_backbone: bool = True, + dilation: bool = True, + **kwargs): + super(ResNetBackbone, self).__init__() + backbone = getattr(torchvision.models, backbone_name)( + replace_stride_with_dilation=[False, False, dilation], + pretrained=is_main_process(), + norm_layer=FrozenBatchNorm2d) + for name, parameter in backbone.named_parameters(): + if not train_backbone or 'layer2' not in name and 'layer3' not in name and 'layer4' not in name: + parameter.requires_grad_(False) + return_layers = { + 'layer1': '0', + 'layer2': '1', + 'layer3': '2', + 'layer4': '3' + } + self.body = IntermediateLayerGetter( + backbone, return_layers=return_layers) + output_channels = 512 if backbone_name in ('resnet18', + 'resnet34') else 2048 + self.layer_output_channels = [ + output_channels // 8, output_channels // 4, output_channels // 2, + output_channels + ] + + def forward(self, tensor_list: NestedTensor): + t, b, _, _, _ = tensor_list.tensors.shape + video_frames = rearrange(tensor_list.tensors, + 't b c h w -> (t b) c h w') + padding_masks = rearrange(tensor_list.mask, 't b h w -> (t b) h w') + features_list = self.body(video_frames) + out = [] + for _, f in features_list.items(): + resized_padding_masks = F.interpolate( + padding_masks[None].float(), + size=f.shape[-2:]).to(torch.bool)[0] + f = rearrange(f, '(t b) c h w -> t b c h w', t=t, b=b) + resized_padding_masks = rearrange( + resized_padding_masks, '(t b) h w -> t b h w', t=t, b=b) + out.append(NestedTensor(f, resized_padding_masks)) + return out + + def num_parameters(self): + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + +def init_backbone(backbone_name, **kwargs): + if backbone_name == 'swin-t': + return VideoSwinTransformerBackbone(**kwargs) + elif 'resnet' in backbone_name: + return ResNetBackbone(backbone_name, **kwargs) + assert False, f'error: backbone "{backbone_name}" is not supported' diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/criterion.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d2f0ff537f65506ee026511772889b089b352b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/criterion.py @@ -0,0 +1,198 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# Modified from DETR https://github.com/facebookresearch/detr +import torch +from torch import nn + +from .misc import (get_world_size, interpolate, is_dist_avail_and_initialized, + nested_tensor_from_tensor_list) +from .segmentation import dice_loss, sigmoid_focal_loss + + +class SetCriterion(nn.Module): + """ This class computes the loss for MTTR. + The process happens in two steps: + 1) we compute the hungarian assignment between the ground-truth and predicted sequences. + 2) we supervise each pair of matched ground-truth / prediction sequences (mask + reference prediction) + """ + + def __init__(self, matcher, weight_dict, eos_coef): + """ Create the criterion. + Parameters: + matcher: module able to compute a matching between targets and proposals + weight_dict: dict containing as key the names of the losses and as values their relative weight. + eos_coef: relative classification weight applied to the un-referred category + """ + super().__init__() + self.matcher = matcher + self.weight_dict = weight_dict + self.eos_coef = eos_coef + # make sure that only loss functions with non-zero weights are computed: + losses_to_compute = [] + if weight_dict['loss_dice'] > 0 or weight_dict[ + 'loss_sigmoid_focal'] > 0: + losses_to_compute.append('masks') + if weight_dict['loss_is_referred'] > 0: + losses_to_compute.append('is_referred') + self.losses = losses_to_compute + + def forward(self, outputs, targets): + aux_outputs_list = outputs.pop('aux_outputs', None) + # compute the losses for the output of the last decoder layer: + losses = self.compute_criterion( + outputs, targets, losses_to_compute=self.losses) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate decoder layer. + if aux_outputs_list is not None: + aux_losses_to_compute = self.losses.copy() + for i, aux_outputs in enumerate(aux_outputs_list): + losses_dict = self.compute_criterion(aux_outputs, targets, + aux_losses_to_compute) + losses_dict = {k + f'_{i}': v for k, v in losses_dict.items()} + losses.update(losses_dict) + + return losses + + def compute_criterion(self, outputs, targets, losses_to_compute): + # Retrieve the matching between the outputs of the last layer and the targets + indices = self.matcher(outputs, targets) + + # T & B dims are flattened so loss functions can be computed per frame (but with same indices per video). + # also, indices are repeated so so the same indices can be used for frames of the same video. + T = len(targets) + outputs, targets = flatten_temporal_batch_dims(outputs, targets) + # repeat the indices list T times so the same indices can be used for each video frame + indices = T * indices + + # Compute the average number of target masks across all nodes, for normalization purposes + num_masks = sum(len(t['masks']) for t in targets) + num_masks = torch.as_tensor([num_masks], + dtype=torch.float, + device=indices[0][0].device) + if is_dist_avail_and_initialized(): + torch.distributed.all_reduce(num_masks) + num_masks = torch.clamp(num_masks / get_world_size(), min=1).item() + + # Compute all the requested losses + losses = {} + for loss in losses_to_compute: + losses.update( + self.get_loss( + loss, outputs, targets, indices, num_masks=num_masks)) + return losses + + def loss_is_referred(self, outputs, targets, indices, **kwargs): + device = outputs['pred_is_referred'].device + bs = outputs['pred_is_referred'].shape[0] + pred_is_referred = outputs['pred_is_referred'].log_softmax( + dim=-1) # note that log-softmax is used here + target_is_referred = torch.zeros_like(pred_is_referred) + # extract indices of object queries that where matched with text-referred target objects + query_referred_indices = self._get_query_referred_indices( + indices, targets) + # by default penalize compared to the no-object class (last token) + target_is_referred[:, :, :] = torch.tensor([0.0, 1.0], device=device) + if 'is_ref_inst_visible' in targets[ + 0]: # visibility labels are available per-frame for the referred object: + is_ref_inst_visible_per_frame = torch.stack( + [t['is_ref_inst_visible'] for t in targets]) + ref_inst_visible_frame_indices = is_ref_inst_visible_per_frame.nonzero( + ).squeeze() + # keep only the matched query indices of the frames in which the referred object is visible: + visible_query_referred_indices = query_referred_indices[ + ref_inst_visible_frame_indices] + target_is_referred[ref_inst_visible_frame_indices, + visible_query_referred_indices] = torch.tensor( + [1.0, 0.0], device=device) + else: # assume that the referred object is visible in every frame: + target_is_referred[torch.arange(bs), + query_referred_indices] = torch.tensor( + [1.0, 0.0], device=device) + loss = -(pred_is_referred * target_is_referred).sum(-1) + # apply no-object class weights: + eos_coef = torch.full(loss.shape, self.eos_coef, device=loss.device) + eos_coef[torch.arange(bs), query_referred_indices] = 1.0 + loss = loss * eos_coef + bs = len(indices) + loss = loss.sum() / bs # sum and normalize the loss by the batch size + losses = {'loss_is_referred': loss} + return losses + + def loss_masks(self, outputs, targets, indices, num_masks, **kwargs): + assert 'pred_masks' in outputs + + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + src_masks = outputs['pred_masks'] + src_masks = src_masks[src_idx] + masks = [t['masks'] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + # upsample predictions to the target size + src_masks = interpolate( + src_masks[:, None], + size=target_masks.shape[-2:], + mode='bilinear', + align_corners=False) + src_masks = src_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(src_masks.shape) + losses = { + 'loss_sigmoid_focal': + sigmoid_focal_loss(src_masks, target_masks, num_masks), + 'loss_dice': + dice_loss(src_masks, target_masks, num_masks), + } + return losses + + @staticmethod + def _get_src_permutation_idx(indices): + # permute predictions following indices + batch_idx = torch.cat( + [torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) + src_idx = torch.cat([src for (src, _) in indices]) + return batch_idx, src_idx + + @staticmethod + def _get_tgt_permutation_idx(indices): + # permute targets following indices + batch_idx = torch.cat( + [torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) + tgt_idx = torch.cat([tgt for (_, tgt) in indices]) + return batch_idx, tgt_idx + + @staticmethod + def _get_query_referred_indices(indices, targets): + """ + extract indices of object queries that where matched with text-referred target objects + """ + query_referred_indices = [] + for (query_idxs, target_idxs), target in zip(indices, targets): + ref_query_idx = query_idxs[torch.where( + target_idxs == target['referred_instance_idx'])[0]] + query_referred_indices.append(ref_query_idx) + query_referred_indices = torch.cat(query_referred_indices) + return query_referred_indices + + def get_loss(self, loss, outputs, targets, indices, **kwargs): + loss_map = { + 'masks': self.loss_masks, + 'is_referred': self.loss_is_referred, + } + assert loss in loss_map, f'do you really want to compute {loss} loss?' + return loss_map[loss](outputs, targets, indices, **kwargs) + + +def flatten_temporal_batch_dims(outputs, targets): + for k in outputs.keys(): + if isinstance(outputs[k], torch.Tensor): + outputs[k] = outputs[k].flatten(0, 1) + else: # list + outputs[k] = [i for step_t in outputs[k] for i in step_t] + targets = [ + frame_t_target for step_t in targets for frame_t_target in step_t + ] + return outputs, targets diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/matcher.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..4f9b88e5eede2fc1facb7e19d150eb4d11f5d4b8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/matcher.py @@ -0,0 +1,163 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# Modified from DETR https://github.com/facebookresearch/detr +# Module to compute the matching cost and solve the corresponding LSAP. + +import torch +from scipy.optimize import linear_sum_assignment +from torch import nn + +from .misc import interpolate, nested_tensor_from_tensor_list + + +class HungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + + For efficiency reasons, the targets don't include the no_object. Because of this, in general, + there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, + while the others are un-matched (and thus treated as non-objects). + """ + + def __init__(self, cost_is_referred: float = 1, cost_dice: float = 1): + """Creates the matcher + + Params: + cost_is_referred: This is the relative weight of the reference cost in the total matching cost + cost_dice: This is the relative weight of the dice cost in the total matching cost + """ + super().__init__() + self.cost_is_referred = cost_is_referred + self.cost_dice = cost_dice + assert cost_is_referred != 0 or cost_dice != 0, 'all costs cant be 0' + + @torch.inference_mode() + def forward(self, outputs, targets): + """ Performs the matching + + Params: + outputs: A dict that contains at least these entries: + "pred_is_referred": Tensor of dim [time, batch_size, num_queries, 2] with the reference logits + "pred_masks": Tensor of dim [time, batch_size, num_queries, H, W] with the predicted masks logits + + targets: A list of lists of targets (outer - time steps, inner - batch samples). each target is a dict + which contain mask and reference ground truth information for a single frame. + + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_masks) + """ + t, bs, num_queries = outputs['pred_masks'].shape[:3] + + # We flatten to compute the cost matrices in a batch + out_masks = outputs['pred_masks'].flatten( + 1, 2) # [t, batch_size * num_queries, mask_h, mask_w] + + # preprocess and concat the target masks + tgt_masks = [[ + m for v in t_step_batch for m in v['masks'].unsqueeze(1) + ] for t_step_batch in targets] + # pad the target masks to a uniform shape + tgt_masks, valid = list( + zip(*[ + nested_tensor_from_tensor_list(t).decompose() + for t in tgt_masks + ])) + tgt_masks = torch.stack(tgt_masks).squeeze(2) + + # upsample predicted masks to target mask size + out_masks = interpolate( + out_masks, + size=tgt_masks.shape[-2:], + mode='bilinear', + align_corners=False) + + # Compute the soft-tokens cost: + if self.cost_is_referred > 0: + cost_is_referred = compute_is_referred_cost(outputs, targets) + else: + cost_is_referred = 0 + + # Compute the DICE coefficient between the masks: + if self.cost_dice > 0: + cost_dice = -dice_coef(out_masks, tgt_masks) + else: + cost_dice = 0 + + # Final cost matrix + C = self.cost_is_referred * cost_is_referred + self.cost_dice * cost_dice + C = C.view(bs, num_queries, -1).cpu() + + num_traj_per_batch = [ + len(v['masks']) for v in targets[0] + ] # number of instance trajectories in each batch + indices = [ + linear_sum_assignment(c[i]) + for i, c in enumerate(C.split(num_traj_per_batch, -1)) + ] + device = out_masks.device + return [(torch.as_tensor(i, dtype=torch.int64, device=device), + torch.as_tensor(j, dtype=torch.int64, device=device)) + for i, j in indices] + + +def dice_coef(inputs, targets, smooth=1.0): + """ + Compute the DICE coefficient, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid().flatten(2).unsqueeze(2) + targets = targets.flatten(2).unsqueeze(1) + numerator = 2 * (inputs * targets).sum(-1) + denominator = inputs.sum(-1) + targets.sum(-1) + coef = (numerator + smooth) / (denominator + smooth) + coef = coef.mean( + 0) # average on the temporal dim to get instance trajectory scores + return coef + + +def compute_is_referred_cost(outputs, targets): + pred_is_referred = outputs['pred_is_referred'].flatten(1, 2).softmax( + dim=-1) # [t, b*nq, 2] + device = pred_is_referred.device + t = pred_is_referred.shape[0] + # number of instance trajectories in each batch + num_traj_per_batch = torch.tensor([len(v['masks']) for v in targets[0]], + device=device) + total_trajectories = num_traj_per_batch.sum() + # note that ref_indices are shared across time steps: + ref_indices = torch.tensor( + [v['referred_instance_idx'] for v in targets[0]], device=device) + # convert ref_indices to fit flattened batch targets: + ref_indices += torch.cat( + (torch.zeros(1, dtype=torch.long, + device=device), num_traj_per_batch.cumsum(0)[:-1])) + # number of instance trajectories in each batch + target_is_referred = torch.zeros((t, total_trajectories, 2), device=device) + # 'no object' class by default (for un-referred objects) + target_is_referred[:, :, :] = torch.tensor([0.0, 1.0], device=device) + if 'is_ref_inst_visible' in targets[0][ + 0]: # visibility labels are available per-frame for the referred object: + is_ref_inst_visible = torch.stack([ + torch.stack([t['is_ref_inst_visible'] for t in t_step]) + for t_step in targets + ]).permute(1, 0) + for ref_idx, is_visible in zip(ref_indices, is_ref_inst_visible): + is_visible = is_visible.nonzero().squeeze() + target_is_referred[is_visible, + ref_idx, :] = torch.tensor([1.0, 0.0], + device=device) + else: # assume that the referred object is visible in every frame: + target_is_referred[:, ref_indices, :] = torch.tensor([1.0, 0.0], + device=device) + cost_is_referred = -(pred_is_referred.unsqueeze(2) + * target_is_referred.unsqueeze(1)).sum(dim=-1).mean( + dim=0) + return cost_is_referred diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/misc.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..ecf34b8cadc4322685fa89ddc4d3b52cef4cab58 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/misc.py @@ -0,0 +1,234 @@ +# Modified from DETR https://github.com/facebookresearch/detr +# Misc functions. +# Mostly copy-paste from torchvision references. + +import pickle +from typing import List, Optional + +import torch +import torch.distributed as dist +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +from torch import Tensor + +if float(torchvision.__version__.split('.')[1]) < 7.0: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to('cuda') + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device='cuda') + size_list = [torch.tensor([0], device='cuda') for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append( + torch.empty((max_size, ), dtype=torch.uint8, device='cuda')) + if local_size != max_size: + padding = torch.empty( + size=(max_size - local_size, ), dtype=torch.uint8, device='cuda') + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + """ + This function receives a list of image tensors and returns a NestedTensor of the padded images, along with their + padding masks (true for padding areas, false otherwise). + """ + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[:img.shape[0], :img.shape[1], :img.shape[2]].copy_(img) + m[:img.shape[1], :img.shape[2]] = False + return NestedTensor(tensor, mask) + + +def nested_tensor_from_videos_list(videos_list: List[Tensor]): + """ + This function receives a list of videos (each of shape [T, C, H, W]) and returns a NestedTensor of the padded + videos (shape [T, B, C, PH, PW], along with their padding masks (true for padding areas, false otherwise, of shape + [T, B, PH, PW]. + """ + max_size = _max_by_axis([list(img.shape) for img in videos_list]) + padded_batch_shape = [len(videos_list)] + max_size + b, t, c, h, w = padded_batch_shape + dtype = videos_list[0].dtype + device = videos_list[0].device + padded_videos = torch.zeros(padded_batch_shape, dtype=dtype, device=device) + videos_pad_masks = torch.ones((b, t, h, w), + dtype=torch.bool, + device=device) + for vid_frames, pad_vid_frames, vid_pad_m in zip(videos_list, + padded_videos, + videos_pad_masks): + pad_vid_frames[:vid_frames.shape[0], :, :vid_frames. + shape[2], :vid_frames.shape[3]].copy_(vid_frames) + vid_pad_m[:vid_frames.shape[0], :vid_frames.shape[2], :vid_frames. + shape[3]] = False + # transpose the temporal and batch dims and create a NestedTensor: + return NestedTensor( + padded_videos.transpose(0, 1), videos_pad_masks.transpose(0, 1)) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def interpolate(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if float(torchvision.__version__.split('.')[1]) < 7.0: + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, + mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, + mode, align_corners) diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/mttr.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/mttr.py new file mode 100644 index 0000000000000000000000000000000000000000..136208cd08519db9cb74cb9f2b9158f0d3a9cacd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/mttr.py @@ -0,0 +1,132 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR + +import torch +import torch.nn.functional as F +from einops import rearrange +from torch import nn + +from .backbone import init_backbone +from .misc import NestedTensor +from .multimodal_transformer import MultimodalTransformer +from .segmentation import FPNSpatialDecoder + + +class MTTR(nn.Module): + """ The main module of the Multimodal Tracking Transformer """ + + def __init__(self, + num_queries, + mask_kernels_dim=8, + aux_loss=False, + transformer_cfg_dir=None, + **kwargs): + """ + Parameters: + num_queries: number of object queries, ie detection slot. This is the maximal number of objects + MTTR can detect in a single image. In our paper we use 50 in all settings. + mask_kernels_dim: dim of the segmentation kernels and of the feature maps outputted by the spatial decoder. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + """ + super().__init__() + self.backbone = init_backbone(**kwargs) + assert transformer_cfg_dir is not None + self.transformer = MultimodalTransformer( + transformer_cfg_dir=transformer_cfg_dir, **kwargs) + d_model = self.transformer.d_model + self.is_referred_head = nn.Linear( + d_model, + 2) # binary 'is referred?' prediction head for object queries + self.instance_kernels_head = MLP( + d_model, d_model, output_dim=mask_kernels_dim, num_layers=2) + self.obj_queries = nn.Embedding( + num_queries, d_model) # pos embeddings for the object queries + self.vid_embed_proj = nn.Conv2d( + self.backbone.layer_output_channels[-1], d_model, kernel_size=1) + self.spatial_decoder = FPNSpatialDecoder( + d_model, self.backbone.layer_output_channels[:-1][::-1], + mask_kernels_dim) + self.aux_loss = aux_loss + + def forward(self, samples: NestedTensor, valid_indices, text_queries): + """The forward expects a NestedTensor, which consists of: + - samples.tensor: Batched frames of shape [time x batch_size x 3 x H x W] + - samples.mask: A binary mask of shape [time x batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_is_referred": The reference prediction logits for all queries. + Shape: [time x batch_size x num_queries x 2] + - "pred_masks": The mask logits for all queries. + Shape: [time x batch_size x num_queries x H_mask x W_mask] + - "aux_outputs": Optional, only returned when auxiliary losses are activated. It is a list of + dictionaries containing the two above keys for each decoder layer. + """ + backbone_out = self.backbone(samples) + # keep only the valid frames (frames which are annotated): + # (for example, in a2d-sentences only the center frame in each window is annotated). + for layer_out in backbone_out: + valid_indices = valid_indices.to(layer_out.tensors.device) + layer_out.tensors = layer_out.tensors.index_select( + 0, valid_indices) + layer_out.mask = layer_out.mask.index_select(0, valid_indices) + bbone_final_layer_output = backbone_out[-1] + vid_embeds, vid_pad_mask = bbone_final_layer_output.decompose() + + T, B, _, _, _ = vid_embeds.shape + vid_embeds = rearrange(vid_embeds, 't b c h w -> (t b) c h w') + vid_embeds = self.vid_embed_proj(vid_embeds) + vid_embeds = rearrange( + vid_embeds, '(t b) c h w -> t b c h w', t=T, b=B) + + transformer_out = self.transformer(vid_embeds, vid_pad_mask, + text_queries, + self.obj_queries.weight) + # hs is: [L, T, B, N, D] where L is number of decoder layers + # vid_memory is: [T, B, D, H, W] + # txt_memory is a list of length T*B of [S, C] where S might be different for each sentence + # encoder_middle_layer_outputs is a list of [T, B, H, W, D] + hs, vid_memory, txt_memory = transformer_out + + vid_memory = rearrange(vid_memory, 't b d h w -> (t b) d h w') + bbone_middle_layer_outputs = [ + rearrange(o.tensors, 't b d h w -> (t b) d h w') + for o in backbone_out[:-1][::-1] + ] + decoded_frame_features = self.spatial_decoder( + vid_memory, bbone_middle_layer_outputs) + decoded_frame_features = rearrange( + decoded_frame_features, '(t b) d h w -> t b d h w', t=T, b=B) + instance_kernels = self.instance_kernels_head(hs) # [L, T, B, N, C] + # output masks is: [L, T, B, N, H_mask, W_mask] + output_masks = torch.einsum('ltbnc,tbchw->ltbnhw', instance_kernels, + decoded_frame_features) + outputs_is_referred = self.is_referred_head(hs) # [L, T, B, N, 2] + + layer_outputs = [] + for pm, pir in zip(output_masks, outputs_is_referred): + layer_out = {'pred_masks': pm, 'pred_is_referred': pir} + layer_outputs.append(layer_out) + out = layer_outputs[ + -1] # the output for the last decoder layer is used by default + if self.aux_loss: + out['aux_outputs'] = layer_outputs[:-1] + return out + + def num_parameters(self): + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + +class MLP(nn.Module): + """ Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList( + nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/multimodal_transformer.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/multimodal_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..f750437aeafa663789ca6e5c3d835ce881afc035 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/multimodal_transformer.py @@ -0,0 +1,443 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# MTTR Multimodal Transformer class. +# Modified from DETR https://github.com/facebookresearch/detr + +import copy +import os +from typing import Optional + +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from torch import Tensor, nn +from transformers import RobertaModel, RobertaTokenizerFast + +from .position_encoding_2d import PositionEmbeddingSine2D + +os.environ[ + 'TOKENIZERS_PARALLELISM'] = 'false' # this disables a huggingface tokenizer warning (printed every epoch) + + +class MultimodalTransformer(nn.Module): + + def __init__(self, + num_encoder_layers=3, + num_decoder_layers=3, + text_encoder_type='roberta-base', + freeze_text_encoder=True, + transformer_cfg_dir=None, + **kwargs): + super().__init__() + self.d_model = kwargs['d_model'] + encoder_layer = TransformerEncoderLayer(**kwargs) + self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers) + decoder_layer = TransformerDecoderLayer(**kwargs) + self.decoder = TransformerDecoder( + decoder_layer, + num_decoder_layers, + norm=nn.LayerNorm(self.d_model), + return_intermediate=True) + self.pos_encoder_2d = PositionEmbeddingSine2D() + self._reset_parameters() + + if text_encoder_type != 'roberta-base': + transformer_cfg_dir = text_encoder_type + self.text_encoder = RobertaModel.from_pretrained(transformer_cfg_dir) + self.text_encoder.pooler = None # this pooler is never used, this is a hack to avoid DDP problems... + self.tokenizer = RobertaTokenizerFast.from_pretrained( + transformer_cfg_dir) + self.freeze_text_encoder = freeze_text_encoder + if freeze_text_encoder: + for p in self.text_encoder.parameters(): + p.requires_grad_(False) + + self.txt_proj = FeatureResizer( + input_feat_size=self.text_encoder.config.hidden_size, + output_feat_size=self.d_model, + dropout=kwargs['dropout'], + ) + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, vid_embeds, vid_pad_mask, text_queries, obj_queries): + device = vid_embeds.device + t, b, _, h, w = vid_embeds.shape + + txt_memory, txt_pad_mask = self.forward_text(text_queries, device) + # add temporal dim to txt memory & padding mask: + txt_memory = repeat(txt_memory, 's b c -> s (t b) c', t=t) + txt_pad_mask = repeat(txt_pad_mask, 'b s -> (t b) s', t=t) + + vid_embeds = rearrange(vid_embeds, 't b c h w -> (h w) (t b) c') + # Concat the image & text embeddings on the sequence dimension + encoder_src_seq = torch.cat((vid_embeds, txt_memory), dim=0) + seq_mask = torch.cat( + (rearrange(vid_pad_mask, 't b h w -> (t b) (h w)'), txt_pad_mask), + dim=1) + # vid_pos_embed is: [T*B, H, W, d_model] + vid_pos_embed = self.pos_encoder_2d( + rearrange(vid_pad_mask, 't b h w -> (t b) h w'), self.d_model) + # use zeros in place of pos embeds for the text sequence: + pos_embed = torch.cat( + (rearrange(vid_pos_embed, 't_b h w c -> (h w) t_b c'), + torch.zeros_like(txt_memory)), + dim=0) + + memory = self.encoder( + encoder_src_seq, src_key_padding_mask=seq_mask, + pos=pos_embed) # [S, T*B, C] + vid_memory = rearrange( + memory[:h * w, :, :], + '(h w) (t b) c -> t b c h w', + h=h, + w=w, + t=t, + b=b) + txt_memory = memory[h * w:, :, :] + txt_memory = rearrange(txt_memory, 's t_b c -> t_b s c') + txt_memory = [ + t_mem[~pad_mask] + for t_mem, pad_mask in zip(txt_memory, txt_pad_mask) + ] # remove padding + + # add T*B dims to query embeds (was: [N, C], where N is the number of object queries): + obj_queries = repeat(obj_queries, 'n c -> n (t b) c', t=t, b=b) + tgt = torch.zeros_like(obj_queries) # [N, T*B, C] + + # hs is [L, N, T*B, C] where L is number of layers in the decoder + hs = self.decoder( + tgt, + memory, + memory_key_padding_mask=seq_mask, + pos=pos_embed, + query_pos=obj_queries) + hs = rearrange(hs, 'l n (t b) c -> l t b n c', t=t, b=b) + return hs, vid_memory, txt_memory + + def forward_text(self, text_queries, device): + tokenized_queries = self.tokenizer.batch_encode_plus( + text_queries, padding='longest', return_tensors='pt') + tokenized_queries = tokenized_queries.to(device) + with torch.inference_mode(mode=self.freeze_text_encoder): + encoded_text = self.text_encoder(**tokenized_queries) + # Transpose memory because pytorch's attention expects sequence first + tmp_last_hidden_state = encoded_text.last_hidden_state.clone() + txt_memory = rearrange(tmp_last_hidden_state, 'b s c -> s b c') + txt_memory = self.txt_proj( + txt_memory) # change text embeddings dim to model dim + # Invert attention mask that we get from huggingface because its the opposite in pytorch transformer + txt_pad_mask = tokenized_queries.attention_mask.ne(1).bool() # [B, S] + return txt_memory, txt_pad_mask + + def num_parameters(self): + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + +class TransformerEncoder(nn.Module): + + def __init__(self, encoder_layer, num_layers, norm=None): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward(self, + src, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + output = src + + for layer in self.layers: + output = layer( + output, + src_mask=mask, + src_key_padding_mask=src_key_padding_mask, + pos=pos) + + if self.norm is not None: + output = self.norm(output) + + return output + + +class TransformerDecoder(nn.Module): + + def __init__(self, + decoder_layer, + num_layers, + norm=None, + return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + output = tgt + + intermediate = [] + + for layer in self.layers: + output = layer( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + tgt_key_padding_mask=tgt_key_padding_mask, + memory_key_padding_mask=memory_key_padding_mask, + pos=pos, + query_pos=query_pos) + if self.return_intermediate: + intermediate.append(self.norm(output)) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate) + + return output.unsqueeze(0) + + +class TransformerEncoderLayer(nn.Module): + + def __init__(self, + d_model, + nheads, + dim_feedforward=2048, + dropout=0.1, + activation='relu', + normalize_before=False, + **kwargs): + super().__init__() + self.self_attn = nn.MultiheadAttention( + d_model, nheads, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(src, pos) + src2 = self.self_attn( + q, + k, + value=src, + attn_mask=src_mask, + key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + src = src + self.dropout2(src2) + src = self.norm2(src) + return src + + def forward_pre(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + src2 = self.norm1(src) + q = k = self.with_pos_embed(src2, pos) + src2 = self.self_attn( + q, + k, + value=src2, + attn_mask=src_mask, + key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src2 = self.norm2(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src2)))) + src = src + self.dropout2(src2) + return src + + def forward(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(src, src_mask, src_key_padding_mask, pos) + return self.forward_post(src, src_mask, src_key_padding_mask, pos) + + +class TransformerDecoderLayer(nn.Module): + + def __init__(self, + d_model, + nheads, + dim_feedforward=2048, + dropout=0.1, + activation='relu', + normalize_before=False, + **kwargs): + super().__init__() + self.self_attn = nn.MultiheadAttention( + d_model, nheads, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nheads, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn( + q, + k, + value=tgt, + attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout3(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward_pre(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm1(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn( + q, + k, + value=tgt2, + attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt2 = self.norm2(tgt) + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt2 = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout3(tgt2) + return tgt + + def forward(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, memory, tgt_mask, memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask, pos, query_pos) + return self.forward_post(tgt, memory, tgt_mask, memory_mask, + tgt_key_padding_mask, memory_key_padding_mask, + pos, query_pos) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class FeatureResizer(nn.Module): + """ + This class takes as input a set of embeddings of dimension C1 and outputs a set of + embedding of dimension C2, after a linear transformation, dropout and normalization (LN). + """ + + def __init__(self, input_feat_size, output_feat_size, dropout, do_ln=True): + super().__init__() + self.do_ln = do_ln + # Object feature encoding + self.fc = nn.Linear(input_feat_size, output_feat_size, bias=True) + self.layer_norm = nn.LayerNorm(output_feat_size, eps=1e-12) + self.dropout = nn.Dropout(dropout) + + def forward(self, encoder_features): + x = self.fc(encoder_features) + if self.do_ln: + x = self.layer_norm(x) + output = self.dropout(x) + return output + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == 'relu': + return F.relu + if activation == 'gelu': + return F.gelu + if activation == 'glu': + return F.glu + raise RuntimeError(F'activation should be relu/gelu, not {activation}.') diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/position_encoding_2d.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/position_encoding_2d.py new file mode 100644 index 0000000000000000000000000000000000000000..f9ef05a1c3eda616f3a734e13103bd1bc621d42e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/position_encoding_2d.py @@ -0,0 +1,57 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# Modified from DETR https://github.com/facebookresearch/detr +# 2D sine positional encodings for the visual features in the multimodal transformer. + +import math + +import torch +from torch import Tensor, nn + + +class PositionEmbeddingSine2D(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, temperature=10000, normalize=True, scale=None): + super().__init__() + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError('normalize should be True if scale is passed') + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, mask: Tensor, hidden_dim: int): + """ + @param mask: a tensor of shape [B, H, W] + @param hidden_dim: int + @return: + """ + num_pos_feats = hidden_dim // 2 + + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange( + num_pos_feats, dtype=torch.float32, device=mask.device) + dim_t = self.temperature**(2 * (dim_t // 2) / num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), + dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3) + return pos diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/postprocessing.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/postprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..b979268847d918ff5e2980d94900f9327965abb4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/postprocessing.py @@ -0,0 +1,119 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR + +import numpy as np +import pycocotools.mask as mask_util +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + + +class A2DSentencesPostProcess(nn.Module): + """ + This module converts the model's output into the format expected by the coco api for the given task + """ + + def __init__(self): + super(A2DSentencesPostProcess, self).__init__() + + @torch.inference_mode() + def forward(self, outputs, resized_padded_sample_size, + resized_sample_sizes, orig_sample_sizes): + """ Perform the computation + Parameters: + outputs: raw outputs of the model + resized_padded_sample_size: size of samples (input to model) after size augmentation + padding. + resized_sample_sizes: size of samples after size augmentation but without padding. + orig_sample_sizes: original size of the samples (no augmentations or padding) + """ + pred_is_referred = outputs['pred_is_referred'] + prob = F.softmax(pred_is_referred, dim=-1) + scores = prob[..., 0] + pred_masks = outputs['pred_masks'] + pred_masks = F.interpolate( + pred_masks, + size=resized_padded_sample_size, + mode='bilinear', + align_corners=False) + pred_masks = (pred_masks.sigmoid() > 0.5) + processed_pred_masks, rle_masks = [], [] + for f_pred_masks, resized_size, orig_size in zip( + pred_masks, resized_sample_sizes, orig_sample_sizes): + f_mask_h, f_mask_w = resized_size # resized shape without padding + # remove the samples' padding + f_pred_masks_no_pad = f_pred_masks[:, :f_mask_h, : + f_mask_w].unsqueeze(1) + # resize the samples back to their original dataset (target) size for evaluation + f_pred_masks_processed = F.interpolate( + f_pred_masks_no_pad.float(), size=orig_size, mode='nearest') + f_pred_rle_masks = [ + mask_util.encode( + np.array( + mask[0, :, :, np.newaxis], dtype=np.uint8, + order='F'))[0] + for mask in f_pred_masks_processed.cpu() + ] + processed_pred_masks.append(f_pred_masks_processed) + rle_masks.append(f_pred_rle_masks) + predictions = [{ + 'scores': s, + 'masks': m, + 'rle_masks': rle + } for s, m, rle in zip(scores, processed_pred_masks, rle_masks)] + return predictions + + +class ReferYoutubeVOSPostProcess(nn.Module): + """ + This module converts the model's output into the format expected by the coco api for the given task + """ + + def __init__(self): + super(ReferYoutubeVOSPostProcess, self).__init__() + + @torch.inference_mode() + def forward(self, outputs, videos_metadata, samples_shape_with_padding): + """ Perform the computation + Parameters: + outputs: raw outputs of the model + videos_metadata: a dictionary with each video's metadata. + samples_shape_with_padding: size of the batch frames with padding. + """ + pred_is_referred = outputs['pred_is_referred'] + prob_is_referred = F.softmax(pred_is_referred, dim=-1) + # note we average on the temporal dim to compute score per trajectory: + trajectory_scores = prob_is_referred[..., 0].mean(dim=0) + pred_trajectory_indices = torch.argmax(trajectory_scores, dim=-1) + pred_masks = rearrange(outputs['pred_masks'], + 't b nq h w -> b t nq h w') + # keep only the masks of the chosen trajectories: + b = pred_masks.shape[0] + pred_masks = pred_masks[torch.arange(b), :, pred_trajectory_indices] + # resize the predicted masks to the size of the model input (which might include padding) + pred_masks = F.interpolate( + pred_masks, + size=samples_shape_with_padding, + mode='bilinear', + align_corners=False) + # apply a threshold to create binary masks: + pred_masks = (pred_masks.sigmoid() > 0.5) + # remove the padding per video (as videos might have different resolutions and thus different padding): + preds_by_video = [] + for video_pred_masks, video_metadata in zip(pred_masks, + videos_metadata): + # size of the model input batch frames without padding: + resized_h, resized_w = video_metadata['resized_frame_size'] + video_pred_masks = video_pred_masks[:, :resized_h, : + resized_w].unsqueeze( + 1) # remove the padding + # resize the masks back to their original frames dataset size for evaluation: + original_frames_size = video_metadata['original_frame_size'] + tuple_size = tuple(original_frames_size.cpu()) + video_pred_masks = F.interpolate( + video_pred_masks.float(), size=tuple_size, mode='nearest') + video_pred_masks = video_pred_masks.to(torch.uint8).cpu() + # combine the predicted masks and the video metadata to create a final predictions dict: + video_pred = {**video_metadata, **{'pred_masks': video_pred_masks}} + preds_by_video.append(video_pred) + return preds_by_video diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/segmentation.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..b3228820e95d9f5cc6f4a42c31f1786b9ef8381c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/segmentation.py @@ -0,0 +1,137 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# Modified from DETR https://github.com/facebookresearch/detr + +from typing import List + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor + + +class FPNSpatialDecoder(nn.Module): + """ + An FPN-like spatial decoder. Generates high-res, semantically rich features which serve as the base for creating + instance segmentation masks. + """ + + def __init__(self, context_dim, fpn_dims, mask_kernels_dim=8): + super().__init__() + + inter_dims = [ + context_dim, context_dim // 2, context_dim // 4, context_dim // 8, + context_dim // 16 + ] + self.lay1 = torch.nn.Conv2d(context_dim, inter_dims[0], 3, padding=1) + self.gn1 = torch.nn.GroupNorm(8, inter_dims[0]) + self.lay2 = torch.nn.Conv2d(inter_dims[0], inter_dims[1], 3, padding=1) + self.gn2 = torch.nn.GroupNorm(8, inter_dims[1]) + self.lay3 = torch.nn.Conv2d(inter_dims[1], inter_dims[2], 3, padding=1) + self.gn3 = torch.nn.GroupNorm(8, inter_dims[2]) + self.lay4 = torch.nn.Conv2d(inter_dims[2], inter_dims[3], 3, padding=1) + self.gn4 = torch.nn.GroupNorm(8, inter_dims[3]) + self.adapter1 = torch.nn.Conv2d(fpn_dims[0], inter_dims[1], 1) + self.adapter2 = torch.nn.Conv2d(fpn_dims[1], inter_dims[2], 1) + self.context_dim = context_dim + + self.add_extra_layer = len(fpn_dims) == 3 + if self.add_extra_layer: + self.adapter3 = torch.nn.Conv2d(fpn_dims[2], inter_dims[3], 1) + self.lay5 = torch.nn.Conv2d( + inter_dims[3], inter_dims[4], 3, padding=1) + self.gn5 = torch.nn.GroupNorm(8, inter_dims[4]) + self.out_lay = torch.nn.Conv2d( + inter_dims[4], mask_kernels_dim, 3, padding=1) + else: + self.out_lay = torch.nn.Conv2d( + inter_dims[3], mask_kernels_dim, 3, padding=1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_uniform_(m.weight, a=1) + nn.init.constant_(m.bias, 0) + + def forward(self, x: Tensor, layer_features: List[Tensor]): + x = self.lay1(x) + x = self.gn1(x) + x = F.relu(x) + x = self.lay2(x) + x = self.gn2(x) + x = F.relu(x) + + cur_fpn = self.adapter1(layer_features[0]) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode='nearest') + x = self.lay3(x) + x = self.gn3(x) + x = F.relu(x) + + cur_fpn = self.adapter2(layer_features[1]) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode='nearest') + x = self.lay4(x) + x = self.gn4(x) + x = F.relu(x) + + if self.add_extra_layer: + cur_fpn = self.adapter3(layer_features[2]) + x = cur_fpn + F.interpolate( + x, size=cur_fpn.shape[-2:], mode='nearest') + x = self.lay5(x) + x = self.gn5(x) + x = F.relu(x) + + x = self.out_lay(x) + return x + + def num_parameters(self): + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + +def dice_loss(inputs, targets, num_masks): + """ + Compute the DICE loss, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid() + numerator = 2 * (inputs * targets).sum(1) + denominator = inputs.sum(-1) + targets.sum(-1) + loss = 1 - (numerator + 1) / (denominator + 1) + return loss.sum() / num_masks + + +def sigmoid_focal_loss(inputs, + targets, + num_masks, + alpha: float = 0.25, + gamma: float = 2): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + alpha: (optional) Weighting factor in range (0,1) to balance + positive vs negative examples. Default = -1 (no weighting). + gamma: Exponent of the modulating factor (1 - p_t) to + balance easy vs hard examples. + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = F.binary_cross_entropy_with_logits( + inputs, targets, reduction='none') + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t)**gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.mean(1).sum() / num_masks diff --git a/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/swin_transformer.py b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..faaf6e10040b146bd224e8b65946511b9ff7313c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/referring_video_object_segmentation/utils/swin_transformer.py @@ -0,0 +1,732 @@ +# The implementation is adopted from MTTR, +# made publicly available under the Apache 2.0 License at https://github.com/mttr2021/MTTR +# Modified from Video-Swin-Transformer https://github.com/SwinTransformer/Video-Swin-Transformer + +from functools import lru_cache, reduce +from operator import mul + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from einops import rearrange +from timm.models.layers import DropPath, trunc_normal_ + + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, D, H, W, C) + window_size (tuple[int]): window size + + Returns: + windows: (B*num_windows, window_size*window_size, C) + """ + B, D, H, W, C = x.shape + x = x.view(B, D // window_size[0], window_size[0], H // window_size[1], + window_size[1], W // window_size[2], window_size[2], C) + windows = x.permute(0, 1, 3, 5, 2, 4, 6, + 7).contiguous().view(-1, reduce(mul, window_size), C) + return windows + + +def window_reverse(windows, window_size, B, D, H, W): + """ + Args: + windows: (B*num_windows, window_size, window_size, C) + window_size (tuple[int]): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, D, H, W, C) + """ + x = windows.view(B, D // window_size[0], H // window_size[1], + W // window_size[2], window_size[0], window_size[1], + window_size[2], -1) + x = x.permute(0, 1, 4, 2, 5, 3, 6, 7).contiguous().view(B, D, H, W, -1) + return x + + +def get_window_size(x_size, window_size, shift_size=None): + use_window_size = list(window_size) + if shift_size is not None: + use_shift_size = list(shift_size) + for i in range(len(x_size)): + if x_size[i] <= window_size[i]: + use_window_size[i] = x_size[i] + if shift_size is not None: + use_shift_size[i] = 0 + + if shift_size is None: + return tuple(use_window_size) + else: + return tuple(use_window_size), tuple(use_shift_size) + + +class WindowAttention3D(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The temporal length, height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wd, Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + wd, wh, ww = window_size + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * wd - 1) * (2 * wh - 1) * (2 * ww - 1), + num_heads)) # 2*Wd-1 * 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_d = torch.arange(self.window_size[0]) + coords_h = torch.arange(self.window_size[1]) + coords_w = torch.arange(self.window_size[2]) + coords = torch.stack(torch.meshgrid(coords_d, coords_h, + coords_w)) # 3, Wd, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 3, Wd*Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 3, Wd*Wh*Ww, Wd*Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wd*Wh*Ww, Wd*Wh*Ww, 3 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 2] += self.window_size[2] - 1 + + relative_coords[:, :, 0] *= (2 * self.window_size[1] + - 1) * (2 * self.window_size[2] - 1) + relative_coords[:, :, 1] *= (2 * self.window_size[2] - 1) + relative_position_index = relative_coords.sum(-1) # Wd*Wh*Ww, Wd*Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, N, N) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # B_, nH, N, C + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index[:N, :N].reshape(-1)].reshape( + N, N, -1) # Wd*Wh*Ww,Wd*Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wd*Wh*Ww, Wd*Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) # B_, nH, N, N + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock3D(nn.Module): + """ Swin Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (tuple[int]): Window size. + shift_size (tuple[int]): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + window_size=(2, 7, 7), + shift_size=(0, 0, 0), + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + use_checkpoint=False): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.use_checkpoint = use_checkpoint + + assert 0 <= self.shift_size[0] < self.window_size[ + 0], 'shift_size must in 0-window_size' + assert 0 <= self.shift_size[1] < self.window_size[ + 1], 'shift_size must in 0-window_size' + assert 0 <= self.shift_size[2] < self.window_size[ + 2], 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention3D( + dim, + window_size=self.window_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + def forward_part1(self, x, mask_matrix): + B, D, H, W, C = x.shape + window_size, shift_size = get_window_size((D, H, W), self.window_size, + self.shift_size) + + x = self.norm1(x) + # pad feature maps to multiples of window size + pad_l = pad_t = pad_d0 = 0 + pad_d1 = (window_size[0] - D % window_size[0]) % window_size[0] + pad_b = (window_size[1] - H % window_size[1]) % window_size[1] + pad_r = (window_size[2] - W % window_size[2]) % window_size[2] + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b, pad_d0, pad_d1)) + _, Dp, Hp, Wp, _ = x.shape + # cyclic shift + if any(i > 0 for i in shift_size): + shifted_x = torch.roll( + x, + shifts=(-shift_size[0], -shift_size[1], -shift_size[2]), + dims=(1, 2, 3)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + # partition windows + x_windows = window_partition(shifted_x, + window_size) # B*nW, Wd*Wh*Ww, C + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask) # B*nW, Wd*Wh*Ww, C + # merge windows + attn_windows = attn_windows.view(-1, *(window_size + (C, ))) + shifted_x = window_reverse(attn_windows, window_size, B, Dp, Hp, + Wp) # B D' H' W' C + # reverse cyclic shift + if any(i > 0 for i in shift_size): + x = torch.roll( + shifted_x, + shifts=(shift_size[0], shift_size[1], shift_size[2]), + dims=(1, 2, 3)) + else: + x = shifted_x + + if pad_d1 > 0 or pad_r > 0 or pad_b > 0: + x = x[:, :D, :H, :W, :].contiguous() + return x + + def forward_part2(self, x): + return self.drop_path(self.mlp(self.norm2(x))) + + def forward(self, x, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, D, H, W, C). + mask_matrix: Attention mask for cyclic shift. + """ + + shortcut = x + if self.use_checkpoint: + x = checkpoint.checkpoint(self.forward_part1, x, mask_matrix) + else: + x = self.forward_part1(x, mask_matrix) + x = shortcut + self.drop_path(x) + + if self.use_checkpoint: + x = x + checkpoint.checkpoint(self.forward_part2, x) + else: + x = x + self.forward_part2(x) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ Forward function. + + Args: + x: Input feature, tensor size (B, D, H, W, C). + """ + B, D, H, W, C = x.shape + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, :, 0::2, 0::2, :] # B D H/2 W/2 C + x1 = x[:, :, 1::2, 0::2, :] # B D H/2 W/2 C + x2 = x[:, :, 0::2, 1::2, :] # B D H/2 W/2 C + x3 = x[:, :, 1::2, 1::2, :] # B D H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B D H/2 W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +# cache each stage results +@lru_cache() +def compute_mask(D, H, W, window_size, shift_size, device): + img_mask = torch.zeros((1, D, H, W, 1), device=device) # 1 Dp Hp Wp 1 + cnt = 0 + for d in slice(-window_size[0]), slice(-window_size[0], + -shift_size[0]), slice( + -shift_size[0], None): + for h in slice(-window_size[1]), slice(-window_size[1], + -shift_size[1]), slice( + -shift_size[1], None): + for w in slice(-window_size[2]), slice(-window_size[2], + -shift_size[2]), slice( + -shift_size[2], None): + img_mask[:, d, h, w, :] = cnt + cnt += 1 + mask_windows = window_partition(img_mask, + window_size) # nW, ws[0]*ws[1]*ws[2], 1 + mask_windows = mask_windows.squeeze(-1) # nW, ws[0]*ws[1]*ws[2] + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + return attn_mask + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (tuple[int]): Local window size. Default: (1,7,7). + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=(1, 7, 7), + mlp_ratio=4., + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = tuple(i // 2 for i in window_size) + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock3D( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=(0, 0, 0) if (i % 2 == 0) else self.shift_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + use_checkpoint=use_checkpoint, + ) for i in range(depth) + ]) + + self.downsample = downsample + if self.downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + + def forward(self, x): + """ Forward function. + + Args: + x: Input feature, tensor size (B, C, D, H, W). + """ + # calculate attention mask for SW-MSA + B, C, D, H, W = x.shape + window_size, shift_size = get_window_size((D, H, W), self.window_size, + self.shift_size) + x = rearrange(x, 'b c d h w -> b d h w c') + Dp = int(np.ceil(D / window_size[0])) * window_size[0] + Hp = int(np.ceil(H / window_size[1])) * window_size[1] + Wp = int(np.ceil(W / window_size[2])) * window_size[2] + attn_mask = compute_mask(Dp, Hp, Wp, window_size, shift_size, x.device) + for blk in self.blocks: + x = blk(x, attn_mask) + x = x.view(B, D, H, W, -1) + + if self.downsample is not None: + x = self.downsample(x) + x = rearrange(x, 'b d h w c -> b c d h w') + return x + + +class PatchEmbed3D(nn.Module): + """ Video to Patch Embedding. + + Args: + patch_size (int): Patch token size. Default: (2,4,4). + in_chans (int): Number of input video channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=(2, 4, 4), + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv3d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, D, H, W = x.size() + if W % self.patch_size[2] != 0: + x = F.pad(x, (0, self.patch_size[2] - W % self.patch_size[2])) + if H % self.patch_size[1] != 0: + x = F.pad(x, + (0, 0, 0, self.patch_size[1] - H % self.patch_size[1])) + if D % self.patch_size[0] != 0: + x = F.pad( + x, + (0, 0, 0, 0, 0, self.patch_size[0] - D % self.patch_size[0])) + + x = self.proj(x) # B C D Wh Ww + if self.norm is not None: + D, Wh, Ww = x.size(2), x.size(3), x.size(4) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, D, Wh, Ww) + + return x + + +class SwinTransformer3D(nn.Module): + """ Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + + Args: + patch_size (int | tuple(int)): Patch size. Default: (4,4,4). + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: Truee + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer: Normalization layer. Default: nn.LayerNorm. + patch_norm (bool): If True, add normalization after patch embedding. Default: False. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + """ + + def __init__(self, + pretrained=None, + pretrained2d=True, + patch_size=(4, 4, 4), + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=(2, 7, 7), + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + patch_norm=False, + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrained = pretrained + self.pretrained2d = pretrained2d + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.patch_norm = patch_norm + self.frozen_stages = frozen_stages + self.window_size = window_size + self.patch_size = patch_size + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed3D( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging + if i_layer < self.num_layers - 1 else None, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + self.num_features = int(embed_dim * 2**(self.num_layers - 1)) + + # add a norm layer for each output + self.norm = norm_layer(self.num_features) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1: + self.pos_drop.eval() + for i in range(0, self.frozen_stages): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def inflate_weights(self, logger): + """Inflate the swin2d parameters to swin3d. + + The differences between swin3d and swin2d mainly lie in an extra + axis. To utilize the pretrained parameters in 2d model, + the weight of swin2d models should be inflated to fit in the shapes of + the 3d counterpart. + + Args: + logger (logging.Logger): The logger used to print + debugging infomation. + """ + checkpoint = torch.load(self.pretrained, map_location='cpu') + state_dict = checkpoint['model'] + + # delete relative_position_index since we always re-init it + relative_position_index_keys = [ + k for k in state_dict.keys() if 'relative_position_index' in k + ] + for k in relative_position_index_keys: + del state_dict[k] + + # delete attn_mask since we always re-init it + attn_mask_keys = [k for k in state_dict.keys() if 'attn_mask' in k] + for k in attn_mask_keys: + del state_dict[k] + + state_dict['patch_embed.proj.weight'] = state_dict[ + 'patch_embed.proj.weight'].unsqueeze(2).repeat( + 1, 1, self.patch_size[0], 1, 1) / self.patch_size[0] + + # bicubic interpolate relative_position_bias_table if not match + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for k in relative_position_bias_table_keys: + relative_position_bias_table_pretrained = state_dict[k] + relative_position_bias_table_current = self.state_dict()[k] + L1, nH1 = relative_position_bias_table_pretrained.size() + L2, nH2 = relative_position_bias_table_current.size() + L2 = (2 * self.window_size[1] - 1) * (2 * self.window_size[2] - 1) + wd = self.window_size[0] + if nH1 != nH2: + logger.warning(f'Error in loading {k}, passing') + else: + if L1 != L2: + S1 = int(L1**0.5) + relative_position_bias_table_pretrained_resized = torch.nn.functional.interpolate( + relative_position_bias_table_pretrained.permute( + 1, 0).view(1, nH1, S1, S1), + size=(2 * self.window_size[1] - 1, + 2 * self.window_size[2] - 1), + mode='bicubic') + relative_position_bias_table_pretrained = relative_position_bias_table_pretrained_resized.view( + nH2, L2).permute(1, 0) + state_dict[k] = relative_position_bias_table_pretrained.repeat( + 2 * wd - 1, 1) + + msg = self.load_state_dict(state_dict, strict=False) + logger.info(msg) + logger.info(f"=> loaded successfully '{self.pretrained}'") + del checkpoint + torch.cuda.empty_cache() + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + x = self.pos_drop(x) + + for layer in self.layers: + x = layer(x.contiguous()) + + x = rearrange(x, 'n c d h w -> n d h w c') + x = self.norm(x) + x = rearrange(x, 'n d h w c -> n c d h w') + + return x + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer3D, self).train(mode) + self._freeze_stages() diff --git a/runtime/Lib/site_packages/models/cv/robust_image_classification/__init__.py b/runtime/Lib/site_packages/models/cv/robust_image_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e71b2376c48a0f8db1ccd20e979fc5dfbea6bb43 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/robust_image_classification/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .easyrobust_model import EasyRobustModel + +else: + _import_structure = { + 'easyrobust_model': ['EasyRobustModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/robust_image_classification/easyrobust_model.py b/runtime/Lib/site_packages/models/cv/robust_image_classification/easyrobust_model.py new file mode 100644 index 0000000000000000000000000000000000000000..96c0d391d374b9c14880e614502928025a68626a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/robust_image_classification/easyrobust_model.py @@ -0,0 +1,83 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import torch +import torch.nn as nn + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks + + +def normalize_fn(tensor, mean, std): + """Differentiable version of torchvision.functional.normalize""" + # here we assume the color channel is in at dim=1 + mean = mean[None, :, None, None] + std = std[None, :, None, None] + return tensor.sub(mean).div(std) + + +class NormalizeByChannelMeanStd(nn.Module): + + def __init__(self, mean, std): + super(NormalizeByChannelMeanStd, self).__init__() + if not isinstance(mean, torch.Tensor): + mean = torch.tensor(mean) + if not isinstance(std, torch.Tensor): + std = torch.tensor(std) + self.register_buffer('mean', mean) + self.register_buffer('std', std) + + def forward(self, tensor): + return normalize_fn(tensor, self.mean, self.std) + + def extra_repr(self): + return 'mean={}, std={}'.format(self.mean, self.std) + + +@MODELS.register_module( + Tasks.image_classification, module_name=Models.easyrobust_model) +class EasyRobustModel(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + import easyrobust.models + from timm.models import create_model + from mmcls.datasets import ImageNet + import modelscope.models.cv.image_classification.backbones + from modelscope.utils.hub import read_config + + super().__init__(model_dir) + + self.config_type = 'ms_config' + self.CLASSES = ImageNet.CLASSES + cfg = read_config(model_dir) + cfg.model.mm_model.pretrained = None + self.cls_model = create_model( + cfg.model.mm_model['type'], pretrained=False, num_classes=1000) + + model_pth_path = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + normalize = NormalizeByChannelMeanStd( + mean=cfg.model.mm_model['mean'], std=cfg.model.mm_model['std']) + checkpoint = torch.load(model_pth_path, map_location='cpu') + + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + if '0.mean' in state_dict.keys() and '0.std' in state_dict.keys(): + self.cls_model = nn.Sequential(normalize, self.cls_model) + self.cls_model.load_state_dict(state_dict) + else: + self.cls_model.load_state_dict(state_dict) + self.cls_model = nn.Sequential(normalize, self.cls_model) + + self.cfg = cfg + self.ms_model_dir = model_dir + + def forward(self, inputs): + logits = self.cls_model(inputs['img']) + score = torch.nn.functional.softmax(logits, dim=1).cpu().numpy() + return score diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..089a397f3959d9606be1ce721a2571c07fe9ddf6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .s2net_model import PanoramaDepthEstimation + +else: + _import_structure = { + 's2net_model': ['PanoramaDepthEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/__init__.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cc4e1291611754f678996378feed4e77831673f8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .config import get_config +from .model import EffnetSphDecoderNet, ResnetSphDecoderNet, SwinSphDecoderNet +from .util_helper import compute_hp_info, render_depth_map diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/config.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/config.py new file mode 100644 index 0000000000000000000000000000000000000000..00a230a310b5785ad6055bc45eb5ea352664a775 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/config.py @@ -0,0 +1,164 @@ +# -------------------------------------------------------- +# We adopt this configuration method like swin-transfomer +# (https://github.com/microsoft/Swin-Transformer.git), +# made publicly available under the MIT License +# --------------------------------------------------------' + +import os + +import yaml +from yacs.config import CfgNode + +cfg = CfgNode() + +# Base config files +cfg.BASE = [''] + +# ----------------------------------------------------------------------------- +# Data settings +# ----------------------------------------------------------------------------- +cfg.DATA = CfgNode() +# Batch size for a single GPU, could be overwritten by command line argument +# Number of data loading threads +cfg.DATA.NUM_WORKERS = 8 +# Pin CPU memory in DataLoader for more efficient (sometimes) transfer to GPU. +cfg.DATA.PIN_MEMORY = True +# some dataset supported +cfg.DATA.DATASET_NAME = 'Matterport3D' # Stanford3D, PanoSUNCG3D, 3D60, Pano3D +cfg.DATA.IMG_WIDTH = 1024 +cfg.DATA.IMG_HEIGHT = 512 + +# ----------------------------------------------------------------------------- +# Model settings +# ----------------------------------------------------------------------------- +cfg.MODEL = CfgNode() +cfg.MODEL.DECODER_DIM = 256 +cfg.MODEL.USE_CAF_FUSION = True +cfg.MODEL.MAX_DEPTH = 10.0 + +# ----------------------------------------------------------------------------- +# Backbone settings +# ----------------------------------------------------------------------------- +cfg.BACKBONE = CfgNode() +cfg.BACKBONE.TYPE = 'swin' + +# for swin +cfg.BACKBONE.NAME = 'swin_base_patch4_window7_224' +cfg.BACKBONE.VERSION = 'base' +cfg.BACKBONE.PRETRAIN_RES = 224 +cfg.BACKBONE.PRETRAIN_IMAGENET = '22k' +cfg.BACKBONE.DROP_PATH_RATE = 0.5 +cfg.BACKBONE.EMBED_DIM = 128 +cfg.BACKBONE.DEPTHS = [2, 2, 18, 2] +cfg.BACKBONE.NUM_HEADS = [4, 8, 16, 32] +cfg.BACKBONE.WINDOW_SIZE = 7 +cfg.BACKBONE.FROZEN_STAGES = -1 +cfg.BACKBONE.PRETRAIN_LR_SCALE = 1.0 + +# for resnet +cfg.BACKBONE.RESNET_LAYER_NUM = 50 + +# ----------------------------------------------------------------------------- +# Training settings +# ----------------------------------------------------------------------------- +cfg.TRAIN = CfgNode() +# device +cfg.TRAIN.RANK = 0 +cfg.TRAIN.WORLD_SIZE = 1 +cfg.TRAIN.BACKEND = 'nccl' +cfg.TRAIN.IP_ADDR = 'localhost:' +cfg.TRAIN.PORT = '8000' +cfg.TRAIN.INIT_METHOD = 'tcp://' + cfg.TRAIN.IP_ADDR + cfg.TRAIN.PORT +cfg.TRAIN.BATCH_SIZE = 1 +cfg.TRAIN.START_EPOCH = 0 +cfg.TRAIN.EPOCHS = 300 +cfg.TRAIN.WARMUP_EPOCHS = 5 +cfg.TRAIN.WEIGHT_DECAY = 0.05 +cfg.TRAIN.BASE_LR = 0.001 +cfg.TRAIN.WARMUP_LR = 0.001 +cfg.TRAIN.MIN_LR = 0.001 +# Clip gradient norm +cfg.TRAIN.CLIP_GRAD = 5.0 +# Auto resume from latest checkpoint +cfg.TRAIN.AUTO_RESUME = True +cfg.TRAIN.RESUME = '' +# Gradient accumulation steps +cfg.TRAIN.ACCUMULATION_STEPS = 0 +# LR scheduler +cfg.TRAIN.LR_SCHEDULER = CfgNode() +cfg.TRAIN.LR_SCHEDULER.NAME = 'step' +# Epoch interval to decay LR, used in StepLRScheduler +cfg.TRAIN.LR_SCHEDULER.DECAY_EPOCHS = 3 +# LR decay rate, used in StepLRScheduler +cfg.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.9 +# Optimizer +cfg.TRAIN.OPTIMIZER = CfgNode() +cfg.TRAIN.OPTIMIZER.NAME = 'adamw' +# Optimizer Epsilon +cfg.TRAIN.OPTIMIZER.EPS = 1e-8 +# Optimizer BetasWEIGHT_DECAY +cfg.TRAIN.OPTIMIZER.BETAS = (0.9, 0.999) +# SGD momentum +cfg.TRAIN.OPTIMIZER.MOMENTUM = 0.9 +cfg.TRAIN.PRETRAINED_MODEL = '' +cfg.TRAIN.LOSS_TYPE = 'RMSLE' # or 'BerHu' + +# ----------------------------------------------------------------------------- +# Augmentation settings +# ----------------------------------------------------------------------------- +# trained on depth estimation +cfg.AUG = CfgNode() +cfg.AUG.BRIGHTNESS = 0.2 +cfg.AUG.CONTRAST = 0.2 +cfg.AUG.SATURATION = 0.2 +cfg.AUG.HUE = 0.1 + +# ----------------------------------------------------------------------------- +# Testing or validating settings +# ----------------------------------------------------------------------------- +cfg.TESTING = CfgNode() +cfg.TESTING.BATCH_SIZE = 1 +cfg.TESTING.RANK = 0 +cfg.TESTING.WORLD_SIZE = 1 +cfg.TESTING.BACKEND = 'nccl' +cfg.TESTING.IP_ADDR = 'localhost:' +cfg.TESTING.PORT = '8000' +cfg.TESTING.INIT_METHOD = 'tcp://' + cfg.TESTING.IP_ADDR + cfg.TESTING.PORT +cfg.TESTING.MEDIAN_ALIGN = False +# ----------------------------------------------------------------------------- +# Misc settings +# ----------------------------------------------------------------------------- +# Mixed precision opt level, if O0, no amp is used ('O0', 'O1', 'O2') +cfg.AMP_OPT_LEVEL = 'O0' +# Frequency to save checkpoint +cfg.SAVE_FREQ = 1 +# Frequency to logging info +cfg.PRINT_FREQ = 5 +# Fixed random seed +cfg.SEED = 0 +cfg.OUT_ROOT_DIR = './train' +cfg.EVAL_MODE = False + + +def update_cfg_from_file(config, cfg_file): + config.defrost() + with open(cfg_file, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.FullLoader) + + for cfg in yaml_cfg.setdefault('BASE', ['']): + if cfg: + update_cfg_from_file(config, + os.path.join(os.path.dirname(cfg_file), cfg)) + print('=> merge config from {}'.format(cfg_file)) + config.merge_from_file(cfg_file) + config.freeze() + + +def get_config(cfg_file): + """Get a yacs CfgNode object with default values.""" + # Return a clone so that the defaults will not be altered + # This is for the "local variable" use pattern + config = cfg.clone() + update_cfg_from_file(config, cfg_file) + + return config diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/decoder.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..9b7eaaca17dd2315990a095fe93192d150ecabef --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/decoder.py @@ -0,0 +1,459 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from torch import einsum + +from .util_helper import (precompute_pixelization_maps, + precompute_position_encoding) + + +class SphConv2d(nn.Module): + + def __init__(self, in_dims, out_dims, k, neigh_map, groups=1, bias=True): + super().__init__() + self.neighbor_map = neigh_map + self.conv = nn.Conv1d( + in_dims, + out_dims, + kernel_size=k**2, + stride=k**2, + groups=groups, + bias=bias) + + def forward(self, fmap): + fmap = fmap.squeeze(2) # remove row-dim + x = F.pad(fmap, (0, 1, 0, 0, 0, 0), mode='constant', value=0.0) + vec = x[:, :, self.neighbor_map] # index_select + return self.conv(vec).unsqueeze(2) + + +# classes +class Residual(nn.Module): + + def __init__(self, fn): + super().__init__() + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(x, **kwargs) + x + + +class SphPixelization(nn.Module): + """ + A projection which converts ERP format feature maps to spherical feature maps, + The indices are precomputed. + """ + + def __init__(self, index): + super(SphPixelization, self).__init__() + + self.x0, self.y0 = index[0, :].long(), index[1, :].long() + self.x1, self.y1 = index[2, :].long(), index[3, :].long() + self.wa, self.wb = index[4, :].float(), index[5, :].float() + self.wc, self.wd = index[6, :].float(), index[7, :].float() + + def forward(self, x): + # a projection from ERP feature map to spherical feature maps by bilinear interpolation + out = self.wa * x[:, :, self.y0, self.x0] + self.wb * x[:, :, self.y1, self.x0] \ + + self.wc * x[:, :, self.y0, self.x1] + self.wd * x[:, :, self.y1, self.x1] + out = out.unsqueeze(-2) + return out + + +class SphPosEmbedding(nn.Module): + """Spherical positional embedding""" + + def __init__(self, dim, pos_enc): + super(SphPosEmbedding, self).__init__() + self.pos_enc = pos_enc.float() + self.pos_emb = nn.Conv2d(3, dim, 1) + + def forward(self, x): + b, c, w, h = x.shape + pos_enc = self.pos_enc.unsqueeze(0).repeat(b, 1, 1, 1) + x += self.pos_emb(pos_enc) + return x + + +class SphInterpolate(nn.Module): + """Spherical pixels interpolation""" + + def __init__(self, scale_factor): + super(SphInterpolate, self).__init__() + self.scale_factor = scale_factor + + def forward(self, x): + return torch.repeat_interleave(x, self.scale_factor, dim=-1) + + +class LayerNorm(nn.Module): + + def __init__(self, dim, eps=1e-8): + super().__init__() + self.eps = eps + self.g = nn.Parameter(torch.ones(1, dim, 1, 1)) + self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) + + def forward(self, x): + std = torch.var(x, dim=1, unbiased=False, keepdim=True).sqrt() + mean = torch.mean(x, dim=1, keepdim=True) + return (x - mean) / (std + self.eps) * self.g + self.b + + +class PreNorm(nn.Module): + + def __init__(self, dim, fn): + super().__init__() + self.norm = LayerNorm(dim) + self.fn = fn + + def forward(self, x, **kwargs): + x = self.norm(x) + return self.fn(x, **kwargs) + + +class FeedForward(nn.Module): + + def __init__(self, dim, mult=4, dropout=0.): + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(dim, dim * mult, 1), nn.GELU(), nn.Dropout(dropout), + nn.Conv2d(dim * mult, dim, 1), nn.Dropout(dropout)) + + def forward(self, x): + return self.net(x) + + +class SphGSA(nn.Module): + """ + Global self attention on spherical surface. + To balance memory usage and performance, + we use a strategy like twins-svt(https://github.com/Meituan-AutoML/Twins) + """ + + def __init__(self, dim_in, dim_out, heads=8, dim_head=64, dropout=0., k=4): + super().__init__() + inner_dim = dim_head * heads + self.heads = heads + self.scale = dim_head**-0.5 + + self.to_q = nn.Conv2d(dim_in, inner_dim, 1, bias=False) + self.to_kv = nn.Conv2d(dim_in, inner_dim * 2, k, stride=k, bias=False) + + self.to_out = nn.Sequential( + nn.Conv2d(inner_dim, dim_out, 1), nn.Dropout(dropout)) + self.k = k + + def forward(self, fmap): + shape, k = fmap.shape, self.k + [b, n, x, y] = shape + h = self.heads + # windows number + w = y // (k**2) + q = self.to_q(fmap) + fmap = rearrange( + fmap, 'b n x (y p1 p2) -> (b x y) n p1 p2', p1=k, p2=k) + kv = self.to_kv(fmap) + kv = rearrange(kv, '(b w) n x y -> b n x (w y)', w=w) + k, v = kv.chunk(2, dim=1) + q, k, v = map( + lambda t: rearrange(t, 'b (h d) x y -> (b h) (x y) d', h=h), + (q, k, v)) + + dots = einsum('b i d, b j d -> b i j', q, k) * self.scale + attn = dots.softmax(dim=-1) + + out = einsum('b i j, b j d -> b i d', attn, v) + out = rearrange(out, '(b h) (x y) d -> b (h d) x y', h=h, y=y) + return self.to_out(out) + + +def make_skip_connection(in_dim, out_dim, pix_index_maps=None): + """ + establish skip connections between encoders and decoders + we call this process as SPP(spherial pixelization projection) + """ + out_dim1, out_dim2, out_dim3, out_dim4 = out_dim, out_dim, out_dim, out_dim + skip_con1 = nn.Sequential( + SphPixelization(pix_index_maps[0]), nn.Conv2d(in_dim[0], out_dim1, 1), + nn.ReLU(False)) + skip_con2 = nn.Sequential( + SphPixelization(pix_index_maps[1]), nn.Conv2d(in_dim[1], out_dim2, 1), + nn.ReLU(False)) + skip_con3 = nn.Sequential( + SphPixelization(pix_index_maps[2]), nn.Conv2d(in_dim[2], out_dim3, 1), + nn.ReLU(False)) + skip_con4 = nn.Sequential( + SphPixelization(pix_index_maps[3]), nn.Conv2d(in_dim[3], out_dim4, 1), + nn.ReLU(False)) + return skip_con1, skip_con2, skip_con3, skip_con4 + + +class ResidualConvBlock(nn.Module): + + def __init__(self, in_dim, activation, bn, neighbor_map): + super().__init__() + self.bn = bn + self.groups = 1 + self.conv1 = SphConv2d( + in_dim, + in_dim, + k=3, + bias=not self.bn, + groups=self.groups, + neigh_map=neighbor_map) + self.conv2 = SphConv2d( + in_dim, + in_dim, + k=3, + bias=not self.bn, + groups=self.groups, + neigh_map=neighbor_map) + + if self.bn is True: + self.bn1 = nn.BatchNorm2d(features_in) + self.bn2 = nn.BatchNorm2d(features_in) + self.activation = activation + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + out = self.activation(x) + out = self.conv1(out) + if self.bn is True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn is True: + out = self.bn2(out) + return self.skip_add.add(out, x) + + +class SphCrossAttFusionBlock(nn.Module): + + def __init__(self, dim, heads=8, dim_head=64, dropout=0., k0=4, k1=4): + super().__init__() + inner_dim = dim_head * heads + self.heads = heads + self.scale = dim_head**-0.5 + self.to_q0 = nn.Conv2d(dim, inner_dim, 1, bias=False) + self.to_kv0 = nn.Conv2d(dim, inner_dim * 2, k0, stride=k0, bias=False) + self.to_q1 = nn.Conv2d(dim, inner_dim, 1, bias=False) + self.to_kv1 = nn.Conv2d(dim, inner_dim * 2, k1, stride=k1, bias=False) + + self.to_out0 = nn.Sequential( + nn.Conv2d(inner_dim, dim, 1), nn.Dropout(dropout)) + self.to_out1 = nn.Sequential( + nn.Conv2d(inner_dim, dim, 1), nn.Dropout(dropout)) + self.add = nn.quantized.FloatFunctional() + self.k0, self.k1 = k0, k1 + + def forward(self, *xs): + x0, x1 = xs[0], xs[1] + assert x0.shape == x1.shape + shape, k0, k1 = x0.shape, self.k0, self.k1 + [b, n, x, y] = shape + h = self.heads + + # for feature map x0 + w = y // (k0**2) + q0 = self.to_q0(x0) + x0_w = rearrange( + x0, 'b n x (y p1 p2) -> (b x y) n p1 p2', p1=k0, p2=k0) + kv0 = self.to_kv0(x0_w) + kv0 = rearrange(kv0, '(b w) n x y -> b n x (w y)', w=w) + k0, v0 = kv0.chunk(2, dim=1) + q0, k0, v0 = map( + lambda t: rearrange(t, 'b (h d) x y -> (b h) (x y) d', h=h), + (q0, k0, v0)) + + # for feature map x1 + w = y // (k1**2) + q1 = self.to_q1(x1) + x1_w = rearrange( + x1, 'b n x (y p1 p2) -> (b x y) n p1 p2', p1=k1, p2=k1) + kv1 = self.to_kv1(x1_w) + kv1 = rearrange(kv1, '(b w) n x y -> b n x (w y)', w=w) + k1, v1 = kv1.chunk(2, dim=1) + q1, k1, v1 = map( + lambda t: rearrange(t, 'b (h d) x y -> (b h) (x y) d', h=h), + (q1, k1, v1)) + + # fusion + dots = einsum('b i d, b j d -> b i j', q0, k1) * self.scale + attn = dots.softmax(dim=-1) + out0 = einsum('b i j, b j d -> b i d', attn, v1) + out0 = rearrange(out0, '(b h) (x y) d -> b (h d) x y', h=h, y=y) + out0 = self.to_out0(out0) + + dots = einsum('b i d, b j d -> b i j', q1, k0) * self.scale + attn = dots.softmax(dim=-1) + out1 = einsum('b i j, b j d -> b i d', attn, v0) + out1 = rearrange(out1, '(b h) (x y) d -> b (h d) x y', h=h, y=y) + out1 = self.to_out1(out1) + return 0.5 * self.add.add(x1 + out0, x0 + out1) + + +class SphCAFBlock(nn.Module): + + def __init__(self, dim, pos_enc, k0=4, k1=4): + super().__init__() + self.process0 = nn.Sequential( + SphPosEmbedding(dim, pos_enc=pos_enc), PreNorm(dim, nn.Identity())) + self.process1 = nn.Sequential( + SphPosEmbedding(dim, pos_enc=pos_enc), PreNorm(dim, nn.Identity())) + self.fusion = SphCrossAttFusionBlock(dim, k0=k0, k1=k1) + self.feedforward = Residual(PreNorm(dim, FeedForward(dim))) + + def forward(self, *xs): + x0, x1 = xs[0], xs[1] + x0, x1 = self.process0(x0), self.process1(x1) + out = self.feedforward(self.fusion(x0, x1)) + return out + + +class FeatureFusion(nn.Module): + + def __init__(self, + enable_caf, + dim, + neighbor_map, + activation=nn.ReLU(False), + bn=False, + k0=4, + k1=4, + pos_enc=None): + super(FeatureFusion, self).__init__() + self.enable_CAF = enable_caf + self.skip_add = None if self.enable_CAF else nn.quantized.FloatFunctional( + ) + self.resConfUnit2 = ResidualConvBlock( + dim, activation, bn, neighbor_map=neighbor_map) + self.resConfUnit1 = ResidualConvBlock( + dim, activation, bn, neighbor_map=neighbor_map) + self.fusionBlock = SphCAFBlock( + dim, pos_enc, k0=k0, + k1=k1) if self.enable_CAF else self.skip_add.add + self.up_sample = SphInterpolate(scale_factor=4) + + def forward(self, *xs): + output = xs[0] + res = self.resConfUnit1(xs[1]) + output = self.fusionBlock(output, res) + output = self.resConfUnit2(output) + output = self.up_sample(output) + return output + + +class FeatureFusionBottom(nn.Module): + + def __init__(self, + enable_caf, + dim, + neighbor_map, + activation=nn.ReLU(False), + bn=False, + k=1, + pos_enc=None): + super(FeatureFusionBottom, self).__init__() + self.enable_CAF = enable_caf + self.resConfUnit = nn.Sequential( + SphPosEmbedding(dim, pos_enc=pos_enc), + Residual(PreNorm(dim, SphGSA(dim, dim, k=k))), + Residual(PreNorm(dim, FeedForward(dim, mult=4, dropout=0.))), + ) if self.enable_CAF else ResidualConvBlock( + dim, activation, bn, neighbor_map=neighbor_map) + self.up_sample = SphInterpolate(scale_factor=4) + + def forward(self, *xs): + output = xs[0] + output = self.resConfUnit(output) + output = self.up_sample(output) + return output + + +class SphDecoder(nn.Module): + + def __init__(self, + cfg, + neighbor_maps, + num_ch_enc, + use_bn=False, + img_size=(512, 1024)): + super(SphDecoder, self).__init__() + self.use_caf = cfg.MODEL.USE_CAF_FUSION + self.pix_sample_index_maps = precompute_pixelization_maps( + nsides=[32, 16, 8, 4], + initial_img_size=[img_size[0] // 4, img_size[1] // 4]) + + self.pos_encodings = precompute_position_encoding( + nsides=[32, 16, 8, 4]) if self.use_caf else None + self.ffBlock1 = FeatureFusion( + cfg.MODEL.USE_CAF_FUSION, + cfg.MODEL.DECODER_DIM, + neighbor_map=neighbor_maps[32], + bn=use_bn, + k0=4, + k1=4, + pos_enc=self.pos_encodings[0] if self.use_caf else None) + self.ffBlock2 = FeatureFusion( + cfg.MODEL.USE_CAF_FUSION, + cfg.MODEL.DECODER_DIM, + neighbor_map=neighbor_maps[16], + bn=use_bn, + k0=2, + k1=2, + pos_enc=self.pos_encodings[1] if self.use_caf else None) + self.ffBlock3 = FeatureFusion( + cfg.MODEL.USE_CAF_FUSION, + cfg.MODEL.DECODER_DIM, + neighbor_map=neighbor_maps[8], + bn=use_bn, + k0=1, + k1=1, + pos_enc=self.pos_encodings[2] if self.use_caf else None) + self.ffBlock4 = FeatureFusionBottom( + cfg.MODEL.USE_CAF_FUSION, + cfg.MODEL.DECODER_DIM, + neighbor_map=neighbor_maps[4], + bn=use_bn, + k=1, + pos_enc=self.pos_encodings[3] if self.use_caf else None) + self.head = nn.Sequential( + SphConv2d( + cfg.MODEL.DECODER_DIM, + cfg.MODEL.DECODER_DIM // 4, + k=3, + neigh_map=neighbor_maps[64]), nn.ReLU(True), + SphInterpolate(scale_factor=4), + SphConv2d( + cfg.MODEL.DECODER_DIM // 4, + cfg.MODEL.DECODER_DIM // 8, + k=3, + neigh_map=neighbor_maps[128]), nn.ReLU(True), + nn.Conv2d( + cfg.MODEL.DECODER_DIM // 8, + 1, + kernel_size=1, + stride=1, + padding=0), nn.ReLU(True)) + self.skip_con1, self.skip_con2, self.skip_con3, self.skip_con4 = \ + make_skip_connection(num_ch_enc, cfg.MODEL.DECODER_DIM, pix_index_maps=self.pix_sample_index_maps) + + def forward(self, enc_feat1, enc_feat2, enc_feat3, enc_feat4): + sph_skip_feat1 = self.skip_con1(enc_feat1) + sph_skip_feat2 = self.skip_con2(enc_feat2) + sph_skip_feat3 = self.skip_con3(enc_feat3) + sph_skip_feat4 = self.skip_con4(enc_feat4) + + sph_dec_feat4 = self.ffBlock4(sph_skip_feat4) + sph_dec_feat3 = self.ffBlock3(sph_dec_feat4, sph_skip_feat3) + sph_dec_feat2 = self.ffBlock2(sph_dec_feat3, sph_skip_feat2) + sph_dec_feat1 = self.ffBlock1(sph_dec_feat2, sph_skip_feat1) + out = self.head(sph_dec_feat1) + return out diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/model.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/model.py new file mode 100644 index 0000000000000000000000000000000000000000..8b6c8c03ea5caabb4e269a3e6937adc3905fd690 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/model.py @@ -0,0 +1,156 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn + +from .decoder import SphDecoder +from .resnet import resnet18, resnet34, resnet50, resnet101, resnet152 +# import backbone +from .swin_transformer import SwinTransformer +from .util_helper import compute_hp_neighmap + + +class SwinSphDecoderNet(nn.Module): + + def __init__(self, cfg, pretrained=False): + super(SwinSphDecoderNet, self).__init__() + # compute healpixel + self.neighbor_maps_all_levels = compute_hp_neighmap( + [128, 64, 32, 16, 8, 4]) + self.vit_encoder = SwinTransformer( + pretrain_img_size=cfg.BACKBONE.PRETRAIN_RES, + embed_dim=cfg.BACKBONE.EMBED_DIM, + depths=cfg.BACKBONE.DEPTHS, + num_heads=cfg.BACKBONE.NUM_HEADS, + window_size=cfg.BACKBONE.WINDOW_SIZE, + drop_path_rate=cfg.BACKBONE.DROP_PATH_RATE, + frozen_stages=cfg.BACKBONE.FROZEN_STAGES) + + ch = cfg.BACKBONE.EMBED_DIM + self.num_ch_enc = [ch, 2 * ch, 4 * ch, 8 * ch] + self.decoder = SphDecoder( + cfg=cfg, + neighbor_maps=self.neighbor_maps_all_levels, + num_ch_enc=self.num_ch_enc, + img_size=(cfg.DATA.IMG_HEIGHT, cfg.DATA.IMG_WIDTH)) + + cfg.defrost() + if pretrained: + # no offline pretrained model + if cfg.TRAIN.PRETRAINED_MODEL == '': + cfg.TRAIN.PRETRAINED_MODEL = 'modelzoo://swin_transformer/swin_' + cfg.BACKBONE.VERSION \ + + '_patch4_window' + str(cfg.BACKBONE.WINDOW_SIZE) + '_' + str(cfg.BACKBONE.PRETRAIN_RES) + if cfg.BACKBONE.PRETRAIN_IMAGENET == '1k': + cfg.TRAIN.PRETRAINED_MODEL += '.pth' + elif cfg.BACKBONE.PRETRAIN_IMAGENET == '22k': + cfg.TRAIN.PRETRAINED_MODEL += '_22k.pth' + else: + cfg.TRAIN.PRETRAINED_MODEL = '' + cfg.freeze() + if cfg.TRAIN.PRETRAINED_MODEL != '': + self.init_weights(cfg.TRAIN.PRETRAINED_MODEL) + + def init_weights(self, pretrained_model): + self.vit_encoder.init_weights(pretrained_model) + + def forward(self, x): + out = self.vit_encoder(x) + out = self.decoder(out[0], out[1], out[2], out[3]) + return out + + +class ResnetSphDecoderNet(nn.Module): + + def __init__(self, cfg, pretrained=False): + super(ResNetSphDecoder, self).__init__() + self.neighbor_maps_all_levels = compute_hp_neighmap( + [128, 64, 32, 16, 8, 4]) + encoder = { + 18: resnet18, + 34: resnet34, + 50: resnet50, + 101: resnet101, + 152: resnet152 + } + self.encoder = encoder[cfg.BACKBONE.RESNET_LAYER_NUM](pretrained) + + self.num_ch_enc = np.array([64, 64, 128, 256, 512]) + if cfg.BACKBONE.RESNET_LAYER_NUM > 34: + self.num_ch_enc[1:] *= 4 + + if cfg.BACKBONE.RESNET_LAYER_NUM < 18: + self.num_ch_enc = np.array([16, 24, 32, 96, 320]) + self.num_layers = cfg.BACKBONE.RESNET_LAYER_NUM + + self.decoder = SphDecoder( + cfg=cfg, + neighbor_maps=self.neighbor_maps_all_levels, + num_ch_enc=self.num_ch_enc[1:], + img_size=(cfg.DATA.IMG_HEIGHT, cfg.DATA.IMG_WIDTH)) + + def forward(self, x): + if self.num_layers < 18: + enc_feat0, enc_feat1, enc_feat2, enc_feat3, enc_feat4 \ + = self.encoder(x) + else: + x = self.encoder.conv1(x) + x = self.encoder.relu(self.encoder.bn1(x)) + x = self.encoder.maxpool(x) + enc_feat1 = self.encoder.layer1(x) + enc_feat2 = self.encoder.layer2(enc_feat1) + enc_feat3 = self.encoder.layer3(enc_feat2) + enc_feat4 = self.encoder.layer4(enc_feat3) + out = self.decoder(enc_feat1, enc_feat2, enc_feat3, enc_feat4) + return out + + +class EfficientNetEncoder(nn.Module): + + def __init__(self, backend): + super(EfficientNetEncoder, self).__init__() + self.original_model = backend + + def forward(self, x): + features = [x] + for k, v in self.original_model._modules.items(): + if k == 'blocks': + for ki, vi in v._modules.items(): + features.append(vi(features[-1])) + else: + features.append(v(features[-1])) + return features + + +class EffnetSphDecoderNet(nn.Module): + + def __init__(self, cfg, pretrained=False): + super(EffNetSphDecoder, self).__init__() + self.neighbor_maps_all_levels = compute_hp_neighmap( + [128, 64, 32, 16, 8, 4]) + basemodel_name = 'tf_efficientnet_b5_ap' + basemodel = torch.hub.load( + 'rwightman/gen-efficientnet-pytorch', + basemodel_name, + pretrained=pretrained) + # Remove last layer + basemodel.global_pool = nn.Identity() + basemodel.classifier = nn.Identity() + self.encoder = EfficientNetEncoder(basemodel) + + self.num_ch_enc = np.array([24, 40, 64, 176, 2048]) + self.decoder = SphDecoder( + cfg=cfg, + neighbor_maps=self.neighbor_maps_all_levels, + num_ch_enc=self.num_ch_enc[1:], + img_size=(cfg.DATA.IMG_HEIGHT, cfg.DATA.IMG_WIDTH)) + self.conv4 = nn.Conv2d(2048, 1024, kernel_size=1, stride=1, padding=1) + self.relu = nn.ReLU(False) + + def forward(self, x): + enc_feat = self.encoder(x) + enc_feat1, enc_feat2, enc_feat3, enc_feat4 = \ + enc_feat[5], enc_feat[6], enc_feat[8], enc_feat[11] + enc_feat4 = self.relu(self.conv4(enc_feat4)) + out = self.decoder(enc_feat1, enc_feat2, enc_feat3, enc_feat4) + return out diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/resnet.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..0be686126b34c18587fdea74e952388e13716469 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/resnet.py @@ -0,0 +1,459 @@ +# Modified from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py, +# made publicly available under the BSD License +import torch +import torch.nn as nn + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + +__all__ = [ + 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50_32x4d', 'resnext101_32x8d', 'wide_resnet50_2', + 'wide_resnet101_2' +] + +model_urls = { + 'resnet18': + 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': + 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': + 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': + 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': + 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': + 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': + 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': + 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': + 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, padding, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=padding, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + + self.conv1 = conv3x3(inplanes, planes, 1, stride) + + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=False) + + self.conv2 = conv3x3(planes, planes, 1) + + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(identity) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition" https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + + self.conv2 = conv3x3(width, width, 1, stride, groups, dilation) + + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=False) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, + block, + layers, + num_input_images=1, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None): + super(ResNet, self).__init__() + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + self.conv1 = nn.Conv2d( + 3 * num_input_images, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias=False, + padding_mode='zeros') + + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=False) + + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer( + block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer( + block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer( + block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + """ + # remove fcn as we don't need it for a depth prediction task + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + """ + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + """ + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.fc(x) + """ + + return x + + def forward(self, x): + return self._forward_impl(x) + + +def _resnet(arch, + block, + layers, + pretrained, + progress, + num_input_images=1, + **kwargs): + model = ResNet(block, layers, num_input_images=num_input_images, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url( + model_urls[arch], + model_dir='../../checkpoints/', + progress=progress) + + state_dict['conv1.weight'] = torch.cat( + [state_dict['conv1.weight']] * num_input_images, + 1) / num_input_images + + model_dict = model.state_dict() + state_dict = {k: v for k, v in state_dict.items() if k in model_dict} + model_dict.update(state_dict) + model.load_state_dict(model_dict) + + return model + + +def resnet18(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, + progress, **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], pretrained, + progress, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/swin_transformer.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..4ca3ce5ec6a248ae349442e3dd12b459279c352e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/swin_transformer.py @@ -0,0 +1,696 @@ +# -------------------------------------------------------- +# Swin Transformer +# Copyright (c) 2021 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Written by Ze Liu, Yutong Lin, Yixuan Wei +# -------------------------------------------------------- + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from .util_helper import load_checkpoint + + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """ Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """ Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, 'input feature has wrong size' + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, + Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """ Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, 'input feature has wrong size' + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=4, + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, + (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """ Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1] + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], + patches_resolution[1])) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if + (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + if isinstance(pretrained, str): + self.apply(_init_weights) + load_checkpoint(self, pretrained, strict=False) + elif pretrained is None: + self.apply(_init_weights) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic') + x = (x + absolute_pos_embed).flatten(2).transpose(1, + 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, + self.num_features[i]).permute(0, 3, 1, + 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/util_helper.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/util_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..138078e43f4629206dad6939b0496e3f0f87a24d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/networks/util_helper.py @@ -0,0 +1,461 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import pkgutil +import warnings +from importlib import import_module + +import healpy as hp +import numpy as np +import torch +import torch.distributed as dist +import torchvision +from torch.nn import functional as F +from torch.nn.parallel import DataParallel, DistributedDataParallel +from torch.utils import model_zoo + +try: + from apex import amp +except ImportError: + amp = None + + +@torch.no_grad() +def compute_hp_neighmap(nsides): + """ Precomputing the neighborhood indices table of each pixel on spherical surface. + + Precomputing the neighborhood indices of each pixel on spherical surface. + The neighbor map provides the surrounding pixel ids for each pixel. + Hierarchical Equal Area isoLatitude Pixelization (HEALPix) is used. + The tool package 'healpy' can be found on https://github.com/healpy/healpy + It may take a little long time for high resolution(eg. nside = 128) + + Args: + nsides: the pixelization resolution, could be 4, 8, 16, 32, 64, 128 + which corresponds to pixle num 12 x nside^2 + + Returns: + neighbors: a dict-like neighborhood map corresponding to each resolution. + """ + + neighbours = {} + for i in range(len(nsides)): + npix = hp.nside2npix(nsides[i]) + neighbour_map = torch.ones( + 9 * npix, dtype=torch.long, requires_grad=False) + for p in range(npix): + local_neighbor = hp.pixelfunc.get_all_neighbours( + nsides[i], p, nest=True) + local_neighbor = np.insert(local_neighbor, 4, p) + ind = np.where(local_neighbor == -1)[0] + # padding method, `local_neighbor[ind] = p` may be better + local_neighbor[ind] = npix + # fill -1 when 8-neighbors not exist(only 7 neighbors) + neighbour_map[9 * p:9 * p + 9] = torch.tensor(local_neighbor) + + neighbours[nsides[i]] = neighbour_map + return neighbours + + +def precompute_pixelization_maps(nsides, initial_img_size=(128, 256)): + """ Precomputing the mapping from multi-resolution ERP image to multi-resolution spherical pixels. + + Precomputing the mapping from multi-resolution(sacle factor = 2) ERP image to + multi-resolution(sacle factor = 4)spherical surface. Each pixel on spherical surface + is back projected to ERP image and the indices are recorded and returned. + Bilinear interpolation is used. + Args: + nsides: the pixelization resolution, could be [32, 16, 8, 4] + initial_img_size: the initial image size, scale factor = 2 + Returns: + index maps: a list-like indice maps corresponding to each nside. + """ + index_maps = [] + ini_h, ini_w = initial_img_size[0], initial_img_size[1] + for i in range(len(nsides)): + h, w = ini_h // (2**i), ini_w // (2**i) + pixel_num_sp = hp.nside2npix(nsides[i]) + pixel_idx = np.arange(pixel_num_sp) + sp_ll = hp.pix2ang(nsides[i], pixel_idx, nest=True, lonlat=True) + x, y = sp_ll[0] / 360.0 * w, (sp_ll[1] + 90.0) / 180.0 * h + x, y = x.reshape(1, -1), y.reshape(1, -1) + x0, y0 = np.floor(x).astype(np.int32), np.floor(y).astype(np.int32) + x1, y1 = x0 + 1, y0 + 1 + x0, y0 = np.clip(x0, 0, w - 1), np.clip(y0, 0, h - 1) + x1, y1 = np.clip(x1, 0, w - 1), np.clip(y1, 0, h - 1) + wa = (x1 - x) * (y1 - y) + wb = (x1 - x) * (y - y0) + wc = (x - x0) * (y1 - y) + wd = (x - x0) * (y - y0) + index_maps.append( + torch.from_numpy( + np.concatenate([x0, y0, x1, y1, wa, wb, wc, wd], + axis=0)).cuda()) + return index_maps + + +def precompute_position_encoding(nsides): + """Precomputing spherical coordinates of spherical pixels for each nside""" + + pos_encodings = [] + for i in range(len(nsides)): + pixel_num_sp = hp.nside2npix(nsides[i]) + pixel_idx = np.arange(pixel_num_sp) + dir_x, dir_y, dir_z = hp.pix2vec(nsides[i], pixel_idx, nest=True) + dir_x, dir_y, dir_z = dir_x.reshape(1, -1), dir_y.reshape( + 1, -1), dir_y.reshape(1, -1) + pos_encodings.append( + torch.from_numpy(np.concatenate([dir_x, dir_y, dir_z], + axis=0)).unsqueeze(1).cuda()) + return pos_encodings + + +def auto_resume_helper(output_dir): + checkpoints = os.listdir(output_dir) + checkpoints = [ckpt for ckpt in checkpoints if ckpt.endswith('pth')] + if len(checkpoints) > 0: + latest_checkpoint = max( + [os.path.join(output_dir, d) for d in checkpoints], + key=os.path.getmtime) + resume_file = latest_checkpoint + else: + resume_file = None + return resume_file + + +def load_checkpoint_file(cfg, model, optimizer, lr_scheduler): + if cfg.TRAIN.RESUME.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + cfg.TRAIN.RESUME, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(cfg.TRAIN.RESUME, map_location='cpu') + model.load_state_dict(checkpoint['model'], strict=True) + rel_error = float('inf') + if not cfg.EVAL_MODE and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + cfg.defrost() + cfg.TRAIN.START_EPOCH = checkpoint['epoch'] + 1 + cfg.freeze() + if 'amp' in checkpoint and cfg.AMP_OPT_LEVEL != 'O0' and checkpoint[ + 'config'].AMP_OPT_LEVEL != 'O0': + amp.load_state_dict(checkpoint['amp']) + if 'rel_error' in checkpoint: + rel_error = checkpoint['rel_error'] + + del checkpoint + torch.cuda.empty_cache() + return rel_error + + +def save_checkpoint(cfg, epoch, model, rel_error, optimizer, lr_scheduler, + out_dir): + save_state = { + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'rel_error': rel_error, + 'epoch': epoch, + 'config': cfg + } + if cfg.AMP_OPT_LEVEL != 'O0': + save_state['amp'] = amp.state_dict() + + save_path = os.path.join(out_dir, f'ckpt_epoch_{epoch}.pth') + torch.save(save_state, save_path) + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= dist.get_world_size() + return rt + + +def get_grad_norm(parameters, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item()**norm_type + total_norm = total_norm**(1. / norm_type) + return total_norm + + +def to_numpy(tensor): + return tensor.detach().cpu().numpy( + ) if tensor.requires_grad else tensor.cpu().numpy() + + +# Load swin pretrained model, code from Swin-Transformer-Semantic-Segmentation +# https://github.com/SwinTransformer/Swin-Transformer-Semantic-Segmentation/blob/main/mmcv_custom/checkpoint.py +################################################################################################################ +TORCH_VERSION = torch.__version__ + + +def is_module_wrapper(module): + module_wrappers = (DataParallel, DistributedDataParallel) + return isinstance(module, module_wrappers) + + +def get_dist_info(): + if TORCH_VERSION < '1.0': + initialized = dist._initialized + else: + if dist.is_available(): + initialized = dist.is_initialized() + else: + initialized = False + if initialized: + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + return rank, world_size + + +def load_state_dict(module, state_dict, strict=False): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, '===== The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + else: + if not os.path.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, filename, map_location='cpu', strict=False): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + # strip prefix of state_dict + tmp = {} + for k, v in state_dict.items(): + if k.startswith('module.'): + tmp[k[7:]] = v + else: + tmp[k] = v + state_dict = tmp + del tmp + # if list(state_dict.keys())[0].startswith('module.'): + # state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = { + k.replace('encoder.', ''): v + for k, v in state_dict.items() if k.startswith('encoder.') + } + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H * W: + print('Error in loading absolute_pos_embed, pass') + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view( + N2, H, W, C2).permute(0, 3, 1, 2) + + # interpolate position bias table if needed + relative_position_bias_table_keys = [ + k for k in state_dict.keys() if 'relative_position_bias_table' in k + ] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + print(f'Error in loading {table_key}, pass') + else: + if L1 != L2: + S1 = int(L1**0.5) + S2 = int(L2**0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), + mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view( + nH2, L2).permute(1, 0) + + # load state_dict + load_state_dict(model, state_dict, strict) + return checkpoint + + +def render_depth_map(hp_data, image_to_sp): + return hp_data[:, :, :, image_to_sp].squeeze(2) + + +def compute_hp_info(nside=128, img_size=(512, 1024)): + hp_info = {} + h, w = img_size[0], img_size[1] + pixel_num_sp = hp.nside2npix(nside) + pixel_idx = np.arange(pixel_num_sp) + sp_ll = hp.pix2ang(nside, pixel_idx, nest=True, lonlat=True) + + x, y = sp_ll[0] / 360.0 * w, (sp_ll[1] + 90.0) / 180.0 * h + x, y = x.reshape(1, -1), y.reshape(1, -1) + x0, y0 = np.floor(x).astype(np.int32), np.floor(y).astype(np.int32) + x1, y1 = x0 + 1, y0 + 1 + x0, y0 = np.clip(x0, 0, w - 1), np.clip(y0, 0, h - 1) + x1, y1 = np.clip(x1, 0, w - 1), np.clip(y1, 0, h - 1) + wa = (x1 - x) * (y1 - y) + wb = (x1 - x) * (y - y0) + wc = (x - x0) * (y1 - y) + wd = (x - x0) * (y - y0) + sp_to_image = np.concatenate([x0, y0, x1, y1, wa, wb, wc, wd], axis=0) + + sp_xyz = hp.pix2vec(nside, pixel_idx, nest=True) + sp_xyz = np.stack([sp_xyz[0], sp_xyz[1], sp_xyz[2]], axis=1) + + # consider half pixels + theta = np.arange(h).reshape(h, 1) * np.pi / h + np.pi / h / 2 + theta = np.pi - np.repeat(theta, w, axis=1) + theta = theta.flatten() + phi = np.arange(w).reshape(1, w) * 2 * np.pi / w + np.pi / w + phi = np.repeat(phi, h, axis=0) + phi = phi.flatten() + # see healpy doc: + # https://healpy.readthedocs.io/en/latest/generated/healpy.pixelfunc.ang2pix.html#healpy.pixelfunc.ang2pix + # `lonlat=False` to use colattitude representation + image_to_sp = hp.pixelfunc.ang2pix( + nside, theta, phi, nest=True, lonlat=False) + hp_info['hp_dir'] = sp_xyz + hp_info['hp_pix_num'] = pixel_num_sp + hp_info['hp_to_image_map'] = sp_to_image + hp_info['image_to_sp_map'] = torch.from_numpy(image_to_sp.reshape(h, w)) + return hp_info diff --git a/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/s2net_model.py b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/s2net_model.py new file mode 100644 index 0000000000000000000000000000000000000000..7e8cd1cdd6a49bc03c669694a34c134bf236b555 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/s2net_panorama_depth_estimation/s2net_model.py @@ -0,0 +1,95 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import numpy as np +import torch +from torchvision import transforms + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.s2net_panorama_depth_estimation.networks import ( + EffnetSphDecoderNet, ResnetSphDecoderNet, SwinSphDecoderNet) +from modelscope.models.cv.s2net_panorama_depth_estimation.networks.config import \ + get_config +from modelscope.models.cv.s2net_panorama_depth_estimation.networks.util_helper import ( + compute_hp_info, render_depth_map) +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@MODELS.register_module( + Tasks.panorama_depth_estimation, module_name=Models.s2net_depth_estimation) +class PanoramaDepthEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """ + An end-to-end deep network for monocular panorama depth estimation on a unit spherical surface. + This is the official implementation of paper S2Net: Accurate Panorama Depth Estimation on Spherical Surface, + https://arxiv.org/abs/2301.05845. + Args: + model_dir: the path of the pretrained model file + """ + super().__init__(model_dir, **kwargs) + if 'device' in kwargs: + self.device = create_device(kwargs['device']) + else: + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + # load network config + cfg_path = osp.join(model_dir, 'model.yaml') + cfg = get_config(cfg_path) + encoder_model_dict = { + 'swin': SwinSphDecoderNet, + 'resNet': ResnetSphDecoderNet, + 'effnet': EffnetSphDecoderNet + } + + # load model + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model {model_path}') + model_dict = torch.load(model_path, map_location=torch.device('cpu')) + + self.w = cfg.DATA.IMG_HEIGHT + self.h = cfg.DATA.IMG_WIDTH + self.max_depth_meters = 10.0 + self.to_tensor = transforms.ToTensor() + self.normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + # build model + model_type = encoder_model_dict[cfg.BACKBONE.TYPE] + self.model = model_type(cfg, pretrained=False) + + # load state dict + self.model.to(self.device) + self.model.load_state_dict(model_dict['model'], strict=True) + self.model.eval() + + nside = 128 + self.hp_info = compute_hp_info( + nside, (cfg.DATA.IMG_HEIGHT, cfg.DATA.IMG_WIDTH)) + + logger.info(f'model init done! Device:{self.device}') + + def forward(self, rgb): + """ + Args: + rgb: equirectangular panorama images + The torch size of rgb should be [n, 3, 512, 1024] + Returns: + S2net model outputs containing the predicted equirectangular depth images in metric + """ + equi_inputs = rgb.to(self.device) + return self.model(equi_inputs) + + def postprocess(self, pred_depths_hp): + depth_maps = render_depth_map(pred_depths_hp, + self.hp_info['image_to_sp_map'])[0] + results = {OutputKeys.DEPTHS: depth_maps} + return results diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/__init__.py b/runtime/Lib/site_packages/models/cv/salient_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b3b5b5fa775cfd8a3ac41634a930fbf13a06bc20 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .salient_model import SalientDetection + +else: + _import_structure = { + 'salient_model': ['SalientDetection'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/__init__.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6df5101ad0c345390158e6470685d94b2c05b405 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/__init__.py @@ -0,0 +1,4 @@ +# The implementation is adopted from U-2-Net, made publicly available under the Apache 2.0 License +# source code avaiable via https://github.com/xuebinqin/U-2-Net +from .senet import SENet +from .u2net import U2NET diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/Res2Net_v1b.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/Res2Net_v1b.py new file mode 100644 index 0000000000000000000000000000000000000000..5f92ed8bd0f19099745d2c45441d64a442fc4c15 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/Res2Net_v1b.py @@ -0,0 +1,186 @@ +# Implementation in this file is modified based on SINet-V2,made publicly available under the Apache 2.0 License +# publicly available at https://github.com/GewelsJI/SINet-V2 +import math + +import torch +import torch.nn as nn + +__all__ = ['Res2Net', 'res2net50_v1b_26w_4s'] + + +class Bottle2neck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + baseWidth=26, + scale=4, + stype='normal'): + """ Constructor + Args: + inplanes: input channel dimensionality + planes: output channel dimensionality + stride: conv stride. Replaces pooling layer. + downsample: None when stride = 1 + baseWidth: basic width of conv3x3 + scale: number of scale. + type: 'normal': normal set. 'stage': first block of a new stage. + """ + super(Bottle2neck, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = nn.Conv2d( + inplanes, width * scale, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(width * scale) + if scale == 1: + self.nums = 1 + else: + self.nums = scale - 1 + if stype == 'stage': + self.pool = nn.AvgPool2d(kernel_size=3, stride=stride, padding=1) + convs = [] + bns = [] + for i in range(self.nums): + convs.append( + nn.Conv2d( + width, + width, + kernel_size=3, + stride=stride, + padding=1, + bias=False)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.conv3 = nn.Conv2d( + width * scale, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stype = stype + self.scale = scale + self.width = width + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0 or self.stype == 'stage': + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + if self.scale != 1 and self.stype == 'normal': + out = torch.cat((out, spx[self.nums]), 1) + elif self.scale != 1 and self.stype == 'stage': + out = torch.cat((out, self.pool(spx[self.nums])), 1) + out = self.conv3(out) + out = self.bn3(out) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + + +class Res2Net(nn.Module): + + def __init__(self, block, layers, baseWidth=26, scale=4, num_classes=1000): + self.inplanes = 64 + super(Res2Net, self).__init__() + self.baseWidth = baseWidth + self.scale = scale + self.conv1 = nn.Sequential( + nn.Conv2d(3, 32, 3, 2, 1, bias=False), nn.BatchNorm2d(32), + nn.ReLU(inplace=True), nn.Conv2d(32, 32, 3, 1, 1, bias=False), + nn.BatchNorm2d(32), nn.ReLU(inplace=True), + nn.Conv2d(32, 64, 3, 1, 1, bias=False)) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Linear(512 * block.expansion, num_classes) + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False), + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=1, + bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + layers = [] + layers.append( + block( + self.inplanes, + planes, + stride, + downsample=downsample, + stype='stage', + baseWidth=self.baseWidth, + scale=self.scale)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + baseWidth=self.baseWidth, + scale=self.scale)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x + + +def res2net50_v1b_26w_4s(backbone_path, pretrained=False, **kwargs): + """Constructs a Res2Net-50_v1b_26w_4s lib. + Args: + pretrained (bool): If True, returns a lib pre-trained on ImageNet + """ + model = Res2Net(Bottle2neck, [3, 4, 6, 3], baseWidth=26, scale=4, **kwargs) + if pretrained: + model_state = torch.load(backbone_path) + model.load_state_dict(model_state) + return model diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/__init__.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5a97ef5d7f5d0cfb881b464172db180dd8504d5c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/backbone/__init__.py @@ -0,0 +1,5 @@ +# Implementation in this file is modified based on SINet-V2,made publicly available under the Apache 2.0 License +# publicly available at https://github.com/GewelsJI/SINet-V2 +from .Res2Net_v1b import res2net50_v1b_26w_4s + +__all__ = ['res2net50_v1b_26w_4s'] diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/modules.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..09796bd37c3f93e961d8a85dadfe679137a45c48 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/modules.py @@ -0,0 +1,178 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .utils import ConvBNReLU + + +class AreaLayer(nn.Module): + + def __init__(self, in_channel, out_channel): + super(AreaLayer, self).__init__() + self.lbody = nn.Sequential( + nn.Conv2d(out_channel, out_channel, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True)) + self.hbody = nn.Sequential( + nn.Conv2d(in_channel, out_channel, 1), nn.BatchNorm2d(out_channel), + nn.ReLU(inplace=True)) + self.body = nn.Sequential( + nn.Conv2d(2 * out_channel, out_channel, 3, 1, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True), + nn.Conv2d(out_channel, out_channel, 3, 1, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True), + nn.Conv2d(out_channel, 1, 1)) + + def forward(self, xl, xh): + xl1 = self.lbody(xl) + xl1 = F.interpolate( + xl1, size=xh.size()[2:], mode='bilinear', align_corners=True) + xh1 = self.hbody(xh) + x = torch.cat((xl1, xh1), dim=1) + x_out = self.body(x) + return x_out + + +class EdgeLayer(nn.Module): + + def __init__(self, in_channel, out_channel): + super(EdgeLayer, self).__init__() + self.lbody = nn.Sequential( + nn.Conv2d(out_channel, out_channel, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True)) + self.hbody = nn.Sequential( + nn.Conv2d(in_channel, out_channel, 1), nn.BatchNorm2d(out_channel), + nn.ReLU(inplace=True)) + self.bodye = nn.Sequential( + nn.Conv2d(2 * out_channel, out_channel, 3, 1, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True), + nn.Conv2d(out_channel, out_channel, 3, 1, 1), + nn.BatchNorm2d(out_channel), nn.ReLU(inplace=True), + nn.Conv2d(out_channel, 1, 1)) + + def forward(self, xl, xh): + xl1 = self.lbody(xl) + xh1 = self.hbody(xh) + xh1 = F.interpolate( + xh1, size=xl.size()[2:], mode='bilinear', align_corners=True) + x = torch.cat((xl1, xh1), dim=1) + x_out = self.bodye(x) + return x_out + + +class EBlock(nn.Module): + + def __init__(self, inchs, outchs): + super(EBlock, self).__init__() + self.elayer = nn.Sequential( + ConvBNReLU(inchs + 1, outchs, kernel_size=3, padding=1, stride=1), + ConvBNReLU(outchs, outchs, 1)) + self.salayer = nn.Sequential( + nn.Conv2d(2, 1, 3, 1, 1, bias=False), + nn.BatchNorm2d(1, momentum=0.01), nn.Sigmoid()) + + def forward(self, x, edgeAtten): + x = torch.cat((x, edgeAtten), dim=1) + ex = self.elayer(x) + ex_max = torch.max(ex, 1, keepdim=True)[0] + ex_mean = torch.mean(ex, dim=1, keepdim=True) + xei_compress = torch.cat((ex_max, ex_mean), dim=1) + + scale = self.salayer(xei_compress) + x_out = ex * scale + return x_out + + +class StructureE(nn.Module): + + def __init__(self, inchs, outchs, EM): + super(StructureE, self).__init__() + self.ne_modules = int(inchs / EM) + NM = int(outchs / self.ne_modules) + elayes = [] + for i in range(self.ne_modules): + emblock = EBlock(EM, NM) + elayes.append(emblock) + self.emlayes = nn.ModuleList(elayes) + self.body = nn.Sequential( + ConvBNReLU(outchs, outchs, 3, 1, 1), ConvBNReLU(outchs, outchs, 1)) + + def forward(self, x, edgeAtten): + if edgeAtten.size() != x.size(): + edgeAtten = F.interpolate( + edgeAtten, x.size()[2:], mode='bilinear', align_corners=False) + xx = torch.chunk(x, self.ne_modules, dim=1) + efeas = [] + for i in range(self.ne_modules): + xei = self.emlayes[i](xx[i], edgeAtten) + efeas.append(xei) + efeas = torch.cat(efeas, dim=1) + x_out = self.body(efeas) + return x_out + + +class ABlock(nn.Module): + + def __init__(self, inchs, outchs, k): + super(ABlock, self).__init__() + self.alayer = nn.Sequential( + ConvBNReLU(inchs, outchs, k, 1, k // 2), + ConvBNReLU(outchs, outchs, 1)) + self.arlayer = nn.Sequential( + ConvBNReLU(inchs, outchs, k, 1, k // 2), + ConvBNReLU(outchs, outchs, 1)) + self.fusion = ConvBNReLU(2 * outchs, outchs, 1) + + def forward(self, x, areaAtten): + xa = x * areaAtten + xra = x * (1 - areaAtten) + xout = self.fusion(torch.cat((xa, xra), dim=1)) + return xout + + +class AMFusion(nn.Module): + + def __init__(self, inchs, outchs, AM): + super(AMFusion, self).__init__() + self.k = [3, 3, 5, 5] + self.conv_up = ConvBNReLU(inchs, outchs, 3, 1, 1) + self.up = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=True) + self.na_modules = int(outchs / AM) + alayers = [] + for i in range(self.na_modules): + layer = ABlock(AM, AM, self.k[i]) + alayers.append(layer) + self.alayers = nn.ModuleList(alayers) + self.fusion_0 = ConvBNReLU(outchs, outchs, 3, 1, 1) + self.fusion_e = nn.Sequential( + nn.Conv2d( + outchs, outchs, kernel_size=(3, 1), padding=(1, 0), + bias=False), nn.BatchNorm2d(outchs), nn.ReLU(inplace=True), + nn.Conv2d( + outchs, outchs, kernel_size=(1, 3), padding=(0, 1), + bias=False), nn.BatchNorm2d(outchs), nn.ReLU(inplace=True)) + self.fusion_e1 = nn.Sequential( + nn.Conv2d( + outchs, outchs, kernel_size=(5, 1), padding=(2, 0), + bias=False), nn.BatchNorm2d(outchs), nn.ReLU(inplace=True), + nn.Conv2d( + outchs, outchs, kernel_size=(1, 5), padding=(0, 2), + bias=False), nn.BatchNorm2d(outchs), nn.ReLU(inplace=True)) + self.fusion = ConvBNReLU(3 * outchs, outchs, 1) + + def forward(self, xl, xh, xhm): + xh1 = self.up(self.conv_up(xh)) + x = xh1 + xl + xm = self.up(torch.sigmoid(xhm)) + xx = torch.chunk(x, self.na_modules, dim=1) + xxmids = [] + for i in range(self.na_modules): + xi = self.alayers[i](xx[i], xm) + xxmids.append(xi) + xfea = torch.cat(xxmids, dim=1) + x0 = self.fusion_0(xfea) + x1 = self.fusion_e(xfea) + x2 = self.fusion_e1(xfea) + x_out = self.fusion(torch.cat((x0, x1, x2), dim=1)) + return x_out diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/senet.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/senet.py new file mode 100644 index 0000000000000000000000000000000000000000..37cf42bedccbad6768dbf6cd80a3a7b0ee5723e4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/senet.py @@ -0,0 +1,74 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .backbone import res2net50_v1b_26w_4s as res2net +from .modules import AMFusion, AreaLayer, EdgeLayer, StructureE +from .utils import ASPP, CBAM, ConvBNReLU + + +class SENet(nn.Module): + + def __init__(self, backbone_path=None, pretrained=False): + super(SENet, self).__init__() + resnet50 = res2net(backbone_path, pretrained) + self.layer0_1 = nn.Sequential(resnet50.conv1, resnet50.bn1, + resnet50.relu) + self.maxpool = resnet50.maxpool + self.layer1 = resnet50.layer1 + self.layer2 = resnet50.layer2 + self.layer3 = resnet50.layer3 + self.layer4 = resnet50.layer4 + self.aspp3 = ASPP(1024, 256) + self.aspp4 = ASPP(2048, 256) + self.cbblock3 = CBAM(inchs=256, kernel_size=5) + self.cbblock4 = CBAM(inchs=256, kernel_size=5) + self.up = nn.Upsample( + mode='bilinear', scale_factor=2, align_corners=False) + self.conv_up = ConvBNReLU(512, 512, 1) + self.aux_edge = EdgeLayer(512, 256) + self.aux_area = AreaLayer(512, 256) + self.layer1_enhance = StructureE(256, 128, 128) + self.layer2_enhance = StructureE(512, 256, 128) + self.layer3_decoder = AMFusion(512, 256, 128) + self.layer2_decoder = AMFusion(256, 128, 128) + self.out_conv_8 = nn.Conv2d(256, 1, 1) + self.out_conv_4 = nn.Conv2d(128, 1, 1) + + def forward(self, x): + layer0 = self.layer0_1(x) + layer0s = self.maxpool(layer0) + layer1 = self.layer1(layer0s) + layer2 = self.layer2(layer1) + layer3 = self.layer3(layer2) + layer4 = self.layer4(layer3) + layer3_eh = self.cbblock3(self.aspp3(layer3)) + layer4_eh = self.cbblock4(self.aspp4(layer4)) + layer34 = self.conv_up( + torch.cat((self.up(layer4_eh), layer3_eh), dim=1)) + edge_atten = self.aux_edge(layer1, layer34) + area_atten = self.aux_area(layer1, layer34) + edge_atten_ = torch.sigmoid(edge_atten) + layer1_eh = self.layer1_enhance(layer1, edge_atten_) + layer2_eh = self.layer2_enhance(layer2, edge_atten_) + layer2_fu = self.layer3_decoder(layer2_eh, layer34, area_atten) + out_8 = self.out_conv_8(layer2_fu) + layer1_fu = self.layer2_decoder(layer1_eh, layer2_fu, out_8) + out_4 = self.out_conv_4(layer1_fu) + out_16 = F.interpolate( + area_atten, + size=x.size()[2:], + mode='bilinear', + align_corners=False) + out_8 = F.interpolate( + out_8, size=x.size()[2:], mode='bilinear', align_corners=False) + out_4 = F.interpolate( + out_4, size=x.size()[2:], mode='bilinear', align_corners=False) + edge_out = F.interpolate( + edge_atten_, + size=x.size()[2:], + mode='bilinear', + align_corners=False) + + return out_4.sigmoid(), out_8.sigmoid(), out_16.sigmoid(), edge_out diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/u2net.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/u2net.py new file mode 100644 index 0000000000000000000000000000000000000000..05dbf7ad1029158ebb46a8b3d951bd768fadcfe3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/u2net.py @@ -0,0 +1,301 @@ +# The implementation is adopted from U-2-Net, made publicly available under the Apache 2.0 License +# source code avaiable via https://github.com/xuebinqin/U-2-Net +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class REBNCONV(nn.Module): + + def __init__(self, in_ch=3, out_ch=3, dirate=1): + super(REBNCONV, self).__init__() + self.conv_s1 = nn.Conv2d( + in_ch, out_ch, 3, padding=1 * dirate, dilation=1 * dirate) + self.bn_s1 = nn.BatchNorm2d(out_ch) + self.relu_s1 = nn.ReLU(inplace=True) + + def forward(self, x): + hx = x + xout = self.relu_s1(self.bn_s1(self.conv_s1(hx))) + return xout + + +def _upsample_like(src, tar): + """upsample tensor 'src' to have the same spatial size with tensor 'tar'.""" + src = F.upsample(src, size=tar.shape[2:], mode='bilinear') + return src + + +class RSU7(nn.Module): + + def __init__(self, in_ch=3, mid_ch=12, out_ch=3): + super(RSU7, self).__init__() + self.rebnconvin = REBNCONV(in_ch, out_ch, dirate=1) + self.rebnconv1 = REBNCONV(out_ch, mid_ch, dirate=1) + self.pool1 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv2 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool2 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv3 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool3 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv4 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool4 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv5 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool5 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv6 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.rebnconv7 = REBNCONV(mid_ch, mid_ch, dirate=2) + self.rebnconv6d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv5d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv4d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv3d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv2d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv1d = REBNCONV(mid_ch * 2, out_ch, dirate=1) + + def forward(self, x): + hx = x + hxin = self.rebnconvin(hx) + hx1 = self.rebnconv1(hxin) + hx = self.pool1(hx1) + hx2 = self.rebnconv2(hx) + hx = self.pool2(hx2) + hx3 = self.rebnconv3(hx) + hx = self.pool3(hx3) + hx4 = self.rebnconv4(hx) + hx = self.pool4(hx4) + hx5 = self.rebnconv5(hx) + hx = self.pool5(hx5) + hx6 = self.rebnconv6(hx) + hx7 = self.rebnconv7(hx6) + hx6d = self.rebnconv6d(torch.cat((hx7, hx6), 1)) + hx6dup = _upsample_like(hx6d, hx5) + hx5d = self.rebnconv5d(torch.cat((hx6dup, hx5), 1)) + hx5dup = _upsample_like(hx5d, hx4) + hx4d = self.rebnconv4d(torch.cat((hx5dup, hx4), 1)) + hx4dup = _upsample_like(hx4d, hx3) + hx3d = self.rebnconv3d(torch.cat((hx4dup, hx3), 1)) + hx3dup = _upsample_like(hx3d, hx2) + hx2d = self.rebnconv2d(torch.cat((hx3dup, hx2), 1)) + hx2dup = _upsample_like(hx2d, hx1) + hx1d = self.rebnconv1d(torch.cat((hx2dup, hx1), 1)) + return hx1d + hxin + + +class RSU6(nn.Module): + + def __init__(self, in_ch=3, mid_ch=12, out_ch=3): + super(RSU6, self).__init__() + + self.rebnconvin = REBNCONV(in_ch, out_ch, dirate=1) + self.rebnconv1 = REBNCONV(out_ch, mid_ch, dirate=1) + self.pool1 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv2 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool2 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv3 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool3 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv4 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool4 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv5 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.rebnconv6 = REBNCONV(mid_ch, mid_ch, dirate=2) + self.rebnconv5d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv4d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv3d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv2d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv1d = REBNCONV(mid_ch * 2, out_ch, dirate=1) + + def forward(self, x): + hx = x + hxin = self.rebnconvin(hx) + hx1 = self.rebnconv1(hxin) + hx = self.pool1(hx1) + hx2 = self.rebnconv2(hx) + hx = self.pool2(hx2) + hx3 = self.rebnconv3(hx) + hx = self.pool3(hx3) + hx4 = self.rebnconv4(hx) + hx = self.pool4(hx4) + hx5 = self.rebnconv5(hx) + hx6 = self.rebnconv6(hx5) + hx5d = self.rebnconv5d(torch.cat((hx6, hx5), 1)) + hx5dup = _upsample_like(hx5d, hx4) + hx4d = self.rebnconv4d(torch.cat((hx5dup, hx4), 1)) + hx4dup = _upsample_like(hx4d, hx3) + hx3d = self.rebnconv3d(torch.cat((hx4dup, hx3), 1)) + hx3dup = _upsample_like(hx3d, hx2) + hx2d = self.rebnconv2d(torch.cat((hx3dup, hx2), 1)) + hx2dup = _upsample_like(hx2d, hx1) + hx1d = self.rebnconv1d(torch.cat((hx2dup, hx1), 1)) + return hx1d + hxin + + +class RSU5(nn.Module): + + def __init__(self, in_ch=3, mid_ch=12, out_ch=3): + super(RSU5, self).__init__() + + self.rebnconvin = REBNCONV(in_ch, out_ch, dirate=1) + self.rebnconv1 = REBNCONV(out_ch, mid_ch, dirate=1) + self.pool1 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv2 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool2 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv3 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool3 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv4 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.rebnconv5 = REBNCONV(mid_ch, mid_ch, dirate=2) + self.rebnconv4d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv3d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv2d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv1d = REBNCONV(mid_ch * 2, out_ch, dirate=1) + + def forward(self, x): + hx = x + hxin = self.rebnconvin(hx) + hx1 = self.rebnconv1(hxin) + hx = self.pool1(hx1) + hx2 = self.rebnconv2(hx) + hx = self.pool2(hx2) + hx3 = self.rebnconv3(hx) + hx = self.pool3(hx3) + hx4 = self.rebnconv4(hx) + hx5 = self.rebnconv5(hx4) + hx4d = self.rebnconv4d(torch.cat((hx5, hx4), 1)) + hx4dup = _upsample_like(hx4d, hx3) + hx3d = self.rebnconv3d(torch.cat((hx4dup, hx3), 1)) + hx3dup = _upsample_like(hx3d, hx2) + hx2d = self.rebnconv2d(torch.cat((hx3dup, hx2), 1)) + hx2dup = _upsample_like(hx2d, hx1) + hx1d = self.rebnconv1d(torch.cat((hx2dup, hx1), 1)) + return hx1d + hxin + + +class RSU4(nn.Module): + + def __init__(self, in_ch=3, mid_ch=12, out_ch=3): + super(RSU4, self).__init__() + + self.rebnconvin = REBNCONV(in_ch, out_ch, dirate=1) + self.rebnconv1 = REBNCONV(out_ch, mid_ch, dirate=1) + self.pool1 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv2 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.pool2 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.rebnconv3 = REBNCONV(mid_ch, mid_ch, dirate=1) + self.rebnconv4 = REBNCONV(mid_ch, mid_ch, dirate=2) + self.rebnconv3d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv2d = REBNCONV(mid_ch * 2, mid_ch, dirate=1) + self.rebnconv1d = REBNCONV(mid_ch * 2, out_ch, dirate=1) + + def forward(self, x): + + hx = x + hxin = self.rebnconvin(hx) + hx1 = self.rebnconv1(hxin) + hx = self.pool1(hx1) + hx2 = self.rebnconv2(hx) + hx = self.pool2(hx2) + hx3 = self.rebnconv3(hx) + hx4 = self.rebnconv4(hx3) + hx3d = self.rebnconv3d(torch.cat((hx4, hx3), 1)) + hx3dup = _upsample_like(hx3d, hx2) + hx2d = self.rebnconv2d(torch.cat((hx3dup, hx2), 1)) + hx2dup = _upsample_like(hx2d, hx1) + hx1d = self.rebnconv1d(torch.cat((hx2dup, hx1), 1)) + return hx1d + hxin + + +class RSU4F(nn.Module): + + def __init__(self, in_ch=3, mid_ch=12, out_ch=3): + super(RSU4F, self).__init__() + + self.rebnconvin = REBNCONV(in_ch, out_ch, dirate=1) + self.rebnconv1 = REBNCONV(out_ch, mid_ch, dirate=1) + self.rebnconv2 = REBNCONV(mid_ch, mid_ch, dirate=2) + self.rebnconv3 = REBNCONV(mid_ch, mid_ch, dirate=4) + self.rebnconv4 = REBNCONV(mid_ch, mid_ch, dirate=8) + self.rebnconv3d = REBNCONV(mid_ch * 2, mid_ch, dirate=4) + self.rebnconv2d = REBNCONV(mid_ch * 2, mid_ch, dirate=2) + self.rebnconv1d = REBNCONV(mid_ch * 2, out_ch, dirate=1) + + def forward(self, x): + + hx = x + hxin = self.rebnconvin(hx) + hx1 = self.rebnconv1(hxin) + hx2 = self.rebnconv2(hx1) + hx3 = self.rebnconv3(hx2) + hx4 = self.rebnconv4(hx3) + hx3d = self.rebnconv3d(torch.cat((hx4, hx3), 1)) + hx2d = self.rebnconv2d(torch.cat((hx3d, hx2), 1)) + hx1d = self.rebnconv1d(torch.cat((hx2d, hx1), 1)) + return hx1d + hxin + + +class U2NET(nn.Module): + + def __init__(self, in_ch=3, out_ch=1): + super(U2NET, self).__init__() + + # encoder + self.stage1 = RSU7(in_ch, 32, 64) + self.pool12 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.stage2 = RSU6(64, 32, 128) + self.pool23 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.stage3 = RSU5(128, 64, 256) + self.pool34 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.stage4 = RSU4(256, 128, 512) + self.pool45 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.stage5 = RSU4F(512, 256, 512) + self.pool56 = nn.MaxPool2d(2, stride=2, ceil_mode=True) + self.stage6 = RSU4F(512, 256, 512) + # decoder + self.stage5d = RSU4F(1024, 256, 512) + self.stage4d = RSU4(1024, 128, 256) + self.stage3d = RSU5(512, 64, 128) + self.stage2d = RSU6(256, 32, 64) + self.stage1d = RSU7(128, 16, 64) + self.side1 = nn.Conv2d(64, out_ch, 3, padding=1) + self.side2 = nn.Conv2d(64, out_ch, 3, padding=1) + self.side3 = nn.Conv2d(128, out_ch, 3, padding=1) + self.side4 = nn.Conv2d(256, out_ch, 3, padding=1) + self.side5 = nn.Conv2d(512, out_ch, 3, padding=1) + self.side6 = nn.Conv2d(512, out_ch, 3, padding=1) + self.outconv = nn.Conv2d(6 * out_ch, out_ch, 1) + + def forward(self, x): + + hx = x + hx1 = self.stage1(hx) + hx = self.pool12(hx1) + hx2 = self.stage2(hx) + hx = self.pool23(hx2) + hx3 = self.stage3(hx) + hx = self.pool34(hx3) + hx4 = self.stage4(hx) + hx = self.pool45(hx4) + hx5 = self.stage5(hx) + hx = self.pool56(hx5) + hx6 = self.stage6(hx) + hx6up = _upsample_like(hx6, hx5) + + hx5d = self.stage5d(torch.cat((hx6up, hx5), 1)) + hx5dup = _upsample_like(hx5d, hx4) + hx4d = self.stage4d(torch.cat((hx5dup, hx4), 1)) + hx4dup = _upsample_like(hx4d, hx3) + hx3d = self.stage3d(torch.cat((hx4dup, hx3), 1)) + hx3dup = _upsample_like(hx3d, hx2) + hx2d = self.stage2d(torch.cat((hx3dup, hx2), 1)) + hx2dup = _upsample_like(hx2d, hx1) + hx1d = self.stage1d(torch.cat((hx2dup, hx1), 1)) + d1 = self.side1(hx1d) + d2 = self.side2(hx2d) + d2 = _upsample_like(d2, d1) + d3 = self.side3(hx3d) + d3 = _upsample_like(d3, d1) + d4 = self.side4(hx4d) + d4 = _upsample_like(d4, d1) + d5 = self.side5(hx5d) + d5 = _upsample_like(d5, d1) + d6 = self.side6(hx6) + d6 = _upsample_like(d6, d1) + d0 = self.outconv(torch.cat((d1, d2, d3, d4, d5, d6), 1)) + return torch.sigmoid(d0), torch.sigmoid(d1), torch.sigmoid( + d2), torch.sigmoid(d3), torch.sigmoid(d4), torch.sigmoid( + d5), torch.sigmoid(d6) diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/models/utils.py b/runtime/Lib/site_packages/models/cv/salient_detection/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dac5a7db5bd0c2cd0c22460a07bd1f249ac7c941 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/models/utils.py @@ -0,0 +1,105 @@ +# Implementation in this file is modified based on deeplabv3 +# Originally MIT license,publicly available at https://github.com/fregu856/deeplabv3/blob/master/model/aspp.py +# Implementation in this file is modified based on attention-module +# Originally MIT license,publicly available at https://github.com/Jongchan/attention-module/blob/master/MODELS/cbam.py +import torch +import torch.nn as nn + + +class ConvBNReLU(nn.Module): + + def __init__(self, + inplanes, + planes, + kernel_size=3, + stride=1, + padding=0, + dilation=1, + bias=False): + super(ConvBNReLU, self).__init__() + self.block = nn.Sequential( + nn.Conv2d( + inplanes, + planes, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias), nn.BatchNorm2d(planes), nn.ReLU(inplace=True)) + + def forward(self, x): + return self.block(x) + + +class ASPP(nn.Module): + + def __init__(self, in_dim, out_dim): + super(ASPP, self).__init__() + mid_dim = 128 + self.conv1 = ConvBNReLU(in_dim, mid_dim, kernel_size=1) + self.conv2 = ConvBNReLU( + in_dim, mid_dim, kernel_size=3, padding=2, dilation=2) + self.conv3 = ConvBNReLU( + in_dim, mid_dim, kernel_size=3, padding=5, dilation=5) + self.conv4 = ConvBNReLU( + in_dim, mid_dim, kernel_size=3, padding=7, dilation=7) + self.conv5 = ConvBNReLU(in_dim, mid_dim, kernel_size=1, padding=0) + self.fuse = ConvBNReLU(5 * mid_dim, out_dim, 3, 1, 1) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + + def forward(self, x): + conv1 = self.conv1(x) + conv2 = self.conv2(x) + conv3 = self.conv3(x) + conv4 = self.conv4(x) + xg = self.conv5(self.global_pooling(x)) + conv5 = nn.Upsample((x.shape[2], x.shape[3]), mode='nearest')(xg) + return self.fuse(torch.cat((conv1, conv2, conv3, conv4, conv5), 1)) + + +class ChannelAttention(nn.Module): + + def __init__(self, inchs, ratio=16): + super(ChannelAttention, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.max_pool = nn.AdaptiveMaxPool2d(1) + self.fc = nn.Sequential( + nn.Conv2d(inchs, inchs // 16, 1, bias=False), nn.ReLU(), + nn.Conv2d(inchs // 16, inchs, 1, bias=False)) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = self.fc(self.avg_pool(x)) + max_out = self.fc(self.max_pool(x)) + out = avg_out + max_out + return self.sigmoid(out) + + +class SpatialAttention(nn.Module): + + def __init__(self, kernel_size=7): + super(SpatialAttention, self).__init__() + + self.conv1 = nn.Conv2d( + 2, 1, kernel_size, padding=kernel_size // 2, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + + +class CBAM(nn.Module): + + def __init__(self, inchs, kernel_size=7): + super().__init__() + self.calayer = ChannelAttention(inchs=inchs) + self.saLayer = SpatialAttention(kernel_size=kernel_size) + + def forward(self, x): + xca = self.calayer(x) * x + xsa = self.saLayer(xca) * xca + return xsa diff --git a/runtime/Lib/site_packages/models/cv/salient_detection/salient_model.py b/runtime/Lib/site_packages/models/cv/salient_detection/salient_model.py new file mode 100644 index 0000000000000000000000000000000000000000..e25166c8b5e4e46e1af5e74edab058b09bd9462a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/salient_detection/salient_model.py @@ -0,0 +1,77 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp + +import cv2 +import torch +from PIL import Image +from torchvision import transforms + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .models import U2NET, SENet + + +@MODELS.register_module( + Tasks.semantic_segmentation, module_name=Models.detection) +class SalientDetection(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, *args, **kwargs) + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + self.norm_mean = [0.485, 0.456, 0.406] + self.norm_std = [0.229, 0.224, 0.225] + self.norm_size = (320, 320) + + config_path = osp.join(model_dir, 'config.py') + if osp.exists(config_path) is False: + self.model = U2NET(3, 1) + else: + self.model = SENet(backbone_path=None, pretrained=False) + config = Config.from_file(config_path) + self.norm_mean = config.norm_mean + self.norm_std = config.norm_std + self.norm_size = config.norm_size + checkpoint = torch.load(model_path, map_location='cpu') + self.transform_input = transforms.Compose([ + transforms.Resize(self.norm_size), + transforms.ToTensor(), + transforms.Normalize(mean=self.norm_mean, std=self.norm_std) + ]) + self.model.load_state_dict(checkpoint) + self.model.eval() + + def inference(self, data): + """data is tensor 3 * H * W ---> return tensor H * W .""" + data = data.unsqueeze(0) + if next(self.model.parameters()).is_cuda: + data = data.to( + torch.device([next(self.model.parameters()).device][0])) + + with torch.no_grad(): + results = self.model(data) + + if next(self.model.parameters()).is_cuda: + return results[0][0, 0, :, :].cpu() + return results[0][0, 0, :, :] + + def preprocess(self, image): + """image is numpy.""" + data = self.transform_input(Image.fromarray(image)) + return data.float() + + def postprocess(self, inputs): + """resize .""" + data = inputs['data'] + w = inputs['img_w'] + h = inputs['img_h'] + data_norm = (data - torch.min(data)) / ( + torch.max(data) - torch.min(data)) + data_norm_np = (data_norm.numpy() * 255).astype('uint8') + data_norm_rst = cv2.resize(data_norm_np, (w, h)) + + return data_norm_rst diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..072628bd3dedd2e553f4031df38d262b85883c33 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .shop_seg_base import SHOPSEG + +else: + _import_structure = {'shop_seg_base': ['SHOPSEG']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/common.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/common.py new file mode 100644 index 0000000000000000000000000000000000000000..8cb940a5e5cccdada90943293b173b91fd6eb560 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/common.py @@ -0,0 +1,57 @@ +# Base modules are adapted from https://github.com/open-mmlab/mmcv/, +# originally Apache 2.0 License, Copyright (c) 2018-2022 OpenMMLab, +# https://github.com/open-mmlab/mmsegmentation/, +# originally Apache 2.0 License, Copyright (c) 2020-2021 OpenMMLab, +# and adapted from https://github.com/raoyongming/DenseCLIP/, +# originally MIT License, Copyright (c) 2022 Rao, Yongming. + +import warnings + +import torch.nn as nn +import torch.nn.functional as F + + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > input_w: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +class Upsample(nn.Module): + + def __init__(self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None): + super(Upsample, self).__init__() + self.size = size + if isinstance(scale_factor, tuple): + self.scale_factor = tuple(float(factor) for factor in scale_factor) + else: + self.scale_factor = float(scale_factor) if scale_factor else None + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + if not self.size: + size = [int(t * self.scale_factor) for t in x.shape[-2:]] + else: + size = self.size + return resize(x, size, None, self.mode, self.align_corners) diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/head_fpn.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/head_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..0d4027cb7467feb5441bf60fa3596e4d0e77827f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/head_fpn.py @@ -0,0 +1,121 @@ +# Base modules are adapted from https://github.com/open-mmlab/mmcv/, +# originally Apache 2.0 License, Copyright (c) 2018-2022 OpenMMLab, +# https://github.com/open-mmlab/mmsegmentation/, +# originally Apache 2.0 License, Copyright (c) 2020-2021 OpenMMLab, +# and adapted from https://github.com/raoyongming/DenseCLIP/, +# originally MIT License, Copyright (c) 2022 Rao, Yongming. + +import numpy as np +import torch +import torch.nn as nn +from mmcv.cnn import ConvModule +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ + +from .common import Upsample, resize + + +class FPNHead(nn.Module): + """Panoptic Feature Pyramid Networks. + This head is the implementation of `Semantic FPN + `_. + Args: + feature_strides (tuple[int]): The strides for input feature maps. + stack_lateral. All strides suppose to be power of 2. The first + one is of largest resolution. + """ + + def __init__(self, + channels, + num_classes, + dropout_ratio=0.1, + feature_strides=[4, 8, 16, 32], + align_corners=False, + **kwargs): + super(FPNHead, self).__init__() + self.act_cfg = dict(type='ReLU') + self.channels = channels + self.conv_cfg = None + self.norm_cfg = None + self.norm_cfg = dict(type='BN2d', requires_grad=True) + self.align_corners = align_corners + self.dropout_ratio = dropout_ratio + self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.in_index = [0, 1, 2, 3] + assert min(feature_strides) == feature_strides[0] + self.feature_strides = feature_strides + self.scale_heads = nn.ModuleList() + for i in range(len(feature_strides)): + head_length = max( + 1, + int(np.log2(feature_strides[i]) - np.log2(feature_strides[0]))) + scale_head = [] + for k in range(head_length): + scale_head.append( + ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if feature_strides[i] != feature_strides[0]: + scale_head.append( + Upsample( + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners)) + self.scale_heads.append(nn.Sequential(*scale_head)) + + self.apply(self._init_weights) + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + inputs = [inputs[i] for i in self.in_index] + return inputs + + def cls_seg(self, feat): + """Classify each pixel.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.conv_seg(feat) + return output + + def forward(self, inputs): + x = self._transform_inputs(inputs) + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.feature_strides)): + # non inplace + output = output + resize( + self.scale_heads[i](x[i]), + size=output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + + output = self.cls_seg(output) + return output + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight.data, nonlinearity='relu') + if m.bias is not None: + nn.init.constant_(m.bias.data, 0) diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/models.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/models.py new file mode 100644 index 0000000000000000000000000000000000000000..a206e9f1cabdc930f7d1b1411df85185aab199e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/models.py @@ -0,0 +1,900 @@ +# Base modules are adapted from https://github.com/open-mmlab/mmcv/, +# originally Apache 2.0 License, Copyright (c) 2018-2022 OpenMMLab, +# https://github.com/open-mmlab/mmsegmentation/, +# originally Apache 2.0 License, Copyright (c) 2020-2021 OpenMMLab, +# and adapted from https://github.com/raoyongming/DenseCLIP/, +# originally MIT License, Copyright (c) 2022 Rao, Yongming. + +import math +from collections import OrderedDict + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ +from torch import nn + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + self.embed_dim = embed_dim + self.spacial_dim = spacial_dim + + def forward(self, x): + B, C, H, W = x.shape + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + + cls_pos = self.positional_embedding[0:1, :] + spatial_pos = F.interpolate( + self.positional_embedding[1:, ].reshape(1, self.spacial_dim, + self.spacial_dim, + self.embed_dim).permute( + 0, 3, 1, 2), + size=(H, W), + mode='bilinear') + spatial_pos = spatial_pos.reshape(self.embed_dim, H * W).permute(1, 0) + positional_embedding = torch.cat([cls_pos, spatial_pos], dim=0) + + x = x + positional_embedding[:, None, :] + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + x = x.permute(1, 2, 0) + global_feat = x[:, :, 0] + feature_map = x[:, :, 1:].reshape(B, -1, H, W) + return global_feat, feature_map + + +class CLIPResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim=512, + input_resolution=224, + width=64, + pretrained=None, + **kwargs): + super().__init__() + self.pretrained = pretrained + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + def init_weights(self, pretrained=None): + pretrained = pretrained or self.pretrained + if isinstance(pretrained, str): + checkpoint = torch.jit.load( + pretrained, map_location='cpu').float().state_dict() + + state_dict = {} + + for k in checkpoint.keys(): + if k.startswith('visual.'): + new_k = k.replace('visual.', '') + state_dict[new_k] = checkpoint[k] + + u, w = self.load_state_dict(state_dict, False) + print(u, w, 'are misaligned params in CLIPResNet') + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + + outs = [] + x = self.layer1(x) + outs.append(x) + x = self.layer2(x) + outs.append(x) + x = self.layer3(x) + outs.append(x) + x = self.layer4(x) + outs.append(x) + + return tuple(outs) + + +class CLIPResNetWithAttention(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim=1024, + input_resolution=224, + width=64, + pretrained=None, + **kwargs): + super().__init__() + self.pretrained = pretrained + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, 32, + output_dim) + + def init_weights(self, pretrained=None): + pretrained = pretrained or self.pretrained + if isinstance(pretrained, str): + checkpoint = torch.jit.load( + pretrained, map_location='cpu').float().state_dict() + + state_dict = {} + + for k in checkpoint.keys(): + if k.startswith('visual.'): + new_k = k.replace('visual.', '') + state_dict[new_k] = checkpoint[k] + + if 'positional_embedding' in new_k: + if self.attnpool.positional_embedding.shape != state_dict[ + new_k].shape: + print( + f'Resize the pos_embed shape from {state_dict[new_k].shape}' + f' to {self.attnpool.positional_embedding.shape}' + ) + cls_pos = state_dict[new_k][0:1, :] + H = W = self.input_resolution // 32 + old_h = int( + math.sqrt(state_dict[new_k][1:, ].shape[0])) + spatial_pos = F.interpolate( + state_dict[new_k][1:, ].reshape( + 1, old_h, old_h, + cls_pos.shape[1]).permute(0, 3, 1, 2), + size=(H, W), + mode='bilinear') + spatial_pos = spatial_pos.reshape( + cls_pos.shape[1], H * W).permute(1, 0) + positional_embedding = torch.cat( + [cls_pos, spatial_pos], dim=0) + state_dict[new_k] = positional_embedding + assert self.attnpool.positional_embedding.shape == state_dict[ + new_k].shape + + u, w = self.load_state_dict(state_dict, False) + print(u, w, 'are misaligned params in CLIPResNet') + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + + outs = [] + x = self.layer1(x) + outs.append(x) + x = self.layer2(x) + outs.append(x) + x = self.layer3(x) + outs.append(x) + x = self.layer4(x) + outs.append(x) + + x_global, x_local = self.attnpool(x) + outs.append([x_global, x_local]) + + return tuple(outs) + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None, + drop_path=0.): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.drop_path(self.attention(self.ln_1(x))) + x = x + self.drop_path(self.mlp(self.ln_2(x))) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None, + drop_path_rate=0.): + super().__init__() + self.width = width + self.layers = layers + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, layers) + ] # stochastic depth decay rule + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask, dpr[i]) + for i in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class Attention(nn.Module): + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim**-0.5 + + self.q_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.k_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.v_proj = nn.Linear(dim, dim, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, q, k, v): + B, N, C = q.shape + assert k.shape == v.shape + B, M, C = k.shape + q = self.q_proj(q).reshape(B, N, self.num_heads, C // self.num_heads) + k = self.k_proj(k).reshape(B, M, self.num_heads, C // self.num_heads) + v = self.v_proj(v).reshape(B, M, self.num_heads, C // self.num_heads) + + attn = torch.einsum('bnkc,bmkc->bknm', q, k) * self.scale + + attn = attn.softmax(dim=-1) + + x = torch.einsum('bknm,bmkc->bnkc', attn, v).reshape(B, N, C) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class TransformerDecoderLayer(nn.Module): + + def __init__( + self, + d_model, + nhead, + dropout=0.1, + ): + super().__init__() + self.self_attn = Attention(d_model, nhead, proj_drop=dropout) + self.cross_attn = Attention(d_model, nhead, proj_drop=dropout) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.mlp = nn.Sequential( + nn.Linear(d_model, d_model * 4), nn.GELU(), nn.Dropout(dropout), + nn.Linear(d_model * 4, d_model)) + + def forward(self, x, mem): + q = k = v = self.norm1(x) + x = x + self.self_attn(q, k, v) + q = self.norm2(x) + x = x + self.cross_attn(q, mem, mem) + x = x + self.dropout(self.mlp(self.norm3(x))) + return x + + +class CLIPVisionTransformer(nn.Module): + + def __init__(self, + input_resolution=224, + patch_size=32, + width=768, + layers=12, + heads=12, + output_dim=512, + drop_path_rate=0.0, + out_indices=[3, 5, 7, 11], + pretrained=None, + get_embeddings=False, + **kwargs): + super().__init__() + self.pretrained = pretrained + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.spatial_size = input_resolution // patch_size + self.ln_pre = LayerNorm(width) + self.get_embeddings = get_embeddings + + self.transformer = Transformer( + width, layers, heads, drop_path_rate=drop_path_rate) + + self.out_indices = out_indices + + if get_embeddings: + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + embed_dim = width + + if patch_size == 16: + self.fpn1 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.ConvTranspose2d( + embed_dim, embed_dim, kernel_size=2, stride=2), + nn.BatchNorm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d( + embed_dim, embed_dim, kernel_size=2, stride=2), + ) + + self.fpn2 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.ConvTranspose2d( + embed_dim, embed_dim, kernel_size=2, stride=2), + ) + + self.fpn3 = nn.GroupNorm(1, embed_dim) + + self.fpn4 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.MaxPool2d(kernel_size=2, stride=2)) + + elif patch_size == 8: + self.fpn1 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.ConvTranspose2d( + embed_dim, embed_dim, kernel_size=2, stride=2), + ) + + self.fpn2 = nn.GroupNorm(1, embed_dim) + + self.fpn3 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.MaxPool2d(kernel_size=2, stride=2), + ) + + self.fpn4 = nn.Sequential( + nn.GroupNorm(1, embed_dim), + nn.MaxPool2d(kernel_size=4, stride=4), + ) + + def init_weights(self, pretrained=None): + pretrained = pretrained or self.pretrained + if isinstance(pretrained, str): + checkpoint = torch.jit.load( + pretrained, map_location='cpu').float().state_dict() + + state_dict = {} + + for k in checkpoint.keys(): + if k.startswith('visual.'): + new_k = k.replace('visual.', '') + state_dict[new_k] = checkpoint[k] + + if 'positional_embedding' in state_dict.keys(): + if self.positional_embedding.shape != state_dict[ + 'positional_embedding'].shape: + print( + f'Resize the pos_embed shape from {state_dict["positional_embedding"].shape} to' + f' {self.positional_embedding.shape}') + cls_pos = state_dict['positional_embedding'][0:1, :] + spatial_pos = F.interpolate( + state_dict['positional_embedding'][1:, ].reshape( + 1, 14, 14, 768).permute(0, 3, 1, 2), + size=(self.spatial_size, self.spatial_size), + mode='bilinear') + spatial_pos = spatial_pos.reshape( + 768, + self.spatial_size * self.spatial_size).permute(1, 0) + positional_embedding = torch.cat([cls_pos, spatial_pos], + dim=0) + state_dict['positional_embedding'] = positional_embedding + assert self.positional_embedding.shape == state_dict[ + 'positional_embedding'].shape + + u, w = self.load_state_dict(state_dict, False) + print(u, w, 'are misaligned params in vision transformer') + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + B, C, H, W = x.shape + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + x1 = self.class_embedding.to(x.dtype) + x2 = torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([x1 + x2, x], dim=1) + pos = self.positional_embedding.to(x.dtype) + cls_pos = pos[0, :] + self.class_embedding.to(x.dtype) + spatial_pos = F.interpolate( + pos[1:, ].reshape(1, self.spatial_size, self.spatial_size, + C).permute(0, 3, 1, 2), + size=(H, W), + mode='bilinear') + spatial_pos = spatial_pos.reshape(1, C, H * W).permute(0, 2, 1) + pos = torch.cat([cls_pos.reshape(1, 1, C), spatial_pos], dim=1) + x = x + pos + x = self.ln_pre(x) + x = x.permute(1, 0, 2) # NLD -> LND + + gradientcheckpoint = False + + features = [] + for i, blk in enumerate(self.transformer.resblocks): + if gradientcheckpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + + if i in self.out_indices: + xp = x.permute(1, 0, 2)[:, + 1:, :].permute(0, 2, + 1).reshape(B, -1, H, W) + features.append(xp.contiguous()) + + ops = [self.fpn1, self.fpn2, self.fpn3, self.fpn4] + for i in range(len(features)): + features[i] = ops[i](features[i]) + + if self.get_embeddings: + x = x.permute(1, 0, 2) + x = self.ln_post(x) + x = x @ self.proj + + global_embedding = x[:, 0] + visual_embedding = x[:, 1:].reshape(B, H, W, + -1).permute(0, 3, 1, + 2) # B C H W + + features.append([global_embedding, visual_embedding]) + + return tuple(features) + + +class CLIPTextEncoder(nn.Module): + + def __init__(self, + context_length=77, + vocab_size=49408, + transformer_width=512, + transformer_heads=8, + transformer_layers=12, + embed_dim=1024, + out_dim=256, + pretrained=None, + **kwargs): + super().__init__() + + self.pretrained = pretrained + + self.context_length = context_length + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask()) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + + def init_weights(self, pretrained=None): + pretrained = pretrained or self.pretrained + if isinstance(pretrained, str): + checkpoint = torch.jit.load( + pretrained, map_location='cpu').float().state_dict() + + state_dict = {} + + for k in checkpoint.keys(): + if k.startswith('transformer.'): + state_dict[k] = checkpoint[k] + + if k == 'positional_embedding' or k == 'text_projection' or k.startswith( + 'token_embedding') or k.startswith('ln_final'): + if k == 'positional_embedding' and checkpoint[k].size( + 0) > self.context_length: + checkpoint[k] = checkpoint[k][:self.context_length] + print('positional_embedding is tuncated from 77 to', + self.context_length) + state_dict[k] = checkpoint[k] + + u, w = self.load_state_dict(state_dict, False) + print(u, w, 'are misaligned params in text encoder') + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + def forward(self, text): + x = self.token_embedding(text) + x = x + self.positional_embedding + x = x.permute(1, 0, 2) + x = self.transformer(x) + x = x.permute(1, 0, 2) + x = self.ln_final(x) + x = x[torch.arange(x.shape[0]), + text.argmax(dim=-1), ...] @ self.text_projection + return x + + +class CLIPTextContextEncoder(nn.Module): + + def __init__(self, + context_length=22, + vocab_size=49408, + transformer_width=512, + transformer_heads=8, + transformer_layers=12, + embed_dim=1024, + out_dim=256, + pretrained=None, + **kwargs): + super().__init__() + + self.pretrained = pretrained + + self.context_length = context_length + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask()) + + self.embed_dim = embed_dim + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + + def init_weights(self, pretrained=None): + pretrained = pretrained or self.pretrained + if isinstance(pretrained, str): + checkpoint = torch.jit.load( + pretrained, map_location='cpu').float().state_dict() + + state_dict = {} + + for k in checkpoint.keys(): + if k.startswith('transformer.'): + state_dict[k] = checkpoint[k] + + if k == 'positional_embedding' or k == 'text_projection' or k.startswith( + 'token_embedding') or k.startswith('ln_final'): + if k == 'positional_embedding' and checkpoint[k].size( + 0) > self.context_length: + checkpoint[k] = checkpoint[k][:self.context_length] + print('positional_embedding is tuncated from 77 to', + self.context_length) + state_dict[k] = checkpoint[k] + + u, w = self.load_state_dict(state_dict, False) + print(u, w, 'are misaligned params in text encoder') + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + def forward(self, text, context=None): + x_text = self.token_embedding(text) # n_clas, n_text, C + K, N1, C = x_text.shape # 150类 * 5??? * 512 + B, N2, C = context.shape # 1 * 8 * 512 + + eos_indx = text.argmax(dim=-1) + N2 + eos_indx = eos_indx.reshape(1, K).expand(B, K).reshape(-1) + + x_text = x_text.reshape(1, K, N1, C).expand(B, K, N1, C) + context = context.reshape(B, 1, N2, C).expand(B, K, N2, C) + + x = torch.cat([x_text[:, :, 0:1], context, x_text[:, :, 1:]], + dim=2).reshape(B * K, N1 + N2, C) + x = x + self.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + x = x[torch.arange(x.shape[0]), eos_indx] @ self.text_projection + x = x.reshape(B, K, self.embed_dim) + return x + + +class ContextDecoder(nn.Module): + + def __init__(self, + transformer_width=256, + transformer_heads=4, + transformer_layers=6, + visual_dim=1024, + dropout=0.1, + **kwargs): + super().__init__() + + self.memory_proj = nn.Sequential( + nn.LayerNorm(visual_dim), + nn.Linear(visual_dim, transformer_width), + nn.LayerNorm(transformer_width), + ) + + self.text_proj = nn.Sequential( + nn.LayerNorm(visual_dim), + nn.Linear(visual_dim, transformer_width), + ) + + self.decoder = nn.ModuleList([ + TransformerDecoderLayer(transformer_width, transformer_heads, + dropout) for _ in range(transformer_layers) + ]) + + self.out_proj = nn.Sequential( + nn.LayerNorm(transformer_width), + nn.Linear(transformer_width, visual_dim)) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, text, visual): + B, N, C = visual.shape + visual = self.memory_proj(visual) + x = self.text_proj(text) + + for layer in self.decoder: + x = layer(x, visual) + + return self.out_proj(x) diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/neck_fpn.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/neck_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..d344de713ac2249aaacfd1950c14d83beb2c36fe --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/neck_fpn.py @@ -0,0 +1,216 @@ +# Base modules are adapted from https://github.com/open-mmlab/mmcv/, +# originally Apache 2.0 License, Copyright (c) 2018-2022 OpenMMLab, +# https://github.com/open-mmlab/mmsegmentation/, +# originally Apache 2.0 License, Copyright (c) 2020-2021 OpenMMLab, +# and adapted from https://github.com/raoyongming/DenseCLIP/, +# originally MIT License, Copyright (c) 2022 Rao, Yongming. + +import torch.nn as nn +import torch.nn.functional as F +from mmcv.cnn import ConvModule +from timm.models.layers.drop import drop_path +from timm.models.layers.weight_init import trunc_normal_ + +from .common import resize + + +class FPN(nn.Module): + """Feature Pyramid Network. + + This neck is the implementation of `Feature Pyramid Networks for Object + Detection `_. + + Args: + in_channels (list[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale). + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + extra_convs_on_inputs (bool, deprecated): Whether to apply extra convs + on the original feature from the backbone. If True, + it is equivalent to `add_extra_convs='on_input'`. If False, it is + equivalent to set `add_extra_convs='on_output'`. Default to True. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: dict(mode='nearest'). + init_cfg (dict or list[dict], optional): Initialization config dict. + + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest')): + super(FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + if extra_convs_on_inputs: + # For compatibility with previous release + # TODO: deprecate `extra_convs_on_inputs` + self.add_extra_convs = 'on_input' + else: + self.add_extra_convs = 'on_output' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + self.apply(self._init_weights) + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] = laterals[i - 1] + resize( + laterals[i], **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] = laterals[i - 1] + resize( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight.data, nonlinearity='relu') + if m.bias is not None: + nn.init.constant_(m.bias.data, 0) diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_base.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_base.py new file mode 100644 index 0000000000000000000000000000000000000000..3468637079773b9dfbafe70174728f8f1cbbe40a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_base.py @@ -0,0 +1,155 @@ +# Base modules are adapted from https://github.com/open-mmlab/mmcv/, +# originally Apache 2.0 License, Copyright (c) 2018-2022 OpenMMLab, +# https://github.com/open-mmlab/mmsegmentation/, +# originally Apache 2.0 License, Copyright (c) 2020-2021 OpenMMLab, +# and adapted from https://github.com/raoyongming/DenseCLIP/, +# originally MIT License, Copyright (c) 2022 Rao, Yongming. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .head_fpn import FPNHead +from .models import (CLIPTextContextEncoder, CLIPVisionTransformer, + ContextDecoder) +from .neck_fpn import FPN +from .utils import SimpleTokenizer, tokenize + + +class SHOPSEG(nn.Module): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + + def __init__(self, + model_dir, + context_length=22, + context_feature='attention', + score_concat_index=2, + tau=0.07, + token_embed_dim=512, + text_dim=512, + **args): + super(SHOPSEG, self).__init__() + + self.model_dir = model_dir + self.tokenizer = SimpleTokenizer(model_dir + + '/bpe_simple_vocab_16e6.txt.gz') + + backbone = CLIPVisionTransformer( + input_resolution=1024, + patch_size=16, + width=768, + layers=12, + output_dim=512, + drop_path_rate=0.1, + pretrained=False, + get_embeddings=True) + + text_encoder = CLIPTextContextEncoder( + context_length=30, + vocab_size=49408, + transformer_width=512, + transformer_heads=8, + transformer_layers=12, + embed_dim=512, + pretrained=False) + + context_decoder = ContextDecoder( + transformer_width=256, + transformer_heads=4, + transformer_layers=3, + visual_dim=512, + dropout=0.1) + neck = FPN( + in_channels=[768, 768, 768 + 2, 768], out_channels=256, num_outs=4) + head_fpd = FPNHead(channels=256, num_classes=2) + + self.backbone = backbone + self.text_encoder = text_encoder + self.context_decoder = context_decoder + self.context_length = context_length + self.score_concat_index = score_concat_index + + self.context_feature = context_feature + self.tau = tau + context_length = self.text_encoder.context_length - self.context_length + self.contexts = nn.Parameter( + torch.randn(1, context_length, token_embed_dim)) + nn.init.trunc_normal_(self.contexts) + self.gamma = nn.Parameter(torch.ones(text_dim) * 1e-4) + + self.neck = neck + self.head_fpn = head_fpd + + self.tau = 0.07 + + def encode_text(self, text, context_length): + output = tokenize(self.tokenizer, text, context_length, True) + return output + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + return x + + def after_extract_feat(self, x, name_list): + x_orig = list(x[0:4]) + global_feat, visual_embeddings = x[4] + B, C, H, W = visual_embeddings.shape + if self.context_feature == 'attention': + x1 = global_feat.reshape(B, C, 1) + x2 = visual_embeddings.reshape(B, C, H * W) + visual_context = torch.cat([x1, x2], dim=2).permute(0, 2, 1) + texts = torch.cat([ + self.encode_text(c, context_length=self.context_length) + for c in name_list + ]) + x1 = texts.to(global_feat.device) + x1 = self.text_encoder(x1, self.contexts) + text_embeddings = x1.expand(B, -1, -1) + # update text_embeddings by visual_context! + # (B, 1, C) + text_diff = self.context_decoder(text_embeddings, visual_context) + # (B, K, C) + text_embeddings = text_embeddings + self.gamma * text_diff + + # compute score map and concat + B, K, C = text_embeddings.shape + visual_embeddings = F.normalize(visual_embeddings, dim=1, p=2) + text = F.normalize(text_embeddings, dim=2, p=2) + score_map_list = [] + bsz = B + for i in range(bsz): + ind = 2 * i + sub_text = torch.cat( + [text[i:i + 1, ind:ind + 1], text[i:i + 1, ind + 1:ind + 2]], + dim=1) # 1 * 2 * h * w + + sub_score_map = torch.einsum('bchw,bkc->bkhw', + visual_embeddings[i:i + 1], + sub_text) # 1 * 2 * h * w + score_map_list.append(sub_score_map) + score_map = torch.cat(score_map_list, dim=0) # b * 2 * h * w + x_orig[self.score_concat_index] = torch.cat( + [x_orig[self.score_concat_index], score_map], dim=1) + return x_orig, score_map + + def forward(self, img, text_list=None): + if text_list is None: + bsz = img.size()[0] + text_list = ['foregeound'] * bsz + x = self.extract_feat(img) + _x_orig = [x[i] for i in range(4)] + name_list = [] + for name in text_list: + name_list.append('others') + name_list.append(name[0:20]) + x_orig, score_map = self.after_extract_feat(x, name_list) + x_orig = list(self.neck(x_orig)) + _x_orig = x_orig + pred = self.head_fpn(_x_orig) + return pred diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_model.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ac0d67fa550a5ee5ccc5a48651418ff62c8337c3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/shop_seg_model.py @@ -0,0 +1,117 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.shop_segmentation import SHOPSEG +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['ShopSegmentation'] + + +@MODELS.register_module( + Tasks.shop_segmentation, module_name=Models.shop_segmentation) +class ShopSegmentation(TorchModel): + """ shop segmentation model. + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + self.model = SHOPSEG(model_dir=model_dir) + pretrained_params = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location='cpu') + self.model.load_state_dict(pretrained_params) + self.model.eval() + if device_id >= 0 and torch.cuda.is_available(): + self.model.to('cuda:{}'.format(device_id)) + logger.info('Use GPU: {}'.format(device_id)) + else: + device_id = -1 + logger.info('Use CPU for inference') + self.device_id = device_id + + def preprocess(self, img, size=1024): + mean = [0.48145466, 0.4578275, 0.40821073] + std = [0.26862954, 0.26130258, 0.27577711] + h, w, c = img.shape + max_hw = max(h, w) + ratio = 1.0 * size / max_hw + crop_h, crop_w = int(ratio * h), int(ratio * w) + pil_img = Image.fromarray(img) + pil_img = pil_img.resize((crop_w, crop_h), Image.BILINEAR) + np_img = np.array(pil_img, dtype=np.float32) / 255. + + for j in range(3): + np_img[:, :, j] = (np_img[:, :, j] - mean[j]) / std[j] + + img_pad = np.zeros((size, size, 3), dtype=np.float32) + img_pad[:crop_h, :crop_w] = np_img + + img_pad = torch.from_numpy(img_pad).permute(2, 0, + 1).unsqueeze(0).float() + return img_pad, h, w, crop_h, crop_w + + def postprocess(self, tensors, crop_h, crop_w, ori_h, ori_w): + output = np.clip(tensors * 255., a_min=0, a_max=255.) + crop_output = np.array(output[:crop_h, :crop_w], dtype=np.uint8) + + pil_output = Image.fromarray(crop_output) + pil_output = pil_output.resize((ori_w, ori_h), Image.BILINEAR) + np_output = np.array(pil_output, dtype=np.uint8) + + np_output[np_output < 128] = 0 + np_output[np_output >= 128] = 255 + np_output = np.uint8(np_output) + return np_output + + def forward(self, image): + """ + image should be numpy array, dtype=np.uint8, shape: height*width*3 + """ + image_tensor, ori_h, ori_w, crop_h, crop_w = self.preprocess( + image, size=1024) + pred = self.inference(image_tensor) + msk = self.postprocess(pred, crop_h, crop_w, ori_h, ori_w, size=1024) + + outputs = {OutputKeys.MASKS: msk} + return outputs + + def inference(self, image): + """ + image should be tensor, 1 * 3 * 1024 * 1024 + """ + with torch.no_grad(): + if self.device_id == -1: + output = self.model(image) + else: + device = torch.device('cuda', self.device_id) + output = self.model(image.to(device)) + output = F.interpolate(output, size=(1024, 1024), mode='bilinear') + output = F.softmax(output, dim=1) + output = torch.argmax(output, dim=1) + output = output[0] + if self.device_id == -1: + pred = output.data.numpy() + else: + pred = output.data.cpu().numpy() + + del output + return pred diff --git a/runtime/Lib/site_packages/models/cv/shop_segmentation/utils.py b/runtime/Lib/site_packages/models/cv/shop_segmentation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4035b0ef2606a880f5344a94d7935c1d30c8d866 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/shop_segmentation/utils.py @@ -0,0 +1,198 @@ +# CLIP Tokenizer +# Adapted from https://github.com/openai/CLIP. +# Originally MIT License, Copyright (c) 2021 OpenAI. + +import gzip +import html +import os +from functools import lru_cache +from typing import Any, List, Union + +import ftfy +import regex as re +import torch + + +@lru_cache() +def default_bpe(): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'bpe_simple_vocab_16e6.txt.gz') + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + error_list = [] + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception as err: + error_list.append(err) + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + +def tokenize(tokenizer, + texts, + context_length: int = 77, + truncate: bool = False) -> torch.LongTensor: + """ + Returns the tokenized representation of given input string(s) + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + context_length : int + The context length to use; all CLIP models use 77 as the context length + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = tokenizer.encoder['<|startoftext|>'] + eot_token = tokenizer.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + tokenizer.encode(text) + [eot_token] + for text in texts] + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError( + f'Input {texts[i]} is too long for context length {context_length}' + ) + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/__init__.py b/runtime/Lib/site_packages/models/cv/skin_retouching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/__init__.py b/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_module.py b/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_module.py new file mode 100644 index 0000000000000000000000000000000000000000..5db9c44c893b73613cde1137390b13991d7066fa --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_module.py @@ -0,0 +1,66 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + + +class ConvBNActiv(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bn=True, + sample='none-3', + activ='relu', + bias=False): + super(ConvBNActiv, self).__init__() + + if sample == 'down-7': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=bias) + elif sample == 'down-5': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=5, + stride=2, + padding=2, + bias=bias) + elif sample == 'down-3': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=bias) + else: + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + + if bn: + self.bn = nn.BatchNorm2d(out_channels) + + if activ == 'relu': + self.activation = nn.ReLU() + elif activ == 'leaky': + self.activation = nn.LeakyReLU(negative_slope=0.2) + + def forward(self, images): + + outputs = self.conv(images) + if hasattr(self, 'bn'): + outputs = self.bn(outputs) + if hasattr(self, 'activation'): + outputs = self.activation(outputs) + + return outputs diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_unet_in.py b/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_unet_in.py new file mode 100644 index 0000000000000000000000000000000000000000..c0be1a52edd06409c14a75639165ff2ec0f209bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/detection_model/detection_unet_in.py @@ -0,0 +1,67 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..weights_init import weights_init +from .detection_module import ConvBNActiv + + +class DetectionUNet(nn.Module): + + def __init__(self, + n_channels, + n_classes, + up_sampling_node='nearest', + init_weights=True): + super(DetectionUNet, self).__init__() + + self.n_classes = n_classes + self.up_sampling_node = up_sampling_node + + self.ec_images_1 = ConvBNActiv( + n_channels, 64, bn=False, sample='down-3') + self.ec_images_2 = ConvBNActiv(64, 128, sample='down-3') + self.ec_images_3 = ConvBNActiv(128, 256, sample='down-3') + self.ec_images_4 = ConvBNActiv(256, 512, sample='down-3') + self.ec_images_5 = ConvBNActiv(512, 512, sample='down-3') + self.ec_images_6 = ConvBNActiv(512, 512, sample='down-3') + + self.dc_images_6 = ConvBNActiv(512 + 512, 512, activ='leaky') + self.dc_images_5 = ConvBNActiv(512 + 512, 512, activ='leaky') + self.dc_images_4 = ConvBNActiv(512 + 256, 256, activ='leaky') + self.dc_images_3 = ConvBNActiv(256 + 128, 128, activ='leaky') + self.dc_images_2 = ConvBNActiv(128 + 64, 64, activ='leaky') + self.dc_images_1 = nn.Conv2d(64 + n_channels, n_classes, kernel_size=1) + + if init_weights: + self.apply(weights_init()) + + def forward(self, input_images): + + ec_images = {} + + ec_images['ec_images_0'] = input_images + ec_images['ec_images_1'] = self.ec_images_1(input_images) + ec_images['ec_images_2'] = self.ec_images_2(ec_images['ec_images_1']) + ec_images['ec_images_3'] = self.ec_images_3(ec_images['ec_images_2']) + ec_images['ec_images_4'] = self.ec_images_4(ec_images['ec_images_3']) + ec_images['ec_images_5'] = self.ec_images_5(ec_images['ec_images_4']) + ec_images['ec_images_6'] = self.ec_images_6(ec_images['ec_images_5']) + # -------------- + # images decoder + # -------------- + logits = ec_images['ec_images_6'] + + for _ in range(6, 0, -1): + + ec_images_skip = 'ec_images_{:d}'.format(_ - 1) + dc_conv = 'dc_images_{:d}'.format(_) + + logits = F.interpolate( + logits, scale_factor=2, mode=self.up_sampling_node) + logits = torch.cat((logits, ec_images[ec_images_skip]), dim=1) + + logits = getattr(self, dc_conv)(logits) + + return logits diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/__init__.py b/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/gconv.py b/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/gconv.py new file mode 100644 index 0000000000000000000000000000000000000000..8b3eb2fcbd3dbe379bd5eef128d6323d1fa965bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/gconv.py @@ -0,0 +1,208 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + + +class GatedConvBNActiv(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bn=True, + sample='none-3', + activ='relu', + bias=False): + super(GatedConvBNActiv, self).__init__() + + if sample == 'down-7': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=bias) + elif sample == 'down-5': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=5, + stride=2, + padding=2, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=5, + stride=2, + padding=2, + bias=bias) + elif sample == 'down-3': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=bias) + else: + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + + if bn: + self.bn = nn.BatchNorm2d(out_channels) + + if activ == 'relu': + self.activation = nn.ReLU() + elif activ == 'leaky': + self.activation = nn.LeakyReLU(negative_slope=0.2) + + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + + images = self.conv(x) + gates = self.sigmoid(self.gate(x)) + + if hasattr(self, 'bn'): + images = self.bn(images) + if hasattr(self, 'activation'): + images = self.activation(images) + + images = images * gates + + return images + + +class GatedConvBNActiv2(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bn=True, + sample='none-3', + activ='relu', + bias=False): + super(GatedConvBNActiv2, self).__init__() + + if sample == 'down-7': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=bias) + elif sample == 'down-5': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=5, + stride=2, + padding=2, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=5, + stride=2, + padding=2, + bias=bias) + elif sample == 'down-3': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=bias) + else: + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + self.gate = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + + self.conv_skip = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=bias) + + if bn: + self.bn = nn.BatchNorm2d(out_channels) + + if activ == 'relu': + self.activation = nn.ReLU() + elif activ == 'leaky': + self.activation = nn.LeakyReLU(negative_slope=0.2) + + self.sigmoid = nn.Sigmoid() + + def forward(self, f_up, f_skip, mask): + x = torch.cat((f_up, f_skip, mask), dim=1) + images = self.conv(x) + images_skip = self.conv_skip(f_skip) + gates = self.sigmoid(self.gate(x)) + + if hasattr(self, 'bn'): + images = self.bn(images) + images_skip = self.bn(images_skip) + if hasattr(self, 'activation'): + images = self.activation(images) + images_skip = self.activation(images_skip) + + images = images * gates + images_skip * (1 - gates) + + return images diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/inpainting_unet.py b/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/inpainting_unet.py new file mode 100644 index 0000000000000000000000000000000000000000..dd220dd6aaeb1d47f1e9ea59c9b31a88464a003e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/inpainting_model/inpainting_unet.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.skin_retouching.inpainting_model.gconv import \ + GatedConvBNActiv +from ..weights_init import weights_init + + +class RetouchingNet(nn.Module): + + def __init__(self, + in_channels=3, + out_channels=3, + up_sampling_node='nearest', + init_weights=True): + super(RetouchingNet, self).__init__() + + self.freeze_ec_bn = False + self.up_sampling_node = up_sampling_node + + self.ec_images_1 = GatedConvBNActiv( + in_channels, 64, bn=False, sample='down-3') + self.ec_images_2 = GatedConvBNActiv(64, 128, sample='down-3') + self.ec_images_3 = GatedConvBNActiv(128, 256, sample='down-3') + self.ec_images_4 = GatedConvBNActiv(256, 512, sample='down-3') + self.ec_images_5 = GatedConvBNActiv(512, 512, sample='down-3') + self.ec_images_6 = GatedConvBNActiv(512, 512, sample='down-3') + + self.dc_images_6 = GatedConvBNActiv(512 + 512, 512, activ='leaky') + self.dc_images_5 = GatedConvBNActiv(512 + 512, 512, activ='leaky') + self.dc_images_4 = GatedConvBNActiv(512 + 256, 256, activ='leaky') + self.dc_images_3 = GatedConvBNActiv(256 + 128, 128, activ='leaky') + self.dc_images_2 = GatedConvBNActiv(128 + 64, 64, activ='leaky') + self.dc_images_1 = GatedConvBNActiv( + 64 + in_channels, + out_channels, + bn=False, + sample='none-3', + activ=None, + bias=True) + + self.tanh = nn.Tanh() + + if init_weights: + self.apply(weights_init()) + + def forward(self, input_images, input_masks): + + ec_images = {} + + ec_images['ec_images_0'] = torch.cat((input_images, input_masks), + dim=1) + ec_images['ec_images_1'] = self.ec_images_1(ec_images['ec_images_0']) + ec_images['ec_images_2'] = self.ec_images_2(ec_images['ec_images_1']) + ec_images['ec_images_3'] = self.ec_images_3(ec_images['ec_images_2']) + + ec_images['ec_images_4'] = self.ec_images_4(ec_images['ec_images_3']) + ec_images['ec_images_5'] = self.ec_images_5(ec_images['ec_images_4']) + ec_images['ec_images_6'] = self.ec_images_6(ec_images['ec_images_5']) + + # -------------- + # images decoder + # -------------- + dc_images = ec_images['ec_images_6'] + for _ in range(6, 0, -1): + ec_images_skip = 'ec_images_{:d}'.format(_ - 1) + dc_conv = 'dc_images_{:d}'.format(_) + + dc_images = F.interpolate( + dc_images, scale_factor=2, mode=self.up_sampling_node) + dc_images = torch.cat((dc_images, ec_images[ec_images_skip]), + dim=1) + + dc_images = getattr(self, dc_conv)(dc_images) + + outputs = self.tanh(dc_images) + + return outputs + + def train(self, mode=True): + + super().train(mode) + + if self.freeze_ec_bn: + for name, module in self.named_modules(): + if isinstance(module, nn.BatchNorm2d): + module.eval() diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/__init__.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/box_utils.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/box_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a4aeffd11cc127ea43c2a2b092fcb9851b51afc4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/box_utils.py @@ -0,0 +1,272 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +from typing import List, Tuple, Union + +import numpy as np +import torch + + +def point_form(boxes: torch.Tensor) -> torch.Tensor: + """Convert prior_boxes to (x_min, y_min, x_max, y_max) representation for comparison to point form \ + ground truth data. + + Args: + boxes: center-size default boxes from priorbox layers. + Return: + boxes: Converted x_min, y_min, x_max, y_max form of boxes. + """ + return torch.cat( + (boxes[:, :2] - boxes[:, 2:] / 2, boxes[:, :2] + boxes[:, 2:] / 2), + dim=1) + + +def center_size(boxes: torch.Tensor) -> torch.Tensor: + """Convert prior_boxes to (cx, cy, w, h) representation for comparison to center-size form ground truth data. + Args: + boxes: point_form boxes + Return: + boxes: Converted x_min, y_min, x_max, y_max form of boxes. + """ + return torch.cat( + ((boxes[:, 2:] + boxes[:, :2]) / 2, boxes[:, 2:] - boxes[:, :2]), + dim=1) + + +def intersect(box_a: torch.Tensor, box_b: torch.Tensor) -> torch.Tensor: + """ We resize both tensors to [A,B,2] without new malloc: + [A, 2] -> [A, 1, 2] -> [A, B, 2] + [B, 2] -> [1, B, 2] -> [A, B, 2] + Then we compute the area of intersect between box_a and box_b. + Args: + box_a: bounding boxes, Shape: [A, 4]. + box_b: bounding boxes, Shape: [B, 4]. + Return: + intersection area, Shape: [A, B]. + """ + A = box_a.size(0) + B = box_b.size(0) + max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2), + box_b[:, 2:].unsqueeze(0).expand(A, B, 2)) + min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2), + box_b[:, :2].unsqueeze(0).expand(A, B, 2)) + inter = torch.clamp((max_xy - min_xy), min=0) + return inter[:, :, 0] * inter[:, :, 1] + + +def jaccard(box_a: torch.Tensor, box_b: torch.Tensor) -> torch.Tensor: + """Compute the jaccard overlap of two sets of boxes. The jaccard overlap is simply the intersection over + union of two boxes. Here we operate on ground truth boxes and default boxes. + E.g.: + A ∩ B / A ∪ B = A ∩ B / (area(A) + area(B) - A ∩ B) + Args: + box_a: Ground truth bounding boxes, Shape: [num_objects,4] + box_b: Prior boxes from priorbox layers, Shape: [num_priors,4] + Return: + jaccard overlap: Shape: [box_a.size(0), box_b.size(0)] + """ + inter = intersect(box_a, box_b) + area_a = (box_a[:, 2] - box_a[:, 0]) * (box_a[:, 3] - box_a[:, 1]) + area_a = area_a.unsqueeze(1).expand_as(inter) # [A,B] + area_b = (box_b[:, 2] - box_b[:, 0]) * (box_b[:, 3] - box_b[:, 1]) + area_b = area_b.unsqueeze(0).expand_as(inter) # [A,B] + union = area_a + area_b - inter + return inter / union + + +def matrix_iof(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """ + return iof of a and b, numpy version for data augmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + return area_i / np.maximum(area_a[:, np.newaxis], 1) + + +def match( + threshold: float, + box_gt: torch.Tensor, + priors: torch.Tensor, + variances: List[float], + labels_gt: torch.Tensor, + landmarks_gt: torch.Tensor, + box_t: torch.Tensor, + label_t: torch.Tensor, + landmarks_t: torch.Tensor, + batch_id: int, +) -> None: + """Match each prior box with the ground truth box of the highest jaccard overlap, encode the bounding + boxes, then return the matched indices corresponding to both confidence and location preds. + + Args: + threshold: The overlap threshold used when matching boxes. + box_gt: Ground truth boxes, Shape: [num_obj, 4]. + priors: Prior boxes from priorbox layers, Shape: [n_priors, 4]. + variances: Variances corresponding to each prior coord, Shape: [num_priors, 4]. + labels_gt: All the class labels for the image, Shape: [num_obj, 2]. + landmarks_gt: Ground truth landms, Shape [num_obj, 10]. + box_t: Tensor to be filled w/ endcoded location targets. + label_t: Tensor to be filled w/ matched indices for labels predictions. + landmarks_t: Tensor to be filled w/ endcoded landmarks targets. + batch_id: current batch index + Return: + The matched indices corresponding to 1)location 2)confidence 3)landmarks preds. + """ + # Compute iou between gt and priors + overlaps = jaccard(box_gt, point_form(priors)) + # (Bipartite Matching) + # [1, num_objects] best prior for each ground truth + best_prior_overlap, best_prior_idx = overlaps.max(1, keepdim=True) + + # ignore hard gt + valid_gt_idx = best_prior_overlap[:, 0] >= 0.2 + best_prior_idx_filter = best_prior_idx[valid_gt_idx, :] + if best_prior_idx_filter.shape[0] <= 0: + box_t[batch_id] = 0 + label_t[batch_id] = 0 + return + + # [1, num_priors] best ground truth for each prior + best_truth_overlap, best_truth_idx = overlaps.max(0, keepdim=True) + best_truth_idx.squeeze_(0) + best_truth_overlap.squeeze_(0) + best_prior_idx.squeeze_(1) + best_prior_idx_filter.squeeze_(1) + best_prior_overlap.squeeze_(1) + best_truth_overlap.index_fill_(0, best_prior_idx_filter, + 2) # ensure best prior + # TODO refactor: index best_prior_idx with long tensor + # ensure every gt matches with its prior of max overlap + for j in range(best_prior_idx.size(0)): + best_truth_idx[best_prior_idx[j]] = j + + matches = box_gt[best_truth_idx] # Shape: [num_priors, 4] + labels = labels_gt[best_truth_idx] # Shape: [num_priors] + # label as background + labels[best_truth_overlap < threshold] = 0 + loc = encode(matches, priors, variances) + + matches_landm = landmarks_gt[best_truth_idx] + landmarks_gt = encode_landm(matches_landm, priors, variances) + box_t[batch_id] = loc # [num_priors, 4] encoded offsets to learn + label_t[batch_id] = labels # [num_priors] top class label for each prior + landmarks_t[batch_id] = landmarks_gt + + +def encode(matched, priors, variances): + """Encode the variances from the priorbox layers into the ground truth boxes + we have matched (based on jaccard overlap) with the prior boxes. + Args: + matched: (tensor) Coords of ground truth for each prior in point-form + Shape: [num_priors, 4]. + priors: (tensor) Prior boxes in center-offset form + Shape: [num_priors,4]. + variances: (list[float]) Variances of priorboxes + Return: + encoded boxes (tensor), Shape: [num_priors, 4] + """ + + # dist b/t match center and prior's center + g_cxcy = (matched[:, :2] + matched[:, 2:]) / 2 - priors[:, :2] + # encode variance + g_cxcy /= variances[0] * priors[:, 2:] + # match wh / prior wh + g_wh = (matched[:, 2:] - matched[:, :2]) / priors[:, 2:] + g_wh = torch.log(g_wh) / variances[1] + # return target for smooth_l1_loss + return torch.cat([g_cxcy, g_wh], 1) # [num_priors,4] + + +def encode_landm( + matched: torch.Tensor, priors: torch.Tensor, + variances: Union[List[float], Tuple[float, float]]) -> torch.Tensor: + """Encode the variances from the priorbox layers into the ground truth boxes we have matched + (based on jaccard overlap) with the prior boxes. + Args: + matched: Coords of ground truth for each prior in point-form + Shape: [num_priors, 10]. + priors: Prior boxes in center-offset form + Shape: [num_priors,4]. + variances: Variances of priorboxes + Return: + encoded landmarks, Shape: [num_priors, 10] + """ + + # dist b/t match center and prior's center + matched = torch.reshape(matched, (matched.size(0), 5, 2)) + priors_cx = priors[:, 0].unsqueeze(1).expand(matched.size(0), + 5).unsqueeze(2) + priors_cy = priors[:, 1].unsqueeze(1).expand(matched.size(0), + 5).unsqueeze(2) + priors_w = priors[:, 2].unsqueeze(1).expand(matched.size(0), + 5).unsqueeze(2) + priors_h = priors[:, 3].unsqueeze(1).expand(matched.size(0), + 5).unsqueeze(2) + priors = torch.cat([priors_cx, priors_cy, priors_w, priors_h], dim=2) + g_cxcy = matched[:, :, :2] - priors[:, :, :2] + # encode variance + g_cxcy = g_cxcy // variances[0] * priors[:, :, 2:] + # return target for smooth_l1_loss + return g_cxcy.reshape(g_cxcy.size(0), -1) + + +# Adapted from https://github.com/Hakuyume/chainer-ssd +def decode(loc: torch.Tensor, priors: torch.Tensor, + variances: Union[List[float], Tuple[float, float]]) -> torch.Tensor: + """Decode locations from predictions using priors to undo the encoding we did for offset regression at train time. + Args: + loc: location predictions for loc layers, + Shape: [num_priors, 4] + priors: Prior boxes in center-offset form. + Shape: [num_priors, 4]. + variances: Variances of priorboxes + Return: + decoded bounding box predictions + """ + + boxes = torch.cat( + ( + priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1]), + ), + 1, + ) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def decode_landm( + pre: torch.Tensor, priors: torch.Tensor, + variances: Union[List[float], Tuple[float, float]]) -> torch.Tensor: + """Decode landmarks from predictions using priors to undo the encoding we did for offset regression at train time. + Args: + pre: landmark predictions for loc layers, + Shape: [num_priors, 10] + priors: Prior boxes in center-offset form. + Shape: [num_priors, 4]. + variances: Variances of priorboxes + Return: + decoded landmark predictions + """ + return torch.cat( + ( + priors[:, :2] + pre[:, :2] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:], + ), + dim=1, + ) + + +def log_sum_exp(x: torch.Tensor) -> torch.Tensor: + """Utility function for computing log_sum_exp while determining This will be used to determine unaveraged + confidence loss across all examples in a batch. + Args: + x: conf_preds from conf layers + """ + x_max = x.data.max() + return torch.log(torch.sum(torch.exp(x - x_max), 1, keepdim=True)) + x_max diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/net.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/net.py new file mode 100644 index 0000000000000000000000000000000000000000..e9b0297b6bea8b42579a22afa7da2e943fe1d835 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/net.py @@ -0,0 +1,124 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +from typing import Dict, List + +import torch +import torch.nn.functional as F +from torch import nn + + +def conv_bn(inp: int, + oup: int, + stride: int = 1, + leaky: float = 0) -> nn.Sequential: + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + ) + + +def conv_bn_no_relu(inp: int, oup: int, stride: int) -> nn.Sequential: + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + ) + + +def conv_bn1X1(inp: int, + oup: int, + stride: int, + leaky: float = 0) -> nn.Sequential: + return nn.Sequential( + nn.Conv2d(inp, oup, 1, stride, padding=0, bias=False), + nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + ) + + +def conv_dw(inp: int, + oup: int, + stride: int, + leaky: float = 0.1) -> nn.Sequential: + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.LeakyReLU(negative_slope=leaky, inplace=True), + ) + + +class SSH(nn.Module): + + def __init__(self, in_channel: int, out_channel: int) -> None: + super().__init__() + if out_channel % 4 != 0: + raise ValueError( + f'Expect out channel % 4 == 0, but we got {out_channel % 4}') + + leaky: float = 0 + if out_channel <= 64: + leaky = 0.1 + self.conv3X3 = conv_bn_no_relu(in_channel, out_channel // 2, stride=1) + + self.conv5X5_1 = conv_bn( + in_channel, out_channel // 4, stride=1, leaky=leaky) + self.conv5X5_2 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + self.conv7X7_2 = conv_bn( + out_channel // 4, out_channel // 4, stride=1, leaky=leaky) + self.conv7x7_3 = conv_bn_no_relu( + out_channel // 4, out_channel // 4, stride=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + conv3X3 = self.conv3X3(x) + + conv5X5_1 = self.conv5X5_1(x) + conv5X5 = self.conv5X5_2(conv5X5_1) + + conv7X7_2 = self.conv7X7_2(conv5X5_1) + conv7X7 = self.conv7x7_3(conv7X7_2) + + out = torch.cat([conv3X3, conv5X5, conv7X7], dim=1) + + return F.relu(out) + + +class FPN(nn.Module): + + def __init__(self, in_channels_list: List[int], out_channels: int) -> None: + super().__init__() + leaky = 0.0 + if out_channels <= 64: + leaky = 0.1 + + self.output1 = conv_bn1X1( + in_channels_list[0], out_channels, stride=1, leaky=leaky) + self.output2 = conv_bn1X1( + in_channels_list[1], out_channels, stride=1, leaky=leaky) + self.output3 = conv_bn1X1( + in_channels_list[2], out_channels, stride=1, leaky=leaky) + + self.merge1 = conv_bn(out_channels, out_channels, leaky=leaky) + self.merge2 = conv_bn(out_channels, out_channels, leaky=leaky) + + def forward(self, x: Dict[str, torch.Tensor]) -> List[torch.Tensor]: + y = list(x.values()) + + output1 = self.output1(y[0]) + output2 = self.output2(y[1]) + output3 = self.output3(y[2]) + + up3 = F.interpolate( + output3, size=[output2.size(2), output2.size(3)], mode='nearest') + output2 = output2 + up3 + output2 = self.merge2(output2) + + up2 = F.interpolate( + output2, size=[output1.size(2), output1.size(3)], mode='nearest') + output1 = output1 + up2 + output1 = self.merge1(output1) + + return [output1, output2, output3] diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/network.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/network.py new file mode 100644 index 0000000000000000000000000000000000000000..3b197ca9524d2accbc63112ead3573ca50e3275a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/network.py @@ -0,0 +1,146 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +from typing import Dict, Tuple + +import torch +from torch import nn +from torchvision import models +from torchvision.models import _utils + +from .net import FPN, SSH + + +class ClassHead(nn.Module): + + def __init__(self, in_channels: int = 512, num_anchors: int = 3) -> None: + super().__init__() + self.conv1x1 = nn.Conv2d( + in_channels, + num_anchors * 2, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + return out.view(out.shape[0], -1, 2) + + +class BboxHead(nn.Module): + + def __init__(self, in_channels: int = 512, num_anchors: int = 3): + super().__init__() + self.conv1x1 = nn.Conv2d( + in_channels, + num_anchors * 4, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + return out.view(out.shape[0], -1, 4) + + +class LandmarkHead(nn.Module): + + def __init__(self, in_channels: int = 512, num_anchors: int = 3): + super().__init__() + self.conv1x1 = nn.Conv2d( + in_channels, + num_anchors * 10, + kernel_size=(1, 1), + stride=1, + padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.conv1x1(x) + out = out.permute(0, 2, 3, 1).contiguous() + return out.view(out.shape[0], -1, 10) + + +class RetinaFace(nn.Module): + + def __init__(self, name: str, pretrained: bool, in_channels: int, + return_layers: Dict[str, int], out_channels: int) -> None: + super().__init__() + + if name == 'Resnet50': + backbone = models.resnet50(pretrained=pretrained) + else: + raise NotImplementedError( + f'Only Resnet50 backbone is supported but got {name}') + + self.body = _utils.IntermediateLayerGetter(backbone, return_layers) + in_channels_stage2 = in_channels + in_channels_list = [ + in_channels_stage2 * 2, + in_channels_stage2 * 4, + in_channels_stage2 * 8, + ] + self.fpn = FPN(in_channels_list, out_channels) + self.ssh1 = SSH(out_channels, out_channels) + self.ssh2 = SSH(out_channels, out_channels) + self.ssh3 = SSH(out_channels, out_channels) + + self.ClassHead = self._make_class_head( + fpn_num=3, in_channels=out_channels) + self.BboxHead = self._make_bbox_head( + fpn_num=3, in_channels=out_channels) + self.LandmarkHead = self._make_landmark_head( + fpn_num=3, in_channels=out_channels) + + @staticmethod + def _make_class_head(fpn_num: int = 3, + in_channels: int = 64, + anchor_num: int = 2) -> nn.ModuleList: + classhead = nn.ModuleList() + for _ in range(fpn_num): + classhead.append(ClassHead(in_channels, anchor_num)) + return classhead + + @staticmethod + def _make_bbox_head(fpn_num: int = 3, + in_channels: int = 64, + anchor_num: int = 2) -> nn.ModuleList: + bboxhead = nn.ModuleList() + for _ in range(fpn_num): + bboxhead.append(BboxHead(in_channels, anchor_num)) + return bboxhead + + @staticmethod + def _make_landmark_head(fpn_num: int = 3, + in_channels: int = 64, + anchor_num: int = 2) -> nn.ModuleList: + landmarkhead = nn.ModuleList() + for _ in range(fpn_num): + landmarkhead.append(LandmarkHead(in_channels, anchor_num)) + return landmarkhead + + def forward( + self, inputs: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + out = self.body(inputs) + + # FPN + fpn = self.fpn(out) + + # SSH + feature1 = self.ssh1(fpn[0]) + feature2 = self.ssh2(fpn[1]) + feature3 = self.ssh3(fpn[2]) + features = [feature1, feature2, feature3] + + bbox_regressions = torch.cat( + [self.BboxHead[i](feature) for i, feature in enumerate(features)], + dim=1) + classifications = torch.cat( + [self.ClassHead[i](feature) for i, feature in enumerate(features)], + dim=1) + ldm_regressions = [ + self.LandmarkHead[i](feature) for i, feature in enumerate(features) + ] + ldm_regressions = torch.cat(ldm_regressions, dim=1) + + return bbox_regressions, classifications, ldm_regressions diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/predict_single.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/predict_single.py new file mode 100644 index 0000000000000000000000000000000000000000..659a11342fc20061ecc06977626e5243b4c6e845 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/predict_single.py @@ -0,0 +1,152 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +""" +There is a lot of post processing of the predictions. +""" +from typing import Dict, List, Union + +import albumentations as A +import numpy as np +import torch +from torch.nn import functional as F +from torchvision.ops import nms + +from ..utils import pad_to_size, unpad_from_size +from .box_utils import decode, decode_landm +from .network import RetinaFace +from .prior_box import priorbox +from .utils import tensor_from_rgb_image + + +class Model: + + def __init__(self, max_size: int = 960, device: str = 'cpu') -> None: + self.model = RetinaFace( + name='Resnet50', + pretrained=False, + return_layers={ + 'layer2': 1, + 'layer3': 2, + 'layer4': 3 + }, + in_channels=256, + out_channels=256, + ).to(device) + self.device = device + self.transform = A.Compose( + [A.LongestMaxSize(max_size=max_size, p=1), + A.Normalize(p=1)]) + self.max_size = max_size + self.prior_box = priorbox( + min_sizes=[[16, 32], [64, 128], [256, 512]], + steps=[8, 16, 32], + clip=False, + image_size=(self.max_size, self.max_size), + ).to(device) + self.variance = [0.1, 0.2] + + def load_state_dict(self, state_dict: Dict[str, torch.Tensor]) -> None: + self.model.load_state_dict(state_dict) + + def eval(self): + self.model.eval() + + def predict_jsons( + self, + image: np.array, + confidence_threshold: float = 0.7, + nms_threshold: float = 0.4) -> List[Dict[str, Union[List, float]]]: + with torch.no_grad(): + original_height, original_width = image.shape[:2] + + scale_landmarks = torch.from_numpy( + np.tile([self.max_size, self.max_size], + 5)).to(self.device).float() + scale_bboxes = torch.from_numpy( + np.tile([self.max_size, self.max_size], + 2)).to(self.device).float() + + transformed_image = self.transform(image=image)['image'] + + paded = pad_to_size( + target_size=(self.max_size, self.max_size), + image=transformed_image) + + pads = paded['pads'] + + torched_image = tensor_from_rgb_image(paded['image']).to( + self.device) + + loc, conf, land = self.model(torched_image.unsqueeze(0)) + + conf = F.softmax(conf, dim=-1) + + annotations: List[Dict[str, Union[List, float]]] = [] + + boxes = decode(loc.data[0], self.prior_box, self.variance) + + boxes *= scale_bboxes + scores = conf[0][:, 1] + + landmarks = decode_landm(land.data[0], self.prior_box, + self.variance) + landmarks *= scale_landmarks + + # ignore low scores + valid_index = scores > confidence_threshold + boxes = boxes[valid_index] + landmarks = landmarks[valid_index] + scores = scores[valid_index] + + # Sort from high to low + order = scores.argsort(descending=True) + boxes = boxes[order] + landmarks = landmarks[order] + scores = scores[order] + + # do NMS + keep = nms(boxes, scores, nms_threshold) + boxes = boxes[keep, :].int() + + if boxes.shape[0] == 0: + return [{'bbox': [], 'score': -1, 'landmarks': []}] + + landmarks = landmarks[keep] + + scores = scores[keep].cpu().numpy().astype(np.float64) + boxes = boxes.cpu().numpy() + landmarks = landmarks.cpu().numpy() + landmarks = landmarks.reshape([-1, 2]) + + unpadded = unpad_from_size(pads, bboxes=boxes, keypoints=landmarks) + + resize_coeff = max(original_height, original_width) / self.max_size + + boxes = (unpadded['bboxes'] * resize_coeff).astype(int) + landmarks = (unpadded['keypoints'].reshape(-1, 10) + * resize_coeff).astype(int) + + for box_id, bbox in enumerate(boxes): + x_min, y_min, x_max, y_max = bbox + + x_min = np.clip(x_min, 0, original_width - 1) + x_max = np.clip(x_max, x_min + 1, original_width - 1) + + if x_min >= x_max: + continue + + y_min = np.clip(y_min, 0, original_height - 1) + y_max = np.clip(y_max, y_min + 1, original_height - 1) + + if y_min >= y_max: + continue + + annotations += [{ + 'bbox': + bbox.tolist(), + 'score': + scores[box_id], + 'landmarks': + landmarks[box_id].reshape(-1, 2).tolist(), + }] + + return annotations diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/prior_box.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/prior_box.py new file mode 100644 index 0000000000000000000000000000000000000000..863a676c8dc6517e83b00bd24a66ea607138baa5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/prior_box.py @@ -0,0 +1,28 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +from itertools import product +from math import ceil + +import torch + + +def priorbox(min_sizes, steps, clip, image_size): + feature_maps = [[ceil(image_size[0] / step), + ceil(image_size[1] / step)] for step in steps] + + anchors = [] + for k, f in enumerate(feature_maps): + t_min_sizes = min_sizes[k] + for i, j in product(range(f[0]), range(f[1])): + for min_size in t_min_sizes: + s_kx = min_size / image_size[1] + s_ky = min_size / image_size[0] + dense_cx = [x * steps[k] / image_size[1] for x in [j + 0.5]] + dense_cy = [y * steps[k] / image_size[0] for y in [i + 0.5]] + for cy, cx in product(dense_cy, dense_cx): + anchors += [cx, cy, s_kx, s_ky] + + # back to torch land + output = torch.Tensor(anchors).view(-1, 4) + if clip: + output.clamp_(max=1, min=0) + return output diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/utils.py b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c6b97484afaec2d660691ec82a9ee5709b0b5269 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/retinaface/utils.py @@ -0,0 +1,70 @@ +# Implementation in this file is modifed from source code avaiable via https://github.com/ternaus/retinaface +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import cv2 +import numpy as np +import torch + + +def load_checkpoint(file_path: Union[Path, str], + rename_in_layers: Optional[dict] = None) -> Dict[str, Any]: + """Loads PyTorch checkpoint, optionally renaming layer names. + Args: + file_path: path to the torch checkpoint. + rename_in_layers: {from_name: to_name} + ex: {"model.0.": "", + "model.": ""} + Returns: + """ + checkpoint = torch.load( + file_path, map_location=lambda storage, loc: storage) + + if rename_in_layers is not None: + model_state_dict = checkpoint['state_dict'] + + result = {} + for key, value in model_state_dict.items(): + for key_r, value_r in rename_in_layers.items(): + key = re.sub(key_r, value_r, key) + + result[key] = value + + checkpoint['state_dict'] = result + + return checkpoint + + +def tensor_from_rgb_image(image: np.ndarray) -> torch.Tensor: + image = np.transpose(image, (2, 0, 1)) + return torch.from_numpy(image) + + +def vis_annotations(image: np.ndarray, + annotations: List[Dict[str, Any]]) -> np.ndarray: + vis_image = image.copy() + + for annotation in annotations: + landmarks = annotation['landmarks'] + + colors = [(255, 0, 0), (128, 255, 0), (255, 178, 102), (102, 128, 255), + (0, 255, 255)] + + for landmark_id, (x, y) in enumerate(landmarks): + vis_image = cv2.circle( + vis_image, (x, y), + radius=3, + color=colors[landmark_id], + thickness=3) + + x_min, y_min, x_max, y_max = annotation['bbox'] + + x_min = np.clip(x_min, 0, x_max - 1) + y_min = np.clip(y_min, 0, y_max - 1) + + vis_image = cv2.rectangle( + vis_image, (x_min, y_min), (x_max, y_max), + color=(0, 255, 0), + thickness=2) + return vis_image diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/unet_deploy.py b/runtime/Lib/site_packages/models/cv/skin_retouching/unet_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..0ff75b85fe35e933a30afdf2bc42340e96af1b09 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/unet_deploy.py @@ -0,0 +1,144 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .weights_init import weights_init + +warnings.filterwarnings(action='ignore') + + +class double_conv(nn.Module): + '''(conv => BN => ReLU) * 2''' + + def __init__(self, in_ch, out_ch): + super(double_conv, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(in_ch, out_ch, 3, padding=1), nn.BatchNorm2d(out_ch), + nn.ReLU(inplace=True), nn.Conv2d(out_ch, out_ch, 3, padding=1), + nn.BatchNorm2d(out_ch), nn.ReLU(inplace=True)) + + def forward(self, x): + x = self.conv(x) + return x + + +class inconv(nn.Module): + + def __init__(self, in_ch, out_ch): + super(inconv, self).__init__() + self.conv = double_conv(in_ch, out_ch) + + def forward(self, x): + x = self.conv(x) + return x + + +class down(nn.Module): + + def __init__(self, in_ch, out_ch): + super(down, self).__init__() + self.mpconv = nn.Sequential( + nn.MaxPool2d(2), double_conv(in_ch, out_ch)) + + def forward(self, x): + x = self.mpconv(x) + return x + + +class up(nn.Module): + + def __init__(self, in_ch, out_ch, bilinear=True): + super(up, self).__init__() + + if bilinear: + self.up = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=True) + else: + self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2) + + self.conv = double_conv(in_ch, out_ch) + + def forward(self, x1, x2): + x1 = self.up(x1) + + diffY = x2.size()[2] - x1.size()[2] + diffX = x2.size()[3] - x1.size()[3] + + x1 = F.pad( + x1, + (diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2)) + + x = torch.cat([x2, x1], dim=1) + x = self.conv(x) + return x + + +class outconv(nn.Module): + + def __init__(self, in_ch, out_ch): + super(outconv, self).__init__() + self.conv = nn.Conv2d(in_ch, out_ch, 1) + + def forward(self, x): + x = self.conv(x) + return x + + +class UNet(nn.Module): + + def __init__(self, + n_channels, + n_classes, + deep_supervision=False, + init_weights=True): + super(UNet, self).__init__() + self.deep_supervision = deep_supervision + self.inc = inconv(n_channels, 64) + self.down1 = down(64, 128) + self.down2 = down(128, 256) + self.down3 = down(256, 512) + self.down4 = down(512, 512) + self.up1 = up(1024, 256) + self.up2 = up(512, 128) + self.up3 = up(256, 64) + self.up4 = up(128, 64) + self.outc = outconv(64, n_classes) + + self.dsoutc4 = outconv(256, n_classes) + self.dsoutc3 = outconv(128, n_classes) + self.dsoutc2 = outconv(64, n_classes) + self.dsoutc1 = outconv(64, n_classes) + + self.sigmoid = nn.Sigmoid() + + if init_weights: + self.apply(weights_init()) + + def forward(self, x): + x1 = self.inc(x) + x2 = self.down1(x1) + x3 = self.down2(x2) + x4 = self.down3(x3) + x5 = self.down4(x4) + x44 = self.up1(x5, x4) + x33 = self.up2(x44, x3) + x22 = self.up3(x33, x2) + x11 = self.up4(x22, x1) + x0 = self.outc(x11) + x0 = self.sigmoid(x0) + if self.deep_supervision: + x11 = F.interpolate( + self.dsoutc1(x11), x0.shape[2:], mode='bilinear') + x22 = F.interpolate( + self.dsoutc2(x22), x0.shape[2:], mode='bilinear') + x33 = F.interpolate( + self.dsoutc3(x33), x0.shape[2:], mode='bilinear') + x44 = F.interpolate( + self.dsoutc4(x44), x0.shape[2:], mode='bilinear') + + return x0, x11, x22, x33, x44 + else: + return x0 diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/utils.py b/runtime/Lib/site_packages/models/cv/skin_retouching/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eb0da6b946f285444989a1b3ed2be23e87fdbab7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/utils.py @@ -0,0 +1,328 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import time +from typing import Dict, List, Optional, Tuple, Union + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange + +__all__ = [ + 'gen_diffuse_mask', 'get_crop_bbox', 'get_roi_without_padding', + 'patch_aggregation_overlap', 'patch_partition_overlap', 'preprocess_roi', + 'resize_on_long_side', 'roi_to_tensor', 'smooth_border_mg', 'whiten_img' +] + + +def resize_on_long_side(img, long_side=800): + src_height = img.shape[0] + src_width = img.shape[1] + + if src_height > src_width: + scale = long_side * 1.0 / src_height + _img = cv2.resize( + img, (int(src_width * scale), long_side), + interpolation=cv2.INTER_LINEAR) + else: + scale = long_side * 1.0 / src_width + _img = cv2.resize( + img, (long_side, int(src_height * scale)), + interpolation=cv2.INTER_LINEAR) + + return _img, scale + + +def get_crop_bbox(detecting_results): + boxes = [] + for anno in detecting_results: + if anno['score'] == -1: + break + boxes.append({ + 'x1': anno['bbox'][0], + 'y1': anno['bbox'][1], + 'x2': anno['bbox'][2], + 'y2': anno['bbox'][3] + }) + face_count = len(boxes) + + suitable_bboxes = [] + for i in range(face_count): + face_bbox = boxes[i] + + face_bbox_width = abs(face_bbox['x2'] - face_bbox['x1']) + face_bbox_height = abs(face_bbox['y2'] - face_bbox['y1']) + + face_bbox_center = ((face_bbox['x1'] + face_bbox['x2']) / 2, + (face_bbox['y1'] + face_bbox['y2']) / 2) + + square_bbox_length = face_bbox_height if face_bbox_height > face_bbox_width else face_bbox_width + enlarge_ratio = 1.5 + square_bbox_length = int(enlarge_ratio * square_bbox_length) + + sideScale = 1 + + square_bbox = { + 'x1': + int(face_bbox_center[0] - sideScale * square_bbox_length / 2), + 'x2': + int(face_bbox_center[0] + sideScale * square_bbox_length / 2), + 'y1': + int(face_bbox_center[1] - sideScale * square_bbox_length / 2), + 'y2': int(face_bbox_center[1] + sideScale * square_bbox_length / 2) + } + + suitable_bboxes.append(square_bbox) + + return suitable_bboxes + + +def get_roi_without_padding(img, bbox): + crop_t = max(bbox['y1'], 0) + crop_b = min(bbox['y2'], img.shape[0]) + crop_l = max(bbox['x1'], 0) + crop_r = min(bbox['x2'], img.shape[1]) + roi = img[crop_t:crop_b, crop_l:crop_r] + return roi, 0, [crop_t, crop_b, crop_l, crop_r] + + +def roi_to_tensor(img): + img = torch.from_numpy(img.transpose((2, 0, 1)))[None, ...] + + return img + + +def preprocess_roi(img): + img = img.float() / 255.0 + img = (img - 0.5) * 2 + + return img + + +def patch_partition_overlap(image, p1, p2, padding=32): + + B, C, H, W = image.size() + h, w = H // p1, W // p2 + image = F.pad( + image, + pad=(padding, padding, padding, padding, 0, 0), + mode='constant', + value=0) + + patch_list = [] + for i in range(h): + for j in range(w): + patch = image[:, :, p1 * i:p1 * (i + 1) + padding * 2, + p2 * j:p2 * (j + 1) + padding * 2] + patch_list.append(patch) + + output = torch.cat( + patch_list, dim=0) # (b h w) c (p1 + 2 * padding) (p2 + 2 * padding) + return output + + +def patch_aggregation_overlap(image, h, w, padding=32): + + image = image[:, :, padding:-padding, padding:-padding] + + output = rearrange(image, '(b h w) c p1 p2 -> b c (h p1) (w p2)', h=h, w=w) + + return output + + +def smooth_border_mg(diffuse_mask, mg): + mg = mg - 0.5 + diffuse_mask = F.interpolate( + diffuse_mask, mg.shape[:2], mode='bilinear')[0].permute(1, 2, 0) + mg = mg * diffuse_mask + mg = mg + 0.5 + return mg + + +def whiten_img(image, skin_mask, whitening_degree, flag_bigKernal=False): + """ + image: rgb + """ + dilate_kernalsize = 30 + if flag_bigKernal: + dilate_kernalsize = 80 + new_kernel1 = cv2.getStructuringElement( + cv2.MORPH_ELLIPSE, (dilate_kernalsize, dilate_kernalsize)) + new_kernel2 = cv2.getStructuringElement( + cv2.MORPH_ELLIPSE, (dilate_kernalsize, dilate_kernalsize)) + if len(skin_mask.shape) == 3: + skin_mask = skin_mask[:, :, -1] + skin_mask = cv2.dilate(skin_mask, new_kernel1, 1) + skin_mask = cv2.erode(skin_mask, new_kernel2, 1) + skin_mask = cv2.blur(skin_mask, (20, 20)) / 255.0 + skin_mask = skin_mask.squeeze() + skin_mask = torch.from_numpy(skin_mask).to(image.device) + skin_mask = torch.stack([skin_mask, skin_mask, skin_mask], dim=0)[None, + ...] + skin_mask[:, 1:, :, :] *= 0.75 + + whiten_mg = skin_mask * 0.2 * whitening_degree + 0.5 + assert len(whiten_mg.shape) == 4 + whiten_mg = F.interpolate( + whiten_mg, image.shape[:2], mode='bilinear')[0].permute(1, 2, + 0).half() + output_pred = image.half() + output_pred = output_pred / 255.0 + output_pred = ( + -2 * whiten_mg + 1 + ) * output_pred * output_pred + 2 * whiten_mg * output_pred # value: 0~1 + output_pred = output_pred * 255.0 + output_pred = output_pred.byte() + + output_pred = output_pred.cpu().numpy() + return output_pred + + +def gen_diffuse_mask(out_channels=3): + mask_size = 500 + diffuse_with = 20 + a = np.ones(shape=(mask_size, mask_size), dtype=np.float32) + + for i in range(mask_size): + for j in range(mask_size): + if i >= diffuse_with and i <= ( + mask_size - diffuse_with) and j >= diffuse_with and j <= ( + mask_size - diffuse_with): + a[i, j] = 1.0 + elif i <= diffuse_with: + a[i, j] = i * 1.0 / diffuse_with + elif i > (mask_size - diffuse_with): + a[i, j] = (mask_size - i) * 1.0 / diffuse_with + + for i in range(mask_size): + for j in range(mask_size): + if j <= diffuse_with: + a[i, j] = min(a[i, j], j * 1.0 / diffuse_with) + elif j > (mask_size - diffuse_with): + a[i, j] = min(a[i, j], (mask_size - j) * 1.0 / diffuse_with) + a = np.dstack([a] * out_channels) + return a + + +def pad_to_size( + target_size: Tuple[int, int], + image: np.array, + bboxes: Optional[np.ndarray] = None, + keypoints: Optional[np.ndarray] = None, +) -> Dict[str, Union[np.ndarray, Tuple[int, int, int, int]]]: + """Pads the image on the sides to the target_size + + Args: + target_size: (target_height, target_width) + image: + bboxes: np.array with shape (num_boxes, 4). Each row: [x_min, y_min, x_max, y_max] + keypoints: np.array with shape (num_keypoints, 2), each row: [x, y] + + Returns: + { + "image": padded_image, + "pads": (x_min_pad, y_min_pad, x_max_pad, y_max_pad), + "bboxes": shifted_boxes, + "keypoints": shifted_keypoints + } + + """ + target_height, target_width = target_size + + image_height, image_width = image.shape[:2] + + if target_width < image_width: + raise ValueError(f'Target width should bigger than image_width' + f'We got {target_width} {image_width}') + + if target_height < image_height: + raise ValueError(f'Target height should bigger than image_height' + f'We got {target_height} {image_height}') + + if image_height == target_height: + y_min_pad = 0 + y_max_pad = 0 + else: + y_pad = target_height - image_height + y_min_pad = y_pad // 2 + y_max_pad = y_pad - y_min_pad + + if image_width == target_width: + x_min_pad = 0 + x_max_pad = 0 + else: + x_pad = target_width - image_width + x_min_pad = x_pad // 2 + x_max_pad = x_pad - x_min_pad + + result = { + 'pads': (x_min_pad, y_min_pad, x_max_pad, y_max_pad), + 'image': + cv2.copyMakeBorder(image, y_min_pad, y_max_pad, x_min_pad, x_max_pad, + cv2.BORDER_CONSTANT), + } + + if bboxes is not None: + bboxes[:, 0] += x_min_pad + bboxes[:, 1] += y_min_pad + bboxes[:, 2] += x_min_pad + bboxes[:, 3] += y_min_pad + + result['bboxes'] = bboxes + + if keypoints is not None: + keypoints[:, 0] += x_min_pad + keypoints[:, 1] += y_min_pad + + result['keypoints'] = keypoints + + return result + + +def unpad_from_size( + pads: Tuple[int, int, int, int], + image: Optional[np.array] = None, + bboxes: Optional[np.ndarray] = None, + keypoints: Optional[np.ndarray] = None, +) -> Dict[str, np.ndarray]: + """Crops patch from the center so that sides are equal to pads. + + Args: + image: + pads: (x_min_pad, y_min_pad, x_max_pad, y_max_pad) + bboxes: np.array with shape (num_boxes, 4). Each row: [x_min, y_min, x_max, y_max] + keypoints: np.array with shape (num_keypoints, 2), each row: [x, y] + + Returns: cropped image + + { + "image": cropped_image, + "bboxes": shifted_boxes, + "keypoints": shifted_keypoints + } + + """ + x_min_pad, y_min_pad, x_max_pad, y_max_pad = pads + + result = {} + + if image is not None: + height, width = image.shape[:2] + result['image'] = image[y_min_pad:height - y_max_pad, + x_min_pad:width - x_max_pad] + + if bboxes is not None: + bboxes[:, 0] -= x_min_pad + bboxes[:, 1] -= y_min_pad + bboxes[:, 2] -= x_min_pad + bboxes[:, 3] -= y_min_pad + + result['bboxes'] = bboxes + + if keypoints is not None: + keypoints[:, 0] -= x_min_pad + keypoints[:, 1] -= y_min_pad + + result['keypoints'] = keypoints + + return result diff --git a/runtime/Lib/site_packages/models/cv/skin_retouching/weights_init.py b/runtime/Lib/site_packages/models/cv/skin_retouching/weights_init.py new file mode 100644 index 0000000000000000000000000000000000000000..ae62d4a407b44b895209b8abdc4b833dc94cb2c2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/skin_retouching/weights_init.py @@ -0,0 +1,37 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + + +def weights_init(init_type='kaiming', gain=0.02): + + def init_func(m): + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + + if init_type == 'normal': + nn.init.normal_(m.weight.data, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight.data, gain=gain) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight.data, gain=gain) + + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias.data, 0.0) + + elif classname.find('BatchNorm2d') != -1: + nn.init.normal_(m.weight.data, 1.0, gain) + nn.init.constant_(m.bias.data, 0.0) + + return init_func + + +def spectral_norm(module, mode=True): + + if mode: + return nn.utils.spectral_norm(module) + + return module diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..33d4828bb2bb8ed6440a7a9f563c9911ab1264c4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .realtime_video_detector import RealtimeVideoDetector +else: + _import_structure = { + 'realtime_video_detector': ['RealtimeVideoDetector'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/data/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/data/data_augment.py b/runtime/Lib/site_packages/models/cv/stream_yolo/data/data_augment.py new file mode 100644 index 0000000000000000000000000000000000000000..b52a65fe900aa01ffe11b2be6ba9b070799bc358 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/data/data_augment.py @@ -0,0 +1,69 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX +""" +Data augmentation functionality. Passed as callable transformations to +Dataset classes. + +The data augmentation procedures were interpreted from @weiliu89's SSD paper +http://arxiv.org/abs/1512.02325 +""" + +import math +import random + +import cv2 +import numpy as np + +from ..utils import xyxy2cxcywh + + +def preproc(img, input_size, swap=(2, 0, 1)): + if len(img.shape) == 3: + padded_img = np.ones( + (input_size[0], input_size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(input_size, dtype=np.uint8) * 114 + + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[:int(img.shape[0] * r), :int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + +class ValTransform: + """ + Defines the transformations that should be applied to test PIL image + for input into the network + + dimension -> tensorize -> color adj + + Arguments: + resize (int): input dimension to SSD + rgb_means ((int,int,int)): average RGB of the dataset + (104,117,123) + swap ((int,int,int)): final order of channels + + Returns: + transform (transform) : callable transform to be applied to test/val + data + """ + + def __init__(self, swap=(2, 0, 1), legacy=False): + self.swap = swap + self.legacy = legacy + + # assume input is cv2 img for now + def __call__(self, img, res, input_size): + img, _ = preproc(img, input_size, self.swap) + if self.legacy: + img = img[::-1, :, :].copy() + img /= 255.0 + img -= np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1) + img /= np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1) + return img, np.zeros((1, 5)) diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ddd82580f8fdc0cce3c09d3025bb29746cdae62 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/__init__.py @@ -0,0 +1,4 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +from .base_exp import BaseExp +from .build import get_exp_by_name diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/base_exp.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/base_exp.py new file mode 100644 index 0000000000000000000000000000000000000000..a4278cbfcbbe5b4b1d5b0a97ac6670d2955da543 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/base_exp.py @@ -0,0 +1,12 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +from abc import ABCMeta, abstractmethod + +from torch.nn import Module + + +class BaseExp(metaclass=ABCMeta): + + @abstractmethod + def get_model(self) -> Module: + pass diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/build.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/build.py new file mode 100644 index 0000000000000000000000000000000000000000..ac179a3768dcc9dffe11754f643b7b30676cdc36 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/build.py @@ -0,0 +1,14 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +import os +import sys + + +def get_exp_by_name(exp_name): + exp = exp_name.replace('-', + '_') # convert string like "yolox-s" to "yolox_s" + if exp == 'streamyolo': + from .default import StreamYoloExp as YoloXExp + else: + pass + return YoloXExp() diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c8928d6fd1ced666e0811bf7039e383d59ff4f4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/__init__.py @@ -0,0 +1,2 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX +from .streamyolo import StreamYoloExp diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/streamyolo.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/streamyolo.py new file mode 100644 index 0000000000000000000000000000000000000000..5a62c8fc8c6ba1bde305e19096fcd1eaa7fe3200 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/default/streamyolo.py @@ -0,0 +1,43 @@ +# The implementation is based on StreamYOLO, available at https://github.com/yancie-yjr/StreamYOLO +import os +import sys + +import torch + +from ..yolox_base import Exp as YoloXExp + + +class StreamYoloExp(YoloXExp): + + def __init__(self): + super(YoloXExp, self).__init__() + self.depth = 1.0 + self.width = 1.0 + self.num_classes = 8 + self.test_size = (600, 960) + self.test_conf = 0.3 + self.nmsthre = 0.65 + + def get_model(self): + from ...models import StreamYOLO, DFPPAFPN, TALHead + + def init_yolo(M): + for m in M.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eps = 1e-3 + m.momentum = 0.03 + + if getattr(self, 'model', None) is None: + in_channels = [256, 512, 1024] + backbone = DFPPAFPN( + self.depth, self.width, in_channels=in_channels) + head = TALHead( + self.num_classes, + self.width, + in_channels=in_channels, + gamma=1.0, + ignore_thr=0.5, + ignore_value=1.6) + self.model = StreamYOLO(backbone, head) + + return self.model diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/exp/yolox_base.py b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/yolox_base.py new file mode 100644 index 0000000000000000000000000000000000000000..c5159a9fa08e1059a275b69b0519aef2a8ebd1c2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/exp/yolox_base.py @@ -0,0 +1,58 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX +import os +import random + +import torch +import torch.distributed as dist +import torch.nn as nn + +from .base_exp import BaseExp + + +class Exp(BaseExp): + + def __init__(self): + super().__init__() + + # ---------------- model config ---------------- # + # detect classes number of model + self.num_classes = 80 + # factor of model depth + self.depth = 1.00 + # factor of model width + self.width = 1.00 + # activation name. For example, if using "relu", then "silu" will be replaced to "relu". + self.act = 'silu' + # ----------------- testing config ------------------ # + # output image size during evaluation/test + self.test_size = (640, 640) + # confidence threshold during evaluation/test, + # boxes whose scores are less than test_conf will be filtered + self.test_conf = 0.01 + # nms threshold + self.nmsthre = 0.65 + + def get_model(self): + from ..models import YOLOX, YOLOPAFPN, YOLOXHead + + def init_yolo(M): + for m in M.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eps = 1e-3 + m.momentum = 0.03 + + if getattr(self, 'model', None) is None: + in_channels = [256, 512, 1024] + backbone = YOLOPAFPN( + self.depth, self.width, in_channels=in_channels, act=self.act) + head = YOLOXHead( + self.num_classes, + self.width, + in_channels=in_channels, + act=self.act) + self.model = YOLOX(backbone, head) + + self.model.apply(init_yolo) + self.model.head.initialize_biases(1e-2) + self.model.train() + return self.model diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0988bd5f9bc6681793687a6b701d994b11ec0a16 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/__init__.py @@ -0,0 +1,6 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +from .darknet import CSPDarknet, Darknet +from .dfp_pafpn import DFPPAFPN +from .streamyolo import StreamYOLO +from .tal_head import TALHead diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/darknet.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/darknet.py new file mode 100644 index 0000000000000000000000000000000000000000..8ece2a1e42bd445c9bf82d9e7dce945f81d5105e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/darknet.py @@ -0,0 +1,189 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +from torch import nn + +from .network_blocks import (BaseConv, CSPLayer, DWConv, Focus, ResLayer, + SPPBottleneck) + + +class Darknet(nn.Module): + # number of blocks from dark2 to dark5. + depth2blocks = {21: [1, 2, 2, 1], 53: [2, 8, 8, 4]} + + def __init__( + self, + depth, + in_channels=3, + stem_out_channels=32, + out_features=('dark3', 'dark4', 'dark5'), + ): + """ + Args: + depth (int): depth of darknet used in model, usually use [21, 53] for this param. + in_channels (int): number of input channels, for example, use 3 for RGB image. + stem_out_channels (int): number of output channels of darknet stem. + It decides channels of darknet layer2 to layer5. + out_features (Tuple[str]): desired output layer name. + """ + super().__init__() + assert out_features, 'please provide output features of Darknet' + self.out_features = out_features + self.stem = nn.Sequential( + BaseConv( + in_channels, stem_out_channels, ksize=3, stride=1, + act='lrelu'), + *self.make_group_layer(stem_out_channels, num_blocks=1, stride=2), + ) + in_channels = stem_out_channels * 2 # 64 + + num_blocks = Darknet.depth2blocks[depth] + # create darknet with `stem_out_channels` and `num_blocks` layers. + # to make model structure more clear, we don't use `for` statement in python. + self.dark2 = nn.Sequential( + *self.make_group_layer(in_channels, num_blocks[0], stride=2)) + in_channels *= 2 # 128 + self.dark3 = nn.Sequential( + *self.make_group_layer(in_channels, num_blocks[1], stride=2)) + in_channels *= 2 # 256 + self.dark4 = nn.Sequential( + *self.make_group_layer(in_channels, num_blocks[2], stride=2)) + in_channels *= 2 # 512 + + self.dark5 = nn.Sequential( + *self.make_group_layer(in_channels, num_blocks[3], stride=2), + *self.make_spp_block([in_channels, in_channels * 2], + in_channels * 2), + ) + + def make_group_layer(self, + in_channels: int, + num_blocks: int, + stride: int = 1): + 'starts with conv layer then has `num_blocks` `ResLayer`' + return [ + BaseConv( + in_channels, + in_channels * 2, + ksize=3, + stride=stride, + act='lrelu'), + *[(ResLayer(in_channels * 2)) for _ in range(num_blocks)], + ] + + def make_spp_block(self, filters_list, in_filters): + m = nn.Sequential(*[ + BaseConv(in_filters, filters_list[0], 1, stride=1, act='lrelu'), + BaseConv( + filters_list[0], filters_list[1], 3, stride=1, act='lrelu'), + SPPBottleneck( + in_channels=filters_list[1], + out_channels=filters_list[0], + activation='lrelu', + ), + BaseConv( + filters_list[0], filters_list[1], 3, stride=1, act='lrelu'), + BaseConv( + filters_list[1], filters_list[0], 1, stride=1, act='lrelu'), + ]) + return m + + def forward(self, x): + outputs = {} + x = self.stem(x) + outputs['stem'] = x + x = self.dark2(x) + outputs['dark2'] = x + x = self.dark3(x) + outputs['dark3'] = x + x = self.dark4(x) + outputs['dark4'] = x + x = self.dark5(x) + outputs['dark5'] = x + return {k: v for k, v in outputs.items() if k in self.out_features} + + +class CSPDarknet(nn.Module): + + def __init__( + self, + dep_mul, + wid_mul, + out_features=('dark3', 'dark4', 'dark5'), + depthwise=False, + act='silu', + ): + super().__init__() + assert out_features, 'please provide output features of Darknet' + self.out_features = out_features + Conv = DWConv if depthwise else BaseConv + + base_channels = int(wid_mul * 64) # 64 + base_depth = max(round(dep_mul * 3), 1) # 3 + + # stem + self.stem = Focus(3, base_channels, ksize=3, act=act) + + # dark2 + self.dark2 = nn.Sequential( + Conv(base_channels, base_channels * 2, 3, 2, act=act), + CSPLayer( + base_channels * 2, + base_channels * 2, + n=base_depth, + depthwise=depthwise, + act=act, + ), + ) + + # dark3 + self.dark3 = nn.Sequential( + Conv(base_channels * 2, base_channels * 4, 3, 2, act=act), + CSPLayer( + base_channels * 4, + base_channels * 4, + n=base_depth * 3, + depthwise=depthwise, + act=act, + ), + ) + + # dark4 + self.dark4 = nn.Sequential( + Conv(base_channels * 4, base_channels * 8, 3, 2, act=act), + CSPLayer( + base_channels * 8, + base_channels * 8, + n=base_depth * 3, + depthwise=depthwise, + act=act, + ), + ) + + # dark5 + self.dark5 = nn.Sequential( + Conv(base_channels * 8, base_channels * 16, 3, 2, act=act), + SPPBottleneck( + base_channels * 16, base_channels * 16, activation=act), + CSPLayer( + base_channels * 16, + base_channels * 16, + n=base_depth, + shortcut=False, + depthwise=depthwise, + act=act, + ), + ) + + def forward(self, x): + outputs = {} + x = self.stem(x) + outputs['stem'] = x + x = self.dark2(x) + outputs['dark2'] = x + x = self.dark3(x) + outputs['dark3'] = x + x = self.dark4(x) + outputs['dark4'] = x + x = self.dark5(x) + outputs['dark5'] = x + return {k: v for k, v in outputs.items() if k in self.out_features} diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/dfp_pafpn.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/dfp_pafpn.py new file mode 100644 index 0000000000000000000000000000000000000000..01284791f930bb153c98d85ec6dbe361aa567b00 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/dfp_pafpn.py @@ -0,0 +1,307 @@ +# The implementation is based on StreamYOLO, available at https://github.com/yancie-yjr/StreamYOLO +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .darknet import CSPDarknet +from .network_blocks import BaseConv, CSPLayer, DWConv + + +class DFPPAFPN(nn.Module): + """ + YOLOv3 model. Darknet 53 is the default backbone of this model. + """ + + def __init__( + self, + depth=1.0, + width=1.0, + in_features=('dark3', 'dark4', 'dark5'), + in_channels=[256, 512, 1024], + depthwise=False, + act='silu', + ): + super().__init__() + self.backbone = CSPDarknet(depth, width, depthwise=depthwise, act=act) + self.in_features = in_features + self.in_channels = in_channels + Conv = DWConv if depthwise else BaseConv + + self.lateral_conv0 = BaseConv( + int(in_channels[2] * width), + int(in_channels[1] * width), + 1, + 1, + act=act) + self.C3_p4 = CSPLayer( + int(2 * in_channels[1] * width), + int(in_channels[1] * width), + round(3 * depth), + False, + depthwise=depthwise, + act=act, + ) # cat + + self.reduce_conv1 = BaseConv( + int(in_channels[1] * width), + int(in_channels[0] * width), + 1, + 1, + act=act) + self.C3_p3 = CSPLayer( + int(2 * in_channels[0] * width), + int(in_channels[0] * width), + round(3 * depth), + False, + depthwise=depthwise, + act=act, + ) + + # bottom-up conv + self.bu_conv2 = Conv( + int(in_channels[0] * width), + int(in_channels[0] * width), + 3, + 2, + act=act) + self.C3_n3 = CSPLayer( + int(2 * in_channels[0] * width), + int(in_channels[1] * width), + round(3 * depth), + False, + depthwise=depthwise, + act=act, + ) + + # bottom-up conv + self.bu_conv1 = Conv( + int(in_channels[1] * width), + int(in_channels[1] * width), + 3, + 2, + act=act) + self.C3_n4 = CSPLayer( + int(2 * in_channels[1] * width), + int(in_channels[2] * width), + round(3 * depth), + False, + depthwise=depthwise, + act=act, + ) + + self.jian2 = Conv( + in_channels=int(in_channels[0] * width), + out_channels=int(in_channels[0] * width) // 2, + ksize=1, + stride=1, + act=act, + ) + + self.jian1 = Conv( + in_channels=int(in_channels[1] * width), + out_channels=int(in_channels[1] * width) // 2, + ksize=1, + stride=1, + act=act, + ) + + self.jian0 = Conv( + in_channels=int(in_channels[2] * width), + out_channels=int(in_channels[2] * width) // 2, + ksize=1, + stride=1, + act=act, + ) + + def off_forward(self, input): + """ + Args: + inputs: input images. + + Returns: + Tuple[Tensor]: FPN feature. + """ + + # backbone + rurrent_out_features = self.backbone(torch.split(input, 3, dim=1)[0]) + rurrent_features = [rurrent_out_features[f] for f in self.in_features] + [rurrent_x2, rurrent_x1, rurrent_x0] = rurrent_features + + rurrent_fpn_out0 = self.lateral_conv0(rurrent_x0) # 1024->512/32 + rurrent_f_out0 = F.interpolate( + rurrent_fpn_out0, size=rurrent_x1.shape[2:4], + mode='nearest') # 512/16 + rurrent_f_out0 = torch.cat([rurrent_f_out0, rurrent_x1], + 1) # 512->1024/16 + rurrent_f_out0 = self.C3_p4(rurrent_f_out0) # 1024->512/16 + + rurrent_fpn_out1 = self.reduce_conv1(rurrent_f_out0) # 512->256/16 + rurrent_f_out1 = F.interpolate( + rurrent_fpn_out1, size=rurrent_x2.shape[2:4], + mode='nearest') # 256/8 + rurrent_f_out1 = torch.cat([rurrent_f_out1, rurrent_x2], + 1) # 256->512/8 + rurrent_pan_out2 = self.C3_p3(rurrent_f_out1) # 512->256/8 + + rurrent_p_out1 = self.bu_conv2(rurrent_pan_out2) # 256->256/16 + rurrent_p_out1 = torch.cat([rurrent_p_out1, rurrent_fpn_out1], + 1) # 256->512/16 + rurrent_pan_out1 = self.C3_n3(rurrent_p_out1) # 512->512/16 + + rurrent_p_out0 = self.bu_conv1(rurrent_pan_out1) # 512->512/32 + rurrent_p_out0 = torch.cat([rurrent_p_out0, rurrent_fpn_out0], + 1) # 512->1024/32 + rurrent_pan_out0 = self.C3_n4(rurrent_p_out0) # 1024->1024/32 + + ##### + + support_out_features = self.backbone(torch.split(input, 3, dim=1)[1]) + support_features = [support_out_features[f] for f in self.in_features] + [support_x2, support_x1, support_x0] = support_features + + support_fpn_out0 = self.lateral_conv0(support_x0) # 1024->512/32 + support_f_out0 = F.interpolate( + support_fpn_out0, size=support_x1.shape[2:4], + mode='nearest') # 512/16 + support_f_out0 = torch.cat([support_f_out0, support_x1], + 1) # 512->1024/16 + support_f_out0 = self.C3_p4(support_f_out0) # 1024->512/16 + + support_fpn_out1 = self.reduce_conv1(support_f_out0) # 512->256/16 + support_f_out1 = F.interpolate( + support_fpn_out1, size=support_x2.shape[2:4], + mode='nearest') # 256/8 + support_f_out1 = torch.cat([support_f_out1, support_x2], + 1) # 256->512/8 + support_pan_out2 = self.C3_p3(support_f_out1) # 512->256/8 + + support_p_out1 = self.bu_conv2(support_pan_out2) # 256->256/16 + support_p_out1 = torch.cat([support_p_out1, support_fpn_out1], + 1) # 256->512/16 + support_pan_out1 = self.C3_n3(support_p_out1) # 512->512/16 + + support_p_out0 = self.bu_conv1(support_pan_out1) # 512->512/32 + support_p_out0 = torch.cat([support_p_out0, support_fpn_out0], + 1) # 512->1024/32 + support_pan_out0 = self.C3_n4(support_p_out0) # 1024->1024/32 + + # 0.5 channel + pan_out2 = torch.cat( + [self.jian2(rurrent_pan_out2), + self.jian2(support_pan_out2)], + dim=1) + rurrent_pan_out2 + pan_out1 = torch.cat( + [self.jian1(rurrent_pan_out1), + self.jian1(support_pan_out1)], + dim=1) + rurrent_pan_out1 + pan_out0 = torch.cat( + [self.jian0(rurrent_pan_out0), + self.jian0(support_pan_out0)], + dim=1) + rurrent_pan_out0 + + outputs = (pan_out2, pan_out1, pan_out0) + + return outputs + + def online_forward(self, input, buffer=None, node='star'): + """ + Args: + inputs: input images. + + Returns: + Tuple[Tensor]: FPN feature. + """ + + # backbone + rurrent_out_features = self.backbone(input) + rurrent_features = [rurrent_out_features[f] for f in self.in_features] + [rurrent_x2, rurrent_x1, rurrent_x0] = rurrent_features + + rurrent_fpn_out0 = self.lateral_conv0(rurrent_x0) # 1024->512/32 + rurrent_f_out0 = F.interpolate( + rurrent_fpn_out0, size=rurrent_x1.shape[2:4], + mode='nearest') # 512/16 + rurrent_f_out0 = torch.cat([rurrent_f_out0, rurrent_x1], + 1) # 512->1024/16 + rurrent_f_out0 = self.C3_p4(rurrent_f_out0) # 1024->512/16 + + rurrent_fpn_out1 = self.reduce_conv1(rurrent_f_out0) # 512->256/16 + rurrent_f_out1 = F.interpolate( + rurrent_fpn_out1, size=rurrent_x2.shape[2:4], + mode='nearest') # 256/8 + rurrent_f_out1 = torch.cat([rurrent_f_out1, rurrent_x2], + 1) # 256->512/8 + rurrent_pan_out2 = self.C3_p3(rurrent_f_out1) # 512->256/8 + + rurrent_p_out1 = self.bu_conv2(rurrent_pan_out2) # 256->256/16 + rurrent_p_out1 = torch.cat([rurrent_p_out1, rurrent_fpn_out1], + 1) # 256->512/16 + rurrent_pan_out1 = self.C3_n3(rurrent_p_out1) # 512->512/16 + + rurrent_p_out0 = self.bu_conv1(rurrent_pan_out1) # 512->512/32 + rurrent_p_out0 = torch.cat([rurrent_p_out0, rurrent_fpn_out0], + 1) # 512->1024/32 + rurrent_pan_out0 = self.C3_n4(rurrent_p_out0) # 1024->1024/32 + + ##### + if node == 'star': + pan_out2 = torch.cat( + [self.jian2(rurrent_pan_out2), + self.jian2(rurrent_pan_out2)], + dim=1) + rurrent_pan_out2 + pan_out1 = torch.cat( + [self.jian1(rurrent_pan_out1), + self.jian1(rurrent_pan_out1)], + dim=1) + rurrent_pan_out1 + pan_out0 = torch.cat( + [self.jian0(rurrent_pan_out0), + self.jian0(rurrent_pan_out0)], + dim=1) + rurrent_pan_out0 + elif node == 'buffer': + + [support_pan_out2, support_pan_out1, support_pan_out0] = buffer + + pan_out2 = torch.cat( + [self.jian2(rurrent_pan_out2), + self.jian2(support_pan_out2)], + dim=1) + rurrent_pan_out2 + pan_out1 = torch.cat( + [self.jian1(rurrent_pan_out1), + self.jian1(support_pan_out1)], + dim=1) + rurrent_pan_out1 + pan_out0 = torch.cat( + [self.jian0(rurrent_pan_out0), + self.jian0(support_pan_out0)], + dim=1) + rurrent_pan_out0 + + outputs = (pan_out2, pan_out1, pan_out0) + + buffer_ = (rurrent_pan_out2, rurrent_pan_out1, rurrent_pan_out0) + + return outputs, buffer_ + + def forward(self, input, buffer=None, mode='off_pipe'): + + if mode == 'off_pipe': + # Glops caculate mode + if input.size()[1] == 3: + input = torch.cat([input, input], dim=1) + output = self.off_forward(input) + # offline train mode + elif input.size()[1] == 6: + output = self.off_forward(input) + + return output + + elif mode == 'on_pipe': + # online star state + if buffer is None: + output, buffer_ = self.online_forward(input, node='star') + # online inference + else: + assert len(buffer) == 3 + assert input.size()[1] == 3 + output, buffer_ = self.online_forward( + input, buffer=buffer, node='buffer') + + return output, buffer_ diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/network_blocks.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/network_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..88bd55c7e38a07ce40a2b3599411dc40f2d653a3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/network_blocks.py @@ -0,0 +1,212 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX +import torch +import torch.nn as nn + + +def get_activation(name='silu', inplace=True): + if name == 'silu': + module = nn.SiLU(inplace=inplace) + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + return module + + +class BaseConv(nn.Module): + """A Conv2d -> Batchnorm -> silu/leaky relu block""" + + def __init__(self, + in_channels, + out_channels, + ksize, + stride, + groups=1, + bias=False, + act='silu'): + super(BaseConv, self).__init__() + # same padding + pad = (ksize - 1) // 2 + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=ksize, + stride=stride, + padding=pad, + groups=groups, + bias=bias, + ) + self.bn = nn.BatchNorm2d(out_channels) + self.act = get_activation(act, inplace=True) + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class DWConv(nn.Module): + """Depthwise Conv + Conv""" + + def __init__(self, in_channels, out_channels, ksize, stride=1, act='silu'): + super(DWConv, self).__init__() + self.dconv = BaseConv( + in_channels, + in_channels, + ksize=ksize, + stride=stride, + groups=in_channels, + act=act, + ) + self.pconv = BaseConv( + in_channels, out_channels, ksize=1, stride=1, groups=1, act=act) + + def forward(self, x): + x = self.dconv(x) + return self.pconv(x) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__( + self, + in_channels, + out_channels, + shortcut=True, + expansion=0.5, + depthwise=False, + act='silu', + ): + super().__init__() + hidden_channels = int(out_channels * expansion) + Conv = DWConv if depthwise else BaseConv + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act) + self.use_add = shortcut and in_channels == out_channels + + def forward(self, x): + y = self.conv2(self.conv1(x)) + if self.use_add: + y = y + x + return y + + +class ResLayer(nn.Module): + 'Residual layer with `in_channels` inputs.' + + def __init__(self, in_channels: int): + super().__init__() + mid_channels = in_channels // 2 + self.layer1 = BaseConv( + in_channels, mid_channels, ksize=1, stride=1, act='lrelu') + self.layer2 = BaseConv( + mid_channels, in_channels, ksize=3, stride=1, act='lrelu') + + def forward(self, x): + out = self.layer2(self.layer1(x)) + return x + out + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP""" + + def __init__(self, + in_channels, + out_channels, + kernel_sizes=(5, 9, 13), + activation='silu'): + super().__init__() + hidden_channels = in_channels // 2 + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=activation) + self.m = nn.ModuleList([ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ]) + conv2_channels = hidden_channels * (len(kernel_sizes) + 1) + self.conv2 = BaseConv( + conv2_channels, out_channels, 1, stride=1, act=activation) + + def forward(self, x): + x = self.conv1(x) + x = torch.cat([x] + [m(x) for m in self.m], dim=1) + x = self.conv2(x) + return x + + +class CSPLayer(nn.Module): + """C3 in yolov5, CSP Bottleneck with 3 convolutions""" + + def __init__( + self, + in_channels, + out_channels, + n=1, + shortcut=True, + expansion=0.5, + depthwise=False, + act='silu', + ): + """ + Args: + in_channels (int): input channels. + out_channels (int): output channels. + n (int): number of Bottlenecks. Default value: 1. + """ + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + hidden_channels = int(out_channels * expansion) # hidden channels + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv3 = BaseConv( + 2 * hidden_channels, out_channels, 1, stride=1, act=act) + module_list = [ + Bottleneck( + hidden_channels, + hidden_channels, + shortcut, + 1.0, + depthwise, + act=act) for _ in range(n) + ] + self.m = nn.Sequential(*module_list) + + def forward(self, x): + x_1 = self.conv1(x) + x_2 = self.conv2(x) + x_1 = self.m(x_1) + x = torch.cat((x_1, x_2), dim=1) + return self.conv3(x) + + +class Focus(nn.Module): + """Focus width and height information into channel space.""" + + def __init__(self, + in_channels, + out_channels, + ksize=1, + stride=1, + act='silu'): + super().__init__() + self.conv = BaseConv( + in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) + patch_top_left = x[..., ::2, ::2] + patch_top_right = x[..., ::2, 1::2] + patch_bot_left = x[..., 1::2, ::2] + patch_bot_right = x[..., 1::2, 1::2] + x = torch.cat( + ( + patch_top_left, + patch_bot_left, + patch_top_right, + patch_bot_right, + ), + dim=1, + ) + return self.conv(x) diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/streamyolo.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/streamyolo.py new file mode 100644 index 0000000000000000000000000000000000000000..b3ec350418eb98379502c0ed22e8244ffe18514f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/streamyolo.py @@ -0,0 +1,41 @@ +# The implementation is based on StreamYOLO, available at https://github.com/yancie-yjr/StreamYOLO +import torch.nn as nn + +from .dfp_pafpn import DFPPAFPN +from .tal_head import TALHead + + +class StreamYOLO(nn.Module): + """ + YOLOX model module. The module list is defined by create_yolov3_modules function. + The network returns loss values from three YOLO layers during training + and detection results during test. + """ + + def __init__(self, backbone=None, head=None): + super().__init__() + if backbone is None: + backbone = DFPPAFPN() + if head is None: + head = TALHead(20) + + self.backbone = backbone + self.head = head + + def forward(self, x, targets=None, buffer=None, mode='off_pipe'): + # fpn output content features of [dark3, dark4, dark5] + assert mode in ['off_pipe', 'on_pipe'] + + if mode == 'off_pipe': + fpn_outs = self.backbone(x, buffer=buffer, mode='off_pipe') + if self.training: + pass + else: + outputs = self.head(fpn_outs, imgs=x) + + return outputs + elif mode == 'on_pipe': + fpn_outs, buffer_ = self.backbone(x, buffer=buffer, mode='on_pipe') + outputs = self.head(fpn_outs) + + return outputs, buffer_ diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/models/tal_head.py b/runtime/Lib/site_packages/models/cv/stream_yolo/models/tal_head.py new file mode 100644 index 0000000000000000000000000000000000000000..7a82f8c62d97917d5d452871c57097aa44597d1b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/models/tal_head.py @@ -0,0 +1,170 @@ +# The implementation is based on StreamYOLO, available at https://github.com/yancie-yjr/StreamYOLO +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .network_blocks import BaseConv, DWConv + + +class TALHead(nn.Module): + + def __init__( + self, + num_classes, + width=1.0, + strides=[8, 16, 32], + in_channels=[256, 512, 1024], + act='silu', + depthwise=False, + gamma=1.5, + ignore_thr=0.2, + ignore_value=0.2, + ): + """ + Args: + act (str): activation type of conv. Defalut value: "silu". + depthwise (bool): wheather apply depthwise conv in conv branch. Defalut value: False. + """ + super().__init__() + + self.gamma = gamma + self.ignore_thr = ignore_thr + self.ignore_value = ignore_value + + self.n_anchors = 1 + self.num_classes = num_classes + self.decode_in_inference = True # for deploy, set to False + + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + self.cls_preds = nn.ModuleList() + self.reg_preds = nn.ModuleList() + self.obj_preds = nn.ModuleList() + self.stems = nn.ModuleList() + Conv = DWConv if depthwise else BaseConv + + for i in range(len(in_channels)): + self.stems.append( + BaseConv( + in_channels=int(in_channels[i] * width), + out_channels=int(256 * width), + ksize=1, + stride=1, + act=act, + )) + self.cls_convs.append( + nn.Sequential(*[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ])) + self.reg_convs.append( + nn.Sequential(*[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ])) + self.cls_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * self.num_classes, + kernel_size=1, + stride=1, + padding=0, + )) + self.reg_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=4, + kernel_size=1, + stride=1, + padding=0, + )) + self.obj_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * 1, + kernel_size=1, + stride=1, + padding=0, + )) + + self.strides = strides + self.grids = [torch.zeros(1)] * len(in_channels) + self.expanded_strides = [None] * len(in_channels) + + def forward(self, xin, labels=None, imgs=None): + outputs = [] + for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate( + zip(self.cls_convs, self.reg_convs, self.strides, xin)): + x = self.stems[k](x) + cls_x = x + reg_x = x + + cls_feat = cls_conv(cls_x) + cls_output = self.cls_preds[k](cls_feat) + + reg_feat = reg_conv(reg_x) + reg_output = self.reg_preds[k](reg_feat) + obj_output = self.obj_preds[k](reg_feat) + + if self.training: + pass + + else: + output = torch.cat( + [reg_output, + obj_output.sigmoid(), + cls_output.sigmoid()], 1) + + outputs.append(output) + + if self.training: + pass + else: + self.hw = [x.shape[-2:] for x in outputs] + outputs = torch.cat([x.flatten(start_dim=2) for x in outputs], + dim=2).permute(0, 2, 1) + if self.decode_in_inference: + return self.decode_outputs(outputs, dtype=xin[0].type()) + else: + return outputs + + def decode_outputs(self, outputs, dtype): + grids = [] + strides = [] + for (hsize, wsize), stride in zip(self.hw, self.strides): + yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)]) + grid = torch.stack((xv, yv), 2).view(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + strides.append(torch.full((*shape, 1), stride)) + + grids = torch.cat(grids, dim=1).type(dtype) + strides = torch.cat(strides, dim=1).type(dtype) + + outputs[..., :2] = (outputs[..., :2] + grids) * strides + outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides + return outputs diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/realtime_video_detector.py b/runtime/Lib/site_packages/models/cv/stream_yolo/realtime_video_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d62d7287bfdaf5ac9b783517f15964bd6eab0e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/realtime_video_detector.py @@ -0,0 +1,125 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import argparse +import logging as logger +import os +import os.path as osp +import time + +import cv2 +import json +import numpy as np +import torch +from tqdm import tqdm + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.preprocessors import LoadImage +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .data.data_augment import ValTransform +from .exp import get_exp_by_name +from .utils import postprocess, timestamp_format + + +@MODELS.register_module( + group_key=Tasks.video_object_detection, + module_name=Models.realtime_video_object_detection) +class RealtimeVideoDetector(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + + # model type + self.exp = get_exp_by_name(self.config.model_type) + + # build model + self.model = self.exp.get_model() + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_BIN_FILE) + ckpt = torch.load(model_path, map_location='cpu') + + # load the model state dict + self.model.load_state_dict(ckpt['model']) + self.model.eval() + + # params setting + self.exp.num_classes = self.config.num_classes + self.confthre = self.config.conf_thr + self.num_classes = self.exp.num_classes + self.nmsthre = self.exp.nmsthre + self.test_size = self.exp.test_size + self.preproc = ValTransform(legacy=False) + self.current_buffer = None + self.label_mapping = self.config['labels'] + + def inference(self, img): + with torch.no_grad(): + outputs, self.current_buffer = self.model( + img, buffer=self.current_buffer, mode='on_pipe') + return outputs + + def forward(self, inputs): + return self.inference_video(inputs) + + def preprocess(self, img): + img = LoadImage.convert_to_ndarray(img) + height, width = img.shape[:2] + self.ratio = min(self.test_size[0] / img.shape[0], + self.test_size[1] / img.shape[1]) + + img, _ = self.preproc(img, None, self.test_size) + img = torch.from_numpy(img).unsqueeze(0) + img = img.float() + + # Video decoding and preprocessing automatically are not supported by Pipeline/Model + # Sending preprocessed video frame tensor to GPU buffer self-adaptively + if next(self.model.parameters()).is_cuda: + img = img.to(next(self.model.parameters()).device) + return img + + def postprocess(self, input): + outputs = postprocess( + input, + self.num_classes, + self.confthre, + self.nmsthre, + class_agnostic=True) + + if len(outputs) == 1 and (outputs[0] is not None): + bboxes = outputs[0][:, 0:4].cpu().numpy() / self.ratio + scores = outputs[0][:, 5].cpu().numpy() + labels = outputs[0][:, 6].cpu().int().numpy() + pred_label_names = [] + for lab in labels: + pred_label_names.append(self.label_mapping[lab]) + else: + bboxes = np.asarray([]) + scores = np.asarray([]) + pred_label_names = np.asarray([]) + + return bboxes, scores, pred_label_names + + def inference_video(self, v_path): + outputs = [] + desc = 'Detecting video: {}'.format(v_path) + for frame_idx, (frame, result) in enumerate( + tqdm(self.inference_video_iter(v_path), desc=desc)): + result = result + (timestamp_format(seconds=frame_idx + / self.fps), ) + outputs.append(result) + + return outputs + + def inference_video_iter(self, v_path): + capture = cv2.VideoCapture(v_path) + self.fps = capture.get(cv2.CAP_PROP_FPS) + while capture.isOpened(): + ret, frame = capture.read() + if not ret: + break + output = self.preprocess(frame) + output = self.inference(output) + output = self.postprocess(output) + yield frame, output diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/utils/__init__.py b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..46ad7d850a22cc2782378dc2c8229ff0b171d0e7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/__init__.py @@ -0,0 +1,9 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +from .boxes import * # noqa +from .format import * # noqa + +__all__ = [ + 'bboxes_iou', 'meshgrid', 'postprocess', 'xyxy2cxcywh', 'xyxy2xywh', + 'timestamp_format' +] diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/utils/boxes.py b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..b29a3a04971f9bc405dae1c6e395e58cbaefeec6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/boxes.py @@ -0,0 +1,107 @@ +# The implementation is based on YOLOX, available at https://github.com/Megvii-BaseDetection/YOLOX + +import torch +import torchvision + +_TORCH_VER = [int(x) for x in torch.__version__.split('.')[:2]] + + +def meshgrid(*tensors): + if _TORCH_VER >= [1, 10]: + return torch.meshgrid(*tensors, indexing='ij') + else: + return torch.meshgrid(*tensors) + + +def xyxy2xywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + return bboxes + + +def xyxy2cxcywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5 + return bboxes + + +def postprocess(prediction, + num_classes, + conf_thre=0.7, + nms_thre=0.45, + class_agnostic=False): + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max( + image_pred[:, 5:5 + num_classes], 1, keepdim=True) + + conf_mask = image_pred[:, 4] * class_conf.squeeze() + conf_mask = (conf_mask >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat( + (image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + if class_agnostic: + nms_out_index = torchvision.ops.nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + nms_thre, + ) + else: + nms_out_index = torchvision.ops.batched_nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + detections[:, 6], + nms_thre, + ) + + detections = detections[nms_out_index] + if output[i] is None: + output[i] = detections + else: + output[i] = torch.cat((output[i], detections)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) diff --git a/runtime/Lib/site_packages/models/cv/stream_yolo/utils/format.py b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/format.py new file mode 100644 index 0000000000000000000000000000000000000000..c3d7a4c6ee5a855e0f3412831718e10348bbd940 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/stream_yolo/utils/format.py @@ -0,0 +1,9 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + + +def timestamp_format(seconds): + m, s = divmod(seconds, 60) + h, m = divmod(m, 60) + time = '%02d:%02d:%06.3f' % (h, m, s) + return time diff --git a/runtime/Lib/site_packages/models/cv/super_resolution/__init__.py b/runtime/Lib/site_packages/models/cv/super_resolution/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7187c57ae7e1d80fa2928079a2d9d95a19e6627c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/super_resolution/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .rrdbnet_arch import RRDBNet + from .ecbsr_model import ECBSRModel + +else: + _import_structure = { + 'rrdbnet_arch': ['RRDBNet'], + 'ecbsr_model': ['ECBSRModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/super_resolution/arch_util.py b/runtime/Lib/site_packages/models/cv/super_resolution/arch_util.py new file mode 100644 index 0000000000000000000000000000000000000000..99711a1141c9da23629d1e6384e88e78c3f6f5ea --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/super_resolution/arch_util.py @@ -0,0 +1,228 @@ +# The implementation is adopted from BasicSR, made public available under the Apache 2.0 License +# at https://github.com/XPixelGroup/BasicSR/blob/master/basicsr/archs/arch_util.py +import collections.abc +import math +import warnings +from itertools import repeat + +import torch +import torchvision +from torch import nn as nn +from torch.nn import functional as F +from torch.nn import init as init +from torch.nn.modules.batchnorm import _BatchNorm + + +@torch.no_grad() +def default_init_weights(module_list, scale=1, bias_fill=0, **kwargs): + """Initialize network weights. + + Args: + module_list (list[nn.Module] | nn.Module): Modules to be initialized. + scale (float): Scale initialized weights, especially for residual + blocks. Default: 1. + bias_fill (float): The value to fill bias. Default: 0 + kwargs (dict): Other arguments for initialization function. + """ + if not isinstance(module_list, list): + module_list = [module_list] + for module in module_list: + for m in module.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, **kwargs) + m.weight.data *= scale + if m.bias is not None: + m.bias.data.fill_(bias_fill) + elif isinstance(m, nn.Linear): + init.kaiming_normal_(m.weight, **kwargs) + m.weight.data *= scale + if m.bias is not None: + m.bias.data.fill_(bias_fill) + elif isinstance(m, _BatchNorm): + init.constant_(m.weight, 1) + if m.bias is not None: + m.bias.data.fill_(bias_fill) + + +def make_layer(basic_block, num_basic_block, **kwarg): + """Make layers by stacking the same blocks. + + Args: + basic_block (nn.module): nn.module class for basic block. + num_basic_block (int): number of blocks. + + Returns: + nn.Sequential: Stacked blocks in nn.Sequential. + """ + layers = [] + for _ in range(num_basic_block): + layers.append(basic_block(**kwarg)) + return nn.Sequential(*layers) + + +class ResidualBlockNoBN(nn.Module): + """Residual block without BN. + + It has a style of: + ---Conv-ReLU-Conv-+- + |________________| + + Args: + num_feat (int): Channel number of intermediate features. + Default: 64. + res_scale (float): Residual scale. Default: 1. + pytorch_init (bool): If set to True, use pytorch default init, + otherwise, use default_init_weights. Default: False. + """ + + def __init__(self, num_feat=64, res_scale=1, pytorch_init=False): + super(ResidualBlockNoBN, self).__init__() + self.res_scale = res_scale + self.conv1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=True) + self.conv2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=True) + self.relu = nn.ReLU(inplace=True) + + if not pytorch_init: + default_init_weights([self.conv1, self.conv2], 0.1) + + def forward(self, x): + identity = x + out = self.conv2(self.relu(self.conv1(x))) + return identity + out * self.res_scale + + +class Upsample(nn.Sequential): + """Upsample module. + + Args: + scale (int): Scale factor. Supported scales: 2^n and 3. + num_feat (int): Channel number of intermediate features. + """ + + def __init__(self, scale, num_feat): + m = [] + if (scale & (scale - 1)) == 0: # scale = 2^n + for _ in range(int(math.log(scale, 2))): + m.append(nn.Conv2d(num_feat, 4 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(2)) + elif scale == 3: + m.append(nn.Conv2d(num_feat, 9 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(3)) + else: + raise ValueError( + f'scale {scale} is not supported. Supported scales: 2^n and 3.' + ) + super(Upsample, self).__init__(*m) + + +def flow_warp(x, + flow, + interp_mode='bilinear', + padding_mode='zeros', + align_corners=True): + """Warp an image or feature map with optical flow. + + Args: + x (Tensor): Tensor with size (n, c, h, w). + flow (Tensor): Tensor with size (n, h, w, 2), normal value. + interp_mode (str): 'nearest' or 'bilinear'. Default: 'bilinear'. + padding_mode (str): 'zeros' or 'border' or 'reflection'. + Default: 'zeros'. + align_corners (bool): Before pytorch 1.3, the default value is + align_corners=True. After pytorch 1.3, the default value is + align_corners=False. Here, we use the True as default. + + Returns: + Tensor: Warped image or feature map. + """ + assert x.size()[-2:] == flow.size()[1:3] + _, _, h, w = x.size() + # create mesh grid + grid_y, grid_x = torch.meshgrid( + torch.arange(0, h).type_as(x), + torch.arange(0, w).type_as(x)) + grid = torch.stack((grid_x, grid_y), 2).float() # W(x), H(y), 2 + grid.requires_grad = False + + vgrid = grid + flow + # scale grid to [-1,1] + vgrid_x = 2.0 * vgrid[:, :, :, 0] / max(w - 1, 1) - 1.0 + vgrid_y = 2.0 * vgrid[:, :, :, 1] / max(h - 1, 1) - 1.0 + vgrid_scaled = torch.stack((vgrid_x, vgrid_y), dim=3) + output = F.grid_sample( + x, + vgrid_scaled, + mode=interp_mode, + padding_mode=padding_mode, + align_corners=align_corners) + + # TODO, what if align_corners=False + return output + + +def resize_flow(flow, + size_type, + sizes, + interp_mode='bilinear', + align_corners=False): + """Resize a flow according to ratio or shape. + + Args: + flow (Tensor): Precomputed flow. shape [N, 2, H, W]. + size_type (str): 'ratio' or 'shape'. + sizes (list[int | float]): the ratio for resizing or the final output + shape. + 1) The order of ratio should be [ratio_h, ratio_w]. For + downsampling, the ratio should be smaller than 1.0 (i.e., ratio + < 1.0). For upsampling, the ratio should be larger than 1.0 (i.e., + ratio > 1.0). + 2) The order of output_size should be [out_h, out_w]. + interp_mode (str): The mode of interpolation for resizing. + Default: 'bilinear'. + align_corners (bool): Whether align corners. Default: False. + + Returns: + Tensor: Resized flow. + """ + _, _, flow_h, flow_w = flow.size() + if size_type == 'ratio': + output_h, output_w = int(flow_h * sizes[0]), int(flow_w * sizes[1]) + elif size_type == 'shape': + output_h, output_w = sizes[0], sizes[1] + else: + raise ValueError( + f'Size type should be ratio or shape, but got type {size_type}.') + + input_flow = flow.clone() + ratio_h = output_h / flow_h + ratio_w = output_w / flow_w + input_flow[:, 0, :, :] *= ratio_w + input_flow[:, 1, :, :] *= ratio_h + resized_flow = F.interpolate( + input=input_flow, + size=(output_h, output_w), + mode=interp_mode, + align_corners=align_corners) + return resized_flow + + +# TODO: may write a cpp file +def pixel_unshuffle(x, scale): + """ Pixel unshuffle. + + Args: + x (Tensor): Input feature with shape (b, c, hh, hw). + scale (int): Downsample ratio. + + Returns: + Tensor: the pixel unshuffled feature. + """ + b, c, hh, hw = x.size() + out_channel = c * (scale**2) + + assert hh % scale == 0 and hw % scale == 0 + + h = hh // scale + w = hw // scale + x_view = x.view(b, c, h, scale, w, scale) + return x_view.permute(0, 1, 3, 5, 2, 4).reshape(b, out_channel, h, w) diff --git a/runtime/Lib/site_packages/models/cv/super_resolution/ecb.py b/runtime/Lib/site_packages/models/cv/super_resolution/ecb.py new file mode 100644 index 0000000000000000000000000000000000000000..4ddf734c41deae8ac4526f6efc1b9611e8bd6d15 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/super_resolution/ecb.py @@ -0,0 +1,272 @@ +# The implementation is adopted from ECBSR, +# made publicly available under the Apache 2.0 License at +# https://github.com/xindongzhang/ECBSR/blob/main/models/ecb.py + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class SeqConv3x3(nn.Module): + + def __init__(self, seq_type, inp_planes, out_planes, depth_multiplier): + super(SeqConv3x3, self).__init__() + + self.type = seq_type + self.inp_planes = inp_planes + self.out_planes = out_planes + + if self.type == 'conv1x1-conv3x3': + self.mid_planes = int(out_planes * depth_multiplier) + conv0 = torch.nn.Conv2d( + self.inp_planes, self.mid_planes, kernel_size=1, padding=0) + self.k0 = conv0.weight + self.b0 = conv0.bias + + conv1 = torch.nn.Conv2d( + self.mid_planes, self.out_planes, kernel_size=3) + self.k1 = conv1.weight + self.b1 = conv1.bias + + elif self.type == 'conv1x1-sobelx': + conv0 = torch.nn.Conv2d( + self.inp_planes, self.out_planes, kernel_size=1, padding=0) + self.k0 = conv0.weight + self.b0 = conv0.bias + + # init scale & bias + scale = torch.randn(size=(self.out_planes, 1, 1, 1)) * 1e-3 + self.scale = nn.Parameter(scale) + # bias = 0.0 + # bias = [bias for c in range(self.out_planes)] + # bias = torch.FloatTensor(bias) + bias = torch.randn(self.out_planes) * 1e-3 + bias = torch.reshape(bias, (self.out_planes, )) + self.bias = nn.Parameter(bias) + # init mask + self.mask = torch.zeros((self.out_planes, 1, 3, 3), + dtype=torch.float32) + for i in range(self.out_planes): + self.mask[i, 0, 0, 0] = 1.0 + self.mask[i, 0, 1, 0] = 2.0 + self.mask[i, 0, 2, 0] = 1.0 + self.mask[i, 0, 0, 2] = -1.0 + self.mask[i, 0, 1, 2] = -2.0 + self.mask[i, 0, 2, 2] = -1.0 + self.mask = nn.Parameter(data=self.mask, requires_grad=False) + + elif self.type == 'conv1x1-sobely': + conv0 = torch.nn.Conv2d( + self.inp_planes, self.out_planes, kernel_size=1, padding=0) + self.k0 = conv0.weight + self.b0 = conv0.bias + + # init scale & bias + scale = torch.randn(size=(self.out_planes, 1, 1, 1)) * 1e-3 + self.scale = nn.Parameter(torch.FloatTensor(scale)) + # bias = 0.0 + # bias = [bias for c in range(self.out_planes)] + # bias = torch.FloatTensor(bias) + bias = torch.randn(self.out_planes) * 1e-3 + bias = torch.reshape(bias, (self.out_planes, )) + self.bias = nn.Parameter(torch.FloatTensor(bias)) + # init mask + self.mask = torch.zeros((self.out_planes, 1, 3, 3), + dtype=torch.float32) + for i in range(self.out_planes): + self.mask[i, 0, 0, 0] = 1.0 + self.mask[i, 0, 0, 1] = 2.0 + self.mask[i, 0, 0, 2] = 1.0 + self.mask[i, 0, 2, 0] = -1.0 + self.mask[i, 0, 2, 1] = -2.0 + self.mask[i, 0, 2, 2] = -1.0 + self.mask = nn.Parameter(data=self.mask, requires_grad=False) + + elif self.type == 'conv1x1-laplacian': + conv0 = torch.nn.Conv2d( + self.inp_planes, self.out_planes, kernel_size=1, padding=0) + self.k0 = conv0.weight + self.b0 = conv0.bias + + # init scale & bias + scale = torch.randn(size=(self.out_planes, 1, 1, 1)) * 1e-3 + self.scale = nn.Parameter(torch.FloatTensor(scale)) + # bias = 0.0 + # bias = [bias for c in range(self.out_planes)] + # bias = torch.FloatTensor(bias) + bias = torch.randn(self.out_planes) * 1e-3 + bias = torch.reshape(bias, (self.out_planes, )) + self.bias = nn.Parameter(torch.FloatTensor(bias)) + # init mask + self.mask = torch.zeros((self.out_planes, 1, 3, 3), + dtype=torch.float32) + for i in range(self.out_planes): + self.mask[i, 0, 0, 1] = 1.0 + self.mask[i, 0, 1, 0] = 1.0 + self.mask[i, 0, 1, 2] = 1.0 + self.mask[i, 0, 2, 1] = 1.0 + self.mask[i, 0, 1, 1] = -4.0 + self.mask = nn.Parameter(data=self.mask, requires_grad=False) + else: + raise ValueError('the type of seqconv is not supported!') + + def forward(self, x): + if self.type == 'conv1x1-conv3x3': + # conv-1x1 + y0 = F.conv2d(input=x, weight=self.k0, bias=self.b0, stride=1) + # explicitly padding with bias + y0 = F.pad(y0, (1, 1, 1, 1), 'constant', 0) + b0_pad = self.b0.view(1, -1, 1, 1) + y0[:, :, 0:1, :] = b0_pad + y0[:, :, -1:, :] = b0_pad + y0[:, :, :, 0:1] = b0_pad + y0[:, :, :, -1:] = b0_pad + # conv-3x3 + y1 = F.conv2d(input=y0, weight=self.k1, bias=self.b1, stride=1) + else: + y0 = F.conv2d(input=x, weight=self.k0, bias=self.b0, stride=1) + # explicitly padding with bias + y0 = F.pad(y0, (1, 1, 1, 1), 'constant', 0) + b0_pad = self.b0.view(1, -1, 1, 1) + y0[:, :, 0:1, :] = b0_pad + y0[:, :, -1:, :] = b0_pad + y0[:, :, :, 0:1] = b0_pad + y0[:, :, :, -1:] = b0_pad + # conv-3x3 + y1 = F.conv2d( + input=y0, + weight=self.scale * self.mask, + bias=self.bias, + stride=1, + groups=self.out_planes) + return y1 + + def rep_params(self): + device = self.k0.get_device() + if device < 0: + device = None + + if self.type == 'conv1x1-conv3x3': + # re-param conv kernel + RK = F.conv2d(input=self.k1, weight=self.k0.permute(1, 0, 2, 3)) + # re-param conv bias + RB = torch.ones( + 1, self.mid_planes, 3, 3, device=device) * self.b0.view( + 1, -1, 1, 1) + RB = F.conv2d(input=RB, weight=self.k1).view(-1, ) + self.b1 + else: + tmp = self.scale * self.mask + k1 = torch.zeros((self.out_planes, self.out_planes, 3, 3), + device=device) + for i in range(self.out_planes): + k1[i, i, :, :] = tmp[i, 0, :, :] + b1 = self.bias + # re-param conv kernel + RK = F.conv2d(input=k1, weight=self.k0.permute(1, 0, 2, 3)) + # re-param conv bias + RB = torch.ones( + 1, self.out_planes, 3, 3, device=device) * self.b0.view( + 1, -1, 1, 1) + RB = F.conv2d(input=RB, weight=k1).view(-1, ) + b1 + return RK, RB + + +class ECB(nn.Module): + + def __init__(self, + inp_planes, + out_planes, + depth_multiplier, + act_type='prelu', + with_idt=False): + super(ECB, self).__init__() + + self.depth_multiplier = depth_multiplier + self.inp_planes = inp_planes + self.out_planes = out_planes + self.act_type = act_type + + if with_idt and (self.inp_planes == self.out_planes): + self.with_idt = True + else: + self.with_idt = False + + self.conv3x3 = torch.nn.Conv2d( + self.inp_planes, self.out_planes, kernel_size=3, padding=1) + self.conv1x1_3x3 = SeqConv3x3('conv1x1-conv3x3', self.inp_planes, + self.out_planes, self.depth_multiplier) + self.conv1x1_sbx = SeqConv3x3('conv1x1-sobelx', self.inp_planes, + self.out_planes, -1) + self.conv1x1_sby = SeqConv3x3('conv1x1-sobely', self.inp_planes, + self.out_planes, -1) + self.conv1x1_lpl = SeqConv3x3('conv1x1-laplacian', self.inp_planes, + self.out_planes, -1) + + if self.act_type == 'prelu': + self.act = nn.PReLU(num_parameters=self.out_planes) + elif self.act_type == 'relu': + self.act = nn.ReLU(inplace=True) + elif self.act_type == 'rrelu': + self.act = nn.RReLU(lower=-0.05, upper=0.05) + elif self.act_type == 'softplus': + self.act = nn.Softplus() + elif self.act_type == 'linear': + pass + else: + raise ValueError('The type of activation if not support!') + + def forward(self, x): + if self.training: + y = self.conv3x3(x) + \ + self.conv1x1_3x3(x) + \ + self.conv1x1_sbx(x) + \ + self.conv1x1_sby(x) + \ + self.conv1x1_lpl(x) + if self.with_idt: + y += x + else: + RK, RB = self.rep_params() + y = F.conv2d(input=x, weight=RK, bias=RB, stride=1, padding=1) + if self.act_type != 'linear': + y = self.act(y) + return y + + def rep_params(self): + K0, B0 = self.conv3x3.weight, self.conv3x3.bias + K1, B1 = self.conv1x1_3x3.rep_params() + K2, B2 = self.conv1x1_sbx.rep_params() + K3, B3 = self.conv1x1_sby.rep_params() + K4, B4 = self.conv1x1_lpl.rep_params() + RK, RB = (K0 + K1 + K2 + K3 + K4), (B0 + B1 + B2 + B3 + B4) + + if self.with_idt: + device = RK.get_device() + if device < 0: + device = None + K_idt = torch.zeros( + self.out_planes, self.out_planes, 3, 3, device=device) + for i in range(self.out_planes): + K_idt[i, i, 1, 1] = 1.0 + B_idt = 0.0 + RK, RB = RK + K_idt, RB + B_idt + return RK, RB + + +if __name__ == '__main__': + + # # test seq-conv + x = torch.randn(1, 3, 5, 5).cuda() + conv = SeqConv3x3('conv1x1-conv3x3', 3, 3, 2).cuda() + y0 = conv(x) + RK, RB = conv.rep_params() + y1 = F.conv2d(input=x, weight=RK, bias=RB, stride=1, padding=1) + print(y0 - y1) + + # test ecb + x = torch.randn(1, 3, 5, 5).cuda() * 200 + ecb = ECB(3, 3, 2, act_type='linear', with_idt=True).cuda() + y0 = ecb(x) + + RK, RB = ecb.rep_params() + y1 = F.conv2d(input=x, weight=RK, bias=RB, stride=1, padding=1) + print(y0 - y1) diff --git a/runtime/Lib/site_packages/models/cv/super_resolution/ecbsr_model.py b/runtime/Lib/site_packages/models/cv/super_resolution/ecbsr_model.py new file mode 100644 index 0000000000000000000000000000000000000000..6f54d2e44fc6fef9350f4237bc3a6384249e4b59 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/super_resolution/ecbsr_model.py @@ -0,0 +1,102 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, Union + +import torch +import torch.cuda +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .ecb import ECB + +logger = get_logger() +__all__ = ['ECBSRModel'] + + +@MODELS.register_module(Tasks.image_super_resolution, module_name=Models.ecbsr) +class ECBSRModel(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the image denoise model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + + # network architecture + self.module_nums = self.config.model.model_args.module_nums + self.channel_nums = self.config.model.model_args.channel_nums + self.scale = self.config.model.model_args.scale + self.colors = self.config.model.model_args.colors + self.with_idt = self.config.model.model_args.with_idt + self.act_type = self.config.model.model_args.act_type + + backbone = [] + backbone += [ + ECB(self.colors, + self.channel_nums, + depth_multiplier=2.0, + act_type=self.act_type, + with_idt=self.with_idt) + ] + for i in range(self.module_nums): + backbone += [ + ECB(self.channel_nums, + self.channel_nums, + depth_multiplier=2.0, + act_type=self.act_type, + with_idt=self.with_idt) + ] + backbone += [ + ECB(self.channel_nums, + self.colors * self.scale * self.scale, + depth_multiplier=2.0, + act_type='linear', + with_idt=self.with_idt) + ] + + self.backbone = nn.Sequential(*backbone) + self.upsampler = nn.PixelShuffle(self.scale) + + self.interp = nn.Upsample(scale_factor=self.scale, mode='nearest') + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + output = self.backbone(input) + output = self.upsampler(output) + self.interp(input) + return {'outputs': output} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + return self._inference_forward(**inputs) + + @classmethod + def _instantiate(cls, **kwargs): + model_file = kwargs.get('am_model_name', ModelFile.TORCH_MODEL_FILE) + model_dir = kwargs['model_dir'] + ckpt_path = os.path.join(model_dir, model_file) + logger.info(f'loading model from {ckpt_path}') + model_dir = kwargs.pop('model_dir') + model = cls(model_dir=model_dir, **kwargs) + ckpt_path = os.path.join(model_dir, model_file) + model.load_state_dict(torch.load(ckpt_path, map_location='cpu')) + return model diff --git a/runtime/Lib/site_packages/models/cv/super_resolution/rrdbnet_arch.py b/runtime/Lib/site_packages/models/cv/super_resolution/rrdbnet_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..8c84f796afd72983a8c17fd6745ca3039477013d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/super_resolution/rrdbnet_arch.py @@ -0,0 +1,131 @@ +# The implementation is adopted from BasicSR, made public available under the Apache 2.0 License +# at https://github.com/XPixelGroup/BasicSR/blob/master/basicsr/archs/rrdbnet_arch.py +import torch +from torch import nn as nn +from torch.nn import functional as F + +from .arch_util import default_init_weights, make_layer, pixel_unshuffle + + +class ResidualDenseBlock(nn.Module): + """Residual Dense Block. + + Used in RRDB block in ESRGAN. + + Args: + num_feat (int): Channel number of intermediate features. + num_grow_ch (int): Channels for each growth. + """ + + def __init__(self, num_feat=64, num_grow_ch=32): + super(ResidualDenseBlock, self).__init__() + self.conv1 = nn.Conv2d(num_feat, num_grow_ch, 3, 1, 1) + self.conv2 = nn.Conv2d(num_feat + num_grow_ch, num_grow_ch, 3, 1, 1) + self.conv3 = nn.Conv2d(num_feat + 2 * num_grow_ch, num_grow_ch, 3, 1, + 1) + self.conv4 = nn.Conv2d(num_feat + 3 * num_grow_ch, num_grow_ch, 3, 1, + 1) + self.conv5 = nn.Conv2d(num_feat + 4 * num_grow_ch, num_feat, 3, 1, 1) + + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + # initialization + default_init_weights( + [self.conv1, self.conv2, self.conv3, self.conv4, self.conv5], 0.1) + + def forward(self, x): + x1 = self.lrelu(self.conv1(x)) + x2 = self.lrelu(self.conv2(torch.cat((x, x1), 1))) + x3 = self.lrelu(self.conv3(torch.cat((x, x1, x2), 1))) + x4 = self.lrelu(self.conv4(torch.cat((x, x1, x2, x3), 1))) + x5 = self.conv5(torch.cat((x, x1, x2, x3, x4), 1)) + # Emperically, we use 0.2 to scale the residual for better performance + return x5 * 0.2 + x + + +class RRDB(nn.Module): + """Residual in Residual Dense Block. + + Used in RRDB-Net in ESRGAN. + + Args: + num_feat (int): Channel number of intermediate features. + num_grow_ch (int): Channels for each growth. + """ + + def __init__(self, num_feat, num_grow_ch=32): + super(RRDB, self).__init__() + self.rdb1 = ResidualDenseBlock(num_feat, num_grow_ch) + self.rdb2 = ResidualDenseBlock(num_feat, num_grow_ch) + self.rdb3 = ResidualDenseBlock(num_feat, num_grow_ch) + + def forward(self, x): + out = self.rdb1(x) + out = self.rdb2(out) + out = self.rdb3(out) + # Emperically, we use 0.2 to scale the residual for better performance + return out * 0.2 + x + + +class RRDBNet(nn.Module): + """Networks consisting of Residual in Residual Dense Block, which is used + in ESRGAN. + + ESRGAN: Enhanced Super-Resolution Generative Adversarial Networks. + + We extend ESRGAN for scale x2 and scale x1. + Note: This is one option for scale 1, scale 2 in RRDBNet. + We first employ the pixel-unshuffle (an inverse operation of pixelshuffle to reduce the spatial size + and enlarge the channel size before feeding inputs into the main ESRGAN architecture. + + Args: + num_in_ch (int): Channel number of inputs. + num_out_ch (int): Channel number of outputs. + num_feat (int): Channel number of intermediate features. + Default: 64 + num_block (int): Block number in the trunk network. Defaults: 23 + num_grow_ch (int): Channels for each growth. Default: 32. + """ + + def __init__(self, + num_in_ch, + num_out_ch, + scale=4, + num_feat=64, + num_block=23, + num_grow_ch=32): + super(RRDBNet, self).__init__() + self.scale = scale + if scale == 2: + num_in_ch = num_in_ch * 4 + elif scale == 1: + num_in_ch = num_in_ch * 16 + self.conv_first = nn.Conv2d(num_in_ch, num_feat, 3, 1, 1) + self.body = make_layer( + RRDB, num_block, num_feat=num_feat, num_grow_ch=num_grow_ch) + self.conv_body = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + # upsample + self.conv_up1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_up2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_hr = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1) + + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def forward(self, x): + if self.scale == 2: + feat = pixel_unshuffle(x, scale=2) + elif self.scale == 1: + feat = pixel_unshuffle(x, scale=4) + else: + feat = x + feat = self.conv_first(feat) + body_feat = self.conv_body(self.body(feat)) + feat = feat + body_feat + # upsample + feat = self.lrelu( + self.conv_up1(F.interpolate(feat, scale_factor=2, mode='nearest'))) + feat = self.lrelu( + self.conv_up2(F.interpolate(feat, scale_factor=2, mode='nearest'))) + out = self.conv_last(self.lrelu(self.conv_hr(feat))) + return out diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/__init__.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3a1bc048dd3eaa8aa777b6c827ce1b8ff0c77f24 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .surface_recon_common import SurfaceReconCommon + +else: + _import_structure = {'surface_recon_common': ['SurfaceReconCommon']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/dataset.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5d47f6ac180084ccdbac5024d84e8abc46859698 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/dataset.py @@ -0,0 +1,289 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/Totoro97/NeuS/blob/main/models/dataset.py +# Copyright (c) 2021 Peng Wang. All Rights Reserved. +# ------------------------------------------------------------------------ + +import os +from glob import glob + +import cv2 as cv +import numpy as np +import torch +from scipy.spatial.transform import Rotation as Rot +from scipy.spatial.transform import Slerp + + +def load_K_Rt_from_P(filename, P=None): + if P is None: + lines = open(filename).read().splitlines() + if len(lines) == 4: + lines = lines[1:] + lines = [[x[0], x[1], x[2], x[3]] + for x in (x.split(' ') for x in lines)] + P = np.asarray(lines).astype(np.float32).squeeze() + + out = cv.decomposeProjectionMatrix(P) + K = out[0] + R = out[1] + t = out[2] + + K = K / K[2, 2] + intrinsics = np.eye(4) + intrinsics[:3, :3] = K + + pose = np.eye(4, dtype=np.float32) + pose[:3, :3] = R.transpose() + pose[:3, 3] = (t[:3] / t[3])[:, 0] + + return intrinsics, pose + + +class Dataset: + + def __init__(self, data_dir, device): + super(Dataset, self).__init__() + print('Load data: Begin') + self.device = device + self.data_dir = data_dir + print('data_dir: ', self.data_dir) + + camera_dict = np.load( + os.path.join(self.data_dir, 'cameras_sphere.npz')) + self.camera_dict = camera_dict + self.images_lis = sorted( + glob(os.path.join(self.data_dir, 'image/*.png'))) + self.n_images = len(self.images_lis) + print('found %d images' % self.n_images) + + self.world_mats_np = [ + camera_dict['world_mat_%d' % idx].astype(np.float32) + for idx in range(self.n_images) + ] + self.scale_mats_np = [ + camera_dict['scale_mat_%d' % idx].astype(np.float32) + for idx in range(self.n_images) + ] + + self.intrinsics_all = [] + self.pose_all = [] + for scale_mat, world_mat in zip(self.scale_mats_np, + self.world_mats_np): + P = world_mat @ scale_mat + P = P[:3, :4] + intrinsics, pose = load_K_Rt_from_P(None, P) + self.intrinsics_all.append(torch.from_numpy(intrinsics).float()) + self.pose_all.append(torch.from_numpy(pose).float()) + + self.intrinsics_all = torch.stack(self.intrinsics_all).to( + self.device) # [n_images, 4, 4] + self.intrinsics_all_inv = torch.inverse( + self.intrinsics_all) # [n_images, 4, 4] + self.focal = self.intrinsics_all[0][0, 0] + self.pose_all = torch.stack(self.pose_all).to( + self.device) # [n_images, 4, 4] + + object_bbox_min = np.array([-1.01, -1.01, -1.01, 1.0]) + object_bbox_max = np.array([1.01, 1.01, 1.01, 1.0]) + # Object scale mat: region of interest to **extract mesh** + object_scale_mat = np.load( + os.path.join(self.data_dir, 'cameras_sphere.npz'))['scale_mat_0'] + object_bbox_min = np.linalg.inv( + self.scale_mats_np[0]) @ object_scale_mat @ object_bbox_min[:, + None] + object_bbox_max = np.linalg.inv( + self.scale_mats_np[0]) @ object_scale_mat @ object_bbox_max[:, + None] + self.object_bbox_min = object_bbox_min[:3, 0] + self.object_bbox_max = object_bbox_max[:3, 0] + + print('Load data: End') + + def gen_rays_at(self, img_idx, resolution_level=1): + """ + Generate rays at world space from one camera. + """ + level = resolution_level + tx = torch.linspace(0, self.W - 1, self.W // level) + ty = torch.linspace(0, self.H - 1, self.H // level) + pixels_x, pixels_y = torch.meshgrid(tx, ty) + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # W, H, 3 + p = torch.matmul(self.intrinsics_all_inv[img_idx, None, None, :3, :3], + p[:, :, :, None]).squeeze() # W, H, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # W, H, 3 + rays_v = torch.matmul(self.pose_all[img_idx, None, None, :3, :3], + rays_v[:, :, :, None]).squeeze() # W, H, 3 + rays_o = self.pose_all[img_idx, None, None, :3, + 3].expand(rays_v.shape) # W, H, 3 + return rays_o.transpose(0, 1), rays_v.transpose(0, 1) + + def gen_rays_o_at(self, img_idx): + """ + Generate rays_o at world space from one camera. + """ + rays_o = self.pose_all[img_idx, :3, 3] + return rays_o + + # add + def gen_rays_at_camera(self, pose, resolution_level=1): + """ + Generate rays at world space from one camera. + """ + level = resolution_level + tx = torch.linspace(0, self.W - 1, self.W // level) + ty = torch.linspace(0, self.H - 1, self.H // level) + pixels_x, pixels_y = torch.meshgrid(tx, ty) + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # W, H, 3 + p = torch.matmul(self.intrinsics_all_inv[0, None, None, :3, :3], + p[:, :, :, None]).squeeze() # W, H, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # W, H, 3 + rays_v = torch.matmul(pose[:3, :3], rays_v[:, :, :, + None]).squeeze() # W, H, 3 + rays_o = pose[:3, 3].expand(rays_v.shape) # W, H, 3 + return rays_o.transpose(0, 1), rays_v.transpose(0, 1) + + def gen_random_rays_at(self, img_idx, batch_size): + """ + Generate random rays at world space from one camera. + """ + pixels_x = torch.randint(low=0, high=self.W, size=[batch_size]) # bs + pixels_y = torch.randint(low=0, high=self.H, size=[batch_size]) # bs + color = self.images[img_idx][(pixels_y, pixels_x)] # batch_size, 3 + mask = self.masks[img_idx][(pixels_y, pixels_x)] # batch_size, 3 + + depth = self.depths[img_idx][(pixels_y, pixels_x)] # batch_size, 1 + + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], + dim=-1).float() # batch_size, 3 + p = torch.matmul(self.intrinsics_all_inv[img_idx, None, :3, :3], + p[:, :, None]).squeeze() # batch_size, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # batch_size, 3 + rays_v = torch.matmul(self.pose_all[img_idx, None, :3, :3], + rays_v[:, :, None]).squeeze() # batch_size, 3 + rays_o = self.pose_all[img_idx, None, :3, + 3].expand(rays_v.shape) # batch_size, 3 + return torch.cat( + [rays_o.cpu(), + rays_v.cpu(), color, mask[:, :1], depth[:, None]], + dim=-1).cuda() # batch_size, 10 + + def gen_random_rays_at_mask(self, img_idx, batch_size): + """ + Generate random rays at world space from one camera. + """ + pixels_x = torch.randint(low=0, high=self.W, size=[batch_size]) # bs + pixels_y = torch.randint(low=0, high=self.H, size=[batch_size]) # bs + color = self.images[img_idx][(pixels_y, pixels_x)] # batch_size, 3 + mask = self.masks[img_idx][(pixels_y, pixels_x)] # batch_size, 3 + + depth = self.depths[img_idx][(pixels_y, pixels_x)] # batch_size, 1 + + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], + dim=-1).float() # batch_size, 3 + p = torch.matmul(self.intrinsics_all_inv[img_idx, None, :3, :3], + p[:, :, None]).squeeze() # batch_size, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # batch_size, 3 + rays_v = torch.matmul(self.pose_all[img_idx, None, :3, :3], + rays_v[:, :, None]).squeeze() # batch_size, 3 + rays_o = self.pose_all[img_idx, None, :3, + 3].expand(rays_v.shape) # batch_size, 3 + return torch.cat( + [rays_o.cpu(), + rays_v.cpu(), color, mask[:, :1], depth[:, None]], + dim=-1).cuda() # batch_size, 10 + + def gen_rays_between(self, idx_0, idx_1, ratio, resolution_level=1): + """ + Interpolate pose between two cameras. + """ + level = resolution_level + tx = torch.linspace(0, self.W - 1, self.W // level) + ty = torch.linspace(0, self.H - 1, self.H // level) + pixels_x, pixels_y = torch.meshgrid(tx, ty) + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # W, H, 3 + p = torch.matmul(self.intrinsics_all_inv[0, None, None, :3, :3], + p[:, :, :, None]).squeeze() # W, H, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # W, H, 3 + trans = self.pose_all[idx_0, :3, 3] * ( + 1.0 - ratio) + self.pose_all[idx_1, :3, 3] * ratio + pose_0 = self.pose_all[idx_0].detach().cpu().numpy() + pose_1 = self.pose_all[idx_1].detach().cpu().numpy() + pose_0 = np.linalg.inv(pose_0) + pose_1 = np.linalg.inv(pose_1) + rot_0 = pose_0[:3, :3] + rot_1 = pose_1[:3, :3] + rots = Rot.from_matrix(np.stack([rot_0, rot_1])) + key_times = [0, 1] + slerp = Slerp(key_times, rots) + rot = slerp(ratio) + pose = np.diag([1.0, 1.0, 1.0, 1.0]) + pose = pose.astype(np.float32) + pose[:3, :3] = rot.as_matrix() + pose[:3, 3] = ((1.0 - ratio) * pose_0 + ratio * pose_1)[:3, 3] + pose = np.linalg.inv(pose) + rot = torch.from_numpy(pose[:3, :3]).cuda() + trans = torch.from_numpy(pose[:3, 3]).cuda() + rays_v = torch.matmul(rot[None, None, :3, :3], + rays_v[:, :, :, None]).squeeze() # W, H, 3 + rays_o = trans[None, None, :3].expand(rays_v.shape) # W, H, 3 + return rays_o.transpose(0, 1), rays_v.transpose(0, 1), pose + + def gen_rays_across(self, idx_0, idx_1, ratio, resolution_level=1): + """ + Interpolate pose between two cameras. + """ + level = resolution_level + tx = torch.linspace(0, self.W - 1, self.W // level) + ty = torch.linspace(0, self.H - 1, self.H // level) + pixels_x, pixels_y = torch.meshgrid(tx, ty) + p = torch.stack( + [pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # W, H, 3 + p = torch.matmul(self.intrinsics_all_inv[0, None, None, :3, :3], + p[:, :, :, None]).squeeze() # W, H, 3 + rays_v = p / torch.linalg.norm( + p, ord=2, dim=-1, keepdim=True) # W, H, 3 + trans = self.pose_all[idx_0, :3, 3] * ( + 1.0 - ratio) + self.pose_all[idx_1, :3, 3] * ratio + pose_0 = self.pose_all[idx_0].detach().cpu().numpy() + pose_1 = self.pose_all[idx_1].detach().cpu().numpy() + pose_0 = np.linalg.inv(pose_0) + pose_1 = np.linalg.inv(pose_1) + rot_0 = pose_0[:3, :3] + rot_1 = pose_1[:3, :3] + rots = Rot.from_matrix(np.stack([rot_0, rot_1])) + key_times = [0, 1] + slerp = Slerp(key_times, rots) + rot = slerp(ratio) + pose = np.diag([1.0, 1.0, 1.0, 1.0]) + pose = pose.astype(np.float32) + pose[:3, :3] = rot.as_matrix() + pose[:3, 3] = ((1.0 - ratio) * pose_0 + ratio * pose_1)[:3, 3] + pose = np.linalg.inv(pose) + rot = torch.from_numpy(pose[:3, :3]).cuda() + trans = torch.from_numpy(pose[:3, 3]).cuda() + rays_v = torch.matmul(rot[None, None, :3, :3], + rays_v[:, :, :, None]).squeeze() # W, H, 3 + rays_o = trans[None, None, :3].expand(rays_v.shape) # W, H, 3 + return rays_o.transpose(0, 1), rays_v.transpose(0, 1), pose + + def near_far_from_sphere(self, rays_o, rays_d): + a = torch.sum(rays_d**2, dim=-1, keepdim=True) + b = 2.0 * torch.sum(rays_o * rays_d, dim=-1, keepdim=True) + mid = 0.5 * (-b) / a + near = mid - 1.0 + far = mid + 1.0 + return near, far + + def image_at(self, idx, resolution_level): + img = cv.imread(self.images_lis[idx]) + return (cv.resize(img, (self.W // resolution_level, + self.H // resolution_level))).clip(0, 255) diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/fields.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/fields.py new file mode 100644 index 0000000000000000000000000000000000000000..d0444759d79c5620273f6513b30a28e69f9fd3ad --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/fields.py @@ -0,0 +1,390 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/Totoro97/NeuS/blob/main/models/fields.py +# Copyright (c) 2021 Peng Wang. All Rights Reserved. +# ------------------------------------------------------------------------ + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils import spectral_norm + + +class SDFNetwork(nn.Module): + + def __init__(self, + d_in, + d_out, + d_hidden, + n_layers, + skip_in=(4, ), + multires=0, + bias=0.5, + scale=1, + geometric_init=True, + weight_norm=True, + inside_outside=False): + super(SDFNetwork, self).__init__() + + dims = [d_in] + [d_hidden for _ in range(n_layers)] + [d_out] + + self.embed_fn_fine = None + + if multires > 0: + embed_fn, input_ch = get_embedder(multires, input_dims=d_in) + self.embed_fn_fine = embed_fn + dims[0] = input_ch + + self.num_layers = len(dims) + self.skip_in = skip_in + self.scale = scale + + for layer in range(0, self.num_layers - 1): + if layer + 1 in self.skip_in: + out_dim = dims[layer + 1] - dims[0] + else: + out_dim = dims[layer + 1] + + lin = nn.Linear(dims[layer], out_dim) + + if geometric_init: + if layer == self.num_layers - 2: + if not inside_outside: + torch.nn.init.normal_( + lin.weight, + mean=np.sqrt(np.pi) / np.sqrt(dims[layer]), + std=0.0001) + torch.nn.init.constant_(lin.bias, -bias) + else: + torch.nn.init.normal_( + lin.weight, + mean=-np.sqrt(np.pi) / np.sqrt(dims[layer]), + std=0.0001) + torch.nn.init.constant_(lin.bias, bias) + elif multires > 0 and layer == 0: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.constant_(lin.weight[:, 3:], 0.0) + torch.nn.init.normal_(lin.weight[:, :3], 0.0, + np.sqrt(2) / np.sqrt(out_dim)) + elif multires > 0 and layer in self.skip_in: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, + np.sqrt(2) / np.sqrt(out_dim)) + torch.nn.init.constant_(lin.weight[:, -(dims[0] - 3):], + 0.0) + else: + torch.nn.init.constant_(lin.bias, 0.0) + torch.nn.init.normal_(lin.weight, 0.0, + np.sqrt(2) / np.sqrt(out_dim)) + + if weight_norm: + lin = nn.utils.weight_norm(lin) + + setattr(self, 'lin' + str(layer), lin) + + self.activation = nn.Softplus(beta=100) + + def forward(self, inputs): + inputs = inputs * self.scale + + if self.embed_fn_fine is not None: + inputs = self.embed_fn_fine(inputs) + + x = inputs + for layer in range(0, self.num_layers - 1): + lin = getattr(self, 'lin' + str(layer)) + + if layer in self.skip_in: + x = torch.cat([x, inputs], 1) / np.sqrt(2) + + x = lin(x) + + if layer < self.num_layers - 2: + x = self.activation(x) + return torch.cat([x[:, :1] / self.scale, x[:, 1:]], dim=-1) + + def sdf(self, x): + return self.forward(x)[:, :1] + + def sdf_hidden_appearance(self, x): + return self.forward(x) + + def gradient(self, x): + x.requires_grad_(True) + with torch.enable_grad(): + y = self.sdf(x) + + d_output = torch.ones_like(y, requires_grad=False, device=y.device) + gradients = torch.autograd.grad( + outputs=y, + inputs=x, + grad_outputs=d_output, + create_graph=True, + retain_graph=True, + only_inputs=True)[0] + return gradients.unsqueeze(1) + + +class RenderingNetwork(nn.Module): + + def __init__(self, + d_feature, + mode, + d_in, + d_out, + d_hidden, + n_layers, + weight_norm=True, + multires_view=0, + squeeze_out=True): + super().__init__() + + self.mode = mode + self.squeeze_out = squeeze_out + dims = [d_in + d_feature] + [d_hidden + for _ in range(n_layers)] + [d_out] + + self.embedview_fn = None + if multires_view > 0: + embedview_fn, input_ch = get_embedder(multires_view) + self.embedview_fn = embedview_fn + dims[0] += (input_ch - 3) + + self.num_layers = len(dims) + + for layer in range(0, self.num_layers - 1): + out_dim = dims[layer + 1] + lin = nn.Linear(dims[layer], out_dim) + + if weight_norm: + lin = nn.utils.weight_norm(lin) + + setattr(self, 'lin' + str(layer), lin) + + self.relu = nn.ReLU() + + def forward(self, points, normals, view_dirs, feature_vectors): + if self.embedview_fn is not None: + view_dirs = self.embedview_fn(view_dirs) + + rendering_input = None + + if self.mode == 'idr': + rendering_input = torch.cat( + [points, view_dirs, normals, feature_vectors], dim=-1) + elif self.mode == 'no_view_dir': + rendering_input = torch.cat([points, normals, feature_vectors], + dim=-1) + elif self.mode == 'no_normal': + rendering_input = torch.cat([points, view_dirs, feature_vectors], + dim=-1) + + x = rendering_input + + for layer in range(0, self.num_layers - 1): + lin = getattr(self, 'lin' + str(layer)) + x = lin(x) + if layer < self.num_layers - 2: + x = self.relu(x) + if self.squeeze_out: + x = torch.sigmoid(x) + return x + + +class NeRF(nn.Module): + + def __init__(self, + D=8, + W=256, + d_in=3, + d_in_view=3, + multires=0, + multires_view=0, + output_ch=4, + skips=[4], + use_viewdirs=False): + super(NeRF, self).__init__() + self.D = D + self.W = W + self.d_in = d_in + self.d_in_view = d_in_view + self.input_ch = 3 + self.input_ch_view = 3 + self.embed_fn = None + self.embed_fn_view = None + + if multires > 0: + embed_fn, input_ch = get_embedder(multires, input_dims=d_in) + self.embed_fn = embed_fn + self.input_ch = input_ch + + if multires_view > 0: + embed_fn_view, input_ch_view = get_embedder( + multires_view, input_dims=d_in_view) + self.embed_fn_view = embed_fn_view + self.input_ch_view = input_ch_view + + self.skips = skips + self.use_viewdirs = use_viewdirs + + self.pts_linears = nn.ModuleList([nn.Linear(self.input_ch, W)] + [ + nn.Linear(W, W) if i not in + self.skips else nn.Linear(W + self.input_ch, W) + for i in range(D - 1) + ]) + + self.views_linears = nn.ModuleList( + [nn.Linear(self.input_ch_view + W, W // 2)]) + + if use_viewdirs: + self.feature_linear = nn.Linear(W, W) + self.alpha_linear = nn.Linear(W, 1) + self.rgb_linear = nn.Linear(W // 2, 3) + else: + self.output_linear = nn.Linear(W, output_ch) + + def forward(self, input_pts, input_views): + if self.embed_fn is not None: + input_pts = self.embed_fn(input_pts) + if self.embed_fn_view is not None: + input_views = self.embed_fn_view(input_views) + + h = input_pts + for i, l in enumerate(self.pts_linears): + h = self.pts_linears[i](h) + h = F.relu(h) + if i in self.skips: + h = torch.cat([input_pts, h], -1) + + if self.use_viewdirs: + alpha = self.alpha_linear(h) + feature = self.feature_linear(h) + h = torch.cat([feature, input_views], -1) + + for i, l in enumerate(self.views_linears): + h = self.views_linears[i](h) + h = F.relu(h) + + rgb = self.rgb_linear(h) + return alpha, rgb + else: + assert False + + +class SingleVarianceNetwork(nn.Module): + + def __init__(self, init_val): + super(SingleVarianceNetwork, self).__init__() + self.register_parameter('variance', + nn.Parameter(torch.tensor(init_val))) + + def forward(self, x): + return torch.ones([len(x), 1], + device=self.variance.device) * torch.exp( + self.variance * 10.0) + + +class Mean(nn.Module): + + def __init__(self, dim: list, keepdim=False): + super().__init__() + self.dim = dim + self.keepdim = keepdim + + def forward(self, x): + return torch.mean(x, self.dim, self.keepdim) + + +class Discriminator(nn.Module): + + def __init__(self, channel=32, patch=True): + super().__init__() + self.imsize = 32 + self.nc = 3 + + self.channel = channel + self.patch = patch + in_channel = 3 + layer = [] + for idx in range(3): + layer.extend([ + spectral_norm( + nn.Conv2d( + in_channel, channel * (2**idx), 3, stride=2, + padding=1)), + nn.LeakyReLU(inplace=True), + spectral_norm( + nn.Conv2d( + channel * (2**idx), + channel * (2**idx), + 3, + stride=1, + padding=1)), + nn.LeakyReLU(inplace=True), + ]) + in_channel = channel * (2**idx) + self.body = nn.Sequential(*layer) + if self.patch: + self.head = spectral_norm(nn.Conv2d(in_channel, 1, 1, padding=0)) + else: + self.head = nn.Sequential(Mean([1, 2]), nn.Linear(in_channel, 1)) + + def forward(self, x): + x = x[:, :self.nc] + x = x.view(-1, self.imsize, self.imsize, self.nc).permute(0, 3, 1, 2) + x = self.body(x) + x = self.head(x) + return x + + +class Embedder: + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.create_embedding_fn() + + def create_embedding_fn(self): + embed_fns = [] + d = self.kwargs['input_dims'] + out_dim = 0 + if self.kwargs['include_input']: + embed_fns.append(lambda x: x) + out_dim += d + + max_freq = self.kwargs['max_freq_log2'] + N_freqs = self.kwargs['num_freqs'] + + if self.kwargs['log_sampling']: + freq_bands = 2.**torch.linspace(0., max_freq, N_freqs) + else: + freq_bands = torch.linspace(2.**0., 2.**max_freq, N_freqs) + + for freq in freq_bands: + for p_fn in self.kwargs['periodic_fns']: + embed_fns.append( + lambda x, p_fn=p_fn, freq=freq: p_fn(x * freq)) + out_dim += d + + self.embed_fns = embed_fns + self.out_dim = out_dim + + def embed(self, inputs): + return torch.cat([fn(inputs) for fn in self.embed_fns], -1) + + +def get_embedder(multires, input_dims=3): + embed_kwargs = { + 'include_input': True, + 'input_dims': input_dims, + 'max_freq_log2': multires - 1, + 'num_freqs': multires, + 'log_sampling': True, + 'periodic_fns': [torch.sin, torch.cos], + } + + embedder_obj = Embedder(**embed_kwargs) + + def embed(x, eo=embedder_obj): + return eo.embed(x) + + return embed, embedder_obj.out_dim diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/renderer.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..cbe2e5408aaccd6bf57720a38ef63152186adbf3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/renderer.py @@ -0,0 +1,388 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/Totoro97/NeuS/blob/main/models/renderer.py +# Copyright (c) 2021 Peng Wang. +# Copyright (c) Alibaba, Inc. and its affiliates. +# ------------------------------------------------------------------------ + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .fields import RenderingNetwork, SDFNetwork, SingleVarianceNetwork +from .utils import extract_geometry, sample_pdf + + +class SurfaceRenderer(nn.Module): + + def __init__(self, conf, device): + super().__init__() + self.conf = conf + self.device = device + self.sdf_network = SDFNetwork(**self.conf['sdf_network']).to( + self.device) + self.variance_network = SingleVarianceNetwork( + **self.conf['variance_network']).to(self.device) + self.color_network = RenderingNetwork( + **self.conf['rendering_network']).to(self.device) + self.light_network = RenderingNetwork(**self.conf['light_network']).to( + self.device) + self.n_samples = self.conf['neus_renderer']['n_samples'] + self.n_importance = self.conf['neus_renderer']['n_importance'] + self.n_outside = self.conf['neus_renderer']['n_outside'] + self.up_sample_steps = self.conf['neus_renderer']['up_sample_steps'] + self.perturb = self.conf['neus_renderer']['perturb'] + + def extract_geometry(self, + bound_min, + bound_max, + resolution, + threshold=0.0, + device='cuda'): + return extract_geometry( + bound_min, + bound_max, + resolution=resolution, + threshold=threshold, + query_func=lambda pts: -self.sdf_network.sdf(pts), + device=device) + + def render_core_outside(self, + rays_o, + rays_d, + z_vals, + sample_dist, + nerf, + background_rgb=None): + batch_size, n_samples = z_vals.shape + + dists = z_vals[..., 1:] - z_vals[..., :-1] + dists = torch.cat( + [dists, + torch.Tensor([sample_dist]).expand(dists[..., :1].shape)], -1) + mid_z_vals = z_vals + dists * 0.5 + + pts = rays_o[:, None, :] + rays_d[:, None, :] * mid_z_vals[ + ..., :, None] # batch_size, n_samples, 3 + + dis_to_center = torch.linalg.norm( + pts, ord=2, dim=-1, keepdim=True).clip(1.0, 1e10) + pts = torch.cat([pts / dis_to_center, 1.0 / dis_to_center], + dim=-1) # batch_size, n_samples, 4 + + dirs = rays_d[:, None, :].expand(batch_size, n_samples, 3) + + pts = pts.reshape(-1, 3 + int(self.n_outside > 0)) + dirs = dirs.reshape(-1, 3) + + density, sampled_color = nerf(pts, dirs) + alpha = 1.0 - torch.exp( + -F.softplus(density.reshape(batch_size, n_samples)) * dists) + alpha = alpha.reshape(batch_size, n_samples) + weights = alpha * torch.cumprod( + torch.cat([torch.ones([batch_size, 1]), 1. - alpha + 1e-7], -1), + -1)[:, :-1] + sampled_color = sampled_color.reshape(batch_size, n_samples, 3) + color = (weights[:, :, None] * sampled_color).sum(dim=1) + if background_rgb is not None: + color = color + background_rgb * ( + 1.0 - weights.sum(dim=-1, keepdim=True)) + + return { + 'color': color, + 'sampled_color': sampled_color, + 'alpha': alpha, + 'weights': weights, + } + + def up_sample(self, rays_o, rays_d, z_vals, sdf, n_importance, inv_s): + batch_size, n_samples = z_vals.shape + pts = rays_o[:, None, :] + rays_d[:, None, :] * z_vals[..., :, None] + radius = torch.linalg.norm(pts, ord=2, dim=-1, keepdim=False) + inside_sphere = (radius[:, :-1] < 1.0) | (radius[:, 1:] < 1.0) + sdf = sdf.reshape(batch_size, n_samples) + prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:] + prev_z_vals, next_z_vals = z_vals[:, :-1], z_vals[:, 1:] + mid_sdf = (prev_sdf + next_sdf) * 0.5 + cos_val = (next_sdf - prev_sdf) / (next_z_vals - prev_z_vals + 1e-5) + + prev_cos_val = torch.cat( + [torch.zeros([batch_size, 1]).to(self.device), cos_val[:, :-1]], + dim=-1) + cos_val = torch.stack([prev_cos_val, cos_val], dim=-1) + cos_val, _ = torch.min(cos_val, dim=-1, keepdim=False) + cos_val = cos_val.clip(-1e3, 0.0) * inside_sphere + + dist = (next_z_vals - prev_z_vals) + prev_esti_sdf = mid_sdf - cos_val * dist * 0.5 + next_esti_sdf = mid_sdf + cos_val * dist * 0.5 + prev_cdf = torch.sigmoid(prev_esti_sdf * inv_s) + next_cdf = torch.sigmoid(next_esti_sdf * inv_s) + alpha = (prev_cdf - next_cdf + 1e-5) / (prev_cdf + 1e-5) + weights = alpha * torch.cumprod( + torch.cat([ + torch.ones([batch_size, 1]).to(self.device), 1. - alpha + 1e-7 + ], -1), -1)[:, :-1] + + z_samples = sample_pdf( + z_vals, weights, n_importance, det=True, + device=self.device).detach() + return z_samples + + def cat_z_vals(self, rays_o, rays_d, z_vals, new_z_vals, sdf, last=False): + batch_size, n_samples = z_vals.shape + _, n_importance = new_z_vals.shape + pts = rays_o[:, + None, :] + rays_d[:, None, :] * new_z_vals[..., :, None] + z_vals = torch.cat([z_vals, new_z_vals], dim=-1) + z_vals, index = torch.sort(z_vals, dim=-1) + + if not last: + new_sdf = self.sdf_network.sdf(pts.reshape(-1, 3)).reshape( + batch_size, n_importance) + sdf = torch.cat([sdf, new_sdf], dim=-1) + xx = torch.arange(batch_size)[:, None].expand( + batch_size, n_samples + n_importance).reshape(-1) + index = index.reshape(-1) + sdf = sdf[(xx, index)].reshape(batch_size, + n_samples + n_importance) + + return z_vals, sdf + + def render_core(self, + rays_o, + rays_d, + z_vals, + sample_dist, + sdf_network, + deviation_network, + color_network, + light_network, + depth_z=None, + background_alpha=None, + bg_sampled_color=None, + background_rgb=None, + cos_anneal_ratio=0.0): + batch_size, n_samples = z_vals.shape + + dists = z_vals[..., 1:] - z_vals[..., :-1] + dists = torch.cat([ + dists, + torch.Tensor([sample_dist]).expand(dists[..., :1].shape).to( + self.device) + ], -1) + mid_z_vals = z_vals + dists * 0.5 + + pts = rays_o[:, + None, :] + rays_d[:, None, :] * mid_z_vals[..., :, None] + dirs = rays_d[:, None, :].expand(pts.shape) + + pts = pts.reshape(-1, 3) + dirs = dirs.reshape(-1, 3) + + sdf_nn_output = sdf_network(pts) + sdf = sdf_nn_output[:, :1] + feature_vector = sdf_nn_output[:, 1:] + + gradients = sdf_network.gradient(pts).squeeze() + sampled_albedo = color_network(pts, gradients, dirs, + feature_vector).reshape( + batch_size, n_samples, 3) + sampled_light = light_network(pts, gradients, dirs, + feature_vector).reshape( + batch_size, n_samples, 3) + sampled_color = sampled_albedo * sampled_light + + inv_s = deviation_network(torch.zeros([1, 3]))[:, :1].clip(1e-6, 1e6) + inv_s = inv_s.expand(batch_size * n_samples, 1) + + true_cos = (dirs * gradients).sum(-1, keepdim=True) + iter_cos_p1 = F.relu(-true_cos * 0.5 + 0.5) * (1.0 - cos_anneal_ratio) + iter_cos = -(iter_cos_p1 + F.relu(-true_cos) * cos_anneal_ratio) + + estimated_next_sdf = sdf + iter_cos * dists.reshape(-1, 1) * 0.5 + estimated_prev_sdf = sdf - iter_cos * dists.reshape(-1, 1) * 0.5 + + prev_cdf = torch.sigmoid(estimated_prev_sdf * inv_s) + next_cdf = torch.sigmoid(estimated_next_sdf * inv_s) + + p = prev_cdf - next_cdf + c = prev_cdf + + alpha = ((p + 1e-5) / (c + 1e-5)).reshape(batch_size, + n_samples).clip(0.0, 1.0) + + pts_norm = torch.linalg.norm( + pts, ord=2, dim=-1, keepdim=True).reshape(batch_size, n_samples) + inside_sphere = (pts_norm < 1.0).float().detach() + relax_inside_sphere = (pts_norm < 1.2).float().detach() + + if background_alpha is not None: + alpha = alpha * inside_sphere + background_alpha[:, :n_samples] * ( + 1.0 - inside_sphere) + alpha = torch.cat([alpha, background_alpha[:, n_samples:]], dim=-1) + foreground_color = sampled_color * inside_sphere[:, :, None] + background_color = bg_sampled_color[:, :n_samples] * ( + 1.0 - inside_sphere)[:, :, None] + sampled_color = foreground_color + background_color + + sampled_color = torch.cat( + [sampled_color, bg_sampled_color[:, n_samples:]], dim=1) + + beta = torch.cat([ + torch.ones([batch_size, 1], device=alpha.device), 1. - alpha + 1e-7 + ], -1) + weights = alpha * torch.cumprod(beta, -1)[:, :-1] + weights_sum = weights.sum(dim=-1, keepdim=True) + + color = (sampled_color * weights[:, :, None]).sum(dim=1) + if background_rgb is not None: + color = color + background_rgb * (1.0 - weights_sum) + + albedo = (sampled_albedo * weights[:, :, None]).sum(dim=1) + + depth = (mid_z_vals * weights).sum(dim=1) + if depth_z is not None: + pts_depth = rays_o[:, None, :] + rays_d[:, None, :] * depth_z[ + ..., :, None] # n_rays, n_samples, 3 + pts_depth = pts_depth.reshape(-1, 3) + sdf_depth = sdf_network(pts_depth)[:, :1] + else: + sdf_depth = None + + gradients_norm = torch.linalg.norm( + gradients.reshape(batch_size, n_samples, 3), ord=2, dim=-1) + gradient_error = (gradients_norm - 1.0)**2 + gradient_error = (relax_inside_sphere * gradient_error).sum() + gradient_error = gradient_error / (relax_inside_sphere.sum() + 1e-5) + + return { + 'color': color, + 'albedo': albedo, + 'depth': depth, + 'sdf': sdf, + 'sdf_depth': sdf_depth, + 'dists': dists, + 'gradients': gradients.reshape(batch_size, n_samples, 3), + 's_val': 1.0 / inv_s, + 'mid_z_vals': mid_z_vals, + 'weights': weights, + 'cdf': c.reshape(batch_size, n_samples), + 'gradient_error': gradient_error, + 'inside_sphere': inside_sphere + } + + def render(self, + rays_o, + rays_d, + near, + far, + depth_z=None, + perturb_overwrite=-1, + background_rgb=None, + cos_anneal_ratio=0.0): + batch_size = len(rays_o) + sample_dist = 2.0 / self.n_samples # Assuming the region of interest is a unit sphere + z_vals = torch.linspace(0.0, 1.0, self.n_samples).to(self.device) + z_vals = near + (far - near) * z_vals[None, :] + + z_vals_outside = None + if self.n_outside > 0: + z_vals_end = 1.0 - 1.0 / (self.n_outside + 1.0) + z_vals_outside = torch.linspace(1e-3, z_vals_end, self.n_outside) + + n_samples = self.n_samples + perturb = self.perturb + + if perturb_overwrite >= 0: + perturb = perturb_overwrite + if perturb > 0: + t_rand = (torch.rand([batch_size, 1]).to(self.device) - 0.5) + z_vals = z_vals + t_rand * 2.0 / self.n_samples + + if self.n_outside > 0: + mids = .5 * ( + z_vals_outside[..., 1:] + z_vals_outside[..., :-1]) + upper = torch.cat([mids, z_vals_outside[..., -1:]], -1) + lower = torch.cat([z_vals_outside[..., :1], mids], -1) + t_rand = torch.rand([batch_size, z_vals_outside.shape[-1]]) + z_vals_outside = lower[None, :] + (upper + - lower)[None, :] * t_rand + + if self.n_outside > 0: + z_vals_outside = far / torch.flip( + z_vals_outside, dims=[-1]) + 1.0 / self.n_samples + + background_alpha = None + background_sampled_color = None + + # Up sample + if self.n_importance > 0: + with torch.no_grad(): + pts = rays_o[:, None, :] + rays_d[:, None, :] * z_vals[..., :, + None] + sdf = self.sdf_network.sdf(pts.reshape(-1, 3)).reshape( + batch_size, self.n_samples) + + for i in range(self.up_sample_steps): + new_z_vals = self.up_sample( + rays_o, rays_d, z_vals, sdf, + self.n_importance // self.up_sample_steps, 64 * 2**i) + z_vals, sdf = self.cat_z_vals( + rays_o, + rays_d, + z_vals, + new_z_vals, + sdf, + last=(i + 1 == self.up_sample_steps)) + + n_samples = self.n_samples + self.n_importance + + if self.n_outside > 0: + z_vals_feed = torch.cat([z_vals, z_vals_outside], dim=-1) + z_vals_feed, _ = torch.sort(z_vals_feed, dim=-1) + ret_outside = self.render_core_outside(rays_o, rays_d, z_vals_feed, + sample_dist, self.nerf) + + background_sampled_color = ret_outside['sampled_color'] + background_alpha = ret_outside['alpha'] + + ret_fine = self.render_core( + rays_o, + rays_d, + z_vals, + sample_dist, + self.sdf_network, + self.variance_network, + self.color_network, + self.light_network, + depth_z=depth_z, + background_rgb=background_rgb, + background_alpha=background_alpha, + background_sampled_color=background_sampled_color, + cos_anneal_ratio=cos_anneal_ratio) + + color_fine = ret_fine['color'] + albedo_fine = ret_fine['albedo'] + depth_fine = ret_fine['depth'] + sdf_depth = ret_fine['sdf_depth'] + weights = ret_fine['weights'] + weights_sum = weights.sum(dim=-1, keepdim=True) + gradients = ret_fine['gradients'] + s_val = ret_fine['s_val'].reshape(batch_size, n_samples).mean( + dim=-1, keepdim=True) + + return { + 'color_fine': color_fine, + 'albedo_fine': albedo_fine, + 'depth_fine': depth_fine, + 'sdf_depth': sdf_depth, + 's_val': s_val, + 'cdf_fine': ret_fine['cdf'], + 'weight_sum': weights_sum, + 'weight_max': torch.max(weights, dim=-1, keepdim=True)[0], + 'gradients': gradients, + 'weights': weights, + 'mid_z_vals': ret_fine['mid_z_vals'], + 'gradient_error': ret_fine['gradient_error'], + 'inside_sphere': ret_fine['inside_sphere'] + } diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/surface_recon_common.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/surface_recon_common.py new file mode 100644 index 0000000000000000000000000000000000000000..d45383eb336f0559a58e5d63b1fe68db8e45f64a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/surface_recon_common.py @@ -0,0 +1,160 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import numpy as np +import torch +import trimesh + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .dataset import Dataset +from .renderer import SurfaceRenderer + +logger = get_logger() + +__all__ = ['SurfaceReconCommon'] + + +@MODELS.register_module( + Tasks.surface_recon_common, module_name=Models.surface_recon_common) +class SurfaceReconCommon(TorchModel): + + def __init__(self, model_dir, network_cfg, **kwargs): + """initialize the surface reconstruction model for common objects. + + Args: + model_dir (str): the model path. + network_cfg (dict): args of network config + """ + super().__init__(model_dir, **kwargs) + logger.info('model params:{}'.format(kwargs)) + + if torch.cuda.is_available(): + self.device = torch.device('cuda') + else: + raise Exception('GPU is required') + + logger.info(network_cfg) + + self.renderer = SurfaceRenderer(network_cfg, device=self.device) + self.ckpt_path = os.path.join(model_dir, 'model.pth') + if not os.path.exists(self.ckpt_path): + raise Exception('model path not found') + self.load_checkpoint(self.ckpt_path) + logger.info('load models from %s' % self.ckpt_path) + + self.n_rays = network_cfg['n_rays'] + + def load_checkpoint(self, ckpt_path): + checkpoint = torch.load(ckpt_path, map_location=self.device) + for name, module in self.renderer.named_modules(): + saved_name = name + '_fine' + if saved_name in checkpoint: + module.load_state_dict(checkpoint[saved_name]) + + def surface_reconstruction(self, + data_dir, + save_dir, + color=False, + n_directions=8): + + self.dataset = Dataset(data_dir, self.device) + + bound_min = torch.tensor( + self.dataset.object_bbox_min, dtype=torch.float32).to(self.device) + bound_max = torch.tensor( + self.dataset.object_bbox_max, dtype=torch.float32).to(self.device) + + vertices, triangles = \ + self.renderer.extract_geometry(bound_min, bound_max, resolution=512, threshold=0.0, + device=self.device) + if color: + pt_vertices = torch.from_numpy(vertices).cuda().reshape(-1, 1, + 3).float() + idx_list = np.linspace( + 0, + self.dataset.n_images, + n_directions, + endpoint=False, + dtype=int) + rays_o_list = [] + for idx in idx_list: + rays_o = self.dataset.pose_all[idx, :3, 3] + rays_o_list.append(rays_o) + + rgb_final = None + diff_final = None + for rays_o in rays_o_list: + rays_o = rays_o.reshape(1, 3).repeat(vertices.shape[0], + 1).float() + + rays_d = pt_vertices.reshape(-1, 3) - rays_o + rays_d = rays_d / torch.norm(rays_d, dim=-1).reshape(-1, 1) + dist = torch.norm(pt_vertices.reshape(-1, 3) - rays_o, dim=-1) + + rays_o = rays_o.reshape(-1, 3).split(self.n_rays) + rays_d = rays_d.reshape(-1, 3).split(self.n_rays) + dist = dist.reshape(-1).split(self.n_rays) + out_rgb_fine = [] + depth_diff = [] + for i, (rays_o_batch, + rays_d_batch) in enumerate(zip(rays_o, rays_d)): + near, far = self.dataset.near_far_from_sphere( + rays_o_batch, rays_d_batch) + render_out = self.renderer.render( + rays_o_batch, + rays_d_batch, + near, + far, + cos_anneal_ratio=1.0, + background_rgb=None) + + # out_rgb_fine.append(render_out['color_fine'].detach().cpu().numpy()) + out_rgb_fine.append( + render_out['albedo_fine'].detach().cpu().numpy()) + + weights = render_out['weights'] + mid_z_vals = render_out['mid_z_vals'] + n_samples = self.renderer.n_samples + self.renderer.n_importance + depth_batch = (mid_z_vals[:, :n_samples] + * weights[:, :n_samples]).sum( + dim=1).detach().cpu().numpy() + dist_batch = dist[i].detach().cpu().numpy() + depth_diff.append(np.abs(depth_batch - dist_batch)) + + del render_out + + out_rgb_fine = np.concatenate( + out_rgb_fine, axis=0).reshape(vertices.shape[0], 3) + depth_diff = np.concatenate( + depth_diff, axis=0).reshape(vertices.shape[0]) + + if rgb_final is None: + rgb_final = out_rgb_fine.copy() + diff_final = depth_diff.copy() + else: + ind = diff_final > depth_diff + ind = ind.reshape(-1) + rgb_final[ind] = out_rgb_fine[ind] + diff_final[ind] = depth_diff[ind] + + vertices = vertices * self.dataset.scale_mats_np[0][ + 0, 0] + self.dataset.scale_mats_np[0][:3, 3][None] + + if color: + logger.info('save mesh with color') + vert_colors = (255 * np.clip(rgb_final[..., ::-1], 0, 1)).astype( + np.uint8) + mesh = trimesh.Trimesh( + vertices, triangles, vertex_colors=vert_colors) + else: + mesh = trimesh.Trimesh(vertices, triangles) + + outpath = os.path.join(save_dir, 'mesh.ply') + mesh.export(outpath) + + logger.info('surface econstruction done, export mesh to %s' % outpath) diff --git a/runtime/Lib/site_packages/models/cv/surface_recon_common/utils.py b/runtime/Lib/site_packages/models/cv/surface_recon_common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9c990393636ac40a0341a9653dcb15f544259f69 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/surface_recon_common/utils.py @@ -0,0 +1,85 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/Totoro97/NeuS/blob/main/models/renderer.py +# Copyright (c) 2021 Peng Wang. +# ------------------------------------------------------------------------ + +import mcubes +import numpy as np +import torch + + +def extract_fields(bound_min, + bound_max, + resolution, + query_func, + device='cuda'): + + N = 64 + X = torch.linspace(bound_min[0], bound_max[0], resolution).split(N) + Y = torch.linspace(bound_min[1], bound_max[1], resolution).split(N) + Z = torch.linspace(bound_min[2], bound_max[2], resolution).split(N) + + u = np.zeros([resolution, resolution, resolution], dtype=np.float32) + with torch.no_grad(): + for xi, xs in enumerate(X): + for yi, ys in enumerate(Y): + for zi, zs in enumerate(Z): + xx, yy, zz = torch.meshgrid(xs, ys, zs) + xx = xx.reshape(-1, 1) + yy = yy.reshape(-1, 1) + zz = zz.reshape(-1, 1) + pts = torch.cat([xx, yy, zz], dim=-1) + pts = pts.to(device) + val = query_func(pts).reshape( + len(xs), len(ys), len(zs)).detach().cpu().numpy() + u[xi * N:xi * N + len(xs), yi * N:yi * N + len(ys), + zi * N:zi * N + len(zs)] = val + return u + + +def extract_geometry(bound_min, bound_max, resolution, threshold, query_func, + device): + print('threshold: {}'.format(threshold)) + u = extract_fields(bound_min, bound_max, resolution, query_func, device) + vertices, triangles = mcubes.marching_cubes(u, threshold) + b_max_np = bound_max.detach().cpu().numpy() + b_min_np = bound_min.detach().cpu().numpy() + + vertices = vertices / (resolution - 1.0) * ( + b_max_np - b_min_np)[None, :] + b_min_np[None, :] + return vertices, triangles + + +def sample_pdf(bins, weights, n_samples, det=False, device='cuda'): + # This implementation is from NeRF + # Get pdf + weights = weights + 1e-5 # prevent nans + pdf = weights / torch.sum(weights, -1, keepdim=True) + cdf = torch.cumsum(pdf, -1) + cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], -1) + # Take uniform samples + if det: + u = torch.linspace( + 0. + 0.5 / n_samples, 1. - 0.5 / n_samples, + steps=n_samples).to(device) + u = u.expand(list(cdf.shape[:-1]) + [n_samples]) + else: + u = torch.rand(list(cdf.shape[:-1]) + [n_samples]).to(device) + + # Invert CDF + u = u.contiguous() + inds = torch.searchsorted(cdf, u, right=True) + below = torch.max(torch.zeros_like(inds - 1), inds - 1) + above = torch.min((cdf.shape[-1] - 1) * torch.ones_like(inds), inds) + inds_g = torch.stack([below, above], -1) # (batch, N_samples, 2) + + matched_shape = [inds_g.shape[0], inds_g.shape[1], cdf.shape[-1]] + cdf_g = torch.gather(cdf.unsqueeze(1).expand(matched_shape), 2, inds_g) + bins_g = torch.gather(bins.unsqueeze(1).expand(matched_shape), 2, inds_g) + + denom = (cdf_g[..., 1] - cdf_g[..., 0]) + denom = torch.where(denom < 1e-5, torch.ones_like(denom), denom) + t = (u - cdf_g[..., 0]) / denom + samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0]) + + return samples diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/__init__.py b/runtime/Lib/site_packages/models/cv/table_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e88f7f67d01283fbcc56bf2ce798f0606152bce7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/table_recognition/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model_lore import LoreModel + +else: + _import_structure = {'model_lore': ['LoreModel']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/lineless_table_process.py b/runtime/Lib/site_packages/models/cv/table_recognition/lineless_table_process.py new file mode 100644 index 0000000000000000000000000000000000000000..a04d531d68a1f90164eb8157c52dc9d1c7f50a99 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/table_recognition/lineless_table_process.py @@ -0,0 +1,449 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from CenterNet, +# made publicly available under the MIT License at https://github.com/xingyizhou/CenterNet.git +# ------------------------------------------------------------------------------ + +import cv2 +import numpy as np +import shapely +import torch +import torch.nn as nn +from shapely.geometry import MultiPoint, Point, Polygon + + +def _gather_feat(feat, ind, mask=None): + # mandatory + dim = feat.size(2) + ind = ind.unsqueeze(2).expand(ind.size(0), ind.size(1), dim) + feat = feat.gather(1, ind) + if mask is not None: + mask = mask.unsqueeze(2).expand_as(feat) + feat = feat[mask] + feat = feat.view(-1, dim) + return feat + + +def _tranpose_and_gather_feat(feat, ind): + # mandatory + feat = feat.permute(0, 2, 3, 1).contiguous() + feat = feat.view(feat.size(0), -1, feat.size(3)) + feat = _gather_feat(feat, ind) + return feat + + +def _get_4ps_feat(cc_match, output): + # mandatory + if isinstance(output, dict): + feat = output['cr'] + else: + feat = output + feat = feat.permute(0, 2, 3, 1).contiguous() + feat = feat.contiguous().view(feat.size(0), -1, feat.size(3)) + feat = feat.unsqueeze(3).expand( + feat.size(0), feat.size(1), feat.size(2), 4) + + dim = feat.size(2) + + cc_match = cc_match.unsqueeze(2).expand( + cc_match.size(0), cc_match.size(1), dim, cc_match.size(2)) + if not (isinstance(output, dict)): + cc_match = torch.where( + cc_match < feat.shape[1], cc_match, (feat.shape[0] - 1) + * torch.ones(cc_match.shape).to(torch.int64).cuda()) + cc_match = torch.where( + cc_match >= 0, cc_match, + torch.zeros(cc_match.shape).to(torch.int64).cuda()) + feat = feat.gather(1, cc_match) + return feat + + +def _nms(heat, name, kernel=3): + pad = (kernel - 1) // 2 + + hmax = nn.functional.max_pool2d( + heat, (kernel, kernel), stride=1, padding=pad) + # save_map(hmax.cpu().numpy()[0],name) + keep = (hmax == heat).float() + return heat * keep, keep + + +def _topk(scores, K=40): + batch, cat, height, width = scores.size() + + topk_scores, topk_inds = torch.topk(scores.view(batch, cat, -1), K) + + topk_inds = topk_inds % ( + torch.Tensor([height]).to(torch.int64).cuda() + * torch.Tensor([width]).to(torch.int64).cuda()) + topk_ys = (topk_inds / torch.Tensor([width]).cuda()).int().float() + topk_xs = (topk_inds + % torch.Tensor([width]).to(torch.int64).cuda()).int().float() + + topk_score, topk_ind = torch.topk(topk_scores.view(batch, -1), K) + topk_clses = (topk_ind // K).int() + topk_inds = _gather_feat(topk_inds.view(batch, -1, 1), + topk_ind).view(batch, K) + topk_ys = _gather_feat(topk_ys.view(batch, -1, 1), topk_ind).view(batch, K) + topk_xs = _gather_feat(topk_xs.view(batch, -1, 1), topk_ind).view(batch, K) + + return topk_score, topk_inds, topk_clses, topk_ys, topk_xs + + +def corner_decode(mk, st_reg, mk_reg=None, K=400): + batch, cat, height, width = mk.size() + mk, keep = _nms(mk, 'mk.0.maxpool') + scores, inds, clses, ys, xs = _topk(mk, K=K) + if mk_reg is not None: + reg = _tranpose_and_gather_feat(mk_reg, inds) + reg = reg.view(batch, K, 2) + xs = xs.view(batch, K, 1) + reg[:, :, 0:1] + ys = ys.view(batch, K, 1) + reg[:, :, 1:2] + else: + xs = xs.view(batch, K, 1) + 0.5 + ys = ys.view(batch, K, 1) + 0.5 + scores = scores.view(batch, K, 1) + st_Reg = _tranpose_and_gather_feat(st_reg, inds) + bboxes_vec = [ + xs - st_Reg[..., 0:1], ys - st_Reg[..., 1:2], xs - st_Reg[..., 2:3], + ys - st_Reg[..., 3:4], xs - st_Reg[..., 4:5], ys - st_Reg[..., 5:6], + xs - st_Reg[..., 6:7], ys - st_Reg[..., 7:8] + ] + bboxes = torch.cat(bboxes_vec, dim=2) + corner_dict = { + 'scores': scores, + 'inds': inds, + 'ys': ys, + 'xs': xs, + 'gboxes': bboxes + } + return scores, inds, ys, xs, bboxes, corner_dict + + +def ctdet_4ps_decode(heat, + wh, + ax, + cr, + corner_dict=None, + reg=None, + cat_spec_wh=False, + K=100, + wiz_rev=False): + + batch, cat, height, width = heat.size() + # heat = torch.sigmoid(heat) + # perform nms on heatmaps + heat, keep = _nms(heat, 'hm.0.maxpool') + + scores, inds, clses, ys, xs = _topk(heat, K=K) + if reg is not None: + reg = _tranpose_and_gather_feat(reg, inds) + reg = reg.view(batch, K, 2) + xs = xs.view(batch, K, 1) + reg[:, :, 0:1] + ys = ys.view(batch, K, 1) + reg[:, :, 1:2] + else: + xs = xs.view(batch, K, 1) + 0.5 + ys = ys.view(batch, K, 1) + 0.5 + wh = _tranpose_and_gather_feat(wh, inds) + ax = _tranpose_and_gather_feat(ax, inds) + + if cat_spec_wh: + wh = wh.view(batch, K, cat, 8) + clses_ind = clses.view(batch, K, 1, 1).expand(batch, K, 1, 8).long() + wh = wh.gather(2, clses_ind).view(batch, K, 8) + else: + wh = wh.view(batch, K, 8) + clses = clses.view(batch, K, 1).float() + scores = scores.view(batch, K, 1) + + bboxes_vec = [ + xs - wh[..., 0:1], ys - wh[..., 1:2], xs - wh[..., 2:3], + ys - wh[..., 3:4], xs - wh[..., 4:5], ys - wh[..., 5:6], + xs - wh[..., 6:7], ys - wh[..., 7:8] + ] + bboxes = torch.cat(bboxes_vec, dim=2) + + cc_match = torch.cat( + [(xs - wh[..., 0:1]) + width * torch.round(ys - wh[..., 1:2]), + (xs - wh[..., 2:3]) + width * torch.round(ys - wh[..., 3:4]), + (xs - wh[..., 4:5]) + width * torch.round(ys - wh[..., 5:6]), + (xs - wh[..., 6:7]) + width * torch.round(ys - wh[..., 7:8])], + dim=2) + + cc_match = torch.round(cc_match).to(torch.int64) + + cr_feat = _get_4ps_feat(cc_match, cr) + cr_feat = cr_feat.sum(axis=3) + + detections = torch.cat([bboxes, scores, clses], dim=2) + + return detections, keep, ax, cr_feat + + +def get_3rd_point(a, b): + direct = a - b + return b + np.array([-direct[1], direct[0]], dtype=np.float32) + + +def affine_transform(pt, t): + new_pt = np.array([pt[0], pt[1], 1.], dtype=np.float32).T + new_pt = np.dot(t, new_pt) + return new_pt[:2] + + +def get_dir(src_point, rot_rad): + sn, cs = np.sin(rot_rad), np.cos(rot_rad) + + src_result = [0, 0] + src_result[0] = src_point[0] * cs - src_point[1] * sn + src_result[1] = src_point[0] * sn + src_point[1] * cs + + return src_result + + +def get_affine_transform(center, + scale, + rot, + output_size, + shift=np.array([0, 0], dtype=np.float32), + inv=0): + if not isinstance(scale, np.ndarray) and not isinstance(scale, list): + scale = np.array([scale, scale], dtype=np.float32) + + scale_tmp = scale + src_w = scale_tmp[0] + dst_w = output_size[0] + dst_h = output_size[1] + + rot_rad = np.pi * rot / 180 + src_dir = get_dir([0, src_w * -0.5], rot_rad) + dst_dir = np.array([0, dst_w * -0.5], np.float32) + + src = np.zeros((3, 2), dtype=np.float32) + dst = np.zeros((3, 2), dtype=np.float32) + src[0, :] = center + scale_tmp * shift # [0,0] # + src[1, :] = center + src_dir + scale_tmp * shift # scale # + dst[0, :] = [dst_w * 0.5, dst_h * 0.5] # [0,0] # + dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5], + np.float32) + dst_dir # output_size # + + src[2:, :] = get_3rd_point(src[0, :], src[1, :]) + dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :]) + + if inv: + trans = cv2.getAffineTransform(np.float32(dst), np.float32(src)) + else: + trans = cv2.getAffineTransform(np.float32(src), np.float32(dst)) + + return trans + + +def get_affine_transform_upper_left(center, + scale, + rot, + output_size, + shift=np.array([0, 0], dtype=np.float32), + inv=0): + if not isinstance(scale, np.ndarray) and not isinstance(scale, list): + scale = np.array([scale, scale], dtype=np.float32) + + src = np.zeros((3, 2), dtype=np.float32) + dst = np.zeros((3, 2), dtype=np.float32) + src[0, :] = center + dst[0, :] = [0, 0] + if center[0] < center[1]: + src[1, :] = [scale[0], center[1]] + dst[1, :] = [output_size[0], 0] + else: + src[1, :] = [center[0], scale[0]] + dst[1, :] = [0, output_size[0]] + src[2:, :] = get_3rd_point(src[0, :], src[1, :]) + dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :]) + + if inv: + trans = cv2.getAffineTransform(np.float32(dst), np.float32(src)) + else: + trans = cv2.getAffineTransform(np.float32(src), np.float32(dst)) + + return trans + + +def transform_preds(coords, center, scale, output_size, rot=0): + target_coords = np.zeros(coords.shape) + trans = get_affine_transform(center, scale, rot, output_size, inv=1) + for p in range(coords.shape[0]): + target_coords[p, 0:2] = affine_transform(coords[p, 0:2], trans) + return target_coords + + +def transform_preds_upper_left(coords, center, scale, output_size, rot=0): + target_coords = np.zeros(coords.shape) + + trans = get_affine_transform_upper_left( + center, scale, rot, output_size, inv=1) + for p in range(coords.shape[0]): + target_coords[p, 0:2] = affine_transform(coords[p, 0:2], trans) + return target_coords + + +def ctdet_4ps_post_process_upper_left(dets, c, s, h, w, num_classes, rot=0): + # dets: batch x max_dets x dim + # return 1-based class det dict + ret = [] + for i in range(dets.shape[0]): + top_preds = {} + dets[i, :, 0:2] = transform_preds_upper_left(dets[i, :, 0:2], c[i], + s[i], (w, h), rot) + dets[i, :, 2:4] = transform_preds_upper_left(dets[i, :, 2:4], c[i], + s[i], (w, h), rot) + dets[i, :, 4:6] = transform_preds_upper_left(dets[i, :, 4:6], c[i], + s[i], (w, h), rot) + dets[i, :, 6:8] = transform_preds_upper_left(dets[i, :, 6:8], c[i], + s[i], (w, h), rot) + classes = dets[i, :, -1] + for j in range(num_classes): + inds = (classes == j) + tmp_top_pred = [ + dets[i, inds, :8].astype(np.float32), + dets[i, inds, 8:9].astype(np.float32) + ] + top_preds[j + 1] = np.concatenate(tmp_top_pred, axis=1).tolist() + ret.append(top_preds) + return ret + + +def ctdet_corner_post_process(corner_st_reg, c, s, h, w, num_classes): + for i in range(corner_st_reg.shape[0]): + corner_st_reg[i, :, 0:2] = transform_preds(corner_st_reg[i, :, 0:2], + c[i], s[i], (w, h)) + corner_st_reg[i, :, 2:4] = transform_preds(corner_st_reg[i, :, 2:4], + c[i], s[i], (w, h)) + corner_st_reg[i, :, 4:6] = transform_preds(corner_st_reg[i, :, 4:6], + c[i], s[i], (w, h)) + corner_st_reg[i, :, 6:8] = transform_preds(corner_st_reg[i, :, 6:8], + c[i], s[i], (w, h)) + corner_st_reg[i, :, 8:10] = transform_preds(corner_st_reg[i, :, 8:10], + c[i], s[i], (w, h)) + return corner_st_reg + + +def merge_outputs(detections): + # thresh_conf, thresh_min, thresh_max = 0.1, 0.5, 0.7 + num_classes, max_per_image = 2, 3000 + results = {} + for j in range(1, num_classes + 1): + results[j] = np.concatenate([detection[j] for detection in detections], + axis=0).astype(np.float32) + scores = np.hstack([results[j][:, 8] for j in range(1, num_classes + 1)]) + if len(scores) > max_per_image: + kth = len(scores) - max_per_image + thresh = np.partition(scores, kth)[kth] + for j in range(1, num_classes + 1): + keep_inds = (results[j][:, 8] >= thresh) + results[j] = results[j][keep_inds] + return results + + +def filter(results, logi, ps): + # this function select boxes + batch_size, feat_dim = logi.shape[0], logi.shape[2] + + num_valid = sum(results[1][:, 8] >= 0.15) + + slct_logi = np.zeros((batch_size, num_valid, feat_dim), dtype=np.float32) + slct_dets = np.zeros((batch_size, num_valid, 8), dtype=np.int32) + for i in range(batch_size): + for j in range(num_valid): + slct_logi[i, j, :] = logi[i, j, :].cpu() + slct_dets[i, j, :] = ps[i, j, :].cpu() + + return torch.Tensor(slct_logi).cuda(), torch.Tensor(slct_dets).cuda() + + +def normalized_ps(ps, vocab_size): + ps = torch.round(ps).to(torch.int64) + ps = torch.where(ps < vocab_size, ps, (vocab_size - 1) + * torch.ones(ps.shape).to(torch.int64).cuda()) + ps = torch.where(ps >= 0, ps, torch.zeros(ps.shape).to(torch.int64).cuda()) + return ps + + +def process_detect_output(output, meta): + K, MK = 3000, 5000 + num_classes = 2 + scale = 1.0 + + hm = output['hm'].sigmoid_() + wh = output['wh'] + reg = output['reg'] + st = output['st'] + ax = output['ax'] + cr = output['cr'] + + scores, inds, ys, xs, st_reg, corner_dict = corner_decode( + hm[:, 1:2, :, :], st, reg, K=MK) + dets, keep, logi, cr = ctdet_4ps_decode( + hm[:, 0:1, :, :], wh, ax, cr, corner_dict, reg=reg, K=K, wiz_rev=False) + raw_dets = dets + dets = dets.detach().cpu().numpy() + dets = dets.reshape(1, -1, dets.shape[2]) + dets = ctdet_4ps_post_process_upper_left(dets.copy(), + [meta['c'].cpu().numpy()], + [meta['s']], meta['out_height'], + meta['out_width'], 2) + for j in range(1, num_classes + 1): + dets[0][j] = np.array(dets[0][j], dtype=np.float32).reshape(-1, 9) + dets[0][j][:, :8] /= scale + dets = dets[0] + detections = [dets] + + logi = logi + cr + results = merge_outputs(detections) + slct_logi_feat, slct_dets_feat = filter(results, logi, raw_dets[:, :, :8]) + slct_dets_feat = normalized_ps(slct_dets_feat, 256) + slct_output_dets = results[1][:slct_logi_feat.shape[1], :8] + + return slct_logi_feat, slct_dets_feat, slct_output_dets + + +def process_logic_output(logi): + logi_floor = logi.floor() + dev = logi - logi_floor + logi = torch.where(dev > 0.5, logi_floor + 1, logi_floor) + + return logi + + +def load_lore_model(model, checkpoint, mtype): + state_dict_ = checkpoint['state_dict'] + state_dict = {} + # convert data_parallal to model + for k in state_dict_: + if k.startswith('module') and not k.startswith('module_list'): + state_dict[k[7:]] = state_dict_[k] + else: + if mtype == 'model': + if k.startswith('model'): + state_dict[k[6:]] = state_dict_[k] + else: + continue + else: + if k.startswith('processor'): + state_dict[k[10:]] = state_dict_[k] + else: + continue + model_state_dict = model.state_dict() + # check loaded parameters and created model parameters + for k in state_dict: + if k in model_state_dict: + if state_dict[k].shape != model_state_dict[k].shape: + print('Skip loading parameter {}, required shape{}, ' + 'loaded shape{}.'.format(k, model_state_dict[k].shape, + state_dict[k].shape)) + state_dict[k] = model_state_dict[k] + else: + print('Drop parameter {}.'.format(k)) + for k in model_state_dict: + if not (k in state_dict): + print('No param {}.'.format(k)) + state_dict[k] = model_state_dict[k] + model.load_state_dict(state_dict, strict=False) diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/model_lore.py b/runtime/Lib/site_packages/models/cv/table_recognition/model_lore.py new file mode 100644 index 0000000000000000000000000000000000000000..d21b3fbeda81150a6174dad5f2682562c803c2d1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/table_recognition/model_lore.py @@ -0,0 +1,88 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy +import math +from os.path import join +from typing import Any, Dict + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .lineless_table_process import (get_affine_transform, + get_affine_transform_upper_left, + load_lore_model, process_detect_output, + process_logic_output) +from .modules.lore_detector import LoreDetectModel +from .modules.lore_processor import LoreProcessModel + +LOGGER = get_logger() + + +@MODELS.register_module(Tasks.lineless_table_recognition, + Models.lineless_table_recognition) +class LoreModel(TorchModel): + ''' + The model first locates table cells in the input image by key point segmentation. + Then the logical locations are predicted along with the spatial locations + employing two cascading regressors. + See details in paper "LORE: Logical Location Regression Network for Table Structure Recognition" + (https://arxiv.org/abs/2303.03730). + ''' + + def __init__(self, model_dir: str, **kwargs): + '''initialize the LORE model from the `model_dir` path. + + Args: + model_dir (str): the model path. + ''' + super(LoreModel, self).__init__() + + model_path = join(model_dir, ModelFile.TORCH_MODEL_FILE) + checkpoint = torch.load(model_path, map_location='cpu') + # init detect infer model + self.detect_infer_model = LoreDetectModel() + load_lore_model(self.detect_infer_model, checkpoint, 'model') + # init process infer model + self.process_infer_model = LoreProcessModel() + load_lore_model(self.process_infer_model, checkpoint, 'processor') + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """ + Args: + img (`torch.Tensor`): image tensor, + shape of each tensor is [3, H, W]. + + Return: + dets (`torch.Tensor`): the locations of detected table cells, + shape of each tensor is [N_cell, 8]. + dets (`torch.Tensor`): the logical coordinates of detected table cells, + shape of each tensor is [N_cell, 4]. + meta (`Dict`): the meta info of original image. + """ + outputs = self.detect_infer_model(input['img']) + output = outputs[-1] + meta = input['meta'] + slct_logi_feat, slct_dets_feat, slct_output_dets = process_detect_output( + output, meta) + _, slct_logi = self.process_infer_model( + slct_logi_feat, dets=slct_dets_feat.to(torch.int64)) + return { + 'dets': slct_output_dets, + 'logi': slct_logi, + 'meta': input['meta'] + } + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + slct_dets = inputs['dets'] + slct_logi = process_logic_output(inputs['logi']) + result = { + OutputKeys.POLYGONS: slct_dets, + OutputKeys.BOXES: np.array(slct_logi[0].cpu().numpy()) + } + return result diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/modules/__init__.py b/runtime/Lib/site_packages/models/cv/table_recognition/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_detector.py b/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..d7e75a4fe73f2b0311320a456f5df267a949265b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_detector.py @@ -0,0 +1,385 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from CenterNet, +# made publicly available under the MIT License at https://github.com/xingyizhou/CenterNet.git +# ------------------------------------------------------------------------------ + +import copy +import math +from os.path import join + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=0, + bias=False) + + +class ChannelAttention(nn.Module): + + def __init__(self, in_planes, ratio=16): + super(ChannelAttention, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.max_pool = nn.AdaptiveMaxPool2d(1) + + self.fc1 = nn.Conv2d(in_planes, in_planes // ratio, 1, bias=False) + self.relu1 = nn.ReLU() + self.fc2 = nn.Conv2d(in_planes // ratio, in_planes, 1, bias=False) + + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x)))) + max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x)))) + + out = avg_out + max_out + + return self.sigmoid(out) + + +class SpatialAttention(nn.Module): + + def __init__(self): + super(SpatialAttention, self).__init__() + + self.conv1 = nn.Conv2d(2, 1, 3, padding=1, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.BN_MOMENTUM = 0.1 + self.conv1 = nn.Conv2d( + inplanes, planes, kernel_size=3, stride=stride, padding=1) + self.bn1 = nn.BatchNorm2d(planes, momentum=self.BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm2d(planes, momentum=self.BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + self.planes = planes + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + if self.downsample is not None: + residual = self.downsample(residual) + + out += residual + out = self.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.BN_MOMENTUM = 0.1 + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes, momentum=self.BN_MOMENTUM) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(planes, momentum=self.BN_MOMENTUM) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d( + planes * self.expansion, momentum=self.BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class LoreDetectModel(nn.Module): + """ + A key point-based detector with ResNet backbone. In this model, it is trained for table cell detection. + See details in paper "LORE: Logical Location Regression Network for Table Structure Recognition" + (https://arxiv.org/abs/2303.03730) + """ + + def __init__(self, **kwargs): + ''' + Args: + ''' + self.BN_MOMENTUM = 0.1 + self.inplanes = 64 + self.deconv_with_bias = False + self.block = BasicBlock + self.layers = [2, 2, 2, 2] + self.heads = { + 'hm': 2, + 'st': 8, + 'wh': 8, + 'ax': 256, + 'cr': 256, + 'reg': 2 + } + self.head_conv = 64 + + super(LoreDetectModel, self).__init__() + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64, momentum=self.BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer( + self.block, 64, self.layers[0], stride=2) + self.layer2 = self._make_layer( + self.block, 128, self.layers[1], stride=2) + self.layer3 = self._make_layer( + self.block, 256, self.layers[2], stride=2) + self.layer4 = self._make_layer( + self.block, 256, self.layers[3], stride=2) + + self.adaption3 = nn.Conv2d( + 256, 256, kernel_size=1, stride=1, padding=0, bias=False) + self.adaption2 = nn.Conv2d( + 128, 256, kernel_size=1, stride=1, padding=0, bias=False) + self.adaption1 = nn.Conv2d( + 64, 256, kernel_size=1, stride=1, padding=0, bias=False) + self.adaption0 = nn.Conv2d( + 64, 256, kernel_size=1, stride=1, padding=0, bias=False) + + self.adaptionU1 = nn.Conv2d( + 256, 256, kernel_size=1, stride=1, padding=0, bias=False) + + # used for deconv layers + self.deconv_layers1 = self._make_deconv_layer( + 1, + [256], + [4], + ) + self.deconv_layers2 = self._make_deconv_layer( + 1, + [256], + [4], + ) + self.deconv_layers3 = self._make_deconv_layer( + 1, + [256], + [4], + ) + self.deconv_layers4 = self._make_deconv_layer( + 1, + [256], + [4], + ) + + self.hm_maxpool = nn.MaxPool2d(kernel_size=3, stride=1, padding=1) + self.hm_sigmoid = nn.Sigmoid() + self.mk_maxpool = nn.MaxPool2d(kernel_size=3, stride=1, padding=1) + self.mk_sigmoid = nn.Sigmoid() + + for head in sorted(self.heads): + num_output = self.heads[head] + if self.head_conv > 0 and (head == 'reg' or head == 'mk_reg'): + inchannel = 256 + fc = nn.Sequential( + nn.Conv2d( + inchannel, + self.head_conv, + kernel_size=3, + padding=1, + bias=True), nn.ReLU(inplace=True), + nn.Conv2d( + self.head_conv, + num_output, + kernel_size=1, + stride=1, + padding=0)) + elif self.head_conv > 0: + inchannel = 256 + fc = nn.Sequential( + nn.Conv2d( + inchannel, + self.head_conv, + kernel_size=3, + padding=1, + bias=True), nn.ReLU(inplace=True), + nn.Conv2d( + self.head_conv, + self.head_conv, + kernel_size=3, + padding=1, + bias=True), nn.ReLU(inplace=True), + nn.Conv2d( + self.head_conv, + self.head_conv, + kernel_size=3, + padding=1, + bias=True), nn.ReLU(inplace=True), + nn.Conv2d( + self.head_conv, + self.head_conv, + kernel_size=3, + padding=1, + bias=True), nn.ReLU(inplace=True), + nn.Conv2d( + self.head_conv, + num_output, + kernel_size=1, + stride=1, + padding=0)) + else: + inchannel = 256 + fc = nn.Conv2d( + in_channels=inchannel, + out_channels=num_output, + kernel_size=1, + stride=1, + padding=0) + self.__setattr__(head, fc) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d( + planes * block.expansion, momentum=self.BN_MOMENTUM), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _get_deconv_cfg(self, deconv_kernel, index): + if deconv_kernel == 4: + padding = 1 + output_padding = 0 + elif deconv_kernel == 3: + padding = 1 + output_padding = 1 + elif deconv_kernel == 2: + padding = 0 + output_padding = 0 + elif deconv_kernel == 7: + padding = 3 + output_padding = 0 + + return deconv_kernel, padding, output_padding + + def _make_deconv_layer(self, num_layers, num_filters, num_kernels): + assert num_layers == len(num_filters), \ + 'ERROR: num_deconv_layers is different len(num_deconv_filters)' + assert num_layers == len(num_kernels), \ + 'ERROR: num_deconv_layers is different len(num_deconv_filters)' + + layers = [] + for i in range(num_layers): + kernel, padding, output_padding = \ + self._get_deconv_cfg(num_kernels[i], i) + + planes = num_filters[i] + layers.append( + nn.ConvTranspose2d( + in_channels=self.inplanes, + out_channels=planes, + kernel_size=kernel, + stride=2, + padding=padding, + output_padding=output_padding, + bias=self.deconv_with_bias)) + layers.append(nn.BatchNorm2d(planes, momentum=self.BN_MOMENTUM)) + layers.append(nn.ReLU(inplace=True)) + self.inplanes = planes + + return nn.Sequential(*layers) + + def forward(self, x): + """ + Args: + x : Input image, a tensor of [batch_size, channel, w, h]. + + Returns: + ret : A dict of tensors, the keys are corresponding to the keys of head as initialized, + and the value is tensors of [batch_size, dim_key ,w, h], + where dim_key is different according to different keys. For example, + in this implementation, the dim_keys are 2, 8, 8, 256, 256, 2. + """ + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x0 = self.maxpool(x) + x1 = self.layer1(x0) + x2 = self.layer2(x1) + x3 = self.layer3(x2) + x4 = self.layer4(x3) + + x3_ = self.deconv_layers1(x4) + x3_ = self.adaption3(x3) + x3_ + + x2_ = self.deconv_layers2(x3_) + x2_ = self.adaption2(x2) + x2_ + + x1_ = self.deconv_layers3(x2_) + x1_ = self.adaption1(x1) + x1_ + + x0_ = self.deconv_layers4(x1_) + self.adaption0(x0) + x0_ = self.adaptionU1(x0_) + + ret = {} + + for head in self.heads: + ret[head] = self.__getattr__(head)(x0_) + return [ret] diff --git a/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_processor.py b/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..c643b334cf5f67c46d44418f198892db807db7da --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/table_recognition/modules/lore_processor.py @@ -0,0 +1,440 @@ +# ------------------------------------------------------------------------------ +# Part of implementation is adopted from CenterNet, +# made publicly available under the MIT License at https://github.com/xingyizhou/CenterNet.git +# ------------------------------------------------------------------------------ + +import copy +import math +from os.path import join + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + + +def get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class Encoder(nn.Module): + + def __init__(self, input_size, hidden_size, N, heads, dropout): + super().__init__() + self.N = N + self.pe = PositionalEncoder(hidden_size, dropout=dropout) + self.layers = get_clones(EncoderLayer(hidden_size, heads, dropout), N) + self.norm = Norm(hidden_size) + + def forward(self, x, mask=None, require_att=False): + att = None + for i in range(self.N): + if mask is None: + if i == (self.N - 1): + x, att = self.layers[i](x, require_att=True) + else: + x = self.layers[i](x) + else: + x = self.layers[i](x, mask) + if require_att: + return x, att + else: + return x + + +class Decoder(nn.Module): + + def __init__(self, hidden_size, output_size): + super(Decoder, self).__init__() + self.linear = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + nn.ReLU(inplace=True), + nn.Linear(hidden_size, output_size), + nn.ReLU(inplace=True) # newly added + ) + + def forward(self, x): + out = self.linear(x) + return out + + +class Transformer(nn.Module): + + def __init__(self, input_size, hidden_size, output_size, n_layers, heads, + dropout): + super().__init__() + self.linear = nn.Linear(input_size, hidden_size) + self.encoder = Encoder(input_size, hidden_size, n_layers, heads, + dropout) + self.decoder = Decoder(hidden_size, output_size) + + def forward(self, x, mask=None, require_att=False): + x = self.linear(x) + att = None + if mask is None: + # evaluation model + if require_att: + embedding, att = self.encoder(x, require_att=True) + else: + embedding = self.encoder(x) + + output = self.decoder(embedding) + + if require_att: + return output, att + else: + return output + else: + if require_att: + embedding, att = self.encoder(x, mask, require_att=True) + else: + embedding = self.encoder(x, mask) + + output = self.decoder(embedding) + return output + + +class Norm(nn.Module): + + def __init__(self, d_model, eps=1e-6): + super().__init__() + + self.size = d_model + # create two learnable parameters to calibrate normalisation + self.alpha = nn.Parameter(torch.ones(self.size)) + self.bias = nn.Parameter(torch.zeros(self.size)) + self.eps = eps + + def forward(self, x): + norm = self.alpha * (x - x.mean(dim=-1, keepdim=True)) \ + / (x.std(dim=-1, keepdim=True) + self.eps) + self.bias + return norm + + +def attention(q, k, v, d_k, mask=None, dropout=None): + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(d_k) + + if mask is not None: + if len(mask.shape) == 2: + mask = mask.unsqueeze(1) + mask = mask.unsqueeze(3) + mask = mask.to(torch.float32) + mask2d = torch.matmul(mask, mask.transpose(-2, -1)).expand( + scores.shape[0], scores.shape[1], scores.shape[2], + scores.shape[3]) + elif len(mask.shape) == 3: + mask = mask.unsqueeze(1) + mask = mask.to(torch.float32) + mask2d = mask.expand(scores.shape[0], scores.shape[1], + scores.shape[2], scores.shape[3]) + + scores = scores.masked_fill(mask2d == 0, -1e9) + + scores = F.softmax(scores, dim=-1) + if dropout is not None: + scores = dropout(scores) + + output = torch.matmul(scores, v) + return output + + +def attention_score(q, k, v, d_k): + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(d_k) + scores = F.softmax(scores, dim=-1) + return scores + + +class MultiHeadAttention(nn.Module): + + def __init__(self, heads, d_model, dropout=0.1): + super().__init__() + + self.d_model = d_model + self.d_k = d_model // heads + self.h = heads + + self.q_linear = nn.Linear(d_model, d_model) + self.v_linear = nn.Linear(d_model, d_model) + self.k_linear = nn.Linear(d_model, d_model) + + self.dropout = nn.Dropout(dropout) + self.out = nn.Linear(d_model, d_model) + + def attention_map(self, q, k, v, mask=None): + bs = q.size(0) + + # perform linear operation and split into N heads + k = self.k_linear(k).view(bs, -1, self.h, self.d_k) + q = self.q_linear(q).view(bs, -1, self.h, self.d_k) + v = self.v_linear(v).view(bs, -1, self.h, self.d_k) + + # transpose to get dimensions bs * N * sl * d_model + k = k.transpose(1, 2) + q = q.transpose(1, 2) + v = v.transpose(1, 2) + + scores = attention_score(q, k, v, self.d_k) + + return scores + + def forward(self, q, k, v, mask=None): + + bs = q.size(0) + + # perform linear operation and split into N heads + k = self.k_linear(k).view(bs, -1, self.h, self.d_k) + q = self.q_linear(q).view(bs, -1, self.h, self.d_k) + v = self.v_linear(v).view(bs, -1, self.h, self.d_k) + + # transpose to get dimensions bs * N * sl * d_model + k = k.transpose(1, 2) + q = q.transpose(1, 2) + v = v.transpose(1, 2) + + # calculate attention using function we will define next + scores = attention(q, k, v, self.d_k, mask, self.dropout) + # concatenate heads and put through final linear layer + + concat = scores.transpose(1, 2).contiguous() \ + .view(bs, -1, self.d_model) + output = self.out(concat) + + return output + + +class FeedForward(nn.Module): + + def __init__(self, d_model, d_ff=2048, dropout=0.1): + super().__init__() + + # We set d_ff as a default to 2048 + self.linear_1 = nn.Linear(d_model, d_ff) + self.dropout = nn.Dropout(dropout) + self.linear_2 = nn.Linear(d_ff, d_model) + + def forward(self, x): + x = self.dropout(F.relu(self.linear_1(x))) + x = self.linear_2(x) + return x + + +class Embedder(nn.Module): + + def __init__(self, vocab_size, d_model): + super().__init__() + self.d_model = d_model + self.embed = nn.Embedding(vocab_size, d_model) + + def forward(self, x): + return self.embed(x) + + +class PositionalEncoder(nn.Module): + + def __init__(self, d_model, max_seq_len=900, dropout=0.1): + super().__init__() + self.d_model = d_model + self.dropout = nn.Dropout(dropout) + # create constant 'pe' matrix with values dependant on + # pos and i + pe = torch.zeros(max_seq_len, d_model) + for pos in range(max_seq_len): + for i in range(0, d_model, 2): + sin_coef = 10000**((2 * i) / d_model) + cos_coef = 10000**((2 * (i + 1)) / d_model) + pe[pos, i] = math.sin(pos / sin_coef) + pe[pos, i + 1] = math.cos(pos / cos_coef) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + + def forward(self, x): + # make embeddings relatively larger + x = x * math.sqrt(self.d_model) + # add constant to embedding + seq_len = x.size(1) + pe = Variable(self.pe[:, :seq_len], requires_grad=False) + if x.is_cuda: + pe.cuda() + x = x + pe + return self.dropout(x) + + +class EncoderLayer(nn.Module): + + def __init__(self, d_model, heads, dropout=0.1): + super().__init__() + self.norm_1 = Norm(d_model) + self.norm_2 = Norm(d_model) + self.attn = MultiHeadAttention(heads, d_model, dropout=dropout) + self.ff = FeedForward(d_model, dropout=dropout) + self.dropout_1 = nn.Dropout(dropout) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x, mask=None, require_att=False): + x2 = self.norm_1(x) + xc = x2.clone() + + if mask is None: + x = x + self.dropout_1(self.attn(x2, x2, x2)) + else: + x = x + self.dropout_1(self.attn(x2, x2, x2, mask)) + + x2 = self.norm_2(x) + x = x + self.dropout_2(self.ff(x2)) + + if require_att: + att = self.attn.attention_map(xc, xc, xc) + return x, att + else: + return x + + +class DecoderLayer(nn.Module): + + def __init__(self, d_model, heads, dropout=0.1): + super().__init__() + self.norm_1 = Norm(d_model) + self.norm_2 = Norm(d_model) + self.norm_3 = Norm(d_model) + + self.dropout_1 = nn.Dropout(dropout) + self.dropout_2 = nn.Dropout(dropout) + self.dropout_3 = nn.Dropout(dropout) + + self.attn_1 = MultiHeadAttention(heads, d_model, dropout=dropout) + self.attn_2 = MultiHeadAttention(heads, d_model, dropout=dropout) + self.ff = FeedForward(d_model, dropout=dropout) + + def forward(self, x, e_outputs, src_mask, trg_mask): + x2 = self.norm_1(x) + x = x + self.dropout_1(self.attn_1(x2, x2, x2, trg_mask)) + x2 = self.norm_2(x) + x = x + self.dropout_2(self.attn_2(x2, e_outputs, e_outputs, src_mask)) + x2 = self.norm_3(x) + x = x + self.dropout_3(self.ff(x2)) + return x + + +class Stacker(nn.Module): + ''' + The architecture of the stacking regressor, which takes the dense representations and + logical locations of table cells to make more accurate prediction of logical locations. + ''' + + def __init__(self, + input_size, + hidden_size, + output_size, + layers, + heads=8, + dropout=0.1): + """ + Args: + input_size : The dim of logical locations which is always 4. + hidden_size : The dim of hidden states which is 256 by default. + output_size : The dim of logical locations which is always 4. + layers : Number of layers of self-attention mechanism, which is 4 in this implementation. + """ + super(Stacker, self).__init__() + self.logi_encoder = nn.Sequential( + nn.Linear(input_size, hidden_size), nn.ReLU(inplace=True), + nn.Linear(hidden_size, hidden_size), nn.ReLU(inplace=True)) + self.tsfm = Transformer(2 * hidden_size, hidden_size, output_size, + layers, heads, dropout) + + def forward(self, outputs, logi, mask=None, require_att=False): + """ + Args: + outputs : The dense representation of table cells, a tensor of [batch_size, number_of_objects, hidden_size]. + logi : The logical location of table cells, a tensor of [batch_size, number_of_objects, 4]. + mask : The mask of cells, a tensor of [batch_size, number_of_objects], not None only in training stage. + require_att : If True, the model will also generate the attention maps of table cells. + + Returns: + stacked_axis : The predicted logical location of cells, a tensor of [batch_size, number_of_objects, 4]. + att : The attention map of table cells. + """ + logi_embeddings = self.logi_encoder(logi) + + cat_embeddings = torch.cat((logi_embeddings, outputs), dim=2) + + if mask is None: + if require_att: + stacked_axis, att = self.tsfm(cat_embeddings) + else: + stacked_axis = self.tsfm(cat_embeddings) + else: + stacked_axis = self.tsfm(cat_embeddings, mask=mask) + + if require_att: + return stacked_axis, att + else: + return stacked_axis + + +class LoreProcessModel(nn.Module): + ''' + The logical location prediction head of LORE. It contains a base regressor and a stacking regressor. + They both consist of several self-attention blocks. + See details in paper "LORE: Logical Location Regression Network for Table Structure Recognition" + (https://arxiv.org/abs/2303.03730). + ''' + + def __init__(self, **kwargs): + ''' + Args: + ''' + super(LoreProcessModel, self).__init__() + + self.input_size = 256 + self.output_size = 4 + self.hidden_size = 256 + self.max_fmp_size = 256 + self.stacking_layers = 4 + self.tsfm_layers = 4 + self.num_heads = 8 + self.att_dropout = 0.1 + self.stacker = Stacker(self.output_size, self.hidden_size, + self.output_size, self.stacking_layers) + self.tsfm_axis = Transformer(self.input_size, self.hidden_size, + self.output_size, self.tsfm_layers, + self.num_heads, self.att_dropout) + self.x_position_embeddings = nn.Embedding(self.max_fmp_size, + self.hidden_size) + self.y_position_embeddings = nn.Embedding(self.max_fmp_size, + self.hidden_size) + + def forward(self, outputs, batch=None, cc_match=None, dets=None): + """ + Args: + outputs : The dense representation of table cells from the detection part of LORE, + a tensor of [batch_size, number_of_objects, hidden_size]. + batch : The detection results of other source, such as external OCR systems. + dets : The detection results of each table cells, a tensor of [batch_size, number_of_objects, 8]. + + Returns: + logi_axis : The output logical location of base regressor, + a tensor of [batch_size, number_of_objects, 4]. + stacked_axis : The output logical location of stacking regressor, + a tensor of [batch_size, number_of_objects, 4]. + """ + if batch is None: + # evaluation mode + vis_feat = outputs + + if batch is None: + if dets is None: + logic_axis = self.tsfm_axis(vis_feat) + stacked_axis = self.stacker(vis_feat, logic_axis) + else: + left_pe = self.x_position_embeddings(dets[:, :, 0]) + upper_pe = self.y_position_embeddings(dets[:, :, 1]) + right_pe = self.x_position_embeddings(dets[:, :, 2]) + lower_pe = self.y_position_embeddings(dets[:, :, 5]) + feat = vis_feat + left_pe + upper_pe + right_pe + lower_pe + + logic_axis = self.tsfm_axis(feat) + + stacked_axis = self.stacker(feat, logic_axis) + + return logic_axis, stacked_axis diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/__init__.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aefaa6984f575d1b0a2fef9a99e511838a0000bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .lseg_base import TextDrivenSegmentation diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/clip.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..1cec5f397147a022a1964e0a18b60cadbf8c18c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/clip.py @@ -0,0 +1,169 @@ +# CLIP +# Adapted from https://github.com/openai/CLIP. +# Originally MIT License, Copyright (c) 2021 OpenAI. + +import hashlib +import os +import urllib +import warnings +from typing import Any, List, Union + +import torch +from PIL import Image +from pkg_resources import packaging +from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, + ToTensor) +from tqdm import tqdm + +from .model import build_model +from .simple_tokenizer import SimpleTokenizer as _Tokenizer + +try: + from torchvision.transforms import InterpolationMode + BICUBIC = InterpolationMode.BICUBIC +except ImportError: + BICUBIC = Image.BICUBIC + +if packaging.version.parse( + torch.__version__) < packaging.version.parse('1.7.1'): + warnings.warn('PyTorch version 1.7.1 or higher is recommended') +__all__ = ['load', 'tokenize'] + + +def _convert_image_to_rgb(image): + return image.convert('RGB') + + +def _transform(n_px): + return Compose([ + Resize(n_px, interpolation=BICUBIC), + CenterCrop(n_px), + _convert_image_to_rgb, + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)), + ]) + + +def load(name: str, + device: Union[str, torch.device] = 'cuda' + if torch.cuda.is_available() else 'cpu', + jit: bool = False, + root: str = None): + + if not jit: + model = build_model().to(device) + if str(device) == 'cpu': + model.float() + return model, _transform(model.visual.input_resolution) + + # patch the device names + device_holder = torch.jit.trace( + lambda: torch.ones([]).to(torch.device(device)), example_inputs=[]) + device_node = [ + n for n in device_holder.graph.findAllNodes('prim::Constant') + if 'Device' in repr(n) + ][-1] + + def patch_device(module): + try: + graphs = [module.graph] if hasattr(module, 'graph') else [] + except RuntimeError: + graphs = [] + + if hasattr(module, 'forward1'): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes('prim::Constant'): + if 'value' in node.attributeNames() and str( + node['value']).startswith('cuda'): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_image) + patch_device(model.encode_text) + + # patch dtype to float32 on CPU + if str(device) == 'cpu': + float_holder = torch.jit.trace( + lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode('aten::to').inputs())[1] + float_node = float_input.node() + + def patch_float(module): + try: + graphs = [module.graph] if hasattr(module, 'graph') else [] + except RuntimeError: + graphs = [] + + if hasattr(module, 'forward1'): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes('aten::to'): + inputs = list(node.inputs()) + for i in [ + 1, 2 + ]: # dtype can be the second or third argument to aten::to() + if inputs[i].node()['value'] == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_image) + patch_float(model.encode_text) + + model.float() + + return model, _transform(model.input_resolution.item()) + + +def tokenize( + _tokenizer, + texts: Union[str, List[str]], + context_length: int = 77, + truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]: + """ + Returns the tokenized representation of given input string(s) + + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + + context_length : int + The context length to use; all CLIP models use 77 as the context length + + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]. + We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long. + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = _tokenizer.encoder['<|startoftext|>'] + eot_token = _tokenizer.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] + for text in texts] + if packaging.version.parse( + torch.__version__) < packaging.version.parse('1.8.0'): + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + else: + result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError( + f'Input {texts[i]} is too long for context length {context_length}' + ) + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_base.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_base.py new file mode 100644 index 0000000000000000000000000000000000000000..c79861a7d414d888f13c93e6b1695bcdcd4aa4e9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_base.py @@ -0,0 +1,26 @@ +# Adapted from https://github.com/isl-org/lang-seg. +# Originally MIT License, Copyright (c) 2021 Intelligent Systems Lab Org. + +import torch +import torch.nn as nn + +from .lseg_net import LSeg + + +class TextDrivenSegmentation(nn.Module): + + def __init__(self, model_dir): + super(TextDrivenSegmentation, self).__init__() + self.net = LSeg(model_dir=model_dir) + self.model_dir = model_dir + + def forward(self, img, txt_list): + b = img.size()[0] + batch_name_list = txt_list + xout_list = [] + for i in range(b): + labelset = ['others', batch_name_list[i]] + xout = self.net(img[i:i + 1], labelset=labelset) + xout_list.append(xout) + score_map = torch.cat(xout_list, dim=0) + return score_map diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_blocks.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..56d4a65de2363fe47c348619382b29e81f7e935e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_blocks.py @@ -0,0 +1,332 @@ +# Adapted from https://github.com/isl-org/lang-seg. +# Originally MIT License, Copyright (c) 2021 Intelligent Systems Lab Org. + +import torch +import torch.nn as nn + +from .lseg_vit import _make_pretrained_clip_vitl16_384, forward_vit + + +def _make_encoder( + backbone, + features, + use_pretrained=True, + groups=1, + expand=False, + exportable=True, + hooks=None, + use_vit_only=False, + use_readout='ignore', + enable_attention_hooks=False, +): + if backbone == 'clip_vitl16_384': + clip_pretrained, pretrained = _make_pretrained_clip_vitl16_384( + use_pretrained, + hooks=hooks, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + scratch = _make_scratch([256, 512, 1024, 1024], + features, + groups=groups, + expand=expand) + else: + raise NotImplementedError(f"Backbone '{backbone}' not implemented") + + return clip_pretrained, pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand is True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + + return scratch + + +class Interpolate(nn.Module): + """Interpolation module.""" + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block.""" + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode='bilinear', align_corners=True) + + return output + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + if self.bn is True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn is True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn is True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + ): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand is True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1, + ) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners) + + output = self.out_conv(output) + return output diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_model.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3813563ab2633c7da22839c5ddf39a3ef8f77d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_model.py @@ -0,0 +1,109 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.text_driven_segmentation import \ + TextDrivenSegmentation +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() +__all__ = ['TextDrivenSeg'] + + +@MODELS.register_module( + Tasks.text_driven_segmentation, + module_name=Models.text_driven_segmentation) +class TextDrivenSeg(TorchModel): + """ text driven segmentation model. + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.model = TextDrivenSegmentation(model_dir=model_dir) + pretrained_params = torch.load('{}/{}'.format( + model_dir, ModelFile.TORCH_MODEL_BIN_FILE)) + self.model.load_state_dict(pretrained_params) + self.model.eval() + if device_id >= 0 and torch.cuda.is_available(): + self.model.to('cuda:{}'.format(device_id)) + logger.info('Use GPU: {}'.format(device_id)) + else: + device_id = -1 + logger.info('Use CPU for inference') + self.device_id = device_id + + def preprocess(self, img, size=640): + mean = [0.48145466, 0.4578275, 0.40821073] + std = [0.26862954, 0.26130258, 0.27577711] + h, w, c = img.shape + max_hw = max(h, w) + ratio = 1.0 * size / max_hw + crop_h, crop_w = int(ratio * h), int(ratio * w) + pil_img = Image.fromarray(img) + pil_img = pil_img.resize((crop_w, crop_h), Image.BILINEAR) + np_img = np.array(pil_img, dtype=np.float32) / 255. + for j in range(3): + np_img[:, :, j] = (np_img[:, :, j] - mean[j]) / std[j] + img_pad = np.zeros((size, size, 3), dtype=np.float32) + img_pad[:crop_h, :crop_w] = np_img + img_pad = torch.from_numpy(img_pad).permute(2, 0, + 1).unsqueeze(0).float() + return img_pad, h, w, crop_h, crop_w + + def postprocess(self, tensors, crop_h, crop_w, ori_h, ori_w): + output = np.clip(tensors * 255., a_min=0, a_max=255.) + crop_output = np.array(output[:crop_h, :crop_w], dtype=np.uint8) + pil_output = Image.fromarray(crop_output) + pil_output = pil_output.resize((ori_w, ori_h), Image.BILINEAR) + np_output = np.array(pil_output, dtype=np.uint8) + np_output[np_output < 128] = 0 + np_output[np_output >= 128] = 255 + np_output = np.uint8(np_output) + return np_output + + def forward(self, image, text): + """ + image should be numpy array, dtype=np.uint8, shape: height*width*3 + """ + image_tensor, ori_h, ori_w, crop_h, crop_w = self.preprocess( + image, size=640) + pred = self.inference(image_tensor, text) + msk = self.postprocess(pred, crop_h, crop_w, ori_h, ori_w, size=640) + outputs = {OutputKeys.MASKS: msk} + return outputs + + def inference(self, image, text): + """ + image should be tensor, 1 * 3 * 640 * 640 + """ + with torch.no_grad(): + if self.device_id == -1: + output = self.model(image, [text]) + else: + device = torch.device('cuda', self.device_id) + output = self.model(image.to(device), [text]) + output = F.interpolate(output, size=(640, 640), mode='bilinear') + output = F.softmax(output, dim=1) + output = torch.argmax(output, dim=1) + output = output[0] + if self.device_id == -1: + pred = output.data.numpy() + else: + pred = output.data.cpu().numpy() + del output + return pred diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_net.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_net.py new file mode 100644 index 0000000000000000000000000000000000000000..541a4a386e7c0ffa387d7b875d3d355ae96b2a35 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_net.py @@ -0,0 +1,195 @@ +# Adapted from https://github.com/isl-org/lang-seg. +# Originally MIT License, Copyright (c) 2021 Intelligent Systems Lab Org. + +import numpy as np +import torch +import torch.nn as nn + +from . import clip +from .lseg_blocks import (FeatureFusionBlock, FeatureFusionBlock_custom, + Interpolate, _make_encoder, forward_vit) +from .simple_tokenizer import SimpleTokenizer + + +class depthwise_clipseg_conv(nn.Module): + + def __init__(self): + super(depthwise_clipseg_conv, self).__init__() + self.depthwise = nn.Conv2d(1, 1, kernel_size=3, padding=1) + + def depthwise_clipseg(self, x, channels): + x = torch.cat( + [self.depthwise(x[:, i].unsqueeze(1)) for i in range(channels)], + dim=1) + return x + + def forward(self, x): + channels = x.shape[1] + out = self.depthwise_clipseg(x, channels) + return out + + +class depthwise_conv(nn.Module): + + def __init__(self, kernel_size=3, stride=1, padding=1): + super(depthwise_conv, self).__init__() + self.depthwise = nn.Conv2d( + 1, 1, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x): + # support for 4D tensor with NCHW + C, H, W = x.shape[1:] + x = x.reshape(-1, 1, H, W) + x = self.depthwise(x) + x = x.view(-1, C, H, W) + return x + + +class depthwise_block(nn.Module): + + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(depthwise_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + def forward(self, x, act=True): + x = self.depthwise(x) + if act: + x = self.activation(x) + return x + + +class bottleneck_block(nn.Module): + + def __init__(self, kernel_size=3, stride=1, padding=1, activation='relu'): + super(bottleneck_block, self).__init__() + self.depthwise = depthwise_conv(kernel_size=3, stride=1, padding=1) + if activation == 'relu': + self.activation = nn.ReLU() + elif activation == 'lrelu': + self.activation = nn.LeakyReLU() + elif activation == 'tanh': + self.activation = nn.Tanh() + + def forward(self, x, act=True): + sum_layer = x.max(dim=1, keepdim=True)[0] + x = self.depthwise(x) + x = x + sum_layer + if act: + x = self.activation(x) + return x + + +class BaseModel(torch.nn.Module): + + def load(self, path): + """Load model from file. + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device('cpu')) + + if 'optimizer' in parameters: + parameters = parameters['model'] + + self.load_state_dict(parameters) + + +def _make_fusion_block(features, use_bn): + return FeatureFusionBlock_custom( + features, + activation=nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + ) + + +class LSeg(BaseModel): + + def __init__( + self, + features=256, + backbone='clip_vitl16_384', + readout='project', + use_bn=True, + model_dir=None, + ): + super(LSeg, self).__init__() + hooks = { + 'clip_vitl16_384': [5, 11, 17, 23], + } + + # Instantiate backbone and reassemble blocks + self.clip_pretrained, self.pretrained, self.scratch = _make_encoder( + backbone, + features, + groups=1, + expand=False, + exportable=False, + hooks=hooks[backbone], + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.logit_scale = nn.Parameter(torch.ones([]) + * np.log(1 / 0.07)).exp() + self.out_c = 512 + self.scratch.head1 = nn.Conv2d(features, self.out_c, kernel_size=1) + + self.scratch.output_conv = nn.Sequential( + Interpolate(scale_factor=2, mode='bilinear', align_corners=True), ) + + self.tau = 0.07 + self.model_dir = model_dir + self.tokenizer = SimpleTokenizer(model_dir + + '/bpe_simple_vocab_16e6.txt.gz') + + def forward(self, x, labelset=''): + text = clip.tokenize(self.tokenizer, labelset) + + layer_1, layer_2, layer_3, layer_4 = forward_vit(self.pretrained, x) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + text = text.to(x.device) + text_features = self.clip_pretrained.encode_text(text) + + image_features = self.scratch.head1(path_1) + + imshape = image_features.shape + image_features = image_features.permute(0, 2, 3, + 1).reshape(-1, self.out_c) + + # normalized features + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + logits_per_image = image_features @ text_features.t() / self.tau + + out = logits_per_image.float().view(imshape[0], imshape[2], imshape[3], + -1).permute(0, 3, 1, 2) + + out = self.scratch.output_conv(out) + + return out diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_vit.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..5298832f3a5092c912f8b3b7df928fe32b734b7e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/lseg_vit.py @@ -0,0 +1,541 @@ +# Adapted from https://github.com/isl-org/lang-seg. +# Originally MIT License, Copyright (c) 2021 Intelligent Systems Lab Org. + +import math +import types + +import timm +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from . import clip + +activations = {} + + +def get_activation(name): + + def hook(model, input, output): + activations[name] = output + + return hook + + +attention = {} + + +def get_attention(name): + + def hook(module, input, output): + x = input[0] + B, N, C = x.shape + qkv = ( + module.qkv(x).reshape(B, N, 3, module.num_heads, + C // module.num_heads).permute( + 2, 0, 3, 1, 4)) + q, k, _ = ( + qkv[0], + qkv[1], + qkv[2], + ) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * module.scale + + attn = attn.softmax(dim=-1) # [:,:,1,1:] + attention[name] = attn + + return hook + + +def get_mean_attention_map(attn, token, shape): + attn = attn[:, :, token, 1:] + attn = attn.unflatten(2, torch.Size([shape[2] // 16, + shape[3] // 16])).float() + attn = torch.nn.functional.interpolate( + attn, size=shape[2:], mode='bicubic', align_corners=False).squeeze(0) + + all_attn = torch.mean(attn, 0) + + return all_attn + + +class Slice(nn.Module): + + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index:] + + +class AddReadout(nn.Module): + + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index:] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential( + nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index:]) + features = torch.cat((x[:, self.start_index:], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +def forward_vit(pretrained, x): + b, c, h, w = x.shape + + # encoder + _ = pretrained.model.forward_flex(x) + + layer_1 = pretrained.activations['1'] + layer_2 = pretrained.activations['2'] + layer_3 = pretrained.activations['3'] + layer_4 = pretrained.activations['4'] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size([ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ]), + )) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3:len(pretrained.act_postprocess1)]( + layer_1) + layer_2 = pretrained.act_postprocess2[3:len(pretrained.act_postprocess2)]( + layer_2) + layer_3 = pretrained.act_postprocess3[3:len(pretrained.act_postprocess3)]( + layer_3) + layer_4 = pretrained.act_postprocess4[3:len(pretrained.act_postprocess4)]( + layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, :self.start_index], + posemb[0, self.start_index:], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, + -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate( + posemb_grid, size=(gs_h, gs_w), mode='bilinear') + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed(self.pos_embed, h // self.patch_size[1], + w // self.patch_size[0]) + + B = x.shape[0] + + if hasattr(self.patch_embed, 'backbone'): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[ + -1] # last feature if backbone outputs list/tuple of features + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, 'dist_token', None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + pos_embed + x = self.pos_drop(x) + + gradient_checkpoint = False + for blk in self.blocks: + if gradient_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + + x = self.norm(x) + + return x + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == 'ignore': + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == 'add': + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == 'project': + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def adapt_input_conv(in_chans, conv_weight): + conv_type = conv_weight.dtype + conv_weight = conv_weight.float( + ) # Some weights are in torch.half, ensure it's float for sum on CPU + O, II, J, K = conv_weight.shape + if in_chans == 1: + if II > 3: + assert conv_weight.shape[1] % 3 == 0 + # For models with space2depth stems + conv_weight = conv_weight.reshape(O, II // 3, 3, J, K) + conv_weight = conv_weight.sum(dim=2, keepdim=False) + else: + conv_weight = conv_weight.sum(dim=1, keepdim=True) + elif in_chans != 3: + if II != 3: + raise NotImplementedError( + 'Weight format not supported by conversion.') + else: + # NOTE this strategy should be better than random init, but there could be other combinations of + # the original RGB input layer weights that'd work better for specific cases. + repeat = int(math.ceil(in_chans / 3)) + conv_weight = conv_weight.repeat(1, repeat, 1, + 1)[:, :in_chans, :, :] + conv_weight *= (3 / float(in_chans)) + conv_weight = conv_weight.to(conv_type) + return conv_weight + + +@torch.no_grad() +def _load_weights(model, checkpoint_path, prefix=''): + """ Load weights from .npz checkpoints for official Google Brain Flax implementation + """ + import numpy as np + + def _n2p(w, t=True): + if w.ndim == 4 and w.shape[0] == w.shape[1] == w.shape[2] == 1: + w = w.flatten() + if t: + if w.ndim == 4: + w = w.transpose([3, 2, 0, 1]) + elif w.ndim == 3: + w = w.transpose([2, 0, 1]) + elif w.ndim == 2: + w = w.transpose([1, 0]) + return torch.from_numpy(w) + + w = np.load(checkpoint_path) + if not prefix and 'opt/target/embedding/kernel' in w: + prefix = 'opt/target/' + + if hasattr(model.patch_embed, 'backbone'): + # hybrid + backbone = model.patch_embed.backbone + stem_only = not hasattr(backbone, 'stem') + stem = backbone if stem_only else backbone.stem + stem.conv.weight.copy_( + adapt_input_conv(stem.conv.weight.shape[1], + _n2p(w[f'{prefix}conv_root/kernel']))) + stem.norm.weight.copy_(_n2p(w[f'{prefix}gn_root/scale'])) + stem.norm.bias.copy_(_n2p(w[f'{prefix}gn_root/bias'])) + if not stem_only: + for i, stage in enumerate(backbone.stages): + for j, block in enumerate(stage.blocks): + bp = f'{prefix}block{i + 1}/unit{j + 1}/' + for r in range(3): + getattr(block, f'conv{r + 1}').weight.copy_( + _n2p(w[f'{bp}conv{r + 1}/kernel'])) + getattr(block, f'norm{r + 1}').weight.copy_( + _n2p(w[f'{bp}gn{r + 1}/scale'])) + getattr(block, f'norm{r + 1}').bias.copy_( + _n2p(w[f'{bp}gn{r + 1}/bias'])) + if block.downsample is not None: + block.downsample.conv.weight.copy_( + _n2p(w[f'{bp}conv_proj/kernel'])) + block.downsample.norm.weight.copy_( + _n2p(w[f'{bp}gn_proj/scale'])) + block.downsample.norm.bias.copy_( + _n2p(w[f'{bp}gn_proj/bias'])) + embed_conv_w = _n2p(w[f'{prefix}embedding/kernel']) + else: + embed_conv_w = adapt_input_conv(model.patch_embed.proj.weight.shape[1], + _n2p(w[f'{prefix}embedding/kernel'])) + model.patch_embed.proj.weight.copy_(embed_conv_w) + model.patch_embed.proj.bias.copy_(_n2p(w[f'{prefix}embedding/bias'])) + model.cls_token.copy_(_n2p(w[f'{prefix}cls'], t=False)) + pos_embed_w = _n2p( + w[f'{prefix}Transformer/posembed_input/pos_embedding'], t=False) + if pos_embed_w.shape != model.pos_embed.shape: + pos_embed_w = resize_pos_embed( # resize pos embedding when different size from pretrained weights + pos_embed_w, model.pos_embed, getattr(model, 'num_prefix_tokens', + 1), + model.patch_embed.grid_size) + model.pos_embed.copy_(pos_embed_w) + model.norm.weight.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/scale'])) + model.norm.bias.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/bias'])) + if isinstance( + model.head, nn.Linear + ) and model.head.bias.shape[0] == w[f'{prefix}head/bias'].shape[-1]: + model.head.weight.copy_(_n2p(w[f'{prefix}head/kernel'])) + model.head.bias.copy_(_n2p(w[f'{prefix}head/bias'])) + # NOTE representation layer has been removed, not used in latest 21k/1k pretrained weights + # if isinstance(getattr(model.pre_logits, 'fc', None), nn.Linear) and f'{prefix}pre_logits/bias' in w: + # model.pre_logits.fc.weight.copy_(_n2p(w[f'{prefix}pre_logits/kernel'])) + # model.pre_logits.fc.bias.copy_(_n2p(w[f'{prefix}pre_logits/bias'])) + for i, block in enumerate(model.blocks.children()): + block_prefix = f'{prefix}Transformer/encoderblock_{i}/' + mha_prefix = block_prefix + 'MultiHeadDotProductAttention_1/' + block.norm1.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/scale'])) + block.norm1.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/bias'])) + block.attn.qkv.weight.copy_( + torch.cat([ + _n2p(w[f'{mha_prefix}{n}/kernel'], t=False).flatten(1).T + for n in ('query', 'key', 'value') + ])) + block.attn.qkv.bias.copy_( + torch.cat([ + _n2p(w[f'{mha_prefix}{n}/bias'], t=False).reshape(-1) + for n in ('query', 'key', 'value') + ])) + block.attn.proj.weight.copy_( + _n2p(w[f'{mha_prefix}out/kernel']).flatten(1)) + block.attn.proj.bias.copy_(_n2p(w[f'{mha_prefix}out/bias'])) + for r in range(2): + getattr(block.mlp, f'fc{r + 1}').weight.copy_( + _n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/kernel'])) + getattr(block.mlp, f'fc{r + 1}').bias.copy_( + _n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/bias'])) + block.norm2.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/scale'])) + block.norm2.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/bias'])) + + +def resize_pos_embed(posemb, posemb_new, num_prefix_tokens=1, gs_new=()): + # Rescale the grid of position embeddings when loading from state_dict. Adapted from + # https://github.com/google-research/vision_transformer/blob/00883dd691c63a6830751563748663526e811cee/vit_jax/checkpoint.py#L224 + ntok_new = posemb_new.shape[1] + if num_prefix_tokens: + posemb_prefix, posemb_grid = posemb[:, :num_prefix_tokens], posemb[ + 0, num_prefix_tokens:] + ntok_new -= num_prefix_tokens + else: + posemb_prefix, posemb_grid = posemb[:, :0], posemb[0] + gs_old = int(math.sqrt(len(posemb_grid))) + if not len(gs_new): # backwards compatibility + gs_new = [int(math.sqrt(ntok_new))] * 2 + assert len(gs_new) >= 2 + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, + -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate( + posemb_grid, size=gs_new, mode='bicubic', align_corners=False) + posemb_grid = posemb_grid.permute(0, 2, 3, + 1).reshape(1, gs_new[0] * gs_new[1], -1) + posemb = torch.cat([posemb_prefix, posemb_grid], dim=1) + return posemb + + +def _make_pretrained_clip_vitl16_384(pretrained, + use_readout='ignore', + hooks=None, + enable_attention_hooks=False): + clip_pretrained, _ = clip.load('ViT-B/32', device='cpu', jit=False) + + # model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + model = timm.create_model('vit_large_patch16_384', pretrained=False) + hooks = [5, 11, 17, 23] if hooks is None else hooks + pretrained = _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + enable_attention_hooks=enable_attention_hooks, + ) + return clip_pretrained, pretrained + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout='ignore', + start_index=1, + enable_attention_hooks=False, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook( + get_activation('1')) + pretrained.model.blocks[hooks[1]].register_forward_hook( + get_activation('2')) + pretrained.model.blocks[hooks[2]].register_forward_hook( + get_activation('3')) + pretrained.model.blocks[hooks[3]].register_forward_hook( + get_activation('4')) + + pretrained.activations = activations + + if enable_attention_hooks: + pretrained.model.blocks[hooks[0]].attn.register_forward_hook( + get_attention('attn_1')) + pretrained.model.blocks[hooks[1]].attn.register_forward_hook( + get_attention('attn_2')) + pretrained.model.blocks[hooks[2]].attn.register_forward_hook( + get_attention('attn_3')) + pretrained.model.blocks[hooks[3]].attn.register_forward_hook( + get_attention('attn_4')) + pretrained.attention = attention + + readout_oper = get_readout_oper(vit_features, features, use_readout, + start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, + pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model) + + return pretrained diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/model.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/model.py new file mode 100644 index 0000000000000000000000000000000000000000..f98d480dad59d4c3365f6d255494b896945c204f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/model.py @@ -0,0 +1,456 @@ +# Adapted from https://github.com/isl-org/lang-seg. +# Originally MIT License, Copyright (c) 2021 Intelligent Systems Lab Org. + +from collections import OrderedDict +from typing import Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.relu1 = nn.ReLU(inplace=True) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.relu2 = nn.ReLU(inplace=True) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu3 = nn.ReLU(inplace=True) + + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu1(self.bn1(self.conv1(x))) + out = self.relu2(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu3(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x[:1], + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + return x.squeeze(0) + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.relu1 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.relu2 = nn.ReLU(inplace=True) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.relu3 = nn.ReLU(inplace=True) + self.avgpool = nn.AvgPool2d(2) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, width, layers, heads, attn_mask=None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + x1 = self.class_embedding.to(x.dtype) + x2 = torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([x1 + x2, x], dim=1) # shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int): + super().__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisionTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask()) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image): + return self.visual(image.type(self.dtype)) + + def encode_text(self, text): + x = self.token_embedding(text).type(self.dtype) + x = x + self.positional_embedding.type(self.dtype) + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + x = x[torch.arange(x.shape[0]), + text.argmax(dim=-1)] @ self.text_projection + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=1, keepdim=True) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(ll): + if isinstance(ll, (nn.Conv1d, nn.Conv2d, nn.Linear)): + ll.weight.data = ll.weight.data.half() + if ll.bias is not None: + ll.bias.data = ll.bias.data.half() + + if isinstance(ll, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(ll, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ['text_projection', 'proj']: + if hasattr(ll, name): + attr = getattr(ll, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +def build_model(): + model = CLIP(512, 224, 12, 768, 32, 77, 49408, 512, 8, 12) + convert_weights(model) + return model.eval() diff --git a/runtime/Lib/site_packages/models/cv/text_driven_segmentation/simple_tokenizer.py b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/simple_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..361d67c6db64ddab286f72e996026c586c8b51ef --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_driven_segmentation/simple_tokenizer.py @@ -0,0 +1,155 @@ +# CLIP +# Adapted from https://github.com/openai/CLIP. +# Originally MIT License, Copyright (c) 2021 OpenAI. + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def default_bpe(): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'bpe_simple_vocab_16e6.txt.gz') + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + error_list = [] + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception as err: + new_word.extend(word[i:]) + error_list.append(err) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/Tex2Texture.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/Tex2Texture.py new file mode 100644 index 0000000000000000000000000000000000000000..5e6eee3ee4935ce2948865a6b86946593b6a8b5b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/Tex2Texture.py @@ -0,0 +1,660 @@ +# Copyright © Alibaba, Inc. and its affiliates. +# The implementation here is modifed based on StableDiffusionControlNetInpaintPipeline, +# originally Apache 2.0 License and public available at +# https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py + +import os +from typing import Any, Callable, Dict, List, Optional, Union + +import cv2 +import numpy as np +import PIL +import PIL.Image as Image +import torch +import torchvision.transforms as transforms +from diffusers import (AutoencoderKL, ControlNetModel, DiffusionPipeline, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetInpaintPipeline, + StableDiffusionInpaintPipeline, StableDiffusionPipeline, + UNet2DConditionModel) +from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.utils import (deprecate, is_accelerate_available, + is_accelerate_version, is_compiled_module, + logging, randn_tensor, replace_example_docstring) +from pytorch3d.io import load_obj, load_objs_as_meshes, save_obj + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.text_texture_generation.lib2.camera import * +from modelscope.models.cv.text_texture_generation.lib2.init_view import * +from modelscope.models.cv.text_texture_generation.utils import * +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> init_image = load_image(image_path) + >>> init_image = init_image.resize((512, 512)) + >>> generator = torch.Generator(device="cpu").manual_seed(1) + >>> mask_image = load_image(mask_path) + >>> mask_image = mask_image.resize((512, 512)) + >>> def make_inpaint_condition(image, image_mask): + ... image = np.array(image.convert("RGB")).astype(np.float32) / 255.0 + ... image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0 + ... assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size" + ... image[image_mask > 0.5] = -1.0 # set as masked pixel + ... image = np.expand_dims(image, 0).transpose(0, 3, 1, 2) + ... image = torch.from_numpy(image) + ... return image + >>> control_image = make_inpaint_condition(init_image, mask_image) + >>> controlnet = ControlNetModel.from_pretrained( + ... "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16 + ... ) + >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16 + ... ) + >>> pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + >>> pipe.enable_model_cpu_offload() + >>> image = pipe( + ... "a handsome man with ray-ban sunglasses", + ... num_inference_steps=20, + ... generator=generator, + ... eta=1.0, + ... image=init_image, + ... mask_image=mask_image, + ... control_image=control_image, + ... ).images[0] + ``` +""" + + +@MODELS.register_module( + Tasks.text_texture_generation, module_name=Models.text_texture_generation) +class Tex2Texture(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + """The Tex2Texture is modified based on TEXTure and Text2Tex, publicly available at + https://github.com/TEXTurePaper/TEXTurePaper & + https://github.com/daveredrum/Text2Tex + Args: + model_dir: the root directory of the model files + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + if torch.cuda.is_available(): + self.device = torch.device('cuda') + logger.info('Use GPU: {}'.format(self.device)) + else: + print('no gpu avaiable') + exit() + + model_path = model_dir + '/base_model/' + controlmodel_path = model_dir + '/control_model/' + inpaintmodel_path = model_dir + '/inpaint_model/' + torch_dtype = kwargs.get('torch_dtype', torch.float16) + self.controlnet = ControlNetModel.from_pretrained( + controlmodel_path, torch_dtype=torch_dtype).to(self.device) + self.inpaintmodel = StableDiffusionInpaintPipeline.from_pretrained( + inpaintmodel_path, + torch_dtype=torch_dtype, + ).to(self.device) + self.pipe = StableDiffusionControlinpaintPipeline.from_pretrained( + model_path, controlnet=self.controlnet, + torch_dtype=torch_dtype).to(self.device) + logger.info('model load over') + + def init_mesh(self, mesh_path): + verts, faces, aux = load_obj(mesh_path, device=self.device) + mesh = load_objs_as_meshes([mesh_path], device=self.device) + return mesh, verts, faces, aux + + def normalize_mesh(self, mesh): + bbox = mesh.get_bounding_boxes() + num_verts = mesh.verts_packed().shape[0] + mesh_center = bbox.mean(dim=2).repeat(num_verts, 1) + mesh = mesh.offset_verts(-mesh_center) + lens = bbox[0, :, 1] - bbox[0, :, 0] + max_len = lens.max() + scale = 0.9 / max_len + scale = scale.unsqueeze(0).repeat(num_verts) + # mesh.scale_verts_(scale) + new_mesh = mesh.scale_verts(scale) + return new_mesh.verts_packed(), new_mesh, mesh_center, scale + + def save_normalized_obj(self, verts, faces, aux, path='normalized.obj'): + print('=> saving normalized mesh file...') + obj_path = path + save_obj( + obj_path, + verts=verts, + faces=faces.verts_idx, + decimal_places=5, + verts_uvs=aux.verts_uvs, + faces_uvs=faces.textures_idx, + texture_map=aux.texture_images[list(aux.texture_images.keys())[0]]) + + def mesh_normalized(self, mesh_path, save_path='normalized.obj'): + mesh, verts, faces, aux = self.init_mesh(mesh_path) + verts, mesh, mesh_center, scale = self.normalize_mesh(mesh) + self.save_normalized_obj(verts, faces, aux, save_path) + return mesh, verts, faces, aux, mesh_center, scale + + +def prepare_mask_and_masked_image(image, + mask, + height, + width, + return_image=False): + if image is None: + raise ValueError('`image` input cannot be undefined.') + + if mask is None: + raise ValueError('`mask_image` input cannot be undefined.') + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError( + f'`image` is a torch.Tensor but `mask` (type: {type(mask)} is not' + ) + + # Batch single image + if image.ndim == 3: + assert image.shape[ + 0] == 3, 'Image outside a batch should be of shape (3, H, W)' + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, 'Image and Mask must have 4 dimensions' + assert image.shape[-2:] == mask.shape[ + -2:], 'Image and Mask must have the same spatial dimensions' + assert image.shape[0] == mask.shape[ + 0], 'Image and Mask must have the same batch size' + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError('Image should be in [-1, 1] range') + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError('Mask should be in [0, 1] range') + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError( + f'`mask` is a torch.Tensor but `image` (type: {type(image)} is not' + ) + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [ + i.resize((width, height), resample=PIL.Image.LANCZOS) + for i in image + ] + image = [np.array(i.convert('RGB'))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [ + i.resize((width, height), resample=PIL.Image.LANCZOS) + for i in mask + ] + mask = np.concatenate( + [np.array(m.convert('L'))[None, None, :] for m in mask], + axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + masked_image = image * (mask < 0.5) + + # n.b. ensure backwards compatibility as old function does not return image + if return_image: + return mask, masked_image, image + + return mask, masked_image + + +class StableDiffusionControlinpaintPipeline( + StableDiffusionControlNetInpaintPipeline): + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[torch.Tensor, PIL.Image.Image] = None, + mask_image: Union[torch.Tensor, PIL.Image.Image] = None, + control_image: Union[torch.FloatTensor, PIL.Image.Image, np.ndarray, + List[torch.FloatTensor], List[PIL.Image.Image], + List[np.ndarray], ] = None, + height: Optional[int] = None, + width: Optional[int] = None, + strength: float = 1.0, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, + List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = 'pil', + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], + None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 0.5, + guess_mode: bool = False, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, + `List[List[torch.FloatTensor]]`, or `List[List[PIL.Image.Image]]`): + The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If + the type is specified as `Torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can + also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If + height and/or width are passed, `image` is resized according to them. If multiple ControlNets are + specified in init, images must be passed as a list such that each element of the list can be correctly + batched for input to a single controlnet. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + strength (`float`, *optional*, defaults to 1.): + Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be + between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the + `strength`. The number of denoising steps depends on the amount of noise initially added. When + `strength` is 1, added noise will be maximum and the denoising process will run for the full number of + iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked + portion of the reference `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5): + The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original unet. If multiple ControlNets are specified in init, you can set the + corresponding scale as a list. Note that by default, we use a smaller conditioning scale for inpainting + than for [`~StableDiffusionControlNetPipeline.__call__`]. + guess_mode (`bool`, *optional*, defaults to `False`): + In this mode, the ControlNet encoder will try best to recognize the content of the input image even if + you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + # 0. Default height and width to unet + height, width = self._default_height_width(height, width, image) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + control_image, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + controlnet_conditioning_scale, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + controlnet = self.controlnet._orig_mod if is_compiled_module( + self.controlnet) else self.controlnet + + if isinstance(controlnet, MultiControlNetModel) and isinstance( + controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale + ] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions if isinstance( + controlnet, ControlNetModel) else + controlnet.nets[0].config.global_pool_conditions) + guess_mode = guess_mode or global_pool_conditions + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get('scale', None) + if cross_attention_kwargs is not None else None) + prompt_embeds = self._encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Prepare image + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + guess_mode=guess_mode, + ) + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + else: + assert False + + # 4. Preprocess mask and image - resizes image and mask w.r.t height and width + mask, masked_image, init_image = prepare_mask_and_masked_image( + image, mask_image, height, width, return_image=True) + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps=num_inference_steps, + strength=strength, + device=device) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size + * num_images_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + num_channels_unet = self.unet.config.in_channels + return_image_latents = num_channels_unet == 4 + latents_outputs = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + image=init_image, + timestep=latent_timestep, + is_strength_max=is_strength_max, + return_noise=True, + return_image_latents=return_image_latents, + ) + + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + + # 7. Prepare mask latent variables + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + prompt_embeds.dtype, + device, + generator, + do_classifier_free_guidance, + ) + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat( + [latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t) + + # controlnet(s) inference + if guess_mode and do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input( + control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=controlnet_conditioning_scale, + guess_mode=guess_mode, + return_dict=False, + ) + + if guess_mode and do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [ + torch.cat([torch.zeros_like(d), d]) + for d in down_block_res_samples + ] + mid_block_res_sample = torch.cat([ + torch.zeros_like(mid_block_res_sample), + mid_block_res_sample + ]) + + # predict the noise residual + if num_channels_unet == 9: + latent_model_input = torch.cat( + [latent_model_input, mask, masked_image_latents], + dim=1) + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * ( + noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + **extra_step_kwargs, + return_dict=False)[0] + + if num_channels_unet == 4: + init_latents_proper = image_latents[:1] + init_mask = mask[:1] + + if i < len(timesteps) - 1: + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([t])) + + latents = (1 - init_mask + ) * init_latents_proper + init_mask * latents + + if i == len(timesteps) - 1 or ((i + 1) % self.scheduler.order + == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr( + self, + 'final_offload_hook') and self.final_offload_hook is not None: + self.unet.to('cpu') + self.controlnet.to('cpu') + torch.cuda.empty_cache() + + if not output_type == 'latent': + image = self.vae.decode( + latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker( + image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess( + image, output_type=output_type, do_denormalize=do_denormalize) + + if hasattr( + self, + 'final_offload_hook') and self.final_offload_hook is not None: + self.final_offload_hook.offload() + if not return_dict: + return (image, has_nsfw_concept) + return StableDiffusionPipelineOutput( + images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/__init__.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/__init__.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/camera.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/camera.py new file mode 100644 index 0000000000000000000000000000000000000000..cd36bcd951a7608d1331cea1f0efd9c10d58eb4b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/camera.py @@ -0,0 +1,165 @@ +# customized +import sys + +import numpy as np +import torch +from pytorch3d.renderer import PerspectiveCameras, look_at_view_transform +from sklearn.metrics.pairwise import cosine_similarity + +from modelscope.models.cv.text_texture_generation.lib2.init_view import \ + VIEWPOINTS + +sys.path.append('.') + +# ---------------- UTILS ---------------------- + + +def degree_to_radian(d): + return d * np.pi / 180 + + +def radian_to_degree(r): + return 180 * r / np.pi + + +def xyz_to_polar(xyz): + """ assume y-axis is the up axis """ + + x, y, z = xyz + + theta = 180 * np.arccos(z) / np.pi + phi = 180 * np.arccos(y) / np.pi + + return theta, phi + + +def polar_to_xyz(theta, phi, dist): + """ assume y-axis is the up axis """ + + theta = degree_to_radian(theta) + phi = degree_to_radian(phi) + + x = np.sin(phi) * np.sin(theta) * dist + y = np.cos(phi) * dist + z = np.sin(phi) * np.cos(theta) * dist + + return [x, y, z] + + +# ---------------- VIEWPOINTS ---------------------- + + +def filter_viewpoints(pre_viewpoints: dict, viewpoints: dict): + """ return the binary mask of viewpoints to be filtered """ + + filter_mask = [0 for _ in viewpoints.keys()] + for i, v in viewpoints.items(): + x_v, y_v, z_v = polar_to_xyz(v['azim'], 90 - v['elev'], v['dist']) + + for _, pv in pre_viewpoints.items(): + x_pv, y_pv, z_pv = polar_to_xyz(pv['azim'], 90 - pv['elev'], + pv['dist']) + sim = cosine_similarity( + np.array([[x_v, y_v, z_v]]), np.array([[x_pv, y_pv, z_pv]]))[0, + 0] + + if sim > 0.9: + filter_mask[i] = 1 + + return filter_mask + + +def init_viewpoints(init_dist, + init_elev, + init_azim, + use_principle=True, + use_shapenet=False, + use_objaverse=False): + sample_space = 12 + (dist_list, elev_list, azim_list, + sector_list) = init_predefined_viewpoints(sample_space, init_dist, + init_elev) + + # punishments for views -> in case always selecting the same view + view_punishments = [1 for _ in range(len(dist_list))] + + if use_principle: + (dist_list, elev_list, azim_list, sector_list, + view_punishments) = init_principle_viewpoints(dist_list, elev_list, + azim_list, sector_list, + view_punishments, + use_shapenet, + use_objaverse) + azim_list = [v - init_azim for v in azim_list] + elev_list = [v - init_elev for v in elev_list] + + return dist_list, elev_list, azim_list, sector_list, view_punishments + + +def init_principle_viewpoints(dist_list, + elev_list, + azim_list, + sector_list, + view_punishments, + use_shapenet=False, + use_objaverse=False): + if use_shapenet: + key = 'shapenet' + + pre_elev_list = [v for v in VIEWPOINTS[key]['elev']] + pre_azim_list = [v for v in VIEWPOINTS[key]['azim']] + pre_sector_list = [v for v in VIEWPOINTS[key]['sector']] + + num_principle = 10 + pre_dist_list = [dist_list[0] for _ in range(num_principle)] + pre_view_punishments = [0 for _ in range(num_principle)] + + elif use_objaverse: + key = 'objaverse' + + pre_elev_list = [v for v in VIEWPOINTS[key]['elev']] + pre_azim_list = [v for v in VIEWPOINTS[key]['azim']] + pre_sector_list = [v for v in VIEWPOINTS[key]['sector']] + + num_principle = 10 + pre_dist_list = [dist_list[0] for _ in range(num_principle)] + pre_view_punishments = [0 for _ in range(num_principle)] + else: + num_principle = 12 + pre_elev_list = [v for v in VIEWPOINTS[num_principle]['elev']] + pre_azim_list = [v for v in VIEWPOINTS[num_principle]['azim']] + pre_sector_list = [v for v in VIEWPOINTS[num_principle]['sector']] + pre_dist_list = [dist_list[0] for _ in range(num_principle)] + pre_view_punishments = [0 for _ in range(num_principle)] + + dist_list = pre_dist_list + dist_list + elev_list = pre_elev_list + elev_list + azim_list = pre_azim_list + azim_list + sector_list = pre_sector_list + sector_list + view_punishments = pre_view_punishments + view_punishments + + return dist_list, elev_list, azim_list, sector_list, view_punishments + + +def init_predefined_viewpoints(sample_space, init_dist, init_elev): + viewpoints = VIEWPOINTS[sample_space] + + assert sample_space == len(viewpoints['sector']) + + dist_list = [init_dist + for _ in range(sample_space)] # always the same dist + elev_list = [viewpoints['elev'][i] for i in range(sample_space)] + azim_list = [viewpoints['azim'][i] for i in range(sample_space)] + sector_list = [viewpoints['sector'][i] for i in range(sample_space)] + + return dist_list, elev_list, azim_list, sector_list + + +def init_camera(dist, elev, azim, image_size, device): + R, T = look_at_view_transform(dist, elev, azim) + image_size = torch.tensor([image_size, image_size]).unsqueeze(0) + T[0][2] = dist + cameras = PerspectiveCameras( + R=R, T=T, device=device, image_size=image_size) + + return cameras diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/init_view.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/init_view.py new file mode 100644 index 0000000000000000000000000000000000000000..5bc44e7a7ba1665edffd26e2f941145a59c323be --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/init_view.py @@ -0,0 +1,229 @@ +PALETTE = { + 0: [255, 255, 255], # white - background + 1: [204, 50, 50], # red - old + 2: [231, 180, 22], # yellow - update + 3: [45, 201, 55] # green - new +} + +QUAD_WEIGHTS = { + 0: 0, # background + 1: 0.1, # old + 2: 0.5, # update + 3: 1 # new +} + +VIEWPOINTS = { + 2: { + 'azim': [0, 180], + 'elev': [0, 0], + 'sector': ['front', 'back'] + }, + 4: { + 'azim': [ + 45, + 315, + 135, + 225, + ], + 'elev': [ + 0, + 0, + 0, + 0, + ], + 'sector': [ + 'front right', + 'front left', + 'back right', + 'back left', + ] + }, + 6: { + 'azim': [0, 90, 270, 0, 180, 0], + 'elev': [0, 0, 0, 90, 0, -90], + 'sector': [ + 'front', + 'right', + 'left', + 'top', + 'back', + 'bottom', + ] + }, + 10: { + 'azim': [270, 315, 225, 0, 180, 45, 135, 90, 270, 270], + 'elev': [15, 15, 15, 15, 15, 15, 15, 15, 90, -90], + 'sector': [ + 'front', + 'front right', + 'front left', + 'right', + 'left', + 'back right', + 'back left', + 'back', + 'top', + 'bottom', + ] + }, + 12: { + 'azim': [ + 0, + 45, + 315, + 135, + 225, + 180, + 45, + 315, + 90, + 270, + 90, + 270, + ], + 'elev': [ + 0, + 0, + 0, + 0, + 0, + 0, + 30, + 30, + 15, + 15, + 90, + -90, + ], + 'sector': [ + 'front', + 'front right', + 'front left', + 'back right', + 'back left', + 'back', + 'front right', + 'front left', + 'right', + 'left', + 'top', + 'bottom', + ] + }, + 36: { + 'azim': [ + 45, + 315, + 135, + 225, + 0, + 45, + 315, + 90, + 270, + 135, + 225, + 180, + 0, + 45, + 315, + 90, + 270, + 135, + 225, + 180, + 22.5, + 337.5, + 67.5, + 292.5, + 112.5, + 247.5, + 157.5, + 202.5, + 22.5, + 337.5, + 67.5, + 292.5, + 112.5, + 247.5, + 157.5, + 202.5, + ], + 'elev': [ + 0, + 0, + 0, + 0, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 60, + 60, + 60, + 60, + 60, + 60, + 60, + 60, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 45, + 45, + 45, + 45, + 45, + 45, + 45, + 45, + ], + 'sector': [ + 'front right', + 'front left', + 'back right', + 'back left', + 'front', + 'front right', + 'front left', + 'right', + 'left', + 'back right', + 'back left', + 'back', + 'top front', + 'top right', + 'top left', + 'top right', + 'top left', + 'top right', + 'top left', + 'top back', + 'front right', + 'front left', + 'front right', + 'front left', + 'back right', + 'back left', + 'back right', + 'back left', + 'front right', + 'front left', + 'front right', + 'front left', + 'back right', + 'back left', + 'back right', + 'back left', + ] + } +} diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/projection.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/projection.py new file mode 100644 index 0000000000000000000000000000000000000000..4209eae8448cbbf21fc570373880bea247b7b4a1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/projection.py @@ -0,0 +1,655 @@ +import os +import random +# customized +import sys +from typing import NamedTuple, Sequence + +import cv2 +import numpy as np +import torch +from PIL import Image +from pytorch3d.io import save_obj +from pytorch3d.ops import interpolate_face_attributes +from pytorch3d.renderer import (AmbientLights, MeshRasterizer, + MeshRendererWithFragments, + RasterizationSettings, SoftPhongShader, + TexturesUV) +from pytorch3d.renderer.mesh.shader import ShaderBase +from torchvision import transforms +from tqdm import tqdm + +from modelscope.models.cv.text_texture_generation.lib2.camera import \ + init_camera +from modelscope.models.cv.text_texture_generation.lib2.init_view import * +from modelscope.models.cv.text_texture_generation.lib2.viusel import ( + visualize_outputs, visualize_quad_mask) + +sys.path.append('.') + + +class BlendParams(NamedTuple): + sigma: float = 1e-4 + gamma: float = 1e-4 + background_color: Sequence = (1, 1, 1) + + +class FlatTexelShader(ShaderBase): + + def __init__(self, + device='cpu', + cameras=None, + lights=None, + materials=None, + blend_params=None): + super().__init__(device, cameras, lights, materials, blend_params) + + def forward(self, fragments, meshes, **_kwargs): + texels = meshes.sample_textures(fragments) + texels[(fragments.pix_to_face == -1), :] = 0 + return texels.squeeze(-2) + + +def init_soft_phong_shader(camera, blend_params, device): + lights = AmbientLights(device=device) + shader = SoftPhongShader( + cameras=camera, + lights=lights, + device=device, + blend_params=blend_params) + + return shader + + +def init_flat_texel_shader(camera, device): + shader = FlatTexelShader(cameras=camera, device=device) + return shader + + +def init_renderer(camera, shader, image_size, faces_per_pixel): + raster_settings = RasterizationSettings( + image_size=image_size, faces_per_pixel=faces_per_pixel) + renderer = MeshRendererWithFragments( + rasterizer=MeshRasterizer( + cameras=camera, raster_settings=raster_settings), + shader=shader) + + return renderer + + +@torch.no_grad() +def render(mesh, renderer, pad_value=10): + + def phong_normal_shading(meshes, fragments) -> torch.Tensor: + faces = meshes.faces_packed() # (F, 3) + vertex_normals = meshes.verts_normals_packed() # (V, 3) + faces_normals = vertex_normals[faces] + pixel_normals = interpolate_face_attributes(fragments.pix_to_face, + fragments.bary_coords, + faces_normals) + + return pixel_normals + + def similarity_shading(meshes, fragments): + faces = meshes.faces_packed() # (F, 3) + vertex_normals = meshes.verts_normals_packed() # (V, 3) + faces_normals = vertex_normals[faces] + vertices = meshes.verts_packed() # (V, 3) + face_positions = vertices[faces] + view_directions = torch.nn.functional.normalize( + (renderer.shader.cameras.get_camera_center().reshape(1, 1, 3) + - face_positions), + p=2, + dim=2) + cosine_similarity = torch.nn.CosineSimilarity(dim=2)(faces_normals, + view_directions) + pixel_similarity = interpolate_face_attributes( + fragments.pix_to_face, fragments.bary_coords, + cosine_similarity.unsqueeze(-1)) + + return pixel_similarity + + def get_relative_depth_map(fragments, pad_value=pad_value): + absolute_depth = fragments.zbuf[..., 0] # B, H, W + no_depth = -1 + + depth_min, depth_max = absolute_depth[absolute_depth != no_depth].min( + ), absolute_depth[absolute_depth != no_depth].max() + target_min, target_max = 50, 255 + + depth_value = absolute_depth[absolute_depth != no_depth] + depth_value = depth_max - depth_value # reverse values + + depth_value /= (depth_max - depth_min) + depth_value = depth_value * (target_max - target_min) + target_min + + relative_depth = absolute_depth.clone() + relative_depth[absolute_depth != no_depth] = depth_value + relative_depth[absolute_depth == no_depth] = pad_value + + return relative_depth + + images, fragments = renderer(mesh) + normal_maps = phong_normal_shading(mesh, fragments).squeeze(-2) + similarity_maps = similarity_shading(mesh, fragments).squeeze(-2) # -1 - 1 + depth_maps = get_relative_depth_map(fragments) + + # normalize similarity mask to 0 - 1 + similarity_maps = torch.abs(similarity_maps) # 0 - 1 + + # HACK erode, eliminate isolated dots + non_zero_similarity = (similarity_maps > 0).float() + non_zero_similarity = (non_zero_similarity * 255.).cpu().numpy().astype( + np.uint8)[0] + non_zero_similarity = cv2.erode( + non_zero_similarity, kernel=np.ones((3, 3), np.uint8), iterations=2) + non_zero_similarity = torch.from_numpy(non_zero_similarity).to( + similarity_maps.device).unsqueeze(0) / 255. + similarity_maps = non_zero_similarity.unsqueeze(-1) * similarity_maps + return images, normal_maps, similarity_maps, depth_maps, fragments + + +@torch.no_grad() +def check_visible_faces(mesh, fragments): + pix_to_face = fragments.pix_to_face + visible_map = pix_to_face.unique() # (num_visible_faces) + return visible_map + + +def get_all_4_locations(values_y, values_x): + y_0 = torch.floor(values_y) + y_1 = torch.ceil(values_y) + x_0 = torch.floor(values_x) + x_1 = torch.ceil(values_x) + + return torch.cat([y_0, y_0, y_1, y_1], + 0).long(), torch.cat([x_0, x_1, x_0, x_1], 0).long() + + +def compose_quad_mask(new_mask_image, update_mask_image, old_mask_image, + device): + """ + compose quad mask: + -> 0: background + -> 1: old + -> 2: update + -> 3: new + """ + + new_mask_tensor = transforms.ToTensor()(new_mask_image).to(device) + update_mask_tensor = transforms.ToTensor()(update_mask_image).to(device) + old_mask_tensor = transforms.ToTensor()(old_mask_image).to(device) + + all_mask_tensor = new_mask_tensor + update_mask_tensor + old_mask_tensor + + quad_mask_tensor = torch.zeros_like(all_mask_tensor) + quad_mask_tensor[old_mask_tensor == 1] = 1 + quad_mask_tensor[update_mask_tensor == 1] = 2 + quad_mask_tensor[new_mask_tensor == 1] = 3 + + return old_mask_tensor, update_mask_tensor, new_mask_tensor, all_mask_tensor, quad_mask_tensor + + +def compute_view_heat(similarity_tensor, quad_mask_tensor): + num_total_pixels = quad_mask_tensor.reshape(-1).shape[0] + heat = 0 + for idx in QUAD_WEIGHTS: + heat += (quad_mask_tensor + == idx).sum() * QUAD_WEIGHTS[idx] / num_total_pixels + + return heat + + +def select_viewpoint(selected_view_ids, + view_punishments, + mode, + dist_list, + elev_list, + azim_list, + sector_list, + view_idx, + similarity_texture_cache, + exist_texture, + mesh, + faces, + verts_uvs, + image_size, + faces_per_pixel, + init_image_dir, + mask_image_dir, + normal_map_dir, + depth_map_dir, + similarity_map_dir, + device, + use_principle=False): + if mode == 'sequential': + + num_views = len(dist_list) + + dist = dist_list[view_idx % num_views] + elev = elev_list[view_idx % num_views] + azim = azim_list[view_idx % num_views] + sector = sector_list[view_idx % num_views] + + selected_view_ids.append(view_idx % num_views) + + elif mode == 'heuristic': + + if use_principle and view_idx < 6: + + selected_view_idx = view_idx + + else: + + selected_view_idx = None + max_heat = 0 + + print('=> selecting next view...') + view_heat_list = [] + for sample_idx in tqdm(range(len(dist_list))): + + view_heat, *_ = render_one_view_and_build_masks( + dist_list[sample_idx], elev_list[sample_idx], + azim_list[sample_idx], sample_idx, sample_idx, + view_punishments, similarity_texture_cache, exist_texture, + mesh, faces, verts_uvs, image_size, faces_per_pixel, + init_image_dir, mask_image_dir, normal_map_dir, + depth_map_dir, similarity_map_dir, device) + + if view_heat > max_heat: + selected_view_idx = sample_idx + max_heat = view_heat + + view_heat_list.append(view_heat.item()) + + print(view_heat_list) + print('select view {} with heat {}'.format(selected_view_idx, + max_heat)) + + dist = dist_list[selected_view_idx] + elev = elev_list[selected_view_idx] + azim = azim_list[selected_view_idx] + sector = sector_list[selected_view_idx] + + selected_view_ids.append(selected_view_idx) + + view_punishments[selected_view_idx] *= 0.01 + + elif mode == 'random': + + selected_view_idx = random.choice(range(len(dist_list))) + + dist = dist_list[selected_view_idx] + elev = elev_list[selected_view_idx] + azim = azim_list[selected_view_idx] + sector = sector_list[selected_view_idx] + + selected_view_ids.append(selected_view_idx) + + else: + raise NotImplementedError() + + return dist, elev, azim, sector, selected_view_ids, view_punishments + + +@torch.no_grad() +def build_backproject_mask(mesh, faces, verts_uvs, cameras, reference_image, + faces_per_pixel, image_size, uv_size, device): + # construct pixel UVs + renderer_scaled = init_renderer( + cameras, + shader=init_soft_phong_shader( + camera=cameras, blend_params=BlendParams(), device=device), + image_size=image_size, + faces_per_pixel=faces_per_pixel) + fragments_scaled = renderer_scaled.rasterizer(mesh) + + # get UV coordinates for each pixel + faces_verts_uvs = verts_uvs[faces.textures_idx] + + pixel_uvs = interpolate_face_attributes(fragments_scaled.pix_to_face, + fragments_scaled.bary_coords, + faces_verts_uvs) # NxHsxWsxKx2 + pixel_uvs = pixel_uvs.permute(0, 3, 1, 2, 4).reshape(-1, 2) + + texture_locations_y, texture_locations_x = get_all_4_locations( + (1 - pixel_uvs[:, 1]).reshape(-1) * (uv_size - 1), + pixel_uvs[:, 0].reshape(-1) * (uv_size - 1)) + + K = faces_per_pixel + + texture_values = torch.from_numpy( + np.array(reference_image.resize( + (image_size, image_size)))).float() / 255. + texture_values = texture_values.to(device).unsqueeze(0).expand( + [4, -1, -1, -1]).unsqueeze(0).expand([K, -1, -1, -1, -1]) + + # texture + texture_tensor = torch.zeros(uv_size, uv_size, 3).to(device) + texture_tensor[texture_locations_y, + texture_locations_x, :] = texture_values.reshape(-1, 3) + + return texture_tensor[:, :, 0] + + +@torch.no_grad() +def build_diffusion_mask(mesh_stuff, + renderer, + exist_texture, + similarity_texture_cache, + target_value, + device, + image_size, + smooth_mask=False, + view_threshold=0.01): + mesh, faces, verts_uvs = mesh_stuff + mask_mesh = mesh.clone() # NOTE in-place operation - DANGER!!! + + # visible mask => the whole region + exist_texture_expand = exist_texture.unsqueeze(0).unsqueeze(-1).expand( + -1, -1, -1, 3).to(device) + mask_mesh.textures = TexturesUV( + maps=torch.ones_like(exist_texture_expand), + faces_uvs=faces.textures_idx[None, ...], + verts_uvs=verts_uvs[None, ...], + sampling_mode='nearest') + # visible_mask_tensor, *_ = render(mask_mesh, renderer) + visible_mask_tensor, _, similarity_map_tensor, *_ = render( + mask_mesh, renderer) + # faces that are too rotated away from the viewpoint will be treated as invisible + valid_mask_tensor = (similarity_map_tensor >= view_threshold).float() + visible_mask_tensor *= valid_mask_tensor + + # nonexist mask <=> new mask + exist_texture_expand = exist_texture.unsqueeze(0).unsqueeze(-1).expand( + -1, -1, -1, 3).to(device) + mask_mesh.textures = TexturesUV( + maps=1 - exist_texture_expand, + faces_uvs=faces.textures_idx[None, ...], + verts_uvs=verts_uvs[None, ...], + sampling_mode='nearest') + new_mask_tensor, *_ = render(mask_mesh, renderer) + new_mask_tensor *= valid_mask_tensor + + # exist mask => visible mask - new mask + exist_mask_tensor = visible_mask_tensor - new_mask_tensor + exist_mask_tensor[ + exist_mask_tensor < 0] = 0 # NOTE dilate can lead to overflow + + # all update mask + mask_mesh.textures = TexturesUV( + maps=( + similarity_texture_cache.argmax(0) == target_value + # # only consider the views that have already appeared before + # similarity_texture_cache[0:target_value+1].argmax(0) == target_value + ).float().unsqueeze(0).unsqueeze(-1).expand(-1, -1, -1, 3).to(device), + faces_uvs=faces.textures_idx[None, ...], + verts_uvs=verts_uvs[None, ...], + sampling_mode='nearest') + all_update_mask_tensor, *_ = render(mask_mesh, renderer) + + # current update mask => intersection between all update mask and exist mask + update_mask_tensor = exist_mask_tensor * all_update_mask_tensor + + # keep mask => exist mask - update mask + old_mask_tensor = exist_mask_tensor - update_mask_tensor + + # convert + new_mask = new_mask_tensor[0].cpu().float().permute(2, 0, 1) + new_mask = transforms.ToPILImage()(new_mask).convert('L') + + update_mask = update_mask_tensor[0].cpu().float().permute(2, 0, 1) + update_mask = transforms.ToPILImage()(update_mask).convert('L') + + old_mask = old_mask_tensor[0].cpu().float().permute(2, 0, 1) + old_mask = transforms.ToPILImage()(old_mask).convert('L') + + exist_mask = exist_mask_tensor[0].cpu().float().permute(2, 0, 1) + exist_mask = transforms.ToPILImage()(exist_mask).convert('L') + + return new_mask, update_mask, old_mask, exist_mask + + +@torch.no_grad() +def render_one_view(mesh, dist, elev, azim, image_size, faces_per_pixel, + device): + # render the view + # print(image_size) + cameras = init_camera(dist, elev, azim, image_size, device) + renderer = init_renderer( + cameras, + shader=init_soft_phong_shader( + camera=cameras, blend_params=BlendParams(), device=device), + image_size=image_size, + faces_per_pixel=faces_per_pixel) + + init_images_tensor, normal_maps_tensor, similarity_tensor, depth_maps_tensor, fragments = render( + mesh, renderer) + # print(init_images_tensor.shape, torch.max(init_images_tensor), torch.min(init_images_tensor)) + cv2.imwrite('img.png', + (np.array(init_images_tensor.squeeze(0)[:, :, :3].cpu()) + * 255).astype(np.uint8)) + return (cameras, renderer, init_images_tensor, normal_maps_tensor, + similarity_tensor, depth_maps_tensor, fragments) + + +@torch.no_grad() +def build_similarity_texture_cache_for_all_views(mesh, faces, verts_uvs, + dist_list, elev_list, + azim_list, image_size, + image_size_scaled, uv_size, + faces_per_pixel, device): + num_candidate_views = len(dist_list) + similarity_texture_cache = torch.zeros(num_candidate_views, uv_size, + uv_size).to(device) + + print('=> building similarity texture cache for all views...') + for i in tqdm(range(num_candidate_views)): + cameras, _, _, _, similarity_tensor, _, _ = render_one_view( + mesh, dist_list[i], elev_list[i], azim_list[i], image_size, + faces_per_pixel, device) + + similarity_texture_cache[i] = build_backproject_mask( + mesh, faces, verts_uvs, cameras, + transforms.ToPILImage()(similarity_tensor[0, :, :, + 0]).convert('RGB'), + faces_per_pixel, image_size_scaled, uv_size, device) + + return similarity_texture_cache + + +@torch.no_grad() +def render_one_view_and_build_masks(dist, + elev, + azim, + selected_view_idx, + view_idx, + view_punishments, + similarity_texture_cache, + exist_texture, + mesh, + faces, + verts_uvs, + image_size, + faces_per_pixel, + init_image_dir, + mask_image_dir, + normal_map_dir, + depth_map_dir, + similarity_map_dir, + device, + save_intermediate=False, + smooth_mask=False, + view_threshold=0.01): + # render the view + (cameras, renderer, init_images_tensor, normal_maps_tensor, + similarity_tensor, depth_maps_tensor, + fragments) = render_one_view(mesh, dist, elev, azim, image_size, + faces_per_pixel, device) + + init_image = init_images_tensor[0].cpu() + init_image = init_image.permute(2, 0, 1) + init_image = transforms.ToPILImage()(init_image).convert('RGB') + + normal_map = normal_maps_tensor[0].cpu() + normal_map = normal_map.permute(2, 0, 1) + normal_map = transforms.ToPILImage()(normal_map).convert('RGB') + + depth_map = depth_maps_tensor[0].cpu().numpy() + depth_map = Image.fromarray(depth_map).convert('L') + + similarity_map = similarity_tensor[0, :, :, 0].cpu() + similarity_map = transforms.ToPILImage()(similarity_map).convert('L') + + flat_renderer = init_renderer( + cameras, + shader=init_flat_texel_shader(camera=cameras, device=device), + image_size=image_size, + faces_per_pixel=faces_per_pixel) + new_mask_image, update_mask_image, old_mask_image, exist_mask_image = build_diffusion_mask( + (mesh, faces, verts_uvs), + flat_renderer, + exist_texture, + similarity_texture_cache, + selected_view_idx, + device, + image_size, + smooth_mask=smooth_mask, + view_threshold=view_threshold) + # NOTE the view idx is the absolute idx in the sample space (i.e. `selected_view_idx`) + # it should match with `similarity_texture_cache` + + (old_mask_tensor, update_mask_tensor, new_mask_tensor, all_mask_tensor, + quad_mask_tensor) = compose_quad_mask(new_mask_image, update_mask_image, + old_mask_image, device) + + view_heat = compute_view_heat(similarity_tensor, quad_mask_tensor) + view_heat *= view_punishments[selected_view_idx] + + # save intermediate results + if save_intermediate: + init_image.save( + os.path.join(init_image_dir, '{}.png'.format(view_idx))) + normal_map.save( + os.path.join(normal_map_dir, '{}.png'.format(view_idx))) + depth_map.save(os.path.join(depth_map_dir, '{}.png'.format(view_idx))) + similarity_map.save( + os.path.join(similarity_map_dir, '{}.png'.format(view_idx))) + + new_mask_image.save( + os.path.join(mask_image_dir, '{}_new.png'.format(view_idx))) + update_mask_image.save( + os.path.join(mask_image_dir, '{}_update.png'.format(view_idx))) + old_mask_image.save( + os.path.join(mask_image_dir, '{}_old.png'.format(view_idx))) + exist_mask_image.save( + os.path.join(mask_image_dir, '{}_exist.png'.format(view_idx))) + + visualize_quad_mask(mask_image_dir, quad_mask_tensor, view_idx, + view_heat, device) + + return (view_heat, renderer, cameras, fragments, init_image, normal_map, + depth_map, init_images_tensor, normal_maps_tensor, + depth_maps_tensor, similarity_tensor, old_mask_image, + update_mask_image, new_mask_image, old_mask_tensor, + update_mask_tensor, new_mask_tensor, all_mask_tensor, + quad_mask_tensor) + + +def save_full_obj(output_dir, obj_name, verts, faces, verts_uvs, faces_uvs, + projected_texture, device): + print('=> saving OBJ file...') + texture_map = transforms.ToTensor()(projected_texture).to(device) + texture_map = texture_map.permute(1, 2, 0) + obj_path = os.path.join(output_dir, obj_name) + + save_obj( + obj_path, + verts=verts, + faces=faces, + decimal_places=5, + verts_uvs=verts_uvs, + faces_uvs=faces_uvs, + texture_map=texture_map) + + +@torch.no_grad() +def backproject_from_image(mesh, faces, verts_uvs, cameras, reference_image, + new_mask_image, update_mask_image, init_texture, + exist_texture, image_size, uv_size, faces_per_pixel, + device): + # construct pixel UVs + renderer_scaled = init_renderer( + cameras, + shader=init_soft_phong_shader( + camera=cameras, blend_params=BlendParams(), device=device), + image_size=image_size, + faces_per_pixel=faces_per_pixel) + fragments_scaled = renderer_scaled.rasterizer(mesh) + + # get UV coordinates for each pixel + faces_verts_uvs = verts_uvs[faces.textures_idx] + + pixel_uvs = interpolate_face_attributes(fragments_scaled.pix_to_face, + fragments_scaled.bary_coords, + faces_verts_uvs) # NxHsxWsxKx2 + pixel_uvs = pixel_uvs.permute(0, 3, 1, 2, + 4).reshape(pixel_uvs.shape[-2], + pixel_uvs.shape[1], + pixel_uvs.shape[2], 2) + + # the update mask has to be on top of the diffusion mask + new_mask_image_tensor = transforms.ToTensor()(new_mask_image).to( + device).unsqueeze(-1) + update_mask_image_tensor = transforms.ToTensor()(update_mask_image).to( + device).unsqueeze(-1) + + project_mask_image_tensor = torch.logical_or( + update_mask_image_tensor, new_mask_image_tensor).float() + project_mask_image = project_mask_image_tensor * 255. + project_mask_image = Image.fromarray( + project_mask_image[0, :, :, 0].cpu().numpy().astype(np.uint8)) + + project_mask_image_scaled = project_mask_image.resize( + (image_size, image_size), ) + # Image.Resampling.NEAREST + # ) + project_mask_image_tensor_scaled = transforms.ToTensor()( + project_mask_image_scaled).to(device) + + pixel_uvs_masked = pixel_uvs[project_mask_image_tensor_scaled == 1] + + texture_locations_y, texture_locations_x = get_all_4_locations( + (1 - pixel_uvs_masked[:, 1]).reshape(-1) * (uv_size - 1), + pixel_uvs_masked[:, 0].reshape(-1) * (uv_size - 1)) + + K = pixel_uvs.shape[0] + project_mask_image_tensor_scaled = project_mask_image_tensor_scaled[:, + None, :, :, + None].repeat( + 1, + 4, + 1, + 1, + 3) + + texture_values = torch.from_numpy( + np.array(reference_image.resize((image_size, image_size)))) + texture_values = texture_values.to(device).unsqueeze(0).expand( + [4, -1, -1, -1]).unsqueeze(0).expand([K, -1, -1, -1, -1]) + + texture_values_masked = texture_values.reshape( + -1, 3)[project_mask_image_tensor_scaled.reshape(-1, 3) == 1].reshape( + -1, 3) + + # texture + texture_tensor = torch.from_numpy(np.array(init_texture)).to(device) + texture_tensor[texture_locations_y, + texture_locations_x, :] = texture_values_masked + + init_texture = Image.fromarray(texture_tensor.cpu().numpy().astype( + np.uint8)) + + # update texture cache + exist_texture[texture_locations_y, texture_locations_x] = 1 + + return init_texture, project_mask_image, exist_texture diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/viusel.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/viusel.py new file mode 100644 index 0000000000000000000000000000000000000000..faf466bb24b31d4f3050a873912a54ce5ef50b47 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/lib2/viusel.py @@ -0,0 +1,268 @@ +import os +import sys + +import imageio.v2 as imageio +# visualization +import matplotlib +import matplotlib.cm as cm +import matplotlib.pyplot as plt +import numpy as np +import torch +from PIL import Image + +from modelscope.models.cv.text_texture_generation.lib2.camera import \ + polar_to_xyz +from modelscope.models.cv.text_texture_generation.lib2.init_view import * + +matplotlib.use('Agg') + +sys.path.append('.') + + +def visualize_quad_mask(mask_image_dir, quad_mask_tensor, view_idx, view_score, + device): + quad_mask_tensor = quad_mask_tensor.unsqueeze(-1).repeat(1, 1, 1, 3) + quad_mask_image_tensor = torch.zeros_like(quad_mask_tensor) + + for idx in PALETTE: + selected = quad_mask_tensor[quad_mask_tensor == idx].reshape(-1, 3) + selected = torch.FloatTensor( + PALETTE[idx]).to(device).unsqueeze(0).repeat(selected.shape[0], 1) + + quad_mask_image_tensor[quad_mask_tensor == idx] = selected.reshape(-1) + + quad_mask_image_np = quad_mask_image_tensor[0].cpu().numpy().astype( + np.uint8) + quad_mask_image = Image.fromarray(quad_mask_image_np).convert('RGB') + quad_mask_image.save( + os.path.join(mask_image_dir, + '{}_quad_{:.5f}.png'.format(view_idx, view_score))) + + +def visualize_outputs(output_dir, init_image_dir, mask_image_dir, + inpainted_image_dir, num_views): + # subplot settings + num_col = 3 + num_row = 1 + sus = 4 + + summary_image_dir = os.path.join(output_dir, 'summary') + os.makedirs(summary_image_dir, exist_ok=True) + + # graph settings + print('=> visualizing results...') + for view_idx in range(num_views): + plt.switch_backend('agg') + fig = plt.figure(dpi=100) + fig.set_size_inches(sus * num_col, sus * (num_row + 1)) + fig.set_facecolor('white') + + # rendering + plt.subplot2grid((num_row, num_col), (0, 0)) + plt.imshow( + Image.open( + os.path.join(init_image_dir, '{}.png'.format(view_idx)))) + plt.text( + 0, + 0, + 'Rendering', + fontsize=16, + color='black', + backgroundcolor='white') + plt.axis('off') + + # mask + plt.subplot2grid((num_row, num_col), (0, 1)) + plt.imshow( + Image.open( + os.path.join(mask_image_dir, + '{}_project.png'.format(view_idx)))) + plt.text( + 0, + 0, + 'Project Mask', + fontsize=16, + color='black', + backgroundcolor='white') + plt.set_cmap(cm.Greys_r) + plt.axis('off') + + # inpainted + plt.subplot2grid((num_row, num_col), (0, 2)) + plt.imshow( + Image.open( + os.path.join(inpainted_image_dir, '{}.png'.format(view_idx)))) + plt.text( + 0, + 0, + 'Inpainted', + fontsize=16, + color='black', + backgroundcolor='white') + plt.axis('off') + + plt.savefig( + os.path.join(summary_image_dir, '{}.png'.format(view_idx)), + bbox_inches='tight') + fig.clf() + + # generate GIF + images = [ + imageio.imread( + os.path.join(summary_image_dir, '{}.png'.format(view_idx))) + for view_idx in range(num_views) + ] + imageio.mimsave( + os.path.join(summary_image_dir, 'output.gif'), images, duration=1) + + print('=> done!') + + +def visualize_principle_viewpoints(output_dir, dist_list, elev_list, + azim_list): + theta_list = [e for e in azim_list] + phi_list = [90 - e for e in elev_list] + DIST = dist_list[0] + + xyz_list = [ + polar_to_xyz(theta, phi, DIST) + for theta, phi in zip(theta_list, phi_list) + ] + + xyz_np = np.array(xyz_list) + color_np = np.array([[0, 0, 0]]).repeat(xyz_np.shape[0], 0) + + ax = plt.axes(projection='3d') + SCALE = 0.8 + ax.set_xlim((-DIST, DIST)) + ax.set_ylim((-DIST, DIST)) + ax.set_zlim((-SCALE * DIST, SCALE * DIST)) + + ax.scatter( + xyz_np[:, 0], + xyz_np[:, 2], + xyz_np[:, 1], + s=100, + c=color_np, + depthshade=True, + label='Principle views') + ax.scatter([0], [0], [0], + c=[[1, 0, 0]], + s=100, + depthshade=True, + label='Object center') + + # draw hemisphere + # theta inclination angle + # phi azimuthal angle + n_theta = 50 # number of values for theta + n_phi = 200 # number of values for phi + r = DIST # radius of sphere + + # theta, phi = np.mgrid[0.0:0.5*np.pi:n_theta*1j, 0.0:2.0*np.pi:n_phi*1j] + theta, phi = np.mgrid[0.0:1 * np.pi:n_theta * 1j, + 0.0:2.0 * np.pi:n_phi * 1j] + + x = r * np.sin(theta) * np.cos(phi) + y = r * np.sin(theta) * np.sin(phi) + z = r * np.cos(theta) + + ax.plot_surface(x, y, z, rstride=1, cstride=1, alpha=0.25, linewidth=1) + + # Make the grid + ax.quiver( + xyz_np[:, 0], + xyz_np[:, 2], + xyz_np[:, 1], + -xyz_np[:, 0], + -xyz_np[:, 2], + -xyz_np[:, 1], + normalize=True, + length=0.3) + + ax.set_xlabel('X Label') + ax.set_ylabel('Z Label') + ax.set_zlabel('Y Label') + + ax.view_init(30, 35) + ax.legend() + + plt.show() + + plt.savefig(os.path.join(output_dir, 'principle_viewpoints.png')) + + +def visualize_refinement_viewpoints(output_dir, selected_view_ids, dist_list, + elev_list, azim_list): + theta_list = [azim_list[i] for i in selected_view_ids] + phi_list = [90 - elev_list[i] for i in selected_view_ids] + DIST = dist_list[0] + + xyz_list = [ + polar_to_xyz(theta, phi, DIST) + for theta, phi in zip(theta_list, phi_list) + ] + + xyz_np = np.array(xyz_list) + color_np = np.array([[0, 0, 0]]).repeat(xyz_np.shape[0], 0) + + fig = plt.figure() + ax = plt.axes(projection='3d') + SCALE = 0.8 + ax.set_xlim((-DIST, DIST)) + ax.set_ylim((-DIST, DIST)) + ax.set_zlim((-SCALE * DIST, SCALE * DIST)) + + ax.scatter( + xyz_np[:, 0], + xyz_np[:, 2], + xyz_np[:, 1], + c=color_np, + depthshade=True, + label='Refinement views') + ax.scatter([0], [0], [0], + c=[[1, 0, 0]], + s=100, + depthshade=True, + label='Object center') + + # draw hemisphere + # theta inclination angle + # phi azimuthal angle + n_theta = 50 # number of values for theta + n_phi = 200 # number of values for phi + r = DIST # radius of sphere + + # theta, phi = np.mgrid[0.0:0.5*np.pi:n_theta*1j, 0.0:2.0*np.pi:n_phi*1j] + theta, phi = np.mgrid[0.0:1 * np.pi:n_theta * 1j, + 0.0:2.0 * np.pi:n_phi * 1j] + + x = r * np.sin(theta) * np.cos(phi) + y = r * np.sin(theta) * np.sin(phi) + z = r * np.cos(theta) + + ax.plot_surface(x, y, z, rstride=1, cstride=1, alpha=0.25, linewidth=1) + + # Make the grid + ax.quiver( + xyz_np[:, 0], + xyz_np[:, 2], + xyz_np[:, 1], + -xyz_np[:, 0], + -xyz_np[:, 2], + -xyz_np[:, 1], + normalize=True, + length=0.3) + + ax.set_xlabel('X Label') + ax.set_ylabel('Z Label') + ax.set_zlabel('Y Label') + + ax.view_init(30, 35) + ax.legend() + + plt.show() + + plt.savefig(os.path.join(output_dir, 'refinement_viewpoints.png')) + + fig.clear() diff --git a/runtime/Lib/site_packages/models/cv/text_texture_generation/utils.py b/runtime/Lib/site_packages/models/cv/text_texture_generation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..09c40ddd231adfaedbcda6b262e87ce3ab993d9f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_texture_generation/utils.py @@ -0,0 +1,91 @@ +# common utils +import os + +import imageio.v2 as imageio +import torch +# pytorch3d +from pytorch3d.io import load_obj, load_objs_as_meshes +from pytorch3d.renderer import (AmbientLights, MeshRasterizer, + MeshRendererWithFragments, PerspectiveCameras, + RasterizationSettings, SoftPhongShader, + look_at_view_transform) +from torchvision import transforms +from tqdm import tqdm + +IMAGE_SIZE = 768 + + +def init_mesh(model_path, device): + verts, faces, aux = load_obj(model_path, device=device) + mesh = load_objs_as_meshes([model_path], device=device) + return mesh, verts, faces, aux + + +def init_camera(num_views, dist, elev, azim, view_idx, device): + interval = 360 // num_views + azim = (azim + interval * view_idx) % 360 + R, T = look_at_view_transform(dist, elev, azim) + T[0][2] = dist + image_size = torch.tensor([IMAGE_SIZE, IMAGE_SIZE]).unsqueeze(0) + focal_length = torch.tensor(2.0) + cameras = PerspectiveCameras( + focal_length=focal_length, + R=R, + T=T, + device=device, + image_size=image_size) + return cameras, dist, elev, azim + + +def init_renderer(camera, device): + raster_settings = RasterizationSettings(image_size=IMAGE_SIZE) + lights = AmbientLights(device=device) + renderer = MeshRendererWithFragments( + rasterizer=MeshRasterizer( + cameras=camera, raster_settings=raster_settings), + shader=SoftPhongShader(cameras=camera, lights=lights, device=device)) + + return renderer + + +def generation_gif(mesh_path): + num_views = 72 + if torch.cuda.is_available(): + DEVICE = torch.device('cuda:0') + torch.cuda.set_device(DEVICE) + else: + print('no gpu avaiable') + exit() + output_dir = 'GIF-{}'.format(num_views) + os.makedirs(output_dir, exist_ok=True) + + mesh, verts, faces, aux = init_mesh(mesh_path, DEVICE) + + # rendering + print('=> rendering...') + for view_idx in tqdm(range(num_views)): + init_image_path = os.path.join(output_dir, '{}.png'.format(view_idx)) + dist = 1.8 + elev = 15 + azim = 0 + + cameras, dist, elev, azim = init_camera(num_views, dist, elev, azim, + view_idx, DEVICE) + renderer = init_renderer(cameras, DEVICE) + init_images_tensor, fragments = renderer(mesh) + + # save images + init_image = init_images_tensor[0].cpu() + init_image = init_image.permute(2, 0, 1) + init_image = transforms.ToPILImage()(init_image).convert('RGB') + init_image.save(init_image_path) + + # generate GIF + images = [ + imageio.imread(os.path.join(output_dir, '{}.png').format(v_id)) + for v_id in range(args.num_views) + ] + imageio.mimsave( + os.path.join(output_dir, 'output.gif'), images, duration=0.1) + imageio.mimsave(os.path.join(output_dir, 'output.mp4'), images, fps=25) + print('=> done!') diff --git a/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/__init__.py b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fb01e4253ff9565a2c81f7cabf7fd52cf6ee5e3b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .pipeline_base import StableDiffusionBlendExtendPipeline + from .pipeline_sr import StableDiffusionControlNetImg2ImgPanoPipeline + +else: + _import_structure = { + 'pipeline_base': ['StableDiffusionBlendExtendPipeline'], + 'pipeline_sr': ['StableDiffusionControlNetImg2ImgPanoPipeline'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_base.py b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_base.py new file mode 100644 index 0000000000000000000000000000000000000000..bcfab512e8781493aed91f5d49a4f03db5cc6ccf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_base.py @@ -0,0 +1,855 @@ +# Copyright © Alibaba, Inc. and its affiliates. +# The implementation here is modifed based on diffusers.StableDiffusionPipeline, +# originally Apache 2.0 License and public available at +# https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py + +import inspect +import re +import warnings +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from diffusers import (AutoencoderKL, DiffusionPipeline, + StableDiffusionPipeline, UNet2DConditionModel) +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models.vae import DecoderOutput +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import \ + StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import (deprecate, is_accelerate_available, + is_accelerate_version, logging, randn_tensor, + replace_example_docstring) +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import EulerAncestralDiscreteScheduler + >>> modelscope.models.cv.text_to_360panorama_image import StableDiffusionBlendExtendPipeline + >>> model_id = "damo/cv_diffusion_text-to-360panorama-image_generation/sd-base" + >>> pipe = StableDiffusionBlendExtendPipeline.from_pretrained(model_id, torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + >>> pipe.vae.enable_tiling() + >>> pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + >>> # remove following line if xformers is not installed + >>> pipe.enable_xformers_memory_efficient_attention() + >>> pipe.enable_model_cpu_offload() + >>> prompt = "a living room" + >>> image = pipe(prompt).images[0] + ``` +""" + +re_attention = re.compile( + r""" +\\\(| +\\\)| +\\\[| +\\]| +\\\\| +\\| +\(| +\[| +:([+-]?[.\d]+)\)| +\)| +]| +[^\\()\[\]:]+| +: +""", + re.X, +) + + +def parse_prompt_attention(text): + """ + Parses a string with attention tokens and returns a list of pairs: text and its associated weight. + Accepted tokens are: + (abc) - increases attention to abc by a multiplier of 1.1 + (abc:3.12) - increases attention to abc by a multiplier of 3.12 + [abc] - decreases attention to abc by a multiplier of 1.1 + """ + + res = [] + round_brackets = [] + square_brackets = [] + + round_bracket_multiplier = 1.1 + square_bracket_multiplier = 1 / 1.1 + + def multiply_range(start_position, multiplier): + for p in range(start_position, len(res)): + res[p][1] *= multiplier + + for m in re_attention.finditer(text): + text = m.group(0) + weight = m.group(1) + + if text.startswith('\\'): + res.append([text[1:], 1.0]) + elif text == '(': + round_brackets.append(len(res)) + elif text == '[': + square_brackets.append(len(res)) + elif weight is not None and len(round_brackets) > 0: + multiply_range(round_brackets.pop(), float(weight)) + elif text == ')' and len(round_brackets) > 0: + multiply_range(round_brackets.pop(), round_bracket_multiplier) + elif text == ']' and len(square_brackets) > 0: + multiply_range(square_brackets.pop(), square_bracket_multiplier) + else: + res.append([text, 1.0]) + + for pos in round_brackets: + multiply_range(pos, round_bracket_multiplier) + + for pos in square_brackets: + multiply_range(pos, square_bracket_multiplier) + + if len(res) == 0: + res = [['', 1.0]] + + # merge runs of identical weights + i = 0 + while i + 1 < len(res): + if res[i][1] == res[i + 1][1]: + res[i][0] += res[i + 1][0] + res.pop(i + 1) + else: + i += 1 + + return res + + +def get_prompts_with_weights(pipe: DiffusionPipeline, prompt: List[str], + max_length: int): + r""" + Tokenize a list of prompts and return its tokens with weights of each token. + + No padding, starting or ending token is included. + """ + tokens = [] + weights = [] + truncated = False + for text in prompt: + texts_and_weights = parse_prompt_attention(text) + text_token = [] + text_weight = [] + for word, weight in texts_and_weights: + # tokenize and discard the starting and the ending token + token = pipe.tokenizer(word).input_ids[1:-1] + text_token += token + # copy the weight by length of token + text_weight += [weight] * len(token) + # stop if the text is too long (longer than truncation limit) + if len(text_token) > max_length: + truncated = True + break + # truncate + if len(text_token) > max_length: + truncated = True + text_token = text_token[:max_length] + text_weight = text_weight[:max_length] + tokens.append(text_token) + weights.append(text_weight) + if truncated: + logger.warning( + 'Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples' + ) + return tokens, weights + + +def pad_tokens_and_weights(tokens, + weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=True, + chunk_length=77): + r""" + Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length. + """ + max_embeddings_multiples = (max_length - 2) // (chunk_length - 2) + weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length + for i in range(len(tokens)): + tokens[i] = [ + bos + ] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos] + if no_boseos_middle: + weights[i] = [1.0] + weights[i] + [1.0] * ( + max_length - 1 - len(weights[i])) + else: + w = [] + if len(weights[i]) == 0: + w = [1.0] * weights_length + else: + for j in range(max_embeddings_multiples): + w.append(1.0) # weight for starting token in this chunk + w += weights[i][j * (chunk_length - 2):min( + len(weights[i]), (j + 1) * (chunk_length - 2))] + w.append(1.0) # weight for ending token in this chunk + w += [1.0] * (weights_length - len(w)) + weights[i] = w[:] + + return tokens, weights + + +def get_unweighted_text_embeddings( + pipe: DiffusionPipeline, + text_input: torch.Tensor, + chunk_length: int, + no_boseos_middle: Optional[bool] = True, +): + """ + When the length of tokens is a multiple of the capacity of the text encoder, + it should be split into chunks and sent to the text encoder individually. + """ + max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2) + if max_embeddings_multiples > 1: + text_embeddings = [] + for i in range(max_embeddings_multiples): + # extract the i-th chunk + text_input_chunk = text_input[:, i * (chunk_length - 2):(i + 1) + * (chunk_length - 2) + 2].clone() + + # cover the head and the tail by the starting and the ending tokens + text_input_chunk[:, 0] = text_input[0, 0] + text_input_chunk[:, -1] = text_input[0, -1] + text_embedding = pipe.text_encoder(text_input_chunk)[0] + + if no_boseos_middle: + if i == 0: + # discard the ending token + text_embedding = text_embedding[:, :-1] + elif i == max_embeddings_multiples - 1: + # discard the starting token + text_embedding = text_embedding[:, 1:] + else: + # discard both starting and ending tokens + text_embedding = text_embedding[:, 1:-1] + + text_embeddings.append(text_embedding) + text_embeddings = torch.concat(text_embeddings, axis=1) + else: + text_embeddings = pipe.text_encoder(text_input)[0] + return text_embeddings + + +def get_weighted_text_embeddings( + pipe: DiffusionPipeline, + prompt: Union[str, List[str]], + uncond_prompt: Optional[Union[str, List[str]]] = None, + max_embeddings_multiples: Optional[int] = 3, + no_boseos_middle: Optional[bool] = False, + skip_parsing: Optional[bool] = False, + skip_weighting: Optional[bool] = False, +): + r""" + Prompts can be assigned with local weights using brackets. For example, + prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful', + and the embedding tokens corresponding to the words get multiplied by a constant, 1.1. + + Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean. + + Args: + pipe (`DiffusionPipeline`): + Pipe to provide access to the tokenizer and the text encoder. + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + uncond_prompt (`str` or `List[str]`): + The unconditional prompt or prompts for guide the image generation. If unconditional prompt + is provided, the embeddings of prompt and uncond_prompt are concatenated. + max_embeddings_multiples (`int`, *optional*, defaults to `3`): + The max multiple length of prompt embeddings compared to the max output length of text encoder. + no_boseos_middle (`bool`, *optional*, defaults to `False`): + If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and + ending token in each of the chunk in the middle. + skip_parsing (`bool`, *optional*, defaults to `False`): + Skip the parsing of brackets. + skip_weighting (`bool`, *optional*, defaults to `False`): + Skip the weighting. When the parsing is skipped, it is forced True. + """ + max_length = (pipe.tokenizer.model_max_length + - 2) * max_embeddings_multiples + 2 + if isinstance(prompt, str): + prompt = [prompt] + + if not skip_parsing: + prompt_tokens, prompt_weights = get_prompts_with_weights( + pipe, prompt, max_length - 2) + if uncond_prompt is not None: + if isinstance(uncond_prompt, str): + uncond_prompt = [uncond_prompt] + uncond_tokens, uncond_weights = get_prompts_with_weights( + pipe, uncond_prompt, max_length - 2) + else: + prompt_tokens = [ + token[1:-1] for token in pipe.tokenizer( + prompt, max_length=max_length, truncation=True).input_ids + ] + prompt_weights = [[1.0] * len(token) for token in prompt_tokens] + if uncond_prompt is not None: + if isinstance(uncond_prompt, str): + uncond_prompt = [uncond_prompt] + uncond_tokens = [ + token[1:-1] for token in pipe.tokenizer( + uncond_prompt, max_length=max_length, + truncation=True).input_ids + ] + uncond_weights = [[1.0] * len(token) for token in uncond_tokens] + + # round up the longest length of tokens to a multiple of (model_max_length - 2) + max_length = max([len(token) for token in prompt_tokens]) + if uncond_prompt is not None: + max_length = max(max_length, + max([len(token) for token in uncond_tokens])) + + max_embeddings_multiples = min( + max_embeddings_multiples, + (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1, + ) + max_embeddings_multiples = max(1, max_embeddings_multiples) + max_length = (pipe.tokenizer.model_max_length + - 2) * max_embeddings_multiples + 2 + + # pad the length of tokens and weights + bos = pipe.tokenizer.bos_token_id + eos = pipe.tokenizer.eos_token_id + pad = getattr(pipe.tokenizer, 'pad_token_id', eos) + prompt_tokens, prompt_weights = pad_tokens_and_weights( + prompt_tokens, + prompt_weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=no_boseos_middle, + chunk_length=pipe.tokenizer.model_max_length, + ) + prompt_tokens = torch.tensor( + prompt_tokens, dtype=torch.long, device=pipe.device) + if uncond_prompt is not None: + uncond_tokens, uncond_weights = pad_tokens_and_weights( + uncond_tokens, + uncond_weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=no_boseos_middle, + chunk_length=pipe.tokenizer.model_max_length, + ) + uncond_tokens = torch.tensor( + uncond_tokens, dtype=torch.long, device=pipe.device) + + # get the embeddings + text_embeddings = get_unweighted_text_embeddings( + pipe, + prompt_tokens, + pipe.tokenizer.model_max_length, + no_boseos_middle=no_boseos_middle, + ) + prompt_weights = torch.tensor( + prompt_weights, + dtype=text_embeddings.dtype, + device=text_embeddings.device) + if uncond_prompt is not None: + uncond_embeddings = get_unweighted_text_embeddings( + pipe, + uncond_tokens, + pipe.tokenizer.model_max_length, + no_boseos_middle=no_boseos_middle, + ) + uncond_weights = torch.tensor( + uncond_weights, + dtype=uncond_embeddings.dtype, + device=uncond_embeddings.device) + + # assign weights to the prompts and normalize in the sense of mean + # TODO: should we normalize by chunk or in a whole (current implementation)? + if (not skip_parsing) and (not skip_weighting): + previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to( + text_embeddings.dtype) + text_embeddings *= prompt_weights.unsqueeze(-1) + current_mean = text_embeddings.float().mean(axis=[-2, -1]).to( + text_embeddings.dtype) + text_embeddings *= (previous_mean + / current_mean).unsqueeze(-1).unsqueeze(-1) + if uncond_prompt is not None: + previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to( + uncond_embeddings.dtype) + uncond_embeddings *= uncond_weights.unsqueeze(-1) + current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to( + uncond_embeddings.dtype) + uncond_embeddings *= (previous_mean + / current_mean).unsqueeze(-1).unsqueeze(-1) + + if uncond_prompt is not None: + return text_embeddings, uncond_embeddings + return text_embeddings, None + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std( + dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + ( + 1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionBlendExtendPipeline(StableDiffusionPipeline): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *Textual-Inversion*: [`loaders.TextualInversionLoaderMixin.load_textual_inversion`] + - *LoRA*: [`loaders.LoraLoaderMixin.load_lora_weights`] + - *Ckpt*: [`loaders.FromCkptMixin.from_ckpt`] + + as well as the following saving methods: + - *LoRA*: [`loaders.LoraLoaderMixin.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/ + en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + _optional_components = ['safety_checker', 'feature_extractor'] + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + max_embeddings_multiples=3, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `list(int)`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + max_embeddings_multiples (`int`, *optional*, defaults to `3`): + The max multiple length of prompt embeddings compared to the max output length of text encoder. + """ + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [''] * batch_size + elif isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] * batch_size + if batch_size != len(negative_prompt): + raise ValueError( + f'`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:' + f' {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches' + ' the batch size of `prompt`.') + if prompt_embeds is None or negative_prompt_embeds is None: + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = self.maybe_convert_prompt( + negative_prompt, self.tokenizer) + + prompt_embeds1, negative_prompt_embeds1 = get_weighted_text_embeddings( + pipe=self, + prompt=prompt, + uncond_prompt=negative_prompt + if do_classifier_free_guidance else None, + max_embeddings_multiples=max_embeddings_multiples, + ) + if prompt_embeds is None: + prompt_embeds = prompt_embeds1 + if negative_prompt_embeds is None: + negative_prompt_embeds = negative_prompt_embeds1 + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, + seq_len, -1) + + if do_classifier_free_guidance: + bs_embed, seq_len, _ = negative_prompt_embeds.shape + negative_prompt_embeds = negative_prompt_embeds.repeat( + 1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view( + bs_embed * num_images_per_prompt, seq_len, -1) + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def blend_v(self, a, b, blend_extent): + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, + y, :] = a[:, :, -blend_extent + + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * ( + y / blend_extent) + return b + + def blend_h(self, a, b, blend_extent): + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + + x] * (1 - x / blend_extent) + b[:, :, :, x] * ( + x / blend_extent) + return b + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, + List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = 'pil', + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], + None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). + guidance_rescale (`float`, *optional*, defaults to 0.7): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + def tiled_decode( + self, + z: torch.FloatTensor, + return_dict: bool = True + ) -> Union[DecoderOutput, torch.FloatTensor]: + r"""Decode a batch of images using a tiled decoder. + + Args: + When this option is enabled, the VAE will split the input tensor into tiles to compute decoding in several + steps. This is useful to keep memory use constant regardless of image size. + The end result of tiled decoding is: different from non-tiled decoding due to each tile using a different + decoder. To avoid tiling artifacts, the tiles overlap and are blended together to form a smooth output. + You may still see tile-sized changes in the look of the output, but they should be much less noticeable. + z (`torch.FloatTensor`): Input batch of latent vectors. return_dict (`bool`, *optional*, defaults to + `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + _tile_overlap_factor = 1 - self.tile_overlap_factor + overlap_size = int(self.tile_latent_min_size + * _tile_overlap_factor) + blend_extent = int(self.tile_sample_min_size + * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + w = z.shape[3] + z = torch.cat([z, z[:, :, :, :w // 4]], dim=-1) + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + tile = z[:, :, i:i + self.tile_latent_min_size, :] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + vae_scale_factor = decoded.shape[-1] // tile.shape[-1] + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append( + self.blend_h( + tile[:, :, :row_limit, w * vae_scale_factor:], + tile[:, :, :row_limit, :w * vae_scale_factor], + tile.shape[-1] - w * vae_scale_factor)) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec, ) + + return DecoderOutput(sample=dec) + + self.vae.tiled_decode = tiled_decode.__get__(self.vae, AutoencoderKL) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, height, width, callback_steps, + negative_prompt, prompt_embeds, + negative_prompt_embeds) + self.blend_extend = width // self.vae_scale_factor // 32 + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get('scale', None) + if cross_attention_kwargs is not None else None) + prompt_embeds = self._encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len( + timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat( + [latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * ( + noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_text, + guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + **extra_step_kwargs, + return_dict=False)[0] + + # call the callback, if provided + condition_i = i == len(timesteps) - 1 + condition_warm = (i + 1) > num_warmup_steps and ( + i + 1) % self.scheduler.order == 0 + if condition_i or condition_warm: + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + latents = self.blend_h(latents, latents, self.blend_extend) + latents = self.blend_h(latents, latents, self.blend_extend) + latents = latents[:, :, :, :width // self.vae_scale_factor] + + if not output_type == 'latent': + image = self.vae.decode( + latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker( + image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess( + image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload last model to CPU + if hasattr( + self, + 'final_offload_hook') and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput( + images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_sr.py b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_sr.py new file mode 100644 index 0000000000000000000000000000000000000000..fab559d90fb435d858b2b799d9921082bae5e6c4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_to_360panorama_image/pipeline_sr.py @@ -0,0 +1,1208 @@ +# Copyright © Alibaba, Inc. and its affiliates. +# The implementation here is modifed based on diffusers.StableDiffusionControlNetImg2ImgPipeline, +# originally Apache 2.0 License and public available at +# https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py + +import copy +import inspect +import os +import re +import warnings +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from diffusers import (AutoencoderKL, DiffusionPipeline, + StableDiffusionControlNetImg2ImgPipeline, + UNet2DConditionModel) +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import ControlNetModel +from diffusers.models.vae import DecoderOutput +from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import \ + StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import (PIL_INTERPOLATION, deprecate, + is_accelerate_available, is_accelerate_version, + is_compiled_module, logging, randn_tensor, + replace_example_docstring) +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from PIL import Image + >>> modelscope.models.cv.text_to_360panorama_image import StableDiffusionControlNetImg2ImgPanoPipeline + >>> base_model_path = "damo/cv_diffusion_text-to-360panorama-image_generation/sr-base" + >>> controlnet_path = "damo/cv_diffusion_text-to-360panorama-image_generation/sr-control" + >>> controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16) + >>> pipe = StableDiffusionControlNetImg2ImgPanoPipeline.from_pretrained(base_model_path, controlnet=controlnet, + ... torch_dtype=torch.float16) + >>> pipe.vae.enable_tiling() + >>> # remove following line if xformers is not installed + >>> pipe.enable_xformers_memory_efficient_attention() + >>> pipe.enable_model_cpu_offload() + >>> input_image_path = 'data/test/images/test_to_360panorama_image/test.png' + >>> image = Image.open(input_image_path) + >>> image = pipe( + ... "futuristic-looking woman", + ... num_inference_steps=20, + ... image=image, + ... height=768, + ... width=1536, + ... control_image=image, + ... ).images[0] + + ``` +""" + +re_attention = re.compile( + r""" +\\\(| +\\\)| +\\\[| +\\]| +\\\\| +\\| +\(| +\[| +:([+-]?[.\d]+)\)| +\)| +]| +[^\\()\[\]:]+| +: +""", + re.X, +) + + +def parse_prompt_attention(text): + """ + Parses a string with attention tokens and returns a list of pairs: text and its associated weight. + Accepted tokens are: + (abc) - increases attention to abc by a multiplier of 1.1 + (abc:3.12) - increases attention to abc by a multiplier of 3.12 + [abc] - decreases attention to abc by a multiplier of 1.1 + """ + + res = [] + round_brackets = [] + square_brackets = [] + + round_bracket_multiplier = 1.1 + square_bracket_multiplier = 1 / 1.1 + + def multiply_range(start_position, multiplier): + for p in range(start_position, len(res)): + res[p][1] *= multiplier + + for m in re_attention.finditer(text): + text = m.group(0) + weight = m.group(1) + + if text.startswith('\\'): + res.append([text[1:], 1.0]) + elif text == '(': + round_brackets.append(len(res)) + elif text == '[': + square_brackets.append(len(res)) + elif weight is not None and len(round_brackets) > 0: + multiply_range(round_brackets.pop(), float(weight)) + elif text == ')' and len(round_brackets) > 0: + multiply_range(round_brackets.pop(), round_bracket_multiplier) + elif text == ']' and len(square_brackets) > 0: + multiply_range(square_brackets.pop(), square_bracket_multiplier) + else: + res.append([text, 1.0]) + + for pos in round_brackets: + multiply_range(pos, round_bracket_multiplier) + + for pos in square_brackets: + multiply_range(pos, square_bracket_multiplier) + + if len(res) == 0: + res = [['', 1.0]] + + # merge runs of identical weights + i = 0 + while i + 1 < len(res): + if res[i][1] == res[i + 1][1]: + res[i][0] += res[i + 1][0] + res.pop(i + 1) + else: + i += 1 + + return res + + +def get_prompts_with_weights(pipe: DiffusionPipeline, prompt: List[str], + max_length: int): + r""" + Tokenize a list of prompts and return its tokens with weights of each token. + + No padding, starting or ending token is included. + """ + tokens = [] + weights = [] + truncated = False + for text in prompt: + texts_and_weights = parse_prompt_attention(text) + text_token = [] + text_weight = [] + for word, weight in texts_and_weights: + # tokenize and discard the starting and the ending token + token = pipe.tokenizer(word).input_ids[1:-1] + text_token += token + # copy the weight by length of token + text_weight += [weight] * len(token) + # stop if the text is too long (longer than truncation limit) + if len(text_token) > max_length: + truncated = True + break + # truncate + if len(text_token) > max_length: + truncated = True + text_token = text_token[:max_length] + text_weight = text_weight[:max_length] + tokens.append(text_token) + weights.append(text_weight) + if truncated: + logger.warning( + 'Prompt was truncated. Try to shorten the prompt or increase max_embeddings_multiples' + ) + return tokens, weights + + +def pad_tokens_and_weights(tokens, + weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=True, + chunk_length=77): + r""" + Pad the tokens (with starting and ending tokens) and weights (with 1.0) to max_length. + """ + max_embeddings_multiples = (max_length - 2) // (chunk_length - 2) + weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length + for i in range(len(tokens)): + tokens[i] = [ + bos + ] + tokens[i] + [pad] * (max_length - 1 - len(tokens[i]) - 1) + [eos] + if no_boseos_middle: + weights[i] = [1.0] + weights[i] + [1.0] * ( + max_length - 1 - len(weights[i])) + else: + w = [] + if len(weights[i]) == 0: + w = [1.0] * weights_length + else: + for j in range(max_embeddings_multiples): + w.append(1.0) # weight for starting token in this chunk + w += weights[i][j * (chunk_length - 2):min( + len(weights[i]), (j + 1) * (chunk_length - 2))] + w.append(1.0) # weight for ending token in this chunk + w += [1.0] * (weights_length - len(w)) + weights[i] = w[:] + + return tokens, weights + + +def get_unweighted_text_embeddings( + pipe: DiffusionPipeline, + text_input: torch.Tensor, + chunk_length: int, + no_boseos_middle: Optional[bool] = True, +): + """ + When the length of tokens is a multiple of the capacity of the text encoder, + it should be split into chunks and sent to the text encoder individually. + """ + max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2) + if max_embeddings_multiples > 1: + text_embeddings = [] + for i in range(max_embeddings_multiples): + # extract the i-th chunk + text_input_chunk = text_input[:, i * (chunk_length - 2):(i + 1) + * (chunk_length - 2) + 2].clone() + + # cover the head and the tail by the starting and the ending tokens + text_input_chunk[:, 0] = text_input[0, 0] + text_input_chunk[:, -1] = text_input[0, -1] + text_embedding = pipe.text_encoder(text_input_chunk)[0] + + if no_boseos_middle: + if i == 0: + # discard the ending token + text_embedding = text_embedding[:, :-1] + elif i == max_embeddings_multiples - 1: + # discard the starting token + text_embedding = text_embedding[:, 1:] + else: + # discard both starting and ending tokens + text_embedding = text_embedding[:, 1:-1] + + text_embeddings.append(text_embedding) + text_embeddings = torch.concat(text_embeddings, axis=1) + else: + text_embeddings = pipe.text_encoder(text_input)[0] + return text_embeddings + + +def get_weighted_text_embeddings( + pipe: DiffusionPipeline, + prompt: Union[str, List[str]], + uncond_prompt: Optional[Union[str, List[str]]] = None, + max_embeddings_multiples: Optional[int] = 3, + no_boseos_middle: Optional[bool] = False, + skip_parsing: Optional[bool] = False, + skip_weighting: Optional[bool] = False, +): + r""" + Prompts can be assigned with local weights using brackets. For example, + prompt 'A (very beautiful) masterpiece' highlights the words 'very beautiful', + and the embedding tokens corresponding to the words get multiplied by a constant, 1.1. + + Also, to regularize of the embedding, the weighted embedding would be scaled to preserve the original mean. + + Args: + pipe (`DiffusionPipeline`): + Pipe to provide access to the tokenizer and the text encoder. + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + uncond_prompt (`str` or `List[str]`): + The unconditional prompt or prompts for guide the image generation. If unconditional prompt + is provided, the embeddings of prompt and uncond_prompt are concatenated. + max_embeddings_multiples (`int`, *optional*, defaults to `3`): + The max multiple length of prompt embeddings compared to the max output length of text encoder. + no_boseos_middle (`bool`, *optional*, defaults to `False`): + If the length of text token is multiples of the capacity of text encoder, whether reserve the starting and + ending token in each of the chunk in the middle. + skip_parsing (`bool`, *optional*, defaults to `False`): + Skip the parsing of brackets. + skip_weighting (`bool`, *optional*, defaults to `False`): + Skip the weighting. When the parsing is skipped, it is forced True. + """ + max_length = (pipe.tokenizer.model_max_length + - 2) * max_embeddings_multiples + 2 + if isinstance(prompt, str): + prompt = [prompt] + + if not skip_parsing: + prompt_tokens, prompt_weights = get_prompts_with_weights( + pipe, prompt, max_length - 2) + if uncond_prompt is not None: + if isinstance(uncond_prompt, str): + uncond_prompt = [uncond_prompt] + uncond_tokens, uncond_weights = get_prompts_with_weights( + pipe, uncond_prompt, max_length - 2) + else: + prompt_tokens = [ + token[1:-1] for token in pipe.tokenizer( + prompt, max_length=max_length, truncation=True).input_ids + ] + prompt_weights = [[1.0] * len(token) for token in prompt_tokens] + if uncond_prompt is not None: + if isinstance(uncond_prompt, str): + uncond_prompt = [uncond_prompt] + uncond_tokens = [ + token[1:-1] for token in pipe.tokenizer( + uncond_prompt, max_length=max_length, + truncation=True).input_ids + ] + uncond_weights = [[1.0] * len(token) for token in uncond_tokens] + + # round up the longest length of tokens to a multiple of (model_max_length - 2) + max_length = max([len(token) for token in prompt_tokens]) + if uncond_prompt is not None: + max_length = max(max_length, + max([len(token) for token in uncond_tokens])) + + max_embeddings_multiples = min( + max_embeddings_multiples, + (max_length - 1) // (pipe.tokenizer.model_max_length - 2) + 1, + ) + max_embeddings_multiples = max(1, max_embeddings_multiples) + max_length = (pipe.tokenizer.model_max_length + - 2) * max_embeddings_multiples + 2 + + # pad the length of tokens and weights + bos = pipe.tokenizer.bos_token_id + eos = pipe.tokenizer.eos_token_id + pad = getattr(pipe.tokenizer, 'pad_token_id', eos) + prompt_tokens, prompt_weights = pad_tokens_and_weights( + prompt_tokens, + prompt_weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=no_boseos_middle, + chunk_length=pipe.tokenizer.model_max_length, + ) + prompt_tokens = torch.tensor( + prompt_tokens, dtype=torch.long, device=pipe.device) + if uncond_prompt is not None: + uncond_tokens, uncond_weights = pad_tokens_and_weights( + uncond_tokens, + uncond_weights, + max_length, + bos, + eos, + pad, + no_boseos_middle=no_boseos_middle, + chunk_length=pipe.tokenizer.model_max_length, + ) + uncond_tokens = torch.tensor( + uncond_tokens, dtype=torch.long, device=pipe.device) + + # get the embeddings + text_embeddings = get_unweighted_text_embeddings( + pipe, + prompt_tokens, + pipe.tokenizer.model_max_length, + no_boseos_middle=no_boseos_middle, + ) + prompt_weights = torch.tensor( + prompt_weights, + dtype=text_embeddings.dtype, + device=text_embeddings.device) + if uncond_prompt is not None: + uncond_embeddings = get_unweighted_text_embeddings( + pipe, + uncond_tokens, + pipe.tokenizer.model_max_length, + no_boseos_middle=no_boseos_middle, + ) + uncond_weights = torch.tensor( + uncond_weights, + dtype=uncond_embeddings.dtype, + device=uncond_embeddings.device) + + # assign weights to the prompts and normalize in the sense of mean + # TODO: should we normalize by chunk or in a whole (current implementation)? + if (not skip_parsing) and (not skip_weighting): + previous_mean = text_embeddings.float().mean(axis=[-2, -1]).to( + text_embeddings.dtype) + text_embeddings *= prompt_weights.unsqueeze(-1) + current_mean = text_embeddings.float().mean(axis=[-2, -1]).to( + text_embeddings.dtype) + text_embeddings *= (previous_mean + / current_mean).unsqueeze(-1).unsqueeze(-1) + if uncond_prompt is not None: + previous_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to( + uncond_embeddings.dtype) + uncond_embeddings *= uncond_weights.unsqueeze(-1) + current_mean = uncond_embeddings.float().mean(axis=[-2, -1]).to( + uncond_embeddings.dtype) + uncond_embeddings *= (previous_mean + / current_mean).unsqueeze(-1).unsqueeze(-1) + + if uncond_prompt is not None: + return text_embeddings, uncond_embeddings + return text_embeddings, None + + +def prepare_image(image): + if isinstance(image, torch.Tensor): + # Batch single image + if image.ndim == 3: + image = image.unsqueeze(0) + + image = image.to(dtype=torch.float32) + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + image = [np.array(i.convert('RGB'))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + return image + + +class StableDiffusionControlNetImg2ImgPanoPipeline( + StableDiffusionControlNetImg2ImgPipeline): + r""" + Pipeline for text-to-image generation using Stable Diffusion with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *Textual-Inversion*: [`loaders.TextualInversionLoaderMixin.load_textual_inversion`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/ + model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets + as a list, the outputs from each ControlNet are added together to create one combined additional + conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + _optional_components = ['safety_checker', 'feature_extractor'] + + def check_inputs( + self, + prompt, + image, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + controlnet_conditioning_scale=1.0, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError( + f'`height` and `width` have to be divisible by 8 but are {height} and {width}.' + ) + condition_1 = callback_steps is not None + condition_2 = not isinstance(callback_steps, + int) or callback_steps <= 0 + if (callback_steps is None) or (condition_1 and condition_2): + raise ValueError( + f'`callback_steps` has to be a positive integer but is {callback_steps} of type' + f' {type(callback_steps)}.') + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f'Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to' + ' only forward one of the two.') + elif prompt is None and prompt_embeds is None: + raise ValueError( + 'Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined.' + ) + elif prompt is not None and (not isinstance(prompt, str) + and not isinstance(prompt, list)): + raise ValueError( + f'`prompt` has to be of type `str` or `list` but is {type(prompt)}' + ) + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f'Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:' + f' {negative_prompt_embeds}. Please make sure to only forward one of the two.' + ) + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + '`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but' + f' got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`' + f' {negative_prompt_embeds.shape}.') + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f'You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}' + ' prompts. The conditionings will be fixed across the prompts.' + ) + # Check `image` + is_compiled = hasattr( + F, 'scaled_dot_product_attention') and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule) + if (isinstance(self.controlnet, ControlNetModel) or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel)): + self.check_image(image, prompt, prompt_embeds) + elif (isinstance(self.controlnet, MultiControlNetModel) or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel)): + if not isinstance(image, list): + raise TypeError( + 'For multiple controlnets: `image` must be type `list`') + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError( + 'A single batch of multiple conditionings are supported at the moment.' + ) + elif len(image) != len(self.controlnet.nets): + raise ValueError( + 'For multiple controlnets: `image` must have the same length as the number of controlnets.' + ) + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + # Check `controlnet_conditioning_scale` + if (isinstance(self.controlnet, ControlNetModel) or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel)): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError( + 'For single controlnet: `controlnet_conditioning_scale` must be type `float`.' + ) + elif (isinstance(self.controlnet, MultiControlNetModel) or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel)): + if isinstance(controlnet_conditioning_scale, list): + if any( + isinstance(i, list) + for i in controlnet_conditioning_scale): + raise ValueError( + 'A single batch of multiple conditionings are supported at the moment.' + ) + elif isinstance( + controlnet_conditioning_scale, + list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets): + raise ValueError( + 'For multiple controlnets: When `controlnet_conditioning_scale` ' + 'is specified as `list`, it must have' + ' the same length as the number of controlnets') + else: + assert False + + def _default_height_width(self, height, width, image): + # NOTE: It is possible that a list of images have different + # dimensions for each image, so just checking the first image + # is not _exactly_ correct, but it is simple. + while isinstance(image, list): + image = image[0] + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[2] + height = (height // 8) * 8 # round down to nearest multiple of 8 + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[3] + width = (width // 8) * 8 # round down to nearest multiple of 8 + return height, width + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + max_embeddings_multiples=3, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `list(int)`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + max_embeddings_multiples (`int`, *optional*, defaults to `3`): + The max multiple length of prompt embeddings compared to the max output length of text encoder. + """ + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [''] * batch_size + elif isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] * batch_size + if batch_size != len(negative_prompt): + raise ValueError( + f'`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:' + f' {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches' + ' the batch size of `prompt`.') + if prompt_embeds is None or negative_prompt_embeds is None: + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = self.maybe_convert_prompt( + negative_prompt, self.tokenizer) + + prompt_embeds1, negative_prompt_embeds1 = get_weighted_text_embeddings( + pipe=self, + prompt=prompt, + uncond_prompt=negative_prompt + if do_classifier_free_guidance else None, + max_embeddings_multiples=max_embeddings_multiples, + ) + if prompt_embeds is None: + prompt_embeds = prompt_embeds1 + if negative_prompt_embeds is None: + negative_prompt_embeds = negative_prompt_embeds1 + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, + seq_len, -1) + + if do_classifier_free_guidance: + bs_embed, seq_len, _ = negative_prompt_embeds.shape + negative_prompt_embeds = negative_prompt_embeds.repeat( + 1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view( + bs_embed * num_images_per_prompt, seq_len, -1) + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def denoise_latents(self, latents, t, prompt_embeds, control_image, + controlnet_conditioning_scale, guess_mode, + cross_attention_kwargs, do_classifier_free_guidance, + guidance_scale, extra_step_kwargs, + views_scheduler_status): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat( + [latents] * 2) if do_classifier_free_guidance else latents + self.scheduler.__dict__.update(views_scheduler_status[0]) + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t) + # controlnet(s) inference + if guess_mode and do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + controlnet_latent_model_input = latents + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + else: + controlnet_latent_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + down_block_res_samples, mid_block_res_sample = self.controlnet( + controlnet_latent_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=controlnet_conditioning_scale, + guess_mode=guess_mode, + return_dict=False, + ) + if guess_mode and do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [ + torch.cat([torch.zeros_like(d), d]) + for d in down_block_res_samples + ] + mid_block_res_sample = torch.cat( + [torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + return_dict=False, + )[0] + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * ( + noise_pred_text - noise_pred_uncond) + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + return latents + + def blend_v(self, a, b, blend_extent): + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, + y, :] = a[:, :, -blend_extent + + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * ( + y / blend_extent) + return b + + def blend_h(self, a, b, blend_extent): + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + + x] * (1 - x / blend_extent) + b[:, :, :, x] * ( + x / blend_extent) + return b + + def get_blocks(self, latents, control_image, tile_latent_min_size, + overlap_size): + rows_latents = [] + rows_control_images = [] + for i in range(0, latents.shape[2] - overlap_size, overlap_size): + row_latents = [] + row_control_images = [] + for j in range(0, latents.shape[3] - overlap_size, overlap_size): + latents_input = latents[:, :, i:i + tile_latent_min_size, + j:j + tile_latent_min_size] + c_start_i = self.vae_scale_factor * i + c_end_i = self.vae_scale_factor * (i + tile_latent_min_size) + c_start_j = self.vae_scale_factor * j + c_end_j = self.vae_scale_factor * (j + tile_latent_min_size) + control_image_input = control_image[:, :, c_start_i:c_end_i, + c_start_j:c_end_j] + row_latents.append(latents_input) + row_control_images.append(control_image_input) + rows_latents.append(row_latents) + rows_control_images.append(row_control_images) + return rows_latents, rows_control_images + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[torch.FloatTensor, PIL.Image.Image, + List[torch.FloatTensor], List[PIL.Image.Image]] = None, + control_image: Union[torch.FloatTensor, PIL.Image.Image, + List[torch.FloatTensor], + List[PIL.Image.Image]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + strength: float = 0.8, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, + List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = 'pil', + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], + None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + guess_mode: bool = False, + context_size: int = 768, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, + `List[List[torch.FloatTensor]]`, or `List[List[PIL.Image.Image]]`): + The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If + the type is specified as `Torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can + also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If + height and/or width are passed, `image` is resized according to them. If multiple ControlNets are + specified in init, images must be passed as a list such that each element of the list can be correctly + batched for input to a single controlnet. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/ + src/diffusers/models/cross_attention.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original unet. If multiple ControlNets are specified in init, you can set the + corresponding scale as a list. Note that by default, we use a smaller conditioning scale for inpainting + than for [`~StableDiffusionControlNetPipeline.__call__`]. + guess_mode (`bool`, *optional*, defaults to `False`): + In this mode, the ControlNet encoder will try best to recognize the content of the input image even if + you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. + context_size ('int', *optional*, defaults to '768'): + tiled size when denoise the latents. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + def tiled_decode( + self, + z: torch.FloatTensor, + return_dict: bool = True + ) -> Union[DecoderOutput, torch.FloatTensor]: + r"""Decode a batch of images using a tiled decoder. + + Args: + When this option is enabled, the VAE will split the input tensor into tiles to compute decoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled + decoding is: different from non-tiled decoding due to each tile using a different decoder. + To avoid tiling artifacts, the tiles overlap and are blended together to form a smooth output. + You may still see tile-sized changes in the look of the output, but they should be much less noticeable. + z (`torch.FloatTensor`): Input batch of latent vectors. return_dict (`bool`, *optional*, defaults to + `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + _tile_overlap_factor = 1 - self.tile_overlap_factor + overlap_size = int(self.tile_latent_min_size + * _tile_overlap_factor) + blend_extent = int(self.tile_sample_min_size + * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + w = z.shape[3] + z = torch.cat([z, z[:, :, :, :w // 4]], dim=-1) + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + tile = z[:, :, i:i + self.tile_latent_min_size, :] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + vae_scale_factor = decoded.shape[-1] // tile.shape[-1] + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append( + self.blend_h( + tile[:, :, :row_limit, w * vae_scale_factor:], + tile[:, :, :row_limit, :w * vae_scale_factor], + tile.shape[-1] - w * vae_scale_factor)) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec, ) + + return DecoderOutput(sample=dec) + + self.vae.tiled_decode = tiled_decode.__get__(self.vae, AutoencoderKL) + + # 0. Default height and width to unet + height, width = self._default_height_width(height, width, image) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + control_image, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + controlnet_conditioning_scale, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + controlnet = self.controlnet._orig_mod if is_compiled_module( + self.controlnet) else self.controlnet + + if isinstance(controlnet, MultiControlNetModel) and isinstance( + controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale + ] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions if isinstance( + controlnet, ControlNetModel) else + controlnet.nets[0].config.global_pool_conditions) + guess_mode = guess_mode or global_pool_conditions + + # 3. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + # 4. Prepare image, and controlnet_conditioning_image + image = prepare_image(image) + + # 5. Prepare image + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + guess_mode=guess_mode, + ) + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + else: + assert False + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size + * num_images_per_prompt) + + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + views_scheduler_status = [copy.deepcopy(self.scheduler.__dict__)] + # value = torch.zeros_like(latents) + _, _, height, width = control_image.size() + tile_latent_min_size = context_size // self.vae_scale_factor + tile_overlap_factor = 0.5 + overlap_size = int(tile_latent_min_size * (1 - tile_overlap_factor)) + blend_extent = int(tile_latent_min_size * tile_overlap_factor) + row_limit = tile_latent_min_size - blend_extent + w = latents.shape[3] + latents = torch.cat([latents, latents[:, :, :, :overlap_size]], dim=-1) + control_image_extend = control_image[:, :, :, :overlap_size + * self.vae_scale_factor] + control_image = torch.cat([control_image, control_image_extend], + dim=-1) + + # 8. Denoising loop + num_warmup_steps = len( + timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latents_input, control_image_input = self.get_blocks( + latents, control_image, tile_latent_min_size, overlap_size) + rows = [] + for latents_input_, control_image_input_ in zip( + latents_input, control_image_input): + num_block = len(latents_input_) + # get batched latents_input + latents_input_ = torch.cat( + latents_input_[:num_block], dim=0) + # get batched prompt_embeds + prompt_embeds_ = torch.cat( + [prompt_embeds.chunk(2)[0]] * num_block + + [prompt_embeds.chunk(2)[1]] * num_block, + dim=0) + # get batched control_image_input + control_image_input_ = torch.cat( + [ + x[0, :, :, ][None, :, :, :] + for x in control_image_input_[:num_block] + ] + [ + x[1, :, :, ][None, :, :, :] + for x in control_image_input_[:num_block] + ], + dim=0) + latents_output = self.denoise_latents( + latents_input_, t, prompt_embeds_, + control_image_input_, controlnet_conditioning_scale, + guess_mode, cross_attention_kwargs, + do_classifier_free_guidance, guidance_scale, + extra_step_kwargs, views_scheduler_status) + rows.append(list(latents_output.chunk(num_block))) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, + blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + if j == 0: + tile = self.blend_h(row[-1], tile, blend_extent) + if i != len(rows) - 1: + if j == len(row) - 1: + result_row.append(tile[:, :, :row_limit, :]) + else: + result_row.append( + tile[:, :, :row_limit, :row_limit]) + else: + if j == len(row) - 1: + result_row.append(tile[:, :, :, :]) + else: + result_row.append(tile[:, :, :, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + latents = torch.cat(result_rows, dim=2) + + # call the callback, if provided + condition_i = i == len(timesteps) - 1 + condition_warm = (i + 1) > num_warmup_steps and ( + i + 1) % self.scheduler.order == 0 + if condition_i or condition_warm: + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + latents = latents[:, :, :, :w] + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr( + self, + 'final_offload_hook') and self.final_offload_hook is not None: + self.unet.to('cpu') + self.controlnet.to('cpu') + torch.cuda.empty_cache() + + if not output_type == 'latent': + image = self.vae.decode( + latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker( + image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess( + image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload last model to CPU + if hasattr( + self, + 'final_offload_hook') and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput( + images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/runtime/Lib/site_packages/models/cv/text_to_head/__init__.py b/runtime/Lib/site_packages/models/cv/text_to_head/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/text_to_head/text_to_head_model.py b/runtime/Lib/site_packages/models/cv/text_to_head/text_to_head_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ed09b8faf06ad0ab8ea422ab45948e305293d24f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/text_to_head/text_to_head_model.py @@ -0,0 +1,55 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from collections import OrderedDict + +import cv2 +import numpy as np +import torch +from diffusers import (ControlNetModel, DDIMScheduler, + StableDiffusionControlNetPipeline) +from diffusers.utils import load_image + +from modelscope.models import MODELS, TorchModel + + +@MODELS.register_module('text-to-head', 'text_to_head') +class TextToHeadModel(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + """The HeadReconModel is implemented based on HRN, publicly available at + https://github.com/youngLBW/HRN + + Args: + model_dir: the root directory of the model files + """ + super().__init__(model_dir, *args, **kwargs) + + self.model_dir = model_dir + + base_model_path = os.path.join(model_dir, 'base_model') + controlnet_path = os.path.join(model_dir, 'control_net') + + controlnet = ControlNetModel.from_pretrained( + controlnet_path, torch_dtype=torch.float16) + self.face_gen_pipeline = StableDiffusionControlNetPipeline.from_pretrained( + base_model_path, controlnet=controlnet, torch_dtype=torch.float16) + self.face_gen_pipeline.scheduler = DDIMScheduler.from_config( + self.face_gen_pipeline.scheduler.config) + self.face_gen_pipeline.enable_model_cpu_offload() + + self.add_prompt = ', 4K, good looking face, epic realistic, Sony a7, sharp, ' \ + 'skin detail pores, soft light, uniform illumination' + self.neg_prompt = 'ugly, cross eye, bangs, teeth, glasses, hat, dark, shadow' + + control_pose_path = os.path.join(self.model_dir, 'control_pose.jpg') + self.control_pose = load_image(control_pose_path) + + def forward(self, input): + prompt = input['text'] + self.add_prompt + image = self.face_gen_pipeline( + prompt, + negative_prompt=self.neg_prompt, + image=self.control_pose, + num_inference_steps=20).images[0] # PIL.Image + + return image diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6c2f89ee5b68ddbd54fe2382b1706fb6b84ffbc6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model_zoo import get_zennet + +else: + _import_structure = { + 'model_zoo': ['get_zennet'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/basic_blocks.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/basic_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..50548dcc901f19ef5eb14c4c682ae4ff63a3145a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/basic_blocks.py @@ -0,0 +1,1309 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import uuid + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from .global_utils import (create_netblock_list_from_str_inner, + get_right_parentheses_index) + + +class PlainNetBasicBlockClass(nn.Module): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=1, + no_create=False, + block_name=None, + **kwargs): + super(PlainNetBasicBlockClass, self).__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.no_create = no_create + self.block_name = block_name + if self.block_name is None: + self.block_name = 'uuid{}'.format(uuid.uuid4().hex) + + def forward(self, x): + raise RuntimeError('Not implemented') + + def __str__(self): + return type(self).__name__ + '({},{},{})'.format( + self.in_channels, self.out_channels, self.stride) + + def __repr__(self): + return type(self).__name__ + '({}|{},{},{})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride) + + def get_output_resolution(self, input_resolution): + raise RuntimeError('Not implemented') + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert PlainNetBasicBlockClass.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + in_channels = int(param_str_split[0]) + out_channels = int(param_str_split[1]) + stride = int(param_str_split[2]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + @classmethod + def is_instance_from_str(cls, s): + if s.startswith(cls.__name__ + '(') and s[-1] == ')': + return True + else: + return False + + +class AdaptiveAvgPool(PlainNetBasicBlockClass): + + def __init__(self, out_channels, output_size, no_create=False, **kwargs): + super(AdaptiveAvgPool, self).__init__(**kwargs) + self.in_channels = out_channels + self.out_channels = out_channels + self.output_size = output_size + self.no_create = no_create + if not no_create: + self.netblock = nn.AdaptiveAvgPool2d( + output_size=(self.output_size, self.output_size)) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return type(self).__name__ + '({},{})'.format( + self.out_channels // self.output_size**2, self.output_size) + + def __repr__(self): + return type(self).__name__ + '({}|{},{})'.format( + self.block_name, self.out_channels // self.output_size**2, + self.output_size) + + def get_output_resolution(self, input_resolution): + return self.output_size + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert AdaptiveAvgPool.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('AdaptiveAvgPool('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + out_channels = int(param_str_split[0]) + output_size = int(param_str_split[1]) + return AdaptiveAvgPool( + out_channels=out_channels, + output_size=output_size, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + +class BN(PlainNetBasicBlockClass): + + def __init__(self, + out_channels=None, + copy_from=None, + no_create=False, + **kwargs): + super(BN, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + assert isinstance(copy_from, nn.BatchNorm2d) + self.in_channels = copy_from.weight.shape[0] + self.out_channels = copy_from.weight.shape[0] + assert out_channels is None or out_channels == self.out_channels + self.netblock = copy_from + + else: + self.in_channels = out_channels + self.out_channels = out_channels + if no_create: + return + else: + self.netblock = nn.BatchNorm2d(num_features=self.out_channels) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return 'BN({})'.format(self.out_channels) + + def __repr__(self): + return 'BN({}|{})'.format(self.block_name, self.out_channels) + + def get_output_resolution(self, input_resolution): + return input_resolution + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert BN.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('BN('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + out_channels = int(param_str) + return BN( + out_channels=out_channels, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + +class ConvKX(PlainNetBasicBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + groups=1, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKX, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + assert isinstance(copy_from, nn.Conv2d) + self.in_channels = copy_from.in_channels + self.out_channels = copy_from.out_channels + self.kernel_size = copy_from.kernel_size[0] + self.stride = copy_from.stride[0] + self.groups = copy_from.groups + assert in_channels is None or in_channels == self.in_channels + assert out_channels is None or out_channels == self.out_channels + assert kernel_size is None or kernel_size == self.kernel_size + assert stride is None or stride == self.stride + self.netblock = copy_from + else: + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.groups = groups + self.kernel_size = kernel_size + self.padding = (self.kernel_size - 1) // 2 + if no_create or self.in_channels == 0 or self.out_channels == 0 or self.kernel_size == 0 \ + or self.stride == 0: + return + else: + self.netblock = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=False, + groups=self.groups) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return type(self).__name__ + '({},{},{},{})'.format( + self.in_channels, self.out_channels, self.kernel_size, self.stride) + + def __repr__(self): + return type(self).__name__ + '({}|{},{},{},{})'.format( + self.block_name, self.in_channels, self.out_channels, + self.kernel_size, self.stride) + + def get_output_resolution(self, input_resolution): + return input_resolution // self.stride + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert cls.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + split_str = param_str.split(',') + in_channels = int(split_str[0]) + out_channels = int(split_str[1]) + kernel_size = int(split_str[2]) + stride = int(split_str[3]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class ConvDW(PlainNetBasicBlockClass): + + def __init__(self, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvDW, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + assert isinstance(copy_from, nn.Conv2d) + self.in_channels = copy_from.in_channels + self.out_channels = copy_from.out_channels + self.kernel_size = copy_from.kernel_size[0] + self.stride = copy_from.stride[0] + assert self.in_channels == self.out_channels + assert out_channels is None or out_channels == self.out_channels + assert kernel_size is None or kernel_size == self.kernel_size + assert stride is None or stride == self.stride + + self.netblock = copy_from + else: + + self.in_channels = out_channels + self.out_channels = out_channels + self.stride = stride + self.kernel_size = kernel_size + + self.padding = (self.kernel_size - 1) // 2 + if no_create or self.in_channels == 0 or self.out_channels == 0 or self.kernel_size == 0 \ + or self.stride == 0: + return + else: + self.netblock = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + bias=False, + groups=self.in_channels) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return 'ConvDW({},{},{})'.format(self.out_channels, self.kernel_size, + self.stride) + + def __repr__(self): + return 'ConvDW({}|{},{},{})'.format(self.block_name, self.out_channels, + self.kernel_size, self.stride) + + def get_output_resolution(self, input_resolution): + return input_resolution // self.stride + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert ConvDW.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('ConvDW('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + split_str = param_str.split(',') + out_channels = int(split_str[0]) + kernel_size = int(split_str[1]) + stride = int(split_str[2]) + return ConvDW( + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class ConvKXG2(ConvKX): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKXG2, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + copy_from=copy_from, + no_create=no_create, + groups=2, + **kwargs) + + +class ConvKXG4(ConvKX): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKXG4, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + copy_from=copy_from, + no_create=no_create, + groups=4, + **kwargs) + + +class ConvKXG8(ConvKX): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKXG8, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + copy_from=copy_from, + no_create=no_create, + groups=8, + **kwargs) + + +class ConvKXG16(ConvKX): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKXG16, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + copy_from=copy_from, + no_create=no_create, + groups=16, + **kwargs) + + +class ConvKXG32(ConvKX): + + def __init__(self, + in_channels=None, + out_channels=None, + kernel_size=None, + stride=None, + copy_from=None, + no_create=False, + **kwargs): + super(ConvKXG32, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + copy_from=copy_from, + no_create=no_create, + groups=32, + **kwargs) + + +class Flatten(PlainNetBasicBlockClass): + + def __init__(self, out_channels, no_create=False, **kwargs): + super(Flatten, self).__init__(**kwargs) + self.in_channels = out_channels + self.out_channels = out_channels + self.no_create = no_create + + def forward(self, x): + return torch.flatten(x, 1) + + def __str__(self): + return 'Flatten({})'.format(self.out_channels) + + def __repr__(self): + return 'Flatten({}|{})'.format(self.block_name, self.out_channels) + + def get_output_resolution(self, input_resolution): + return 1 + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert Flatten.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('Flatten('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + out_channels = int(param_str) + return Flatten( + out_channels=out_channels, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class Linear(PlainNetBasicBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + bias=True, + copy_from=None, + no_create=False, + **kwargs): + super(Linear, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + assert isinstance(copy_from, nn.Linear) + self.in_channels = copy_from.weight.shape[1] + self.out_channels = copy_from.weight.shape[0] + self.use_bias = copy_from.bias is not None + assert in_channels is None or in_channels == self.in_channels + assert out_channels is None or out_channels == self.out_channels + + self.netblock = copy_from + else: + + self.in_channels = in_channels + self.out_channels = out_channels + self.use_bias = bias + if not no_create: + self.netblock = nn.Linear( + self.in_channels, self.out_channels, bias=self.use_bias) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return 'Linear({},{},{})'.format(self.in_channels, self.out_channels, + int(self.use_bias)) + + def __repr__(self): + return 'Linear({}|{},{},{})'.format(self.block_name, self.in_channels, + self.out_channels, + int(self.use_bias)) + + def get_output_resolution(self, input_resolution): + assert input_resolution == 1 + return 1 + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert Linear.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('Linear('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + split_str = param_str.split(',') + in_channels = int(split_str[0]) + out_channels = int(split_str[1]) + use_bias = int(split_str[2]) + + return Linear( + in_channels=in_channels, + out_channels=out_channels, + bias=use_bias == 1, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + +class MaxPool(PlainNetBasicBlockClass): + + def __init__(self, + out_channels, + kernel_size, + stride, + no_create=False, + **kwargs): + super(MaxPool, self).__init__(**kwargs) + self.in_channels = out_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.padding = (kernel_size - 1) // 2 + self.no_create = no_create + if not no_create: + self.netblock = nn.MaxPool2d( + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding) + + def forward(self, x): + return self.netblock(x) + + def __str__(self): + return 'MaxPool({},{},{})'.format(self.out_channels, self.kernel_size, + self.stride) + + def __repr__(self): + return 'MaxPool({}|{},{},{})'.format(self.block_name, + self.out_channels, + self.kernel_size, self.stride) + + def get_output_resolution(self, input_resolution): + return input_resolution // self.stride + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert MaxPool.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('MaxPool('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + out_channels = int(param_str_split[0]) + kernel_size = int(param_str_split[1]) + stride = int(param_str_split[2]) + return MaxPool( + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class Sequential(PlainNetBasicBlockClass): + + def __init__(self, block_list, no_create=False, **kwargs): + super(Sequential, self).__init__(**kwargs) + self.block_list = block_list + if not no_create: + self.module_list = nn.ModuleList(block_list) + self.in_channels = block_list[0].in_channels + self.out_channels = block_list[-1].out_channels + self.no_create = no_create + res = 1024 + for block in self.block_list: + res = block.get_output_resolution(res) + self.stride = 1024 // res + + def forward(self, x): + output = x + for inner_block in self.block_list: + output = inner_block(output) + return output + + def __str__(self): + s = 'Sequential(' + for inner_block in self.block_list: + s += str(inner_block) + s += ')' + return s + + def __repr__(self): + return str(self) + + def get_output_resolution(self, input_resolution): + the_res = input_resolution + for the_block in self.block_list: + the_res = the_block.get_output_resolution(the_res) + return the_res + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert Sequential.is_instance_from_str(s) + the_right_paraen_idx = get_right_parentheses_index(s) + param_str = s[len('Sequential(') + 1:the_right_paraen_idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, netblocks_dict=bottom_basic_dict, no_create=no_create) + assert len(remaining_s) == 0 + if the_block_list is None or len(the_block_list) == 0: + return None, '' + return Sequential( + block_list=the_block_list, + no_create=no_create, + block_name=tmp_block_name), '' + + +class MultiSumBlock(PlainNetBasicBlockClass): + + def __init__(self, block_list, no_create=False, **kwargs): + super(MultiSumBlock, self).__init__(**kwargs) + self.block_list = block_list + if not no_create: + self.module_list = nn.ModuleList(block_list) + self.in_channels = np.max([x.in_channels for x in block_list]) + self.out_channels = np.max([x.out_channels for x in block_list]) + self.no_create = no_create + + res = 1024 + res = self.block_list[0].get_output_resolution(res) + self.stride = 1024 // res + + def forward(self, x): + output = self.block_list[0](x) + for inner_block in self.block_list[1:]: + output2 = inner_block(x) + output = output + output2 + return output + + def __str__(self): + s = 'MultiSumBlock({}|'.format(self.block_name) + for inner_block in self.block_list: + s += str(inner_block) + ';' + s = s[:-1] + s += ')' + return s + + def __repr__(self): + return str(self) + + def get_output_resolution(self, input_resolution): + the_res = self.block_list[0].get_output_resolution(input_resolution) + for the_block in self.block_list: + assert the_res == the_block.get_output_resolution(input_resolution) + + return the_res + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert MultiSumBlock.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('MultiSumBlock('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + the_s = param_str + + the_block_list = [] + while len(the_s) > 0: + tmp_block_list, remaining_s = create_netblock_list_from_str_inner( + the_s, netblocks_dict=bottom_basic_dict, no_create=no_create) + the_s = remaining_s + if tmp_block_list is None: + pass + elif len(tmp_block_list) == 1: + the_block_list.append(tmp_block_list[0]) + else: + the_block_list.append( + Sequential(block_list=tmp_block_list, no_create=no_create)) + pass + + if len(the_block_list) == 0: + return None, s[idx + 1:] + + return MultiSumBlock( + block_list=the_block_list, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + +class MultiCatBlock(PlainNetBasicBlockClass): + + def __init__(self, block_list, no_create=False, **kwargs): + super(MultiCatBlock, self).__init__(**kwargs) + self.block_list = block_list + if not no_create: + self.module_list = nn.ModuleList(block_list) + self.in_channels = np.max([x.in_channels for x in block_list]) + self.out_channels = np.sum([x.out_channels for x in block_list]) + self.no_create = no_create + + res = 1024 + res = self.block_list[0].get_output_resolution(res) + self.stride = 1024 // res + + def forward(self, x): + output_list = [] + for inner_block in self.block_list: + output = inner_block(x) + output_list.append(output) + + return torch.cat(output_list, dim=1) + + def __str__(self): + s = 'MultiCatBlock({}|'.format(self.block_name) + for inner_block in self.block_list: + s += str(inner_block) + ';' + + s = s[:-1] + s += ')' + return s + + def __repr__(self): + return str(self) + + def get_output_resolution(self, input_resolution): + the_res = self.block_list[0].get_output_resolution(input_resolution) + for the_block in self.block_list: + assert the_res == the_block.get_output_resolution(input_resolution) + + return the_res + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert MultiCatBlock.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('MultiCatBlock('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + the_s = param_str + + the_block_list = [] + while len(the_s) > 0: + tmp_block_list, remaining_s = create_netblock_list_from_str_inner( + the_s, netblocks_dict=bottom_basic_dict, no_create=no_create) + the_s = remaining_s + if tmp_block_list is None: + pass + elif len(tmp_block_list) == 1: + the_block_list.append(tmp_block_list[0]) + else: + the_block_list.append( + Sequential(block_list=tmp_block_list, no_create=no_create)) + + if len(the_block_list) == 0: + return None, s[idx + 1:] + + return MultiCatBlock( + block_list=the_block_list, + block_name=tmp_block_name, + no_create=no_create), s[idx + 1:] + + +class RELU(PlainNetBasicBlockClass): + + def __init__(self, out_channels, no_create=False, **kwargs): + super(RELU, self).__init__(**kwargs) + self.in_channels = out_channels + self.out_channels = out_channels + self.no_create = no_create + + def forward(self, x): + return F.relu(x) + + def __str__(self): + return 'RELU({})'.format(self.out_channels) + + def __repr__(self): + return 'RELU({}|{})'.format(self.block_name, self.out_channels) + + def get_output_resolution(self, input_resolution): + return input_resolution + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert RELU.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('RELU('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + out_channels = int(param_str) + return RELU( + out_channels=out_channels, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class ResBlock(PlainNetBasicBlockClass): + """ + ResBlock(in_channles, inner_blocks_str). If in_channels is missing, use block_list[0].in_channels as in_channels + """ + + def __init__(self, + block_list, + in_channels=None, + stride=None, + no_create=False, + **kwargs): + super(ResBlock, self).__init__(**kwargs) + self.block_list = block_list + self.stride = stride + self.no_create = no_create + if not no_create: + self.module_list = nn.ModuleList(block_list) + + if in_channels is None: + self.in_channels = block_list[0].in_channels + else: + self.in_channels = in_channels + self.out_channels = block_list[-1].out_channels + + if self.stride is None: + tmp_input_res = 1024 + tmp_output_res = self.get_output_resolution(tmp_input_res) + self.stride = tmp_input_res // tmp_output_res + + self.proj = None + if self.stride > 1 or self.in_channels != self.out_channels: + self.proj = nn.Sequential( + nn.Conv2d(self.in_channels, self.out_channels, 1, self.stride), + nn.BatchNorm2d(self.out_channels), + ) + + def forward(self, x): + if len(self.block_list) == 0: + return x + + output = x + for inner_block in self.block_list: + output = inner_block(output) + + if self.proj is not None: + output = output + self.proj(x) + else: + output = output + x + + return output + + def __str__(self): + s = 'ResBlock({},{},'.format(self.in_channels, self.stride) + for inner_block in self.block_list: + s += str(inner_block) + + s += ')' + return s + + def __repr__(self): + s = 'ResBlock({}|{},{},'.format(self.block_name, self.in_channels, + self.stride) + for inner_block in self.block_list: + s += str(inner_block) + + s += ')' + return s + + def get_output_resolution(self, input_resolution): + the_res = input_resolution + for the_block in self.block_list: + the_res = the_block.get_output_resolution(the_res) + + return the_res + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert ResBlock.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + the_stride = None + param_str = s[len('ResBlock('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + first_comma_index = param_str.find(',') + if first_comma_index < 0 or not param_str[0:first_comma_index].isdigit( + ): + in_channels = None + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + else: + in_channels = int(param_str[0:first_comma_index]) + param_str = param_str[first_comma_index + 1:] + second_comma_index = param_str.find(',') + if second_comma_index < 0 or not param_str[ + 0:second_comma_index].isdigit(): + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + else: + the_stride = int(param_str[0:second_comma_index]) + param_str = param_str[second_comma_index + 1:] + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + pass + pass + + assert len(remaining_s) == 0 + if the_block_list is None or len(the_block_list) == 0: + return None, s[idx + 1:] + return ResBlock( + block_list=the_block_list, + in_channels=in_channels, + stride=the_stride, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class ResBlockProj(PlainNetBasicBlockClass): + """ + ResBlockProj(in_channles, inner_blocks_str). If in_channels is missing, use block_list[0].in_channels as in_channels + """ + + def __init__(self, + block_list, + in_channels=None, + stride=None, + no_create=False, + **kwargs): + super(ResBlockProj, self).__init__(**kwargs) + self.block_list = block_list + self.stride = stride + self.no_create = no_create + if not no_create: + self.module_list = nn.ModuleList(block_list) + + if in_channels is None: + self.in_channels = block_list[0].in_channels + else: + self.in_channels = in_channels + self.out_channels = block_list[-1].out_channels + + if self.stride is None: + tmp_input_res = 1024 + tmp_output_res = self.get_output_resolution(tmp_input_res) + self.stride = tmp_input_res // tmp_output_res + + self.proj = nn.Sequential( + nn.Conv2d(self.in_channels, self.out_channels, 1, self.stride), + nn.BatchNorm2d(self.out_channels), + ) + + def forward(self, x): + if len(self.block_list) == 0: + return x + + output = x + for inner_block in self.block_list: + output = inner_block(output) + output = output + self.proj(x) + return output + + def __str__(self): + s = 'ResBlockProj({},{},'.format(self.in_channels, self.stride) + for inner_block in self.block_list: + s += str(inner_block) + + s += ')' + return s + + def __repr__(self): + s = 'ResBlockProj({}|{},{},'.format(self.block_name, self.in_channels, + self.stride) + for inner_block in self.block_list: + s += str(inner_block) + + s += ')' + return s + + def get_output_resolution(self, input_resolution): + the_res = input_resolution + for the_block in self.block_list: + the_res = the_block.get_output_resolution(the_res) + + return the_res + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert ResBlockProj.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + the_stride = None + param_str = s[len('ResBlockProj('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + first_comma_index = param_str.find(',') + if first_comma_index < 0 or not param_str[0:first_comma_index].isdigit( + ): + in_channels = None + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + else: + in_channels = int(param_str[0:first_comma_index]) + param_str = param_str[first_comma_index + 1:] + second_comma_index = param_str.find(',') + if second_comma_index < 0 or not param_str[ + 0:second_comma_index].isdigit(): + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + else: + the_stride = int(param_str[0:second_comma_index]) + param_str = param_str[second_comma_index + 1:] + the_block_list, remaining_s = create_netblock_list_from_str_inner( + param_str, + netblocks_dict=bottom_basic_dict, + no_create=no_create) + pass + pass + + assert len(remaining_s) == 0 + if the_block_list is None or len(the_block_list) == 0: + return None, s[idx + 1:] + return ResBlockProj( + block_list=the_block_list, + in_channels=in_channels, + stride=the_stride, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class SE(PlainNetBasicBlockClass): + + def __init__(self, + out_channels=None, + copy_from=None, + no_create=False, + **kwargs): + super(SE, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + raise RuntimeError('Not implemented') + else: + self.in_channels = out_channels + self.out_channels = out_channels + self.se_ratio = 0.25 + self.se_channels = max( + 1, int(round(self.out_channels * self.se_ratio))) + if no_create or self.out_channels == 0: + return + else: + self.netblock = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d( + in_channels=self.out_channels, + out_channels=self.se_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), nn.BatchNorm2d(self.se_channels), + nn.ReLU(), + nn.Conv2d( + in_channels=self.se_channels, + out_channels=self.out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False), nn.BatchNorm2d(self.out_channels), + nn.Sigmoid()) + + def forward(self, x): + se_x = self.netblock(x) + return se_x * x + + def __str__(self): + return 'SE({})'.format(self.out_channels) + + def __repr__(self): + return 'SE({}|{})'.format(self.block_name, self.out_channels) + + def get_output_resolution(self, input_resolution): + return input_resolution + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert SE.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('SE('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + out_channels = int(param_str) + return SE( + out_channels=out_channels, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +class SwishImplementation(torch.autograd.Function): + + @staticmethod + def forward(ctx, i): + result = i * torch.sigmoid(i) + ctx.save_for_backward(i) + return result + + @staticmethod + def backward(ctx, grad_output): + i = ctx.saved_variables[0] + sigmoid_i = torch.sigmoid(i) + return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i))) + + +class Swish(PlainNetBasicBlockClass): + + def __init__(self, + out_channels=None, + copy_from=None, + no_create=False, + **kwargs): + super(Swish, self).__init__(**kwargs) + self.no_create = no_create + + if copy_from is not None: + raise RuntimeError('Not implemented') + else: + self.in_channels = out_channels + self.out_channels = out_channels + + def forward(self, x): + return SwishImplementation.apply(x) + + def __str__(self): + return 'Swish({})'.format(self.out_channels) + + def __repr__(self): + return 'Swish({}|{})'.format(self.block_name, self.out_channels) + + def get_output_resolution(self, input_resolution): + return input_resolution + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert Swish.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len('Swish('):idx] + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + out_channels = int(param_str) + return Swish( + out_channels=out_channels, + no_create=no_create, + block_name=tmp_block_name), s[idx + 1:] + + +bottom_basic_dict = { + 'AdaptiveAvgPool': AdaptiveAvgPool, + 'BN': BN, + 'ConvDW': ConvDW, + 'ConvKX': ConvKX, + 'ConvKXG2': ConvKXG2, + 'ConvKXG4': ConvKXG4, + 'ConvKXG8': ConvKXG8, + 'ConvKXG16': ConvKXG16, + 'ConvKXG32': ConvKXG32, + 'Flatten': Flatten, + 'Linear': Linear, + 'MaxPool': MaxPool, + 'PlainNetBasicBlockClass': PlainNetBasicBlockClass, + 'RELU': RELU, + 'SE': SE, + 'Swish': Swish, +} + + +def register_netblocks_dict(netblocks_dict: dict): + this_py_file_netblocks_dict = { + 'MultiSumBlock': MultiSumBlock, + 'MultiCatBlock': MultiCatBlock, + 'ResBlock': ResBlock, + 'ResBlockProj': ResBlockProj, + 'Sequential': Sequential, + } + netblocks_dict.update(this_py_file_netblocks_dict) + netblocks_dict.update(bottom_basic_dict) + return netblocks_dict diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/global_utils.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/global_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..022c61a09b2b8b84cb9de28aca86f04943b3992b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/global_utils.py @@ -0,0 +1,65 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + + +def smart_round(x, base=None): + if base is None: + if x > 32 * 8: + round_base = 32 + elif x > 16 * 8: + round_base = 16 + else: + round_base = 8 + else: + round_base = base + + return max(round_base, round(x / float(round_base)) * round_base) + + +def get_right_parentheses_index(s): + left_paren_count = 0 + for index, x in enumerate(s): + + if x == '(': + left_paren_count += 1 + elif x == ')': + left_paren_count -= 1 + if left_paren_count == 0: + return index + else: + pass + return None + + +def create_netblock_list_from_str_inner(s, + no_create=False, + netblocks_dict=None, + **kwargs): + block_list = [] + while len(s) > 0: + is_found_block_class = False + for the_block_class_name in netblocks_dict.keys(): + tmp_idx = s.find('(') + if tmp_idx > 0 and s[0:tmp_idx] == the_block_class_name: + is_found_block_class = True + the_block_class = netblocks_dict[the_block_class_name] + the_block, remaining_s = the_block_class.create_from_str( + s, no_create=no_create, **kwargs) + if the_block is not None: + block_list.append(the_block) + s = remaining_s + if len(s) > 0 and s[0] == ';': + return block_list, s[1:] + break + assert is_found_block_class + return block_list, '' + + +def create_netblock_list_from_str(s, + no_create=False, + netblocks_dict=None, + **kwargs): + the_list, remaining_s = create_netblock_list_from_str_inner( + s, no_create=no_create, netblocks_dict=netblocks_dict, **kwargs) + assert len(remaining_s) == 0 + return the_list diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/master_net.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/master_net.py new file mode 100644 index 0000000000000000000000000000000000000000..e2bc47e0fe06cc47014e60ccdbfd9617de39e1c4 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/master_net.py @@ -0,0 +1,94 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import torch +import torch.nn.functional as F +from torch import nn + +from . import basic_blocks, plain_net_utils + + +class PlainNet(plain_net_utils.PlainNet): + + def __init__(self, + argv=None, + opt=None, + num_classes=None, + plainnet_struct=None, + no_create=False, + no_reslink=None, + no_BN=None, + use_se=None, + dropout=None, + **kwargs): + + module_opt = None + + if no_BN is None: + if module_opt is not None: + no_BN = module_opt.no_BN + else: + no_BN = False + + if no_reslink is None: + if module_opt is not None: + no_reslink = module_opt.no_reslink + else: + no_reslink = False + + if use_se is None: + if module_opt is not None: + use_se = module_opt.use_se + else: + use_se = False + + if dropout is None: + if module_opt is not None: + self.dropout = module_opt.dropout + else: + self.dropout = None + else: + self.dropout = dropout + + super(PlainNet, self).__init__( + argv=argv, + opt=opt, + num_classes=num_classes, + plainnet_struct=plainnet_struct, + no_create=no_create, + no_reslink=no_reslink, + no_BN=no_BN, + use_se=use_se, + **kwargs) + self.last_channels = self.block_list[-1].out_channels + self.fc_linear = basic_blocks.Linear( + in_channels=self.last_channels, + out_channels=self.num_classes, + no_create=no_create) + + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + self.use_se = use_se + + for layer in self.modules(): + if isinstance(layer, nn.BatchNorm2d): + layer.eps = 1e-3 + + def forward(self, x): + output = x + for block_id, the_block in enumerate(self.block_list): + output = the_block(output) + if self.dropout is not None: + dropout_p = float(block_id) / len( + self.block_list) * self.dropout + output = F.dropout( + output, dropout_p, training=self.training, inplace=True) + + output = F.adaptive_avg_pool2d(output, output_size=1) + if self.dropout is not None: + output = F.dropout( + output, self.dropout, training=self.training, inplace=True) + output = torch.flatten(output, 1) + output = self.fc_linear(output) + return output diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/model_zoo.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..a49b053be8deabc8ffab9642e70bfadcab520b2e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/model_zoo.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +from . import master_net + + +def get_zennet(): + model_plainnet_str = ( + 'SuperConvK3BNRELU(3,32,2,1)' + 'SuperResK1K5K1(32,80,2,32,1)SuperResK1K7K1(80,432,2,128,5)' + 'SuperResK1K7K1(432,640,2,192,3)SuperResK1K7K1(640,1008,1,160,5)' + 'SuperResK1K7K1(1008,976,1,160,4)SuperResK1K5K1(976,2304,2,384,5)' + 'SuperResK1K5K1(2304,2496,1,384,5)SuperConvK1BNRELU(2496,3072,1,1)') + use_SE = False + num_classes = 1000 + + model = master_net.PlainNet( + num_classes=num_classes, + plainnet_struct=model_plainnet_str, + use_se=use_SE) + + return model diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/plain_net_utils.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/plain_net_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1f5c88527cfbe0f72cd7d042c858b69378bf917a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/plain_net_utils.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +from torch import nn + +from . import (basic_blocks, super_blocks, super_res_idwexkx, super_res_k1kxk1, + super_res_kxkx) +from .global_utils import create_netblock_list_from_str_inner + + +class PlainNet(nn.Module): + + def __init__(self, + argv=None, + opt=None, + num_classes=None, + plainnet_struct=None, + no_create=False, + **kwargs): + super(PlainNet, self).__init__() + self.argv = argv + self.opt = opt + self.num_classes = num_classes + self.plainnet_struct = plainnet_struct + + self.module_opt = None + + if self.num_classes is None: + self.num_classes = self.module_opt.num_classes + + if self.plainnet_struct is None and self.module_opt.plainnet_struct is not None: + self.plainnet_struct = self.module_opt.plainnet_struct + + if self.plainnet_struct is None: + if hasattr(opt, 'plainnet_struct_txt' + ) and opt.plainnet_struct_txt is not None: + plainnet_struct_txt = opt.plainnet_struct_txt + else: + plainnet_struct_txt = self.module_opt.plainnet_struct_txt + + if plainnet_struct_txt is not None: + with open(plainnet_struct_txt, 'r', encoding='utf-8') as fid: + the_line = fid.readlines()[0].strip() + self.plainnet_struct = the_line + pass + + if self.plainnet_struct is None: + return + + the_s = self.plainnet_struct + + block_list, remaining_s = create_netblock_list_from_str_inner( + the_s, + netblocks_dict=_all_netblocks_dict_, + no_create=no_create, + **kwargs) + assert len(remaining_s) == 0 + + self.block_list = block_list + if not no_create: + self.module_list = nn.ModuleList(block_list) + + def forward(self, x): + output = x + for the_block in self.block_list: + output = the_block(output) + return output + + def __str__(self): + s = '' + for the_block in self.block_list: + s += str(the_block) + return s + + def __repr__(self): + return str(self) + + +_all_netblocks_dict_ = {} +_all_netblocks_dict_ = basic_blocks.register_netblocks_dict( + _all_netblocks_dict_) +_all_netblocks_dict_ = super_blocks.register_netblocks_dict( + _all_netblocks_dict_) +_all_netblocks_dict_ = super_res_kxkx.register_netblocks_dict( + _all_netblocks_dict_) +_all_netblocks_dict_ = super_res_k1kxk1.register_netblocks_dict( + _all_netblocks_dict_) +_all_netblocks_dict_ = super_res_idwexkx.register_netblocks_dict( + _all_netblocks_dict_) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_blocks.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..25862255fb9f1f1cdca4dac541a35d09f2a315cb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_blocks.py @@ -0,0 +1,228 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import uuid + +from torch import nn + +from . import basic_blocks, global_utils +from .global_utils import get_right_parentheses_index + + +class PlainNetSuperBlockClass(basic_blocks.PlainNetBasicBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + no_create=False, + **kwargs): + super(PlainNetSuperBlockClass, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.sub_layers = sub_layers + self.no_create = no_create + self.block_list = None + self.module_list = None + + def forward(self, x): + output = x + for block in self.block_list: + output = block(output) + return output + + def __str__(self): + return type(self).__name__ + '({},{},{},{})'.format( + self.in_channels, self.out_channels, self.stride, self.sub_layers) + + def __repr__(self): + return type(self).__name__ + '({}|{},{},{},{})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride, + self.sub_layers) + + def get_output_resolution(self, input_resolution): + resolution = input_resolution + for block in self.block_list: + resolution = block.get_output_resolution(resolution) + return resolution + + @classmethod + def create_from_str(cls, s, no_create=False, **kwargs): + assert cls.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + in_channels = int(param_str_split[0]) + out_channels = int(param_str_split[1]) + stride = int(param_str_split[2]) + sub_layers = int(param_str_split[3]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + sub_layers=sub_layers, + block_name=tmp_block_name, + no_create=no_create, + **kwargs), s[idx + 1:] + + +class SuperConvKXBNRELU(PlainNetSuperBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + kernel_size=None, + no_create=False, + no_reslink=False, + no_BN=False, + **kwargs): + super(SuperConvKXBNRELU, self).__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.sub_layers = sub_layers + self.kernel_size = kernel_size + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + + full_str = '' + last_channels = in_channels + current_stride = stride + for i in range(self.sub_layers): + if not self.no_BN: + inner_str = 'ConvKX({},{},{},{})BN({})RELU({})'.format( + last_channels, self.out_channels, self.kernel_size, + current_stride, self.out_channels, self.out_channels) + else: + inner_str = 'ConvKX({},{},{},{})RELU({})'.format( + last_channels, self.out_channels, self.kernel_size, + current_stride, self.out_channels) + full_str += inner_str + + last_channels = out_channels + current_stride = 1 + pass + + netblocks_dict = basic_blocks.register_netblocks_dict({}) + self.block_list = global_utils.create_netblock_list_from_str( + full_str, + no_create=no_create, + netblocks_dict=netblocks_dict, + no_reslink=no_reslink, + no_BN=no_BN) + if not no_create: + self.module_list = nn.ModuleList(self.block_list) + else: + self.module_list = None + + def __str__(self): + return type(self).__name__ + '({},{},{},{})'.format( + self.in_channels, self.out_channels, self.stride, self.sub_layers) + + def __repr__(self): + return type( + self + ).__name__ + '({}|in={},out={},stride={},sub_layers={},kernel_size={})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride, + self.sub_layers, self.kernel_size) + + +class SuperConvK1BNRELU(SuperConvKXBNRELU): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperConvK1BNRELU, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + sub_layers=sub_layers, + kernel_size=1, + no_create=no_create, + **kwargs) + + +class SuperConvK3BNRELU(SuperConvKXBNRELU): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperConvK3BNRELU, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + sub_layers=sub_layers, + kernel_size=3, + no_create=no_create, + **kwargs) + + +class SuperConvK5BNRELU(SuperConvKXBNRELU): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperConvK5BNRELU, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + sub_layers=sub_layers, + kernel_size=5, + no_create=no_create, + **kwargs) + + +class SuperConvK7BNRELU(SuperConvKXBNRELU): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperConvK7BNRELU, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + sub_layers=sub_layers, + kernel_size=7, + no_create=no_create, + **kwargs) + + +def register_netblocks_dict(netblocks_dict: dict): + this_py_file_netblocks_dict = { + 'SuperConvK1BNRELU': SuperConvK1BNRELU, + 'SuperConvK3BNRELU': SuperConvK3BNRELU, + 'SuperConvK5BNRELU': SuperConvK5BNRELU, + 'SuperConvK7BNRELU': SuperConvK7BNRELU, + } + netblocks_dict.update(this_py_file_netblocks_dict) + return netblocks_dict diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_idwexkx.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_idwexkx.py new file mode 100644 index 0000000000000000000000000000000000000000..7d0050699b8714d66d6b7ccaa040dca31093de8a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_idwexkx.py @@ -0,0 +1,451 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import uuid + +from torch import nn + +from . import basic_blocks, global_utils +from .global_utils import get_right_parentheses_index +from .super_blocks import PlainNetSuperBlockClass + + +class SuperResIDWEXKX(PlainNetSuperBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + kernel_size=None, + expension=None, + no_create=False, + no_reslink=False, + no_BN=False, + use_se=False, + **kwargs): + super(SuperResIDWEXKX, self).__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.bottleneck_channels = bottleneck_channels + self.sub_layers = sub_layers + self.kernel_size = kernel_size + self.expension = expension + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + + self.use_se = use_se + + full_str = '' + last_channels = in_channels + current_stride = stride + for i in range(self.sub_layers): + inner_str = '' + dw_channels = global_utils.smart_round( + self.bottleneck_channels * self.expension, base=8) + inner_str += 'ConvKX({},{},{},{})'.format(last_channels, + dw_channels, 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(dw_channels) + inner_str += 'RELU({})'.format(dw_channels) + + inner_str += 'ConvDW({},{},{})'.format(dw_channels, + self.kernel_size, + current_stride) + if not self.no_BN: + inner_str += 'BN({})'.format(dw_channels) + inner_str += 'RELU({})'.format(dw_channels) + if self.use_se: + inner_str += 'SE({})'.format(dw_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(dw_channels, + bottleneck_channels, 1, + 1) + if not self.no_BN: + inner_str += 'BN({})'.format(bottleneck_channels) + + if not self.no_reslink: + if i == 0: + res_str = 'ResBlockProj({})RELU({})'.format( + inner_str, self.out_channels) + else: + res_str = 'ResBlock({})RELU({})'.format( + inner_str, self.out_channels) + + else: + res_str = '{}RELU({})'.format(inner_str, self.out_channels) + + full_str += res_str + + inner_str = '' + dw_channels = global_utils.smart_round( + self.out_channels * self.expension, base=8) + inner_str += 'ConvKX({},{},{},{})'.format(bottleneck_channels, + dw_channels, 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(dw_channels) + inner_str += 'RELU({})'.format(dw_channels) + + inner_str += 'ConvDW({},{},{})'.format(dw_channels, + self.kernel_size, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(dw_channels) + inner_str += 'RELU({})'.format(dw_channels) + if self.use_se: + inner_str += 'SE({})'.format(dw_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(dw_channels, + self.out_channels, 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.out_channels) + + if not self.no_reslink: + res_str = 'ResBlock({})RELU({})'.format( + inner_str, self.out_channels) + else: + res_str = '{}RELU({})'.format(inner_str, self.out_channels) + + full_str += res_str + last_channels = out_channels + current_stride = 1 + pass + + netblocks_dict = basic_blocks.register_netblocks_dict({}) + self.block_list = global_utils.create_netblock_list_from_str( + full_str, + netblocks_dict=netblocks_dict, + no_create=no_create, + no_reslink=no_reslink, + no_BN=no_BN, + **kwargs) + if not no_create: + self.module_list = nn.ModuleList(self.block_list) + else: + self.module_list = None + + def __str__(self): + return type(self).__name__ + '({},{},{},{},{})'.format( + self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers) + + def __repr__(self): + return type( + self + ).__name__ + '({}|in={},out={},stride={},btl_channels={},sub_layers={},kernel_size={})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers, self.kernel_size) + + @classmethod + def create_from_str(cls, s, **kwargs): + assert cls.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + in_channels = int(param_str_split[0]) + out_channels = int(param_str_split[1]) + stride = int(param_str_split[2]) + bottleneck_channels = int(param_str_split[3]) + sub_layers = int(param_str_split[4]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + block_name=tmp_block_name, + **kwargs), s[idx + 1:] + + +class SuperResIDWE1K3(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE1K3, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + expension=1.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE2K3(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE2K3, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + expension=2.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE4K3(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE4K3, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + expension=4.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE6K3(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE6K3, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + expension=6.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE1K5(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE1K5, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + expension=1.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE2K5(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE2K5, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + expension=2.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE4K5(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE4K5, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + expension=4.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE6K5(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE6K5, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + expension=6.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE1K7(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE1K7, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + expension=1.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE2K7(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE2K7, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + expension=2.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE4K7(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE4K7, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + expension=4.0, + no_create=no_create, + **kwargs) + + +class SuperResIDWE6K7(SuperResIDWEXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResIDWE6K7, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + expension=6.0, + no_create=no_create, + **kwargs) + + +def register_netblocks_dict(netblocks_dict: dict): + this_py_file_netblocks_dict = { + 'SuperResIDWE1K3': SuperResIDWE1K3, + 'SuperResIDWE2K3': SuperResIDWE2K3, + 'SuperResIDWE4K3': SuperResIDWE4K3, + 'SuperResIDWE6K3': SuperResIDWE6K3, + 'SuperResIDWE1K5': SuperResIDWE1K5, + 'SuperResIDWE2K5': SuperResIDWE2K5, + 'SuperResIDWE4K5': SuperResIDWE4K5, + 'SuperResIDWE6K5': SuperResIDWE6K5, + 'SuperResIDWE1K7': SuperResIDWE1K7, + 'SuperResIDWE2K7': SuperResIDWE2K7, + 'SuperResIDWE4K7': SuperResIDWE4K7, + 'SuperResIDWE6K7': SuperResIDWE6K7, + } + netblocks_dict.update(this_py_file_netblocks_dict) + return netblocks_dict diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_k1kxk1.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_k1kxk1.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca687420ca27df986f396148405059289a9bff8 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_k1kxk1.py @@ -0,0 +1,238 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import uuid + +from torch import nn + +from . import basic_blocks, global_utils +from .global_utils import get_right_parentheses_index +from .super_blocks import PlainNetSuperBlockClass + + +class SuperResK1KXK1(PlainNetSuperBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + kernel_size=None, + no_create=False, + no_reslink=False, + no_BN=False, + use_se=False, + **kwargs): + super(SuperResK1KXK1, self).__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.bottleneck_channels = bottleneck_channels + self.sub_layers = sub_layers + self.kernel_size = kernel_size + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + self.use_se = use_se + + full_str = '' + last_channels = in_channels + current_stride = stride + for i in range(self.sub_layers): + inner_str = '' + + inner_str += 'ConvKX({},{},{},{})'.format(last_channels, + self.bottleneck_channels, + 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.bottleneck_channels) + inner_str += 'RELU({})'.format(self.bottleneck_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(self.bottleneck_channels, + self.bottleneck_channels, + self.kernel_size, + current_stride) + if not self.no_BN: + inner_str += 'BN({})'.format(self.bottleneck_channels) + inner_str += 'RELU({})'.format(self.bottleneck_channels) + if self.use_se: + inner_str += 'SE({})'.format(bottleneck_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(self.bottleneck_channels, + self.out_channels, 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.out_channels) + + if not self.no_reslink: + if i == 0: + res_str = 'ResBlockProj({})RELU({})'.format( + inner_str, out_channels) + else: + res_str = 'ResBlock({})RELU({})'.format( + inner_str, out_channels) + else: + res_str = '{}RELU({})'.format(inner_str, out_channels) + + full_str += res_str + + inner_str = '' + inner_str += 'ConvKX({},{},{},{})'.format(self.out_channels, + self.bottleneck_channels, + 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.bottleneck_channels) + inner_str += 'RELU({})'.format(self.bottleneck_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(self.bottleneck_channels, + self.bottleneck_channels, + self.kernel_size, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.bottleneck_channels) + inner_str += 'RELU({})'.format(self.bottleneck_channels) + if self.use_se: + inner_str += 'SE({})'.format(bottleneck_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(self.bottleneck_channels, + self.out_channels, 1, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.out_channels) + + if not self.no_reslink: + res_str = 'ResBlock({})RELU({})'.format( + inner_str, out_channels) + else: + res_str = '{}RELU({})'.format(inner_str, out_channels) + + full_str += res_str + + last_channels = out_channels + current_stride = 1 + pass + + netblocks_dict = basic_blocks.register_netblocks_dict({}) + self.block_list = global_utils.create_netblock_list_from_str( + full_str, + netblocks_dict=netblocks_dict, + no_create=no_create, + no_reslink=no_reslink, + no_BN=no_BN, + **kwargs) + if not no_create: + self.module_list = nn.ModuleList(self.block_list) + else: + self.module_list = None + + def __str__(self): + return type(self).__name__ + '({},{},{},{},{})'.format( + self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers) + + def __repr__(self): + return type( + self + ).__name__ + '({}|in={},out={},stride={},btl_channels={},sub_layers={},kernel_size={})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers, self.kernel_size) + + @classmethod + def create_from_str(cls, s, **kwargs): + assert cls.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + in_channels = int(param_str_split[0]) + out_channels = int(param_str_split[1]) + stride = int(param_str_split[2]) + bottleneck_channels = int(param_str_split[3]) + sub_layers = int(param_str_split[4]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + block_name=tmp_block_name, + **kwargs), s[idx + 1:] + + +class SuperResK1K3K1(SuperResK1KXK1): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK1K3K1, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + no_create=no_create, + **kwargs) + + +class SuperResK1K5K1(SuperResK1KXK1): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK1K5K1, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + no_create=no_create, + **kwargs) + + +class SuperResK1K7K1(SuperResK1KXK1): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK1K7K1, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + no_create=no_create, + **kwargs) + + +def register_netblocks_dict(netblocks_dict: dict): + this_py_file_netblocks_dict = { + 'SuperResK1K3K1': SuperResK1K3K1, + 'SuperResK1K5K1': SuperResK1K5K1, + 'SuperResK1K7K1': SuperResK1K7K1, + } + netblocks_dict.update(this_py_file_netblocks_dict) + return netblocks_dict diff --git a/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_kxkx.py b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_kxkx.py new file mode 100644 index 0000000000000000000000000000000000000000..a694fdbe6b26217fcfeef410268b2b1b459e3456 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_classfication/super_res_kxkx.py @@ -0,0 +1,202 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The ZenNAS implementation is also open-sourced by the authors, and available at https://github.com/idstcv/ZenNAS. + +import uuid + +from torch import nn + +from . import basic_blocks, global_utils +from .global_utils import get_right_parentheses_index +from .super_blocks import PlainNetSuperBlockClass + + +class SuperResKXKX(PlainNetSuperBlockClass): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + kernel_size=None, + no_create=False, + no_reslink=False, + no_BN=False, + use_se=False, + **kwargs): + super(SuperResKXKX, self).__init__(**kwargs) + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + self.bottleneck_channels = bottleneck_channels + self.sub_layers = sub_layers + self.kernel_size = kernel_size + self.no_create = no_create + self.no_reslink = no_reslink + self.no_BN = no_BN + self.use_se = use_se + + full_str = '' + last_channels = in_channels + current_stride = stride + for i in range(self.sub_layers): + inner_str = '' + + inner_str += 'ConvKX({},{},{},{})'.format(last_channels, + self.bottleneck_channels, + self.kernel_size, + current_stride) + if not self.no_BN: + inner_str += 'BN({})'.format(self.bottleneck_channels) + inner_str += 'RELU({})'.format(self.bottleneck_channels) + if self.use_se: + inner_str += 'SE({})'.format(bottleneck_channels) + + inner_str += 'ConvKX({},{},{},{})'.format(self.bottleneck_channels, + self.out_channels, + self.kernel_size, 1) + if not self.no_BN: + inner_str += 'BN({})'.format(self.out_channels) + + if not self.no_reslink: + if i == 0: + res_str = 'ResBlockProj({})RELU({})'.format( + inner_str, out_channels) + else: + res_str = 'ResBlock({})RELU({})'.format( + inner_str, out_channels) + else: + res_str = '{}RELU({})'.format(inner_str, out_channels) + + full_str += res_str + + last_channels = out_channels + current_stride = 1 + pass + + netblocks_dict = basic_blocks.register_netblocks_dict({}) + self.block_list = global_utils.create_netblock_list_from_str( + full_str, + netblocks_dict=netblocks_dict, + no_create=no_create, + no_reslink=no_reslink, + no_BN=no_BN, + **kwargs) + if not no_create: + self.module_list = nn.ModuleList(self.block_list) + else: + self.module_list = None + + def __str__(self): + return type(self).__name__ + '({},{},{},{},{})'.format( + self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers) + + def __repr__(self): + return type( + self + ).__name__ + '({}|in={},out={},stride={},btl_channels={},sub_layers={},kernel_size={})'.format( + self.block_name, self.in_channels, self.out_channels, self.stride, + self.bottleneck_channels, self.sub_layers, self.kernel_size) + + @classmethod + def create_from_str(cls, s, **kwargs): + assert cls.is_instance_from_str(s) + idx = get_right_parentheses_index(s) + assert idx is not None + param_str = s[len(cls.__name__ + '('):idx] + + tmp_idx = param_str.find('|') + if tmp_idx < 0: + tmp_block_name = 'uuid{}'.format(uuid.uuid4().hex) + else: + tmp_block_name = param_str[0:tmp_idx] + param_str = param_str[tmp_idx + 1:] + + param_str_split = param_str.split(',') + in_channels = int(param_str_split[0]) + out_channels = int(param_str_split[1]) + stride = int(param_str_split[2]) + bottleneck_channels = int(param_str_split[3]) + sub_layers = int(param_str_split[4]) + return cls( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + block_name=tmp_block_name, + **kwargs), s[idx + 1:] + + +class SuperResK3K3(SuperResKXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK3K3, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=3, + no_create=no_create, + **kwargs) + + +class SuperResK5K5(SuperResKXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK5K5, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=5, + no_create=no_create, + **kwargs) + + +class SuperResK7K7(SuperResKXKX): + + def __init__(self, + in_channels=None, + out_channels=None, + stride=None, + bottleneck_channels=None, + sub_layers=None, + no_create=False, + **kwargs): + super(SuperResK7K7, self).__init__( + in_channels=in_channels, + out_channels=out_channels, + stride=stride, + bottleneck_channels=bottleneck_channels, + sub_layers=sub_layers, + kernel_size=7, + no_create=no_create, + **kwargs) + + +def register_netblocks_dict(netblocks_dict: dict): + this_py_file_netblocks_dict = { + 'SuperResK3K3': SuperResK3K3, + 'SuperResK5K5': SuperResK5K5, + 'SuperResK7K7': SuperResK7K7, + } + netblocks_dict.update(this_py_file_netblocks_dict) + return netblocks_dict diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..01c50b4b3108a97bd52ca0f63a0279476a0bccc2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .tinynas_detector import Tinynas_detector + from .tinynas_damoyolo import DamoYolo + +else: + _import_structure = { + 'tinynas_detector': ['TinynasDetector'], + 'tinynas_damoyolo': ['DamoYolo'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_evaluater.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_evaluater.py new file mode 100644 index 0000000000000000000000000000000000000000..6ff194f6c077e9a6b048db456d2392f151243c11 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_evaluater.py @@ -0,0 +1,68 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import torch + +from modelscope.models.cv.tinynas_detection.damo.apis.detector_inference import \ + inference +from modelscope.models.cv.tinynas_detection.damo.detectors.detector import \ + build_local_model +from modelscope.msdatasets.dataset_cls.custom_datasets.damoyolo import ( + build_dataloader, build_dataset) + + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + + +class Evaluater: + + def __init__(self, cfg): + self.cfg = cfg + self.output_dir = cfg.miscs.output_dir + self.exp_name = cfg.miscs.exp_name + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + + self.ckpt = torch.load( + self.cfg.test.checkpoint_path, map_location=self.device) + self.model = build_local_model(self.cfg, self.device) + if 'state_dict' in self.ckpt: + state_dict = self.ckpt['state_dict'] + elif 'model' in self.ckpt: + state_dict = self.ckpt['model'] + self.model.load_state_dict(state_dict) + self.val_loader = self.get_data_loader(self.cfg, False) + + def get_data_loader(self, cfg, distributed=False): + + val_dataset = build_dataset( + cfg, + cfg.dataset.val_image_dir, + cfg.dataset.val_ann, + is_train=False) + + val_loader = build_dataloader( + val_dataset, + cfg.test.augment, + batch_size=cfg.test.batch_size, + num_workers=cfg.miscs.num_workers, + is_train=False, + size_div=32, + distributed=distributed) + + return val_loader + + def evaluate(self): + + output_folder = os.path.join(self.output_dir, self.exp_name, + 'inference') + os.makedirs(output_folder, exist_ok=True) + for data_loader_val in self.val_loader: + inference( + self.model, + data_loader_val, + device=self.device, + output_folder=output_folder, + ) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_inference.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..dcd33834ed27b106517bb6614aae3e27a1301178 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/apis/detector_inference.py @@ -0,0 +1,120 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Copyright (c) Alibaba, Inc. and its affiliates. +import os + +import torch +from tqdm import tqdm + +from modelscope.msdatasets.dataset_cls.custom_datasets.damoyolo import evaluate +from modelscope.utils.logger import get_logger +from modelscope.utils.timer import Timer, get_time_str +from modelscope.utils.torch_utils import (all_gather, get_world_size, + is_master, synchronize) + +logger = get_logger() + + +def compute_on_dataset(model, data_loader, device, timer=None, tta=False): + model.eval() + results_dict = {} + cpu_device = torch.device('cpu') + for _, batch in enumerate(tqdm(data_loader)): + images, targets, image_ids = batch + with torch.no_grad(): + if timer: + timer.tic() + output = model(images.to(device)) + if timer: + # torch.cuda.synchronize() consume much time + timer.toc() + output = [o.to(cpu_device) if o is not None else o for o in output] + results_dict.update( + {img_id: result + for img_id, result in zip(image_ids, output)}) + return results_dict + + +def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu, + multi_gpu_infer): + if multi_gpu_infer: + all_predictions = all_gather(predictions_per_gpu) + else: + all_predictions = [predictions_per_gpu] + if not is_master(): + return + # merge the list of dicts + predictions = {} + for p in all_predictions: + predictions.update(p) + # convert a dict where the key is the index in a list + image_ids = list(sorted(predictions.keys())) + if len(image_ids) != image_ids[-1] + 1: + logger.warning( + 'Number of images that were gathered from multiple processes is' + 'not a contiguous set. Some images might be missing from the' + 'evaluation') + + # convert to a list + predictions = [predictions[i] for i in image_ids] + return predictions + + +def inference( + model, + data_loader, + iou_types=('bbox', ), + box_only=False, + device='cuda', + expected_results=(), + expected_results_sigma_tol=4, + output_folder=None, + multi_gpu_infer=True, +): + # convert to a torch.device for efficiency + device = torch.device(device) + num_devices = get_world_size() + dataset = data_loader.dataset + logger.info('Start evaluation ({} images).'.format(len(dataset))) + total_timer = Timer() + inference_timer = Timer() + total_timer.tic() + predictions = compute_on_dataset(model, data_loader, device, + inference_timer) + # wait for all processes to complete before measuring the time + if multi_gpu_infer: + synchronize() + total_time = total_timer.toc() + total_time_str = get_time_str(total_time) + logger.info( + 'Total run time: {} ({} s / img per device, on {} devices)'.format( + total_time_str, total_time * num_devices / len(dataset), + num_devices)) + total_infer_time = get_time_str(inference_timer.total_time) + logger.info( + 'Model inference time: {} ({} s / img per device, on {} devices)'. + format( + total_infer_time, + inference_timer.total_time * num_devices / len(dataset), + num_devices, + )) + + predictions = _accumulate_predictions_from_multiple_gpus( + predictions, multi_gpu_infer) + if not is_master(): + return + + if output_folder: + torch.save(predictions, os.path.join(output_folder, 'predictions.pth')) + + extra_args = dict( + box_only=box_only, + iou_types=iou_types, + expected_results=expected_results, + expected_results_sigma_tol=expected_results_sigma_tol, + ) + + return evaluate( + dataset=dataset, + predictions=predictions, + output_folder=output_folder, + **extra_args) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..14e2370d51228c3d7f521bc2038d3e9738c004d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/__init__.py @@ -0,0 +1 @@ +# Copyright © Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..14e2370d51228c3d7f521bc2038d3e9738c004d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/__init__.py @@ -0,0 +1 @@ +# Copyright © Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/box_level_augs.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/box_level_augs.py new file mode 100644 index 0000000000000000000000000000000000000000..f7eb2582aadf6ade14310b7c8b94599b836a1afc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/box_level_augs.py @@ -0,0 +1,104 @@ +# This file mainly comes from +# https://github.com/dvlab-research/SA-AutoAug/blob/master/FCOS/fcos_core/augmentations/box_level_augs/box_level_augs.py +# Copyright © Alibaba, Inc. and its affiliates. + +import random + +import numpy as np + +from .color_augs import color_aug_func +from .geometric_augs import geometric_aug_func + + +def _box_sample_prob(bbox, scale_ratios_splits, box_prob=0.3): + scale_ratios, scale_splits = scale_ratios_splits + + ratios = np.array(scale_ratios) + ratios = ratios / ratios.sum() + area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + if area == 0: + return 0 + if area < scale_splits[0]: + scale_ratio = ratios[0] + elif area < scale_splits[1]: + scale_ratio = ratios[1] + else: + scale_ratio = ratios[2] + return box_prob * scale_ratio + + +def _box_aug_per_img(img, + target, + aug_type=None, + scale_ratios=None, + scale_splits=None, + img_prob=0.1, + box_prob=0.3, + level=1): + if random.random() > img_prob: + return img, target + img /= 255.0 + + tag = 'prob' if aug_type in geometric_aug_func else 'area' + scale_ratios_splits = [scale_ratios[tag], scale_splits] + if scale_ratios is None: + box_sample_prob = [box_prob] * len(target.bbox) + else: + box_sample_prob = [ + _box_sample_prob(bbox, scale_ratios_splits, box_prob=box_prob) + for bbox in target.bbox + ] + + if aug_type in color_aug_func: + img_aug = color_aug_func[aug_type]( + img, level, target, [scale_ratios['area'], scale_splits], + box_sample_prob) + elif aug_type in geometric_aug_func: + img_aug, target = geometric_aug_func[aug_type](img, level, target, + box_sample_prob) + else: + raise ValueError('Unknown box-level augmentation function %s.' % + (aug_type)) + out = img_aug * 255.0 + + return out, target + + +class Box_augs(object): + + def __init__(self, box_augs_dict, max_iters, scale_splits, box_prob=0.3): + self.max_iters = max_iters + self.box_prob = box_prob + self.scale_splits = scale_splits + self.policies = box_augs_dict['policies'] + self.scale_ratios = box_augs_dict['scale_ratios'] + + def __call__(self, tensor, target, iteration): + iter_ratio = float(iteration) / self.max_iters + sub_policy = random.choice(self.policies) + + h, w = tensor.shape[-2:] + ratio = min(h, w) / 800 + + scale_splits = [area * ratio for area in self.scale_splits] + if iter_ratio <= 1: + tensor, _ = _box_aug_per_img( + tensor, + target, + aug_type=sub_policy[0][0], + scale_ratios=self.scale_ratios, + scale_splits=scale_splits, + img_prob=sub_policy[0][1] * iter_ratio, + box_prob=self.box_prob, + level=sub_policy[0][2]) + tensor, target = _box_aug_per_img( + tensor, + target, + aug_type=sub_policy[1][0], + scale_ratios=self.scale_ratios, + scale_splits=scale_splits, + img_prob=sub_policy[1][1] * iter_ratio, + box_prob=self.box_prob, + level=sub_policy[1][2]) + + return tensor, target diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/color_augs.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/color_augs.py new file mode 100644 index 0000000000000000000000000000000000000000..41c46e86b7f3178feca7d03f7b628e1d512ec817 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/color_augs.py @@ -0,0 +1,234 @@ +# This file mainly comes from +# https://github.com/dvlab-research/SA-AutoAug/blob/master/FCOS/fcos_core/augmentations/box_level_augs/color_augs.py +# Copyright © Alibaba, Inc. and its affiliates. + +import random + +import torch +import torch.nn.functional as F + +from .gaussian_maps import _merge_gaussian + +_MAX_LEVEL = 10.0 + + +def blend(image1, image2, factor): + """Blend image1 and image2 using 'factor'. + Factor can be above 0.0. A value of 0.0 means only image1 is used. + A value of 1.0 means only image2 is used. A value between 0.0 and + 1.0 means we linearly interpolate the pixel values between the two + images. A value greater than 1.0 "extrapolates" the difference + between the two pixel values, and we clip the results to values + between 0 and 1.0. + """ + + if factor == 0.0: + return image1 + if factor == 1.0: + return image2 + + difference = image2 - image1 + scaled = factor * difference + + # Do addition in float. + temp = image1 + scaled + + # Interpolate + if factor > 0.0 and factor < 1.0: + # Interpolation means we always stay within 0 and 255. + return temp + + # Extrapolate: + # + # We need to clip and then cast. + return torch.clamp(temp, 0.0, 1.0) + + +def solarize(image, threshold=0.5): + # For each pixel in the image, select the pixel + # if the value is less than the threshold. + # Otherwise, subtract 255 from the pixel. + return torch.where(image <= threshold, image, 1.0 - image) + + +def solarize_add(image, addition=0, threshold=0.5): + # For each pixel in the image less than threshold + # we add 'addition' amount to it and then clip the + # pixel value to be between 0 and 255. The value + # of 'addition' is between -128 and 128. + added_image = image + addition + added_image = torch.clamp(added_image, 0.0, 1.0) + return torch.where(image <= threshold, added_image, image) + + +def rgb2gray(rgb): + gray = rgb[0] * 0.2989 + rgb[1] * 0.5870 + rgb[2] * 0.1140 + gray = gray.unsqueeze(0).repeat((3, 1, 1)) + return gray + + +def color(img, factor): + """Equivalent of PIL Color.""" + if img.shape[0] == 0 or img.shape[1] == 0: + return img + + degenerate = rgb2gray(img) + return blend(degenerate, img, factor) + + +def contrast(img, factor): + dtype = img.dtype if torch.is_floating_point(img) else torch.float32 + mean = torch.mean(rgb2gray(img).to(dtype), dim=(-3, -2, -1), keepdim=True) + return blend(mean, img, max(factor, 1e-6)) + + +def brightness(image, factor): + """Equivalent of PIL Brightness.""" + degenerate = torch.zeros(image.shape) + return blend(degenerate, image, factor) + + +def sharpness(image, factor): + """Implements Sharpness function from PIL using TF ops.""" + if image.shape[0] == 0 or image.shape[1] == 0: + return image + channels = image.shape[0] + kernel = torch.Tensor([[1, 1, 1], [1, 5, 1], [1, 1, 1]]).reshape( + 1, 1, 3, 3) / 13.0 + kernel = kernel.repeat((3, 1, 1, 1)) + image_newaxis = image.unsqueeze(0) + image_pad = F.pad(image_newaxis, (1, 1, 1, 1), mode='reflect') + degenerate = F.conv2d(image_pad, weight=kernel, groups=channels).squeeze(0) + return blend(degenerate, image, factor) + + +def equalize(image): + """Implements Equalize function from PIL using PyTorch ops based on: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/ + autoaugment.py#L352""" + image = image * 255 + + def scale_channel(im, c): + """Scale the data in the channel to implement equalize.""" + im = im[c, :, :] + # Compute the histogram of the image channel. + histo = torch.histc(im, bins=256, min=0, max=255) # .type(torch.int32) + # For the purposes of computing the step, filter out the nonzeros. + nonzero_histo = torch.reshape(histo[histo != 0], [-1]) + step = (torch.sum(nonzero_histo) - nonzero_histo[-1]) // 255 + + def build_lut(histo, step): + # Compute the cumulative sum, shifting by step // 2 + # and then normalization by step. + lut = (torch.cumsum(histo, 0) + (step // 2)) // step + # Shift lut, prepending with 0. + lut = torch.cat([torch.zeros(1), lut[:-1]]) + # Clip the counts to be in range. This is done + # in the C code for image.point. + return torch.clamp(lut, 0, 255) + + # If step is zero, return the original image. Otherwise, build + # lut from the full histogram and step and then index from it. + if step == 0: + result = im + else: + # can't index using 2d index. Have to flatten and then reshape + result = torch.gather( + build_lut(histo, step), 0, + im.flatten().long()) + result = result.reshape_as(im) + + return result # .type(torch.uint8) + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image, 0) + s2 = scale_channel(image, 1) + s3 = scale_channel(image, 2) + image = torch.stack([s1, s2, s3], 0) / 255.0 + return image + + +def autocontrast(image): + + def scale_channel(image): + """Scale the 2D image using the autocontrast rule.""" + lo = torch.min(image) + hi = torch.max(image) + + # Scale the image, making the lowest value 0 and the highest value 1. + def scale_values(im): + scale = 1.0 / (hi - lo) + offset = -lo * scale + im = im * scale + offset + im = torch.clamp(im, 0.0, 1.0) + return im + + if hi > lo: + result = scale_values(image) + else: + result = image + + return result + + # Assumes RGB for now. Scales each channel independently + # and then stacks the result. + s1 = scale_channel(image[0, :, :]) + s2 = scale_channel(image[1, :, :]) + s3 = scale_channel(image[2, :, :]) + image = torch.stack([s1, s2, s3], 0) + return image + + +def posterize(image, bits): + """Equivalent of PIL Posterize.""" + image *= 255 + image = image.long() + shift = bits # 8 - bits + image_rightshift = image >> shift + image_leftshift = image_rightshift << shift + image_leftshift = image_leftshift.float() / 255.0 + return image_leftshift + + +def _color_aug_func(img, img_aug, target, scale_ratios_splits, + box_sample_probs): + scale_ratios, scale_splits = scale_ratios_splits + boxes = [ + bbox for i, bbox in enumerate(target.bbox) + if random.random() < box_sample_probs[i] + ] + img_aug = _merge_gaussian(img, img_aug, boxes, scale_ratios, scale_splits) + return img_aug + + +color_aug_func = { + 'AutoContrast': + lambda x, level, target, + scale_ratios_splits, box_sample_probs: _color_aug_func( + x, autocontrast(x), target, scale_ratios_splits, box_sample_probs), + 'Equalize': + lambda x, leve, target, + scale_ratios_splits, box_sample_probs: _color_aug_func( + x, equalize(x), target, scale_ratios_splits, box_sample_probs), + 'SolarizeAdd': + lambda x, level, target, scale_ratios_splits, box_sample_probs: + _color_aug_func(x, solarize_add(x, level / _MAX_LEVEL * 0.4296875), target, + scale_ratios_splits, box_sample_probs), + 'Color': + lambda x, level, target, scale_ratios_splits, box_sample_probs: + _color_aug_func(x, color(x, level / _MAX_LEVEL * 1.8 + 0.1), target, + scale_ratios_splits, box_sample_probs), + 'Contrast': + lambda x, level, target, scale_ratios_splits, box_sample_probs: + _color_aug_func(x, contrast(x, level / _MAX_LEVEL * 1.8 + 0.1), target, + scale_ratios_splits, box_sample_probs), + 'Brightness': + lambda x, level, target, scale_ratios_splits, box_sample_probs: + _color_aug_func(x, brightness(x, level / _MAX_LEVEL * 1.8 + 0.1), target, + scale_ratios_splits, box_sample_probs), + 'Sharpness': + lambda x, level, target, scale_ratios_splits, box_sample_probs: + _color_aug_func(x, sharpness(x, level / _MAX_LEVEL * 1.8 + 0.1), target, + scale_ratios_splits, box_sample_probs), +} diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/gaussian_maps.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/gaussian_maps.py new file mode 100644 index 0000000000000000000000000000000000000000..08acaa211b0bb101a604e975a736307a427b6377 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/gaussian_maps.py @@ -0,0 +1,59 @@ +# This file mainly comes from +# https://github.com/dvlab-research/SA-AutoAug/blob/master/FCOS/fcos_core/augmentations/box_level_augs/gaussian_maps.py +# Copyright © Alibaba, Inc. and its affiliates. + +import math + +import torch + + +def _gaussian_map(img, boxes, scale_splits=None, scale_ratios=None): + g_maps = torch.zeros(*img.shape[1:]).to(img.device) + height, width = img.shape[1], img.shape[2] + + x_range = torch.arange(0, height, 1).to(img.device) + y_range = torch.arange(0, width, 1).to(img.device) + xx, yy = torch.meshgrid(x_range, y_range) + pos = torch.empty(xx.shape + (2, )).to(img.device) + pos[:, :, 0] = xx + pos[:, :, 1] = yy + + for j, box in enumerate(boxes): + y1, x1, y2, x2 = box + x, y, h, w = x1, y1, x2 - x1, y2 - y1 + mean_torch = torch.tensor([x + h // 2, y + w // 2]).to(img.device) + if scale_ratios is None: + scale_ratio = 1.0 + else: + ratio_list = [0.2, 0.4, 0.6, 0.8, 1.0, 2, 4, 6, 8, 10] + if h * w < scale_splits[0]: + scale_ratio = ratio_list[scale_ratios[0]] * scale_splits[0] / ( + h * w) + elif h * w < scale_splits[1]: + scale_ratio = ratio_list[scale_ratios[1]] * ( + scale_splits[0] + scale_splits[1]) / 2.0 / ( + h * w) + elif h * w < scale_splits[2]: + scale_ratio = ratio_list[scale_ratios[2]] * scale_splits[2] / ( + h * w) + else: + scale_ratio = ratio_list[scale_ratios[2]] + + r_var = (scale_ratio * height * width / (2 * math.pi))**0.5 + var_x = torch.tensor([(h / height) * r_var], + dtype=torch.float32).to(img.device) + var_y = torch.tensor([(w / width) * r_var], + dtype=torch.float32).to(img.device) + g_map = torch.exp(-( + ((xx.float() - mean_torch[0])**2 / (2.0 * var_x**2) + + (yy.float() - mean_torch[1])**2 / (2.0 * var_y**2)))).to( + img.device) + g_maps += g_map + return g_maps + + +def _merge_gaussian(img, img_aug, boxes, scale_ratios, scale_splits): + g_maps = _gaussian_map(img, boxes, scale_splits, scale_ratios) + g_maps = g_maps.clamp(min=0, max=1.0) + out = img * (1 - g_maps) + img_aug * g_maps + return out diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/geometric_augs.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/geometric_augs.py new file mode 100644 index 0000000000000000000000000000000000000000..ee61adefb05fb29d2186be0d5a05725bc590cf7b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/box_level_augs/geometric_augs.py @@ -0,0 +1,168 @@ +# This file mainly comes from +# https://github.com/dvlab-research/SA-AutoAug/blob/master/FCOS/fcos_core/augmentations/box_level_augs/geometric_augs.py +# Copyright © Alibaba, Inc. and its affiliates. + +import copy +import random + +import torch +import torchvision.transforms as transforms + +from .gaussian_maps import _gaussian_map + +_MAX_LEVEL = 10.0 +pixel_mean = [102.9801, 115.9465, 122.7717] + + +def scale_area(box, height, width, scale_ratio=1.0): + y1, x1, y2, x2 = box + h, w = x2 - x1, y2 - y1 + h_new, w_new = h * scale_ratio, w * scale_ratio + x1, y1 = max(x1 + h / 2 - h_new / 2, 0), max(y1 + w / 2 - w_new / 2, 0) + x2, y2 = min(x1 + h_new, height), min(y1 + w_new, width) + box_new = torch.Tensor([y1, x1, y2, x2]) + return box_new + + +def _geometric_aug_func(x, + target, + angle=0, + translate=(0, 0), + scale=1, + shear=(0, 0), + hflip=False, + boxes_sample_prob=[], + scale_ratio=1.0): + boxes_and_labels = [(target.bbox[i], target.extra_fields['labels'][i]) + for i in range(len(target.bbox)) + if random.random() < boxes_sample_prob[i]] + boxes = [b_and_l[0] for b_and_l in boxes_and_labels] + labels = [b_and_l[1] for b_and_l in boxes_and_labels] + + if random.random() < 0.5: + angle *= -1 + translate = (-translate[0], -translate[1]) + shear = (-shear[0], -shear[1]) + + height, width = x.shape[1], x.shape[2] + + x_crops = [] + boxes_crops = [] + boxes_new = [] + labels_new = [] + for i, box in enumerate(boxes): + box_crop = scale_area(box, height, width, scale_ratio) + y1, x1, y2, x2 = box_crop.long() + + x_crop = x[:, x1:x2, y1:y2] + boxes_crops.append(box_crop) + + if x1 >= x2 or y1 >= y2: + x_crops.append(x_crop) + continue + + if hflip: + x_crop = x_crop.flip(-1) + elif translate[0] + translate[1] != 0: + offset_y = (y2 + translate[0]).clamp(0, width).long().tolist() - y2 + offset_x = (x2 + translate[1]).clamp(0, + height).long().tolist() - x2 + if offset_x != 0 or offset_y != 0: + offset = [offset_y, offset_x] + boxes_new.append(box + torch.Tensor(offset * 2)) + labels_new.append(labels[i]) + else: + x_crop = transforms.functional.to_pil_image(x_crop.cpu()) + try: + x_crop = transforms.functional.affine( + x_crop, + angle, + translate, + scale, + shear, + resample=2, + fillcolor=tuple([int(i) for i in pixel_mean])) + except Exception: + x_crop = transforms.functional.affine( + x_crop, + angle, + translate, + scale, + shear, + interpolation=2, + fill=tuple([int(i) for i in pixel_mean])) + + x_crop = transforms.functional.to_tensor(x_crop).to(x.device) + x_crops.append(x_crop) + y = _transform(x, x_crops, boxes_crops, translate) + + if translate[0] + translate[1] != 0 and len(boxes_new) > 0: + target.bbox = torch.cat((target.bbox, torch.stack(boxes_new))) + target.extra_fields['labels'] = torch.cat( + (target.extra_fields['labels'], torch.Tensor(labels_new).long())) + + return y, target + + +def _transform(x, x_crops, boxes_crops, translate=(0, 0)): + y = copy.deepcopy(x) + height, width = x.shape[1], x.shape[2] + + for i, box in enumerate(boxes_crops): + y1_c, x1_c, y2_c, x2_c = boxes_crops[i].long() + + y1_c = (y1_c + translate[0]).clamp(0, width).long().tolist() + x1_c = (x1_c + translate[1]).clamp(0, height).long().tolist() + y2_c = (y2_c + translate[0]).clamp(0, width).long().tolist() + x2_c = (x2_c + translate[1]).clamp(0, height).long().tolist() + + y_crop = copy.deepcopy(y[:, x1_c:x2_c, y1_c:y2_c]) + x_crop = x_crops[i][:, :y_crop.shape[1], :y_crop.shape[2]] + + if y_crop.shape[1] * y_crop.shape[2] == 0: + continue + + g_maps = _gaussian_map(x_crop, + [[0, 0, y_crop.shape[2], y_crop.shape[1]]]) + _, _h, _w = y[:, x1_c:x2_c, y1_c:y2_c].shape + y[:, x1_c:x1_c + x_crop.shape[1], + y1_c:y1_c + x_crop.shape[2]] = g_maps * x_crop + ( + 1 - g_maps) * y_crop[:, :x_crop.shape[1], :x_crop.shape[2]] + return y + + +geometric_aug_func = { + 'hflip': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, target, hflip=True, boxes_sample_prob=boxes_sample_probs), + 'rotate': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, + target, + level / _MAX_LEVEL * 30, + boxes_sample_prob=boxes_sample_probs), + 'shearX': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, + target, + shear=(level / _MAX_LEVEL * 15, 0), + boxes_sample_prob=boxes_sample_probs), + 'shearY': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, + target, + shear=(0, level / _MAX_LEVEL * 15), + boxes_sample_prob=boxes_sample_probs), + 'translateX': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, + target, + translate=(level / _MAX_LEVEL * 120.0, 0), + boxes_sample_prob=boxes_sample_probs), + 'translateY': + lambda x, level, target, boxes_sample_probs: _geometric_aug_func( + x, + target, + translate=(0, level / _MAX_LEVEL * 120.0), + boxes_sample_prob=boxes_sample_probs) +} diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/scale_aware_aug.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/scale_aware_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..ec105526df082603e6a83185afb9b8f2fe38bd39 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/augmentations/scale_aware_aug.py @@ -0,0 +1,77 @@ +# This file mainly comes from +# https://github.com/dvlab-research/SA-AutoAug/blob/master/FCOS/fcos_core/augmentations/scale_aware_aug.py +# Copyright © Alibaba, Inc. and its affiliates. + +import copy + +from .box_level_augs.box_level_augs import Box_augs +from .box_level_augs.color_augs import color_aug_func +from .box_level_augs.geometric_augs import geometric_aug_func + + +class SA_Aug(object): + + def __init__(self, iters_per_epoch, start_epoch, total_epochs, + no_aug_epochs, batch_size, num_gpus, num_workers, sada_cfg): + + autoaug_list = sada_cfg.autoaug_params + num_policies = sada_cfg.num_subpolicies + scale_splits = sada_cfg.scale_splits + box_prob = sada_cfg.box_prob + + self.batch_size = batch_size / num_gpus + self.num_workers = num_workers + self.max_iters = (total_epochs - no_aug_epochs) * iters_per_epoch + self.count = start_epoch * iters_per_epoch + if self.num_workers == 0: + self.num_workers += 1 + + box_aug_list = autoaug_list[4:] + color_aug_types = list(color_aug_func.keys()) + geometric_aug_types = list(geometric_aug_func.keys()) + policies = [] + for i in range(num_policies): + _start_pos = i * 6 + sub_policy = [ + ( + color_aug_types[box_aug_list[_start_pos + 0] + % len(color_aug_types)], + box_aug_list[_start_pos + 1] * 0.1, + box_aug_list[_start_pos + 2], + ), # box_color policy + (geometric_aug_types[box_aug_list[_start_pos + 3] + % len(geometric_aug_types)], + box_aug_list[_start_pos + 4] * 0.1, + box_aug_list[_start_pos + 5]) + ] # box_geometric policy + policies.append(sub_policy) + + _start_pos = num_policies * 6 + scale_ratios = { + 'area': [ + box_aug_list[_start_pos + 0], box_aug_list[_start_pos + 1], + box_aug_list[_start_pos + 2] + ], + 'prob': [ + box_aug_list[_start_pos + 3], box_aug_list[_start_pos + 4], + box_aug_list[_start_pos + 5] + ] + } + + box_augs_dict = {'policies': policies, 'scale_ratios': scale_ratios} + + self.box_augs = Box_augs( + box_augs_dict=box_augs_dict, + max_iters=self.max_iters, + scale_splits=scale_splits, + box_prob=box_prob) + + def __call__(self, tensor, target): + iteration = self.count // self.batch_size * self.num_workers + tensor = copy.deepcopy(tensor) + target = copy.deepcopy(target) + tensor, target = self.box_augs(tensor, target, iteration=iteration) + + self.count += 1 + + return tensor, target diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..04654d8a01470f7cd7e7cf87ce571df3e6f16005 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .backbones import build_backbone +from .heads import build_head +from .necks import build_neck diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cf6d07051e90f12502f8987db32e9c237f9ba113 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy + +from .darknet import CSPDarknet +from .tinynas_csp import load_tinynas_net as load_tinynas_net_csp +from .tinynas_res import load_tinynas_net as load_tinynas_net_res + + +def build_backbone(cfg): + backbone_cfg = copy.deepcopy(cfg) + name = backbone_cfg.pop('name') + if name == 'TinyNAS_res': + return load_tinynas_net_res(backbone_cfg) + elif name == 'TinyNAS_csp': + return load_tinynas_net_csp(backbone_cfg) + elif name == 'CSPDarknet': + return CSPDarknet(**backbone_cfg) + else: + print(f'{name} is not supported yet!') diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/darknet.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/darknet.py new file mode 100644 index 0000000000000000000000000000000000000000..27936ff83a2ad1dfe7fd9e79deb9928992b5d851 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/darknet.py @@ -0,0 +1,125 @@ +# Copyright (c) Megvii Inc. All rights reserved. +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +from torch import nn + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.base_ops import ( + BaseConv, CSPLayer, DWConv, Focus, ResLayer, SPPBottleneck) + + +class CSPDarknet(nn.Module): + + def __init__( + self, + dep_mul, + wid_mul, + out_features=('dark3', 'dark4', 'dark5'), + depthwise=False, + act='silu', + reparam=False, + ): + super(CSPDarknet, self).__init__() + assert out_features, 'please provide output features of Darknet' + self.out_features = out_features + Conv = DWConv if depthwise else BaseConv + + base_channels = int(wid_mul * 64) # 64 + base_depth = max(round(dep_mul * 3), 1) # 3 + + # stem + # self.stem = Focus(3, base_channels, ksize=3, act=act) + self.stem = Focus(3, base_channels, 3, act=act) + + # dark2 + self.dark2 = nn.Sequential( + Conv(base_channels, base_channels * 2, 3, 2, act=act), + CSPLayer( + base_channels * 2, + base_channels * 2, + n=base_depth, + depthwise=depthwise, + act=act, + reparam=reparam, + ), + ) + + # dark3 + self.dark3 = nn.Sequential( + Conv(base_channels * 2, base_channels * 4, 3, 2, act=act), + CSPLayer( + base_channels * 4, + base_channels * 4, + n=base_depth * 3, + depthwise=depthwise, + act=act, + reparam=reparam, + ), + ) + + # dark4 + self.dark4 = nn.Sequential( + Conv(base_channels * 4, base_channels * 8, 3, 2, act=act), + CSPLayer( + base_channels * 8, + base_channels * 8, + n=base_depth * 3, + depthwise=depthwise, + act=act, + reparam=reparam, + ), + ) + + # dark5 + self.dark5 = nn.Sequential( + Conv(base_channels * 8, base_channels * 16, 3, 2, act=act), + SPPBottleneck( + base_channels * 16, base_channels * 16, activation=act), + CSPLayer( + base_channels * 16, + base_channels * 16, + n=base_depth, + shortcut=False, + depthwise=depthwise, + act=act, + reparam=reparam, + ), + ) + + def init_weights(self, pretrain=None): + + if pretrain is None: + return + else: + pretrained_dict = torch.load( + pretrain, map_location='cpu')['state_dict'] + new_params = self.state_dict().copy() + for k, v in pretrained_dict.items(): + ks = k.split('.') + if ks[0] == 'fc' or ks[-1] == 'total_ops' or ks[ + -1] == 'total_params': + continue + else: + new_params[k] = v + + self.load_state_dict(new_params) + print(f' load pretrain backbone from {pretrain}') + + def forward(self, x): + outputs = {} + x = self.stem(x) + outputs['stem'] = x + x = self.dark2(x) + outputs['dark2'] = x + x = self.dark3(x) + outputs['dark3'] = x + x = self.dark4(x) + outputs['dark4'] = x + x = self.dark5(x) + outputs['dark5'] = x + features_out = [ + outputs['stem'], outputs['dark2'], outputs['dark3'], + outputs['dark4'], outputs['dark5'] + ] + + return features_out diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_csp.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_csp.py new file mode 100644 index 0000000000000000000000000000000000000000..4b1bb510f9c7c66dc2c7d5174cd40dcc723d3c33 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_csp.py @@ -0,0 +1,295 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.ops import ( + Focus, RepConv, SPPBottleneck, get_activation) +from modelscope.utils.file_utils import read_file + + +class ConvKXBN(nn.Module): + + def __init__(self, in_c, out_c, kernel_size, stride): + super(ConvKXBN, self).__init__() + self.conv1 = nn.Conv2d( + in_c, + out_c, + kernel_size, + stride, (kernel_size - 1) // 2, + groups=1, + bias=False) + self.bn1 = nn.BatchNorm2d(out_c) + + def forward(self, x): + return self.bn1(self.conv1(x)) + + def fuseforward(self, x): + return self.conv1(x) + + +class ConvKXBNRELU(nn.Module): + + def __init__(self, in_c, out_c, kernel_size, stride, act='silu'): + super(ConvKXBNRELU, self).__init__() + self.conv = ConvKXBN(in_c, out_c, kernel_size, stride) + if act is None: + self.activation_function = torch.relu + else: + self.activation_function = get_activation(act) + + def forward(self, x): + output = self.conv(x) + return self.activation_function(output) + + +class ResConvBlock(nn.Module): + + def __init__(self, + in_c, + out_c, + btn_c, + kernel_size, + stride, + act='silu', + reparam=False, + block_type='k1kx'): + super(ResConvBlock, self).__init__() + self.stride = stride + if block_type == 'k1kx': + self.conv1 = ConvKXBN(in_c, btn_c, kernel_size=1, stride=1) + else: + self.conv1 = ConvKXBN( + in_c, btn_c, kernel_size=kernel_size, stride=1) + if not reparam: + self.conv2 = ConvKXBN(btn_c, out_c, kernel_size, stride) + else: + self.conv2 = RepConv( + btn_c, out_c, kernel_size, stride, act='identity') + + self.activation_function = get_activation(act) + + if in_c != out_c and stride != 2: + self.residual_proj = ConvKXBN(in_c, out_c, kernel_size=1, stride=1) + else: + self.residual_proj = None + + def forward(self, x): + if self.residual_proj is not None: + reslink = self.residual_proj(x) + else: + reslink = x + x = self.conv1(x) + x = self.activation_function(x) + x = self.conv2(x) + if self.stride != 2: + x = x + reslink + x = self.activation_function(x) + return x + + +class CSPStem(nn.Module): + + def __init__(self, + in_c, + out_c, + btn_c, + stride, + kernel_size, + num_blocks, + act='silu', + reparam=False, + block_type='k1kx'): + super(CSPStem, self).__init__() + self.in_channels = in_c + self.out_channels = out_c + self.stride = stride + if self.stride == 2: + self.num_blocks = num_blocks - 1 + else: + self.num_blocks = num_blocks + self.kernel_size = kernel_size + self.act = act + self.block_type = block_type + out_c = out_c // 2 + + if act is None: + self.act = torch.relu + else: + self.act = get_activation(act) + self.block_list = nn.ModuleList() + for block_id in range(self.num_blocks): + if self.stride == 1 and block_id == 0: + in_c = in_c // 2 + else: + in_c = out_c + the_block = ResConvBlock( + in_c, + out_c, + btn_c, + kernel_size, + stride=1, + act=act, + reparam=reparam, + block_type=block_type) + self.block_list.append(the_block) + + def forward(self, x): + output = x + for block in self.block_list: + output = block(output) + return output + + +class TinyNAS(nn.Module): + + def __init__(self, + structure_info=None, + out_indices=[2, 3, 4], + with_spp=False, + use_focus=False, + act='silu', + reparam=False): + super(TinyNAS, self).__init__() + self.out_indices = out_indices + self.block_list = nn.ModuleList() + self.stride_list = [] + + for idx, block_info in enumerate(structure_info): + the_block_class = block_info['class'] + if the_block_class == 'ConvKXBNRELU': + if use_focus and idx == 0: + the_block = Focus( + block_info['in'], + block_info['out'], + block_info['k'], + act=act) + else: + the_block = ConvKXBNRELU( + block_info['in'], + block_info['out'], + block_info['k'], + block_info['s'], + act=act) + elif the_block_class == 'SuperResConvK1KX': + the_block = CSPStem( + block_info['in'], + block_info['out'], + block_info['btn'], + block_info['s'], + block_info['k'], + block_info['L'], + act=act, + reparam=reparam, + block_type='k1kx') + elif the_block_class == 'SuperResConvKXKX': + the_block = CSPStem( + block_info['in'], + block_info['out'], + block_info['btn'], + block_info['s'], + block_info['k'], + block_info['L'], + act=act, + reparam=reparam, + block_type='kxkx') + else: + raise NotImplementedError + + self.block_list.append(the_block) + + self.csp_stage = nn.ModuleList() + self.csp_stage.append(self.block_list[0]) + self.csp_stage.append(CSPWrapper(self.block_list[1])) + self.csp_stage.append(CSPWrapper(self.block_list[2])) + self.csp_stage.append( + CSPWrapper((self.block_list[3], self.block_list[4]))) + self.csp_stage.append( + CSPWrapper(self.block_list[5], with_spp=with_spp)) + del self.block_list + + def init_weights(self, pretrain=None): + pass + + def forward(self, x): + output = x + stage_feature_list = [] + for idx, block in enumerate(self.csp_stage): + output = block(output) + if idx in self.out_indices: + stage_feature_list.append(output) + return stage_feature_list + + +class CSPWrapper(nn.Module): + + def __init__(self, convstem, act='relu', reparam=False, with_spp=False): + + super(CSPWrapper, self).__init__() + self.with_spp = with_spp + if isinstance(convstem, tuple): + in_c = convstem[0].in_channels + out_c = convstem[-1].out_channels + hidden_dim = convstem[0].out_channels // 2 + _convstem = nn.ModuleList() + for modulelist in convstem: + for layer in modulelist.block_list: + _convstem.append(layer) + else: + in_c = convstem.in_channels + out_c = convstem.out_channels + hidden_dim = out_c // 2 + _convstem = convstem.block_list + + self.convstem = nn.ModuleList() + for layer in _convstem: + self.convstem.append(layer) + + self.act = get_activation(act) + self.downsampler = ConvKXBNRELU( + in_c, hidden_dim * 2, 3, 2, act=self.act) + if self.with_spp: + self.spp = SPPBottleneck(hidden_dim * 2, hidden_dim * 2) + if len(self.convstem) > 0: + self.conv_start = ConvKXBNRELU( + hidden_dim * 2, hidden_dim, 1, 1, act=self.act) + self.conv_shortcut = ConvKXBNRELU( + hidden_dim * 2, out_c // 2, 1, 1, act=self.act) + self.conv_fuse = ConvKXBNRELU(out_c, out_c, 1, 1, act=self.act) + + def forward(self, x): + x = self.downsampler(x) + if self.with_spp: + x = self.spp(x) + if len(self.convstem) > 0: + shortcut = self.conv_shortcut(x) + x = self.conv_start(x) + for block in self.convstem: + x = block(x) + x = torch.cat((x, shortcut), dim=1) + x = self.conv_fuse(x) + return x + + +def load_tinynas_net(backbone_cfg): + # load masternet model to path + import ast + + net_structure_str = read_file(backbone_cfg.structure_file) + struct_str = ''.join([x.strip() for x in net_structure_str]) + struct_info = ast.literal_eval(struct_str) + for layer in struct_info: + if 'nbitsA' in layer: + del layer['nbitsA'] + if 'nbitsW' in layer: + del layer['nbitsW'] + + model = TinyNAS( + structure_info=struct_info, + out_indices=backbone_cfg.out_indices, + with_spp=backbone_cfg.with_spp, + use_focus=backbone_cfg.use_focus, + act=backbone_cfg.act, + reparam=backbone_cfg.reparam) + + return model diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_res.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_res.py new file mode 100644 index 0000000000000000000000000000000000000000..5ca844a892e75b441077074055d4bdcd209a7b0e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/backbones/tinynas_res.py @@ -0,0 +1,238 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.ops import ( + Focus, RepConv, SPPBottleneck, get_activation) +from modelscope.utils.file_utils import read_file + + +class ConvKXBN(nn.Module): + + def __init__(self, in_c, out_c, kernel_size, stride): + super(ConvKXBN, self).__init__() + self.conv1 = nn.Conv2d( + in_c, + out_c, + kernel_size, + stride, (kernel_size - 1) // 2, + groups=1, + bias=False) + self.bn1 = nn.BatchNorm2d(out_c) + + def forward(self, x): + return self.bn1(self.conv1(x)) + + def fuseforward(self, x): + return self.conv1(x) + + +class ConvKXBNRELU(nn.Module): + + def __init__(self, in_c, out_c, kernel_size, stride, act='silu'): + super(ConvKXBNRELU, self).__init__() + self.conv = ConvKXBN(in_c, out_c, kernel_size, stride) + if act is None: + self.activation_function = torch.relu + else: + self.activation_function = get_activation(act) + + def forward(self, x): + output = self.conv(x) + return self.activation_function(output) + + +class ResConvBlock(nn.Module): + + def __init__(self, + in_c, + out_c, + btn_c, + kernel_size, + stride, + act='silu', + reparam=False, + block_type='k1kx'): + super(ResConvBlock, self).__init__() + self.stride = stride + if block_type == 'k1kx': + self.conv1 = ConvKXBN(in_c, btn_c, kernel_size=1, stride=1) + else: + self.conv1 = ConvKXBN( + in_c, btn_c, kernel_size=kernel_size, stride=1) + + if not reparam: + self.conv2 = ConvKXBN(btn_c, out_c, kernel_size, stride) + else: + self.conv2 = RepConv( + btn_c, out_c, kernel_size, stride, act='identity') + + self.activation_function = get_activation(act) + + if in_c != out_c and stride != 2: + self.residual_proj = ConvKXBN(in_c, out_c, 1, 1) + else: + self.residual_proj = None + + def forward(self, x): + if self.residual_proj is not None: + reslink = self.residual_proj(x) + else: + reslink = x + x = self.conv1(x) + x = self.activation_function(x) + x = self.conv2(x) + if self.stride != 2: + x = x + reslink + x = self.activation_function(x) + return x + + +class SuperResStem(nn.Module): + + def __init__(self, + in_c, + out_c, + btn_c, + kernel_size, + stride, + num_blocks, + with_spp=False, + act='silu', + reparam=False, + block_type='k1kx'): + super(SuperResStem, self).__init__() + if act is None: + self.act = torch.relu + else: + self.act = get_activation(act) + self.block_list = nn.ModuleList() + for block_id in range(num_blocks): + if block_id == 0: + in_channels = in_c + out_channels = out_c + this_stride = stride + this_kernel_size = kernel_size + else: + in_channels = out_c + out_channels = out_c + this_stride = 1 + this_kernel_size = kernel_size + the_block = ResConvBlock( + in_channels, + out_channels, + btn_c, + this_kernel_size, + this_stride, + act=act, + reparam=reparam, + block_type=block_type) + self.block_list.append(the_block) + if block_id == 0 and with_spp: + self.block_list.append( + SPPBottleneck(out_channels, out_channels)) + + def forward(self, x): + output = x + for block in self.block_list: + output = block(output) + return output + + +class TinyNAS(nn.Module): + + def __init__(self, + structure_info=None, + out_indices=[2, 4, 5], + with_spp=False, + use_focus=False, + act='silu', + reparam=False): + super(TinyNAS, self).__init__() + self.out_indices = out_indices + self.block_list = nn.ModuleList() + + for idx, block_info in enumerate(structure_info): + the_block_class = block_info['class'] + if the_block_class == 'ConvKXBNRELU': + if use_focus: + the_block = Focus( + block_info['in'], + block_info['out'], + block_info['k'], + act=act) + else: + the_block = ConvKXBNRELU( + block_info['in'], + block_info['out'], + block_info['k'], + block_info['s'], + act=act) + self.block_list.append(the_block) + elif the_block_class == 'SuperResConvK1KX': + spp = with_spp if idx == len(structure_info) - 1 else False + the_block = SuperResStem( + block_info['in'], + block_info['out'], + block_info['btn'], + block_info['k'], + block_info['s'], + block_info['L'], + spp, + act=act, + reparam=reparam, + block_type='k1kx') + self.block_list.append(the_block) + elif the_block_class == 'SuperResConvKXKX': + spp = with_spp if idx == len(structure_info) - 1 else False + the_block = SuperResStem( + block_info['in'], + block_info['out'], + block_info['btn'], + block_info['k'], + block_info['s'], + block_info['L'], + spp, + act=act, + reparam=reparam, + block_type='kxkx') + self.block_list.append(the_block) + else: + raise NotImplementedError + + def init_weights(self, pretrain=None): + pass + + def forward(self, x): + output = x + stage_feature_list = [] + for idx, block in enumerate(self.block_list): + output = block(output) + if idx in self.out_indices: + stage_feature_list.append(output) + return stage_feature_list + + +def load_tinynas_net(backbone_cfg): + # load masternet model to path + import ast + + net_structure_str = read_file(backbone_cfg.structure_file) + struct_str = ''.join([x.strip() for x in net_structure_str]) + struct_info = ast.literal_eval(struct_str) + for layer in struct_info: + if 'nbitsA' in layer: + del layer['nbitsA'] + if 'nbitsW' in layer: + del layer['nbitsW'] + + model = TinyNAS( + structure_info=struct_info, + out_indices=backbone_cfg.out_indices, + with_spp=backbone_cfg.with_spp, + use_focus=backbone_cfg.use_focus, + act=backbone_cfg.act, + reparam=backbone_cfg.reparam) + + return model diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/base_ops.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/base_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..daf71d05ef3e981816dd6ff3fac757620a4039d3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/base_ops.py @@ -0,0 +1,474 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .repvgg_block import RepVggBlock + + +class SiLU(nn.Module): + """export-friendly version of nn.SiLU()""" + + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +def get_activation(name='silu', inplace=True): + if name == 'silu': + module = nn.SiLU(inplace=inplace) + elif name == 'relu': + module = nn.ReLU(inplace=inplace) + elif name == 'lrelu': + module = nn.LeakyReLU(0.1, inplace=inplace) + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + return module + + +def get_norm(name, out_channels, inplace=True): + if name == 'bn': + module = nn.BatchNorm2d(out_channels) + elif name == 'gn': + module = nn.GroupNorm(num_channels=out_channels, num_groups=32) + return module + + +class BaseConv(nn.Module): + """A Conv2d -> Batchnorm -> silu/leaky relu block""" + + def __init__(self, + in_channels, + out_channels, + ksize, + stride=1, + groups=1, + bias=False, + act='silu', + norm='bn'): + super().__init__() + # same padding + pad = (ksize - 1) // 2 + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=ksize, + stride=stride, + padding=pad, + groups=groups, + bias=bias, + ) + if norm is not None: + self.bn = get_norm(norm, out_channels, inplace=True) + if act is not None: + self.act = get_activation(act, inplace=True) + self.with_norm = norm is not None + self.with_act = act is not None + + def forward(self, x): + x = self.conv(x) + if self.with_norm: + # x = self.norm(x) + x = self.bn(x) + if self.with_act: + x = self.act(x) + return x + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class DepthWiseConv(nn.Module): + + def __init__(self, + in_channels, + out_channels, + ksize, + stride=1, + groups=None, + bias=False, + act='silu', + norm='bn'): + super().__init__() + padding = (ksize - 1) // 2 + self.depthwise = nn.Conv2d( + in_channels, + in_channels, + kernel_size=ksize, + stride=stride, + padding=padding, + groups=in_channels, + bias=bias, + ) + + self.pointwise = nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias) + if norm is not None: + self.dwnorm = get_norm(norm, in_channels, inplace=True) + self.pwnorm = get_norm(norm, out_channels, inplace=True) + if act is not None: + self.act = get_activation(act, inplace=True) + + self.with_norm = norm is not None + self.with_act = act is not None + self.order = ['depthwise', 'dwnorm', 'pointwise', 'act'] + + def forward(self, x): + + for layer_name in self.order: + layer = self.__getattr__(layer_name) + if layer is not None: + x = layer(x) + return x + + +class DWConv(nn.Module): + """Depthwise Conv + Conv""" + + def __init__(self, in_channels, out_channels, ksize, stride=1, act='silu'): + super().__init__() + self.dconv = BaseConv( + in_channels, + in_channels, + ksize=ksize, + stride=stride, + groups=in_channels, + act=act, + ) + self.pconv = BaseConv( + in_channels, out_channels, ksize=1, stride=1, groups=1, act=act) + + def forward(self, x): + x = self.dconv(x) + return self.pconv(x) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__( + self, + in_channels, + out_channels, + shortcut=True, + expansion=0.5, + depthwise=False, + act='silu', + reparam=False, + ): + super().__init__() + hidden_channels = int(out_channels * expansion) + Conv = DWConv if depthwise else BaseConv + k_conv1 = 3 if reparam else 1 + self.conv1 = BaseConv( + in_channels, hidden_channels, k_conv1, stride=1, act=act) + if reparam: + self.conv2 = RepVggBlock( + hidden_channels, out_channels, 3, stride=1, act=act) + else: + self.conv2 = Conv( + hidden_channels, out_channels, 3, stride=1, act=act) + self.use_add = shortcut and in_channels == out_channels + + def forward(self, x): + y = self.conv2(self.conv1(x)) + if self.use_add: + y = y + x + return y + + +class ResLayer(nn.Module): + 'Residual layer with `in_channels` inputs.' + + def __init__(self, in_channels: int): + super().__init__() + mid_channels = in_channels // 2 + self.layer1 = BaseConv( + in_channels, mid_channels, ksize=1, stride=1, act='lrelu') + self.layer2 = BaseConv( + mid_channels, in_channels, ksize=3, stride=1, act='lrelu') + + def forward(self, x): + out = self.layer2(self.layer1(x)) + return x + out + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP""" + + def __init__(self, + in_channels, + out_channels, + kernel_sizes=(5, 9, 13), + activation='silu'): + super().__init__() + hidden_channels = in_channels // 2 + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=activation) + self.m = nn.ModuleList([ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ]) + conv2_channels = hidden_channels * (len(kernel_sizes) + 1) + self.conv2 = BaseConv( + conv2_channels, out_channels, 1, stride=1, act=activation) + + def forward(self, x): + x = self.conv1(x) + x = torch.cat([x] + [m(x) for m in self.m], dim=1) + x = self.conv2(x) + return x + + +class CSPLayer(nn.Module): + """C3 in yolov5, CSP Bottleneck with 3 convolutions""" + + def __init__( + self, + in_channels, + out_channels, + n=1, + shortcut=True, + expansion=0.5, + depthwise=False, + act='silu', + reparam=False, + ): + """ + Args: + in_channels (int): input channels. + out_channels (int): output channels. + n (int): number of Bottlenecks. Default value: 1. + """ + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + hidden_channels = int(out_channels * expansion) # hidden channels + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv3 = BaseConv( + 2 * hidden_channels, out_channels, 1, stride=1, act=act) + module_list = [ + Bottleneck( + hidden_channels, + hidden_channels, + shortcut, + 1.0, + depthwise, + act=act, + reparam=reparam) for _ in range(n) + ] + self.m = nn.Sequential(*module_list) + + def forward(self, x): + x_1 = self.conv1(x) + x_2 = self.conv2(x) + x_1 = self.m(x_1) + x = torch.cat((x_1, x_2), dim=1) + return self.conv3(x) + + +class Focus(nn.Module): + """Focus width and height information into channel space.""" + + def __init__(self, + in_channels, + out_channels, + ksize=1, + stride=1, + act='silu'): + super().__init__() + self.conv = BaseConv( + in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) + patch_top_left = x[..., ::2, ::2] + patch_top_right = x[..., ::2, 1::2] + patch_bot_left = x[..., 1::2, ::2] + patch_bot_right = x[..., 1::2, 1::2] + x = torch.cat( + ( + patch_top_left, + patch_bot_left, + patch_top_right, + patch_bot_right, + ), + dim=1, + ) + return self.conv(x) + + +class fast_Focus(nn.Module): + + def __init__(self, + in_channels, + out_channels, + ksize=1, + stride=1, + act='silu'): + super(Focus, self).__init__() + self.conv1 = self.focus_conv(w1=1.0) + self.conv2 = self.focus_conv(w3=1.0) + self.conv3 = self.focus_conv(w2=1.0) + self.conv4 = self.focus_conv(w4=1.0) + + self.conv = BaseConv( + in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + return self.conv( + torch.cat( + [self.conv1(x), + self.conv2(x), + self.conv3(x), + self.conv4(x)], 1)) + + def focus_conv(self, w1=0.0, w2=0.0, w3=0.0, w4=0.0): + conv = nn.Conv2d(3, 3, 2, 2, groups=3, bias=False) + conv.weight = self.init_weights_constant(w1, w2, w3, w4) + conv.weight.requires_grad = False + return conv + + def init_weights_constant(self, w1=0.0, w2=0.0, w3=0.0, w4=0.0): + return nn.Parameter( + torch.tensor([[[[w1, w2], [w3, w4]]], [[[w1, w2], [w3, w4]]], + [[[w1, w2], [w3, w4]]]])) + + +# shufflenet block +def channel_shuffle(x, groups=2): + bat_size, channels, w, h = x.shape + group_c = channels // groups + x = x.view(bat_size, groups, group_c, w, h) + x = torch.transpose(x, 1, 2).contiguous() + x = x.view(bat_size, -1, w, h) + return x + + +def conv_1x1_bn(in_c, out_c, stride=1): + return nn.Sequential( + nn.Conv2d(in_c, out_c, 1, stride, 0, bias=False), + nn.BatchNorm2d(out_c), nn.ReLU(True)) + + +def conv_bn(in_c, out_c, stride=2): + return nn.Sequential( + nn.Conv2d(in_c, out_c, 3, stride, 1, bias=False), + nn.BatchNorm2d(out_c), nn.ReLU(True)) + + +class ShuffleBlock(nn.Module): + + def __init__(self, in_c, out_c, downsample=False): + super(ShuffleBlock, self).__init__() + self.downsample = downsample + half_c = out_c // 2 + if downsample: + self.branch1 = nn.Sequential( + # 3*3 dw conv, stride = 2 + # nn.Conv2d(in_c, in_c, 3, 2, 1, groups=in_c, bias=False), + nn.Conv2d(in_c, in_c, 3, 1, 1, groups=in_c, bias=False), + nn.BatchNorm2d(in_c), + # 1*1 pw conv + nn.Conv2d(in_c, half_c, 1, 1, 0, bias=False), + nn.BatchNorm2d(half_c), + nn.ReLU(True)) + + self.branch2 = nn.Sequential( + # 1*1 pw conv + nn.Conv2d(in_c, half_c, 1, 1, 0, bias=False), + nn.BatchNorm2d(half_c), + nn.ReLU(True), + # 3*3 dw conv, stride = 2 + # nn.Conv2d(half_c, half_c, 3, 2, 1, groups=half_c, bias=False), + nn.Conv2d(half_c, half_c, 3, 1, 1, groups=half_c, bias=False), + nn.BatchNorm2d(half_c), + # 1*1 pw conv + nn.Conv2d(half_c, half_c, 1, 1, 0, bias=False), + nn.BatchNorm2d(half_c), + nn.ReLU(True)) + else: + # in_c = out_c + assert in_c == out_c + + self.branch2 = nn.Sequential( + # 1*1 pw conv + nn.Conv2d(half_c, half_c, 1, 1, 0, bias=False), + nn.BatchNorm2d(half_c), + nn.ReLU(True), + # 3*3 dw conv, stride = 1 + nn.Conv2d(half_c, half_c, 3, 1, 1, groups=half_c, bias=False), + nn.BatchNorm2d(half_c), + # 1*1 pw conv + nn.Conv2d(half_c, half_c, 1, 1, 0, bias=False), + nn.BatchNorm2d(half_c), + nn.ReLU(True)) + + def forward(self, x): + out = None + if self.downsample: + # if it is downsampling, we don't need to do channel split + out = torch.cat((self.branch1(x), self.branch2(x)), 1) + else: + # channel split + channels = x.shape[1] + c = channels // 2 + x1 = x[:, :c, :, :] + x2 = x[:, c:, :, :] + out = torch.cat((x1, self.branch2(x2)), 1) + return channel_shuffle(out, 2) + + +class ShuffleCSPLayer(nn.Module): + """C3 in yolov5, CSP Bottleneck with 3 convolutions""" + + def __init__( + self, + in_channels, + out_channels, + n=1, + shortcut=True, + expansion=0.5, + depthwise=False, + act='silu', + ): + """ + Args: + in_channels (int): input channels. + out_channels (int): output channels. + n (int): number of Bottlenecks. Default value: 1. + """ + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + hidden_channels = int(out_channels * expansion) # hidden channels + self.conv1 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = BaseConv( + in_channels, hidden_channels, 1, stride=1, act=act) + module_list = [ + Bottleneck( + hidden_channels, + hidden_channels, + shortcut, + 1.0, + depthwise, + act=act) for _ in range(n) + ] + self.m = nn.Sequential(*module_list) + + def forward(self, x): + x_1 = self.conv1(x) + x_2 = self.conv2(x) + x_1 = self.m(x_1) + x = torch.cat((x_1, x_2), dim=1) + # add channel shuffle + return channel_shuffle(x, 2) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/neck_ops.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/neck_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..b04c323d0c54a8048a2c216759229d1a882efc2c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/neck_ops.py @@ -0,0 +1,324 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Swish(nn.Module): + + def __init__(self, inplace=True): + super(Swish, self).__init__() + self.inplace = inplace + + def forward(self, x): + if self.inplace: + x.mul_(F.sigmoid(x)) + return x + else: + return x * F.sigmoid(x) + + +def get_activation(name='silu', inplace=True): + if name is None: + return nn.Identity() + + if isinstance(name, str): + if name == 'silu': + module = nn.SiLU(inplace=inplace) + elif name == 'relu': + module = nn.ReLU(inplace=inplace) + elif name == 'lrelu': + module = nn.LeakyReLU(0.1, inplace=inplace) + elif name == 'swish': + module = Swish(inplace=inplace) + elif name == 'hardsigmoid': + module = nn.Hardsigmoid(inplace=inplace) + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + return module + elif isinstance(name, nn.Module): + return name + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + + +class ConvBNLayer(nn.Module): + + def __init__(self, + ch_in, + ch_out, + filter_size=3, + stride=1, + groups=1, + padding=0, + act=None): + super(ConvBNLayer, self).__init__() + self.conv = nn.Conv2d( + in_channels=ch_in, + out_channels=ch_out, + kernel_size=filter_size, + stride=stride, + padding=padding, + groups=groups, + bias=False) + self.bn = nn.BatchNorm2d(ch_out, ) + self.act = get_activation(act, inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.act(x) + + return x + + +class RepVGGBlock(nn.Module): + + def __init__(self, ch_in, ch_out, act='relu', deploy=False): + super(RepVGGBlock, self).__init__() + self.ch_in = ch_in + self.ch_out = ch_out + self.deploy = deploy + self.in_channels = ch_in + self.groups = 1 + if self.deploy is False: + self.rbr_dense = ConvBNLayer( + ch_in, ch_out, 3, stride=1, padding=1, act=None) + self.rbr_1x1 = ConvBNLayer( + ch_in, ch_out, 1, stride=1, padding=0, act=None) + # self.rbr_identity = nn.BatchNorm2d(num_features=ch_in) if ch_out == ch_in else None + self.rbr_identity = None + else: + self.rbr_reparam = nn.Conv2d( + in_channels=self.ch_in, + out_channels=self.ch_out, + kernel_size=3, + stride=1, + padding=1, + groups=1) + self.act = get_activation(act) if act is None or isinstance( + act, (str, dict)) else act + + def forward(self, x): + if self.deploy: + print('----------deploy----------') + y = self.rbr_reparam(x) + else: + if self.rbr_identity is None: + y = self.rbr_dense(x) + self.rbr_1x1(x) + else: + y = self.rbr_dense(x) + self.rbr_1x1(x) + self.rbr_identity(x) + + y = self.act(y) + return y + + def switch_to_deploy(self): + print('switch') + if not hasattr(self, 'rbr_reparam'): + # return + self.rbr_reparam = nn.Conv2d( + in_channels=self.ch_in, + out_channels=self.ch_out, + kernel_size=3, + stride=1, + padding=1, + groups=1) + print('switch') + kernel, bias = self.get_equivalent_kernel_bias() + self.rbr_reparam.weight.data = kernel + self.rbr_reparam.bias.data = bias + for para in self.parameters(): + para.detach_() + # self.__delattr__(self.rbr_dense) + # self.__delattr__(self.rbr_1x1) + self.__delattr__('rbr_dense') + self.__delattr__('rbr_1x1') + if hasattr(self, 'rbr_identity'): + self.__delattr__('rbr_identity') + if hasattr(self, 'id_tensor'): + self.__delattr__('id_tensor') + self.deploy = True + + def get_equivalent_kernel_bias(self): + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) + kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) + return kernel3x3 + self._pad_1x1_to_3x3_tensor( + kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + # if isinstance(branch, nn.Sequential): + if isinstance(branch, ConvBNLayer): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + +class BasicBlock(nn.Module): + + def __init__(self, ch_in, ch_out, act='relu', shortcut=True): + super(BasicBlock, self).__init__() + assert ch_in == ch_out + # self.conv1 = ConvBNLayer(ch_in, ch_out, 3, stride=1, padding=1, act=act) + # self.conv1 = ConvBNLayer(ch_in, ch_out, 1, stride=1, padding=0, act=act) + self.conv2 = RepVGGBlock(ch_in, ch_out, act=act) + self.shortcut = shortcut + + def forward(self, x): + # y = self.conv1(x) + y = self.conv2(x) + if self.shortcut: + return x + y + else: + return y + + +class BasicBlock_3x3(nn.Module): + + def __init__(self, ch_in, ch_out, act='relu', shortcut=True): + super(BasicBlock_3x3, self).__init__() + assert ch_in == ch_out + self.conv1 = ConvBNLayer( + ch_in, ch_out, 3, stride=1, padding=1, act=act) + # self.conv1 = ConvBNLayer(ch_in, ch_out, 1, stride=1, padding=0, act=act) + self.conv2 = RepVGGBlock(ch_in, ch_out, act=act) + self.shortcut = shortcut + + def forward(self, x): + y = self.conv1(x) + y = self.conv2(y) + if self.shortcut: + return x + y + else: + return y + + +class BasicBlock_3x3_Reverse(nn.Module): + + def __init__(self, ch_in, ch_out, act='relu', shortcut=True): + super(BasicBlock_3x3_Reverse, self).__init__() + assert ch_in == ch_out + self.conv1 = ConvBNLayer( + ch_in, ch_out, 3, stride=1, padding=1, act=act) + # self.conv1 = ConvBNLayer(ch_in, ch_out, 1, stride=1, padding=0, act=act) + self.conv2 = RepVGGBlock(ch_in, ch_out, act=act) + self.shortcut = shortcut + + def forward(self, x): + y = self.conv2(x) + y = self.conv1(y) + if self.shortcut: + return x + y + else: + return y + + +class SPP(nn.Module): + + def __init__( + self, + ch_in, + ch_out, + k, + pool_size, + act='swish', + ): + super(SPP, self).__init__() + self.pool = [] + for i, size in enumerate(pool_size): + pool = nn.MaxPool2d( + kernel_size=size, stride=1, padding=size // 2, ceil_mode=False) + self.add_module('pool{}'.format(i), pool) + self.pool.append(pool) + self.conv = ConvBNLayer(ch_in, ch_out, k, padding=k // 2, act=act) + + def forward(self, x): + outs = [x] + + for pool in self.pool: + outs.append(pool(x)) + y = torch.cat(outs, axis=1) + + y = self.conv(y) + return y + + +class CSPStage(nn.Module): + + def __init__(self, block_fn, ch_in, ch_out, n, act='swish', spp=False): + super(CSPStage, self).__init__() + + ch_mid = int(ch_out // 2) + self.conv1 = ConvBNLayer(ch_in, ch_mid, 1, act=act) + self.conv2 = ConvBNLayer(ch_in, ch_mid, 1, act=act) + # self.conv2 = ConvBNLayer(ch_in, ch_mid, 3, stride=1, padding=1, act=act) + self.convs = nn.Sequential() + + next_ch_in = ch_mid + for i in range(n): + if block_fn == 'BasicBlock': + self.convs.add_module( + str(i), + BasicBlock(next_ch_in, ch_mid, act=act, shortcut=False)) + elif block_fn == 'BasicBlock_3x3': + self.convs.add_module( + str(i), + BasicBlock_3x3(next_ch_in, ch_mid, act=act, shortcut=True)) + elif block_fn == 'BasicBlock_3x3_Reverse': + self.convs.add_module( + str(i), + BasicBlock_3x3_Reverse( + next_ch_in, ch_mid, act=act, shortcut=True)) + else: + raise NotImplementedError + if i == (n - 1) // 2 and spp: + self.convs.add_module( + 'spp', SPP(ch_mid * 4, ch_mid, 1, [5, 9, 13], act=act)) + next_ch_in = ch_mid + # self.convs = nn.Sequential(*convs) + self.conv3 = ConvBNLayer(ch_mid * (n + 1), ch_out, 1, act=act) + + def forward(self, x): + y1 = self.conv1(x) + y2 = self.conv2(x) + + mid_out = [y1] + for conv in self.convs: + y2 = conv(y2) + mid_out.append(y2) + y = torch.cat(mid_out, axis=1) + y = self.conv3(y) + return y diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ops.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..6ca86a90579448d7d7b2c48efa6184079f9fa58c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ops.py @@ -0,0 +1,434 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class SiLU(nn.Module): + """export-friendly version of nn.SiLU()""" + + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +class Swish(nn.Module): + + def __init__(self, inplace=True): + super(Swish, self).__init__() + self.inplace = inplace + + def forward(self, x): + if self.inplace: + x.mul_(F.sigmoid(x)) + return x + else: + return x * F.sigmoid(x) + + +def get_activation(name='silu', inplace=True): + if name is None: + return nn.Identity() + + if isinstance(name, str): + if name == 'silu': + module = nn.SiLU(inplace=inplace) + elif name == 'relu': + module = nn.ReLU(inplace=inplace) + elif name == 'lrelu': + module = nn.LeakyReLU(0.1, inplace=inplace) + elif name == 'swish': + module = Swish(inplace=inplace) + elif name == 'hardsigmoid': + module = nn.Hardsigmoid(inplace=inplace) + elif name == 'identity': + module = nn.Identity() + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + return module + + elif isinstance(name, nn.Module): + return name + + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + + +def get_norm(name, out_channels, inplace=True): + if name == 'bn': + module = nn.BatchNorm2d(out_channels) + else: + raise NotImplementedError + return module + + +class ConvBNAct(nn.Module): + """A Conv2d -> Batchnorm -> silu/leaky relu block""" + + def __init__( + self, + in_channels, + out_channels, + ksize, + stride=1, + groups=1, + bias=False, + act='silu', + norm='bn', + reparam=False, + ): + super().__init__() + # same padding + pad = (ksize - 1) // 2 + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=ksize, + stride=stride, + padding=pad, + groups=groups, + bias=bias, + ) + if norm is not None: + self.bn = get_norm(norm, out_channels, inplace=True) + if act is not None: + self.act = get_activation(act, inplace=True) + self.with_norm = norm is not None + self.with_act = act is not None + + def forward(self, x): + x = self.conv(x) + if self.with_norm: + x = self.bn(x) + if self.with_act: + x = self.act(x) + return x + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP""" + + def __init__(self, + in_channels, + out_channels, + kernel_sizes=(5, 9, 13), + activation='silu'): + super().__init__() + hidden_channels = in_channels // 2 + self.conv1 = ConvBNAct( + in_channels, hidden_channels, 1, stride=1, act=activation) + self.m = nn.ModuleList([ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ]) + conv2_channels = hidden_channels * (len(kernel_sizes) + 1) + self.conv2 = ConvBNAct( + conv2_channels, out_channels, 1, stride=1, act=activation) + + def forward(self, x): + x = self.conv1(x) + x = torch.cat([x] + [m(x) for m in self.m], dim=1) + x = self.conv2(x) + return x + + +class Focus(nn.Module): + """Focus width and height information into channel space.""" + + def __init__(self, + in_channels, + out_channels, + ksize=1, + stride=1, + act='silu'): + super().__init__() + self.conv = ConvBNAct( + in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) + patch_top_left = x[..., ::2, ::2] + patch_top_right = x[..., ::2, 1::2] + patch_bot_left = x[..., 1::2, ::2] + patch_bot_right = x[..., 1::2, 1::2] + x = torch.cat( + ( + patch_top_left, + patch_bot_left, + patch_top_right, + patch_bot_right, + ), + dim=1, + ) + return self.conv(x) + + +class BasicBlock_3x3_Reverse(nn.Module): + + def __init__(self, + ch_in, + ch_hidden_ratio, + ch_out, + act='relu', + shortcut=True): + super(BasicBlock_3x3_Reverse, self).__init__() + assert ch_in == ch_out + ch_hidden = int(ch_in * ch_hidden_ratio) + self.conv1 = ConvBNAct(ch_hidden, ch_out, 3, stride=1, act=act) + self.conv2 = RepConv(ch_in, ch_hidden, 3, stride=1, act=act) + self.shortcut = shortcut + + def forward(self, x): + y = self.conv2(x) + y = self.conv1(y) + if self.shortcut: + return x + y + else: + return y + + +class SPP(nn.Module): + + def __init__( + self, + ch_in, + ch_out, + k, + pool_size, + act='swish', + ): + super(SPP, self).__init__() + self.pool = [] + for i, size in enumerate(pool_size): + pool = nn.MaxPool2d( + kernel_size=size, stride=1, padding=size // 2, ceil_mode=False) + self.add_module('pool{}'.format(i), pool) + self.pool.append(pool) + self.conv = ConvBNAct(ch_in, ch_out, k, act=act) + + def forward(self, x): + outs = [x] + + for pool in self.pool: + outs.append(pool(x)) + y = torch.cat(outs, axis=1) + + y = self.conv(y) + return y + + +class CSPStage(nn.Module): + + def __init__(self, + block_fn, + ch_in, + ch_hidden_ratio, + ch_out, + n, + act='swish', + spp=False): + super(CSPStage, self).__init__() + + split_ratio = 2 + ch_first = int(ch_out // split_ratio) + ch_mid = int(ch_out - ch_first) + self.conv1 = ConvBNAct(ch_in, ch_first, 1, act=act) + self.conv2 = ConvBNAct(ch_in, ch_mid, 1, act=act) + self.convs = nn.Sequential() + + next_ch_in = ch_mid + for i in range(n): + if block_fn == 'BasicBlock_3x3_Reverse': + self.convs.add_module( + str(i), + BasicBlock_3x3_Reverse( + next_ch_in, + ch_hidden_ratio, + ch_mid, + act=act, + shortcut=True)) + else: + raise NotImplementedError + if i == (n - 1) // 2 and spp: + self.convs.add_module( + 'spp', SPP(ch_mid * 4, ch_mid, 1, [5, 9, 13], act=act)) + next_ch_in = ch_mid + self.conv3 = ConvBNAct(ch_mid * n + ch_first, ch_out, 1, act=act) + + def forward(self, x): + y1 = self.conv1(x) + y2 = self.conv2(x) + + mid_out = [y1] + for conv in self.convs: + y2 = conv(y2) + mid_out.append(y2) + y = torch.cat(mid_out, axis=1) + y = self.conv3(y) + return y + + +def conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1): + '''Basic cell for rep-style block, including conv and bn''' + result = nn.Sequential() + result.add_module( + 'conv', + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False)) + result.add_module('bn', nn.BatchNorm2d(num_features=out_channels)) + return result + + +class RepConv(nn.Module): + '''RepConv is a basic rep-style block, including training and deploy status + Code is based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py + ''' + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + dilation=1, + groups=1, + padding_mode='zeros', + deploy=False, + act='relu', + norm=None): + super(RepConv, self).__init__() + self.deploy = deploy + self.groups = groups + self.in_channels = in_channels + self.out_channels = out_channels + + assert kernel_size == 3 + assert padding == 1 + + padding_11 = padding - kernel_size // 2 + + if isinstance(act, str): + self.nonlinearity = get_activation(act) + else: + self.nonlinearity = act + + if deploy: + self.rbr_reparam = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=True, + padding_mode=padding_mode) + + else: + self.rbr_identity = None + self.rbr_dense = conv_bn( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups) + self.rbr_1x1 = conv_bn( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + stride=stride, + padding=padding_11, + groups=groups) + + def forward(self, inputs): + '''Forward process''' + if hasattr(self, 'rbr_reparam'): + return self.nonlinearity(self.rbr_reparam(inputs)) + + if self.rbr_identity is None: + id_out = 0 + else: + id_out = self.rbr_identity(inputs) + + return self.nonlinearity( + self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out) + + def get_equivalent_kernel_bias(self): + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) + kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) + return kernel3x3 + self._pad_1x1_to_3x3_tensor( + kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def switch_to_deploy(self): + if hasattr(self, 'rbr_reparam'): + return + kernel, bias = self.get_equivalent_kernel_bias() + self.rbr_reparam = nn.Conv2d( + in_channels=self.rbr_dense.conv.in_channels, + out_channels=self.rbr_dense.conv.out_channels, + kernel_size=self.rbr_dense.conv.kernel_size, + stride=self.rbr_dense.conv.stride, + padding=self.rbr_dense.conv.padding, + dilation=self.rbr_dense.conv.dilation, + groups=self.rbr_dense.conv.groups, + bias=True) + self.rbr_reparam.weight.data = kernel + self.rbr_reparam.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rbr_dense') + self.__delattr__('rbr_1x1') + if hasattr(self, 'rbr_identity'): + self.__delattr__('rbr_identity') + if hasattr(self, 'id_tensor'): + self.__delattr__('id_tensor') + self.deploy = True diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ota_assigner.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ota_assigner.py new file mode 100644 index 0000000000000000000000000000000000000000..90bd4c22fabec285989cf602374cc78c4e77cf1d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/ota_assigner.py @@ -0,0 +1,435 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Copyright (c) Alibaba, Inc. and its affiliates. + +import warnings + +import torch +import torch.nn.functional as F + +from modelscope.models.cv.tinynas_detection.damo.utils.boxes import \ + bbox_overlaps + + +class BaseAssigner(object): + """Base assigner that assigns boxes to ground truth boxes.""" + + def assign(self, bboxes, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None): + """Assign boxes to either a ground truth boxes or a negative boxes.""" + + +class AssignResult(object): + """Stores assignments between predicted and truth boxes. + Attributes: + num_gts (int): the number of truth boxes considered when computing this + assignment + gt_inds (LongTensor): for each predicted box indicates the 1-based + index of the assigned truth box. 0 means unassigned and -1 means + ignore. + max_overlaps (FloatTensor): the iou between the predicted box and its + assigned truth box. + labels (None | LongTensor): If specified, for each predicted box + indicates the category label of the assigned truth box. + Example: + >>> # An assign result between 4 predicted boxes and 9 true boxes + >>> # where only two boxes were assigned. + >>> num_gts = 9 + >>> max_overlaps = torch.LongTensor([0, .5, .9, 0]) + >>> gt_inds = torch.LongTensor([-1, 1, 2, 0]) + >>> labels = torch.LongTensor([0, 3, 4, 0]) + >>> self = AssignResult(num_gts, gt_inds, max_overlaps, labels) + >>> print(str(self)) # xdoctest: +IGNORE_WANT + + >>> # Force addition of gt labels (when adding gt as proposals) + >>> new_labels = torch.LongTensor([3, 4, 5]) + >>> self.add_gt_(new_labels) + >>> print(str(self)) # xdoctest: +IGNORE_WANT + + """ + + def __init__(self, num_gts, gt_inds, max_overlaps, labels=None): + self.num_gts = num_gts + self.gt_inds = gt_inds + self.max_overlaps = max_overlaps + self.labels = labels + # Interface for possible user-defined properties + self._extra_properties = {} + + @property + def num_preds(self): + """int: the number of predictions in this assignment""" + return len(self.gt_inds) + + def set_extra_property(self, key, value): + """Set user-defined new property.""" + assert key not in self.info + self._extra_properties[key] = value + + def get_extra_property(self, key): + """Get user-defined property.""" + return self._extra_properties.get(key, None) + + @property + def info(self): + """dict: a dictionary of info about the object""" + basic_info = { + 'num_gts': self.num_gts, + 'num_preds': self.num_preds, + 'gt_inds': self.gt_inds, + 'max_overlaps': self.max_overlaps, + 'labels': self.labels, + } + basic_info.update(self._extra_properties) + return basic_info + + @classmethod + def random(cls, **kwargs): + """Create random AssignResult for tests or debugging. + Args: + num_preds: number of predicted boxes + num_gts: number of true boxes + p_ignore (float): probability of a predicted box assinged to an + ignored truth + p_assigned (float): probability of a predicted box not being + assigned + p_use_label (float | bool): with labels or not + rng (None | int | numpy.random.RandomState): seed or state + Returns: + :obj:`AssignResult`: Randomly generated assign results. + Example: + >>> from mmdet.core.bbox.assigners.assign_result import * # NOQA + >>> self = AssignResult.random() + >>> print(self.info) + """ + from mmdet.core.bbox import demodata + rng = demodata.ensure_rng(kwargs.get('rng', None)) + + num_gts = kwargs.get('num_gts', None) + num_preds = kwargs.get('num_preds', None) + p_ignore = kwargs.get('p_ignore', 0.3) + p_assigned = kwargs.get('p_assigned', 0.7) + p_use_label = kwargs.get('p_use_label', 0.5) + num_classes = kwargs.get('p_use_label', 3) + + if num_gts is None: + num_gts = rng.randint(0, 8) + if num_preds is None: + num_preds = rng.randint(0, 16) + + if num_gts == 0: + max_overlaps = torch.zeros(num_preds, dtype=torch.float32) + gt_inds = torch.zeros(num_preds, dtype=torch.int64) + if p_use_label is True or p_use_label < rng.rand(): + labels = torch.zeros(num_preds, dtype=torch.int64) + else: + labels = None + else: + import numpy as np + # Create an overlap for each predicted box + max_overlaps = torch.from_numpy(rng.rand(num_preds)) + + # Construct gt_inds for each predicted box + is_assigned = torch.from_numpy(rng.rand(num_preds) < p_assigned) + # maximum number of assignments constraints + n_assigned = min(num_preds, min(num_gts, is_assigned.sum())) + + assigned_idxs = np.where(is_assigned)[0] + rng.shuffle(assigned_idxs) + assigned_idxs = assigned_idxs[0:n_assigned] + assigned_idxs.sort() + + is_assigned[:] = 0 + is_assigned[assigned_idxs] = True + + is_ignore = torch.from_numpy( + rng.rand(num_preds) < p_ignore) & is_assigned + + gt_inds = torch.zeros(num_preds, dtype=torch.int64) + + true_idxs = np.arange(num_gts) + rng.shuffle(true_idxs) + true_idxs = torch.from_numpy(true_idxs) + gt_inds[is_assigned] = true_idxs[:n_assigned] + + gt_inds = torch.from_numpy( + rng.randint(1, num_gts + 1, size=num_preds)) + gt_inds[is_ignore] = -1 + gt_inds[~is_assigned] = 0 + max_overlaps[~is_assigned] = 0 + + if p_use_label is True or p_use_label < rng.rand(): + if num_classes == 0: + labels = torch.zeros(num_preds, dtype=torch.int64) + else: + labels = torch.from_numpy( + # remind that we set FG labels to [0, num_class-1] + # since mmdet v2.0 + # BG cat_id: num_class + rng.randint(0, num_classes, size=num_preds)) + labels[~is_assigned] = 0 + else: + labels = None + + self = cls(num_gts, gt_inds, max_overlaps, labels) + return self + + def add_gt_(self, gt_labels): + """Add ground truth as assigned results. + Args: + gt_labels (torch.Tensor): Labels of gt boxes + """ + self_inds = torch.arange( + 1, len(gt_labels) + 1, dtype=torch.long, device=gt_labels.device) + self.gt_inds = torch.cat([self_inds, self.gt_inds]) + + self.max_overlaps = torch.cat( + [self.max_overlaps.new_ones(len(gt_labels)), self.max_overlaps]) + + if self.labels is not None: + self.labels = torch.cat([gt_labels, self.labels]) + + +class AlignOTAAssigner(BaseAssigner): + """Computes matching between predictions and ground truth. + Args: + center_radius (int | float, optional): Ground truth center size + to judge whether a prior is in center. Default 2.5. + candidate_topk (int, optional): The candidate top-k which used to + get top-k ious to calculate dynamic-k. Default 10. + iou_weight (int | float, optional): The scale factor for regression + iou cost. Default 3.0. + cls_weight (int | float, optional): The scale factor for classification + cost. Default 1.0. + """ + + def __init__(self, + center_radius=2.5, + candidate_topk=10, + iou_weight=3.0, + cls_weight=1.0): + self.center_radius = center_radius + self.candidate_topk = candidate_topk + self.iou_weight = iou_weight + self.cls_weight = cls_weight + + def assign(self, + pred_scores, + priors, + decoded_bboxes, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + eps=1e-7): + """Assign gt to priors using SimOTA. It will switch to CPU mode when + GPU is out of memory. + Args: + pred_scores (Tensor): Classification scores of one image, + a 2D-Tensor with shape [num_priors, num_classes] + priors (Tensor): All priors of one image, a 2D-Tensor with shape + [num_priors, 4] in [cx, xy, stride_w, stride_y] format. + decoded_bboxes (Tensor): Predicted bboxes, a 2D-Tensor with shape + [num_priors, 4] in [tl_x, tl_y, br_x, br_y] format. + gt_bboxes (Tensor): Ground truth bboxes of one image, a 2D-Tensor + with shape [num_gts, 4] in [tl_x, tl_y, br_x, br_y] format. + gt_labels (Tensor): Ground truth labels of one image, a Tensor + with shape [num_gts]. + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + eps (float): A value added to the denominator for numerical + stability. Default 1e-7. + Returns: + assign_result (obj:`AssignResult`): The assigned result. + """ + try: + assign_result = self._assign(pred_scores, priors, decoded_bboxes, + gt_bboxes, gt_labels, + gt_bboxes_ignore, eps) + return assign_result + except RuntimeError: + origin_device = pred_scores.device + warnings.warn('OOM RuntimeError is raised due to the huge memory ' + 'cost during label assignment. CPU mode is applied ' + 'in this batch. If you want to avoid this issue, ' + 'try to reduce the batch size or image size.') + torch.cuda.empty_cache() + + pred_scores = pred_scores.cpu() + priors = priors.cpu() + decoded_bboxes = decoded_bboxes.cpu() + gt_bboxes = gt_bboxes.cpu().float() + gt_labels = gt_labels.cpu() + + assign_result = self._assign(pred_scores, priors, decoded_bboxes, + gt_bboxes, gt_labels, + gt_bboxes_ignore, eps) + assign_result.gt_inds = assign_result.gt_inds.to(origin_device) + assign_result.max_overlaps = assign_result.max_overlaps.to( + origin_device) + assign_result.labels = assign_result.labels.to(origin_device) + + return assign_result + + def _assign(self, + pred_scores, + priors, + decoded_bboxes, + gt_bboxes, + gt_labels, + gt_bboxes_ignore=None, + eps=1e-7): + """Assign gt to priors using SimOTA. + Args: + pred_scores (Tensor): Classification scores of one image, + a 2D-Tensor with shape [num_priors, num_classes] + priors (Tensor): All priors of one image, a 2D-Tensor with shape + [num_priors, 4] in [cx, xy, stride_w, stride_y] format. + decoded_bboxes (Tensor): Predicted bboxes, a 2D-Tensor with shape + [num_priors, 4] in [tl_x, tl_y, br_x, br_y] format. + gt_bboxes (Tensor): Ground truth bboxes of one image, a 2D-Tensor + with shape [num_gts, 4] in [tl_x, tl_y, br_x, br_y] format. + gt_labels (Tensor): Ground truth labels of one image, a Tensor + with shape [num_gts]. + gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are + labelled as `ignored`, e.g., crowd boxes in COCO. + eps (float): A value added to the denominator for numerical + stability. Default 1e-7. + Returns: + :obj:`AssignResult`: The assigned result. + """ + INF = 100000000 + num_gt = gt_bboxes.size(0) + num_bboxes = decoded_bboxes.size(0) + + # assign 0 by default + assigned_gt_inds = decoded_bboxes.new_full((num_bboxes, ), + 0, + dtype=torch.long) + valid_mask, is_in_boxes_and_center = self.get_in_gt_and_in_center_info( + priors, gt_bboxes) + valid_decoded_bbox = decoded_bboxes[valid_mask] + valid_pred_scores = pred_scores[valid_mask] + num_valid = valid_decoded_bbox.size(0) + + if num_gt == 0 or num_bboxes == 0 or num_valid == 0: + # No ground truth or boxes, return empty assignment + max_overlaps = decoded_bboxes.new_zeros((num_bboxes, )) + if num_gt == 0: + # No truth, assign everything to background + assigned_gt_inds[:] = 0 + if gt_labels is None: + assigned_labels = None + else: + assigned_labels = decoded_bboxes.new_full((num_bboxes, ), + -1, + dtype=torch.long) + return AssignResult( + num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) + + pairwise_ious = bbox_overlaps(valid_decoded_bbox, gt_bboxes) + iou_cost = -torch.log(pairwise_ious + eps) + iou_cost = iou_cost * self.iou_weight + + gt_onehot_label = ( + F.one_hot(gt_labels.to(torch.int64), + pred_scores.shape[-1]).float().unsqueeze(0).repeat( + num_valid, 1, 1)) + + valid_pred_scores = valid_pred_scores.unsqueeze(1).repeat(1, num_gt, 1) + + soft_label = gt_onehot_label * pairwise_ious[..., None] + scale_factor = soft_label - valid_pred_scores + + cls_cost = F.binary_cross_entropy( + valid_pred_scores, soft_label, + reduction='none') * scale_factor.abs().pow(2.0) + + cls_cost = cls_cost.sum(dim=-1) + cls_cost = cls_cost * self.cls_weight + cost_matrix = (cls_cost + iou_cost + (~is_in_boxes_and_center) * INF) + matched_pred_ious, matched_gt_inds = self.dynamic_k_matching( + cost_matrix, pairwise_ious, num_gt, valid_mask) + + # convert to AssignResult format + assigned_gt_inds[valid_mask] = matched_gt_inds + 1 + assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) + assigned_labels[valid_mask] = gt_labels[matched_gt_inds].long() + max_overlaps = assigned_gt_inds.new_full((num_bboxes, ), + -INF, + dtype=torch.float32) + max_overlaps[valid_mask] = matched_pred_ious + return AssignResult( + num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) + + def get_in_gt_and_in_center_info(self, priors, gt_bboxes): + num_gt = gt_bboxes.size(0) + + repeated_x = priors[:, 0].unsqueeze(1).repeat(1, num_gt) + repeated_y = priors[:, 1].unsqueeze(1).repeat(1, num_gt) + repeated_stride_x = priors[:, 2].unsqueeze(1).repeat(1, num_gt) + repeated_stride_y = priors[:, 3].unsqueeze(1).repeat(1, num_gt) + + # is prior centers in gt bboxes, shape: [n_prior, n_gt] + l_ = repeated_x - gt_bboxes[:, 0] + t_ = repeated_y - gt_bboxes[:, 1] + r_ = gt_bboxes[:, 2] - repeated_x + b_ = gt_bboxes[:, 3] - repeated_y + + deltas = torch.stack([l_, t_, r_, b_], dim=1) + is_in_gts = deltas.min(dim=1).values > 0 + is_in_gts_all = is_in_gts.sum(dim=1) > 0 + + # is prior centers in gt centers + gt_cxs = (gt_bboxes[:, 0] + gt_bboxes[:, 2]) / 2.0 + gt_cys = (gt_bboxes[:, 1] + gt_bboxes[:, 3]) / 2.0 + ct_box_l = gt_cxs - self.center_radius * repeated_stride_x + ct_box_t = gt_cys - self.center_radius * repeated_stride_y + ct_box_r = gt_cxs + self.center_radius * repeated_stride_x + ct_box_b = gt_cys + self.center_radius * repeated_stride_y + + cl_ = repeated_x - ct_box_l + ct_ = repeated_y - ct_box_t + cr_ = ct_box_r - repeated_x + cb_ = ct_box_b - repeated_y + + ct_deltas = torch.stack([cl_, ct_, cr_, cb_], dim=1) + is_in_cts = ct_deltas.min(dim=1).values > 0 + is_in_cts_all = is_in_cts.sum(dim=1) > 0 + + # in boxes or in centers, shape: [num_priors] + is_in_gts_or_centers = is_in_gts_all | is_in_cts_all + + # both in boxes and centers, shape: [num_fg, num_gt] + is_in_boxes_and_centers = ( + is_in_gts[is_in_gts_or_centers, :] + & is_in_cts[is_in_gts_or_centers, :]) + return is_in_gts_or_centers, is_in_boxes_and_centers + + def dynamic_k_matching(self, cost, pairwise_ious, num_gt, valid_mask): + matching_matrix = torch.zeros_like(cost) + # select candidate topk ious for dynamic-k calculation + candidate_topk = min(self.candidate_topk, pairwise_ious.size(0)) + topk_ious, _ = torch.topk(pairwise_ious, candidate_topk, dim=0) + # calculate dynamic k for each gt + dynamic_ks = torch.clamp(topk_ious.sum(0).int(), min=1) + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[:, gt_idx], k=dynamic_ks[gt_idx].item(), largest=False) + matching_matrix[:, gt_idx][pos_idx] = 1.0 + + del topk_ious, dynamic_ks, pos_idx + + prior_match_gt_mask = matching_matrix.sum(1) > 1 + if prior_match_gt_mask.sum() > 0: + cost_min, cost_argmin = torch.min( + cost[prior_match_gt_mask, :], dim=1) + matching_matrix[prior_match_gt_mask, :] *= 0.0 + matching_matrix[prior_match_gt_mask, cost_argmin] = 1.0 + # get foreground mask inside box and center prior + fg_mask_inboxes = matching_matrix.sum(1) > 0.0 + valid_mask[valid_mask.clone()] = fg_mask_inboxes + + matched_gt_inds = matching_matrix[fg_mask_inboxes, :].argmax(1) + matched_pred_ious = (matching_matrix + * pairwise_ious).sum(1)[fg_mask_inboxes] + return matched_pred_ious, matched_gt_inds diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/repvgg_block.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/repvgg_block.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c5ddc4bb2e1866a590ef6074b39ce405a4f48d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/repvgg_block.py @@ -0,0 +1,205 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.init as init +from torch.nn.parameter import Parameter + + +def get_activation(name='silu', inplace=True): + if name == 'silu': + module = nn.SiLU(inplace=inplace) + elif name == 'relu': + module = nn.ReLU(inplace=inplace) + elif name == 'lrelu': + module = nn.LeakyReLU(0.1, inplace=inplace) + elif name == 'identity': + module = nn.Identity() + else: + raise AttributeError('Unsupported act type: {}'.format(name)) + return module + + +def conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1): + '''Basic cell for rep-style block, including conv and bn''' + result = nn.Sequential() + result.add_module( + 'conv', + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False)) + result.add_module('bn', nn.BatchNorm2d(num_features=out_channels)) + return result + + +class RepVggBlock(nn.Module): + '''RepVggBlock is a basic rep-style block, including training and deploy status + This code is based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py + ''' + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + dilation=1, + groups=1, + padding_mode='zeros', + deploy=False, + use_se=False, + act='relu', + norm=None): + super(RepVggBlock, self).__init__() + """ Initialization of the class. + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 1 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + padding_mode (string, optional): Default: 'zeros' + deploy: Whether to be deploy status or training status. Default: False + use_se: Whether to use se. Default: False + """ + self.deploy = deploy + self.groups = groups + self.in_channels = in_channels + self.out_channels = out_channels + + assert kernel_size == 3 + assert padding == 1 + + padding_11 = padding - kernel_size // 2 + + if isinstance(act, str): + self.nonlinearity = get_activation(act) + else: + self.nonlinearity = act + + if use_se: + raise NotImplementedError('se block not supported yet') + else: + self.se = nn.Identity() + + if deploy: + self.rbr_reparam = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=True, + padding_mode=padding_mode) + + else: + self.rbr_identity = None + self.rbr_dense = conv_bn( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups) + self.rbr_1x1 = conv_bn( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + stride=stride, + padding=padding_11, + groups=groups) + + def forward(self, inputs): + '''Forward process''' + if hasattr(self, 'rbr_reparam'): + return self.nonlinearity(self.se(self.rbr_reparam(inputs))) + + if self.rbr_identity is None: + id_out = 0 + else: + id_out = self.rbr_identity(inputs) + + return self.nonlinearity( + self.se(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out)) + + def get_equivalent_kernel_bias(self): + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) + kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) + return kernel3x3 + self._pad_1x1_to_3x3_tensor( + kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), + dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to( + branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def switch_to_deploy(self): + if hasattr(self, 'rbr_reparam'): + return + kernel, bias = self.get_equivalent_kernel_bias() + self.rbr_reparam = nn.Conv2d( + in_channels=self.rbr_dense.conv.in_channels, + out_channels=self.rbr_dense.conv.out_channels, + kernel_size=self.rbr_dense.conv.kernel_size, + stride=self.rbr_dense.conv.stride, + padding=self.rbr_dense.conv.padding, + dilation=self.rbr_dense.conv.dilation, + groups=self.rbr_dense.conv.groups, + bias=True) + self.rbr_reparam.weight.data = kernel + self.rbr_reparam.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rbr_dense') + self.__delattr__('rbr_1x1') + if hasattr(self, 'rbr_identity'): + self.__delattr__('rbr_identity') + if hasattr(self, 'id_tensor'): + self.__delattr__('id_tensor') + self.deploy = True diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/utils.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..04b587e11481a6758470ff7b68b2ed4c3f9c939c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/utils.py @@ -0,0 +1,80 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Copyright (c) Alibaba, Inc. and its affiliates. + +from functools import partial + +import torch +import torch.distributed as dist +import torch.nn as nn + + +class Scale(nn.Module): + """A learnable scale parameter. + This layer scales the input by a learnable factor. It multiplies a + learnable scale parameter of shape (1,) with input of any shape. + Args: + scale (float): Initial value of scale factor. Default: 1.0 + """ + + def __init__(self, scale=1.0): + super(Scale, self).__init__() + self.scale = nn.Parameter(torch.tensor(scale, dtype=torch.float)) + + def forward(self, x): + return x * self.scale + + +def multi_apply(func, *args, **kwargs): + """Apply function to a list of arguments. + Note: + This function applies the ``func`` to multiple inputs and + map the multiple outputs of the ``func`` into different + list. Each list contains the same type of outputs corresponding + to different inputs. + Args: + func (Function): A function that will be applied to a list of + arguments + Returns: + tuple(list): A tuple containing multiple list, each list contains \ + a kind of returned results by the function + """ + pfunc = partial(func, **kwargs) if kwargs else func + map_results = map(pfunc, *args) + return tuple(map(list, zip(*map_results))) + + +def unmap(data, count, inds, fill=0): + """Unmap a subset of item (data) back to the original set of items (of size + count)""" + if data.dim() == 1: + ret = data.new_full((count, ), fill) + ret[inds.type(torch.bool)] = data + else: + new_size = (count, ) + data.size()[1:] + ret = data.new_full(new_size, fill) + ret[inds.type(torch.bool), :] = data + return ret + + +def reduce_mean(tensor): + """"Obtain the mean of tensor on different GPUs.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor + + +def images_to_levels(target, num_levels): + """Convert targets by image to targets by feature level. + [target_img0, target_img1] -> [target_level0, target_level1, ...] + """ + target = torch.stack(target, 0) + level_targets = [] + start = 0 + for n in num_levels: + end = start + n + # level_targets.append(target[:, start:end].squeeze(0)) + level_targets.append(target[:, start:end]) + start = end + return level_targets diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/weight_init.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..92548b10e2d2855cf30163fc063abab2f03ff0c3 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/core/weight_init.py @@ -0,0 +1,17 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +import numpy as np +import torch.nn as nn + + +def normal_init(module, mean=0, std=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.normal_(module.weight, mean, std) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def bias_init_with_prob(prior_prob): + """initialize conv/fc bias value according to a given probability value.""" + bias_init = float(-np.log((1 - prior_prob) / prior_prob)) + return bias_init diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8c264404ca71850dfc003244f79909ca3324919c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy + +from .gfocal_v2_tiny import GFocalHead_Tiny +from .zero_head import ZeroHead + + +def build_head(cfg): + + head_cfg = copy.deepcopy(cfg) + name = head_cfg.pop('name') + if name == 'ZeroHead': + return ZeroHead(**head_cfg) + elif name == 'GFocalV2': + return GFocalHead_Tiny(**head_cfg) + else: + raise NotImplementedError diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/gfocal_v2_tiny.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/gfocal_v2_tiny.py new file mode 100644 index 0000000000000000000000000000000000000000..017846d3ba8fe6344d804f1e813b99b9372135c1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/gfocal_v2_tiny.py @@ -0,0 +1,373 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +import functools +from functools import partial + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.base_ops import ( + BaseConv, DWConv) + + +class Scale(nn.Module): + + def __init__(self, scale=1.0): + super(Scale, self).__init__() + self.scale = nn.Parameter(torch.tensor(scale, dtype=torch.float)) + + def forward(self, x): + return x * self.scale + + +def multi_apply(func, *args, **kwargs): + + pfunc = partial(func, **kwargs) if kwargs else func + map_results = map(pfunc, *args) + return tuple(map(list, zip(*map_results))) + + +def xyxy2CxCywh(xyxy, size=None): + x1 = xyxy[..., 0] + y1 = xyxy[..., 1] + x2 = xyxy[..., 2] + y2 = xyxy[..., 3] + + cx = (x1 + x2) / 2 + cy = (y1 + y2) / 2 + + w = x2 - x1 + h = y2 - y1 + if size is not None: + w = w.clamp(min=0, max=size[1]) + h = h.clamp(min=0, max=size[0]) + return torch.stack([cx, cy, w, h], axis=-1) + + +def distance2bbox(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + """ + x1 = points[..., 0] - distance[..., 0] + y1 = points[..., 1] - distance[..., 1] + x2 = points[..., 0] + distance[..., 2] + y2 = points[..., 1] + distance[..., 3] + if max_shape is not None: + x1 = x1.clamp(min=0, max=max_shape[1]) + y1 = y1.clamp(min=0, max=max_shape[0]) + x2 = x2.clamp(min=0, max=max_shape[1]) + y2 = y2.clamp(min=0, max=max_shape[0]) + return torch.stack([x1, y1, x2, y2], -1) + + +def bbox2distance(points, bbox, max_dis=None, eps=0.1): + """Decode bounding box based on distances. + """ + left = points[:, 0] - bbox[:, 0] + top = points[:, 1] - bbox[:, 1] + right = bbox[:, 2] - points[:, 0] + bottom = bbox[:, 3] - points[:, 1] + if max_dis is not None: + left = left.clamp(min=0, max=max_dis - eps) + top = top.clamp(min=0, max=max_dis - eps) + right = right.clamp(min=0, max=max_dis - eps) + bottom = bottom.clamp(min=0, max=max_dis - eps) + return torch.stack([left, top, right, bottom], -1) + + +class Integral(nn.Module): + """A fixed layer for calculating integral result from distribution. + """ + + def __init__(self, reg_max=16): + super(Integral, self).__init__() + self.reg_max = reg_max + self.register_buffer('project', + torch.linspace(0, self.reg_max, self.reg_max + 1)) + + def forward(self, x): + """Forward feature from the regression head to get integral result of + bounding box location. + """ + shape = x.size() + x = F.softmax(x.reshape(*shape[:-1], 4, self.reg_max + 1), dim=-1) + b, nb, ne, _ = x.size() + x = x.reshape(b * nb * ne, self.reg_max + 1) + y = self.project.type_as(x).unsqueeze(1) + x = torch.matmul(x, y).reshape(b, nb, 4) + return x + + +class GFocalHead_Tiny(nn.Module): + """Ref to Generalized Focal Loss V2: Learning Reliable Localization Quality + Estimation for Dense Object Detection. + """ + + def __init__( + self, + num_classes, + in_channels, + stacked_convs=4, # 4 + feat_channels=256, + reg_max=12, + reg_topk=4, + reg_channels=64, + strides=[8, 16, 32], + add_mean=True, + norm='gn', + act='relu', + start_kernel_size=3, + conv_groups=1, + conv_type='BaseConv', + simOTA_cls_weight=1.0, + simOTA_iou_weight=3.0, + octbase=8, + simlqe=False, + use_lqe=True, + **kwargs): + self.simlqe = simlqe + self.num_classes = num_classes + self.in_channels = in_channels + self.strides = strides + self.use_lqe = use_lqe + self.feat_channels = feat_channels if isinstance(feat_channels, list) \ + else [feat_channels] * len(self.strides) + + self.cls_out_channels = num_classes + 1 # add 1 for keep consistance with former models + # and will be deprecated in future. + self.stacked_convs = stacked_convs + self.conv_groups = conv_groups + self.reg_max = reg_max + self.reg_topk = reg_topk + self.reg_channels = reg_channels + self.add_mean = add_mean + self.total_dim = reg_topk + self.start_kernel_size = start_kernel_size + + self.norm = norm + self.act = act + self.conv_module = DWConv if conv_type == 'DWConv' else BaseConv + + if add_mean: + self.total_dim += 1 + + super(GFocalHead_Tiny, self).__init__() + self.integral = Integral(self.reg_max) + + self._init_layers() + + def _build_not_shared_convs(self, in_channel, feat_channels): + self.relu = nn.ReLU(inplace=True) + cls_convs = nn.ModuleList() + reg_convs = nn.ModuleList() + + for i in range(self.stacked_convs): + chn = feat_channels if i > 0 else in_channel + kernel_size = 3 if i > 0 else self.start_kernel_size + cls_convs.append( + self.conv_module( + chn, + feat_channels, + kernel_size, + stride=1, + groups=self.conv_groups, + norm=self.norm, + act=self.act)) + reg_convs.append( + self.conv_module( + chn, + feat_channels, + kernel_size, + stride=1, + groups=self.conv_groups, + norm=self.norm, + act=self.act)) + if self.use_lqe: + if not self.simlqe: + conf_vector = [ + nn.Conv2d(4 * self.total_dim, self.reg_channels, 1) + ] + else: + conf_vector = [ + nn.Conv2d(4 * (self.reg_max + 1), self.reg_channels, 1) + ] + conf_vector += [self.relu] + conf_vector += [nn.Conv2d(self.reg_channels, 1, 1), nn.Sigmoid()] + reg_conf = nn.Sequential(*conf_vector) + else: + reg_conf = None + + return cls_convs, reg_convs, reg_conf + + def _init_layers(self): + """Initialize layers of the head.""" + self.relu = nn.ReLU(inplace=True) + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + self.reg_confs = nn.ModuleList() + + for i in range(len(self.strides)): + cls_convs, reg_convs, reg_conf = self._build_not_shared_convs( + self.in_channels[i], self.feat_channels[i]) + self.cls_convs.append(cls_convs) + self.reg_convs.append(reg_convs) + self.reg_confs.append(reg_conf) + + self.gfl_cls = nn.ModuleList([ + nn.Conv2d( + self.feat_channels[i], self.cls_out_channels, 3, padding=1) + for i in range(len(self.strides)) + ]) + + self.gfl_reg = nn.ModuleList([ + nn.Conv2d( + self.feat_channels[i], 4 * (self.reg_max + 1), 3, padding=1) + for i in range(len(self.strides)) + ]) + + self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + def forward(self, + xin, + labels=None, + imgs=None, + conf_thre=0.05, + nms_thre=0.7): + + # prepare labels during training + b, c, h, w = xin[0].shape + if labels is not None: + gt_bbox_list = [] + gt_cls_list = [] + for label in labels: + gt_bbox_list.append(label.bbox) + gt_cls_list.append((label.get_field('labels') + - 1).long()) # labels starts from 1 + + # prepare priors for label assignment and bbox decode + mlvl_priors_list = [ + self.get_single_level_center_priors( + xin[i].shape[0], + xin[i].shape[-2:], + stride, + dtype=torch.float32, + device=xin[0].device) for i, stride in enumerate(self.strides) + ] + mlvl_priors = torch.cat(mlvl_priors_list, dim=1) + + # forward for bboxes and classification prediction + cls_scores, bbox_preds = multi_apply( + self.forward_single, + xin, + self.cls_convs, + self.reg_convs, + self.gfl_cls, + self.gfl_reg, + self.reg_confs, + self.scales, + ) + flatten_cls_scores = torch.cat(cls_scores, dim=1) + flatten_bbox_preds = torch.cat(bbox_preds, dim=1) + + # calculating losses or bboxes decoded + if self.training: + loss = self.loss(flatten_cls_scores, flatten_bbox_preds, + gt_bbox_list, gt_cls_list, mlvl_priors) + return loss + else: + output = self.get_bboxes(flatten_cls_scores, flatten_bbox_preds, + mlvl_priors) + return output + + def forward_single(self, x, cls_convs, reg_convs, gfl_cls, gfl_reg, + reg_conf, scale): + """Forward feature of a single scale level. + + """ + cls_feat = x + reg_feat = x + + for cls_conv in cls_convs: + cls_feat = cls_conv(cls_feat) + for reg_conv in reg_convs: + reg_feat = reg_conv(reg_feat) + + bbox_pred = scale(gfl_reg(reg_feat)).float() + N, C, H, W = bbox_pred.size() + prob = F.softmax( + bbox_pred.reshape(N, 4, self.reg_max + 1, H, W), dim=2) + if self.use_lqe: + if not self.simlqe: + prob_topk, _ = prob.topk(self.reg_topk, dim=2) + + if self.add_mean: + stat = torch.cat( + [prob_topk, + prob_topk.mean(dim=2, keepdim=True)], + dim=2) + else: + stat = prob_topk + + quality_score = reg_conf( + stat.reshape(N, 4 * self.total_dim, H, W)) + else: + quality_score = reg_conf( + bbox_pred.reshape(N, 4 * (self.reg_max + 1), H, W)) + + cls_score = gfl_cls(cls_feat).sigmoid() * quality_score + else: + cls_score = gfl_cls(cls_feat).sigmoid() + + flatten_cls_score = cls_score.flatten(start_dim=2).transpose(1, 2) + flatten_bbox_pred = bbox_pred.flatten(start_dim=2).transpose(1, 2) + return flatten_cls_score, flatten_bbox_pred + + def get_single_level_center_priors(self, batch_size, featmap_size, stride, + dtype, device): + + h, w = featmap_size + x_range = (torch.arange(0, int(w), dtype=dtype, + device=device)) * stride + y_range = (torch.arange(0, int(h), dtype=dtype, + device=device)) * stride + + x = x_range.repeat(h, 1) + y = y_range.unsqueeze(-1).repeat(1, w) + + y = y.flatten() + x = x.flatten() + strides = x.new_full((x.shape[0], ), stride) + priors = torch.stack([x, y, strides, strides], dim=-1) + + return priors.unsqueeze(0).repeat(batch_size, 1, 1) + + def sample(self, assign_result, gt_bboxes): + pos_inds = torch.nonzero( + assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() + neg_inds = torch.nonzero( + assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() + pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + if gt_bboxes.numel() == 0: + # hack for index error case + assert pos_assigned_gt_inds.numel() == 0 + pos_gt_bboxes = torch.empty_like(gt_bboxes).view(-1, 4) + else: + if len(gt_bboxes.shape) < 2: + gt_bboxes = gt_bboxes.view(-1, 4) + pos_gt_bboxes = gt_bboxes[pos_assigned_gt_inds, :] + + return pos_inds, neg_inds, pos_gt_bboxes, pos_assigned_gt_inds + + def get_bboxes(self, + cls_preds, + reg_preds, + mlvl_center_priors, + img_meta=None): + + dis_preds = self.integral(reg_preds) * mlvl_center_priors[..., 2, None] + bboxes = distance2bbox(mlvl_center_priors[..., :2], dis_preds) + + return cls_preds[..., 0:self.num_classes], bboxes diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/zero_head.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/zero_head.py new file mode 100644 index 0000000000000000000000000000000000000000..8026c3019247685427d4e04299820e8177ef995c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/heads/zero_head.py @@ -0,0 +1,526 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.ops import \ + ConvBNAct +from modelscope.models.cv.tinynas_detection.damo.base_models.core.ota_assigner import \ + AlignOTAAssigner +from modelscope.models.cv.tinynas_detection.damo.base_models.core.utils import ( + Scale, multi_apply, reduce_mean) +from modelscope.models.cv.tinynas_detection.damo.base_models.core.weight_init import ( + bias_init_with_prob, normal_init) +from modelscope.models.cv.tinynas_detection.damo.base_models.losses.gfocal_loss import ( + DistributionFocalLoss, GIoULoss, QualityFocalLoss) +from modelscope.models.cv.tinynas_detection.damo.utils import postprocess + + +def distance2bbox(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + """ + x1 = points[..., 0] - distance[..., 0] + y1 = points[..., 1] - distance[..., 1] + x2 = points[..., 0] + distance[..., 2] + y2 = points[..., 1] + distance[..., 3] + if max_shape is not None: + x1 = x1.clamp(min=0, max=max_shape[1]) + y1 = y1.clamp(min=0, max=max_shape[0]) + x2 = x2.clamp(min=0, max=max_shape[1]) + y2 = y2.clamp(min=0, max=max_shape[0]) + return torch.stack([x1, y1, x2, y2], -1) + + +def bbox2distance(points, bbox, max_dis=None, eps=0.1): + """Decode bounding box based on distances. + """ + left = points[:, 0] - bbox[:, 0] + top = points[:, 1] - bbox[:, 1] + right = bbox[:, 2] - points[:, 0] + bottom = bbox[:, 3] - points[:, 1] + if max_dis is not None: + left = left.clamp(min=0, max=max_dis - eps) + top = top.clamp(min=0, max=max_dis - eps) + right = right.clamp(min=0, max=max_dis - eps) + bottom = bottom.clamp(min=0, max=max_dis - eps) + return torch.stack([left, top, right, bottom], -1) + + +class Integral(nn.Module): + """A fixed layer for calculating integral result from distribution. + """ + + def __init__(self, reg_max=16): + super(Integral, self).__init__() + self.reg_max = reg_max + self.register_buffer('project', + torch.linspace(0, self.reg_max, self.reg_max + 1)) + + def forward(self, x): + """Forward feature from the regression head to get integral result of + bounding box location. + """ + b, hw, _, _ = x.size() + x = x.reshape(b * hw * 4, self.reg_max + 1) + y = self.project.type_as(x).unsqueeze(1) + x = torch.matmul(x, y).reshape(b, hw, 4) + return x + + +class ZeroHead(nn.Module): + """Ref to Generalized Focal Loss V2: Learning Reliable Localization Quality + Estimation for Dense Object Detection. + """ + + def __init__( + self, + num_classes, + in_channels, + stacked_convs=4, # 4 + feat_channels=256, + reg_max=12, + strides=[8, 16, 32], + norm='gn', + act='relu', + nms_conf_thre=0.05, + nms_iou_thre=0.7, + nms=True, + **kwargs): + self.in_channels = in_channels + self.num_classes = num_classes + self.stacked_convs = stacked_convs + self.act = act + self.strides = strides + if stacked_convs == 0: + feat_channels = in_channels + if isinstance(feat_channels, list): + self.feat_channels = feat_channels + else: + self.feat_channels = [feat_channels] * len(self.strides) + # add 1 for keep consistance with former models + self.cls_out_channels = num_classes + 1 + self.reg_max = reg_max + + self.nms = nms + self.nms_conf_thre = nms_conf_thre + self.nms_iou_thre = nms_iou_thre + + self.assigner = AlignOTAAssigner( + center_radius=2.5, cls_weight=1.0, iou_weight=3.0) + + self.feat_size = [torch.zeros(4) for _ in strides] + + super(ZeroHead, self).__init__() + self.integral = Integral(self.reg_max) + self.loss_dfl = DistributionFocalLoss(loss_weight=0.25) + self.loss_cls = QualityFocalLoss( + use_sigmoid=False, beta=2.0, loss_weight=1.0) + self.loss_bbox = GIoULoss(loss_weight=2.0) + + self._init_layers() + + def _build_not_shared_convs(self, in_channel, feat_channels): + cls_convs = nn.ModuleList() + reg_convs = nn.ModuleList() + + for i in range(self.stacked_convs): + chn = feat_channels if i > 0 else in_channel + kernel_size = 3 if i > 0 else 1 + cls_convs.append( + ConvBNAct( + chn, + feat_channels, + kernel_size, + stride=1, + groups=1, + norm='bn', + act=self.act)) + reg_convs.append( + ConvBNAct( + chn, + feat_channels, + kernel_size, + stride=1, + groups=1, + norm='bn', + act=self.act)) + + return cls_convs, reg_convs + + def _init_layers(self): + """Initialize layers of the head.""" + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + + for i in range(len(self.strides)): + cls_convs, reg_convs = self._build_not_shared_convs( + self.in_channels[i], self.feat_channels[i]) + self.cls_convs.append(cls_convs) + self.reg_convs.append(reg_convs) + + self.gfl_cls = nn.ModuleList([ + nn.Conv2d( + self.feat_channels[i], self.cls_out_channels, 3, padding=1) + for i in range(len(self.strides)) + ]) + + self.gfl_reg = nn.ModuleList([ + nn.Conv2d( + self.feat_channels[i], 4 * (self.reg_max + 1), 3, padding=1) + for i in range(len(self.strides)) + ]) + + self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) + + def init_weights(self): + """Initialize weights of the head.""" + for cls_conv in self.cls_convs: + for m in cls_conv: + if isinstance(m, nn.Conv2d): + normal_init(m, std=0.01) + for reg_conv in self.reg_convs: + for m in reg_conv: + if isinstance(m, nn.Conv2d): + normal_init(m, std=0.01) + bias_cls = bias_init_with_prob(0.01) + for i in range(len(self.strides)): + normal_init(self.gfl_cls[i], std=0.01, bias=bias_cls) + normal_init(self.gfl_reg[i], std=0.01) + + def forward(self, xin, labels=None, imgs=None, aux_targets=None): + if self.training: + return self.forward_train(xin=xin, labels=labels, imgs=imgs) + else: + return self.forward_eval(xin=xin, labels=labels, imgs=imgs) + + def forward_train(self, xin, labels=None, imgs=None, aux_targets=None): + + # prepare labels during training + b, c, h, w = xin[0].shape + if labels is not None: + gt_bbox_list = [] + gt_cls_list = [] + for label in labels: + gt_bbox_list.append(label.bbox) + gt_cls_list.append((label.get_field('labels') + - 1).long()) # labels starts from 1 + + # prepare priors for label assignment and bbox decode + mlvl_priors_list = [ + self.get_single_level_center_priors( + xin[i].shape[0], + xin[i].shape[-2:], + stride, + dtype=torch.float32, + device=xin[0].device) for i, stride in enumerate(self.strides) + ] + mlvl_priors = torch.cat(mlvl_priors_list, dim=1) + + # forward for bboxes and classification prediction + cls_scores, bbox_preds, bbox_before_softmax = multi_apply( + self.forward_single, + xin, + self.cls_convs, + self.reg_convs, + self.gfl_cls, + self.gfl_reg, + self.scales, + ) + cls_scores = torch.cat(cls_scores, dim=1) + bbox_preds = torch.cat(bbox_preds, dim=1) + bbox_before_softmax = torch.cat(bbox_before_softmax, dim=1) + + # calculating losses + loss = self.loss( + cls_scores, + bbox_preds, + bbox_before_softmax, + gt_bbox_list, + gt_cls_list, + mlvl_priors, + ) + return loss + + def forward_eval(self, xin, labels=None, imgs=None): + + # prepare priors for label assignment and bbox decode + if self.feat_size[0] != xin[0].shape: + mlvl_priors_list = [ + self.get_single_level_center_priors( + xin[i].shape[0], + xin[i].shape[-2:], + stride, + dtype=torch.float32, + device=xin[0].device) + for i, stride in enumerate(self.strides) + ] + self.mlvl_priors = torch.cat(mlvl_priors_list, dim=1) + self.feat_size[0] = xin[0].shape + + # forward for bboxes and classification prediction + cls_scores, bbox_preds = multi_apply( + self.forward_single, + xin, + self.cls_convs, + self.reg_convs, + self.gfl_cls, + self.gfl_reg, + self.scales, + ) + cls_scores = torch.cat(cls_scores, dim=1)[:, :, :self.num_classes] + bbox_preds = torch.cat(bbox_preds, dim=1) + # batch bbox decode + bbox_preds = self.integral(bbox_preds) * self.mlvl_priors[..., 2, None] + bbox_preds = distance2bbox(self.mlvl_priors[..., :2], bbox_preds) + + if self.nms: + output = postprocess(cls_scores, bbox_preds, self.num_classes, + self.nms_conf_thre, self.nms_iou_thre, imgs) + return output + return cls_scores, bbox_preds + + def forward_single(self, x, cls_convs, reg_convs, gfl_cls, gfl_reg, scale): + """Forward feature of a single scale level. + + """ + cls_feat = x + reg_feat = x + + for cls_conv, reg_conv in zip(cls_convs, reg_convs): + cls_feat = cls_conv(cls_feat) + reg_feat = reg_conv(reg_feat) + + bbox_pred = scale(gfl_reg(reg_feat)).float() + N, C, H, W = bbox_pred.size() + if self.training: + bbox_before_softmax = bbox_pred.reshape(N, 4, self.reg_max + 1, H, + W) + bbox_before_softmax = bbox_before_softmax.flatten( + start_dim=3).permute(0, 3, 1, 2) + bbox_pred = F.softmax( + bbox_pred.reshape(N, 4, self.reg_max + 1, H, W), dim=2) + + cls_score = gfl_cls(cls_feat).sigmoid() + + cls_score = cls_score.flatten(start_dim=2).permute( + 0, 2, 1) # N, h*w, self.num_classes+1 + bbox_pred = bbox_pred.flatten(start_dim=3).permute( + 0, 3, 1, 2) # N, h*w, 4, self.reg_max+1 + if self.training: + return cls_score, bbox_pred, bbox_before_softmax + else: + return cls_score, bbox_pred + + def get_single_level_center_priors(self, batch_size, featmap_size, stride, + dtype, device): + + h, w = featmap_size + x_range = (torch.arange(0, int(w), dtype=dtype, + device=device)) * stride + y_range = (torch.arange(0, int(h), dtype=dtype, + device=device)) * stride + + x = x_range.repeat(h, 1) + y = y_range.unsqueeze(-1).repeat(1, w) + + y = y.flatten() + x = x.flatten() + strides = x.new_full((x.shape[0], ), stride) + priors = torch.stack([x, y, strides, strides], dim=-1) + + return priors.unsqueeze(0).repeat(batch_size, 1, 1) + + def loss( + self, + cls_scores, + bbox_preds, + bbox_before_softmax, + gt_bboxes, + gt_labels, + mlvl_center_priors, + gt_bboxes_ignore=None, + ): + """Compute losses of the head. + + """ + device = cls_scores[0].device + + # get decoded bboxes for label assignment + dis_preds = self.integral(bbox_preds) * mlvl_center_priors[..., 2, + None] + decoded_bboxes = distance2bbox(mlvl_center_priors[..., :2], dis_preds) + cls_reg_targets = self.get_targets( + cls_scores, + decoded_bboxes, + gt_bboxes, + mlvl_center_priors, + gt_labels_list=gt_labels) + + if cls_reg_targets is None: + return None + + (labels_list, label_scores_list, label_weights_list, bbox_targets_list, + bbox_weights_list, dfl_targets_list, num_pos) = cls_reg_targets + + num_total_pos = max( + reduce_mean(torch.tensor(num_pos).type( + torch.float).to(device)).item(), 1.0) + + labels = torch.cat(labels_list, dim=0) + label_scores = torch.cat(label_scores_list, dim=0) + bbox_targets = torch.cat(bbox_targets_list, dim=0) + dfl_targets = torch.cat(dfl_targets_list, dim=0) + + cls_scores = cls_scores.reshape(-1, self.cls_out_channels) + # bbox_preds = bbox_preds.reshape(-1, 4 * (self.reg_max + 1)) + bbox_before_softmax = bbox_before_softmax.reshape( + -1, 4 * (self.reg_max + 1)) + decoded_bboxes = decoded_bboxes.reshape(-1, 4) + + loss_qfl = self.loss_cls( + cls_scores, (labels, label_scores), avg_factor=num_total_pos) + + pos_inds = torch.nonzero( + (labels >= 0) & (labels < self.num_classes), + as_tuple=False).squeeze(1) + + weight_targets = cls_scores.detach() + weight_targets = weight_targets.max(dim=1)[0][pos_inds] + norm_factor = max(reduce_mean(weight_targets.sum()).item(), 1.0) + + if len(pos_inds) > 0: + loss_bbox = self.loss_bbox( + decoded_bboxes[pos_inds], + bbox_targets[pos_inds], + weight=weight_targets, + avg_factor=1.0 * norm_factor, + ) + loss_dfl = self.loss_dfl( + bbox_before_softmax[pos_inds].reshape(-1, self.reg_max + 1), + dfl_targets[pos_inds].reshape(-1), + weight=weight_targets[:, None].expand(-1, 4).reshape(-1), + avg_factor=4.0 * norm_factor, + ) + else: + loss_bbox = bbox_preds.sum() / norm_factor * 0.0 + loss_dfl = bbox_preds.sum() / norm_factor * 0.0 + + total_loss = loss_qfl + loss_bbox + loss_dfl + + return dict( + total_loss=total_loss, + loss_cls=loss_qfl, + loss_bbox=loss_bbox, + loss_dfl=loss_dfl, + ) + + def get_targets(self, + cls_scores, + bbox_preds, + gt_bboxes_list, + mlvl_center_priors, + gt_labels_list=None, + unmap_outputs=True): + """Get targets for GFL head. + + """ + num_imgs = mlvl_center_priors.shape[0] + + if gt_labels_list is None: + gt_labels_list = [None for _ in range(num_imgs)] + + (all_labels, all_label_scores, all_label_weights, all_bbox_targets, + all_bbox_weights, all_dfl_targets, + all_pos_num) = multi_apply(self.get_target_single, mlvl_center_priors, + cls_scores, bbox_preds, gt_bboxes_list, + gt_labels_list) + # no valid anchors + if any([labels is None for labels in all_labels]): + return None + # sampled anchors of all images + all_pos_num = sum(all_pos_num) + + return (all_labels, all_label_scores, all_label_weights, + all_bbox_targets, all_bbox_weights, all_dfl_targets, + all_pos_num) + + def get_target_single(self, + center_priors, + cls_scores, + bbox_preds, + gt_bboxes, + gt_labels, + unmap_outputs=True, + gt_bboxes_ignore=None): + """Compute regression, classification targets for anchors in a single + image. + + """ + # assign gt and sample anchors + + num_valid_center = center_priors.shape[0] + + labels = center_priors.new_full((num_valid_center, ), + self.num_classes, + dtype=torch.long) + label_weights = center_priors.new_zeros( + num_valid_center, dtype=torch.float) + label_scores = center_priors.new_zeros( + num_valid_center, dtype=torch.float) + + bbox_targets = torch.zeros_like(center_priors) + bbox_weights = torch.zeros_like(center_priors) + dfl_targets = torch.zeros_like(center_priors) + + if gt_labels.size(0) == 0: + + return (labels, label_scores, label_weights, bbox_targets, + bbox_weights, dfl_targets, 0) + + assign_result = self.assigner.assign(cls_scores.detach(), + center_priors, + bbox_preds.detach(), gt_bboxes, + gt_labels) + + pos_inds, neg_inds, pos_bbox_targets, pos_assign_gt_inds = self.sample( + assign_result, gt_bboxes) + pos_ious = assign_result.max_overlaps[pos_inds] + + if len(pos_inds) > 0: + labels[pos_inds] = gt_labels[pos_assign_gt_inds] + label_scores[pos_inds] = pos_ious + label_weights[pos_inds] = 1.0 + + bbox_targets[pos_inds, :] = pos_bbox_targets + bbox_weights[pos_inds, :] = 1.0 + dfl_targets[pos_inds, :] = ( + bbox2distance( + center_priors[pos_inds, :2] + / center_priors[pos_inds, None, 2], + pos_bbox_targets / center_priors[pos_inds, None, 2], + self.reg_max)) + if len(neg_inds) > 0: + label_weights[neg_inds] = 1.0 + # map up to original set of anchors + + return (labels, label_scores, label_weights, bbox_targets, + bbox_weights, dfl_targets, pos_inds.size(0)) + + def sample(self, assign_result, gt_bboxes): + pos_inds = torch.nonzero( + assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() + neg_inds = torch.nonzero( + assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() + pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 + + if gt_bboxes.numel() == 0: + # hack for index error case + assert pos_assigned_gt_inds.numel() == 0 + pos_gt_bboxes = torch.empty_like(gt_bboxes).view(-1, 4) + else: + if len(gt_bboxes.shape) < 2: + gt_bboxes = gt_bboxes.view(-1, 4) + pos_gt_bboxes = gt_bboxes[pos_assigned_gt_inds, :] + + return pos_inds, neg_inds, pos_gt_bboxes, pos_assigned_gt_inds diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/distill_loss.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/distill_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..fda2fc471b9f7275652259a65766ff9b14da960b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/distill_loss.py @@ -0,0 +1,173 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class FeatureLoss(nn.Module): + + def __init__(self, + channels_s, + channels_t, + distiller='cwd', + loss_weight=1.0): + super(FeatureLoss, self).__init__() + self.loss_weight = loss_weight + + device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.align_module = nn.ModuleList([ + nn.Conv2d( + channel, tea_channel, kernel_size=1, stride=1, + padding=0).to(device) + for channel, tea_channel in zip(channels_s, channels_t) + ]) + self.norm = [ + nn.BatchNorm2d(tea_channel, affine=False).to(device) + for tea_channel in channels_t + ] + + if (distiller == 'mimic'): + self.feature_loss = MimicLoss(channels_s, channels_t) + elif (distiller == 'mgd'): + self.feature_loss = MGDLoss(channels_s, channels_t) + elif (distiller == 'cwd'): + self.feature_loss = CWDLoss(channels_s, channels_t) + else: + raise NotImplementedError + + def forward(self, y_s, y_t): + assert len(y_s) == len(y_t) + tea_feats = [] + stu_feats = [] + + for idx, (s, t) in enumerate(zip(y_s, y_t)): + s = self.align_module[idx](s) + s = self.norm[idx](s) + t = self.norm[idx](t) + tea_feats.append(t) + stu_feats.append(s) + + loss = self.feature_loss(stu_feats, tea_feats) + return self.loss_weight * loss + + +class MimicLoss(nn.Module): + + def __init__(self, channels_s, channels_t): + super(MimicLoss, self).__init__() + self.mse = nn.MSELoss() + + def forward(self, y_s, y_t): + """Forward computation. + Args: + y_s (list): The student model prediction with + shape (N, C, H, W) in list. + y_t (list): The teacher model prediction with + shape (N, C, H, W) in list. + Return: + torch.Tensor: The calculated loss value of all stages. + """ + assert len(y_s) == len(y_t) + losses = [] + for idx, (s, t) in enumerate(zip(y_s, y_t)): + assert s.shape == t.shape + losses.append(self.mse(s, t)) + loss = sum(losses) + return loss + + +class MGDLoss(nn.Module): + + def __init__(self, + channels_s, + channels_t, + alpha_mgd=0.00002, + lambda_mgd=0.65): + super(MGDLoss, self).__init__() + device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.alpha_mgd = alpha_mgd + self.lambda_mgd = lambda_mgd + + self.generation = [ + nn.Sequential( + nn.Conv2d(channel, channel, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(channel, channel, kernel_size=3, + padding=1)).to(device) for channel in channels_t + ] + + def forward(self, y_s, y_t): + """Forward computation. + Args: + y_s (list): The student model prediction with + shape (N, C, H, W) in list. + y_t (list): The teacher model prediction with + shape (N, C, H, W) in list. + Return: + torch.Tensor: The calculated loss value of all stages. + """ + assert len(y_s) == len(y_t) + losses = [] + for idx, (s, t) in enumerate(zip(y_s, y_t)): + assert s.shape == t.shape + losses.append(self.get_dis_loss(s, t, idx) * self.alpha_mgd) + loss = sum(losses) + return loss + + def get_dis_loss(self, preds_S, preds_T, idx): + loss_mse = nn.MSELoss(reduction='sum') + N, C, H, W = preds_T.shape + + device = preds_S.device + mat = torch.rand((N, 1, H, W)).to(device) + mat = torch.where(mat > 1 - self.lambda_mgd, 0, 1).to(device) + + masked_fea = torch.mul(preds_S, mat) + new_fea = self.generation[idx](masked_fea) + + dis_loss = loss_mse(new_fea, preds_T) / N + + return dis_loss + + +class CWDLoss(nn.Module): + """PyTorch version of `Channel-wise Distillation for Semantic Segmentation. + `_. + """ + + def __init__(self, channels_s, channels_t, tau=1.0): + super(CWDLoss, self).__init__() + self.tau = tau + + def forward(self, y_s, y_t): + """Forward computation. + Args: + y_s (list): The student model prediction with + shape (N, C, H, W) in list. + y_t (list): The teacher model prediction with + shape (N, C, H, W) in list. + Return: + torch.Tensor: The calculated loss value of all stages. + """ + assert len(y_s) == len(y_t) + losses = [] + + for idx, (s, t) in enumerate(zip(y_s, y_t)): + assert s.shape == t.shape + N, C, H, W = s.shape + # normalize in channel diemension + softmax_pred_T = F.softmax( + t.view(-1, W * H) / self.tau, dim=1) # [N*C, H*W] + + logsoftmax = torch.nn.LogSoftmax(dim=1) + cost = torch.sum(softmax_pred_T + * logsoftmax(t.view(-1, W * H) / self.tau) + - softmax_pred_T + * logsoftmax(s.view(-1, W * H) / self.tau)) * ( + self.tau**2) + + losses.append(cost / (C * N)) + loss = sum(losses) + + return loss diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/gfocal_loss.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/gfocal_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..490d41cee4b469a780ad095b27bc31d8413db3f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/losses/gfocal_loss.py @@ -0,0 +1,328 @@ +# This file mainly comes from +# https://github.com/implus/GFocalV2/blob/master/mmdet/models/losses/gfocal_loss.py + +import functools + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.tinynas_detection.damo.utils.boxes import \ + bbox_overlaps + + +def reduce_loss(loss, reduction): + """Reduce loss as specified. + Args: + loss (Tensor): Elementwise loss tensor. + reduction (str): Options are "none", "mean" and "sum". + Return: + Tensor: Reduced loss tensor. + """ + reduction_enum = F._Reduction.get_enum(reduction) + # none: 0, elementwise_mean:1, sum: 2 + if reduction_enum == 0: + return loss + elif reduction_enum == 1: + return loss.mean() + elif reduction_enum == 2: + return loss.sum() + + +def weighted_loss(loss_func): + """Create a weighted version of a given loss function. + + To use this decorator, the loss function must have the signature like + `loss_func(pred, target, **kwargs)`. The function only needs to compute + element-wise loss without any reduction. This decorator will add weight + and reduction arguments to the function. The decorated function will have + the signature like `loss_func(pred, target, weight=None, reduction='mean', + avg_factor=None, **kwargs)`. + + Example: + + >>> import torch + >>> @weighted_loss + >>> def l1_loss(pred, target): + >>> return (pred - target).abs() + + >>> pred = torch.Tensor([0, 2, 3]) + >>> target = torch.Tensor([1, 1, 1]) + >>> weight = torch.Tensor([1, 0, 1]) + + >>> l1_loss(pred, target) + tensor(1.3333) + >>> l1_loss(pred, target, weight) + tensor(1.) + >>> l1_loss(pred, target, reduction='none') + tensor([1., 1., 2.]) + >>> l1_loss(pred, target, weight, avg_factor=2) + tensor(1.5000) + """ + + @functools.wraps(loss_func) + def wrapper(pred, + target, + weight=None, + reduction='mean', + avg_factor=None, + **kwargs): + # get element-wise loss + loss = loss_func(pred, target, **kwargs) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + return wrapper + + +def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): + """Apply element-wise weight and reduce loss. + Args: + loss (Tensor): Element-wise loss. + weight (Tensor): Element-wise weights. + reduction (str): Same as built-in losses of PyTorch. + avg_factor (float): Avarage factor when computing the mean of losses. + Returns: + Tensor: Processed loss values. + """ + # if weight is specified, apply element-wise weight + if weight is not None: + loss = loss * weight + + # if avg_factor is not specified, just reduce the loss + if avg_factor is None: + loss = reduce_loss(loss, reduction) + else: + # if reduction is mean, then average the loss by avg_factor + if reduction == 'mean': + loss = loss.sum() / avg_factor + # if reduction is 'none', then do nothing, otherwise raise an error + elif reduction != 'none': + raise ValueError('avg_factor can not be used with reduction="sum"') + return loss + + +@weighted_loss +def giou_loss(pred, target, eps=1e-7): + r"""`Generalized Intersection over Union: A Metric and A Loss for Bounding + Box Regression `_. + Args: + pred (torch.Tensor): Predicted bboxes of format (x1, y1, x2, y2), + shape (n, 4). + target (torch.Tensor): Corresponding gt bboxes, shape (n, 4). + eps (float): Eps to avoid log(0). + Return: + Tensor: Loss tensor. + """ + gious = bbox_overlaps(pred, target, mode='giou', is_aligned=True, eps=eps) + loss = 1 - gious + return loss + + +class GIoULoss(nn.Module): + + def __init__(self, eps=1e-6, reduction='mean', loss_weight=1.0): + super(GIoULoss, self).__init__() + self.eps = eps + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + if weight is not None and not torch.any(weight > 0): + if pred.dim() == weight.dim() + 1: + weight = weight.unsqueeze(1) + return (pred * weight).sum() # 0 + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if weight is not None and weight.dim() > 1: + # TODO: remove this in the future + # reduce the weight of shape (n, 4) to (n,) to match the + # giou_loss of shape (n,) + assert weight.shape == pred.shape + weight = weight.mean(-1) + loss = self.loss_weight * giou_loss( + pred, + target, + weight, + eps=self.eps, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss + + +@weighted_loss +def distribution_focal_loss(pred, label): + r"""Distribution Focal Loss (DFL) is from `Generalized Focal Loss: Learning + Qualified and Distributed Bounding Boxes for Dense Object Detection + `_. + Args: + pred (torch.Tensor): Predicted general distribution of bounding boxes + (before softmax) with shape (N, n+1), n is the max value of the + integral set `{0, ..., n}` in paper. + label (torch.Tensor): Target distance label for bounding boxes with + shape (N,). + Returns: + torch.Tensor: Loss tensor with shape (N,). + """ + dis_left = label.long() + dis_right = dis_left + 1 + weight_left = dis_right.float() - label + weight_right = label - dis_left.float() + loss = F.cross_entropy(pred, dis_left, reduction='none') * weight_left \ + + F.cross_entropy(pred, dis_right, reduction='none') * weight_right + return loss + + +class DistributionFocalLoss(nn.Module): + r"""Distribution Focal Loss (DFL) is a variant of `Generalized Focal Loss: + Learning Qualified and Distributed Bounding Boxes for Dense Object + Detection `_. + Args: + reduction (str): Options are `'none'`, `'mean'` and `'sum'`. + loss_weight (float): Loss weight of current loss. + """ + + def __init__(self, reduction='mean', loss_weight=1.0): + super(DistributionFocalLoss, self).__init__() + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + Args: + pred (torch.Tensor): Predicted general distribution of bounding + boxes (before softmax) with shape (N, n+1), n is the max value + of the integral set `{0, ..., n}` in paper. + target (torch.Tensor): Target distance label for bounding boxes + with shape (N,). + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_cls = self.loss_weight * distribution_focal_loss( + pred, target, weight, reduction=reduction, avg_factor=avg_factor) + return loss_cls + + +@weighted_loss +def quality_focal_loss(pred, target, beta=2.0, use_sigmoid=True): + r"""Quality Focal Loss (QFL) is from `Generalized Focal Loss: Learning + Qualified and Distributed Bounding Boxes for Dense Object Detection + `_. + Args: + pred (torch.Tensor): Predicted joint representation of classification + and quality (IoU) estimation with shape (N, C), C is the number of + classes. + target (tuple([torch.Tensor])): Target category label with shape (N,) + and target quality label with shape (N,). + beta (float): The beta parameter for calculating the modulating factor. + Defaults to 2.0. + Returns: + torch.Tensor: Loss tensor with shape (N,). + """ + assert len(target) == 2, """target for QFL must be a tuple of two elements, + including category label and quality label, respectively""" + # label denotes the category id, score denotes the quality score + label, score = target + if use_sigmoid: + func = F.binary_cross_entropy_with_logits + else: + func = F.binary_cross_entropy + # negatives are supervised by 0 quality score + pred_sigmoid = pred.sigmoid() if use_sigmoid else pred + scale_factor = pred_sigmoid # 8400, 81 + zerolabel = scale_factor.new_zeros(pred.shape) + loss = func(pred, zerolabel, reduction='none') * scale_factor.pow(beta) + + bg_class_ind = pred.size(1) + pos = ((label >= 0) + & (label < bg_class_ind)).nonzero(as_tuple=False).squeeze(1) + pos_label = label[pos].long() + # positives are supervised by bbox quality (IoU) score + scale_factor = score[pos] - pred_sigmoid[pos, pos_label] + loss[pos, pos_label] = func( + pred[pos, pos_label], score[pos], + reduction='none') * scale_factor.abs().pow(beta) + + loss = loss.sum(dim=1, keepdim=False) + return loss + + +class QualityFocalLoss(nn.Module): + r"""Quality Focal Loss (QFL) is a variant of `Generalized Focal Loss: + Learning Qualified and Distributed Bounding Boxes for Dense Object + Detection `_. + Args: + use_sigmoid (bool): Whether sigmoid operation is conducted in QFL. + Defaults to True. + beta (float): The beta parameter for calculating the modulating factor. + Defaults to 2.0. + reduction (str): Options are "none", "mean" and "sum". + loss_weight (float): Loss weight of current loss. + """ + + def __init__(self, + use_sigmoid=True, + beta=2.0, + reduction='mean', + loss_weight=1.0): + super(QualityFocalLoss, self).__init__() + # assert use_sigmoid is True, 'Only sigmoid in QFL supported now.' + self.use_sigmoid = use_sigmoid + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + Args: + pred (torch.Tensor): Predicted joint representation of + classification and quality (IoU) estimation with shape (N, C), + C is the number of classes. + target (tuple([torch.Tensor])): Target category label with shape + (N,) and target quality label with shape (N,). + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_cls = self.loss_weight * quality_focal_loss( + pred, + target, + weight, + beta=self.beta, + use_sigmoid=self.use_sigmoid, + reduction=reduction, + avg_factor=avg_factor) + return loss_cls diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1b17fd706addbb2f423947084346ce560a4c2fd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy + +from .giraffe_fpn import GiraffeNeck +from .giraffe_fpn_btn import GiraffeNeckV2 + + +def build_neck(cfg): + neck_cfg = copy.deepcopy(cfg) + name = neck_cfg.pop('name') + if name == 'GiraffeNeckV2': + return GiraffeNeckV2(**neck_cfg) + elif name == 'GiraffeNeck': + return GiraffeNeck(**neck_cfg) + else: + raise NotImplementedError diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_config.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_config.py new file mode 100644 index 0000000000000000000000000000000000000000..1e610b16338713c3681f9dc998a3ca8623492eee --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_config.py @@ -0,0 +1,231 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import collections + +import networkx as nx + +Node = collections.namedtuple('Node', ['id', 'inputs', 'type']) + + +def get_graph_info(graph): + input_nodes = [] + output_nodes = [] + Nodes = [] + for node in range(graph.number_of_nodes()): + tmp = list(graph.neighbors(node)) + tmp.sort() + type = -1 + if node < tmp[0]: + input_nodes.append(node) + type = 0 + if node > tmp[-1]: + output_nodes.append(node) + type = 1 + Nodes.append(Node(node, [n for n in tmp if n < node], type)) + return Nodes, input_nodes, output_nodes + + +def nodeid_trans(id, cur_level, num_levels): + if id % 2 == 1: + gap = int(((id + 1) // 2) * num_levels * 2) + else: + a = (num_levels - cur_level) * 2 - 1 + b = ((id + 1) // 2) * num_levels * 2 + gap = int(a + b) + return cur_level + gap + + +def gen_log2n_graph_file(log2n_graph_file, depth_multiplier): + f = open(log2n_graph_file, 'w') + for i in range(depth_multiplier): + for j in [1, 2, 4, 8, 16, 32]: + if i - j < 0: + break + else: + f.write('%d,%d\n' % (i - j, i)) + f.close() + + +def get_log2n_graph(depth_multiplier): + nodes = [] + connnections = [] + + for i in range(depth_multiplier): + nodes.append(i) + for j in [1, 2, 4, 8, 16, 32]: + if i - j < 0: + break + else: + connnections.append((i - j, i)) + return nodes, connnections + + +def get_dense_graph(depth_multiplier): + nodes = [] + connections = [] + + for i in range(depth_multiplier): + nodes.append(i) + for j in range(i): + connections.append((j, i)) + return nodes, connections + + +def giraffeneck_config(min_level, + max_level, + weight_method=None, + depth_multiplier=5, + with_backslash=False, + with_slash=False, + with_skip_connect=False, + skip_connect_type='dense'): + """Graph config with log2n merge and panet""" + if skip_connect_type == 'dense': + nodes, connections = get_dense_graph(depth_multiplier) + elif skip_connect_type == 'log2n': + nodes, connections = get_log2n_graph(depth_multiplier) + graph = nx.Graph() + graph.add_nodes_from(nodes) + graph.add_edges_from(connections) + + drop_node = [] + nodes, input_nodes, output_nodes = get_graph_info(graph) + + weight_method = weight_method or 'fastattn' + + num_levels = max_level - min_level + 1 + node_ids = {min_level + i: [i] for i in range(num_levels)} + node_ids_per_layer = {} + + pnodes = {} + + def update_drop_node(new_id, input_offsets): + if new_id not in drop_node: + new_id = new_id + else: + while new_id in drop_node: + if new_id in pnodes: + for n in pnodes[new_id]['inputs_offsets']: + if n not in input_offsets and n not in drop_node: + input_offsets.append(n) + new_id = new_id - 1 + if new_id not in input_offsets: + input_offsets.append(new_id) + + # top-down layer + for i in range(max_level, min_level - 1, -1): + node_ids_per_layer[i] = [] + for id, node in enumerate(nodes): + input_offsets = [] + if id in input_nodes: + input_offsets.append(node_ids[i][0]) + else: + if with_skip_connect: + for input_id in node.inputs: + new_id = nodeid_trans(input_id, i - min_level, + num_levels) + update_drop_node(new_id, input_offsets) + + # add top2down + new_id = nodeid_trans(id, i - min_level, num_levels) + + # add backslash node + def cal_backslash_node(id): + ind = id // num_levels + mod = id % num_levels + if ind % 2 == 0: # even + if mod == (num_levels - 1): + last = -1 + else: + last = (ind - 1) * num_levels + ( + num_levels - 1 - mod - 1) + else: # odd + if mod == 0: + last = -1 + else: + last = (ind - 1) * num_levels + ( + num_levels - 1 - mod + 1) + + return last + + # add slash node + def cal_slash_node(id): + ind = id // num_levels + mod = id % num_levels + if ind % 2 == 1: # odd + if mod == (num_levels - 1): + last = -1 + else: + last = (ind - 1) * num_levels + ( + num_levels - 1 - mod - 1) + else: # even + if mod == 0: + last = -1 + else: + last = (ind - 1) * num_levels + ( + num_levels - 1 - mod + 1) + + return last + + # add last node + last = new_id - 1 + update_drop_node(last, input_offsets) + + if with_backslash: + backslash = cal_backslash_node(new_id) + if backslash != -1 and backslash not in input_offsets: + input_offsets.append(backslash) + + if with_slash: + slash = cal_slash_node(new_id) + if slash != -1 and slash not in input_offsets: + input_offsets.append(slash) + + if new_id in drop_node: + input_offsets = [] + + pnodes[new_id] = { + 'reduction': 1 << i, + 'inputs_offsets': input_offsets, + 'weight_method': weight_method, + 'is_out': 0, + } + + input_offsets = [] + for out_id in output_nodes: + new_id = nodeid_trans(out_id, i - min_level, num_levels) + input_offsets.append(new_id) + + pnodes[node_ids[i][0] + num_levels * (len(nodes) + 1)] = { + 'reduction': 1 << i, + 'inputs_offsets': input_offsets, + 'weight_method': weight_method, + 'is_out': 1, + } + + pnodes = dict(sorted(pnodes.items(), key=lambda x: x[0])) + return pnodes + + +def get_graph_config(fpn_name, + min_level=3, + max_level=7, + weight_method='concat', + depth_multiplier=5, + with_backslash=False, + with_slash=False, + with_skip_connect=False, + skip_connect_type='dense'): + name_to_config = { + 'giraffeneck': + giraffeneck_config( + min_level=min_level, + max_level=max_level, + weight_method=weight_method, + depth_multiplier=depth_multiplier, + with_backslash=with_backslash, + with_slash=with_slash, + with_skip_connect=with_skip_connect, + skip_connect_type=skip_connect_type), + } + return name_to_config[fpn_name] diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..987358e8aa0aa250e9c513c23c4d25fe8f501127 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn.py @@ -0,0 +1,659 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +import math +from collections import OrderedDict +from functools import partial +from typing import Callable, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm import create_model +from timm.models.layers import (Swish, create_conv2d, create_pool2d, + get_act_layer) + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.base_ops import ( + CSPLayer, ShuffleBlock, ShuffleCSPLayer) +from .giraffe_config import get_graph_config + +_ACT_LAYER = Swish + + +class SequentialList(nn.Sequential): + """ This module exists to work around torchscript typing issues list -> list""" + + def __init__(self, *args): + super(SequentialList, self).__init__(*args) + + def forward(self, x: List[torch.Tensor]) -> List[torch.Tensor]: + for module in self: + x = module(x) + return x + + +class ConvBnAct2d(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=1, + padding='', + bias=False, + norm_layer=nn.BatchNorm2d, + act_layer=_ACT_LAYER): + super(ConvBnAct2d, self).__init__() + + self.conv = create_conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=bias) + self.bn = None if norm_layer is None else norm_layer(out_channels) + self.act = None if act_layer is None else act_layer(inplace=True) + + def forward(self, x): + x = self.conv(x) + if self.bn is not None: + x = self.bn(x) + if self.act is not None: + x = self.act(x) + return x + + +class SeparableConv2d(nn.Module): + """ Separable Conv + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + dilation=1, + padding='', + bias=False, + channel_multiplier=1.0, + pw_kernel_size=1, + norm_layer=nn.BatchNorm2d, + act_layer=_ACT_LAYER): + super(SeparableConv2d, self).__init__() + self.conv_dw = create_conv2d( + in_channels, + int(in_channels * channel_multiplier), + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + depthwise=True) + + self.conv_pw = create_conv2d( + int(in_channels * channel_multiplier), + out_channels, + pw_kernel_size, + padding=padding, + bias=bias) + + self.bn = None if norm_layer is None else norm_layer(out_channels) + self.act = None if act_layer is None else act_layer(inplace=True) + + def forward(self, x): + x = self.conv_dw(x) + x = self.conv_pw(x) + if self.bn is not None: + x = self.bn(x) + if self.act is not None: + x = self.act(x) + return x + + +def _init_weight( + m, + n='', +): + """ Weight initialization as per Tensorflow official implementations. + """ + + def _fan_in_out(w, groups=1): + dimensions = w.dim() + if dimensions < 2: + raise ValueError( + 'Fan in and fan out can not be computed for tensor with fewer than 2 dimensions' + ) + num_input_fmaps = w.size(1) + num_output_fmaps = w.size(0) + receptive_field_size = 1 + if w.dim() > 2: + receptive_field_size = w[0][0].numel() + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + fan_out //= groups + return fan_in, fan_out + + def _glorot_uniform(w, gain=1, groups=1): + fan_in, fan_out = _fan_in_out(w, groups) + gain /= max(1., (fan_in + fan_out) / 2.) # fan avg + limit = math.sqrt(3.0 * gain) + w.data.uniform_(-limit, limit) + + def _variance_scaling(w, gain=1, groups=1): + fan_in, fan_out = _fan_in_out(w, groups) + gain /= max(1., fan_in) # fan in + std = math.sqrt(gain) + w.data.normal_(std=std) + + if isinstance(m, SeparableConv2d): + if 'box_net' in n or 'class_net' in n: + _variance_scaling(m.conv_dw.weight, groups=m.conv_dw.groups) + _variance_scaling(m.conv_pw.weight) + if m.conv_pw.bias is not None: + if 'class_net.predict' in n: + m.conv_pw.bias.data.fill_(-math.log((1 - 0.01) / 0.01)) + else: + m.conv_pw.bias.data.zero_() + else: + _glorot_uniform(m.conv_dw.weight, groups=m.conv_dw.groups) + _glorot_uniform(m.conv_pw.weight) + if m.conv_pw.bias is not None: + m.conv_pw.bias.data.zero_() + elif isinstance(m, ConvBnAct2d): + if 'box_net' in n or 'class_net' in n: + m.conv.weight.data.normal_(std=.01) + if m.conv.bias is not None: + if 'class_net.predict' in n: + m.conv.bias.data.fill_(-math.log((1 - 0.01) / 0.01)) + else: + m.conv.bias.data.zero_() + else: + _glorot_uniform(m.conv.weight) + if m.conv.bias is not None: + m.conv.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + + +def _init_weight_alt( + m, + n='', +): + """ Weight initialization alternative, based on EfficientNet bacbkone init w/ class bias addition + NOTE: this will likely be removed after some experimentation + """ + if isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + if 'class_net.predict' in n: + m.bias.data.fill_(-math.log((1 - 0.01) / 0.01)) + else: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + + +class Interpolate2d(nn.Module): + r"""Resamples a 2d Image + + The input data is assumed to be of the form + `minibatch x channels x [optional depth] x [optional height] x width`. + Hence, for spatial inputs, we expect a 4D Tensor and for volumetric inputs, we expect a 5D Tensor. + + The algorithms available for upsampling are nearest neighbor and linear, + bilinear, bicubic and trilinear for 3D, 4D and 5D input Tensor, + respectively. + + One can either give a :attr:`scale_factor` or the target output :attr:`size` to + calculate the output size. (You cannot give both, as it is ambiguous) + + Args: + size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int], optional): + output spatial sizes + scale_factor (float or Tuple[float] or Tuple[float, float] or Tuple[float, float, float], optional): + multiplier for spatial size. Has to match input size if it is a tuple. + mode (str, optional): the upsampling algorithm: one of ``'nearest'``, + ``'linear'``, ``'bilinear'``, ``'bicubic'`` and ``'trilinear'``. + Default: ``'nearest'`` + align_corners (bool, optional): if ``True``, the corner pixels of the input + and output tensors are aligned, and thus preserving the values at + those pixels. This only has effect when :attr:`mode` is + ``'linear'``, ``'bilinear'``, or ``'trilinear'``. Default: ``False`` + """ + __constants__ = ['size', 'scale_factor', 'mode', 'align_corners', 'name'] + name: str + size: Optional[Union[int, Tuple[int, int]]] + scale_factor: Optional[Union[float, Tuple[float, float]]] + mode: str + align_corners: Optional[bool] + + def __init__(self, + size: Optional[Union[int, Tuple[int, int]]] = None, + scale_factor: Optional[Union[float, Tuple[float, + float]]] = None, + mode: str = 'nearest', + align_corners: bool = False) -> None: + super(Interpolate2d, self).__init__() + self.name = type(self).__name__ + self.size = size + if isinstance(scale_factor, tuple): + self.scale_factor = tuple(float(factor) for factor in scale_factor) + else: + self.scale_factor = float(scale_factor) if scale_factor else None + self.mode = mode + self.align_corners = None if mode == 'nearest' else align_corners + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return F.interpolate( + input, + self.size, + self.scale_factor, + self.mode, + self.align_corners, + recompute_scale_factor=False) + + +class ResampleFeatureMap(nn.Sequential): + + def __init__(self, + in_channels, + out_channels, + reduction_ratio=1., + pad_type='', + downsample=None, + upsample=None, + norm_layer=nn.BatchNorm2d, + apply_bn=False, + conv_after_downsample=False, + redundant_bias=False): + super(ResampleFeatureMap, self).__init__() + downsample = downsample or 'max' + upsample = upsample or 'nearest' + self.in_channels = in_channels + self.out_channels = out_channels + self.reduction_ratio = reduction_ratio + self.conv_after_downsample = conv_after_downsample + + conv = None + if in_channels != out_channels: + conv = ConvBnAct2d( + in_channels, + out_channels, + kernel_size=1, + padding=pad_type, + norm_layer=norm_layer if apply_bn else None, + bias=not apply_bn or redundant_bias, + act_layer=None) + + if reduction_ratio > 1: + if conv is not None and not self.conv_after_downsample: + self.add_module('conv', conv) + if downsample in ('max', 'avg'): + stride_size = int(reduction_ratio) + downsample = create_pool2d( + downsample, + kernel_size=stride_size + 1, + stride=stride_size, + padding=pad_type) + else: + downsample = Interpolate2d( + scale_factor=1. / reduction_ratio, mode=downsample) + self.add_module('downsample', downsample) + if conv is not None and self.conv_after_downsample: + self.add_module('conv', conv) + else: + if conv is not None: + self.add_module('conv', conv) + if reduction_ratio < 1: + scale = int(1 // reduction_ratio) + self.add_module( + 'upsample', + Interpolate2d(scale_factor=scale, mode=upsample)) + + +class GiraffeCombine(nn.Module): + + def __init__(self, + feature_info, + fpn_config, + fpn_channels, + inputs_offsets, + target_reduction, + pad_type='', + downsample=None, + upsample=None, + norm_layer=nn.BatchNorm2d, + apply_resample_bn=False, + conv_after_downsample=False, + redundant_bias=False, + weight_method='attn'): + super(GiraffeCombine, self).__init__() + self.inputs_offsets = inputs_offsets + self.weight_method = weight_method + + self.resample = nn.ModuleDict() + reduction_base = feature_info[0]['reduction'] + + target_channels_idx = int( + math.log(target_reduction // reduction_base, 2)) + for idx, offset in enumerate(inputs_offsets): + if offset < len(feature_info): + in_channels = feature_info[offset]['num_chs'] + input_reduction = feature_info[offset]['reduction'] + else: + node_idx = offset + input_reduction = fpn_config[node_idx]['reduction'] + # in_channels = fpn_config[node_idx]['num_chs'] + input_channels_idx = int( + math.log(input_reduction // reduction_base, 2)) + in_channels = feature_info[input_channels_idx]['num_chs'] + + reduction_ratio = target_reduction / input_reduction + if weight_method == 'concat': + self.resample[str(offset)] = ResampleFeatureMap( + in_channels, + in_channels, + reduction_ratio=reduction_ratio, + pad_type=pad_type, + downsample=downsample, + upsample=upsample, + norm_layer=norm_layer, + apply_bn=apply_resample_bn, + conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias) + else: + self.resample[str(offset)] = ResampleFeatureMap( + in_channels, + fpn_channels[target_channels_idx], + reduction_ratio=reduction_ratio, + pad_type=pad_type, + downsample=downsample, + upsample=upsample, + norm_layer=norm_layer, + apply_bn=apply_resample_bn, + conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias) + + if weight_method == 'attn' or weight_method == 'fastattn': + self.edge_weights = nn.Parameter( + torch.ones(len(inputs_offsets)), requires_grad=True) # WSM + else: + self.edge_weights = None + + def forward(self, x: List[torch.Tensor]): + dtype = x[0].dtype + nodes = [] + if len(self.inputs_offsets) == 0: + return None + for offset, resample in zip(self.inputs_offsets, + self.resample.values()): + input_node = x[offset] + input_node = resample(input_node) + nodes.append(input_node) + + if self.weight_method == 'attn': + normalized_weights = torch.softmax( + self.edge_weights.to(dtype=dtype), dim=0) + out = torch.stack(nodes, dim=-1) * normalized_weights + out = torch.sum(out, dim=-1) + elif self.weight_method == 'fastattn': + edge_weights = nn.functional.relu( + self.edge_weights.to(dtype=dtype)) + weights_sum = torch.sum(edge_weights) + weights_norm = weights_sum + 0.0001 + out = torch.stack([(nodes[i] * edge_weights[i]) / weights_norm + for i in range(len(nodes))], + dim=-1) + + out = torch.sum(out, dim=-1) + elif self.weight_method == 'sum': + out = torch.stack(nodes, dim=-1) + out = torch.sum(out, dim=-1) + elif self.weight_method == 'concat': + out = torch.cat(nodes, dim=1) + else: + raise ValueError('unknown weight_method {}'.format( + self.weight_method)) + return out + + +class GiraffeNode(nn.Module): + """ A simple wrapper used in place of nn.Sequential for torchscript typing + Handles input type List[Tensor] -> output type Tensor + """ + + def __init__(self, combine: nn.Module, after_combine: nn.Module): + super(GiraffeNode, self).__init__() + self.combine = combine + self.after_combine = after_combine + + def forward(self, x: List[torch.Tensor]) -> torch.Tensor: + combine_feat = self.combine(x) + if combine_feat is None: + return None + else: + return self.after_combine(combine_feat) + + +class GiraffeLayer(nn.Module): + + def __init__(self, + feature_info, + fpn_config, + inner_fpn_channels, + outer_fpn_channels, + num_levels=5, + pad_type='', + downsample=None, + upsample=None, + norm_layer=nn.BatchNorm2d, + act_layer=_ACT_LAYER, + apply_resample_bn=False, + conv_after_downsample=True, + conv_bn_relu_pattern=False, + separable_conv=True, + redundant_bias=False, + merge_type='conv'): + super(GiraffeLayer, self).__init__() + self.num_levels = num_levels + self.conv_bn_relu_pattern = False + + self.feature_info = {} + for idx, feat in enumerate(feature_info): + self.feature_info[idx] = feat + + self.fnode = nn.ModuleList() + reduction_base = feature_info[0]['reduction'] + for i, fnode_cfg in fpn_config.items(): + + if fnode_cfg['is_out'] == 1: + fpn_channels = outer_fpn_channels + else: + fpn_channels = inner_fpn_channels + + reduction = fnode_cfg['reduction'] + fpn_channels_idx = int(math.log(reduction // reduction_base, 2)) + combine = GiraffeCombine( + self.feature_info, + fpn_config, + fpn_channels, + tuple(fnode_cfg['inputs_offsets']), + target_reduction=reduction, + pad_type=pad_type, + downsample=downsample, + upsample=upsample, + norm_layer=norm_layer, + apply_resample_bn=apply_resample_bn, + conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias, + weight_method=fnode_cfg['weight_method']) + + after_combine = nn.Sequential() + + in_channels = 0 + out_channels = 0 + for input_offset in fnode_cfg['inputs_offsets']: + in_channels += self.feature_info[input_offset]['num_chs'] + + out_channels = fpn_channels[fpn_channels_idx] + + if merge_type == 'csp': + after_combine.add_module( + 'CspLayer', + CSPLayer( + in_channels, + out_channels, + 2, + shortcut=True, + depthwise=False, + act='silu')) + elif merge_type == 'shuffle': + after_combine.add_module( + 'shuffleBlock', ShuffleBlock(in_channels, in_channels)) + after_combine.add_module( + 'conv1x1', + create_conv2d(in_channels, out_channels, kernel_size=1)) + elif merge_type == 'conv': + after_combine.add_module( + 'conv1x1', + create_conv2d(in_channels, out_channels, kernel_size=1)) + conv_kwargs = dict( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + padding=pad_type, + bias=False, + norm_layer=norm_layer, + act_layer=act_layer) + if not conv_bn_relu_pattern: + conv_kwargs['bias'] = redundant_bias + conv_kwargs['act_layer'] = None + after_combine.add_module('act', act_layer(inplace=True)) + after_combine.add_module( + 'conv', + SeparableConv2d(**conv_kwargs) + if separable_conv else ConvBnAct2d(**conv_kwargs)) + + self.fnode.append( + GiraffeNode(combine=combine, after_combine=after_combine)) + self.feature_info[i] = dict( + num_chs=fpn_channels[fpn_channels_idx], reduction=reduction) + + self.out_feature_info = [] + out_node = list(self.feature_info.keys())[-num_levels::] + for i in out_node: + self.out_feature_info.append(self.feature_info[i]) + + self.feature_info = self.out_feature_info + + def forward(self, x: List[torch.Tensor]): + for fn in self.fnode: + x.append(fn(x)) + return x[-self.num_levels::] + + +class GiraffeNeck(nn.Module): + + def __init__(self, min_level, max_level, num_levels, norm_layer, + norm_kwargs, act_type, fpn_config, fpn_name, fpn_channels, + out_fpn_channels, weight_method, depth_multiplier, + width_multiplier, with_backslash, with_slash, + with_skip_connect, skip_connect_type, separable_conv, + feature_info, merge_type, pad_type, downsample_type, + upsample_type, apply_resample_bn, conv_after_downsample, + redundant_bias, conv_bn_relu_pattern, alternate_init): + super(GiraffeNeck, self).__init__() + + self.num_levels = num_levels + self.min_level = min_level + self.in_features = [0, 1, 2, 3, 4, 5, + 6][self.min_level - 1:self.min_level - 1 + + num_levels] + self.alternate_init = alternate_init + norm_layer = norm_layer or nn.BatchNorm2d + if norm_kwargs: + norm_layer = partial(norm_layer, **norm_kwargs) + act_layer = get_act_layer(act_type) or _ACT_LAYER + fpn_config = fpn_config or get_graph_config( + fpn_name, + min_level=min_level, + max_level=max_level, + weight_method=weight_method, + depth_multiplier=depth_multiplier, + with_backslash=with_backslash, + with_slash=with_slash, + with_skip_connect=with_skip_connect, + skip_connect_type=skip_connect_type) + + # width scale + for i in range(len(fpn_channels)): + fpn_channels[i] = int(fpn_channels[i] * width_multiplier) + + self.resample = nn.ModuleDict() + for level in range(num_levels): + if level < len(feature_info): + in_chs = feature_info[level]['num_chs'] + reduction = feature_info[level]['reduction'] + else: + # Adds a coarser level by downsampling the last feature map + reduction_ratio = 2 + self.resample[str(level)] = ResampleFeatureMap( + in_channels=in_chs, + out_channels=feature_info[level - 1]['num_chs'], + pad_type=pad_type, + downsample=downsample_type, + upsample=upsample_type, + norm_layer=norm_layer, + reduction_ratio=reduction_ratio, + apply_bn=apply_resample_bn, + conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias, + ) + in_chs = feature_info[level - 1]['num_chs'] + reduction = int(reduction * reduction_ratio) + feature_info.append(dict(num_chs=in_chs, reduction=reduction)) + + self.cell = SequentialList() + giraffe_layer = GiraffeLayer( + feature_info=feature_info, + fpn_config=fpn_config, + inner_fpn_channels=fpn_channels, + outer_fpn_channels=out_fpn_channels, + num_levels=num_levels, + pad_type=pad_type, + downsample=downsample_type, + upsample=upsample_type, + norm_layer=norm_layer, + act_layer=act_layer, + separable_conv=separable_conv, + apply_resample_bn=apply_resample_bn, + conv_after_downsample=conv_after_downsample, + conv_bn_relu_pattern=conv_bn_relu_pattern, + redundant_bias=redundant_bias, + merge_type=merge_type) + self.cell.add_module('giraffeNeck', giraffe_layer) + feature_info = giraffe_layer.feature_info + + def init_weights(self, pretrained=False): + for n, m in self.named_modules(): + if 'backbone' not in n: + if self.alternate_init: + _init_weight_alt(m, n) + else: + _init_weight(m, n) + + def forward(self, x: List[torch.Tensor]): + if type(x) is tuple: + x = list(x) + x = [x[f] for f in self.in_features] + for resample in self.resample.values(): + x.append(resample(x[-1])) + x = self.cell(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn_btn.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn_btn.py new file mode 100644 index 0000000000000000000000000000000000000000..9ce58f7c5cde082dea379c4a56abe079a8b413b5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/base_models/necks/giraffe_fpn_btn.py @@ -0,0 +1,132 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + +from modelscope.models.cv.tinynas_detection.damo.base_models.core.ops import ( + ConvBNAct, CSPStage) + + +class GiraffeNeckV2(nn.Module): + + def __init__( + self, + depth=1.0, + hidden_ratio=1.0, + in_features=[2, 3, 4], + in_channels=[256, 512, 1024], + out_channels=[256, 512, 1024], + act='silu', + spp=False, + block_name='BasicBlock', + ): + super().__init__() + self.in_features = in_features + self.in_channels = in_channels + self.out_channels = out_channels + Conv = ConvBNAct + + self.upsample = nn.Upsample(scale_factor=2, mode='nearest') + + # node x3: input x0, x1 + self.bu_conv13 = Conv(in_channels[1], in_channels[1], 3, 2, act=act) + self.merge_3 = CSPStage( + block_name, + in_channels[1] + in_channels[2], + hidden_ratio, + in_channels[2], + round(3 * depth), + act=act, + spp=spp) + + # node x4: input x1, x2, x3 + self.bu_conv24 = Conv(in_channels[0], in_channels[0], 3, 2, act=act) + self.merge_4 = CSPStage( + block_name, + in_channels[0] + in_channels[1] + in_channels[2], + hidden_ratio, + in_channels[1], + round(3 * depth), + act=act, + spp=spp) + + # node x5: input x2, x4 + self.merge_5 = CSPStage( + block_name, + in_channels[1] + in_channels[0], + hidden_ratio, + out_channels[0], + round(3 * depth), + act=act, + spp=spp) + + # node x7: input x4, x5 + self.bu_conv57 = Conv(out_channels[0], out_channels[0], 3, 2, act=act) + self.merge_7 = CSPStage( + block_name, + out_channels[0] + in_channels[1], + hidden_ratio, + out_channels[1], + round(3 * depth), + act=act, + spp=spp) + + # node x6: input x3, x4, x7 + self.bu_conv46 = Conv(in_channels[1], in_channels[1], 3, 2, act=act) + self.bu_conv76 = Conv(out_channels[1], out_channels[1], 3, 2, act=act) + self.merge_6 = CSPStage( + block_name, + in_channels[1] + out_channels[1] + in_channels[2], + hidden_ratio, + out_channels[2], + round(3 * depth), + act=act, + spp=spp) + + def init_weights(self): + pass + + def forward(self, out_features): + """ + Args: + inputs: input images. + + Returns: + Tuple[Tensor]: FPN feature. + """ + + # backbone + [x2, x1, x0] = out_features + + # node x3 + x13 = self.bu_conv13(x1) + x3 = torch.cat([x0, x13], 1) + x3 = self.merge_3(x3) + + # node x4 + x34 = self.upsample(x3) + x24 = self.bu_conv24(x2) + x4 = torch.cat([x1, x24, x34], 1) + x4 = self.merge_4(x4) + + # node x5 + x45 = self.upsample(x4) + x5 = torch.cat([x2, x45], 1) + x5 = self.merge_5(x5) + + # node x8 + # x8 = x5 + + # node x7 + x57 = self.bu_conv57(x5) + x7 = torch.cat([x4, x57], 1) + x7 = self.merge_7(x7) + + # node x6 + x46 = self.bu_conv46(x4) + x76 = self.bu_conv76(x7) + x6 = torch.cat([x3, x46, x76], 1) + x6 = self.merge_6(x6) + + outputs = (x5, x7, x6) + return outputs diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/detector.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/detector.py new file mode 100644 index 0000000000000000000000000000000000000000..5f6b216b0c4a330062eb765e45c6c01d815b9f19 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/detectors/detector.py @@ -0,0 +1,99 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +from torch.nn.parallel import DistributedDataParallel as DDP + +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.cv.tinynas_detection.damo.base_models.backbones import \ + build_backbone +from modelscope.models.cv.tinynas_detection.damo.base_models.heads import \ + build_head +from modelscope.models.cv.tinynas_detection.damo.base_models.necks import \ + build_neck +from modelscope.models.cv.tinynas_detection.damo.structures.image_list import \ + to_image_list +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class Detector(TorchModel): + + def __init__(self, config): + super().__init__() + + self.backbone = build_backbone(config.model.backbone) + self.neck = build_neck(config.model.neck) + self.head = build_head(config.model.head) + + self.config = config + + def init_bn(self, M): + + for m in M.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eps = 1e-3 + m.momentum = 0.03 + + def init_model(self): + + self.apply(self.init_bn) + + self.backbone.init_weights() + self.neck.init_weights() + self.head.init_weights() + + def load_pretrain_detector(self, pretrain_model): + ckpt = torch.load(pretrain_model, map_location='cpu') + if 'model' in ckpt: + state_dict = ckpt['model'] + elif 'state_dict' in ckpt: + state_dict = ckpt['state_dict'] + logger.info(f'Finetune from {pretrain_model}................') + new_state_dict = {} + for k, v in self.state_dict().items(): + k = k.replace('module.', '') + if 'head' in k: + new_state_dict[k] = self.state_dict()[k] + continue + new_state_dict[k] = state_dict[k] + + self.load_state_dict(new_state_dict, strict=True) + + def forward(self, x, targets=None, tea=False, stu=False): + images = to_image_list(x) + feature_outs = self.backbone(images.tensors) # list of tensor + fpn_outs = self.neck(feature_outs) + + if tea: + return fpn_outs + else: + outputs = self.head( + fpn_outs, + targets, + imgs=images, + ) + if stu: + return outputs, fpn_outs + else: + return outputs + + +def build_local_model(config, device): + model = Detector(config) + model.init_model() + model.to(device) + + return model + + +def build_ddp_model(model, local_rank): + model = DDP( + model, + device_ids=[local_rank], + output_device=local_rank, + broadcast_buffers=False, + find_unused_parameters=False) + + return model diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..14e2370d51228c3d7f521bc2038d3e9738c004d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/__init__.py @@ -0,0 +1 @@ +# Copyright © Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/bounding_box.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/bounding_box.py new file mode 100644 index 0000000000000000000000000000000000000000..49fa646b852e6d66ae3e0456f0b026d0d06a3efc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/bounding_box.py @@ -0,0 +1,245 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Copyright © Alibaba, Inc. and its affiliates. + +import torch + +# transpose +FLIP_LEFT_RIGHT = 0 +FLIP_TOP_BOTTOM = 1 + + +class BoxList(object): + """ + This class represents a set of bounding boxes. + The bounding boxes are represented as a Nx4 Tensor. + In order to uniquely determine the bounding boxes with respect + to an image, we also store the corresponding image dimensions. + They can contain extra information that is specific to each bounding box, + such as labels. + """ + + def __init__(self, bbox, image_size, mode='xyxy'): + device = bbox.device if isinstance( + bbox, torch.Tensor) else torch.device('cpu') + bbox = torch.as_tensor(bbox, dtype=torch.float32, device=device) + if bbox.ndimension() != 2: + raise ValueError('bbox should have 2 dimensions, got {}'.format( + bbox.ndimension())) + if bbox.size(-1) != 4: + raise ValueError('last dimension of bbox should have a ' + 'size of 4, got {}'.format(bbox.size(-1))) + if mode not in ('xyxy', 'xywh'): + raise ValueError("mode should be 'xyxy' or 'xywh'") + + self.bbox = bbox + self.size = image_size # (image_width, image_height) + self.mode = mode + self.extra_fields = {} + + def add_field(self, field, field_data): + self.extra_fields[field] = field_data + + def get_field(self, field): + return self.extra_fields[field] + + def has_field(self, field): + return field in self.extra_fields + + def fields(self): + return list(self.extra_fields.keys()) + + def _copy_extra_fields(self, bbox): + for k, v in bbox.extra_fields.items(): + self.extra_fields[k] = v + + def convert(self, mode): + if mode not in ('xyxy', 'xywh'): + raise ValueError("mode should be 'xyxy' or 'xywh'") + if mode == self.mode: + return self + # we only have two modes, so don't need to check + # self.mode + xmin, ymin, xmax, ymax = self._split_into_xyxy() + if mode == 'xyxy': + bbox = torch.cat((xmin, ymin, xmax, ymax), dim=-1) + bbox = BoxList(bbox, self.size, mode=mode) + else: + TO_REMOVE = 0 + bbox = torch.cat( + (xmin, ymin, xmax - xmin + TO_REMOVE, ymax - ymin + TO_REMOVE), + dim=-1) + bbox = BoxList(bbox, self.size, mode=mode) + bbox._copy_extra_fields(self) + return bbox + + def _split_into_xyxy(self): + if self.mode == 'xyxy': + xmin, ymin, xmax, ymax = self.bbox.split(1, dim=-1) + return xmin, ymin, xmax, ymax + elif self.mode == 'xywh': + TO_REMOVE = 0 + xmin, ymin, w, h = self.bbox.split(1, dim=-1) + return ( + xmin, + ymin, + xmin + (w - TO_REMOVE).clamp(min=0), + ymin + (h - TO_REMOVE).clamp(min=0), + ) + else: + raise RuntimeError('Should not be here') + + def resize(self, size, *args, **kwargs): + """ + Returns a resized copy of this bounding box + :param size: The requested size in pixels, as a 2-tuple: + (width, height). + """ + ratios = tuple( + float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) + if ratios[0] == ratios[1]: + ratio = ratios[0] + scaled_box = self.bbox * ratio + bbox = BoxList(scaled_box, size, mode=self.mode) + for k, v in self.extra_fields.items(): + if not isinstance(v, torch.Tensor): + v = v.resize(size, *args, **kwargs) + bbox.add_field(k, v) + return bbox + + ratio_width, ratio_height = ratios + xmin, ymin, xmax, ymax = self._split_into_xyxy() + scaled_xmin = xmin * ratio_width + scaled_xmax = xmax * ratio_width + scaled_ymin = ymin * ratio_height + scaled_ymax = ymax * ratio_height + scaled_box = torch.cat( + (scaled_xmin, scaled_ymin, scaled_xmax, scaled_ymax), dim=-1) + bbox = BoxList(scaled_box, size, mode='xyxy') + for k, v in self.extra_fields.items(): + if not isinstance(v, torch.Tensor): + v = v.resize(size, *args, **kwargs) + bbox.add_field(k, v) + + return bbox.convert(self.mode) + + def transpose(self, method): + """ + Transpose bounding box (flip or rotate in 90 degree steps) + :param method: One of :py:attr:`PIL.Image.FLIP_LEFT_RIGHT`, + :py:attr:`PIL.Image.FLIP_TOP_BOTTOM`, :py:attr:`PIL.Image.ROTATE_90`, + :py:attr:`PIL.Image.ROTATE_180`, :py:attr:`PIL.Image.ROTATE_270`, + :py:attr:`PIL.Image.TRANSPOSE` or :py:attr:`PIL.Image.TRANSVERSE`. + """ + if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): + raise NotImplementedError( + 'Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented') + + image_width, image_height = self.size + xmin, ymin, xmax, ymax = self._split_into_xyxy() + if method == FLIP_LEFT_RIGHT: + TO_REMOVE = 0 + transposed_xmin = image_width - xmax - TO_REMOVE + transposed_xmax = image_width - xmin - TO_REMOVE + transposed_ymin = ymin + transposed_ymax = ymax + elif method == FLIP_TOP_BOTTOM: + transposed_xmin = xmin + transposed_xmax = xmax + transposed_ymin = image_height - ymax + transposed_ymax = image_height - ymin + + transposed_boxes = torch.cat((transposed_xmin, transposed_ymin, + transposed_xmax, transposed_ymax), + dim=-1) + bbox = BoxList(transposed_boxes, self.size, mode='xyxy') + for k, v in self.extra_fields.items(): + if not isinstance(v, torch.Tensor): + v = v.transpose(method) + bbox.add_field(k, v) + return bbox.convert(self.mode) + + def crop(self, box): + """ + Cropss a rectangular region from this bounding box. The box is a + 4-tuple defining the left, upper, right, and lower pixel + coordinate. + """ + xmin, ymin, xmax, ymax = self._split_into_xyxy() + w, h = box[2] - box[0], box[3] - box[1] + cropped_xmin = (xmin - box[0]).clamp(min=0, max=w) + cropped_ymin = (ymin - box[1]).clamp(min=0, max=h) + cropped_xmax = (xmax - box[0]).clamp(min=0, max=w) + cropped_ymax = (ymax - box[1]).clamp(min=0, max=h) + + cropped_box = torch.cat( + (cropped_xmin, cropped_ymin, cropped_xmax, cropped_ymax), dim=-1) + bbox = BoxList(cropped_box, (w, h), mode='xyxy') + for k, v in self.extra_fields.items(): + if not isinstance(v, torch.Tensor): + v = v.crop(box) + bbox.add_field(k, v) + return bbox.convert(self.mode) + + # Tensor-like methods + + def to(self, device): + bbox = BoxList(self.bbox.to(device), self.size, self.mode) + for k, v in self.extra_fields.items(): + if hasattr(v, 'to'): + v = v.to(device) + bbox.add_field(k, v) + return bbox + + def __getitem__(self, item): + bbox = BoxList(self.bbox[item], self.size, self.mode) + for k, v in self.extra_fields.items(): + bbox.add_field(k, v[item]) + return bbox + + def __len__(self): + return self.bbox.shape[0] + + def clip_to_image(self, remove_empty=True): + TO_REMOVE = 0 + self.bbox[:, 0].clamp_(min=0, max=self.size[0] - TO_REMOVE) + self.bbox[:, 1].clamp_(min=0, max=self.size[1] - TO_REMOVE) + self.bbox[:, 2].clamp_(min=0, max=self.size[0] - TO_REMOVE) + self.bbox[:, 3].clamp_(min=0, max=self.size[1] - TO_REMOVE) + if remove_empty: + box = self.bbox + keep = (box[:, 3] > box[:, 1]) & (box[:, 2] > box[:, 0]) + return self[keep] + return self + + def area(self): + box = self.bbox + if self.mode == 'xyxy': + TO_REMOVE = 0 + area = (box[:, 2] - box[:, 0] + TO_REMOVE) * ( + box[:, 3] - box[:, 1] + TO_REMOVE) + elif self.mode == 'xywh': + area = box[:, 2] * box[:, 3] + else: + raise RuntimeError('Should not be here') + + return area + + def copy_with_fields(self, fields, skip_missing=False): + bbox = BoxList(self.bbox, self.size, self.mode) + if not isinstance(fields, (list, tuple)): + fields = [fields] + for field in fields: + if self.has_field(field): + bbox.add_field(field, self.get_field(field)) + elif not skip_missing: + raise KeyError("Field '{}' not found in {}".format( + field, self)) + return bbox + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'num_boxes={}, '.format(len(self)) + s += 'image_width={}, '.format(self.size[0]) + s += 'image_height={}, '.format(self.size[1]) + s += 'mode={})'.format(self.mode) + return s diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/boxlist_ops.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/boxlist_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..6d00a89d82318e503d89953f04c482a5058a08ad --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/boxlist_ops.py @@ -0,0 +1,92 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Copyright © Alibaba, Inc. and its affiliates. + +import torch + +from .bounding_box import BoxList + + +def remove_small_boxes(boxlist, min_size): + """ + Only keep boxes with both sides >= min_size + Arguments: + boxlist (Boxlist) + min_size (int) + """ + xywh_boxes = boxlist.convert('xywh').bbox + _, _, ws, hs = xywh_boxes.unbind(dim=1) + keep = ((ws >= min_size) & (hs >= min_size)).nonzero().squeeze(1) + return boxlist[keep] + + +def boxlist_iou(boxlist1, boxlist2): + """Compute the intersection over union of two set of boxes. + The box order must be (xmin, ymin, xmax, ymax). + Arguments: + box1: (BoxList) bounding boxes, sized [N,4]. + box2: (BoxList) bounding boxes, sized [M,4]. + Returns: + (tensor) iou, sized [N,M]. + Reference: + https://github.com/chainer/chainercv/blob/master/chainercv/utils/bbox/bbox_iou.py + """ + if boxlist1.size != boxlist2.size: + raise RuntimeError( + 'boxlists should have same image size, got {}, {}'.format( + boxlist1, boxlist2)) + + area1 = boxlist1.area() + area2 = boxlist2.area() + + box1, box2 = boxlist1.bbox, boxlist2.bbox + + lt = torch.max(box1[:, None, :2], box2[:, :2]) # [N,M,2] + rb = torch.min(box1[:, None, 2:], box2[:, 2:]) # [N,M,2] + + TO_REMOVE = 1 + + wh = (rb - lt + TO_REMOVE).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + iou = inter / (area1[:, None] + area2 - inter) + return iou + + +def _cat(tensors, dim=0): + """ + Efficient version of torch.cat that avoids a copy if there is only + a single element in a list + """ + assert isinstance(tensors, (list, tuple)) + if len(tensors) == 1: + return tensors[0] + return torch.cat(tensors, dim) + + +def cat_boxlist(bboxes): + """ + Concatenates a list of BoxList (having the same image size) into a + single BoxList + Arguments: + bboxes (list[BoxList]) + """ + assert isinstance(bboxes, (list, tuple)) + assert all(isinstance(bbox, BoxList) for bbox in bboxes) + + size = bboxes[0].size + assert all(bbox.size == size for bbox in bboxes) + + mode = bboxes[0].mode + assert all(bbox.mode == mode for bbox in bboxes) + + fields = set(bboxes[0].fields()) + assert all(set(bbox.fields()) == fields for bbox in bboxes) + + cat_boxes = BoxList( + _cat([bbox.bbox for bbox in bboxes], dim=0), size, mode) + + for field in fields: + data = _cat([bbox.get_field(field) for bbox in bboxes], dim=0) + cat_boxes.add_field(field, data) + + return cat_boxes diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/image_list.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/image_list.py new file mode 100644 index 0000000000000000000000000000000000000000..42aa75251b016144f457c5e088e8c97cfc11712a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/structures/image_list.py @@ -0,0 +1,78 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# Copyright © Alibaba, Inc. and its affiliates. + +from __future__ import division + +import torch + + +class ImageList(object): + """ + Structure that holds a list of images (of possibly + varying sizes) as a single tensor. + This works by padding the images to the same size, + and storing in a field the original sizes of each image + """ + + def __init__(self, tensors, image_sizes, pad_sizes): + """ + Arguments: + tensors (tensor) + image_sizes (list[tuple[int, int]]) + """ + self.tensors = tensors + self.image_sizes = image_sizes + self.pad_sizes = pad_sizes + + def to(self, *args, **kwargs): + cast_tensor = self.tensors.to(*args, **kwargs) + return ImageList(cast_tensor, self.image_sizes, self.pad_sizes) + + +def to_image_list(tensors, size_divisible=0, max_size=None): + """ + tensors can be an ImageList, a torch.Tensor or + an iterable of Tensors. It can't be a numpy array. + When tensors is an iterable of Tensors, it pads + the Tensors with zeros so that they have the same + shape + """ + if isinstance(tensors, torch.Tensor) and size_divisible > 0: + tensors = [tensors] + + if isinstance(tensors, ImageList): + return tensors + elif isinstance(tensors, torch.Tensor): + # single tensor shape can be inferred + if tensors.dim() == 3: + tensors = tensors[None] + assert tensors.dim() == 4 + image_sizes = [tensor.shape[-2:] for tensor in tensors] + return ImageList(tensors, image_sizes, image_sizes) + elif isinstance(tensors, (tuple, list)): + if max_size is None: + max_size = tuple( + max(s) for s in zip(*[img.shape for img in tensors])) + # TODO Ideally, just remove this and let me model handle arbitrary + # input sizs + if size_divisible > 0: + import math + + stride = size_divisible + max_size = list(max_size) + max_size[1] = int(math.ceil(max_size[1] / stride) * stride) + max_size[2] = int(math.ceil(max_size[2] / stride) * stride) + max_size = tuple(max_size) + + batch_shape = (len(tensors), ) + max_size + batched_imgs = tensors[0].new(*batch_shape).zero_() # + 114 + for img, pad_img in zip(tensors, batched_imgs): + pad_img[:img.shape[0], :img.shape[1], :img.shape[2]].copy_(img) + + image_sizes = [im.shape[-2:] for im in tensors] + pad_sizes = [batched_imgs.shape[-2:] for im in batched_imgs] + + return ImageList(batched_imgs, image_sizes, pad_sizes) + else: + raise TypeError('Unsupported type for to_image_list: {}'.format( + type(tensors))) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/__init__.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e93321ace4bdc3d2cd4e68fedb4131ea532fd7cb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/__init__.py @@ -0,0 +1,5 @@ +# Copyright © Alibaba, Inc. and its affiliates. + +from .boxes import adjust_box_anns, postprocess +from .model_utils import ema_model, get_model_info +from .scheduler import cosine_scheduler diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/boxes.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..369767bb322f0d18d36bebb8b9cc4632778b8c08 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/boxes.py @@ -0,0 +1,329 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Copyright (c) Megvii Inc. All rights reserved. +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torchvision + +from modelscope.models.cv.tinynas_detection.damo.structures.bounding_box import \ + BoxList + +__all__ = [ + 'filter_box', + 'postprocess', + 'bboxes_iou', + 'matrix_iou', + 'adjust_box_anns', + 'xyxy2xywh', + 'xyxy2cxcywh', + 'bbox_overlaps', +] + + +def bbox_overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6): + """Calculate overlap between two set of bboxes. + If ``is_aligned `` is ``False``, then calculate the overlaps between each + bbox of bboxes1 and bboxes2, otherwise the overlaps between each aligned + pair of bboxes1 and bboxes2. + Args: + bboxes1 (Tensor): shape (B, m, 4) in format or empty. + bboxes2 (Tensor): shape (B, n, 4) in format or empty. + B indicates the batch dim, in shape (B1, B2, ..., Bn). + If ``is_aligned `` is ``True``, then m and n must be equal. + mode (str): "iou" (intersection over union) or "iof" (intersection over + foreground). + is_aligned (bool, optional): If True, then m and n must be equal. + Default False. + eps (float, optional): A value added to the denominator for numerical + stability. Default 1e-6. + Returns: + Tensor: shape (m, n) if ``is_aligned `` is False else shape (m,) + + Example: + >>> bboxes1 = torch.FloatTensor([ + >>> [0, 0, 10, 10], + >>> [10, 10, 20, 20], + >>> [32, 32, 38, 42], + >>> ]) + >>> bboxes2 = torch.FloatTensor([ + >>> [0, 0, 10, 20], + >>> [0, 10, 10, 19], + >>> [10, 10, 20, 20], + >>> ]) + >>> overlaps = bbox_overlaps(bboxes1, bboxes2) + >>> assert overlaps.shape == (3, 3) + >>> overlaps = bbox_overlaps(bboxes1, bboxes2, is_aligned=True) + >>> assert overlaps.shape == (3, ) + + Example: + >>> empty = torch.empty(0, 4) + >>> nonempty = torch.FloatTensor([[0, 0, 10, 9]]) + >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1) + >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0) + >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0) + """ + + assert mode in ['iou', 'iof', 'giou'], f'Unsupported mode {mode}' + # Either the boxes are empty or the length of boxes's last dimenstion is 4 + assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0) + assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0) + + # Batch dim must be the same + # Batch dim: (B1, B2, ... Bn) + assert bboxes1.shape[:-2] == bboxes2.shape[:-2] + batch_shape = bboxes1.shape[:-2] + + rows = bboxes1.size(-2) + cols = bboxes2.size(-2) + if is_aligned: + assert rows == cols + + if rows * cols == 0: + if is_aligned: + return bboxes1.new(batch_shape + (rows, )) + else: + return bboxes1.new(batch_shape + (rows, cols)) + + area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * ( + bboxes1[..., 3] - bboxes1[..., 1]) + area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * ( + bboxes2[..., 3] - bboxes2[..., 1]) + + if is_aligned: + lt = torch.max(bboxes1[..., :2], bboxes2[..., :2]) # [B, rows, 2] + rb = torch.min(bboxes1[..., 2:], bboxes2[..., 2:]) # [B, rows, 2] + + wh = (rb - lt).clamp(min=0) # [B, rows, 2] + overlap = wh[..., 0] * wh[..., 1] + + if mode in ['iou', 'giou']: + union = area1 + area2 - overlap + else: + union = area1 + if mode == 'giou': + enclosed_lt = torch.min(bboxes1[..., :2], bboxes2[..., :2]) + enclosed_rb = torch.max(bboxes1[..., 2:], bboxes2[..., 2:]) + else: + lt = torch.max(bboxes1[..., :, None, :2], + bboxes2[..., None, :, :2]) # [B, rows, cols, 2] + rb = torch.min(bboxes1[..., :, None, 2:], + bboxes2[..., None, :, 2:]) # [B, rows, cols, 2] + + wh = (rb - lt).clamp(min=0) # [B, rows, cols, 2] + overlap = wh[..., 0] * wh[..., 1] + + if mode in ['iou', 'giou']: + union = area1[..., None] + area2[..., None, :] - overlap + else: + union = area1[..., None] + if mode == 'giou': + enclosed_lt = torch.min(bboxes1[..., :, None, :2], + bboxes2[..., None, :, :2]) + enclosed_rb = torch.max(bboxes1[..., :, None, 2:], + bboxes2[..., None, :, 2:]) + + eps = union.new_tensor([eps]) + union = torch.max(union, eps) + ious = overlap / union + if mode in ['iou', 'iof']: + return ious + # calculate gious + enclose_wh = (enclosed_rb - enclosed_lt).clamp(min=0) + enclose_area = enclose_wh[..., 0] * enclose_wh[..., 1] + enclose_area = torch.max(enclose_area, eps) + gious = ious - (enclose_area - union) / enclose_area + return gious + + +def multiclass_nms(multi_bboxes, + multi_scores, + score_thr, + iou_thr, + max_num=100, + score_factors=None): + """NMS for multi-class bboxes. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_scores (Tensor): shape (n, #class), where the last column + contains scores of the background class, but this will be ignored. + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + nms_thr (float): NMS IoU threshold + max_num (int): if there are more than max_num bboxes after NMS, + only top max_num will be kept. + score_factors (Tensor): The factors multiplied to scores before + applying NMS + + Returns: + tuple: (bboxes, labels), tensors of shape (k, 5) and (k, 1). Labels \ + are 0-based. + """ + num_classes = multi_scores.size(1) + # exclude background category + if multi_bboxes.shape[1] > 4: + bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4) + else: + bboxes = multi_bboxes[:, None].expand( + multi_scores.size(0), num_classes, 4) + scores = multi_scores + # filter out boxes with low scores + valid_mask = scores > score_thr # 1000 * 80 bool + + # We use masked_select for ONNX exporting purpose, + # which is equivalent to bboxes = bboxes[valid_mask] + # (TODO): as ONNX does not support repeat now, + # we have to use this ugly code + # bboxes -> 1000, 4 + bboxes = torch.masked_select( + bboxes, + torch.stack((valid_mask, valid_mask, valid_mask, valid_mask), + -1)).view(-1, 4) # mask-> 1000*80*4, 80000*4 + if score_factors is not None: + scores = scores * score_factors[:, None] + scores = torch.masked_select(scores, valid_mask) + labels = valid_mask.nonzero(as_tuple=False)[:, 1] + + if bboxes.numel() == 0: + bboxes = multi_bboxes.new_zeros((0, 5)) + labels = multi_bboxes.new_zeros((0, ), dtype=torch.long) + scores = multi_bboxes.new_zeros((0, )) + + return bboxes, scores, labels + + keep = torchvision.ops.batched_nms(bboxes, scores, labels, iou_thr) + + if max_num > 0: + keep = keep[:max_num] + + return bboxes[keep], scores[keep], labels[keep] + + +def filter_box(output, scale_range): + """ + output: (N, 5+class) shape + """ + min_scale, max_scale = scale_range + w = output[:, 2] - output[:, 0] + h = output[:, 3] - output[:, 1] + keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale) + return output[keep] + + +def filter_results(boxlist, num_classes, nms_thre): + boxes = boxlist.bbox + scores = boxlist.get_field('scores') + cls = boxlist.get_field('labels') + nms_out_index = torchvision.ops.batched_nms( + boxes, + scores, + cls, + nms_thre, + ) + boxlist = boxlist[nms_out_index] + + return boxlist + + +def postprocess(cls_scores, + bbox_preds, + num_classes, + conf_thre=0.7, + nms_thre=0.45, + imgs=None): + batch_size = bbox_preds.size(0) + output = [None for _ in range(batch_size)] + for i in range(batch_size): + # If none are remaining => process next image + if not bbox_preds[i].size(0): + continue + detections, scores, labels = multiclass_nms(bbox_preds[i], + cls_scores[i], conf_thre, + nms_thre, 500) + detections = torch.cat( + (detections, scores[:, None], scores[:, None], labels[:, None]), + dim=1) + + if output[i] is None: + output[i] = detections + else: + output[i] = torch.cat((output[i], detections)) + + # transfer to BoxList + for i in range(len(output)): + res = output[i] + if res is None or imgs is None: + boxlist = BoxList(torch.zeros(0, 4), (0, 0), mode='xyxy') + boxlist.add_field('objectness', 0) + boxlist.add_field('scores', 0) + boxlist.add_field('labels', -1) + + else: + img_h, img_w = imgs.image_sizes[i] + boxlist = BoxList(res[:, :4], (img_w, img_h), mode='xyxy') + boxlist.add_field('objectness', res[:, 4]) + boxlist.add_field('scores', res[:, 5]) + boxlist.add_field('labels', res[:, 6] + 1) + output[i] = boxlist + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a, b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12) + + +def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max): + bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max) + bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max) + return bbox + + +def xyxy2xywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + return bboxes + + +def xyxy2cxcywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5 + return bboxes diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/model_utils.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..82f947b83f262f907fa3ef15058d3371587f4f08 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/model_utils.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# Copyright (c) Megvii Inc. All rights reserved. +# Copyright © Alibaba, Inc. and its affiliates. + +import math +import time +from copy import deepcopy + +import torch +import torch.nn as nn +from thop import profile +from torch.nn.parallel import DistributedDataParallel as DDP + +__all__ = [ + 'fuse_conv_and_bn', + 'fuse_model', + 'get_model_info', + 'replace_module', + 'ema_model', +] + + +def ema_scheduler(x, ema_momentum): + return ema_momentum * (1 - math.exp(-x / 2000)) + + +class ema_model: + + def __init__(self, student, ema_momentum): + + self.model = deepcopy(student).eval() + self.ema_momentum = ema_momentum + for param in self.model.parameters(): + param.requires_grad_(False) + + def update(self, iters, student): + if isinstance(student, DDP): + student = student.module.state_dict() + else: + student = student.state_dict() + with torch.no_grad(): + momentum = ema_scheduler(iters, self.ema_momentum) + for name, param in self.model.state_dict().items(): + if param.dtype.is_floating_point: + param *= momentum + param += (1.0 - momentum) * student[name].detach() + + +def get_latency(model, inp, iters=500, warmup=2): + + start = time.time() + for i in range(iters): + out = model(inp) + torch.cuda.synchronize() + if i <= warmup: + start = time.time() + latency = (time.time() - start) / (iters - warmup) + + return out, latency + + +def get_model_info(model, tsize): + stride = 640 + model = model.eval() + backbone = model.backbone + neck = model.neck + head = model.head + h, w = tsize + img = torch.randn((1, 3, stride, stride), + device=next(model.parameters()).device) + + bf, bp = profile(deepcopy(backbone), inputs=(img, ), verbose=False) + bo, bl = get_latency(backbone, img, iters=10) + + nf, np = profile(deepcopy(neck), inputs=(bo, ), verbose=False) + no, nl = get_latency(neck, bo, iters=10) + + hf, hp = profile(deepcopy(head), inputs=(no, ), verbose=False) + ho, hl = get_latency(head, no, iters=10) + + _, total_latency = get_latency(model, img) + total_flops = 0 + info = '' + for name, flops, params, latency in zip(('backbone', 'neck', 'head'), + (bf, nf, hf), (bp, np, hp), + (bl, nl, hl)): + params /= 1e6 + flops /= 1e9 + flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops + total_flops += flops + info += f"{name}'s params(M): {params:.2f}, " + \ + f'flops(G): {flops:.2f}, latency(ms): {latency*1000:.3f}\n' + info += f'total latency(ms): {total_latency*1000:.3f}, ' + \ + f'total flops(G): {total_flops:.2f}\n' + return info + + +def fuse_conv_and_bn(conv, bn): + # Fuse convolution and batchnorm layers + # https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + fusedconv = ( + nn.Conv2d( + conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + groups=conv.groups, + bias=True, + ).requires_grad_(False).to(conv.weight.device)) + + # prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # prepare spatial bias + b_conv = ( + torch.zeros(conv.weight.size(0), device=conv.weight.device) + if conv.bias is None else conv.bias) + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div( + torch.sqrt(bn.running_var + bn.eps)) + fusedconv.bias.copy_( + torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def fuse_model(model): + from damo.base_models.core.ops import ConvBNAct + from damo.base_models.backbones.tinynas_res import ConvKXBN + + for m in model.modules(): + if type(m) is ConvBNAct and hasattr(m, 'bn'): + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, 'bn') # remove batchnorm + m.forward = m.fuseforward # update forward + elif type(m) is ConvKXBN and hasattr(m, 'bn1'): + m.conv1 = fuse_conv_and_bn(m.conv1, m.bn1) # update conv + delattr(m, 'bn1') # remove batchnorm + m.forward = m.fuseforward # update forward + + return model + + +def replace_module(module, + replaced_module_type, + new_module_type, + replace_func=None): + """ + Replace given type in module to a new type. mostly used in deploy. + + Args: + module (nn.Module): model to apply replace operation. + replaced_module_type (Type): module type to be replaced. + new_module_type (Type) + replace_func (function): python function to describe replace logic. + Defalut value None. + + Returns: + model (nn.Module): module that already been replaced. + """ + + def default_replace_func(replaced_module_type, new_module_type): + return new_module_type() + + if replace_func is None: + replace_func = default_replace_func + + model = module + if isinstance(module, replaced_module_type): + model = replace_func(replaced_module_type, new_module_type) + else: # recurrsively replace + for name, child in module.named_children(): + new_child = replace_module(child, replaced_module_type, + new_module_type) + if new_child is not child: # child is already replaced + model.add_module(name, new_child) + + return model diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/scheduler.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..0e042db5fe862b03b52e6d42d3cdde30e88d53dd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/damo/utils/scheduler.py @@ -0,0 +1,34 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + + +class cosine_scheduler: + + def __init__(self, + base_lr_per_img, + batch_size, + min_lr_ratio, + total_iters, + no_aug_iters, + warmup_iters, + warmup_start_lr=0): + + self.base_lr = base_lr_per_img * batch_size + self.final_lr = self.base_lr * min_lr_ratio + self.warmup_iters = warmup_iters + self.warmup_start_lr = warmup_start_lr + self.total_iters = total_iters + self.no_aug_iters = no_aug_iters + + def get_lr(self, iters): + + if iters < self.warmup_iters: + lr = (self.base_lr - self.warmup_start_lr) * pow( + iters / float(self.warmup_iters), 2) + self.warmup_start_lr + elif iters >= self.total_iters - self.no_aug_iters: + lr = self.final_lr + else: + lr = self.final_lr + 0.5 * (self.base_lr - self.final_lr) \ + * (1.0 + math.cos(math.pi * (iters - self.warmup_iters) + / (self.total_iters - self.warmup_iters - self.no_aug_iters))) + return lr diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/detector.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/detector.py new file mode 100644 index 0000000000000000000000000000000000000000..4ece5f91f2cf17dfb26eeea6fb9b2489414dd07b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/detector.py @@ -0,0 +1,178 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +import os.path as osp +import pickle + +import torch +import torch.nn as nn +import torchvision + +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.cv.tinynas_detection.damo.base_models.backbones import \ + build_backbone +from modelscope.models.cv.tinynas_detection.damo.base_models.heads import \ + build_head +from modelscope.models.cv.tinynas_detection.damo.base_models.necks import \ + build_neck +from modelscope.outputs.cv_outputs import DetectionOutput +from .utils import parse_config + + +class SingleStageDetector(TorchModel): + """ + The base class of single stage detector. + """ + + def __init__(self, model_dir: str, *args, **kwargs): + """ + init model by cfg + """ + super().__init__(model_dir, *args, **kwargs) + + config_path = osp.join(model_dir, self.config_name) + config = parse_config(config_path) + self.cfg = config + model_path = osp.join(model_dir, config.model.name) + label_map = osp.join(model_dir, config.model.class_map) + self.label_map = pickle.load(open(label_map, 'rb')) + self.size_divisible = config.dataset.size_divisibility + self.num_classes = config.model.head.num_classes + self.conf_thre = config.model.head.nms_conf_thre + self.nms_thre = config.model.head.nms_iou_thre + + if 'TinyNAS' in self.cfg.model.backbone.name: + self.cfg.model.backbone.structure_file = osp.join( + model_dir, self.cfg.model.backbone.structure_file) + self.backbone = build_backbone(self.cfg.model.backbone) + self.neck = build_neck(self.cfg.model.neck) + self.head = build_head(self.cfg.model.head) + self.head.nms = False + self.apply(self.init_bn) + self.onnx_export = False + + self.load_pretrain_model(model_path) + + def load_pretrain_model(self, pretrain_model): + ckpt = torch.load(pretrain_model, map_location='cpu') + if 'model' in ckpt: + state_dict = ckpt['model'] + elif 'state_dict' in ckpt: + state_dict = ckpt['state_dict'] + new_state_dict = {} + for k, v in state_dict.items(): + k = k.replace('module.', '') + new_state_dict[k] = v + self.load_state_dict(new_state_dict, strict=True) + + def init_bn(self, M): + for m in M.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eps = 1e-3 + m.momentum = 0.03 + + def forward(self, x): + if self.training: + pass + else: + x = self.backbone(x) + x = self.neck(x) + cls_scores, bbox_preds = self.head(x) + prediction = torch.cat( + [bbox_preds, cls_scores[..., 0:self.num_classes]], dim=-1) + return prediction + + def postprocess(self, preds): + if self.onnx_export: + return preds + + bboxes, scores, labels_idx = postprocess_gfocal( + preds, self.num_classes, self.conf_thre, self.nms_thre) + bboxes = bboxes.cpu().numpy() + scores = scores.cpu().numpy() + labels_idx = labels_idx.cpu().numpy() + labels = [self.label_map[idx + 1][0]['name'] for idx in labels_idx] + + return DetectionOutput( + boxes=bboxes, + scores=scores, + class_ids=labels, + ) + + +def multiclass_nms(multi_bboxes, + multi_scores, + score_thr, + iou_thr, + max_num=100, + score_factors=None): + """NMS for multi-class bboxes. + + Args: + multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) + multi_scores (Tensor): shape (n, #class), where the last column + contains scores of the background class, but this will be ignored. + score_thr (float): bbox threshold, bboxes with scores lower than it + will not be considered. + nms_thr (float): NMS IoU threshold + max_num (int): if there are more than max_num bboxes after NMS, + only top max_num will be kept. + score_factors (Tensor): The factors multiplied to scores before + applying NMS + + Returns: + tuple: (bboxes, labels), tensors of shape (k, 5) and (k, 1). Labels \ + are 0-based. + """ + num_classes = multi_scores.size(1) + # exclude background category + if multi_bboxes.shape[1] > 4: + bboxes = multi_bboxes.view(multi_scores.size(0), -1, 4) + else: + bboxes = multi_bboxes[:, None].expand( + multi_scores.size(0), num_classes, 4) + scores = multi_scores + # filter out boxes with low scores + valid_mask = scores > score_thr # 1000 * 80 bool + + # We use masked_select for ONNX exporting purpose, + # which is equivalent to bboxes = bboxes[valid_mask] + # (TODO): as ONNX does not support repeat now, + # we have to use this ugly code + # bboxes -> 1000, 4 + bboxes = torch.masked_select( + bboxes, + torch.stack((valid_mask, valid_mask, valid_mask, valid_mask), + -1)).view(-1, 4) # mask-> 1000*80*4, 80000*4 + if score_factors is not None: + scores = scores * score_factors[:, None] + scores = torch.masked_select(scores, valid_mask) + labels = valid_mask.nonzero(as_tuple=False)[:, 1] + + if bboxes.numel() == 0: + bboxes = multi_bboxes.new_zeros((0, 5)) + labels = multi_bboxes.new_zeros((0, ), dtype=torch.long) + scores = multi_bboxes.new_zeros((0, )) + + return bboxes, scores, labels + + keep = torchvision.ops.batched_nms(bboxes, scores, labels, iou_thr) + + if max_num > 0: + keep = keep[:max_num] + + return bboxes[keep], scores[keep], labels[keep] + + +def postprocess_gfocal(prediction, num_classes, conf_thre=0.05, nms_thre=0.7): + assert prediction.shape[0] == 1 + for i, image_pred in enumerate(prediction): + # If none are remaining => process next image + if not image_pred.size(0): + continue + multi_bboxes = image_pred[:, :4] + multi_scores = image_pred[:, 4:] + detections, scores, labels = multiclass_nms(multi_bboxes, multi_scores, + conf_thre, nms_thre, 500) + + return detections, scores, labels diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_damoyolo.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_damoyolo.py new file mode 100644 index 0000000000000000000000000000000000000000..3243e4dba005a61c7dd7afbedf10d6c9e7fd4085 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_damoyolo.py @@ -0,0 +1,18 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from .detector import SingleStageDetector + + +@MODELS.register_module( + Tasks.domain_specific_object_detection, + module_name=Models.tinynas_damoyolo) +@MODELS.register_module( + Tasks.image_object_detection, module_name=Models.tinynas_damoyolo) +class DamoYolo(SingleStageDetector): + + def __init__(self, model_dir, *args, **kwargs): + self.config_name = 'damoyolo.py' + super(DamoYolo, self).__init__(model_dir, *args, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_detector.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..37bb01da38ead2be7e3dbeff490d33b59e418c1a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/tinynas_detector.py @@ -0,0 +1,16 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors at https://github.com/tinyvision/damo-yolo. + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from .detector import SingleStageDetector + + +@MODELS.register_module( + Tasks.image_object_detection, module_name=Models.tinynas_detection) +class TinynasDetector(SingleStageDetector): + + def __init__(self, model_dir, *args, **kwargs): + self.config_name = 'airdet_s.py' + super(TinynasDetector, self).__init__(model_dir, *args, **kwargs) diff --git a/runtime/Lib/site_packages/models/cv/tinynas_detection/utils.py b/runtime/Lib/site_packages/models/cv/tinynas_detection/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..984e1e4efbe232cf1fb4ffa1d88dc6624b38bff5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/tinynas_detection/utils.py @@ -0,0 +1,33 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# The DAMO-YOLO implementation is also open-sourced by the authors, and available +# at https://github.com/tinyvision/damo-yolo. + +import importlib +import os +import shutil +import sys +import tempfile +from os.path import dirname, join + +from easydict import EasyDict + + +def parse_config(filename): + filename = str(filename) + if filename.endswith('.py'): + with tempfile.TemporaryDirectory() as temp_config_dir: + shutil.copyfile(filename, join(temp_config_dir, '_tempconfig.py')) + sys.path.insert(0, temp_config_dir) + mod = importlib.import_module('_tempconfig') + sys.path.pop(0) + cfg_dict = EasyDict({ + name: value + for name, value in mod.__dict__.items() + if not name.startswith('__') + }) + # delete imported module + del sys.modules['_tempconfig'] + else: + raise IOError('Only .py type are supported now!') + + return cfg_dict diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/UNet_for_video_deinterlace.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/UNet_for_video_deinterlace.py new file mode 100644 index 0000000000000000000000000000000000000000..c08c1e8ee44a21d62e0f183e89390d56ed92a6ce --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/UNet_for_video_deinterlace.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from copy import deepcopy +from typing import Any, Dict, Union + +import torch.cuda +import torch.nn.functional as F +from torch.nn.parallel import DataParallel, DistributedDataParallel + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.video_deinterlace.deinterlace_arch import \ + DeinterlaceNet +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() +__all__ = ['UNetForVideoDeinterlace'] + + +def convert(param): + return { + k.replace('module.', ''): v + for k, v in param.items() if 'module.' in k + } + + +@MODELS.register_module( + Tasks.video_deinterlace, module_name=Models.video_deinterlace) +class UNetForVideoDeinterlace(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the video deinterlace model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + + self.model_dir = model_dir + + frenet_path = os.path.join(model_dir, 'deinterlace_fre.pth') + enhnet_path = os.path.join(model_dir, 'deinterlace_mf.pth') + + self.model = DeinterlaceNet() + self._load_pretrained(frenet_path, enhnet_path) + + def _load_pretrained(self, frenet_path, enhnet_path): + state_dict_frenet = torch.load(frenet_path, map_location=self._device) + state_dict_enhnet = torch.load(enhnet_path, map_location=self._device) + + self.model.frenet.load_state_dict(state_dict_frenet, strict=True) + self.model.enhnet.load_state_dict(state_dict_enhnet, strict=True) + logger.info('load model done.') + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'output': self.model(input)} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(input) + del input + torch.cuda.empty_cache() + return {'pred': preds, 'target': target} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + + if 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/__init__.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bef991f2c4d1e62dd356f5c8c21abfc1e39dc925 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .deinterlace_arch import \ + DeinterlaceNet + +else: + _import_structure = {'deinterlace_arch': ['DeinterlaceNet']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/deinterlace_arch.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/deinterlace_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..7f6a18c5f27afd62219022a39095ef4c148549ec --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/deinterlace_arch.py @@ -0,0 +1,27 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch.nn as nn + +from modelscope.models.cv.video_deinterlace.models.enh import DeinterlaceEnh +from modelscope.models.cv.video_deinterlace.models.fre import DeinterlaceFre + + +class DeinterlaceNet(nn.Module): + + def __init__(self): + super(DeinterlaceNet, self).__init__() + self.frenet = DeinterlaceFre() + self.enhnet = DeinterlaceEnh() + + def forward(self, frames): + self.frenet.eval() + self.enhnet.eval() + with torch.no_grad(): + frame1, frame2, frame3 = frames + + F1_out = self.frenet(frame1) + F2_out = self.frenet(frame2) + F3_out = self.frenet(frame3) + + out = self.enhnet([F1_out, F2_out, F3_out]) + + return out diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/__init__.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/archs.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/archs.py new file mode 100644 index 0000000000000000000000000000000000000000..ba0b5aa9dfc75bad980603a46f1e8475f206e5b2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/archs.py @@ -0,0 +1,97 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.fft +import torch.nn as nn +import torch.nn.functional as F + + +class DoubleConv(nn.Module): + """(convolution => [BN] => ReLU) * 2""" + + def __init__(self, in_channels, out_channels, mid_channels=None): + super().__init__() + if not mid_channels: + mid_channels = out_channels + self.double_conv = nn.Sequential( + nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True)) + + def forward(self, x): + return self.double_conv(x) + + +class TripleConv(nn.Module): + """(convolution => [BN] => ReLU) * 3""" + + def __init__(self, in_channels, out_channels, mid_channels=None): + super().__init__() + if not mid_channels: + mid_channels = out_channels + self.triple_conv = nn.Sequential( + nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(mid_channels, mid_channels, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True)) + + def forward(self, x): + return self.triple_conv(x) + + +class DownConv(nn.Module): + """Downscaling with avgpool then double/triple conv""" + + def __init__(self, in_channels, out_channels, num_conv=2): + super().__init__() + if num_conv == 2: + self.pool_conv = nn.Sequential( + nn.AvgPool2d(2), DoubleConv(in_channels, out_channels)) + else: + self.pool_conv = nn.Sequential( + nn.AvgPool2d(2), TripleConv(in_channels, out_channels)) + + def forward(self, x): + return self.pool_conv(x) + + +class UpCatConv(nn.Module): + """Upscaling then double conv""" + + def __init__(self, in_channels, out_channels, bilinear=False): + super().__init__() + + if bilinear: + self.up = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=True) + self.conv = DoubleConv(in_channels, out_channels) + else: + self.up = nn.Upsample( + scale_factor=2, mode='nearest', align_corners=None) + self.conv = DoubleConv(in_channels, out_channels) + + self.subpixel = nn.PixelShuffle(2) + + def interpolate(self, x): + tensor_temp = x + for i in range(3): + tensor_temp = torch.cat((tensor_temp, x), 1) + x = tensor_temp + x = self.subpixel(x) + return x + + def forward(self, x1, x2): + + x1 = self.interpolate(x1) + + diffY = x2.size()[2] - x1.size()[2] + diffX = x2.size()[3] - x1.size()[3] + x1 = F.pad( + x1, + [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2]) + + x = torch.cat([x2, x1], dim=1) + return self.conv(x) diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/deep_fourier_upsampling.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/deep_fourier_upsampling.py new file mode 100644 index 0000000000000000000000000000000000000000..c6e0799d2f8a18d83d75c88e3b9b8ec83516490b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/deep_fourier_upsampling.py @@ -0,0 +1,47 @@ +# The implementation is adopted from Deep Fourier Upsampling, +# made publicly available at https://github.com/manman1995/Deep-Fourier-Upsampling +import numpy as np +import torch +import torch.fft +import torch.nn as nn +import torch.nn.functional as F + + +class freup_Periodicpadding(nn.Module): + + def __init__(self, channels): + super(freup_Periodicpadding, self).__init__() + + self.amp_fuse = nn.Sequential( + nn.Conv2d(channels, channels, 1, 1, 0), + nn.LeakyReLU(0.1, inplace=False), + nn.Conv2d(channels, channels, 1, 1, 0)) + self.pha_fuse = nn.Sequential( + nn.Conv2d(channels, channels, 1, 1, 0), + nn.LeakyReLU(0.1, inplace=False), + nn.Conv2d(channels, channels, 1, 1, 0)) + + self.post = nn.Conv2d(channels, channels, 1, 1, 0) + + def forward(self, x): + + N, C, H, W = x.shape + + fft_x = torch.fft.fft(torch.fft.fft(x, dim=0), dim=1) + mag_x = torch.abs(fft_x) + pha_x = torch.angle(fft_x).detach() + + Mag = self.amp_fuse(mag_x) + Pha = self.pha_fuse(pha_x) + + amp_fuse = Mag.repeat(1, 1, 2, 2) + pha_fuse = Pha.repeat(1, 1, 2, 2) + + real = amp_fuse * torch.cos(pha_fuse) + imag = amp_fuse * torch.sin(pha_fuse) + out = torch.complex(real, imag) + + output = torch.fft.ifft(torch.fft.ifft(out, dim=0), dim=1) + output = torch.abs(output) + + return self.post(output) diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/enh.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/enh.py new file mode 100644 index 0000000000000000000000000000000000000000..e41b99ec027106a3c12ae5565980b80115baeaa2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/enh.py @@ -0,0 +1,71 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.video_deinterlace.models.archs import (DoubleConv, + DownConv, + TripleConv, + UpCatConv) +from modelscope.models.cv.video_deinterlace.models.utils import warp + + +class DeinterlaceEnh(nn.Module): + """Defines a U-Net video enhancement module + + Arg: + num_in_ch (int): Channel number of inputs. Default: 3. + num_feat (int): Channel number of base intermediate features. Default: 64. + """ + + def __init__(self, num_in_ch=3, num_feat=64): + super(DeinterlaceEnh, self).__init__() + self.channel = num_in_ch + + # extra convolutions + self.inconv2_1 = DoubleConv(num_in_ch * 3, 48) + # downsample + self.down2_0 = DownConv(48, 80) + self.down2_1 = DownConv(80, 144) + self.down2_2 = DownConv(144, 256) + self.down2_3 = DownConv(256, 448, num_conv=3) + # upsample + self.up2_3 = UpCatConv(704, 256) + self.up2_2 = UpCatConv(400, 144) + self.up2_1 = UpCatConv(224, 80) + self.up2_0 = UpCatConv(128, 48) + # extra convolutions + self.outconv2_1 = nn.Conv2d(48, num_in_ch, 3, 1, 1, bias=False) + + self.offset_conv1 = nn.Sequential( + nn.Conv2d(num_in_ch * 2, num_feat, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(num_feat, num_feat, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(num_feat, num_in_ch * 2, kernel_size=3, padding=1)) + self.offset_conv2 = nn.Sequential( + nn.Conv2d(num_in_ch * 2, num_feat, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(num_feat, num_feat, kernel_size=3, padding=1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(num_feat, num_in_ch * 2, kernel_size=3, padding=1)) + + def forward(self, frames): + frame1, frame2, frame3 = frames + flow1 = self.offset_conv1(torch.cat([frame1, frame2], 1)) + warp1 = warp(frame1, flow1) + flow3 = self.offset_conv2(torch.cat([frame3, frame2], 1)) + warp3 = warp(frame3, flow3) + x2_0 = self.inconv2_1(torch.cat((warp1, frame2, warp3), 1)) + # downsample + x2_1 = self.down2_0(x2_0) # 1/2 + x2_2 = self.down2_1(x2_1) # 1/4 + x2_3 = self.down2_2(x2_2) # 1/8 + x2_4 = self.down2_3(x2_3) # 1/16 + + x2_5 = self.up2_3(x2_4, x2_3) # 1/8 + x2_5 = self.up2_2(x2_5, x2_2) # 1/4 + x2_5 = self.up2_1(x2_5, x2_1) # 1/2 + x2_5 = self.up2_0(x2_5, x2_0) # 1 + out_final = self.outconv2_1(x2_5) + return out_final diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/fre.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/fre.py new file mode 100644 index 0000000000000000000000000000000000000000..a2144baf75d4a53f7efceef48566f5c2685582bc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/fre.py @@ -0,0 +1,93 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.video_deinterlace.models.archs import (DoubleConv, + DownConv, + TripleConv, + UpCatConv) +from modelscope.models.cv.video_deinterlace.models.deep_fourier_upsampling import \ + freup_Periodicpadding + + +class DeinterlaceFre(nn.Module): + + def __init__(self, num_in_ch=3, num_out_ch=3, ngf=64): + """Defines a video deinterlace module. + input a [b,c,h,w] tensor with range [0,1] as frame, + it will output a [b,c,h,w] tensor with range [0,1] whitout interlace. + + Args: + num_in_ch (int): Channel number of inputs. Default: 3. + num_out_ch (int): Channel number of outputs. Default: 3. + ngf(int): Channel number of features. Default: 64. + """ + super(DeinterlaceFre, self).__init__() + + self.inconv = DoubleConv(num_in_ch, 48) + self.down_0 = DownConv(48, 80) + self.down_1 = DownConv(80, 144) + + self.opfre_0 = freup_Periodicpadding(80) + self.opfre_1 = freup_Periodicpadding(144) + + self.conv_up1 = nn.Conv2d(80, ngf, 3, 1, 1) + self.conv_up2 = nn.Conv2d(144, 80, 3, 1, 1) + + self.conv_hr = nn.Conv2d(ngf, ngf, 3, 1, 1) + self.conv_last = nn.Conv2d(ngf, num_out_ch, 3, 1, 1) + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + self.enh_inconv = DoubleConv(num_in_ch + num_out_ch, 48) + # downsample + self.enh_down_0 = DownConv(48, 80) + self.enh_down_1 = DownConv(80, 144) + self.enh_down_2 = DownConv(144, 256) + self.enh_down_3 = DownConv(256, 448, num_conv=3) + # upsample + self.enh_up_3 = UpCatConv(704, 256) + self.enh_up_2 = UpCatConv(400, 144) + self.enh_up_1 = UpCatConv(224, 80) + self.enh_up_0 = UpCatConv(128, 48) + # extra convolutions + self.enh_outconv = nn.Conv2d(48, num_out_ch, 3, 1, 1, bias=False) + + def interpolate(self, feat, x2, fn): + x1f = fn(feat) + x1 = F.interpolate(feat, scale_factor=2, mode='nearest') + diffY = x2.size()[2] - x1.size()[2] + diffX = x2.size()[3] - x1.size()[3] + x1f = F.pad( + x1f, + [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2]) + x1 = F.pad( + x1, + [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2]) + return x1 + x1f + + def forward(self, x): + x1_0 = self.inconv(x) + # downsample + x1_1 = self.down_0(x1_0) # 1/2 + x1_2 = self.down_1(x1_1) # 1/4 + + feat = self.lrelu( + self.conv_up2(self.interpolate(x1_2, x1_1, self.opfre_1))) + feat = self.lrelu( + self.conv_up1(self.interpolate(feat, x1_0, self.opfre_0))) + x_new = self.conv_last(self.lrelu(self.conv_hr(feat))) + + x2_0 = self.enh_inconv(torch.cat([x_new, x], 1)) + # downsample + x2_1 = self.enh_down_0(x2_0) # 1/2 + x2_2 = self.enh_down_1(x2_1) # 1/4 + x2_3 = self.enh_down_2(x2_2) # 1/8 + x2_4 = self.enh_down_3(x2_3) # 1/16 + + x2_5 = self.enh_up_3(x2_4, x2_3) # 1/8 + x2_5 = self.enh_up_2(x2_5, x2_2) # 1/4 + x2_5 = self.enh_up_1(x2_5, x2_1) # 1/2 + x2_5 = self.enh_up_0(x2_5, x2_0) # 1 + out = self.enh_outconv(x2_5) + return out diff --git a/runtime/Lib/site_packages/models/cv/video_deinterlace/models/utils.py b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f5f1f5b269a3fb8f8195211749621e91864cb84b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_deinterlace/models/utils.py @@ -0,0 +1,107 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch + + +def warp(im, flow): + + def _repeat(x, n_repeats): + rep = torch.ones((1, n_repeats), dtype=torch.int32) + x = torch.matmul(x.view(-1, 1).int(), rep) + return x.view(-1) + + def _repeat2(x, n_repeats): + rep = torch.ones((n_repeats, 1), dtype=torch.int32) + x = torch.matmul(rep, x.view(1, -1).int()) + return x.view(-1) + + def _interpolate(im, x, y): + num_batch, channels, height, width = im.shape + + x = x.float() + y = y.float() + max_y = height - 1 + max_x = width - 1 + + x = _repeat2(torch.arange(0, width), + height * num_batch).float().cuda() + x * 64 + y = _repeat2(_repeat(torch.arange(0, height), width), + num_batch).float().cuda() + y * 64 + + # do sampling + x0 = (torch.floor(x.cpu())).int() + x1 = x0 + 1 + y0 = (torch.floor(y.cpu())).int() + y1 = y0 + 1 + + x0 = torch.clamp(x0, 0, max_x) + x1 = torch.clamp(x1, 0, max_x) + y0 = torch.clamp(y0, 0, max_y) + y1 = torch.clamp(y1, 0, max_y) + dim2 = width + dim1 = width * height + base = _repeat(torch.arange(num_batch) * dim1, height * width) + base_y0 = base + y0 * dim2 + base_y1 = base + y1 * dim2 + idx_a = base_y0 + x0 + idx_b = base_y1 + x0 + idx_c = base_y0 + x1 + idx_d = base_y1 + x1 + + # use indices to lookup pixels in the flat image and restore + im_flat = im.permute(0, 2, 3, 1) + im_flat = im_flat.reshape((-1, channels)).float() + Ia = torch.gather( + im_flat, dim=0, index=torch.unsqueeze(idx_a, 1).long().cuda()) + Ib = torch.gather(im_flat, 0, torch.unsqueeze(idx_b, 1).long().cuda()) + Ic = torch.gather(im_flat, 0, torch.unsqueeze(idx_c, 1).long().cuda()) + Id = torch.gather(im_flat, 0, torch.unsqueeze(idx_d, 1).long().cuda()) + # and finally calculate interpolated values + x0_f = x0.float().cuda() + x1_f = x1.float().cuda() + y0_f = y0.float().cuda() + y1_f = y1.float().cuda() + wa = torch.unsqueeze(((x1_f - x) * (y1_f - y)), 1) + wb = torch.unsqueeze(((x1_f - x) * (y - y0_f)), 1) + wc = torch.unsqueeze(((x - x0_f) * (y1_f - y)), 1) + wd = torch.unsqueeze(((x - x0_f) * (y - y0_f)), 1) + output = wa * Ia + wb * Ib + wc * Ic + wd * Id + return output + + def _meshgrid(height, width): + x_t = torch.matmul( + torch.ones((height, 1)), + torch.unsqueeze(torch.linspace(-0.1, 0.1, width), + 1).permute(1, 0)).cuda() + y_t = torch.matmul( + torch.unsqueeze(torch.linspace(-0.1, 0.1, height), 1), + torch.ones((1, width))).cuda() + + x_t_flat = x_t.reshape((1, -1)) + y_t_flat = y_t.reshape((1, -1)) + + ones = torch.ones_like(x_t_flat).cuda() + grid = torch.cat((x_t_flat, y_t_flat, ones), 0) + + return grid + + def _warp(x_s, y_s, input_dim): + num_batch, num_channels, height, width = input_dim.shape + # out_height, out_width = out_size + + x_s_flat = x_s.reshape(-1) + y_s_flat = y_s.reshape(-1) + + input_transformed = _interpolate(input_dim, x_s_flat, y_s_flat) + output = input_transformed.reshape( + (num_batch, num_channels, height, width)) + return output + + n_dims = int(flow.shape[1]) // 2 + dx = flow[:, :n_dims, :, :] + dy = flow[:, n_dims:, :, :] + + output = torch.cat([ + _warp(dx[:, idx:idx + 1, :, :], dy[:, idx:idx + 1, :, :], + im[:, idx:idx + 1, :, :]) for idx in range(im.shape[1]) + ], 1) + return output diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b05e314a385e8e07b50c6986bb96dd62fdc2496a --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .dro_model import DROEstimation + +else: + _import_structure = { + 'dro_model': ['DROEstimation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/default_config.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/default_config.py new file mode 100644 index 0000000000000000000000000000000000000000..5634a2d259d0ab6a41c1e151ad9815dd13576954 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/configs/default_config.py @@ -0,0 +1,216 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +"""Default dro_sfm configuration parameters (overridable in configs/*.yaml) +""" + +import os + +from yacs.config import CfgNode as CN + +######################################################################################################################## +cfg = CN() +cfg.name = '' # Run name +cfg.debug = False # Debugging flag +######################################################################################################################## +# ARCH +######################################################################################################################## +cfg.arch = CN() +cfg.arch.seed = 42 # Random seed for Pytorch/Numpy initialization +cfg.arch.min_epochs = 1 # Minimum number of epochs +cfg.arch.max_epochs = 50 # Maximum number of epochs +######################################################################################################################## +# CHECKPOINT +######################################################################################################################## +cfg.checkpoint = CN() +cfg.checkpoint.filepath = './results/mdoel' # Checkpoint filepath to save data +cfg.checkpoint.save_top_k = 5 # Number of best models to save +cfg.checkpoint.monitor = 'abs_rel_pp_gt' # Metric to monitor for logging +cfg.checkpoint.monitor_index = 0 # Dataset index for the metric to monitor +cfg.checkpoint.mode = 'auto' # Automatically determine direction of improvement (increase or decrease) +cfg.checkpoint.s3_path = '' # s3 path for AWS model syncing +cfg.checkpoint.s3_frequency = 1 # How often to s3 sync +######################################################################################################################## +# SAVE +######################################################################################################################## +cfg.save = CN() +cfg.save.folder = './results' # Folder where data will be saved +cfg.save.depth = CN() +cfg.save.depth.rgb = True # Flag for saving rgb images +cfg.save.depth.viz = True # Flag for saving inverse depth map visualization +cfg.save.depth.npz = True # Flag for saving numpy depth maps +cfg.save.depth.png = True # Flag for saving png depth maps +######################################################################################################################## +# WANDB +######################################################################################################################## +cfg.wandb = CN() +cfg.wandb.dry_run = True # Wandb dry-run (not logging) +cfg.wandb.name = '' # Wandb run name +cfg.wandb.project = os.environ.get('WANDB_PROJECT', '') # Wandb project +cfg.wandb.entity = os.environ.get('WANDB_ENTITY', '') # Wandb entity +cfg.wandb.tags = [] # Wandb tags +cfg.wandb.dir = '' # Wandb save folder +######################################################################################################################## +# MODEL +######################################################################################################################## +cfg.model = CN() +cfg.model.name = '' # Training model +cfg.model.checkpoint_path = '' # Checkpoint path for model saving +######################################################################################################################## +# MODEL.OPTIMIZER +######################################################################################################################## +cfg.model.optimizer = CN() +cfg.model.optimizer.name = 'Adam' # Optimizer name +cfg.model.optimizer.depth = CN() +cfg.model.optimizer.depth.lr = 0.0002 # Depth learning rate +cfg.model.optimizer.depth.weight_decay = 0.0 # Dept weight decay +cfg.model.optimizer.pose = CN() +cfg.model.optimizer.pose.lr = 0.0002 # Pose learning rate +cfg.model.optimizer.pose.weight_decay = 0.0 # Pose weight decay +cfg.model.optimizer.momentum = 0.9 +######################################################################################################################## +# MODEL.SCHEDULER +######################################################################################################################## +cfg.model.scheduler = CN() +cfg.model.scheduler.name = 'StepLR' # Scheduler name +cfg.model.scheduler.step_size = 10 # Scheduler step size +cfg.model.scheduler.gamma = 0.5 # Scheduler gamma value +cfg.model.scheduler.T_max = 20 # Scheduler maximum number of iterations +cfg.model.scheduler.eta_min = 1e-7 +cfg.model.scheduler.milestones = [10, 15, 20, 25, 30, 35, 40, 45] +######################################################################################################################## +# MODEL.PARAMS +######################################################################################################################## +cfg.model.params = CN() +cfg.model.params.crop = '' # Which crop should be used during evaluation +cfg.model.params.min_depth = 0.0 # Minimum depth value to evaluate +cfg.model.params.max_depth = 80.0 # Maximum depth value to evaluate +######################################################################################################################## +# MODEL.LOSS +######################################################################################################################## +cfg.model.loss = CN() +# +cfg.model.loss.num_scales = 4 # Number of inverse depth scales to use +cfg.model.loss.progressive_scaling = 0.0 # Training percentage to decay number of scales +cfg.model.loss.flip_lr_prob = 0.5 # Probablity of horizontal flippping +cfg.model.loss.rotation_mode = 'euler' # Rotation mode +cfg.model.loss.upsample_depth_maps = True # Resize depth maps to highest resolution +# +cfg.model.loss.ssim_loss_weight = 0.85 # SSIM loss weight +cfg.model.loss.occ_reg_weight = 0.1 # Occlusion regularizer loss weight +cfg.model.loss.smooth_loss_weight = 0.001 # Smoothness loss weight +cfg.model.loss.C1 = 1e-4 # SSIM parameter +cfg.model.loss.C2 = 9e-4 # SSIM parameter +cfg.model.loss.photometric_reduce_op = 'min' # Method for photometric loss reducing +cfg.model.loss.disp_norm = True # Inverse depth normalization +cfg.model.loss.clip_loss = 0.0 # Clip loss threshold variance +cfg.model.loss.padding_mode = 'zeros' # Photometric loss padding mode +cfg.model.loss.automask_loss = True # Automasking to remove static pixels +# +cfg.model.loss.velocity_loss_weight = 0.1 # Velocity supervision loss weight +# +cfg.model.loss.supervised_method = 'sparse-l1' # Method for depth supervision +cfg.model.loss.supervised_num_scales = 4 # Number of scales for supervised learning +cfg.model.loss.supervised_loss_weight = 0.9 # Supervised loss weight +######################################################################################################################## +# MODEL.DEPTH_NET +######################################################################################################################## +cfg.model.depth_net = CN() +cfg.model.depth_net.name = '' # Depth network name +cfg.model.depth_net.checkpoint_path = '' # Depth checkpoint filepath +cfg.model.depth_net.version = '' # Depth network version +cfg.model.depth_net.dropout = 0.0 # Depth network dropout +######################################################################################################################## +# MODEL.POSE_NET +######################################################################################################################## +cfg.model.pose_net = CN() +cfg.model.pose_net.name = '' # Pose network name +cfg.model.pose_net.checkpoint_path = '' # Pose checkpoint filepath +cfg.model.pose_net.version = '' # Pose network version +cfg.model.pose_net.dropout = 0.0 # Pose network dropout +######################################################################################################################## +# MODEL.perpcep_net +######################################################################################################################## +cfg.model.percep_net = CN() +cfg.model.percep_net.name = '' # percep_net network name +cfg.model.percep_net.checkpoint_path = '' # percep_net checkpoint filepath +cfg.model.percep_net.version = '' # percep_net network version +cfg.model.percep_net.dropout = 0.0 # percep_net network dropout +######################################################################################################################## +# DATASETS +######################################################################################################################## +cfg.datasets = CN() +######################################################################################################################## +# DATASETS.AUGMENTATION +######################################################################################################################## +cfg.datasets.augmentation = CN() +cfg.datasets.augmentation.image_shape = (192, 640) # Image shape +cfg.datasets.augmentation.jittering = (0.2, 0.2, 0.2, 0.05 + ) # Color jittering values +######################################################################################################################## +# DATASETS.TRAIN +######################################################################################################################## +cfg.datasets.train = CN() +cfg.datasets.train.batch_size = 8 # Training batch size +cfg.datasets.train.num_workers = 16 # Training number of workers +cfg.datasets.train.back_context = 1 # Training backward context +cfg.datasets.train.forward_context = 1 # Training forward context +cfg.datasets.train.dataset = [] # Training dataset +cfg.datasets.train.path = [] # Training data path +cfg.datasets.train.split = [] # Training split +cfg.datasets.train.depth_type = [''] # Training depth type +cfg.datasets.train.cameras = [ + [] +] # Training cameras (double list, one for each dataset) +cfg.datasets.train.repeat = [ + 1 +] # Number of times training dataset is repeated per epoch +cfg.datasets.train.num_logs = 5 # Number of training images to log +cfg.datasets.train.strides = (1, ) # stride +######################################################################################################################## +# DATASETS.VALIDATION +######################################################################################################################## +cfg.datasets.validation = CN() +cfg.datasets.validation.batch_size = 1 # Validation batch size +cfg.datasets.validation.num_workers = 8 # Validation number of workers +cfg.datasets.validation.back_context = 0 # Validation backward context +cfg.datasets.validation.forward_context = 0 # Validation forward contxt +cfg.datasets.validation.dataset = [] # Validation dataset +cfg.datasets.validation.path = [] # Validation data path +cfg.datasets.validation.split = [] # Validation split +cfg.datasets.validation.depth_type = [''] # Validation depth type +cfg.datasets.validation.cameras = [ + [] +] # Validation cameras (double list, one for each dataset) +cfg.datasets.validation.num_logs = 5 # Number of validation images to log +cfg.datasets.validation.strides = (1, ) # stride +######################################################################################################################## +# DATASETS.TEST +######################################################################################################################## +cfg.datasets.test = CN() +cfg.datasets.test.batch_size = 1 # Test batch size +cfg.datasets.test.num_workers = 8 # Test number of workers +cfg.datasets.test.back_context = 0 # Test backward context +cfg.datasets.test.forward_context = 0 # Test forward context +cfg.datasets.test.dataset = [] # Test dataset +cfg.datasets.test.path = [] # Test data path +cfg.datasets.test.split = [] # Test split +cfg.datasets.test.depth_type = [''] # Test depth type +cfg.datasets.test.cameras = [ + [] +] # Test cameras (double list, one for each dataset) +cfg.datasets.test.num_logs = 5 # Number of test images to log +cfg.datasets.test.strides = (1, ) # stride +######################################################################################################################## +# THESE SHOULD NOT BE CHANGED +######################################################################################################################## +cfg.config = '' # Run configuration file +cfg.default = '' # Run default configuration file +cfg.wandb.url = '' # Wandb URL +cfg.checkpoint.s3_url = '' # s3 URL +cfg.save.pretrained = '' # Pretrained checkpoint +cfg.prepared = False # Prepared flag +######################################################################################################################## + + +def get_cfg_defaults(): + return cfg.clone() diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/dro_model.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/dro_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ccc0d0a790906c9c4334a179ea8837f359bcc8d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/dro_model.py @@ -0,0 +1,190 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import os.path as osp +from glob import glob + +import cv2 +import numpy as np +import torch +import tqdm + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.video_depth_estimation.models.model_wrapper import \ + ModelWrapper +from modelscope.models.cv.video_depth_estimation.utils.augmentations import ( + resize_image, to_tensor) +from modelscope.models.cv.video_depth_estimation.utils.config import \ + parse_test_file +from modelscope.models.cv.video_depth_estimation.utils.depth import ( + inv2depth, viz_inv_depth, write_depth) +from modelscope.models.cv.video_depth_estimation.utils.image import ( + get_intrinsics, load_image, parse_video) +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import ModelFile, Tasks + + +@MODELS.register_module( + Tasks.video_depth_estimation, + module_name=Models.dro_resnet18_depth_estimation) +class DROEstimation(TorchModel): + + def __init__(self, model_dir: str, **kwargs): + """str -- model file root.""" + super().__init__(model_dir, **kwargs) + + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + + # Parse arguments + config, state_dict = parse_test_file(model_path) + + # If no image shape is provided, use the checkpoint one + self.image_shape = config.datasets.augmentation.image_shape + print(f'== input image shape:{self.image_shape}') + + # Initialize model wrapper from checkpoint arguments + self.model_wrapper = ModelWrapper(config, load_datasets=False) + # Restore monodepth_model state + self.model_wrapper.load_state_dict(state_dict) + + # Send model to GPU if available + if torch.cuda.is_available(): + model_wrapper = self.model_wrapper.to('cuda') + else: + model_wrapper = self.model_wrapper + print('cuda is not available, use cpu') + + # Set to eval mode + model_wrapper.eval() + + def forward(self, Inputs): + return self.model_wrapper(Inputs) + + def postprocess(self, Inputs): + return Inputs + + def inference(self, data): + + print('processing video input:.........') + input_type = 'video' + sample_rate = 1 + data_type = 'indoor' + + assert osp.splitext(data['video_path'])[1] in [ + '.mp4', '.avi', '.mov', '.mpeg', '.flv', '.wmv' + ] + input_video_images = os.path.join('tmp/input_video_images') + parse_video(data['video_path'], input_video_images, sample_rate) + # update input + input = input_video_images + + files = [] + for ext in ['png', 'jpg', 'bmp']: + files.extend(glob((os.path.join(input, '*.{}'.format(ext))))) + + if input_type == 'folder': + print('processing folder input:...........') + print(f'folder total frames num: {len(files)}') + files = files[::sample_rate] + + files.sort() + print('Found total {} files'.format(len(files))) + assert len(files) > 2 + + # Process each file + list_of_files = list(zip(files[:-2], files[1:-1], files[2:])) + + depth_list = [] + pose_list = [] + vis_depth_list = [] + depth_upsample_list = [] + vis_depth_upsample_list = [] + + print(f'*********************data_type:{data_type}') + print('inference start.....................') + for fn1, fn2, fn3 in tqdm.tqdm(list_of_files): + depth, vis_depth, depth_upsample, vis_depth_upsample, pose21, pose23, intr, rgb = self.infer_and_save_pose( + [fn1, fn3], fn2, self.model_wrapper, self.image_shape, + data_type) + pose_list.append((pose21, pose23)) + depth_list.append(depth) + vis_depth_list.append(vis_depth.astype(np.uint8)) + depth_upsample_list.append(depth_upsample) + vis_depth_upsample_list.append(vis_depth_upsample.astype(np.uint8)) + + return { + 'depths': depth_list, + 'depths_color': vis_depth_upsample_list, + 'poses': pose_list + } + + @torch.no_grad() + def infer_and_save_pose(self, input_file_refs, input_file, model_wrapper, + image_shape, data_type): + """ + Process a single input file to produce and save visualization + + Parameters + ---------- + input_file_refs : list(str) + Reference image file paths + input_file : str + Image file for pose estimation + model_wrapper : nn.Module + Model wrapper used for inference + image_shape : Image shape + Input image shape + half: bool + use half precision (fp16) + save: str + Save format (npz or png) + """ + + image_raw_wh = load_image(input_file).size + + # Load image + def process_image(filename): + image = load_image(filename) + # Resize and to tensor + intr = get_intrinsics(image.size, image_shape, data_type) # (3, 3) + image = resize_image(image, image_shape) + image = to_tensor(image).unsqueeze(0) + intr = torch.from_numpy(intr).unsqueeze(0) # (1, 3, 3) + # Send image to GPU if available + if torch.cuda.is_available(): + image = image.to('cuda') + intr = intr.to('cuda') + return image, intr + + image_ref = [ + process_image(input_file_ref)[0] + for input_file_ref in input_file_refs + ] + image, intrinsics = process_image(input_file) + + batch = { + 'rgb': image, + 'rgb_context': image_ref, + 'intrinsics': intrinsics + } + + output = self.forward(batch) + inv_depth = output['inv_depths'][0] # (1, 1, h, w) + depth = inv2depth(inv_depth)[0, 0].detach().cpu().numpy() # (h, w) + + pose21 = output['poses'][0].mat[0].detach().cpu().numpy( + ) # (4, 4) # TODO check: targe -> ref[0] + pose23 = output['poses'][1].mat[0].detach().cpu().numpy( + ) # (4, 4) # TODO check: targe -> ref[0] + + vis_depth = viz_inv_depth(inv_depth[0]) * 255 + + vis_depth_upsample = cv2.resize( + vis_depth, image_raw_wh, interpolation=cv2.INTER_LINEAR) + depth_upsample = cv2.resize( + depth, image_raw_wh, interpolation=cv2.INTER_NEAREST) + + return depth, vis_depth, depth_upsample, vis_depth_upsample, pose21, pose23, intrinsics[ + 0].detach().cpu().numpy(), image[0].permute( + 1, 2, 0).detach().cpu().numpy() * 255 diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera.py new file mode 100644 index 0000000000000000000000000000000000000000..426a02e93b7cb1a4dbc6c97833e1d1bd5248cfba --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera.py @@ -0,0 +1,191 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from functools import lru_cache + +import torch +import torch.nn as nn + +from modelscope.models.cv.video_depth_estimation.geometry.camera_utils import \ + scale_intrinsics +from modelscope.models.cv.video_depth_estimation.geometry.pose import Pose +from modelscope.models.cv.video_depth_estimation.utils.image import image_grid + + +class Camera(nn.Module): + """ + Differentiable camera class implementing reconstruction and projection + functions for a pinhole model. + """ + + def __init__(self, K, Tcw=None): + """ + Initializes the Camera class + + Parameters + ---------- + K : torch.Tensor [3,3] + Camera intrinsics + Tcw : Pose + Camera -> World pose transformation + """ + super().__init__() + self.K = K + self.Tcw = Pose.identity(len(K)) if Tcw is None else Tcw + + def __len__(self): + """Batch size of the camera intrinsics""" + return len(self.K) + + def to(self, *args, **kwargs): + """Moves object to a specific device""" + self.K = self.K.to(*args, **kwargs) + self.Tcw = self.Tcw.to(*args, **kwargs) + return self + + @property + def fx(self): + """Focal length in x""" + return self.K[:, 0, 0] + + @property + def fy(self): + """Focal length in y""" + return self.K[:, 1, 1] + + @property + def cx(self): + """Principal point in x""" + return self.K[:, 0, 2] + + @property + def cy(self): + """Principal point in y""" + return self.K[:, 1, 2] + + @property + @lru_cache() + def Twc(self): + """World -> Camera pose transformation (inverse of Tcw)""" + return self.Tcw.inverse() + + @property + @lru_cache() + def Kinv(self): + """Inverse intrinsics (for lifting)""" + Kinv = self.K.clone() + Kinv[:, 0, 0] = 1. / self.fx + Kinv[:, 1, 1] = 1. / self.fy + Kinv[:, 0, 2] = -1. * self.cx / self.fx + Kinv[:, 1, 2] = -1. * self.cy / self.fy + return Kinv + + def scaled(self, x_scale, y_scale=None): + """ + Returns a scaled version of the camera (changing intrinsics) + + Parameters + ---------- + x_scale : float + Resize scale in x + y_scale : float + Resize scale in y. If None, use the same as x_scale + + Returns + ------- + camera : Camera + Scaled version of the current cmaera + """ + # If single value is provided, use for both dimensions + if y_scale is None: + y_scale = x_scale + # If no scaling is necessary, return same camera + if x_scale == 1. and y_scale == 1.: + return self + # Scale intrinsics and return new camera with same Pose + K = scale_intrinsics(self.K.clone(), x_scale, y_scale) + return Camera(K, Tcw=self.Tcw) + + def reconstruct(self, depth, frame='w'): + """ + Reconstructs pixel-wise 3D points from a depth map. + + Parameters + ---------- + depth : torch.Tensor [B,1,H,W] + Depth map for the camera + frame : 'w' + Reference frame: 'c' for camera and 'w' for world + + Returns + ------- + points : torch.tensor [B,3,H,W] + Pixel-wise 3D points + """ + B, C, H, W = depth.shape + assert C == 1 + + # Create flat index grid + grid = image_grid( + B, H, W, depth.dtype, depth.device, normalized=False) # [B,3,H,W] + flat_grid = grid.view(B, 3, -1) # [B,3,HW] + + # Estimate the outward rays in the camera frame + xnorm = (self.Kinv.bmm(flat_grid)).view(B, 3, H, W) + # Scale rays to metric depth + Xc = xnorm * depth + + # If in camera frame of reference + if frame == 'c': + return Xc + # If in world frame of reference + elif frame == 'w': + return self.Twc @ Xc + # If none of the above + else: + raise ValueError('Unknown reference frame {}'.format(frame)) + + def project(self, X, frame='w', normalize=True): + """ + Projects 3D points onto the image plane + + Parameters + ---------- + X : torch.Tensor [B,3,H,W] + 3D points to be projected + frame : 'w' + Reference frame: 'c' for camera and 'w' for world + + Returns + ------- + points : torch.Tensor [B,H,W,2] + 2D projected points that are within the image boundaries + """ + B, C, H, W = X.shape + assert C == 3 + + # Project 3D points onto the camera image plane + if frame == 'c': + Xc = self.K.bmm(X.view(B, 3, -1)) + elif frame == 'w': + Xc = self.K.bmm((self.Tcw @ X).view(B, 3, -1)) + else: + raise ValueError('Unknown reference frame {}'.format(frame)) + + # Normalize points + X = Xc[:, 0] + Y = Xc[:, 1] + Z = Xc[:, 2].clamp(min=1e-5) + if normalize: + Xnorm = 2 * (X / Z) / (W - 1) - 1. # (-1, 1) + Ynorm = 2 * (Y / Z) / (H - 1) - 1. + else: + Xnorm = X / Z + Ynorm = Y / Z + + # Clamp out-of-bounds pixels + # Xmask = ((Xnorm > 1) + (Xnorm < -1)).detach() + # Xnorm[Xmask] = 2. + # Ymask = ((Ynorm > 1) + (Ynorm < -1)).detach() + # Ynorm[Ymask] = 2. + + # Return pixel coordinates + return torch.stack([Xnorm, Ynorm], dim=-1).view(B, H, W, 2) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera_utils.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..08c9d45427f25f9de7bd6ba131467638a54a2159 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/camera_utils.py @@ -0,0 +1,71 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn.functional as funct + +######################################################################################################################## + + +def construct_K(fx, fy, cx, cy, dtype=torch.float, device=None): + """Construct a [3,3] camera intrinsics from pinhole parameters""" + return torch.tensor([[fx, 0, cx], [0, fy, cy], [0, 0, 1]], + dtype=dtype, + device=device) + + +def scale_intrinsics(K, x_scale, y_scale): + """Scale intrinsics given x_scale and y_scale factors""" + K[..., 0, 0] *= x_scale + K[..., 1, 1] *= y_scale + K[..., 0, 2] = (K[..., 0, 2] + 0.5) * x_scale - 0.5 + K[..., 1, 2] = (K[..., 1, 2] + 0.5) * y_scale - 0.5 + return K + + +######################################################################################################################## + + +def view_synthesis(ref_image, + depth, + ref_cam, + cam, + mode='bilinear', + padding_mode='zeros'): + """ + Synthesize an image from another plus a depth map. + + Parameters + ---------- + ref_image : torch.Tensor [B,3,H,W] + Reference image to be warped + depth : torch.Tensor [B,1,H,W] + Depth map from the original image + ref_cam : Camera + Camera class for the reference image + cam : Camera + Camera class for the original image + mode : str + Interpolation mode + padding_mode : str + Padding mode for interpolation + + Returns + ------- + ref_warped : torch.Tensor [B,3,H,W] + Warped reference image in the original frame of reference + """ + assert depth.size(1) == 1 + # Reconstruct world points from target_camera + world_points = cam.reconstruct(depth, frame='w') + # Project world points onto reference camera + ref_coords = ref_cam.project(world_points, frame='w') + + # View-synthesis given the projected reference points + return funct.grid_sample( + ref_image, + ref_coords, + mode=mode, + padding_mode=padding_mode, + align_corners=True) + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose.py new file mode 100644 index 0000000000000000000000000000000000000000..ccb737829dae4a06c26711cc78019b9dbae82458 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose.py @@ -0,0 +1,107 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch + +from modelscope.models.cv.video_depth_estimation.geometry.pose_utils import ( + invert_pose, pose_vec2mat) + +######################################################################################################################## + + +class Pose: + """ + Pose class, that encapsulates a [4,4] transformation matrix + for a specific reference frame + """ + + def __init__(self, mat): + """ + Initializes a Pose object. + + Parameters + ---------- + mat : torch.Tensor [B,4,4] + Transformation matrix + """ + assert tuple(mat.shape[-2:]) == (4, 4) + if mat.dim() == 2: + mat = mat.unsqueeze(0) + assert mat.dim() == 3 + self.mat = mat + + def __len__(self): + """Batch size of the transformation matrix""" + return len(self.mat) + +######################################################################################################################## + + @classmethod + def identity(cls, N=1, device=None, dtype=torch.float): + """Initializes as a [4,4] identity matrix""" + return cls(torch.eye(4, device=device, dtype=dtype).repeat([N, 1, 1])) + + @classmethod + def from_vec(cls, vec, mode): + """Initializes from a [B,6] batch vector""" + mat = pose_vec2mat(vec, mode) # [B,3,4] + pose = torch.eye( + 4, device=vec.device, dtype=vec.dtype).repeat([len(vec), 1, 1]) + pose[:, :3, :3] = mat[:, :3, :3] + pose[:, :3, -1] = mat[:, :3, -1] + return cls(pose) + +######################################################################################################################## + + @property + def shape(self): + """Returns the transformation matrix shape""" + return self.mat.shape + + def item(self): + """Returns the transformation matrix""" + return self.mat + + def repeat(self, *args, **kwargs): + """Repeats the transformation matrix multiple times""" + self.mat = self.mat.repeat(*args, **kwargs) + return self + + def inverse(self): + """Returns a new Pose that is the inverse of this one""" + return Pose(invert_pose(self.mat)) + + def to(self, *args, **kwargs): + """Moves object to a specific device""" + self.mat = self.mat.to(*args, **kwargs) + return self + +######################################################################################################################## + + def transform_pose(self, pose): + """Creates a new pose object that compounds this and another one (self * pose)""" + assert tuple(pose.shape[-2:]) == (4, 4) + return Pose(self.mat.bmm(pose.item())) + + def transform_points(self, points): + """Transforms 3D points using this object""" + assert points.shape[1] == 3 + B, _, H, W = points.shape + out = self.mat[:, :3, :3].bmm(points.view(B, 3, -1)) + \ + self.mat[:, :3, -1].unsqueeze(-1) + return out.view(B, 3, H, W) + + def __matmul__(self, other): + """Transforms the input (Pose or 3D points) using this object""" + if isinstance(other, Pose): + return self.transform_pose(other) + elif isinstance(other, torch.Tensor): + if other.shape[1] == 3 and other.dim() > 2: + assert other.dim() == 3 or other.dim() == 4 + return self.transform_points(other) + else: + raise ValueError('Unknown tensor dimensions {}'.format( + other.shape)) + else: + raise NotImplementedError() + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose_utils.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d58105db3f64721c1e116a97d0ebed29915fc8fb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/geometry/pose_utils.py @@ -0,0 +1,115 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +from torch._C import dtype + + +def mat2euler(mat): + euler = torch.ones(mat.shape[0], 3, dtype=mat.dtype, device=mat.device) + cy_thresh = 1e-6 + # try: + # cy_thresh = np.finfo(mat.dtype).eps * 4 + # except ValueError: + # cy_thresh = np.finfo(np.float).eps * 4.0 + # print("cy_thresh", cy_thresh) + r11, r12, r13, r21, r22, r23, _, _, r33 = mat[:, 0, 0], mat[:, 0, 1], mat[:, 0, 2], \ + mat[:, 1, 0], mat[:, 1, 1], mat[:, 1, 2], \ + mat[:, 2, 0], mat[:, 2, 1], mat[:, 2, 2] + # cy: sqrt((cos(y)*cos(z))**2 + (cos(x)*cos(y))**2) + cy = torch.sqrt(r33 * r33 + r23 * r23) + + mask = cy > cy_thresh + + if torch.sum(mask) > 1: + euler[mask, 0] = torch.atan2(-r23, r33)[mask] + euler[mask, 1] = torch.atan2(r13, cy)[mask] + euler[mask, 2] = torch.atan2(-r12, r11)[mask] + + mask = cy <= cy_thresh + if torch.sum(mask) > 1: + print('mat2euler!!!!!!') + euler[mask, 0] = 0.0 + euler[mask, 1] = torch.atan2(r13, cy) # atan2(sin(y), cy) + euler[mask, 2] = torch.atan2(r21, r22) + + return euler + + +######################################################################################################################## + + +def euler2mat(angle): + """Convert euler angles to rotation matrix""" + B = angle.size(0) + x, y, z = angle[:, 0], angle[:, 1], angle[:, 2] + + cosz = torch.cos(z) + sinz = torch.sin(z) + + zeros = z.detach() * 0 + ones = zeros.detach() + 1 + zmat = torch.stack( + [cosz, -sinz, zeros, sinz, cosz, zeros, zeros, zeros, ones], + dim=1).view(B, 3, 3) + + cosy = torch.cos(y) + siny = torch.sin(y) + + ymat = torch.stack( + [cosy, zeros, siny, zeros, ones, zeros, -siny, zeros, cosy], + dim=1).view(B, 3, 3) + + cosx = torch.cos(x) + sinx = torch.sin(x) + + xmat = torch.stack( + [ones, zeros, zeros, zeros, cosx, -sinx, zeros, sinx, cosx], + dim=1).view(B, 3, 3) + + rot_mat = xmat.bmm(ymat).bmm(zmat) + return rot_mat + + +######################################################################################################################## + + +def pose_vec2mat(vec, mode='euler'): + """Convert Euler parameters to transformation matrix.""" + if mode is None: + return vec + trans, rot = vec[:, :3].unsqueeze(-1), vec[:, 3:] + if mode == 'euler': + rot_mat = euler2mat(rot) + elif mode == 'axis_angle': + from modelscope.models.cv.video_depth_estimation.geometry.pose_trans import axis_angle_to_matrix + rot_mat = axis_angle_to_matrix(rot) + else: + raise ValueError('Rotation mode not supported {}'.format(mode)) + mat = torch.cat([rot_mat, trans], dim=2) # [B,3,4] + return mat + + +######################################################################################################################## + + +def invert_pose(T): + """Inverts a [B,4,4] torch.tensor pose""" + Tinv = torch.eye(4, device=T.device, dtype=T.dtype).repeat([len(T), 1, 1]) + Tinv[:, :3, :3] = torch.transpose(T[:, :3, :3], -2, -1) + Tinv[:, :3, -1] = torch.bmm(-1. * Tinv[:, :3, :3], + T[:, :3, -1].unsqueeze(-1)).squeeze(-1) + return Tinv + + +######################################################################################################################## + + +def invert_pose_numpy(T): + """Inverts a [4,4] np.array pose""" + Tinv = np.copy(T) + R, t = Tinv[:3, :3], Tinv[:3, 3] + Tinv[:3, :3], Tinv[:3, 3] = R.T, -np.matmul(R.T, t) + return Tinv + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_checkpoint.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..761990ff0047a3e6d2b8c7bcf08ccb6700d4be2e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_checkpoint.py @@ -0,0 +1,144 @@ +# The implementation is adopted from Pytorch-Lightning +# https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pytorch_lightning/callbacks/model_checkpoint.py + +import os +import re + +import numpy as np +import torch + + +def save_code(filepath): + """Save code in the models folder""" + os.system('tar cfz {}/code.tar.gz *'.format(filepath)) + + +class ModelCheckpoint: + + def __init__(self, + filepath=None, + monitor='val_loss', + save_top_k=1, + mode='auto', + period=1, + s3_path='', + s3_frequency=5): + super().__init__() + # If save_top_k is zero, save all models + if save_top_k == 0: + save_top_k = 1e6 + # Create checkpoint folder + self.dirpath, self.filename = os.path.split(filepath) + print(self.dirpath, self.filename, filepath) + os.makedirs(self.dirpath, exist_ok=True) + # Store arguments + self.monitor = monitor + self.save_top_k = save_top_k + self.period = period + self.epoch_last_check = None + self.best_k_models = {} + self.kth_best_model = '' + self.best = 0 + # Monitoring modes + torch_inf = torch.tensor(np.Inf) + mode_dict = { + 'min': (torch_inf, 'min'), + 'max': (-torch_inf, 'max'), + 'auto': (-torch_inf, 'max') if 'acc' in self.monitor + or 'a1' in self.monitor or self.monitor.startswith('fmeasure') else + (torch_inf, 'min'), + } + self.kth_value, self.mode = mode_dict[mode] + + self.s3_path = s3_path + self.s3_frequency = s3_frequency + self.s3_enabled = (s3_path != '') and (s3_frequency > 0) + self.save_code = True + + @staticmethod + def _del_model(filepath): + if os.path.isfile(filepath): + os.remove(filepath) + + def _save_model(self, filepath, model): + # Create folder, save model and sync to s3 + os.makedirs(os.path.dirname(filepath), exist_ok=True) + torch.save( + { + 'config': model.config, + 'epoch': model.current_epoch, + 'state_dict': model.state_dict(), + 'optimizer': model.optimizer.state_dict(), + 'scheduler': model.scheduler.state_dict(), + }, filepath) + + def check_monitor_top_k(self, current): + # If we don't have enough models + if len(self.best_k_models) < self.save_top_k: + return True + # Convert to torch if necessary + if not isinstance(current, torch.Tensor): + current = torch.tensor(current) + # Get monitoring operation + monitor_op = { + 'min': torch.lt, + 'max': torch.gt, + }[self.mode] + # Compare and return + return monitor_op(current, self.best_k_models[self.kth_best_model]) + + def format_checkpoint_name(self, epoch, metrics): + metrics['epoch'] = epoch + filename = self.filename + for tmp in re.findall(r'(\{.*?)[:\}]', self.filename): + name = tmp[1:] + filename = filename.replace(tmp, name + '={' + name) + if name not in metrics: + metrics[name] = 0 + filename = filename.format(**metrics) + return os.path.join(self.dirpath, '{}.ckpt'.format(filename)) + + def check_and_save(self, model, metrics): + # Check saving interval + epoch = model.current_epoch + if self.epoch_last_check is not None and \ + (epoch - self.epoch_last_check) < self.period: + return + self.epoch_last_check = epoch + # Prepare filepath + filepath = self.format_checkpoint_name(epoch, metrics) + while os.path.isfile(filepath): + filepath = self.format_checkpoint_name(epoch, metrics) + # Check if saving or not + if self.save_top_k != -1: + current = metrics.get(self.monitor) + assert current, 'Checkpoint metric is not available' + if self.check_monitor_top_k(current): + self._do_check_save(filepath, model, current) + else: + self._save_model(filepath, model) + + def _do_check_save(self, filepath, model, current): + # List of models to delete + del_list = [] + if len(self.best_k_models) == self.save_top_k and self.save_top_k > 0: + delpath = self.kth_best_model + self.best_k_models.pop(self.kth_best_model) + del_list.append(delpath) + # Monitor current models + self.best_k_models[filepath] = current + if len(self.best_k_models) == self.save_top_k: + # Monitor dict has reached k elements + _op = max if self.mode == 'min' else min + self.kth_best_model = _op( + self.best_k_models, key=self.best_k_models.get) + self.kth_value = self.best_k_models[self.kth_best_model] + # Determine best model + _op = min if self.mode == 'min' else max + self.best = _op(self.best_k_models.values()) + # Delete old models + for cur_path in del_list: + if cur_path != filepath: + self._del_model(cur_path) + # Save model + self._save_model(filepath, model) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_utils.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3c839a514fed2226b0d82a65eed2c6ac561222cf --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_utils.py @@ -0,0 +1,70 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +from modelscope.models.cv.video_depth_estimation.utils.types import (is_list, + is_numpy, + is_tensor) + + +def merge_outputs(*outputs): + """ + Merges model outputs for logging + + Parameters + ---------- + outputs : tuple of dict + Outputs to be merged + + Returns + ------- + output : dict + Dictionary with a "metrics" key containing a dictionary with various metrics and + all other keys that are not "loss" (it is handled differently). + """ + ignore = ['loss'] # Keys to ignore + combine = ['metrics'] # Keys to combine + merge = {key: {} for key in combine} + for output in outputs: + # Iterate over all keys + for key, val in output.items(): + # Combine these keys + if key in combine: + for sub_key, sub_val in output[key].items(): + assert sub_key not in merge[key].keys(), \ + 'Combining duplicated key {} to {}'.format(sub_key, key) + merge[key][sub_key] = sub_val + # Ignore these keys + elif key not in ignore: + assert key not in merge.keys(), \ + 'Adding duplicated key {}'.format(key) + merge[key] = val + return merge + + +def stack_batch(batch): + """ + Stack multi-camera batches (B,N,C,H,W becomes BN,C,H,W) + + Parameters + ---------- + batch : dict + Batch + + Returns + ------- + batch : dict + Stacked batch + """ + # If there is multi-camera information + if len(batch['rgb'].shape) == 5: + assert batch['rgb'].shape[ + 0] == 1, 'Only batch size 1 is supported for multi-cameras' + # Loop over all keys + for key in batch.keys(): + # If list, stack every item + if is_list(batch[key]): + if is_tensor(batch[key][0]) or is_numpy(batch[key][0]): + batch[key] = [sample[0] for sample in batch[key]] + # Else, stack single item + else: + batch[key] = batch[key][0] + return batch diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_wrapper.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..c274c48d59cec6cfd0e43dee1e57c4b8d8ea164e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/model_wrapper.py @@ -0,0 +1,319 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import importlib +import random +from collections import OrderedDict + +import numpy as np +import torch + +from modelscope.models.cv.video_depth_estimation.utils.load import ( + filter_args, load_class, load_class_args_create, load_network) +from modelscope.models.cv.video_depth_estimation.utils.misc import pcolor + + +class ModelWrapper(torch.nn.Module): + """ + Top-level torch.nn.Module wrapper around a SfmModel (pose+depth networks). + Designed to use models with high-level Trainer classes (cf. trainers/). + + Parameters + ---------- + config : CfgNode + Model configuration (cf. configs/default_config.py) + """ + + def __init__(self, config, resume=None, logger=None, load_datasets=True): + super().__init__() + + # Store configuration, checkpoint and logger + self.config = config + self.logger = logger + self.resume = resume + + # Set random seed + set_random_seed(config.arch.seed) + + # Task metrics + self.metrics_name = 'depth' + self.metrics_keys = ('abs_rel', 'sqr_rel', 'rmse', 'rmse_log', 'a1', + 'a2', 'a3', 'SILog', 'l1_inv', 'rot_ang', 't_ang', + 't_cm') + self.metrics_modes = ('', '_pp', '_gt', '_pp_gt') + + # Model, optimizers, schedulers and datasets are None for now + self.model = self.optimizer = self.scheduler = None + self.train_dataset = self.validation_dataset = self.test_dataset = None + self.current_epoch = 0 + + # Prepare model + self.prepare_model(resume) + + # Preparations done + self.config.prepared = True + + def prepare_model(self, resume=None): + """Prepare self.model (incl. loading previous state)""" + print0(pcolor('### Preparing Model', 'green')) + self.model = setup_model(self.config.model, self.config.prepared) + # Resume model if available + if resume: + print0( + pcolor( + '### Resuming from {}'.format(resume['file']), + 'magenta', + attrs=['bold'])) + self.model = load_network(self.model, resume['state_dict'], + 'model') + if 'epoch' in resume: + self.current_epoch = resume['epoch'] + + @property + def depth_net(self): + """Returns depth network.""" + return self.model.depth_net + + @property + def pose_net(self): + """Returns pose network.""" + return self.model.pose_net + + @property + def percep_net(self): + """Returns perceptual network.""" + return self.model.percep_net + + @property + def logs(self): + """Returns various logs for tracking.""" + params = OrderedDict() + for param in self.optimizer.param_groups: + params['{}_learning_rate'.format( + param['name'].lower())] = param['lr'] + params['progress'] = self.progress + return { + **params, + **self.model.logs, + } + + @property + def progress(self): + """Returns training progress (current epoch / max. number of epochs)""" + return self.current_epoch / self.config.arch.max_epochs + + def configure_optimizers(self): + """Configure depth and pose optimizers and the corresponding scheduler.""" + + params = [] + # Load optimizer + optimizer = getattr(torch.optim, self.config.model.optimizer.name) + # Depth optimizer + if self.depth_net is not None: + params.append({ + 'name': + 'Depth', + 'params': + self.depth_net.parameters(), + **filter_args(optimizer, self.config.model.optimizer.depth) + }) + # Pose optimizer + if self.pose_net is not None: + params.append({ + 'name': + 'Pose', + 'params': [ + param for param in self.pose_net.parameters() + if param.requires_grad + ], + **filter_args(optimizer, self.config.model.optimizer.pose) + }) + # Create optimizer with parameters + optimizer = optimizer(params) + + # Load and initialize scheduler + scheduler = getattr(torch.optim.lr_scheduler, + self.config.model.scheduler.name) + scheduler = scheduler( + optimizer, **filter_args(scheduler, self.config.model.scheduler)) + + # Create class variables so we can use it internally + self.optimizer = optimizer + self.scheduler = scheduler + + # Return optimizer and scheduler + return optimizer, scheduler + + def forward(self, *args, **kwargs): + """Runs the model and returns the output.""" + assert self.model is not None, 'Model not defined' + return self.model(*args, **kwargs) + + def depth(self, *args, **kwargs): + """Runs the pose network and returns the output.""" + assert self.depth_net is not None, 'Depth network not defined' + return self.depth_net(*args, **kwargs) + + def pose(self, *args, **kwargs): + """Runs the depth network and returns the output.""" + assert self.pose_net is not None, 'Pose network not defined' + return self.pose_net(*args, **kwargs) + + def percep(self, *args, **kwargs): + """Runs the depth network and returns the output.""" + assert self.percep_net is not None, 'Perceptual network not defined' + return self.percep_net(*args, **kwargs) + + +def set_random_seed(seed): + if seed >= 0: + np.random.seed(seed) + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def setup_depth_net(config, prepared, **kwargs): + """ + Create a depth network + + Parameters + ---------- + config : CfgNode + Network configuration + prepared : bool + True if the network has been prepared before + kwargs : dict + Extra parameters for the network + + Returns + ------- + depth_net : nn.Module + Create depth network + """ + print0(pcolor('DepthNet: %s' % config.name, 'yellow')) + if config.name == 'DepthPoseNet': + model_class = getattr( + importlib.import_module( + 'modelscope.models.cv.video_depth_estimation.networks.depth_pose.depth_pose_net' + ), 'DepthPoseNet') + depth_net = model_class(**{**config, **kwargs}) + if not prepared and config.checkpoint_path != '': + depth_net = load_network(depth_net, config.checkpoint_path, + ['depth_net', 'disp_network']) + return depth_net + + +def setup_pose_net(config, prepared, **kwargs): + """ + Create a pose network + + Parameters + ---------- + config : CfgNode + Network configuration + prepared : bool + True if the network has been prepared before + kwargs : dict + Extra parameters for the network + + Returns + ------- + pose_net : nn.Module + Created pose network + """ + print0(pcolor('PoseNet: %s' % config.name, 'yellow')) + pose_net = load_class_args_create( + config.name, + paths=[ + 'modelscope.models.cv.video_depth_estimation.networks.pose', + ], + args={ + **config, + **kwargs + }, + ) + if not prepared and config.checkpoint_path != '': + pose_net = load_network(pose_net, config.checkpoint_path, + ['pose_net', 'pose_network']) + return pose_net + + +def setup_percep_net(config, prepared, **kwargs): + """ + Create a perceputal network + + Parameters + ---------- + config : CfgNode + Network configuration + prepared : bool + True if the network has been prepared before + kwargs : dict + Extra parameters for the network + + Returns + ------- + depth_net : nn.Module + Create depth network + """ + print0(pcolor('PercepNet: %s' % config.name, 'yellow')) + percep_net = load_class_args_create( + config.name, + paths=[ + 'modelscope.models.cv.video_depth_estimation.networks.layers', + ], + args={ + **config, + **kwargs + }, + ) + return percep_net + + +def setup_model(config, prepared, **kwargs): + """ + Create a model + + Parameters + ---------- + config : CfgNode + Model configuration (cf. configs/default_config.py) + prepared : bool + True if the model has been prepared before + kwargs : dict + Extra parameters for the model + + Returns + ------- + model : nn.Module + Created model + """ + print0(pcolor('Model: %s' % config.name, 'yellow')) + config.loss.min_depth = config.params.min_depth + config.loss.max_depth = config.params.max_depth + if config.name == 'SupModelMF': + model_class = getattr( + importlib.import_module( + 'modelscope.models.cv.video_depth_estimation.models.sup_model_mf' + ), 'SupModelMF') + model = model_class(**{**config.loss, **kwargs}) + # Add depth network if required + if model.network_requirements['depth_net']: + config.depth_net.max_depth = config.params.max_depth + config.depth_net.min_depth = config.params.min_depth + model.add_depth_net(setup_depth_net(config.depth_net, prepared)) + # Add pose network if required + if model.network_requirements['pose_net']: + model.add_pose_net(setup_pose_net(config.pose_net, prepared)) + # Add percep_net if required + if model.network_requirements['percep_net']: + model.add_percep_net(setup_percep_net(config.percep_net, prepared)) + # If a checkpoint is provided, load pretrained model + if not prepared and config.checkpoint_path != '': + model = load_network(model, config.checkpoint_path, 'model') + # Return model + return model + + +def print0(string='\n'): + print(string) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sfm_model_mf.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sfm_model_mf.py new file mode 100644 index 0000000000000000000000000000000000000000..bab1119636373e1fe3fdc3402efeeafb6a2c622e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sfm_model_mf.py @@ -0,0 +1,200 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import random + +import torch.nn as nn + +from modelscope.models.cv.video_depth_estimation.geometry.pose import Pose +from modelscope.models.cv.video_depth_estimation.utils.image import \ + flip_lr as flip_lr_img +from modelscope.models.cv.video_depth_estimation.utils.image import ( + flip_lr_intr, flip_mf_model, interpolate_scales) +from modelscope.models.cv.video_depth_estimation.utils.misc import make_list + + +class SfmModelMF(nn.Module): + """ + Model class encapsulating a pose and depth networks. + + Parameters + ---------- + depth_net : nn.Module + Depth network to be used + pose_net : nn.Module + Pose network to be used + rotation_mode : str + Rotation mode for the pose network + flip_lr_prob : float + Probability of flipping when using the depth network + upsample_depth_maps : bool + True if depth map scales are upsampled to highest resolution + kwargs : dict + Extra parameters + """ + + def __init__(self, + depth_net=None, + pose_net=None, + rotation_mode='euler', + flip_lr_prob=0.0, + upsample_depth_maps=False, + min_depth=0.1, + max_depth=100, + **kwargs): + super().__init__() + self.depth_net = depth_net + self.pose_net = pose_net + self.rotation_mode = rotation_mode + self.flip_lr_prob = flip_lr_prob + self.upsample_depth_maps = upsample_depth_maps + self.min_depth = min_depth + self.max_depth = max_depth + self._logs = {} + self._losses = {} + self._network_requirements = { + 'depth_net': True, # Depth network required + 'pose_net': False, # Pose network required + 'percep_net': False, # Pose network required + } + self._train_requirements = { + 'gt_depth': False, # No ground-truth depth required + 'gt_pose': False, # No ground-truth pose required + } + + @property + def logs(self): + """Return logs.""" + return self._logs + + @property + def losses(self): + """Return metrics.""" + return self._losses + + def add_loss(self, key, val): + """Add a new loss to the dictionary and detaches it.""" + self._losses[key] = val.detach() + + @property + def network_requirements(self): + """ + Networks required to run the model + + Returns + ------- + requirements : dict + depth_net : bool + Whether a depth network is required by the model + pose_net : bool + Whether a depth network is required by the model + """ + return self._network_requirements + + @property + def train_requirements(self): + """ + Information required by the model at training stage + + Returns + ------- + requirements : dict + gt_depth : bool + Whether ground truth depth is required by the model at training time + gt_pose : bool + Whether ground truth pose is required by the model at training time + """ + return self._train_requirements + + def add_depth_net(self, depth_net): + """Add a depth network to the model""" + self.depth_net = depth_net + + def add_pose_net(self, pose_net): + """Add a pose network to the model""" + self.pose_net = pose_net + + def compute_inv_depths(self, image, ref_imgs, intrinsics): + """Computes inverse depth maps from single images""" + # Randomly flip and estimate inverse depth maps + flip_lr = random.random( + ) < self.flip_lr_prob if self.training else False + if flip_lr: + intrinsics = flip_lr_intr(intrinsics, width=image.shape[3]) + inv_depths_with_poses = flip_mf_model(self.depth_net, image, ref_imgs, + intrinsics, flip_lr) + inv_depths, poses = inv_depths_with_poses + inv_depths = make_list(inv_depths) + if flip_lr: + inv_depths = [flip_lr_img(inv_d) for inv_d in inv_depths] + # If upsampling depth maps + if self.upsample_depth_maps: + inv_depths = interpolate_scales( + inv_depths, mode='nearest', align_corners=None) + # Return inverse depth maps + return inv_depths, poses + + def compute_poses(self, image, contexts, intrinsics, depth): + """Compute poses from image and a sequence of context images""" + pose_vec = self.pose_net(image, contexts, intrinsics, depth) + if pose_vec is None: + return None + if pose_vec.shape[2] == 6: + return [ + Pose.from_vec(pose_vec[:, i], self.rotation_mode) + for i in range(pose_vec.shape[1]) + ] + else: + return [Pose(pose_vec[:, i]) for i in range(pose_vec.shape[1])] + + def forward(self, batch, return_logs=False): + """ + Processes a batch. + + Parameters + ---------- + batch : dict + Input batch + return_logs : bool + True if logs are stored + + Returns + ------- + output : dict + Dictionary containing predicted inverse depth maps and poses + """ + # Generate inverse depth predictions + inv_depths, pose_vec = self.compute_inv_depths(batch['rgb'], + batch['rgb_context'], + batch['intrinsics']) + # # Generate pose predictions if available + # pose = None + # if 'rgb_context' in batch and self.pose_net is not None: + # pose = self.compute_poses(batch['rgb'], + # batch['rgb_context'], batch["intrinsics"], inv2depth(inv_depths[0])) + # Return output dictionary + if pose_vec.shape[2] == 6: + poses = [ + Pose.from_vec(pose_vec[:, i], self.rotation_mode) + for i in range(pose_vec.shape[1]) + ] + elif (pose_vec.shape[2]) == 4 and (pose_vec.shape[3] == 4): + poses = [Pose(pose_vec[:, i]) for i in range(pose_vec.shape[1])] + else: + # pose_vec shape: (b, n_view, n_iter, 6) + poses = [] + for i in range(pose_vec.shape[1]): + poses_view = [] + for j in range(pose_vec.shape[2]): + poses_view.append( + Pose.from_vec(pose_vec[:, i, j], self.rotation_mode)) + poses.append( + poses_view + ) # ([pose_view1, pose_view2, ....]) each view has n_iter pose + + # print(poses[0][-1].shape, len(poses), len(poses[0]), len(inv_depths), inv_depths[0].shape) + # print(poses[0][-1].mat[0], inv2depth(inv_depths)[-1][0, 0, 12, 40]) + # print("gt", batch["pose_context"][0][0]) + return { + 'inv_depths': inv_depths, + 'poses': poses, + } diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sup_model_mf.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sup_model_mf.py new file mode 100644 index 0000000000000000000000000000000000000000..ba013dcb02eae5f371f6f7f7a9409d5bc14f5c65 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/models/sup_model_mf.py @@ -0,0 +1,134 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +from modelscope.models.cv.video_depth_estimation.models.model_utils import \ + merge_outputs +from modelscope.models.cv.video_depth_estimation.models.sfm_model_mf import \ + SfmModelMF +from modelscope.models.cv.video_depth_estimation.utils.depth import depth2inv + + +class SupModelMF(SfmModelMF): + """ + Model that inherits a depth and pose network from SfmModel and + includes the photometric loss for self-supervised training. + + Parameters + ---------- + kwargs : dict + Extra parameters + """ + + def __init__(self, **kwargs): + # Initializes SfmModel + super().__init__(**kwargs) + # Initializes the photometric loss + + self._network_requirements = { + 'depth_net': True, # Depth network required + 'pose_net': False, # Pose network required + 'percep_net': False, # Pose network required + } + + self._train_requirements = { + 'gt_depth': True, # No ground-truth depth required + 'gt_pose': True, # No ground-truth pose required + } + + # self._photometric_loss = MultiViewPhotometricLoss(**kwargs) + # self._loss = SupervisedDepthPoseLoss(**kwargs) + + @property + def logs(self): + """Return logs.""" + return {**super().logs, **self._photometric_loss.logs} + + def supervised_loss(self, + image, + ref_images, + inv_depths, + gt_depth, + gt_poses, + poses, + intrinsics, + return_logs=False, + progress=0.0): + """ + Calculates the self-supervised photometric loss. + + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Original image + ref_images : list of torch.Tensor [B,3,H,W] + Reference images from context + inv_depths : torch.Tensor [B,1,H,W] + Predicted inverse depth maps from the original image + poses : list of Pose + List containing predicted poses between original and context images + intrinsics : torch.Tensor [B,3,3] + Camera intrinsics + return_logs : bool + True if logs are stored + progress : + Training progress percentage + + Returns + ------- + output : dict + Dictionary containing a "loss" scalar a "metrics" dictionary + """ + return self._loss( + image, + ref_images, + inv_depths, + depth2inv(gt_depth), + gt_poses, + intrinsics, + intrinsics, + poses, + return_logs=return_logs, + progress=progress) + + def forward(self, batch, return_logs=False, progress=0.0): + """ + Processes a batch. + + Parameters + ---------- + batch : dict + Input batch + return_logs : bool + True if logs are stored + progress : + Training progress percentage + + Returns + ------- + output : dict + Dictionary containing a "loss" scalar and different metrics and predictions + for logging and downstream usage. + """ + # Calculate predicted depth and pose output + output = super().forward(batch, return_logs=return_logs) + if not self.training: + # If not training, no need for self-supervised loss + return output + else: + if output['poses'] is None: + return None + # Otherwise, calculate self-supervised loss + self_sup_output = self.supervised_loss( + batch['rgb_original'], + batch['rgb_context_original'], + output['inv_depths'], + batch['depth'], + batch['pose_context'], + output['poses'], + batch['intrinsics'], + return_logs=return_logs, + progress=progress) + # Return loss and metrics + return { + 'loss': self_sup_output['loss'], + **merge_outputs(output, self_sup_output), + } diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/depth_pose_net.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/depth_pose_net.py new file mode 100644 index 0000000000000000000000000000000000000000..c1542ddc403d5f114b32fd1262e9f74b07f480a9 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/depth_pose/depth_pose_net.py @@ -0,0 +1,266 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +from functools import partial + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.video_depth_estimation.geometry.camera import ( + Camera, Pose) +from modelscope.models.cv.video_depth_estimation.networks.layers.resnet.layers import \ + disp_to_depth +from modelscope.models.cv.video_depth_estimation.networks.optim.extractor import \ + ResNetEncoder +from modelscope.models.cv.video_depth_estimation.networks.optim.update import ( + BasicUpdateBlockDepth, BasicUpdateBlockPose, DepthHead, PoseHead, + UpMaskNet) +from modelscope.models.cv.video_depth_estimation.utils.depth import inv2depth + + +class DepthPoseNet(nn.Module): + + def __init__(self, version=None, min_depth=0.1, max_depth=100, **kwargs): + super().__init__() + self.min_depth = min_depth + self.max_depth = max_depth + assert 'it' in version + self.iters = int(version.split('-')[0].split('it')[1]) + self.is_high = 'h' in version + self.out_normalize = 'out' in version + # get seq len in one stage. default: 4. + self.seq_len = 4 + for str in version.split('-'): + if 'seq' in str: + self.seq_len = int(str.split('seq')[1]) + # update iters + self.iters = self.iters // self.seq_len + # intermediate supervision + self.inter_sup = 'inter' in version + + print( + f'=======iters:{self.iters}, sub_seq_len:{self.seq_len}, inter_sup: {self.inter_sup}, ' + f'is_high:{self.is_high}, out_norm:{self.out_normalize}, ' + f'max_depth:{self.max_depth} min_depth:{self.min_depth}========') + + if self.out_normalize: + self.scale_inv_depth = partial( + disp_to_depth, + min_depth=self.min_depth, + max_depth=self.max_depth) + else: + self.scale_inv_depth = lambda x: (x, None) # identity + + # feature network, context network, and update block + self.foutput_dim = 128 + self.feat_ratio = 8 + self.fnet = ResNetEncoder( + out_chs=self.foutput_dim, stride=self.feat_ratio) + + self.depth_head = DepthHead( + input_dim=self.foutput_dim, + hidden_dim=self.foutput_dim, + scale=False) + self.pose_head = PoseHead( + input_dim=self.foutput_dim * 2, hidden_dim=self.foutput_dim) + self.upmask_net = UpMaskNet( + hidden_dim=self.foutput_dim, ratio=self.feat_ratio) + + self.hdim = 128 if self.is_high else 64 + self.cdim = 32 + + self.update_block_depth = BasicUpdateBlockDepth( + hidden_dim=self.hdim, + cost_dim=self.foutput_dim, + ratio=self.feat_ratio, + context_dim=self.cdim) + self.update_block_pose = BasicUpdateBlockPose( + hidden_dim=self.hdim, + cost_dim=self.foutput_dim, + context_dim=self.cdim) + + self.cnet = ResNetEncoder( + out_chs=self.foutput_dim, stride=self.feat_ratio) + self.cnet_depth = ResNetEncoder( + out_chs=self.hdim + self.cdim, + stride=self.feat_ratio, + num_input_images=1) + self.cnet_pose = ResNetEncoder( + out_chs=self.hdim + self.cdim, + stride=self.feat_ratio, + num_input_images=2) + + def upsample_depth(self, depth, mask, ratio=8): + """ Upsample depth field [H/ratio, W/ratio, 2] -> [H, W, 2] using convex combination """ + N, _, H, W = depth.shape + mask = mask.view(N, 1, 9, ratio, ratio, H, W) + mask = torch.softmax(mask, dim=2) + + up_flow = F.unfold(depth, [3, 3], padding=1) + up_flow = up_flow.view(N, 1, 9, 1, 1, H, W) + + up_flow = torch.sum(mask * up_flow, dim=2) + up_flow = up_flow.permute(0, 1, 4, 2, 5, 3) + return up_flow.reshape(N, 1, ratio * H, ratio * W) + + def get_cost_each(self, pose, fmap, fmap_ref, depth, K, ref_K, + scale_factor): + """ + depth: (b, 1, h, w) + fmap, fmap_ref: (b, c, h, w) + """ + pose = Pose.from_vec(pose, 'euler') + + device = depth.device + cam = Camera(K=K.float()).scaled(scale_factor).to( + device) # tcw = Identity + ref_cam = Camera( + K=ref_K.float(), Tcw=pose).scaled(scale_factor).to(device) + + # Reconstruct world points from target_camera + world_points = cam.reconstruct(depth, frame='w') + # Project world points onto reference camera + ref_coords = ref_cam.project( + world_points, frame='w', normalize=True) # (b, h, w,2) + + fmap_warped = F.grid_sample( + fmap_ref, + ref_coords, + mode='bilinear', + padding_mode='zeros', + align_corners=True) # (b, c, h, w) + + cost = (fmap - fmap_warped)**2 + + return cost + + def depth_cost_calc(self, inv_depth, fmap, fmaps_ref, pose_list, K, ref_K, + scale_factor): + cost_list = [] + for pose, fmap_r in zip(pose_list, fmaps_ref): + cost = self.get_cost_each(pose, fmap, fmap_r, inv2depth(inv_depth), + K, ref_K, scale_factor) + cost_list.append(cost) # (b, c,h, w) + # cost = torch.stack(cost_list, dim=1).min(dim=1)[0] + cost = torch.stack(cost_list, dim=1).mean(dim=1) + return cost + + def forward(self, target_image, ref_imgs, intrinsics): + """ Estimate inv depth and poses """ + # run the feature network + fmaps = self.fnet(torch.cat([target_image] + ref_imgs, dim=0)) + fmaps = torch.split( + fmaps, [target_image.shape[0]] * (1 + len(ref_imgs)), dim=0) + fmap1, fmaps_ref = fmaps[0], fmaps[1:] + assert target_image.shape[2] / fmap1.shape[2] == self.feat_ratio + + # initial pose + pose_list_init = [] + for fmap_ref in fmaps_ref: + pose_list_init.append( + self.pose_head(torch.cat([fmap1, fmap_ref], dim=1))) + + # initial depth + inv_depth_init = self.depth_head(fmap1, act_fn=F.sigmoid) + up_mask = self.upmask_net(fmap1) + inv_depth_up_init = self.upsample_depth( + inv_depth_init, up_mask, ratio=self.feat_ratio) + + inv_depth_predictions = [self.scale_inv_depth(inv_depth_up_init)[0]] + pose_predictions = [[pose.clone() for pose in pose_list_init]] + + # run the context network for optimization + if self.iters > 0: + cnet_depth = self.cnet_depth(target_image) + hidden_d, inp_d = torch.split( + cnet_depth, [self.hdim, self.cdim], dim=1) + hidden_d = torch.tanh(hidden_d) + inp_d = torch.relu(inp_d) + + img_pairs = [] + for ref_img in ref_imgs: + img_pairs.append(torch.cat([target_image, ref_img], dim=1)) + cnet_pose_list = self.cnet_pose(img_pairs) + hidden_p_list, inp_p_list = [], [] + for cnet_pose in cnet_pose_list: + hidden_p, inp_p = torch.split( + cnet_pose, [self.hdim, self.cdim], dim=1) + hidden_p_list.append(torch.tanh(hidden_p)) + inp_p_list.append(torch.relu(inp_p)) + + # optimization start................. + pose_list = pose_list_init + inv_depth = inv_depth_init + inv_depth_up = None + for itr in range(self.iters): + inv_depth = inv_depth.detach() + pose_list = [pose.detach() for pose in pose_list] + + # calc cost + pose_cost_func_list = [] + for fmap_ref in fmaps_ref: + pose_cost_func_list.append( + partial( + self.get_cost_each, + fmap=fmap1, + fmap_ref=fmap_ref, + depth=inv2depth(self.scale_inv_depth(inv_depth)[0]), + K=intrinsics, + ref_K=intrinsics, + scale_factor=1.0 / self.feat_ratio)) + + depth_cost_func = partial( + self.depth_cost_calc, + fmap=fmap1, + fmaps_ref=fmaps_ref, + pose_list=pose_list, + K=intrinsics, + ref_K=intrinsics, + scale_factor=1.0 / self.feat_ratio) + + # ######## update depth ########## + hidden_d, up_mask_seqs, inv_depth_seqs = self.update_block_depth( + hidden_d, + depth_cost_func, + inv_depth, + inp_d, + seq_len=self.seq_len, + scale_func=self.scale_inv_depth) + + if not self.inter_sup: + up_mask_seqs, inv_depth_seqs = [up_mask_seqs[-1] + ], [inv_depth_seqs[-1]] + # upsample predictions + for up_mask_i, inv_depth_i in zip(up_mask_seqs, inv_depth_seqs): + inv_depth_up = self.upsample_depth( + inv_depth_i, up_mask_i, ratio=self.feat_ratio) + inv_depth_predictions.append( + self.scale_inv_depth(inv_depth_up)[0]) + inv_depth = inv_depth_seqs[-1] + + # ######## update pose ########### + pose_list_seqs = [None] * len(pose_list) + for i, (pose, + hidden_p) in enumerate(zip(pose_list, hidden_p_list)): + hidden_p, pose_seqs = self.update_block_pose( + hidden_p, + pose_cost_func_list[i], + pose, + inp_p_list[i], + seq_len=self.seq_len) + hidden_p_list[i] = hidden_p + if not self.inter_sup: + pose_seqs = [pose_seqs[-1]] + pose_list_seqs[i] = pose_seqs + + for pose_list_i in zip(*pose_list_seqs): + pose_predictions.append([pose.clone() for pose in pose_list_i]) + + pose_list = list(zip(*pose_list_seqs))[-1] + + if not self.training: + return inv_depth_predictions[-1], \ + torch.stack(pose_predictions[-1], dim=1).view(target_image.shape[0], len(ref_imgs), 6) # (b, n, 6) + + return inv_depth_predictions, \ + torch.stack([torch.stack(poses_ref, dim=1) for poses_ref in pose_predictions], dim=2) # (b, n, iters, 6) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/depth_decoder.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/depth_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..05cbf83310c275837c4e852c6ed76beb5753a4e7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/depth_decoder.py @@ -0,0 +1,265 @@ +# Adapted from monodepth2 +# https://github.com/nianticlabs/monodepth2/blob/master/networks/depth_decoder.py + +from __future__ import absolute_import, division, print_function +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn as nn + +from .layers import Conv3x3, ConvBlock, upsample + + +class DepthDecoder(nn.Module): + + def __init__(self, + num_ch_enc, + scales=range(4), + num_output_channels=1, + use_skips=True): + super(DepthDecoder, self).__init__() + + self.num_output_channels = num_output_channels + self.use_skips = use_skips + self.upsample_mode = 'nearest' + self.scales = scales + + self.num_ch_enc = num_ch_enc + self.num_ch_dec = np.array([16, 32, 64, 128, 256]) + + # decoder + self.convs = OrderedDict() + for i in range(4, -1, -1): + # upconv_0 + num_ch_in = self.num_ch_enc[-1] if i == 4 else self.num_ch_dec[i + + 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 0)] = ConvBlock(num_ch_in, num_ch_out) + + # upconv_1 + num_ch_in = self.num_ch_dec[i] + if self.use_skips and i > 0: + num_ch_in += self.num_ch_enc[i - 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 1)] = ConvBlock(num_ch_in, num_ch_out) + + for s in self.scales: + self.convs[('dispconv', s)] = Conv3x3(self.num_ch_dec[s], + self.num_output_channels) + + self.decoder = nn.ModuleList(list(self.convs.values())) + self.sigmoid = nn.Sigmoid() + + def forward(self, input_features): + self.outputs = {} + + # decoder + x = input_features[-1] + for i in range(4, -1, -1): + x = self.convs[('upconv', i, 0)](x) + x = [upsample(x)] + if self.use_skips and i > 0: + x += [input_features[i - 1]] + x = torch.cat(x, 1) + x = self.convs[('upconv', i, 1)](x) + self.outputs[('feat', i)] = x + if i in self.scales: + self.outputs[('disp', + i)] = self.sigmoid(self.convs[('dispconv', + i)](x)) + + return self.outputs + + +class DepthDecoderShare(nn.Module): + + def __init__(self, + num_ch_enc, + scales=range(4), + num_output_channels=1, + stride=8, + use_skips=True, + num_ch_dec=[16, 32, 64, 128, 256]): + super(DepthDecoderShare, self).__init__() + + self.num_output_channels = num_output_channels + self.use_skips = use_skips + self.upsample_mode = 'nearest' + self.scales = scales + + self.num_ch_enc = num_ch_enc + # self.num_ch_dec = np.array([16, 32, 64, 128, 256]) #(s1:16, s2:32, s4:64, s8:128, s16:256) + self.num_ch_dec = num_ch_dec + + self.stride = stride + # (4:s16, 3:s8, 2:s4, 1:s2, 0:s1) + if self.stride == 8: + self.scale_idx = 3 + elif self.stride == 4: + self.scale_idx = 2 + else: + raise NotImplementedError + + # decoder + self.convs = OrderedDict() + for i in range(4, self.scale_idx - 1, -1): + # upconv_0 + num_ch_in = self.num_ch_enc[-1] if i == 4 else self.num_ch_dec[i + + 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 0)] = ConvBlock(num_ch_in, num_ch_out) + + # upconv_1 + num_ch_in = self.num_ch_dec[i] + if self.use_skips and i > 0: + num_ch_in += self.num_ch_enc[i - 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 1)] = ConvBlock(num_ch_in, num_ch_out) + + self.decoder = nn.ModuleList(list(self.convs.values())) + + def forward(self, input_features): + self.outputs = {} + + # decoder + x = input_features[-1] + self.outputs[('feat', -1)] = x + for i in range(4, self.scale_idx - 1, -1): + x = self.convs[('upconv', i, 0)](x) + x = [upsample(x)] + if self.use_skips and i > 0: + x += [input_features[i - 1]] + x = torch.cat(x, 1) + x = self.convs[('upconv', i, 1)](x) + self.outputs[('feat', i)] = x + + return self.outputs + + +class DepthDecoderShareFeat(nn.Module): + + def __init__(self, + num_ch_enc, + scales=range(4), + num_output_channels=1, + stride=8, + use_skips=True, + num_ch_dec=[16, 32, 64, 128, 256]): + super().__init__() + + self.num_output_channels = num_output_channels + self.use_skips = use_skips + self.upsample_mode = 'nearest' + self.scales = scales + + self.num_ch_enc = num_ch_enc + # self.num_ch_dec = np.array([16, 32, 64, 128, 256]) #(s1:16, s2:32, s4:64, s8:128, s16:256) + self.num_ch_dec = num_ch_dec + + self.stride = stride + # (4:s16, 3:s8, 2:s4, 1:s2, 0:s1) + if self.stride == 8: + self.scale_idx = 3 + elif self.stride == 4: + self.scale_idx = 2 + else: + raise NotImplementedError + + # decoder + self.convs = OrderedDict() + for i in range(4, self.scale_idx - 1, -1): + # upconv_0 + num_ch_in = self.num_ch_enc[-1] if i == 4 else self.num_ch_dec[i + + 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 0)] = nn.Conv2d(num_ch_in, num_ch_out, 3, + 1, 1) + + # upconv_1 + num_ch_in = self.num_ch_dec[i] + if self.use_skips and i > 0: + num_ch_in += self.num_ch_enc[i - 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 1)] = nn.Conv2d(num_ch_in, num_ch_out, 3, + 1, 1) + + self.decoder = nn.ModuleList(list(self.convs.values())) + + def forward(self, input_features): + self.outputs = {} + + # decoder + x = input_features[-1] + self.outputs[('feat', -1)] = x + for i in range(4, self.scale_idx - 1, -1): + x = self.convs[('upconv', i, 0)](x) + x = [upsample(x)] + if self.use_skips and i > 0: + x += [input_features[i - 1]] + x = torch.cat(x, 1) + x = self.convs[('upconv', i, 1)](x) + self.outputs[('feat', i)] = x + + return self.outputs + + +class UnetDecoder(nn.Module): + + def __init__(self, + num_ch_enc, + num_output_channels=1, + stride=8, + out_chs=128, + use_skips=True): + super(UnetDecoder, self).__init__() + + self.num_output_channels = num_output_channels + self.use_skips = use_skips + self.upsample_mode = 'nearest' + + self.num_ch_enc = num_ch_enc + self.num_ch_dec = np.array([16, 32, 64, 128, 256]) + + self.stride = stride + # (4:s16, 3:s8, 2:s4, 1:s2, 0:s1) + if self.stride == 8: + self.scale_idx = 3 + elif self.stride == 4: + self.scale_idx = 2 + else: + raise NotImplementedError + # decoder + self.convs = OrderedDict() + for i in range(4, self.scale_idx - 1, -1): + # upconv_0 + num_ch_in = self.num_ch_enc[-1] if i == 4 else self.num_ch_dec[i + + 1] + num_ch_out = self.num_ch_dec[i] + self.convs[('upconv', i, 0)] = ConvBlock(num_ch_in, num_ch_out) + + # upconv_1 + num_ch_in = self.num_ch_dec[i] + if self.use_skips and i > 0: + num_ch_in += self.num_ch_enc[i - 1] + num_ch_out = self.num_ch_dec[i] if i != self.scale_idx else out_chs + + self.convs[('upconv', i, 1)] = ConvBlock(num_ch_in, num_ch_out) + + self.decoder = nn.ModuleList(list(self.convs.values())) + + def forward(self, input_features): + self.outputs = {} + + # decoder + x = input_features[-1] + for i in range(4, self.scale_idx - 1, -1): + x = self.convs[('upconv', i, 0)](x) + x = [upsample(x)] + if self.use_skips and i > 0: + x += [input_features[i - 1]] + x = torch.cat(x, 1) + x = self.convs[('upconv', i, 1)](x) + self.outputs[('feat', i)] = x + + return self.outputs diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/layers.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..22acb067d29d18f51163ef0a6e6be8149c8e6b7e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/layers.py @@ -0,0 +1,60 @@ +# Adapted from monodepth2 +# https://github.com/nianticlabs/monodepth2/blob/master/layers.py + +from __future__ import absolute_import, division, print_function + +import torch.nn as nn +import torch.nn.functional as F + + +def disp_to_depth(disp, min_depth, max_depth): + """Convert network's sigmoid output into depth prediction + The formula for this conversion is given in the 'additional considerations' + section of the paper. + """ + min_disp = 1 / max_depth + max_disp = 1 / min_depth + scaled_disp = min_disp + (max_disp - min_disp) * disp + depth = 1 / scaled_disp + return scaled_disp, depth + + +class ConvBlock(nn.Module): + """Layer to perform a convolution followed by ELU + """ + + def __init__(self, in_channels, out_channels): + super(ConvBlock, self).__init__() + + self.conv = Conv3x3(in_channels, out_channels) + self.nonlin = nn.ELU(inplace=True) + + def forward(self, x): + out = self.conv(x) + out = self.nonlin(out) + return out + + +class Conv3x3(nn.Module): + """Layer to pad and convolve input + """ + + def __init__(self, in_channels, out_channels, use_refl=True): + super(Conv3x3, self).__init__() + + if use_refl: + self.pad = nn.ReflectionPad2d(1) + else: + self.pad = nn.ZeroPad2d(1) + self.conv = nn.Conv2d(int(in_channels), int(out_channels), 3) + + def forward(self, x): + out = self.pad(x) + out = self.conv(out) + return out + + +def upsample(x): + """Upsample input tensor by a factor of 2 + """ + return F.interpolate(x, scale_factor=2, mode='nearest') diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/pose_decoder.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/pose_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..0c2bbf82646659169e029f2d431e5c7af15f8145 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/pose_decoder.py @@ -0,0 +1,59 @@ +# Adapted from monodepth2 +# https://github.com/nianticlabs/monodepth2/blob/master/networks/pose_decoder.py + +from __future__ import absolute_import, division, print_function +from collections import OrderedDict + +import torch +import torch.nn as nn + + +class PoseDecoder(nn.Module): + + def __init__(self, + num_ch_enc, + num_input_features, + num_frames_to_predict_for=None, + stride=1): + super(PoseDecoder, self).__init__() + + self.num_ch_enc = num_ch_enc + self.num_input_features = num_input_features + + if num_frames_to_predict_for is None: + num_frames_to_predict_for = num_input_features - 1 + self.num_frames_to_predict_for = num_frames_to_predict_for + + self.convs = OrderedDict() + self.convs[('squeeze')] = nn.Conv2d(self.num_ch_enc[-1], 256, 1) + self.convs[('pose', 0)] = nn.Conv2d(num_input_features * 256, 256, 3, + stride, 1) + self.convs[('pose', 1)] = nn.Conv2d(256, 256, 3, stride, 1) + self.convs[('pose', 2)] = nn.Conv2d(256, 6 * num_frames_to_predict_for, + 1) + + self.relu = nn.ReLU() + + self.net = nn.ModuleList(list(self.convs.values())) + + def forward(self, input_features): + last_features = [f[-1] for f in input_features] + + cat_features = [ + self.relu(self.convs['squeeze'](f)) for f in last_features + ] + cat_features = torch.cat(cat_features, 1) + + out = cat_features + for i in range(3): + out = self.convs[('pose', i)](out) + if i != 2: + out = self.relu(out) + out = out.mean(3).mean(2) + + out = 0.01 * out.view(-1, self.num_frames_to_predict_for, 1, 6) + + axisangle = out[..., :3] + translation = out[..., 3:] + + return axisangle, translation diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/resnet_encoder.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/resnet_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..b81ca17ff31940a9adc50ee010cdd83cec16f94b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/layers/resnet/resnet_encoder.py @@ -0,0 +1,111 @@ +# Adapted from monodepth2 +# https://github.com/nianticlabs/monodepth2/blob/master/networks/resnet_encoder.py + +from __future__ import absolute_import, division, print_function + +import numpy as np +import torch +import torch.nn as nn +import torch.utils.model_zoo as model_zoo +import torchvision.models as models + + +class ResNetMultiImageInput(models.ResNet): + """Constructs a resnet model with varying number of input images. + Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py + """ + + def __init__(self, block, layers, num_classes=1000, num_input_images=1): + super(ResNetMultiImageInput, self).__init__(block, layers) + self.inplanes = 64 + self.conv1 = nn.Conv2d( + num_input_images * 3, + 64, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + +def resnet_multiimage_input(num_layers, pretrained=False, num_input_images=1): + """Constructs a ResNet model. + Args: + num_layers (int): Number of resnet layers. Must be 18 or 50 + pretrained (bool): If True, returns a model pre-trained on ImageNet + num_input_images (int): Number of frames stacked as input + """ + assert num_layers in [18, 50], 'Can only run with 18 or 50 layer resnet' + blocks = {18: [2, 2, 2, 2], 50: [3, 4, 6, 3]}[num_layers] + block_type = { + 18: models.resnet.BasicBlock, + 50: models.resnet.Bottleneck + }[num_layers] + model = ResNetMultiImageInput( + block_type, blocks, num_input_images=num_input_images) + + if pretrained: + loaded = model_zoo.load_url( + models.resnet.model_urls['resnet{}'.format(num_layers)]) + loaded['conv1.weight'] = torch.cat( + [loaded['conv1.weight']] * num_input_images, 1) / num_input_images + model.load_state_dict(loaded) + return model + + +class ResnetEncoder(nn.Module): + """Pytorch module for a resnet encoder + """ + + def __init__(self, num_layers, pretrained, num_input_images=1): + super(ResnetEncoder, self).__init__() + self.num_ch_enc = np.array([64, 64, 128, 256, 512]) + + resnets = { + 18: models.resnet18, + 34: models.resnet34, + 50: models.resnet50, + 101: models.resnet101, + 152: models.resnet152 + } + + if num_layers not in resnets: + raise ValueError( + '{} is not a valid number of resnet layers'.format(num_layers)) + + if num_input_images > 1: + self.encoder = resnet_multiimage_input(num_layers, pretrained, + num_input_images) + else: + self.encoder = resnets[num_layers](pretrained) + + if num_layers > 34: + self.num_ch_enc[1:] *= 4 + + def forward(self, input_image): + self.features = [] + x = (input_image - 0.45) / 0.225 + x = self.encoder.conv1(x) + x = self.encoder.bn1(x) + self.features.append(self.encoder.relu(x)) + self.features.append( + self.encoder.layer1(self.encoder.maxpool(self.features[-1]))) + self.features.append(self.encoder.layer2(self.features[-1])) + self.features.append(self.encoder.layer3(self.features[-1])) + self.features.append(self.encoder.layer4(self.features[-1])) + + return self.features diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/extractor.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..abb0882b15a7b30c3d81a24bab28748e8f913b85 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/extractor.py @@ -0,0 +1,429 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +import torchvision.models as models + + +class ResNetEncoder(models.ResNet): + """Constructs a resnet model with varying number of input images. + Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py + """ + + def __init__(self, + num_layers=18, + num_input_images=1, + pretrained=False, + out_chs=32, + stride=8): + layers = {18: [2, 2, 2, 2], 50: [3, 4, 6, 3]}[num_layers] + block = { + 18: models.resnet.BasicBlock, + 50: models.resnet.Bottleneck + }[num_layers] + self.upsample_mode = 'bilinear' + super(ResNetEncoder, self).__init__(block, layers) + + self.inplanes = 64 + self.conv1 = nn.Conv2d( + num_input_images * 3, + 64, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + + self.stride = stride + if stride == 8: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(128, out_chs, 3, 1, padding=1) + + elif stride == 4: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2_fusion = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(64, out_chs, 3, 1, padding=1) + + else: + raise NotImplementedError + + # self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + # del self.layer3 + del self.layer4 + del self.fc + del self.avgpool + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if pretrained: + loaded = model_zoo.load_url( + models.resnet.model_urls['resnet{}'.format(num_layers)]) + loaded['conv1.weight'] = torch.cat( + [loaded['conv1.weight']] * num_input_images, + 1) / num_input_images + loaded_flilter = { + k: v + for k, v in loaded.items() + if 'layer4' not in k and 'fc' not in k + } + try: + print('load pretrained model from:', + models.resnet.model_urls['resnet{}'.format(num_layers)]) + self.load_state_dict(loaded_flilter) + except Exception as e: + print(e) + self.load_state_dict(loaded_flilter, strict=False) + + def forward(self, x): + feats = {} + # if input is list, combine batch dimension + is_list = isinstance(x, tuple) or isinstance(x, list) + if is_list: + num = len(x) + batch_dim = x[0].shape[0] + x = torch.cat(x, dim=0) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + feats['s4'] = x + x = self.layer2(x) + feats['s8'] = x + x = self.layer3(x) + + if self.stride == 8: + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1(x) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + x = self.out_conv(x) + + elif self.stride == 4: + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1(x) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv2(x) + x = self.upconv2_fusion(torch.cat([x, feats['s4']], dim=1)) + + x = self.out_conv(x) + + if is_list: + x = torch.split(x, [batch_dim] * num, dim=0) + + return x + + +######################################################################################################################## +class ResNetEncoderV1(models.ResNet): + """Constructs a resnet model with varying number of input images. + Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py + """ + + def __init__(self, + num_layers=18, + num_input_images=1, + pretrained=True, + out_chs=32, + stride=8): + layers = {18: [2, 2, 2, 2], 50: [3, 4, 6, 3]}[num_layers] + block = { + 18: models.resnet.BasicBlock, + 50: models.resnet.Bottleneck + }[num_layers] + self.upsample_mode = 'nearest' + super().__init__(block, layers) + + self.inplanes = 64 + self.conv1 = nn.Conv2d( + num_input_images * 3, + 64, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + + self.stride = stride + if stride == 8: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(128, out_chs, 3, 1, padding=1) + + elif stride == 4: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2_fusion = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(64, out_chs, 3, 1, padding=1) + + else: + raise NotImplementedError + + # self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + # del self.layer3 + del self.layer4 + del self.fc + del self.avgpool + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if pretrained: + loaded = model_zoo.load_url( + models.resnet.model_urls['resnet{}'.format(num_layers)]) + loaded['conv1.weight'] = torch.cat( + [loaded['conv1.weight']] * num_input_images, + 1) / num_input_images + loaded_flilter = { + k: v + for k, v in loaded.items() + if 'layer4' not in k and 'fc' not in k + } + try: + print('load pretrained model from:', + models.resnet.model_urls['resnet{}'.format(num_layers)]) + self.load_state_dict(loaded_flilter) + except Exception as e: + print(e) + self.load_state_dict(loaded_flilter, strict=False) + + def forward(self, x): + feats = {} + # if input is list, combine batch dimension + is_list = isinstance(x, tuple) or isinstance(x, list) + if is_list: + num = len(x) + batch_dim = x[0].shape[0] + x = torch.cat(x, dim=0) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + feats['s4'] = x + x = self.layer2(x) + feats['s8'] = x + x = self.layer3(x) + + if self.stride == 8: + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1(x) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + x = self.out_conv(x) + + elif self.stride == 4: + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1(x) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv2(x) + x = self.upconv2_fusion(torch.cat([x, feats['s4']], dim=1)) + + x = self.out_conv(x) + + if is_list: + x = torch.split(x, [batch_dim] * num, dim=0) + + return x + + +######################################################################################################################## +class ResNetEncoderV2(models.ResNet): + """Constructs a resnet model with varying number of input images. + Adapted from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py + """ + + def __init__(self, + num_layers=18, + num_input_images=1, + pretrained=True, + out_chs=32, + stride=8): + layers = {18: [2, 2, 2, 2], 50: [3, 4, 6, 3]}[num_layers] + block = { + 18: models.resnet.BasicBlock, + 50: models.resnet.Bottleneck + }[num_layers] + self.upsample_mode = 'bilinear' + super().__init__(block, layers) + + self.inplanes = 64 + self.conv1 = nn.Conv2d( + num_input_images * 3, + 64, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + # del self.layer4 + self.upconv0 = nn.Sequential( + nn.Conv2d(512, 256, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv0_fusion = nn.Sequential( + nn.Conv2d(512, 256, 3, 1, padding=1), nn.ReLU(inplace=True)) + + self.stride = stride + if stride == 8: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(128, out_chs, 3, 1, padding=1) + + elif stride == 4: + self.upconv1 = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv1_fusion = nn.Sequential( + nn.Conv2d(256, 128, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.upconv2_fusion = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, padding=1), nn.ReLU(inplace=True)) + self.out_conv = nn.Conv2d(64, out_chs, 3, 1, padding=1) + + else: + raise NotImplementedError + + del self.fc + del self.avgpool + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if pretrained: + loaded = model_zoo.load_url( + models.resnet.model_urls['resnet{}'.format(num_layers)]) + loaded['conv1.weight'] = torch.cat( + [loaded['conv1.weight']] * num_input_images, + 1) / num_input_images + loaded_flilter = { + k: v + for k, v in loaded.items() + if 'layer4' not in k and 'fc' not in k + } + try: + print('load pretrained model from:', + models.resnet.model_urls['resnet{}'.format(num_layers)]) + self.load_state_dict(loaded_flilter) + except Exception as e: + print(e) + self.load_state_dict(loaded_flilter, strict=False) + + def forward(self, x): + feats = {} + # if input is list, combine batch dimension + is_list = isinstance(x, tuple) or isinstance(x, list) + if is_list: + num = len(x) + batch_dim = x[0].shape[0] + x = torch.cat(x, dim=0) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + feats['s4'] = x + x = self.layer2(x) + feats['s8'] = x + x = self.layer3(x) + + feats['s16'] = x + x = self.layer4(x) + x = self.upconv0(x) + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv0_fusion(torch.cat([x, feats['s16']], dim=1)) + + if self.stride == 8: + x = self.upconv1(x) + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + x = self.out_conv(x) + + elif self.stride == 4: + x = self.upconv1(x) + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv1_fusion(torch.cat([x, feats['s8']], dim=1)) + + x = self.upconv2(x) + x = F.interpolate(x, scale_factor=2, mode=self.upsample_mode) + x = self.upconv2_fusion(torch.cat([x, feats['s4']], dim=1)) + + x = self.out_conv(x) + + if is_list: + x = torch.split(x, [batch_dim] * num, dim=0) + + return x + + +################################################################################ +class FeatBlock(nn.Module): + + def __init__(self, planes=128, out_dim=128): + super().__init__() + self.conv1 = nn.Conv2d(planes, planes, 3, padding=1) + self.conv2 = nn.Conv2d(planes, out_dim, 3, padding=1) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.relu(self.conv1(self.relu(x))) + x = self.conv2(x) + return x diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/update.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/update.py new file mode 100644 index 0000000000000000000000000000000000000000..fc179f84b408df5a2b1e196f6a0aed5a62a712e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/networks/optim/update.py @@ -0,0 +1,234 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class DepthHead(nn.Module): + + def __init__(self, input_dim=256, hidden_dim=128, scale=False): + super(DepthHead, self).__init__() + self.scale = scale + self.conv1 = nn.Conv2d(input_dim, hidden_dim, 3, padding=1) + self.conv2 = nn.Conv2d(hidden_dim, 1, 3, padding=1) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x_d, act_fn=F.tanh): + out = self.conv2(self.relu(self.conv1(x_d))) + return act_fn(out) + + +class PoseHead(nn.Module): + + def __init__(self, input_dim=256, hidden_dim=128): + super(PoseHead, self).__init__() + + self.conv1_pose = nn.Conv2d(input_dim, hidden_dim, 3, padding=1) + self.conv2_pose = nn.Conv2d(hidden_dim, 6, 3, padding=1) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x_p): + out = self.conv2_pose(self.relu(self.conv1_pose(x_p))).mean(3).mean(2) + return torch.cat([out[:, :3], 0.01 * out[:, 3:]], dim=1) + + +class ConvGRU(nn.Module): + + def __init__(self, hidden_dim=128, input_dim=192 + 128): + super(ConvGRU, self).__init__() + self.convz = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + self.convr = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + self.convq = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + + def forward(self, h, x): + hx = torch.cat([h, x], dim=1) + + z = torch.sigmoid(self.convz(hx)) + r = torch.sigmoid(self.convr(hx)) + q = torch.tanh(self.convq(torch.cat([r * h, x], dim=1))) + + h = (1 - z) * h + z * q + return h + + +class SepConvGRU(nn.Module): + + def __init__(self, hidden_dim=128, input_dim=192 + 128): + super(SepConvGRU, self).__init__() + self.convz1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + self.convr1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + self.convq1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + + self.convz2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + self.convr2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + self.convq2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + + def forward(self, h, x): + # horizontal + hx = torch.cat([h, x], dim=1) + z = torch.sigmoid(self.convz1(hx)) + r = torch.sigmoid(self.convr1(hx)) + q = torch.tanh(self.convq1(torch.cat([r * h, x], dim=1))) + h = (1 - z) * h + z * q + + # vertical + hx = torch.cat([h, x], dim=1) + z = torch.sigmoid(self.convz2(hx)) + r = torch.sigmoid(self.convr2(hx)) + q = torch.tanh(self.convq2(torch.cat([r * h, x], dim=1))) + h = (1 - z) * h + z * q + + return h + + +class ProjectionInputDepth(nn.Module): + + def __init__(self, cost_dim, hidden_dim, out_chs): + super().__init__() + self.out_chs = out_chs + self.convc1 = nn.Conv2d(cost_dim, hidden_dim, 1, padding=0) + self.convc2 = nn.Conv2d(hidden_dim, hidden_dim, 3, padding=1) + + self.convd1 = nn.Conv2d(1, hidden_dim, 7, padding=3) + self.convd2 = nn.Conv2d(hidden_dim, 64, 3, padding=1) + + self.convd = nn.Conv2d(64 + hidden_dim, out_chs - 1, 3, padding=1) + + def forward(self, depth, cost): + cor = F.relu(self.convc1(cost)) + cor = F.relu(self.convc2(cor)) + + dfm = F.relu(self.convd1(depth)) + dfm = F.relu(self.convd2(dfm)) + cor_dfm = torch.cat([cor, dfm], dim=1) + + out_d = F.relu(self.convd(cor_dfm)) + + return torch.cat([out_d, depth], dim=1) + + +class ProjectionInputPose(nn.Module): + + def __init__(self, cost_dim, hidden_dim, out_chs): + super().__init__() + self.out_chs = out_chs + self.convc1 = nn.Conv2d(cost_dim, hidden_dim, 1, padding=0) + self.convc2 = nn.Conv2d(hidden_dim, hidden_dim, 3, padding=1) + + self.convp1 = nn.Conv2d(6, hidden_dim, 7, padding=3) + self.convp2 = nn.Conv2d(hidden_dim, 64, 3, padding=1) + + self.convp = nn.Conv2d(64 + hidden_dim, out_chs - 6, 3, padding=1) + + def forward(self, pose, cost): + bs, _, h, w = cost.shape + cor = F.relu(self.convc1(cost)) + cor = F.relu(self.convc2(cor)) + + pfm = F.relu(self.convp1(pose.view(bs, 6, 1, 1).repeat(1, 1, h, w))) + pfm = F.relu(self.convp2(pfm)) + cor_pfm = torch.cat([cor, pfm], dim=1) + + out_p = F.relu(self.convp(cor_pfm)) + return torch.cat( + [out_p, pose.view(bs, 6, 1, 1).repeat(1, 1, h, w)], dim=1) + + +class UpMaskNet(nn.Module): + + def __init__(self, hidden_dim=128, ratio=8): + super(UpMaskNet, self).__init__() + self.mask = nn.Sequential( + nn.Conv2d(hidden_dim, hidden_dim * 2, 3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(hidden_dim * 2, ratio * ratio * 9, 1, padding=0)) + + def forward(self, feat): + # scale mask to balence gradients + mask = .25 * self.mask(feat) + return mask + + +class BasicUpdateBlockDepth(nn.Module): + + def __init__(self, hidden_dim=128, cost_dim=256, ratio=8, context_dim=64): + super(BasicUpdateBlockDepth, self).__init__() + + self.encoder = ProjectionInputDepth( + cost_dim=cost_dim, hidden_dim=hidden_dim, out_chs=hidden_dim) + self.depth_gru = SepConvGRU( + hidden_dim=hidden_dim, + input_dim=self.encoder.out_chs + context_dim) + self.depth_head = DepthHead( + hidden_dim, hidden_dim=hidden_dim, scale=False) + self.mask = nn.Sequential( + nn.Conv2d(hidden_dim, hidden_dim * 2, 3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(hidden_dim * 2, ratio * ratio * 9, 1, padding=0)) + + def forward(self, + net, + cost_func, + inv_depth, + context, + seq_len=4, + scale_func=None): + inv_depth_list = [] + mask_list = [] + for i in range(seq_len): + # TODO detach() + # inv_depth = inv_depth.detach() + input_features = self.encoder(inv_depth, + cost_func(scale_func(inv_depth)[0])) + inp_i = torch.cat([context, input_features], dim=1) + + net = self.depth_gru(net, inp_i) + delta_inv_depth = self.depth_head(net) + # scale mask to balence gradients + mask = .25 * self.mask(net) + + inv_depth = inv_depth + delta_inv_depth + inv_depth_list.append(inv_depth) + mask_list.append(mask) + + return net, mask_list, inv_depth_list + + +class BasicUpdateBlockPose(nn.Module): + + def __init__(self, hidden_dim=128, cost_dim=256, context_dim=64): + super(BasicUpdateBlockPose, self).__init__() + self.encoder = ProjectionInputPose( + cost_dim=cost_dim, hidden_dim=hidden_dim, out_chs=hidden_dim) + self.pose_gru = SepConvGRU( + hidden_dim=hidden_dim, + input_dim=self.encoder.out_chs + context_dim) + self.pose_head = PoseHead(hidden_dim, hidden_dim=hidden_dim) + + def forward(self, net, cost_func, pose, inp, seq_len=4): + pose_list = [] + for i in range(seq_len): + # TODO detach() + # pose = pose.detach() + input_features = self.encoder(pose, cost_func(pose)) + inp_i = torch.cat([inp, input_features], dim=1) + + net = self.pose_gru(net, inp_i) + delta_pose = self.pose_head(net) + + pose = pose + delta_pose + pose_list.append(pose) + + # scale mask to balence gradients + return net, pose_list diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/__init__.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/augmentations.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/augmentations.py new file mode 100644 index 0000000000000000000000000000000000000000..bcde556d3bdd3749b6b304b378027157277dc6c1 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/augmentations.py @@ -0,0 +1,246 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import random + +import cv2 +import numpy as np +import torchvision.transforms as transforms +from PIL import Image + +from modelscope.models.cv.video_depth_estimation.utils.misc import filter_dict + +######################################################################################################################## + + +def resize_image(image, shape, interpolation=Image.Resampling.LANCZOS): + """ + Resizes input image. + + Parameters + ---------- + image : Image.PIL + Input image + shape : tuple [H,W] + Output shape + interpolation : int + Interpolation mode + + Returns + ------- + image : Image.PIL + Resized image + """ + transform = transforms.Resize(shape, interpolation=interpolation) + return transform(image) + + +def resize_depth(depth, shape): + """ + Resizes depth map. + + Parameters + ---------- + depth : np.array [h,w] + Depth map + shape : tuple (H,W) + Output shape + + Returns + ------- + depth : np.array [H,W] + Resized depth map + """ + depth = cv2.resize( + depth, dsize=shape[::-1], interpolation=cv2.INTER_NEAREST) + return np.expand_dims(depth, axis=2) + + +def resize_sample_image_and_intrinsics(sample, + shape, + image_interpolation=Image.Resampling. + LANCZOS): + """ + Resizes the image and intrinsics of a sample + + Parameters + ---------- + sample : dict + Dictionary with sample values + shape : tuple (H,W) + Output shape + image_interpolation : int + Interpolation mode + + Returns + ------- + sample : dict + Resized sample + """ + # Resize image and corresponding intrinsics + image_transform = transforms.Resize( + shape, interpolation=image_interpolation) + (orig_w, orig_h) = sample['rgb'].size + (out_h, out_w) = shape + # Scale intrinsics + for key in filter_dict(sample, ['intrinsics']): + intrinsics = np.copy(sample[key]) + intrinsics[0] *= out_w / orig_w + intrinsics[1] *= out_h / orig_h + sample[key] = intrinsics + # Scale images + for key in filter_dict(sample, [ + 'rgb', + 'rgb_original', + ]): + sample[key] = image_transform(sample[key]) + # Scale context images + for key in filter_dict(sample, [ + 'rgb_context', + 'rgb_context_original', + ]): + sample[key] = [image_transform(k) for k in sample[key]] + # Return resized sample + return sample + + +def resize_sample(sample, shape, image_interpolation=Image.Resampling.LANCZOS): + """ + Resizes a sample, including image, intrinsics and depth maps. + + Parameters + ---------- + sample : dict + Dictionary with sample values + shape : tuple (H,W) + Output shape + image_interpolation : int + Interpolation mode + + Returns + ------- + sample : dict + Resized sample + """ + # Resize image and intrinsics + sample = resize_sample_image_and_intrinsics(sample, shape, + image_interpolation) + # Resize depth maps + for key in filter_dict(sample, [ + 'depth', + ]): + sample[key] = resize_depth(sample[key], shape) + # Resize depth contexts + for key in filter_dict(sample, [ + 'depth_context', + ]): + sample[key] = [resize_depth(k, shape) for k in sample[key]] + # Return resized sample + return sample + + +######################################################################################################################## + + +def to_tensor(image, tensor_type='torch.FloatTensor'): + """Casts an image to a torch.Tensor""" + transform = transforms.ToTensor() + return transform(image).type(tensor_type) + + +def to_tensor_sample(sample, tensor_type='torch.FloatTensor'): + """ + Casts the keys of sample to tensors. + + Parameters + ---------- + sample : dict + Input sample + tensor_type : str + Type of tensor we are casting to + + Returns + ------- + sample : dict + Sample with keys cast as tensors + """ + transform = transforms.ToTensor() + # Convert single items + for key in filter_dict(sample, [ + 'rgb', + 'rgb_original', + 'depth', + ]): + sample[key] = transform(sample[key]).type(tensor_type) + # Convert lists + for key in filter_dict( + sample, ['rgb_context', 'rgb_context_original', 'depth_context']): + sample[key] = [transform(k).type(tensor_type) for k in sample[key]] + # Return converted sample + return sample + + +######################################################################################################################## + + +def duplicate_sample(sample): + """ + Duplicates sample images and contexts to preserve their unaugmented versions. + + Parameters + ---------- + sample : dict + Input sample + + Returns + ------- + sample : dict + Sample including [+"_original"] keys with copies of images and contexts. + """ + # Duplicate single items + for key in filter_dict(sample, ['rgb']): + sample['{}_original'.format(key)] = sample[key].copy() + # Duplicate lists + for key in filter_dict(sample, ['rgb_context']): + sample['{}_original'.format(key)] = [k.copy() for k in sample[key]] + # Return duplicated sample + return sample + + +def colorjitter_sample(sample, parameters, prob=1.0): + """ + Jitters input images as data augmentation. + + Parameters + ---------- + sample : dict + Input sample + parameters : tuple (brightness, contrast, saturation, hue) + Color jittering parameters + prob : float + Jittering probability + + Returns + ------- + sample : dict + Jittered sample + """ + if random.random() < prob: + # Prepare transformation + color_augmentation = transforms.ColorJitter() + brightness, contrast, saturation, hue = parameters + augment_image = color_augmentation.get_params( + brightness=[max(0, 1 - brightness), 1 + brightness], + contrast=[max(0, 1 - contrast), 1 + contrast], + saturation=[max(0, 1 - saturation), 1 + saturation], + hue=[-hue, hue]) + # Jitter single items + for key in filter_dict(sample, ['rgb']): + sample[key] = augment_image(sample[key]) + # Jitter lists + for key in filter_dict(sample, ['rgb_context']): + sample[key] = [augment_image(k) for k in sample[key]] + # Return jittered sample + return sample + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/config.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..727af28768efc3f9b02782fdc6b330165c1774c7 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/config.py @@ -0,0 +1,327 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import os +from datetime import datetime + +import torch +from yacs.config import CfgNode + +from modelscope.models.cv.video_depth_estimation.utils.horovod import on_rank_0 +from modelscope.models.cv.video_depth_estimation.utils.load import ( + backwards_state_dict, load_class) +from modelscope.models.cv.video_depth_estimation.utils.misc import make_list +from modelscope.models.cv.video_depth_estimation.utils.types import (is_cfg, + is_list) + + +def prep_dataset(config): + """ + Expand dataset configuration to match split length + + Parameters + ---------- + config : CfgNode + Dataset configuration + + Returns + ------- + config : CfgNode + Updated dataset configuration + """ + # If there is no dataset, do nothing + if len(config.path) == 0: + return config + # If cameras is not a double list, make it so + if not config.cameras or not is_list(config.cameras[0]): + config.cameras = [config.cameras] + # Get maximum length and expand other arguments to the same length + n = max(len(config.split), len(config.cameras), len(config.depth_type)) + config.dataset = make_list(config.dataset, n) + config.path = make_list(config.path, n) + config.split = make_list(config.split, n) + config.depth_type = make_list(config.depth_type, n) + config.cameras = make_list(config.cameras, n) + if 'repeat' in config: + config.repeat = make_list(config.repeat, n) + # Return updated configuration + return config + + +def set_name(config): + """ + Set run name based on available information + + Parameters + ---------- + config : CfgNode + Model configuration + + Returns + ------- + name : str + Updated run name + """ + # If there is a name already, do nothing + if config.name != '': + return config.name + else: + # Create a name based on available information + return '{}-{}-{}'.format( + os.path.basename(config.default), + os.path.splitext(os.path.basename(config.config))[0], + datetime.now().strftime('%Y.%m.%d-%Hh%Mm%Ss')) + + +@on_rank_0 +def prep_logger_and_checkpoint(model): + """ + Use logger and checkpoint information to update configuration + + Parameters + ---------- + model : nn.Module + Module to update + """ + # Change run name to be the wandb assigned name + if model.logger and not model.config.wandb.dry_run: + model.config.name = model.config.wandb.name = model.logger.run_name + model.config.wandb.url = model.logger.run_url + # If we are saving models we need to update the path + if model.config.checkpoint.filepath != '': + # Change checkpoint filepath + filepath = model.config.checkpoint.filepath.split('/') + filepath[-2] = model.config.name + model.config.checkpoint.filepath = '/'.join(filepath) + # Change callback dirpath + dirpath = os.path.join( + os.path.dirname(model.trainer.checkpoint.dirpath), + model.config.name) + model.trainer.checkpoint.dirpath = dirpath + os.makedirs(dirpath, exist_ok=True) + # Log updated configuration + model.logger.log_config(model.config) + + +def get_default_config(cfg_default): + """Get default configuration from file""" + config = load_class( + 'get_cfg_defaults', + paths=[cfg_default.replace('/', '.')], + concat=False)() + config.merge_from_list(['default', cfg_default]) + return config + + +def merge_cfg_file(config, cfg_file=None): + """Merge configuration file""" + if cfg_file is not None: + config.merge_from_file(cfg_file) + config.merge_from_list(['config', cfg_file]) + return config + + +def merge_cfgs(original, override): + """ + Updates CfgNode with information from another one + + Parameters + ---------- + original : CfgNode + Original configuration node + override : CfgNode + Another configuration node used for overriding + + Returns + ------- + updated : CfgNode + Updated configuration node + """ + for key, value in original.items(): + if key in override.keys(): + if is_cfg(value): # If it's a configuration node, recursion + original[key] = merge_cfgs(original[key], override[key]) + else: # Otherwise, simply update key + original[key] = override[key] + return original + + +def backwards_config(config): + """ + Add or update configuration for backwards compatibility + (no need for it right now, pretrained models are up-to-date with configuration files). + + Parameters + ---------- + config : CfgNode + Model configuration + + Returns + ------- + config : CfgNode + Updated model configuration + """ + # Return updated configuration + return config + + +def parse_train_config(cfg_default, cfg_file): + """ + Parse model configuration for training + + Parameters + ---------- + cfg_default : str + Default **.py** configuration file + cfg_file : str + Configuration **.yaml** file to override the default parameters + + Returns + ------- + config : CfgNode + Parsed model configuration + """ + # Loads default configuration + config = get_default_config(cfg_default) + # Merge configuration file + config = merge_cfg_file(config, cfg_file) + # Return prepared configuration + return prepare_train_config(config) + + +def prepare_train_config(config): + """ + Prepare model configuration for training + + Parameters + ---------- + config : CfgNode + Model configuration + + Returns + ------- + config : CfgNode + Prepared model configuration + """ + # If arguments have already been prepared, don't prepare + if config.prepared: + return config + + # Asserts + assert config.wandb.dry_run or config.wandb.entity != '', \ + 'You need a wandb entity' + assert config.wandb.dry_run or config.wandb.project != '', \ + 'You need a wandb project' + assert config.checkpoint.filepath == '' or \ + (config.checkpoint.monitor_index < len(config.datasets.validation.split)), \ + 'You need to monitor a valid dataset' + + # Prepare datasets + config.datasets.train = prep_dataset(config.datasets.train) + config.datasets.validation = prep_dataset(config.datasets.validation) + config.datasets.test = prep_dataset(config.datasets.test) + # Set name and checkpoint + config.name = set_name(config) + # Return configuration + return config + + +def parse_test_file(ckpt_file, cfg_file=None): + """ + Parse model configuration for testing + + Parameters + ---------- + ckpt_file : str + Checkpoint file, with pretrained model + cfg_file : + Configuration file, to update pretrained model configuration + + Returns + ------- + config : CfgNode + Parsed model configuration + state_dict : dict + Model state dict with pretrained weights + """ + assert ckpt_file.endswith('.ckpt') or ckpt_file.endswith('.pth.tar') or ckpt_file.endswith('.pt'), \ + 'You need to provide a .ckpt or .pth.tar file for checkpoint, not {}'.format(ckpt_file) + assert cfg_file is None or cfg_file.endswith('yaml'), \ + 'You need to provide a .yaml file for configuration, not {}'.format(cfg_file) + cfg_default = 'modelscope/models/cv/video_depth_estimation/configs/default_config' + return parse_test_config(ckpt_file, cfg_default, cfg_file) + + +def parse_test_config(ckpt_file, cfg_default, cfg_file): + """ + Parse model configuration for testing + + Parameters + ---------- + ckpt_file : str + Checkpoint file, with pretrained model + cfg_default : str + Default configuration file, with default values + cfg_file : str + Configuration file with updated information + + Returns + ------- + Returns + ------- + config : CfgNode + Parsed model configuration + state_dict : dict + Model state dict with pretrained weights + """ + if ckpt_file.endswith('.ckpt') or ckpt_file.endswith('.pt'): + # Load checkpoint + ckpt = torch.load(ckpt_file, map_location='cpu') + # Get base configuration + config_default = get_default_config(cfg_default) + # Extract configuration and model state + config_model, state_dict = ckpt['config'], ckpt['state_dict'] + # Override default configuration with model configuration + config = merge_cfgs(config_default, config_model) + # Update configuration for backwards compatibility + config = backwards_config(config) + # If another config file is provided, use it + config = merge_cfg_file(config, cfg_file) + # Backwards compatibility with older models + elif ckpt_file.endswith('.pth.tar'): + # Load model state and update it for backwards compatibility + state_dict = torch.load(ckpt_file, map_location='cpu')['state_dict'] + state_dict = backwards_state_dict(state_dict) + # Get default configuration + config = get_default_config(cfg_default) + # If config file is present, update configuration + config = merge_cfg_file(config, cfg_file) + else: + raise ValueError('Unknown checkpoint {}'.format(ckpt_file)) + # Set pretrained model name + config.save.pretrained = ckpt_file + # Return prepared configuration and model state + return prepare_test_config(config), state_dict + + +def prepare_test_config(config): + """ + Prepare model configuration for testing + + Parameters + ---------- + config : CfgNode + Model configuration + + Returns + ------- + config : CfgNode + Prepared model configuration + """ + # Remove train and validation datasets + config.datasets.train.path = config.datasets.validation.path = [] + config.datasets.test = prep_dataset(config.datasets.test) + # Don't save models or log to wandb + config.wandb.dry_run = True + config.checkpoint.filepath = '' + # Return updated configuration + return config diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/depth.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/depth.py new file mode 100644 index 0000000000000000000000000000000000000000..5fbf6aa6ddc138a8aeac89e637b6790545015893 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/depth.py @@ -0,0 +1,450 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import numpy as np +import torch +import torchvision.transforms as transforms +from matplotlib.cm import get_cmap + +from modelscope.models.cv.video_depth_estimation.utils.image import ( + flip_lr, gradient_x, gradient_y, interpolate_image, load_image) +from modelscope.models.cv.video_depth_estimation.utils.types import (is_seq, + is_tensor) + + +def load_depth(file): + """ + Load a depth map from file + Parameters + ---------- + file : str + Depth map filename (.npz or .png) + + Returns + ------- + depth : np.array [H,W] + Depth map (invalid pixels are 0) + """ + if file.endswith('npz'): + return np.load(file)['depth'] + elif file.endswith('png'): + depth_png = np.array(load_image(file), dtype=int) + assert (np.max(depth_png) > 255), 'Wrong .png depth file' + return depth_png.astype(float) / 256. + else: + raise NotImplementedError('Depth extension not supported.') + + +def write_depth(filename, depth, intrinsics=None): + """ + Write a depth map to file, and optionally its corresponding intrinsics. + + Parameters + ---------- + filename : str + File where depth map will be saved (.npz or .png) + depth : np.array [H,W] + Depth map + intrinsics : np.array [3,3] + Optional camera intrinsics matrix + """ + # If depth is a tensor + if is_tensor(depth): + depth = depth.detach().squeeze().cpu() + # If intrinsics is a tensor + if is_tensor(intrinsics): + intrinsics = intrinsics.detach().cpu() + # If we are saving as a .npz + if filename.endswith('.npz'): + np.savez_compressed(filename, depth=depth, intrinsics=intrinsics) + # If we are saving as a .png + elif filename.endswith('.png'): + depth = transforms.ToPILImage()((depth * 256).int()) + depth.save(filename) + # Something is wrong + else: + raise NotImplementedError('Depth filename not valid.') + + +def viz_inv_depth(inv_depth, + normalizer=None, + percentile=95, + colormap='plasma', + filter_zeros=False): + """ + Converts an inverse depth map to a colormap for visualization. + + Parameters + ---------- + inv_depth : torch.Tensor [B,1,H,W] + Inverse depth map to be converted + normalizer : float + Value for inverse depth map normalization + percentile : float + Percentile value for automatic normalization + colormap : str + Colormap to be used + filter_zeros : bool + If True, do not consider zero values during normalization + + Returns + ------- + colormap : np.array [H,W,3] + Colormap generated from the inverse depth map + """ + # If a tensor is provided, convert to numpy + if is_tensor(inv_depth): + # Squeeze if depth channel exists + if len(inv_depth.shape) == 3: + inv_depth = inv_depth.squeeze(0) + inv_depth = inv_depth.detach().cpu().numpy() + cm = get_cmap(colormap) + if normalizer is None: + normalizer = np.percentile( + inv_depth[inv_depth > 0] if filter_zeros else inv_depth, + percentile) + inv_depth /= (normalizer + 1e-6) + return cm(np.clip(inv_depth, 0., 1.0))[:, :, :3] + + +def inv2depth(inv_depth): + """ + Invert an inverse depth map to produce a depth map + + Parameters + ---------- + inv_depth : torch.Tensor or list of torch.Tensor [B,1,H,W] + Inverse depth map + + Returns + ------- + depth : torch.Tensor or list of torch.Tensor [B,1,H,W] + Depth map + """ + if is_seq(inv_depth): + return [inv2depth(item) for item in inv_depth] + else: + depth = 1. / inv_depth.clamp(min=1e-6) + depth[inv_depth <= 0.] = 0. + return depth + + +def depth2inv(depth): + """ + Invert a depth map to produce an inverse depth map + + Parameters + ---------- + depth : torch.Tensor or list of torch.Tensor [B,1,H,W] + Depth map + + Returns + ------- + inv_depth : torch.Tensor or list of torch.Tensor [B,1,H,W] + Inverse depth map + + """ + if is_seq(depth): + return [depth2inv(item) for item in depth] + else: + inv_depth = 1. / depth.clamp(min=1e-6) + inv_depth[depth <= 0.] = 0. + return inv_depth + + +def inv_depths_normalize(inv_depths): + """ + Inverse depth normalization + + Parameters + ---------- + inv_depths : list of torch.Tensor [B,1,H,W] + Inverse depth maps + + Returns + ------- + norm_inv_depths : list of torch.Tensor [B,1,H,W] + Normalized inverse depth maps + """ + mean_inv_depths = [ + inv_depth.mean(2, True).mean(3, True) for inv_depth in inv_depths + ] + return [ + inv_depth / mean_inv_depth.clamp(min=1e-6) + for inv_depth, mean_inv_depth in zip(inv_depths, mean_inv_depths) + ] + + +def calc_smoothness(inv_depths, images, num_scales): + """ + Calculate smoothness values for inverse depths + + Parameters + ---------- + inv_depths : list of torch.Tensor [B,1,H,W] + Inverse depth maps + images : list of torch.Tensor [B,3,H,W] + Inverse depth maps + num_scales : int + Number of scales considered + + Returns + ------- + smoothness_x : list of torch.Tensor [B,1,H,W] + Smoothness values in direction x + smoothness_y : list of torch.Tensor [B,1,H,W] + Smoothness values in direction y + """ + inv_depths_norm = inv_depths_normalize(inv_depths) + inv_depth_gradients_x = [gradient_x(d) for d in inv_depths_norm] + inv_depth_gradients_y = [gradient_y(d) for d in inv_depths_norm] + + image_gradients_x = [gradient_x(image) for image in images] + image_gradients_y = [gradient_y(image) for image in images] + + weights_x = [ + torch.exp(-torch.mean(torch.abs(g), 1, keepdim=True)) + for g in image_gradients_x + ] + weights_y = [ + torch.exp(-torch.mean(torch.abs(g), 1, keepdim=True)) + for g in image_gradients_y + ] + + # Note: Fix gradient addition + smoothness_x = [ + inv_depth_gradients_x[i] * weights_x[i] for i in range(num_scales) + ] + smoothness_y = [ + inv_depth_gradients_y[i] * weights_y[i] for i in range(num_scales) + ] + return smoothness_x, smoothness_y + + +def fuse_inv_depth(inv_depth, inv_depth_hat, method='mean'): + """ + Fuse inverse depth and flipped inverse depth maps + + Parameters + ---------- + inv_depth : torch.Tensor [B,1,H,W] + Inverse depth map + inv_depth_hat : torch.Tensor [B,1,H,W] + Flipped inverse depth map produced from a flipped image + method : str + Method that will be used to fuse the inverse depth maps + + Returns + ------- + fused_inv_depth : torch.Tensor [B,1,H,W] + Fused inverse depth map + """ + if method == 'mean': + return 0.5 * (inv_depth + inv_depth_hat) + elif method == 'max': + return torch.max(inv_depth, inv_depth_hat) + elif method == 'min': + return torch.min(inv_depth, inv_depth_hat) + else: + raise ValueError('Unknown post-process method {}'.format(method)) + + +def post_process_inv_depth(inv_depth, inv_depth_flipped, method='mean'): + """ + Post-process an inverse and flipped inverse depth map + + Parameters + ---------- + inv_depth : torch.Tensor [B,1,H,W] + Inverse depth map + inv_depth_flipped : torch.Tensor [B,1,H,W] + Inverse depth map produced from a flipped image + method : str + Method that will be used to fuse the inverse depth maps + + Returns + ------- + inv_depth_pp : torch.Tensor [B,1,H,W] + Post-processed inverse depth map + """ + B, C, H, W = inv_depth.shape + inv_depth_hat = flip_lr(inv_depth_flipped) + inv_depth_fused = fuse_inv_depth(inv_depth, inv_depth_hat, method=method) + xs = torch.linspace( + 0., 1., W, device=inv_depth.device, + dtype=inv_depth.dtype).repeat(B, C, H, 1) + mask = 1.0 - torch.clamp(20. * (xs - 0.05), 0., 1.) + mask_hat = flip_lr(mask) + return mask_hat * inv_depth + mask * inv_depth_hat + \ + (1.0 - mask - mask_hat) * inv_depth_fused + + +def compute_depth_metrics(config, gt, pred, use_gt_scale=True): + """ + Compute depth metrics from predicted and ground-truth depth maps + + Parameters + ---------- + config : CfgNode + Metrics parameters + gt : torch.Tensor [B,1,H,W] + Ground-truth depth map + pred : torch.Tensor [B,1,H,W] + Predicted depth map + use_gt_scale : bool + True if ground-truth median-scaling is to be used + + Returns + ------- + metrics : torch.Tensor [7] + Depth metrics (abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3) + """ + crop = config.crop == 'garg' + + # Initialize variables + batch_size, _, gt_height, gt_width = gt.shape + SI = SILog = iabs_diff = irmse = abs_diff = abs_rel = sq_rel = rmse = rmse_log = a1 = a2 = a3 = 0.0 + # Interpolate predicted depth to ground-truth resolution + pred = interpolate_image( + pred, gt.shape, mode='bilinear', align_corners=True) + pred = pred.clamp(min=1e-6) + # If using crop + if config.crop == 'garg': + crop = True + crop_mask = torch.zeros(gt.shape[-2:]).byte().type_as(gt) + y1, y2 = int(0.40810811 * gt_height), int(0.99189189 * gt_height) + x1, x2 = int(0.03594771 * gt_width), int(0.96405229 * gt_width) + crop_mask[y1:y2, x1:x2] = 1 + if config.crop == 'eigen_nyu': + crop = True + crop_mask = torch.zeros(gt.shape[-2:]).byte().type_as(gt) + y1, y2 = 20, 459 + x1, x2 = 24, 615 + crop_mask[y1:y2, x1:x2] = 1 + # For each depth map + for pred_i, gt_i in zip(pred, gt): + gt_i, pred_i = torch.squeeze(gt_i), torch.squeeze(pred_i) + # Keep valid pixels (min/max depth and crop) + valid = (gt_i > config.min_depth) & (gt_i < config.max_depth) + valid = valid & crop_mask.bool() if crop else valid + # Stop if there are no remaining valid pixels + if valid.sum() == 0: + continue + # Keep only valid pixels + gt_i, pred_i = gt_i[valid], pred_i[valid] + # Ground-truth median scaling if needed + if use_gt_scale: + pred_i = pred_i * torch.median(gt_i / pred_i) + pred_i = torch.clamp(pred_i, config.min_depth, config.max_depth) + + # Clamp predicted depth values to min/max values + pred_i = pred_i.clamp(config.min_depth, config.max_depth) + + # Calculate depth metrics + + thresh = torch.max((gt_i / pred_i), (pred_i / gt_i)) + a1 += (thresh < 1.25).float().mean() + a2 += (thresh < 1.25**2).float().mean() + a3 += (thresh < 1.25**3).float().mean() + + diff_i = gt_i - pred_i + abs_diff += torch.mean(torch.abs(diff_i)) + abs_rel += torch.mean(torch.abs(diff_i) / gt_i) + sq_rel += torch.mean(diff_i**2 / gt_i) + rmse += torch.sqrt(torch.mean(diff_i**2)) + rmse_log += torch.sqrt( + torch.mean((torch.log(gt_i) - torch.log(pred_i))**2)) + iabs_diff += torch.mean(torch.abs(1.0 / pred_i - 1.0 / gt_i)) + irmse += torch.sqrt(torch.mean((1.0 / gt_i - 1.0 / pred_i)**2)) + d_i = gt_i.log() - pred_i.log() + SILog += ((d_i**2).mean() - (d_i.sum())**2 / len(d_i)**2)**0.5 + SI += ((diff_i**2).mean() - (diff_i.sum())**2 / len(diff_i)**2)**0.5 + # Return average values for each metric + return torch.tensor([ + metric / batch_size for metric in + [abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3, SILog, iabs_diff] + ]).type_as(gt) + + +def compute_depth_metrics_demon(config, gt, gt_pose, pred, use_gt_scale=True): + # Initialize variables + batch_size, _, gt_height, gt_width = gt.shape + SI = SILog = iabs_diff = irmse = abs_diff = abs_rel = sq_rel = rmse = rmse_log = a1 = a2 = a3 = 0.0 + # Interpolate predicted depth to ground-truth resolution + pred = interpolate_image( + pred, gt.shape, mode='bilinear', align_corners=True) + pred = pred.clamp(min=1e-6) + # For each depth map + for pred_i, gt_i, gt_pose_i in zip(pred, gt, gt_pose): + gt_i, pred_i = torch.squeeze(gt_i), torch.squeeze(pred_i) + # Keep valid pixels (min/max depth and crop) + valid = (gt_i > config.min_depth) & (gt_i < config.max_depth) + # Stop if there are no remaining valid pixels + if valid.sum() == 0: + continue + # Keep only valid pixels + gt_i, pred_i = gt_i[valid], pred_i[valid] + + # Ground-truth median scaling if needed + if use_gt_scale: + # gt normalization + translation_gt = gt_pose_i[:, :3, 3] + translation_norm = torch.sqrt(translation_gt[0].dot( + translation_gt[0])) + gt_i = gt_i / translation_norm + + pred_i = pred_i * torch.median(gt_i / pred_i) + # d1d1 = pred_i * pred_i + # d1d2 = pred_i * gt_i + # mask = d1d2 > 0 + # sum_d1d1 = d1d1[mask].sum() + # sum_d1d2 = d1d2[mask].sum() + # scale = sum_d1d2/sum_d1d1 + # pred_i = pred_i * scale + + # Calculate depth metrics + thresh = torch.max((gt_i / pred_i), (pred_i / gt_i)) + a1 += (thresh < 1.25).float().mean() + a2 += (thresh < 1.25**2).float().mean() + a3 += (thresh < 1.25**3).float().mean() + + diff_i = gt_i - pred_i + abs_diff += torch.mean(torch.abs(diff_i)) + abs_rel += torch.mean(torch.abs(diff_i) / gt_i) + sq_rel += torch.mean(diff_i**2 / gt_i) + rmse += torch.sqrt(torch.mean(diff_i**2)) + rmse_log += torch.sqrt( + torch.mean((torch.log(gt_i) - torch.log(pred_i))**2)) + iabs_diff += torch.mean(torch.abs(1.0 / pred_i - 1.0 / gt_i)) + irmse += torch.sqrt(torch.mean((1.0 / gt_i - 1.0 / pred_i)**2)) + d_i = gt_i.log() - pred_i.log() + SILog += ((d_i**2).mean() - (d_i.sum())**2 / len(d_i)**2)**0.5 + SI += ((diff_i**2).mean() - (diff_i.sum())**2 / len(diff_i)**2)**0.5 + # Return average values for each metric + return torch.tensor([ + metric / batch_size for metric in + [abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3, SILog, iabs_diff] + ]).type_as(gt) + + +def compute_pose_metrics(config, gt, pred): + pr = pred[0].mat.squeeze().detach().cpu().numpy() + gt = gt[0].squeeze().detach().cpu().numpy() + + # seperate rotations and translations + R1, t1 = gt[:3, :3], gt[:3, 3] + R2, t2 = pr[:3, :3], pr[:3, 3] + + costheta = (np.trace(np.dot(R1.T, R2)) - 1.0) / 2.0 + costheta = np.minimum(costheta, 1.0) + rdeg = np.arccos(costheta) * (180 / np.pi) + + t1mag = np.sqrt(np.dot(t1, t1)) + t2mag = np.sqrt(np.dot(t2, t2)) + costheta = np.dot(t1, t2) / (t1mag * t2mag) + tdeg = np.arccos(costheta) * (180 / np.pi) + + # fit scales to translations + a = np.dot(t1, t2) / np.dot(t2, t2) + tcm = 100 * np.sqrt(np.sum((t1 - a * t2)**2, axis=-1)) + + return torch.Tensor([rdeg, tdeg, tcm]) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/horovod.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/horovod.py new file mode 100644 index 0000000000000000000000000000000000000000..2c845c5af3ef69e5f466ce8ee2a6820576115d03 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/horovod.py @@ -0,0 +1,61 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +try: + import horovod.torch as hvd + HAS_HOROVOD = True +except ImportError: + HAS_HOROVOD = False + + +def hvd_disable(): + global HAS_HOROVOD + HAS_HOROVOD = False + + +def hvd_init(): + if HAS_HOROVOD: + hvd.init() + return HAS_HOROVOD + + +def on_rank_0(func): + + def wrapper(*args, **kwargs): + if rank() == 0: + func(*args, **kwargs) + + return wrapper + + +def rank(): + return hvd.rank() if HAS_HOROVOD else 0 + + +def world_size(): + return hvd.size() if HAS_HOROVOD else 1 + + +@on_rank_0 +def print0(string='\n'): + print(string) + + +def reduce_value(value, average, name): + """ + Reduce the mean value of a tensor from all GPUs + + Parameters + ---------- + value : torch.Tensor + Value to be reduced + average : bool + Whether values will be averaged or not + name : str + Value name + + Returns + ------- + value : torch.Tensor + reduced value + """ + return hvd.allreduce(value, average=average, name=name) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image.py new file mode 100644 index 0000000000000000000000000000000000000000..4d58de576797903ff0061dc47b2caef983df01a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image.py @@ -0,0 +1,422 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import os +from functools import lru_cache + +import cv2 +import numpy as np +import torch +import torch.nn.functional as funct +from PIL import Image + +from modelscope.models.cv.video_depth_estimation.utils.misc import same_shape + + +def parse_video(video_file, save_root, sample_rate=10): + os.makedirs(save_root, exist_ok=True) + + cap = cv2.VideoCapture(video_file) + + # Check if camera opened successfully + if (cap.isOpened() is False): + print('Error opening video stream or file') + count = 0 + sample_count = 0 + + while cap.isOpened(): + ret, img = cap.read() + if ret: + if count % sample_rate == 0: + save_path = os.path.join(save_root, + f'{sample_count}'.zfill(6) + '.jpg') + cv2.imwrite(save_path, img) + sample_count += 1 + count += 1 + else: + break + print( + f'video total frames num: {count}, sampled frames num:{sample_count}') + + +def get_intrinsics(image_shape_raw, image_shape, data_type): + if data_type == 'kitti': + intr = np.array([ + 7.215376999999999725e+02, 0.000000000000000000e+00, + 6.095593000000000075e+02, 0.000000000000000000e+00, + 7.215376999999999725e+02, 1.728540000000000134e+02, + 0.000000000000000000e+00, 0.000000000000000000e+00, + 1.000000000000000000e+00 + ]).reshape(3, 3) + elif data_type == 'indoor': + intr = np.array([ + 1170.187988, 0.000000, 647.750000, 0.000000, 1170.187988, + 483.750000, 0.000000, 0.000000, 1.000000 + ]).reshape(3, 3) + else: + # print("fake intrinsics") + w, h = image_shape_raw + fx = w * 1.2 + fy = w * 1.2 + cx = w / 2.0 + cy = h / 2.0 + intr = np.array([[fx, 0., cx], [0., fy, cy], [0., 0., 1.]]) + + orig_w, orig_h = image_shape_raw + out_h, out_w = image_shape + + # Scale intrinsics + intr[0] *= out_w / orig_w + intr[1] *= out_h / orig_h + + return intr + + +def load_image(path): + """ + Read an image using PIL + + Parameters + ---------- + path : str + Path to the image + + Returns + ------- + image : PIL.Image + Loaded image + """ + return Image.open(path) + + +def write_image(filename, image): + """ + Write an image to file. + + Parameters + ---------- + filename : str + File where image will be saved + image : np.array [H,W,3] + RGB image + """ + cv2.imwrite(filename, image[:, :, ::-1]) + + +def flip_lr(image): + """ + Flip image horizontally + + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Image to be flipped + + Returns + ------- + image_flipped : torch.Tensor [B,3,H,W] + Flipped image + """ + assert image.dim() == 4, 'You need to provide a [B,C,H,W] image to flip' + return torch.flip(image, [3]) + + +def flip_lr_intr(intr, width): + """ + Flip image horizontally + + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Image to be flipped + + Returns + ------- + image_flipped : torch.Tensor [B,3,H,W] + Flipped image + """ + assert intr.shape[1:] == (3, 3) + # trans = torch.eye(3, dtype=intr.dtype, device=intr.device) + # trans[0, 0] = -1 + # intr_trans = torch.matmul(trans.unsqueeze(0), intr) + intr[:, 0, 0] = -1 * intr[:, 0, 0] + intr[:, 0, 2] = width - intr[:, 0, 2] + return intr + + +def flip_model(model, image, flip): + """ + Flip input image and flip output inverse depth map + + Parameters + ---------- + model : nn.Module + Module to be used + image : torch.Tensor [B,3,H,W] + Input image + flip : bool + True if the flip is happening + + Returns + ------- + inv_depths : list of torch.Tensor [B,1,H,W] + List of predicted inverse depth maps + """ + if flip: + return [flip_lr(inv_depth) for inv_depth in model(flip_lr(image))] + else: + return model(image) + + +def flip_mf_model(model, image, ref_imgs, intrinsics, flip): + """ + Flip input image and flip output inverse depth map + + Parameters + ---------- + model : nn.Module + Module to be used + image : torch.Tensor [B,3,H,W] + Input image + flip : bool + True if the flip is happening + + Returns + ------- + inv_depths : list of torch.Tensor [B,1,H,W] + List of predicted inverse depth maps + """ + if flip: + if ref_imgs is not None: + return model( + flip_lr(image), [flip_lr(img) for img in ref_imgs], intrinsics) + else: + return model(flip_lr(image), None, intrinsics) + else: + return model(image, ref_imgs, intrinsics) + + +######################################################################################################################## + + +def gradient_x(image): + """ + Calculates the gradient of an image in the x dimension + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Input image + + Returns + ------- + gradient_x : torch.Tensor [B,3,H,W-1] + Gradient of image with respect to x + """ + return image[:, :, :, :-1] - image[:, :, :, 1:] + + +def gradient_y(image): + """ + Calculates the gradient of an image in the y dimension + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Input image + + Returns + ------- + gradient_y : torch.Tensor [B,3,H-1,W] + Gradient of image with respect to y + """ + return image[:, :, :-1, :] - image[:, :, 1:, :] + + +######################################################################################################################## + + +def interpolate_image(image, shape, mode='bilinear', align_corners=True): + """ + Interpolate an image to a different resolution + + Parameters + ---------- + image : torch.Tensor [B,?,h,w] + Image to be interpolated + shape : tuple (H, W) + Output shape + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + image : torch.Tensor [B,?,H,W] + Interpolated image + """ + # Take last two dimensions as shape + if len(shape) > 2: + shape = shape[-2:] + # If the shapes are the same, do nothing + if same_shape(image.shape[-2:], shape): + return image + else: + # Interpolate image to match the shape + return funct.interpolate( + image, size=shape, mode=mode, align_corners=align_corners) + + +def interpolate_scales(images, + shape=None, + mode='bilinear', + align_corners=False): + """ + Interpolate list of images to the same shape + + Parameters + ---------- + images : list of torch.Tensor [B,?,?,?] + Images to be interpolated, with different resolutions + shape : tuple (H, W) + Output shape + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + images : list of torch.Tensor [B,?,H,W] + Interpolated images, with the same resolution + """ + # If no shape is provided, interpolate to highest resolution + if shape is None: + shape = images[0].shape + # Take last two dimensions as shape + if len(shape) > 2: + shape = shape[-2:] + # Interpolate all images + return [ + funct.interpolate( + image, shape, mode=mode, align_corners=align_corners) + for image in images + ] + + +def match_scales(image, + targets, + num_scales, + mode='bilinear', + align_corners=True): + """ + Interpolate one image to produce a list of images with the same shape as targets + + Parameters + ---------- + image : torch.Tensor [B,?,h,w] + Input image + targets : list of torch.Tensor [B,?,?,?] + Tensors with the target resolutions + num_scales : int + Number of considered scales + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + images : list of torch.Tensor [B,?,?,?] + List of images with the same resolutions as targets + """ + # For all scales + images = [] + image_shape = image.shape[-2:] + for i in range(num_scales): + target_shape = targets[i].shape + # If image shape is equal to target shape + if same_shape(image_shape, target_shape): + images.append(image) + else: + # Otherwise, interpolate + images.append( + interpolate_image( + image, + target_shape, + mode=mode, + align_corners=align_corners)) + # Return scaled images + return images + + +######################################################################################################################## + + +@lru_cache(maxsize=None) +def meshgrid(B, H, W, dtype, device, normalized=False): + """ + Create meshgrid with a specific resolution + + Parameters + ---------- + B : int + Batch size + H : int + Height size + W : int + Width size + dtype : torch.dtype + Meshgrid type + device : torch.device + Meshgrid device + normalized : bool + True if grid is normalized between -1 and 1 + + Returns + ------- + xs : torch.Tensor [B,1,W] + Meshgrid in dimension x + ys : torch.Tensor [B,H,1] + Meshgrid in dimension y + """ + if normalized: + xs = torch.linspace(-1, 1, W, device=device, dtype=dtype) + ys = torch.linspace(-1, 1, H, device=device, dtype=dtype) + else: + xs = torch.linspace(0, W - 1, W, device=device, dtype=dtype) + ys = torch.linspace(0, H - 1, H, device=device, dtype=dtype) + ys, xs = torch.meshgrid([ys, xs]) + return xs.repeat([B, 1, 1]), ys.repeat([B, 1, 1]) + + +@lru_cache(maxsize=None) +def image_grid(B, H, W, dtype, device, normalized=False): + """ + Create an image grid with a specific resolution + + Parameters + ---------- + B : int + Batch size + H : int + Height size + W : int + Width size + dtype : torch.dtype + Meshgrid type + device : torch.device + Meshgrid device + normalized : bool + True if grid is normalized between -1 and 1 + + Returns + ------- + grid : torch.Tensor [B,3,H,W] + Image grid containing a meshgrid in x, y and 1 + """ + xs, ys = meshgrid(B, H, W, dtype, device, normalized=normalized) + ones = torch.ones_like(xs) + grid = torch.stack([xs, ys, ones], dim=1) + return grid + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image_gt.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image_gt.py new file mode 100644 index 0000000000000000000000000000000000000000..a751614b870b8ac59cedf4ea805c485188b2c30e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/image_gt.py @@ -0,0 +1,370 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +from functools import lru_cache + +import cv2 +import torch +import torch.nn.functional as funct +from PIL import Image + +from modelscope.models.cv.video_depth_estimation.utils.misc import same_shape + + +def load_image(path): + """ + Read an image using PIL + + Parameters + ---------- + path : str + Path to the image + + Returns + ------- + image : PIL.Image + Loaded image + """ + return Image.open(path) + + +def write_image(filename, image): + """ + Write an image to file. + + Parameters + ---------- + filename : str + File where image will be saved + image : np.array [H,W,3] + RGB image + """ + cv2.imwrite(filename, image[:, :, ::-1]) + + +def flip_lr(image): + """ + Flip image horizontally + + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Image to be flipped + + Returns + ------- + image_flipped : torch.Tensor [B,3,H,W] + Flipped image + """ + assert image.dim() == 4, 'You need to provide a [B,C,H,W] image to flip' + return torch.flip(image, [3]) + + +def flip_lr_intr(intr, width): + """ + Flip image horizontally + + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Image to be flipped + + Returns + ------- + image_flipped : torch.Tensor [B,3,H,W] + Flipped image + """ + assert intr.shape[1:] == (3, 3) + # trans = torch.eye(3, dtype=intr.dtype, device=intr.device) + # trans[0, 0] = -1 + # intr_trans = torch.matmul(trans.unsqueeze(0), intr) + intr[:, 0, 0] = -1 * intr[:, 0, 0] + intr[:, 0, 2] = width - intr[:, 0, 2] + return intr + + +def flip_model(model, image, flip): + """ + Flip input image and flip output inverse depth map + + Parameters + ---------- + model : nn.Module + Module to be used + image : torch.Tensor [B,3,H,W] + Input image + flip : bool + True if the flip is happening + + Returns + ------- + inv_depths : list of torch.Tensor [B,1,H,W] + List of predicted inverse depth maps + """ + if flip: + return [flip_lr(inv_depth) for inv_depth in model(flip_lr(image))] + else: + return model(image) + + +def flip_mf_model(model, + image, + ref_imgs, + intrinsics, + flip, + gt_depth=None, + gt_poses=None): + """ + Flip input image and flip output inverse depth map + + Parameters + ---------- + model : nn.Module + Module to be used + image : torch.Tensor [B,3,H,W] + Input image + flip : bool + True if the flip is happening + + Returns + ------- + inv_depths : list of torch.Tensor [B,1,H,W] + List of predicted inverse depth maps + """ + if flip: + if ref_imgs is not None: + return model( + flip_lr(image), [flip_lr(img) for img in ref_imgs], intrinsics, + None, flip_lr(gt_depth), gt_poses) + else: + return model( + flip_lr(image), None, intrinsics, None, flip_lr(gt_depth), + gt_poses) + else: + return model(image, ref_imgs, intrinsics, None, gt_depth, gt_poses) + + +######################################################################################################################## + + +def gradient_x(image): + """ + Calculates the gradient of an image in the x dimension + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Input image + + Returns + ------- + gradient_x : torch.Tensor [B,3,H,W-1] + Gradient of image with respect to x + """ + return image[:, :, :, :-1] - image[:, :, :, 1:] + + +def gradient_y(image): + """ + Calculates the gradient of an image in the y dimension + Parameters + ---------- + image : torch.Tensor [B,3,H,W] + Input image + + Returns + ------- + gradient_y : torch.Tensor [B,3,H-1,W] + Gradient of image with respect to y + """ + return image[:, :, :-1, :] - image[:, :, 1:, :] + + +######################################################################################################################## + + +def interpolate_image(image, shape, mode='bilinear', align_corners=True): + """ + Interpolate an image to a different resolution + + Parameters + ---------- + image : torch.Tensor [B,?,h,w] + Image to be interpolated + shape : tuple (H, W) + Output shape + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + image : torch.Tensor [B,?,H,W] + Interpolated image + """ + # Take last two dimensions as shape + if len(shape) > 2: + shape = shape[-2:] + # If the shapes are the same, do nothing + if same_shape(image.shape[-2:], shape): + return image + else: + # Interpolate image to match the shape + return funct.interpolate( + image, size=shape, mode=mode, align_corners=align_corners) + + +def interpolate_scales(images, + shape=None, + mode='bilinear', + align_corners=False): + """ + Interpolate list of images to the same shape + + Parameters + ---------- + images : list of torch.Tensor [B,?,?,?] + Images to be interpolated, with different resolutions + shape : tuple (H, W) + Output shape + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + images : list of torch.Tensor [B,?,H,W] + Interpolated images, with the same resolution + """ + # If no shape is provided, interpolate to highest resolution + if shape is None: + shape = images[0].shape + # Take last two dimensions as shape + if len(shape) > 2: + shape = shape[-2:] + # Interpolate all images + return [ + funct.interpolate( + image, shape, mode=mode, align_corners=align_corners) + for image in images + ] + + +def match_scales(image, + targets, + num_scales, + mode='bilinear', + align_corners=True): + """ + Interpolate one image to produce a list of images with the same shape as targets + + Parameters + ---------- + image : torch.Tensor [B,?,h,w] + Input image + targets : list of torch.Tensor [B,?,?,?] + Tensors with the target resolutions + num_scales : int + Number of considered scales + mode : str + Interpolation mode + align_corners : bool + True if corners will be aligned after interpolation + + Returns + ------- + images : list of torch.Tensor [B,?,?,?] + List of images with the same resolutions as targets + """ + # For all scales + images = [] + image_shape = image.shape[-2:] + for i in range(num_scales): + target_shape = targets[i].shape + # If image shape is equal to target shape + if same_shape(image_shape, target_shape): + images.append(image) + else: + # Otherwise, interpolate + images.append( + interpolate_image( + image, + target_shape, + mode=mode, + align_corners=align_corners)) + # Return scaled images + return images + + +######################################################################################################################## + + +@lru_cache(maxsize=None) +def meshgrid(B, H, W, dtype, device, normalized=False): + """ + Create meshgrid with a specific resolution + + Parameters + ---------- + B : int + Batch size + H : int + Height size + W : int + Width size + dtype : torch.dtype + Meshgrid type + device : torch.device + Meshgrid device + normalized : bool + True if grid is normalized between -1 and 1 + + Returns + ------- + xs : torch.Tensor [B,1,W] + Meshgrid in dimension x + ys : torch.Tensor [B,H,1] + Meshgrid in dimension y + """ + if normalized: + xs = torch.linspace(-1, 1, W, device=device, dtype=dtype) + ys = torch.linspace(-1, 1, H, device=device, dtype=dtype) + else: + xs = torch.linspace(0, W - 1, W, device=device, dtype=dtype) + ys = torch.linspace(0, H - 1, H, device=device, dtype=dtype) + ys, xs = torch.meshgrid([ys, xs]) + return xs.repeat([B, 1, 1]), ys.repeat([B, 1, 1]) + + +@lru_cache(maxsize=None) +def image_grid(B, H, W, dtype, device, normalized=False): + """ + Create an image grid with a specific resolution + + Parameters + ---------- + B : int + Batch size + H : int + Height size + W : int + Width size + dtype : torch.dtype + Meshgrid type + device : torch.device + Meshgrid device + normalized : bool + True if grid is normalized between -1 and 1 + + Returns + ------- + grid : torch.Tensor [B,3,H,W] + Image grid containing a meshgrid in x, y and 1 + """ + xs, ys = meshgrid(B, H, W, dtype, device, normalized=normalized) + ones = torch.ones_like(xs) + grid = torch.stack([xs, ys, ones], dim=1) + return grid + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/load.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/load.py new file mode 100644 index 0000000000000000000000000000000000000000..8c2b326c18e603b62735d237a39b033b715bb7ec --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/load.py @@ -0,0 +1,204 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import importlib +import logging +import os +import warnings +from collections import OrderedDict +from inspect import signature + +import torch + +from modelscope.models.cv.video_depth_estimation.utils.horovod import print0 +from modelscope.models.cv.video_depth_estimation.utils.misc import (make_list, + pcolor, + same_shape) +from modelscope.models.cv.video_depth_estimation.utils.types import is_str + + +def set_debug(debug): + """ + Enable or disable debug terminal logging + + Parameters + ---------- + debug : bool + Debugging flag (True to enable) + """ + # Disable logging if requested + if not debug: + os.environ['NCCL_DEBUG'] = '' + os.environ['WANDB_SILENT'] = 'false' + warnings.filterwarnings('ignore') + logging.disable(logging.CRITICAL) + + +def filter_args(func, keys): + """ + Filters a dictionary so it only contains keys that are arguments of a function + + Parameters + ---------- + func : Function + Function for which we are filtering the dictionary + keys : dict + Dictionary with keys we are filtering + + Returns + ------- + filtered : dict + Dictionary containing only keys that are arguments of func + """ + filtered = {} + sign = list(signature(func).parameters.keys()) + for k, v in {**keys}.items(): + if k in sign: + filtered[k] = v + return filtered + + +def filter_args_create(func, keys): + """ + Filters a dictionary so it only contains keys that are arguments of a function + and creates a function with those arguments + + Parameters + ---------- + func : Function + Function for which we are filtering the dictionary + keys : dict + Dictionary with keys we are filtering + + Returns + ------- + func : Function + Function with filtered keys as arguments + """ + return func(**filter_args(func, keys)) + + +def load_class(filename, paths, concat=True): + """ + Look for a file in different locations and return its method with the same name + Optionally, you can use concat to search in path.filename instead + + Parameters + ---------- + filename : str + Name of the file we are searching for + paths : str or list of str + Folders in which the file will be searched + concat : bool + Flag to concatenate filename to each path during the search + + Returns + ------- + method : Function + Loaded method + """ + # for each path in paths + for path in make_list(paths): + # Create full path + full_path = '{}.{}'.format(path, filename) if concat else path + if importlib.util.find_spec(full_path): + # Return method with same name as the file + return getattr(importlib.import_module(full_path), filename) + raise ValueError('Unknown class {}'.format(filename)) + + +def load_class_args_create(filename, paths, args={}, concat=True): + """Loads a class (filename) and returns an instance with filtered arguments (args)""" + class_type = load_class(filename, paths, concat) + return filter_args_create(class_type, args) + + +def load_network(network, path, prefixes=''): + """ + Loads a pretrained network + + Parameters + ---------- + network : nn.Module + Network that will receive the pretrained weights + path : str + File containing a 'state_dict' key with pretrained network weights + prefixes : str or list of str + Layer name prefixes to consider when loading the network + + Returns + ------- + network : nn.Module + Updated network with pretrained weights + """ + prefixes = make_list(prefixes) + # If path is a string + if is_str(path): + saved_state_dict = torch.load(path, map_location='cpu')['state_dict'] + if path.endswith('.pth.tar'): + saved_state_dict = backwards_state_dict(saved_state_dict) + # If state dict is already provided + else: + saved_state_dict = path + # Get network state dict + network_state_dict = network.state_dict() + + updated_state_dict = OrderedDict() + n, n_total = 0, len(network_state_dict.keys()) + for key, val in saved_state_dict.items(): + for prefix in prefixes: + prefix = prefix + '.' + if prefix in key: + idx = key.find(prefix) + len(prefix) + key = key[idx:] + if key in network_state_dict.keys() and \ + same_shape(val.shape, network_state_dict[key].shape): + updated_state_dict[key] = val + n += 1 + try: + network.load_state_dict(updated_state_dict, strict=True) + except Exception as e: + print(e) + network.load_state_dict(updated_state_dict, strict=False) + base_color, attrs = 'cyan', ['bold', 'dark'] + color = 'green' if n == n_total else 'yellow' if n > 0 else 'red' + print0( + pcolor( + '=====###### Pretrained {} loaded:'.format(prefixes[0]), + base_color, + attrs=attrs) + + pcolor(' {}/{} '.format(n, n_total), color, attrs=attrs) + + pcolor('tensors', base_color, attrs=attrs)) + return network + + +def backwards_state_dict(state_dict): + """ + Modify the state dict of older models for backwards compatibility + + Parameters + ---------- + state_dict : dict + Model state dict with pretrained weights + + Returns + ------- + state_dict : dict + Updated model state dict with modified layer names + """ + # List of layer names to change + changes = (('model.model', 'model'), ('pose_network', 'pose_net'), + ('disp_network', 'depth_net')) + # Iterate over all keys and values + updated_state_dict = OrderedDict() + for key, val in state_dict.items(): + # Ad hoc changes due to version changes + key = '{}.{}'.format('model', key) + if 'disp_network' in key: + key = key.replace('conv3.0.weight', 'conv3.weight') + key = key.replace('conv3.0.bias', 'conv3.bias') + # Change layer names + for change in changes: + key = key.replace('{}.'.format(change[0]), '{}.'.format(change[1])) + updated_state_dict[key] = val + # Return updated state dict + return updated_state_dict diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/misc.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..b66a8c744c1ce7681b6117680ccf45bd2fd2a92f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/misc.py @@ -0,0 +1,107 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +from termcolor import colored + +from modelscope.models.cv.video_depth_estimation.utils.types import is_list + +######################################################################################################################## + + +def filter_dict(dictionary, keywords): + """ + Returns only the keywords that are part of a dictionary + + Parameters + ---------- + dictionary : dict + Dictionary for filtering + keywords : list of str + Keywords that will be filtered + + Returns + ------- + keywords : list of str + List containing the keywords that are keys in dictionary + """ + return [key for key in keywords if key in dictionary] + + +######################################################################################################################## + + +def make_list(var, n=None): + """ + Wraps the input into a list, and optionally repeats it to be size n + + Parameters + ---------- + var : Any + Variable to be wrapped in a list + n : int + How much the wrapped variable will be repeated + + Returns + ------- + var_list : list + List generated from var + """ + var = var if is_list(var) else [var] + if n is None: + return var + else: + assert len(var) == 1 or len( + var) == n, 'Wrong list length for make_list' + return var * n if len(var) == 1 else var + + +######################################################################################################################## + + +def same_shape(shape1, shape2): + """ + Checks if two shapes are the same + + Parameters + ---------- + shape1 : tuple + First shape + shape2 : tuple + Second shape + + Returns + ------- + flag : bool + True if both shapes are the same (same length and dimensions) + """ + if len(shape1) != len(shape2): + return False + for i in range(len(shape1)): + if shape1[i] != shape2[i]: + return False + return True + + +######################################################################################################################## + + +def pcolor(string, color, on_color=None, attrs=None): + """ + Produces a colored string for printing + + Parameters + ---------- + string : str + String that will be colored + color : str + Color to use + on_color : str + Background color to use + attrs : list of str + Different attributes for the string + + Returns + ------- + string: str + Colored string + """ + return colored(string, color, on_color, attrs) diff --git a/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/types.py b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/types.py new file mode 100644 index 0000000000000000000000000000000000000000..d9cbd98aeb46d6aff9609dada5a1e3832ed4de4e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_depth_estimation/utils/types.py @@ -0,0 +1,55 @@ +# Part of the implementation is borrowed and modified from PackNet-SfM, +# made publicly available under the MIT License at https://github.com/TRI-ML/packnet-sfm +import numpy as np +import torch +import yacs + +######################################################################################################################## + + +def is_numpy(data): + """Checks if data is a numpy array.""" + return isinstance(data, np.ndarray) + + +def is_tensor(data): + """Checks if data is a torch tensor.""" + return type(data) == torch.Tensor + + +def is_tuple(data): + """Checks if data is a tuple.""" + return isinstance(data, tuple) + + +def is_list(data): + """Checks if data is a list.""" + return isinstance(data, list) + + +def is_dict(data): + """Checks if data is a dictionary.""" + return isinstance(data, dict) + + +def is_str(data): + """Checks if data is a string.""" + return isinstance(data, str) + + +def is_int(data): + """Checks if data is an integer.""" + return isinstance(data, int) + + +def is_seq(data): + """Checks if data is a list or tuple.""" + return is_tuple(data) or is_list(data) + + +def is_cfg(data): + """Checks if data is a configuration node""" + return type(data) == yacs.config.CfgNode + + +######################################################################################################################## diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_arch.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..33486cf9f0b615307fec733aef5974e052786efc --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_arch.py @@ -0,0 +1,53 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +import torch.nn as nn + +from modelscope.models.cv.video_frame_interpolation.flow_model.raft import RAFT +from modelscope.models.cv.video_frame_interpolation.interp_model.IFNet_swin import \ + IFNet +from modelscope.models.cv.video_frame_interpolation.interp_model.refinenet_arch import ( + InterpNet, InterpNetDs) + + +class VFINet(nn.Module): + + def __init__(self, args, Ds_flag=False): + super(VFINet, self).__init__() + self.flownet = RAFT(args) + self.internet = InterpNet() + if Ds_flag: + self.internet_Ds = InterpNetDs() + + def img_trans(self, img_tensor): # in format of RGB + img_tensor = img_tensor / 255.0 + mean = torch.Tensor([0.429, 0.431, 0.397]).view(1, 3, 1, + 1).type_as(img_tensor) + img_tensor -= mean + return img_tensor + + def add_mean(self, x): + mean = torch.Tensor([0.429, 0.431, 0.397]).view(1, 3, 1, 1).type_as(x) + return x + mean + + def forward(self, imgs, timestep=0.5): + self.flownet.eval() + self.internet.eval() + with torch.no_grad(): + img0 = imgs[:, :3] + img1 = imgs[:, 3:6] + img2 = imgs[:, 6:9] + img3 = imgs[:, 9:12] + + _, F10_up = self.flownet(img1, img0, iters=12, test_mode=True) + _, F12_up = self.flownet(img1, img2, iters=12, test_mode=True) + _, F21_up = self.flownet(img2, img1, iters=12, test_mode=True) + _, F23_up = self.flownet(img2, img3, iters=12, test_mode=True) + + img1 = self.img_trans(img1.clone()) + img2 = self.img_trans(img2.clone()) + + It_warp = self.internet( + img1, img2, F10_up, F12_up, F21_up, F23_up, timestep=timestep) + It_warp = self.add_mean(It_warp) + + return It_warp diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_for_video_frame_interpolation.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_for_video_frame_interpolation.py new file mode 100644 index 0000000000000000000000000000000000000000..a7ea00e14a0b0dc9a5bde0bacee9fa2159998ed6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/VFINet_for_video_frame_interpolation.py @@ -0,0 +1,98 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from copy import deepcopy +from typing import Any, Dict, Union + +import torch.cuda +import torch.nn.functional as F +from torch.nn.parallel import DataParallel, DistributedDataParallel + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +# from modelscope.models.cv.video_super_resolution.common import charbonnier_loss +from modelscope.models.cv.video_frame_interpolation.VFINet_arch import VFINet +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() +__all__ = ['VFINetForVideoFrameInterpolation'] + + +def convert(param): + return { + k.replace('module.', ''): v + for k, v in param.items() if 'module.' in k + } + + +@MODELS.register_module( + Tasks.video_frame_interpolation, + module_name=Models.video_frame_interpolation) +class VFINetForVideoFrameInterpolation(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the video frame-interpolation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + + """ + super().__init__(model_dir, *args, **kwargs) + + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + + self.model_dir = model_dir + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + + flownet_path = os.path.join(model_dir, 'raft-sintel.pt') + internet_path = os.path.join(model_dir, 'interpnet.pt') + + self.model = VFINet(self.config.model.network, Ds_flag=True) + self._load_pretrained(flownet_path, internet_path) + + def _load_pretrained(self, flownet_path, internet_path): + state_dict_flownet = torch.load( + flownet_path, map_location=self._device) + state_dict_internet = torch.load( + internet_path, map_location=self._device) + + self.model.flownet.load_state_dict( + convert(state_dict_flownet), strict=True) + self.model.internet.load_state_dict( + convert(state_dict_internet), strict=True) + self.model.internet_Ds.load_state_dict( + convert(state_dict_internet), strict=True) + logger.info('load model done.') + + def _inference_forward(self, input: Tensor) -> Dict[str, Tensor]: + return {'output': self.model(input)} + + def _evaluate_postprocess(self, input: Tensor, + target: Tensor) -> Dict[str, list]: + preds = self.model(input) + del input + torch.cuda.empty_cache() + return {'pred': preds, 'target': target} + + def forward(self, inputs: Dict[str, + Tensor]) -> Dict[str, Union[list, Tensor]]: + """return the result by the model + + Args: + inputs (Tensor): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + + if 'target' in inputs: + return self._evaluate_postprocess(**inputs) + else: + return self._inference_forward(**inputs) diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/__init__.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..657a375ad812ca055367a287902285959fc199b2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .VFINet_arch import VFINet + +else: + _import_structure = {'VFINet_arch': ['VFINet']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/__init__.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/corr.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/corr.py new file mode 100644 index 0000000000000000000000000000000000000000..860094059b27a989ec6c41e87634dd73b979fc54 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/corr.py @@ -0,0 +1,92 @@ +# The implementation is adopted from RAFT, +# made publicly available under the BSD-3-Clause license at https://github.com/princeton-vl/RAFT + +import torch +import torch.nn.functional as F + +from modelscope.models.cv.video_frame_interpolation.utils.utils import ( + bilinear_sampler, coords_grid) + + +class CorrBlock: + + def __init__(self, fmap1, fmap2, num_levels=4, radius=4): + self.num_levels = num_levels + self.radius = radius + self.corr_pyramid = [] + + # all pairs correlation + corr = CorrBlock.corr(fmap1, fmap2) + + batch, h1, w1, dim, h2, w2 = corr.shape + corr = corr.reshape(batch * h1 * w1, dim, h2, w2) + + self.corr_pyramid.append(corr) + for i in range(self.num_levels - 1): + corr = F.avg_pool2d(corr, 2, stride=2) + self.corr_pyramid.append(corr) + + def __call__(self, coords): + r = self.radius + coords = coords.permute(0, 2, 3, 1) + batch, h1, w1, _ = coords.shape + + out_pyramid = [] + for i in range(self.num_levels): + corr = self.corr_pyramid[i] + dx = torch.linspace(-r, r, 2 * r + 1, device=coords.device) + dy = torch.linspace(-r, r, 2 * r + 1, device=coords.device) + delta = torch.stack(torch.meshgrid(dy, dx), axis=-1) + + centroid_lvl = coords.reshape(batch * h1 * w1, 1, 1, 2) / 2**i + delta_lvl = delta.view(1, 2 * r + 1, 2 * r + 1, 2) + coords_lvl = centroid_lvl + delta_lvl + + corr = bilinear_sampler(corr, coords_lvl) + corr = corr.view(batch, h1, w1, -1) + out_pyramid.append(corr) + + out = torch.cat(out_pyramid, dim=-1) + return out.permute(0, 3, 1, 2).contiguous().float() + + @staticmethod + def corr(fmap1, fmap2): + batch, dim, ht, wd = fmap1.shape + fmap1 = fmap1.view(batch, dim, ht * wd) + fmap2 = fmap2.view(batch, dim, ht * wd) + + corr = torch.matmul(fmap1.transpose(1, 2), fmap2) + corr = corr.view(batch, ht, wd, 1, ht, wd) + return corr / torch.sqrt(torch.tensor(dim).float()) + + +class AlternateCorrBlock: + + def __init__(self, fmap1, fmap2, num_levels=4, radius=4): + self.num_levels = num_levels + self.radius = radius + + self.pyramid = [(fmap1, fmap2)] + for i in range(self.num_levels): + fmap1 = F.avg_pool2d(fmap1, 2, stride=2) + fmap2 = F.avg_pool2d(fmap2, 2, stride=2) + self.pyramid.append((fmap1, fmap2)) + + def __call__(self, coords): + coords = coords.permute(0, 2, 3, 1) + B, H, W, _ = coords.shape + dim = self.pyramid[0][0].shape[1] + + corr_list = [] + for i in range(self.num_levels): + r = self.radius + fmap1_i = self.pyramid[0][0].permute(0, 2, 3, 1).contiguous() + fmap2_i = self.pyramid[i][1].permute(0, 2, 3, 1).contiguous() + + coords_i = (coords / 2**i).reshape(B, 1, H, W, 2).contiguous() + corr, = alt_cuda_corr.forward(fmap1_i, fmap2_i, coords_i, r) + corr_list.append(corr.squeeze(1)) + + corr = torch.stack(corr_list, dim=1) + corr = corr.reshape(B, -1, H, W) + return corr / torch.sqrt(torch.tensor(dim).float()) diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/extractor.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ebef47d21c6a5ea5f6c1bd97bf94e07b3b8d0f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/extractor.py @@ -0,0 +1,288 @@ +# The implementation is adopted from RAFT, +# made publicly available under the BSD-3-Clause license at https://github.com/princeton-vl/RAFT + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ResidualBlock(nn.Module): + + def __init__(self, in_planes, planes, norm_fn='group', stride=1): + super(ResidualBlock, self).__init__() + + self.conv1 = nn.Conv2d( + in_planes, planes, kernel_size=3, padding=1, stride=stride) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1) + self.relu = nn.ReLU(inplace=True) + + num_groups = planes // 8 + + if norm_fn == 'group': + self.norm1 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes) + self.norm2 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes) + if not stride == 1: + self.norm3 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes) + + elif norm_fn == 'batch': + self.norm1 = nn.BatchNorm2d(planes) + self.norm2 = nn.BatchNorm2d(planes) + if not stride == 1: + self.norm3 = nn.BatchNorm2d(planes) + + elif norm_fn == 'instance': + self.norm1 = nn.InstanceNorm2d(planes) + self.norm2 = nn.InstanceNorm2d(planes) + if not stride == 1: + self.norm3 = nn.InstanceNorm2d(planes) + + elif norm_fn == 'none': + self.norm1 = nn.Sequential() + self.norm2 = nn.Sequential() + if not stride == 1: + self.norm3 = nn.Sequential() + + if stride == 1: + self.downsample = None + + else: + self.downsample = nn.Sequential( + nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), + self.norm3) + + def forward(self, x): + y = x + y = self.relu(self.norm1(self.conv1(y))) + y = self.relu(self.norm2(self.conv2(y))) + + if self.downsample is not None: + x = self.downsample(x) + + return self.relu(x + y) + + +class BottleneckBlock(nn.Module): + + def __init__(self, in_planes, planes, norm_fn='group', stride=1): + super(BottleneckBlock, self).__init__() + + self.conv1 = nn.Conv2d( + in_planes, planes // 4, kernel_size=1, padding=0) + self.conv2 = nn.Conv2d( + planes // 4, planes // 4, kernel_size=3, padding=1, stride=stride) + self.conv3 = nn.Conv2d(planes // 4, planes, kernel_size=1, padding=0) + self.relu = nn.ReLU(inplace=True) + + num_groups = planes // 8 + + if norm_fn == 'group': + self.norm1 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes // 4) + self.norm2 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes // 4) + self.norm3 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes) + if not stride == 1: + self.norm4 = nn.GroupNorm( + num_groups=num_groups, num_channels=planes) + + elif norm_fn == 'batch': + self.norm1 = nn.BatchNorm2d(planes // 4) + self.norm2 = nn.BatchNorm2d(planes // 4) + self.norm3 = nn.BatchNorm2d(planes) + if not stride == 1: + self.norm4 = nn.BatchNorm2d(planes) + + elif norm_fn == 'instance': + self.norm1 = nn.InstanceNorm2d(planes // 4) + self.norm2 = nn.InstanceNorm2d(planes // 4) + self.norm3 = nn.InstanceNorm2d(planes) + if not stride == 1: + self.norm4 = nn.InstanceNorm2d(planes) + + elif norm_fn == 'none': + self.norm1 = nn.Sequential() + self.norm2 = nn.Sequential() + self.norm3 = nn.Sequential() + if not stride == 1: + self.norm4 = nn.Sequential() + + if stride == 1: + self.downsample = None + + else: + self.downsample = nn.Sequential( + nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride), + self.norm4) + + def forward(self, x): + y = x + y = self.relu(self.norm1(self.conv1(y))) + y = self.relu(self.norm2(self.conv2(y))) + y = self.relu(self.norm3(self.conv3(y))) + + if self.downsample is not None: + x = self.downsample(x) + + return self.relu(x + y) + + +class BasicEncoder(nn.Module): + + def __init__(self, output_dim=128, norm_fn='batch', dropout=0.0): + super(BasicEncoder, self).__init__() + self.norm_fn = norm_fn + + if self.norm_fn == 'group': + self.norm1 = nn.GroupNorm(num_groups=8, num_channels=64) + + elif self.norm_fn == 'batch': + self.norm1 = nn.BatchNorm2d(64) + + elif self.norm_fn == 'instance': + self.norm1 = nn.InstanceNorm2d(64) + + elif self.norm_fn == 'none': + self.norm1 = nn.Sequential() + + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3) + self.relu1 = nn.ReLU(inplace=True) + + self.in_planes = 64 + self.layer1 = self._make_layer(64, stride=1) + self.layer2 = self._make_layer(96, stride=2) + self.layer3 = self._make_layer(128, stride=2) + + # output convolution + self.conv2 = nn.Conv2d(128, output_dim, kernel_size=1) + + self.dropout = None + if dropout > 0: + self.dropout = nn.Dropout2d(p=dropout) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, + (nn.BatchNorm2d, nn.InstanceNorm2d, nn.GroupNorm)): + if m.weight is not None: + nn.init.constant_(m.weight, 1) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + def _make_layer(self, dim, stride=1): + layer1 = ResidualBlock( + self.in_planes, dim, self.norm_fn, stride=stride) + layer2 = ResidualBlock(dim, dim, self.norm_fn, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x): + + # if input is list, combine batch dimension + is_list = isinstance(x, tuple) or isinstance(x, list) + if is_list: + batch_dim = x[0].shape[0] + x = torch.cat(x, dim=0) + + x = self.conv1(x) + x = self.norm1(x) + x = self.relu1(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + + x = self.conv2(x) + + if self.training and self.dropout is not None: + x = self.dropout(x) + + if is_list: + x = torch.split(x, [batch_dim, batch_dim], dim=0) + + return x + + +class SmallEncoder(nn.Module): + + def __init__(self, output_dim=128, norm_fn='batch', dropout=0.0): + super(SmallEncoder, self).__init__() + self.norm_fn = norm_fn + + if self.norm_fn == 'group': + self.norm1 = nn.GroupNorm(num_groups=8, num_channels=32) + + elif self.norm_fn == 'batch': + self.norm1 = nn.BatchNorm2d(32) + + elif self.norm_fn == 'instance': + self.norm1 = nn.InstanceNorm2d(32) + + elif self.norm_fn == 'none': + self.norm1 = nn.Sequential() + + self.conv1 = nn.Conv2d(3, 32, kernel_size=7, stride=2, padding=3) + self.relu1 = nn.ReLU(inplace=True) + + self.in_planes = 32 + self.layer1 = self._make_layer(32, stride=1) + self.layer2 = self._make_layer(64, stride=2) + self.layer3 = self._make_layer(96, stride=2) + + self.dropout = None + if dropout > 0: + self.dropout = nn.Dropout2d(p=dropout) + + self.conv2 = nn.Conv2d(96, output_dim, kernel_size=1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, + (nn.BatchNorm2d, nn.InstanceNorm2d, nn.GroupNorm)): + if m.weight is not None: + nn.init.constant_(m.weight, 1) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + def _make_layer(self, dim, stride=1): + layer1 = BottleneckBlock( + self.in_planes, dim, self.norm_fn, stride=stride) + layer2 = BottleneckBlock(dim, dim, self.norm_fn, stride=1) + layers = (layer1, layer2) + + self.in_planes = dim + return nn.Sequential(*layers) + + def forward(self, x): + + # if input is list, combine batch dimension + is_list = isinstance(x, tuple) or isinstance(x, list) + if is_list: + batch_dim = x[0].shape[0] + x = torch.cat(x, dim=0) + + x = self.conv1(x) + x = self.norm1(x) + x = self.relu1(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.conv2(x) + + if self.training and self.dropout is not None: + x = self.dropout(x) + + if is_list: + x = torch.split(x, [batch_dim, batch_dim], dim=0) + + return x diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/raft.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/raft.py new file mode 100644 index 0000000000000000000000000000000000000000..87b7a2eda819f6082018968be1b6365dc4aa5e1c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/raft.py @@ -0,0 +1,157 @@ +# The implementation is adopted from RAFT, +# made publicly available under the BSD-3-Clause license at https://github.com/princeton-vl/RAFT + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.video_frame_interpolation.flow_model.corr import ( + AlternateCorrBlock, CorrBlock) +from modelscope.models.cv.video_frame_interpolation.flow_model.extractor import ( + BasicEncoder, SmallEncoder) +from modelscope.models.cv.video_frame_interpolation.flow_model.update import ( + BasicUpdateBlock, SmallUpdateBlock) +from modelscope.models.cv.video_frame_interpolation.utils.utils import ( + bilinear_sampler, coords_grid, upflow8) + +autocast = torch.cuda.amp.autocast + + +class RAFT(nn.Module): + + def __init__(self, args): + super(RAFT, self).__init__() + self.args = args + + if args.small: + self.hidden_dim = hdim = 96 + self.context_dim = cdim = 64 + self.args.corr_levels = 4 + self.args.corr_radius = 3 + + else: + self.hidden_dim = hdim = 128 + self.context_dim = cdim = 128 + self.args.corr_levels = 4 + self.args.corr_radius = 4 + + if 'dropout' not in self.args: + self.args.dropout = 0 + + if 'alternate_corr' not in self.args: + self.args.alternate_corr = False + + # feature network, context network, and update block + if args.small: + self.fnet = SmallEncoder( + output_dim=128, norm_fn='instance', dropout=self.args.dropout) + self.cnet = SmallEncoder( + output_dim=hdim + cdim, + norm_fn='none', + dropout=self.args.dropout) + self.update_block = SmallUpdateBlock(self.args, hidden_dim=hdim) + + else: + self.fnet = BasicEncoder( + output_dim=256, norm_fn='instance', dropout=self.args.dropout) + self.cnet = BasicEncoder( + output_dim=hdim + cdim, + norm_fn='batch', + dropout=self.args.dropout) + self.update_block = BasicUpdateBlock(self.args, hidden_dim=hdim) + + def freeze_bn(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + + def initialize_flow(self, img): + """ Flow is represented as difference between two coordinate grids flow = coords1 - coords0""" + N, C, H, W = img.shape + coords0 = coords_grid(N, H // 8, W // 8, device=img.device) + coords1 = coords_grid(N, H // 8, W // 8, device=img.device) + + # optical flow computed as difference: flow = coords1 - coords0 + return coords0, coords1 + + def upsample_flow(self, flow, mask): + """ Upsample flow field [H/8, W/8, 2] -> [H, W, 2] using convex combination """ + N, _, H, W = flow.shape + mask = mask.view(N, 1, 9, 8, 8, H, W) + mask = torch.softmax(mask, dim=2) + + up_flow = F.unfold(8 * flow, [3, 3], padding=1) + up_flow = up_flow.view(N, 2, 9, 1, 1, H, W) + + up_flow = torch.sum(mask * up_flow, dim=2) + up_flow = up_flow.permute(0, 1, 4, 2, 5, 3) + return up_flow.reshape(N, 2, 8 * H, 8 * W) + + def forward(self, + image1, + image2, + iters=12, + flow_init=None, + upsample=True, + test_mode=False): + """ Estimate optical flow between pair of frames """ + + image1 = 2 * (image1 / 255.0) - 1.0 + image2 = 2 * (image2 / 255.0) - 1.0 + + image1 = image1.contiguous() + image2 = image2.contiguous() + + hdim = self.hidden_dim + cdim = self.context_dim + + # run the feature network + with autocast(enabled=self.args.mixed_precision): + fmap1, fmap2 = self.fnet([image1, image2]) + + fmap1 = fmap1.float() + fmap2 = fmap2.float() + if self.args.alternate_corr: + corr_fn = AlternateCorrBlock( + fmap1, fmap2, radius=self.args.corr_radius) + else: + corr_fn = CorrBlock(fmap1, fmap2, radius=self.args.corr_radius) + + # run the context network + with autocast(enabled=self.args.mixed_precision): + cnet = self.cnet(image1) + net, inp = torch.split(cnet, [hdim, cdim], dim=1) + net = torch.tanh(net) + inp = torch.relu(inp) + + coords0, coords1 = self.initialize_flow(image1) + + if flow_init is not None: + coords1 = coords1 + flow_init + + flow_predictions = [] + for itr in range(iters): + coords1 = coords1.detach() + corr = corr_fn(coords1) # index correlation volume + + flow = coords1 - coords0 + with autocast(enabled=self.args.mixed_precision): + net, up_mask, delta_flow = self.update_block( + net, inp, corr, flow) + + # F(t+1) = F(t) + \Delta(t) + coords1 = coords1 + delta_flow + + # upsample predictions + if up_mask is None: + flow_up = upflow8(coords1 - coords0) + else: + flow_up = self.upsample_flow(coords1 - coords0, up_mask) + + flow_predictions.append(flow_up) + + if test_mode: + return coords1 - coords0, flow_up + + return flow_predictions diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/update.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/update.py new file mode 100644 index 0000000000000000000000000000000000000000..29a20db127b7016d6ca129874be3d775dbb186de --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/flow_model/update.py @@ -0,0 +1,160 @@ +# The implementation is adopted from RAFT, +# made publicly available under the BSD-3-Clause license at https://github.com/princeton-vl/RAFT + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class FlowHead(nn.Module): + + def __init__(self, input_dim=128, hidden_dim=256): + super(FlowHead, self).__init__() + self.conv1 = nn.Conv2d(input_dim, hidden_dim, 3, padding=1) + self.conv2 = nn.Conv2d(hidden_dim, 2, 3, padding=1) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + return self.conv2(self.relu(self.conv1(x))) + + +class ConvGRU(nn.Module): + + def __init__(self, hidden_dim=128, input_dim=192 + 128): + super(ConvGRU, self).__init__() + self.convz = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + self.convr = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + self.convq = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, 3, padding=1) + + def forward(self, h, x): + hx = torch.cat([h, x], dim=1) + + z = torch.sigmoid(self.convz(hx)) + r = torch.sigmoid(self.convr(hx)) + q = torch.tanh(self.convq(torch.cat([r * h, x], dim=1))) + + h = (1 - z) * h + z * q + return h + + +class SepConvGRU(nn.Module): + + def __init__(self, hidden_dim=128, input_dim=192 + 128): + super(SepConvGRU, self).__init__() + self.convz1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + self.convr1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + self.convq1 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (1, 5), padding=(0, 2)) + + self.convz2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + self.convr2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + self.convq2 = nn.Conv2d( + hidden_dim + input_dim, hidden_dim, (5, 1), padding=(2, 0)) + + def forward(self, h, x): + # horizontal + hx = torch.cat([h, x], dim=1) + z = torch.sigmoid(self.convz1(hx)) + r = torch.sigmoid(self.convr1(hx)) + q = torch.tanh(self.convq1(torch.cat([r * h, x], dim=1))) + h = (1 - z) * h + z * q + + # vertical + hx = torch.cat([h, x], dim=1) + z = torch.sigmoid(self.convz2(hx)) + r = torch.sigmoid(self.convr2(hx)) + q = torch.tanh(self.convq2(torch.cat([r * h, x], dim=1))) + h = (1 - z) * h + z * q + + return h + + +class SmallMotionEncoder(nn.Module): + + def __init__(self, args): + super(SmallMotionEncoder, self).__init__() + cor_planes = args.corr_levels * (2 * args.corr_radius + 1)**2 + self.convc1 = nn.Conv2d(cor_planes, 96, 1, padding=0) + self.convf1 = nn.Conv2d(2, 64, 7, padding=3) + self.convf2 = nn.Conv2d(64, 32, 3, padding=1) + self.conv = nn.Conv2d(128, 80, 3, padding=1) + + def forward(self, flow, corr): + cor = F.relu(self.convc1(corr)) + flo = F.relu(self.convf1(flow)) + flo = F.relu(self.convf2(flo)) + cor_flo = torch.cat([cor, flo], dim=1) + out = F.relu(self.conv(cor_flo)) + return torch.cat([out, flow], dim=1) + + +class BasicMotionEncoder(nn.Module): + + def __init__(self, args): + super(BasicMotionEncoder, self).__init__() + cor_planes = args.corr_levels * (2 * args.corr_radius + 1)**2 + self.convc1 = nn.Conv2d(cor_planes, 256, 1, padding=0) + self.convc2 = nn.Conv2d(256, 192, 3, padding=1) + self.convf1 = nn.Conv2d(2, 128, 7, padding=3) + self.convf2 = nn.Conv2d(128, 64, 3, padding=1) + self.conv = nn.Conv2d(64 + 192, 128 - 2, 3, padding=1) + + def forward(self, flow, corr): + cor = F.relu(self.convc1(corr)) + cor = F.relu(self.convc2(cor)) + flo = F.relu(self.convf1(flow)) + flo = F.relu(self.convf2(flo)) + + cor_flo = torch.cat([cor, flo], dim=1) + out = F.relu(self.conv(cor_flo)) + return torch.cat([out, flow], dim=1) + + +class SmallUpdateBlock(nn.Module): + + def __init__(self, args, hidden_dim=96): + super(SmallUpdateBlock, self).__init__() + self.encoder = SmallMotionEncoder(args) + self.gru = ConvGRU(hidden_dim=hidden_dim, input_dim=82 + 64) + self.flow_head = FlowHead(hidden_dim, hidden_dim=128) + + def forward(self, net, inp, corr, flow): + motion_features = self.encoder(flow, corr) + inp = torch.cat([inp, motion_features], dim=1) + net = self.gru(net, inp) + delta_flow = self.flow_head(net) + + return net, None, delta_flow + + +class BasicUpdateBlock(nn.Module): + + def __init__(self, args, hidden_dim=128, input_dim=128): + super(BasicUpdateBlock, self).__init__() + self.args = args + self.encoder = BasicMotionEncoder(args) + self.gru = SepConvGRU( + hidden_dim=hidden_dim, input_dim=128 + hidden_dim) + self.flow_head = FlowHead(hidden_dim, hidden_dim=256) + + self.mask = nn.Sequential( + nn.Conv2d(128, 256, 3, padding=1), nn.ReLU(inplace=True), + nn.Conv2d(256, 64 * 9, 1, padding=0)) + + def forward(self, net, inp, corr, flow, upsample=True): + motion_features = self.encoder(flow, corr) + inp = torch.cat([inp, motion_features], dim=1) + + net = self.gru(net, inp) + delta_flow = self.flow_head(net) + + # scale mask to balence gradients + mask = .25 * self.mask(net) + return net, mask, delta_flow diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/IFNet_swin.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/IFNet_swin.py new file mode 100644 index 0000000000000000000000000000000000000000..3e82bde28bc7e725f9518a6b94dc09b0f8c7f041 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/IFNet_swin.py @@ -0,0 +1,434 @@ +# Part of the implementation is borrowed and modified from RIFE, +# publicly available at https://github.com/megvii-research/ECCV2022-RIFE + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import trunc_normal_ + +from modelscope.models.cv.video_frame_interpolation.interp_model.transformer_layers import ( + RTFL, PatchEmbed, PatchUnEmbed) + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + +backwarp_tenGrid = {} + + +def warp(tenInput, tenFlow): + k = (str(tenFlow.device), str(tenFlow.size())) + if k not in backwarp_tenGrid: + tenHorizontal = torch.linspace( + -1.0, 1.0, tenFlow.shape[3], device=device).view( + 1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, + tenFlow.shape[2], -1) + tenVertical = torch.linspace( + -1.0, 1.0, tenFlow.shape[2], + device=device).view(1, 1, tenFlow.shape[2], + 1).expand(tenFlow.shape[0], -1, -1, + tenFlow.shape[3]) + backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], + 1).to(device) + + tmp1 = tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0) + tmp2 = tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0) + tenFlow = torch.cat([tmp1, tmp2], 1) + + g = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1) + return torch.nn.functional.grid_sample( + input=tenInput, + grid=torch.clamp(g, -1, 1), + mode='bilinear', + padding_mode='border', + align_corners=True) + + +def conv_wo_act(in_planes, + out_planes, + kernel_size=3, + stride=1, + padding=1, + dilation=1): + return nn.Sequential( + nn.Conv2d( + in_planes, + out_planes, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=True), ) + + +def conv(in_planes, + out_planes, + kernel_size=3, + stride=1, + padding=1, + dilation=1): + return nn.Sequential( + nn.Conv2d( + in_planes, + out_planes, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=True), nn.PReLU(out_planes)) + + +def conv_bn(in_planes, + out_planes, + kernel_size=3, + stride=1, + padding=1, + dilation=1): + return nn.Sequential( + nn.Conv2d( + in_planes, + out_planes, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=False), nn.BatchNorm2d(out_planes), nn.PReLU(out_planes)) + + +class TransModel(nn.Module): + + def __init__(self, + img_size=64, + patch_size=1, + embed_dim=64, + depths=[[3, 3]], + num_heads=[[2, 2]], + window_size=4, + mlp_ratio=2, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.1, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + use_checkpoint=False, + resi_connection='1conv', + use_crossattn=[[[False, False, False, False], + [True, True, True, True]]]): + super(TransModel, self).__init__() + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.num_features = embed_dim + self.mlp_ratio = mlp_ratio + + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=embed_dim, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + # merge non-overlapping patches into image + self.patch_unembed = PatchUnEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=embed_dim, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + if self.ape: + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, num_patches, embed_dim)) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr0 = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths[0])) + ] # stochastic depth decay rule + + self.layers0 = nn.ModuleList() + num_layers = len(depths[0]) + for i_layer in range(num_layers): + layer = RTFL( + dim=embed_dim, + input_resolution=(patches_resolution[0], + patches_resolution[1]), + depth=depths[0][i_layer], + num_heads=num_heads[0][i_layer], + window_size=window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr0[sum(depths[0][:i_layer]):sum(depths[0][:i_layer + + 1])], + norm_layer=norm_layer, + downsample=None, + use_checkpoint=use_checkpoint, + img_size=(img_size[0], img_size[1]), + patch_size=patch_size, + resi_connection=resi_connection, + use_crossattn=use_crossattn[0][i_layer]) + self.layers0.append(layer) + + self.norm = norm_layer(self.num_features) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'absolute_pos_embed'} + + @torch.jit.ignore + def no_weight_decay_keywords(self): + return {'relative_position_bias_table'} + + def forward_features(self, x, layers): + x_size = (x.shape[2], x.shape[3]) + x = self.patch_embed(x) + if self.ape: + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + + if isinstance(layers, nn.ModuleList): + for layer in layers: + x = layer(x, x_size) + else: + x = layers(x, x_size) + + x = self.norm(x) # B L C + x = self.patch_unembed(x, x_size) + + return x + + def forward(self, x): + out = self.forward_features(x, self.layers0) + return out + + +class IFBlock(nn.Module): + + def __init__(self, in_planes, scale=1, c=64): + super(IFBlock, self).__init__() + self.scale = scale + self.conv0 = nn.Sequential( + conv(in_planes, c // 2, 3, 2, 1), + conv(c // 2, c, 3, 2, 1), + conv(c, c, 3, 1, 1), + ) + + self.trans = TransModel( + img_size=(128 // scale, 128 // scale), + patch_size=1, + embed_dim=c, + depths=[[3, 3]], + num_heads=[[2, 2]]) + + self.conv1 = nn.Sequential( + conv(c, c, 3, 1, 1), + conv(c, c, 3, 1, 1), + ) + + self.up = nn.ConvTranspose2d(c, 4, 4, 2, 1) + + self.conv2 = nn.Conv2d(4, 4, 3, 1, 1) + + def forward(self, x, flow0, flow1): + if self.scale != 1: + x = F.interpolate( + x, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) + flow0 = F.interpolate( + flow0, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) * (1. / self.scale) + flow1 = F.interpolate( + flow1, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) * (1. / self.scale) + + x = torch.cat((x, flow0, flow1), 1) + + x = self.conv0(x) + x = self.trans(x) + x = self.conv1(x) + x + + # upsample 2.0 + x = self.up(x) + + # upsample 2.0 + x = self.conv2(x) + flow = F.interpolate( + x, scale_factor=2.0, mode='bilinear', align_corners=False) * 2.0 + + if self.scale != 1: + flow = F.interpolate( + flow, + scale_factor=self.scale, + mode='bilinear', + align_corners=False) * self.scale + + flow0 = flow[:, :2, :, :] + flow1 = flow[:, 2:, :, :] + + return flow0, flow1 + + +class IFBlock_wo_Swin(nn.Module): + + def __init__(self, in_planes, scale=1, c=64): + super(IFBlock_wo_Swin, self).__init__() + self.scale = scale + self.conv0 = nn.Sequential( + conv(in_planes, c // 2, 3, 2, 1), + conv(c // 2, c, 3, 2, 1), + ) + + self.convblock1 = nn.Sequential(conv(c, c), conv(c, c), conv(c, c)) + self.convblock2 = nn.Sequential(conv(c, c), conv(c, c), conv(c, c)) + + self.up = nn.ConvTranspose2d(c, 4, 4, 2, 1) + + self.conv2 = nn.Conv2d(4, 4, 3, 1, 1) + + def forward(self, x, flow0, flow1): + if self.scale != 1: + x = F.interpolate( + x, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) + flow0 = F.interpolate( + flow0, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) * (1. / self.scale) + flow1 = F.interpolate( + flow1, + scale_factor=1. / self.scale, + mode='bilinear', + align_corners=False) * (1. / self.scale) + + x = torch.cat((x, flow0, flow1), 1) + + x = self.conv0(x) + x = self.convblock1(x) + x + x = self.convblock2(x) + x + # upsample 2.0 + x = self.up(x) + + # upsample 2.0 + x = self.conv2(x) + flow = F.interpolate( + x, scale_factor=2.0, mode='bilinear', align_corners=False) * 2.0 + + if self.scale != 1: + flow = F.interpolate( + flow, + scale_factor=self.scale, + mode='bilinear', + align_corners=False) * self.scale + + flow0 = flow[:, :2, :, :] + flow1 = flow[:, 2:, :, :] + + return flow0, flow1 + + +class IFNet(nn.Module): + + def __init__(self): + super(IFNet, self).__init__() + self.block1 = IFBlock_wo_Swin(16, scale=4, c=128) + self.block2 = IFBlock(16, scale=2, c=64) + self.block3 = IFBlock(16, scale=1, c=32) + + # flow0: flow from img0 to img1 + # flow1: flow from img1 to img0 + def forward(self, img0, img1, flow0, flow1, sc_mode=2): + + if sc_mode == 0: + sc = 0.25 + elif sc_mode == 1: + sc = 0.5 + else: + sc = 1 + + if sc != 1: + img0_sc = F.interpolate( + img0, scale_factor=sc, mode='bilinear', align_corners=False) + img1_sc = F.interpolate( + img1, scale_factor=sc, mode='bilinear', align_corners=False) + flow0_sc = F.interpolate( + flow0, scale_factor=sc, mode='bilinear', + align_corners=False) * sc + flow1_sc = F.interpolate( + flow1, scale_factor=sc, mode='bilinear', + align_corners=False) * sc + else: + img0_sc = img0 + img1_sc = img1 + flow0_sc = flow0 + flow1_sc = flow1 + + warped_img0 = warp(img1_sc, flow0_sc) # -> img0 + warped_img1 = warp(img0_sc, flow1_sc) # -> img1 + flow0_1, flow1_1 = self.block1( + torch.cat((img0_sc, img1_sc, warped_img0, warped_img1), 1), + flow0_sc, flow1_sc) + F0_2 = (flow0_sc + flow0_1) + F1_2 = (flow1_sc + flow1_1) + + warped_img0 = warp(img1_sc, F0_2) # -> img0 + warped_img1 = warp(img0_sc, F1_2) # -> img1 + flow0_2, flow1_2 = self.block2( + torch.cat((img0_sc, img1_sc, warped_img0, warped_img1), 1), F0_2, + F1_2) + F0_3 = (F0_2 + flow0_2) + F1_3 = (F1_2 + flow1_2) + + warped_img0 = warp(img1_sc, F0_3) # -> img0 + warped_img1 = warp(img0_sc, F1_3) # -> img1 + flow0_3, flow1_3 = self.block3( + torch.cat((img0_sc, img1_sc, warped_img0, warped_img1), dim=1), + F0_3, F1_3) + flow_res_0 = flow0_1 + flow0_2 + flow0_3 + flow_res_1 = flow1_1 + flow1_2 + flow1_3 + + if sc != 1: + flow_res_0 = F.interpolate( + flow_res_0, + scale_factor=1 / sc, + mode='bilinear', + align_corners=False) / sc + flow_res_1 = F.interpolate( + flow_res_1, + scale_factor=1 / sc, + mode='bilinear', + align_corners=False) / sc + + F0_4 = flow0 + flow_res_0 + F1_4 = flow1 + flow_res_1 + + return F0_4, F1_4 diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/UNet.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/UNet.py new file mode 100644 index 0000000000000000000000000000000000000000..34b5be1969ba7193e609b2226137b03823748e5e --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/UNet.py @@ -0,0 +1,127 @@ +# Part of the implementation is borrowed and modified from QVI, publicly available at https://github.com/xuxy09/QVI + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class down(nn.Module): + + def __init__(self, inChannels, outChannels, filterSize): + super(down, self).__init__() + self.conv1 = nn.Conv2d( + inChannels, + outChannels, + filterSize, + stride=1, + padding=int((filterSize - 1) / 2)) + self.conv2 = nn.Conv2d( + outChannels, + outChannels, + filterSize, + stride=1, + padding=int((filterSize - 1) / 2)) + + def forward(self, x): + x = F.avg_pool2d(x, 2) + x = F.leaky_relu(self.conv1(x), negative_slope=0.1) + x = F.leaky_relu(self.conv2(x), negative_slope=0.1) + return x + + +class up(nn.Module): + + def __init__(self, inChannels, outChannels): + super(up, self).__init__() + self.conv1 = nn.Conv2d(inChannels, outChannels, 3, stride=1, padding=1) + self.conv2 = nn.Conv2d( + 2 * outChannels, outChannels, 3, stride=1, padding=1) + + def forward(self, x, skpCn): + x = F.interpolate( + x, + size=[skpCn.size(2), skpCn.size(3)], + mode='bilinear', + align_corners=False) + x = F.leaky_relu(self.conv1(x), negative_slope=0.1) + x = F.leaky_relu( + self.conv2(torch.cat((x, skpCn), 1)), negative_slope=0.1) + return x + + +class Small_UNet(nn.Module): + + def __init__(self, inChannels, outChannels): + super(Small_UNet, self).__init__() + self.conv1 = nn.Conv2d(inChannels, 32, 7, stride=1, padding=3) + self.conv2 = nn.Conv2d(32, 32, 7, stride=1, padding=3) + self.down1 = down(32, 64, 5) + self.down2 = down(64, 128, 3) + self.down3 = down(128, 128, 3) + self.up1 = up(128, 128) + self.up2 = up(128, 64) + self.up3 = up(64, 32) + self.conv3 = nn.Conv2d(32, outChannels, 3, stride=1, padding=1) + + def forward(self, x): + x = F.leaky_relu(self.conv1(x), negative_slope=0.1) + s1 = F.leaky_relu(self.conv2(x), negative_slope=0.1) + s2 = self.down1(s1) + s3 = self.down2(s2) + x = self.down3(s3) + x = self.up1(x, s3) + x = self.up2(x, s2) + x1 = self.up3(x, s1) # feature + x = self.conv3(x1) # flow + return x, x1 + + +class Small_UNet_Ds(nn.Module): + + def __init__(self, inChannels, outChannels): + super(Small_UNet_Ds, self).__init__() + self.conv1_1 = nn.Conv2d(inChannels, 32, 5, stride=1, padding=2) + self.conv1_2 = nn.Conv2d(32, 32, 3, stride=1, padding=1) + self.conv2_1 = nn.Conv2d(32, 32, 3, stride=1, padding=1) + self.conv2_2 = nn.Conv2d(32, 32, 3, stride=1, padding=1) + + self.down1 = down(32, 64, 5) + self.down2 = down(64, 128, 3) + self.down3 = down(128, 128, 3) + self.up1 = up(128, 128) + self.up2 = up(128, 64) + self.up3 = up(64, 32) + self.conv3 = nn.Conv2d(32, 32, 3, stride=1, padding=1) + self.conv4 = nn.Conv2d(32, outChannels, 3, stride=1, padding=1) + + def forward(self, x): + + x0 = F.leaky_relu(self.conv1_1(x), negative_slope=0.1) + x0 = F.leaky_relu(self.conv1_2(x0), negative_slope=0.1) + + x = F.interpolate( + x0, + size=[x0.size(2) // 2, x0.size(3) // 2], + mode='bilinear', + align_corners=False) + + x = F.leaky_relu(self.conv2_1(x), negative_slope=0.1) + s1 = F.leaky_relu(self.conv2_2(x), negative_slope=0.1) + + s2 = self.down1(s1) + s3 = self.down2(s2) + x = self.down3(s3) + + x = self.up1(x, s3) + x = self.up2(x, s2) + x1 = self.up3(x, s1) + + x1 = F.interpolate( + x1, + size=[x0.size(2), x0.size(3)], + mode='bilinear', + align_corners=False) + + x1 = F.leaky_relu(self.conv3(x1), negative_slope=0.1) # feature + x = self.conv4(x1) # flow + return x, x1 diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/__init__.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/flow_reversal.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/flow_reversal.py new file mode 100644 index 0000000000000000000000000000000000000000..daac2ead8cf3a60f87e5837a0e9873a92d717995 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/flow_reversal.py @@ -0,0 +1,115 @@ +# The implementation is adopted from QVI, +# made publicly available at https://github.com/xuxy09/QVI + +# class WarpLayer warps image x based on optical flow flo. +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class FlowReversal(nn.Module): + """docstring for WarpLayer""" + + def __init__(self, ): + super(FlowReversal, self).__init__() + + def forward(self, img, flo): + """ + -img: image (N, C, H, W) + -flo: optical flow (N, 2, H, W) + elements of flo is in [0, H] and [0, W] for dx, dy + + """ + + N, C, _, _ = img.size() + + # translate start-point optical flow to end-point optical flow + y = flo[:, 0:1:, :] + x = flo[:, 1:2, :, :] + + x = x.repeat(1, C, 1, 1) + y = y.repeat(1, C, 1, 1) + + x1 = torch.floor(x) + x2 = x1 + 1 + y1 = torch.floor(y) + y2 = y1 + 1 + + # firstly, get gaussian weights + w11, w12, w21, w22 = self.get_gaussian_weights(x, y, x1, x2, y1, y2) + + # secondly, sample each weighted corner + img11, o11 = self.sample_one(img, x1, y1, w11) + img12, o12 = self.sample_one(img, x1, y2, w12) + img21, o21 = self.sample_one(img, x2, y1, w21) + img22, o22 = self.sample_one(img, x2, y2, w22) + + imgw = img11 + img12 + img21 + img22 + o = o11 + o12 + o21 + o22 + + return imgw, o + + def get_gaussian_weights(self, x, y, x1, x2, y1, y2): + w11 = torch.exp(-((x - x1)**2 + (y - y1)**2)) + w12 = torch.exp(-((x - x1)**2 + (y - y2)**2)) + w21 = torch.exp(-((x - x2)**2 + (y - y1)**2)) + w22 = torch.exp(-((x - x2)**2 + (y - y2)**2)) + + return w11, w12, w21, w22 + + def sample_one(self, img, shiftx, shifty, weight): + """ + Input: + -img (N, C, H, W) + -shiftx, shifty (N, c, H, W) + """ + + N, C, H, W = img.size() + + # flatten all (all restored as Tensors) + flat_shiftx = shiftx.view(-1) + flat_shifty = shifty.view(-1) + flat_basex = torch.arange( + 0, H, requires_grad=False).view(-1, 1)[None, + None].cuda().long().repeat( + N, C, 1, W).view(-1) + flat_basey = torch.arange( + 0, W, requires_grad=False).view(1, -1)[None, + None].cuda().long().repeat( + N, C, H, 1).view(-1) + flat_weight = weight.view(-1) + flat_img = img.view(-1) + + idxn = torch.arange( + 0, N, + requires_grad=False).view(N, 1, 1, + 1).long().cuda().repeat(1, C, H, + W).view(-1) + idxc = torch.arange( + 0, C, + requires_grad=False).view(1, C, 1, + 1).long().cuda().repeat(N, 1, H, + W).view(-1) + idxx = flat_shiftx.long() + flat_basex + idxy = flat_shifty.long() + flat_basey + + mask = idxx.ge(0) & idxx.lt(H) & idxy.ge(0) & idxy.lt(W) + + ids = (idxn * C * H * W + idxc * H * W + idxx * W + idxy) + ids_mask = torch.masked_select(ids, mask).clone().cuda() + + img_warp = torch.zeros([ + N * C * H * W, + ]).cuda() + img_warp.put_( + ids_mask, + torch.masked_select(flat_img * flat_weight, mask), + accumulate=True) + + one_warp = torch.zeros([ + N * C * H * W, + ]).cuda() + one_warp.put_( + ids_mask, torch.masked_select(flat_weight, mask), accumulate=True) + + return img_warp.view(N, C, H, W), one_warp.view(N, C, H, W) diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/refinenet_arch.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/refinenet_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..1fa0136fa331d2290a704a1f02953de6d2591410 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/refinenet_arch.py @@ -0,0 +1,488 @@ +# Part of the implementation is borrowed and modified from QVI, publicly available at https://github.com/xuxy09/QVI + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.cv.video_frame_interpolation.interp_model.flow_reversal import \ + FlowReversal +from modelscope.models.cv.video_frame_interpolation.interp_model.IFNet_swin import \ + IFNet +from modelscope.models.cv.video_frame_interpolation.interp_model.UNet import \ + Small_UNet_Ds + + +class AcFusionLayer(nn.Module): + + def __init__(self, ): + super(AcFusionLayer, self).__init__() + + def forward(self, flo10, flo12, flo21, flo23, t=0.5): + return 0.5 * ((t + t**2) * flo12 - (t - t**2) * flo10), \ + 0.5 * (((1 - t) + (1 - t)**2) * flo21 - ((1 - t) - (1 - t)**2) * flo23) + # return 0.375 * flo12 - 0.125 * flo10, 0.375 * flo21 - 0.125 * flo23 + + +class Get_gradient(nn.Module): + + def __init__(self): + super(Get_gradient, self).__init__() + kernel_v = [[0, -1, 0], [0, 0, 0], [0, 1, 0]] + kernel_h = [[0, 0, 0], [-1, 0, 1], [0, 0, 0]] + kernel_h = torch.FloatTensor(kernel_h).unsqueeze(0).unsqueeze(0) + kernel_v = torch.FloatTensor(kernel_v).unsqueeze(0).unsqueeze(0) + self.weight_h = nn.Parameter(data=kernel_h, requires_grad=False) + self.weight_v = nn.Parameter(data=kernel_v, requires_grad=False) + + def forward(self, x): + x0 = x[:, 0] # R + x1 = x[:, 1] # G + x2 = x[:, 2] # B + x0_v = F.conv2d(x0.unsqueeze(1), self.weight_v, padding=1) + x0_h = F.conv2d(x0.unsqueeze(1), self.weight_h, padding=1) + + x1_v = F.conv2d(x1.unsqueeze(1), self.weight_v, padding=1) + x1_h = F.conv2d(x1.unsqueeze(1), self.weight_h, padding=1) + + x2_v = F.conv2d(x2.unsqueeze(1), self.weight_v, padding=1) + x2_h = F.conv2d(x2.unsqueeze(1), self.weight_h, padding=1) + + x0 = torch.sqrt(torch.pow(x0_v, 2) + torch.pow(x0_h, 2) + 1e-6) + x1 = torch.sqrt(torch.pow(x1_v, 2) + torch.pow(x1_h, 2) + 1e-6) + x2 = torch.sqrt(torch.pow(x2_v, 2) + torch.pow(x2_h, 2) + 1e-6) + + x = torch.cat([x0, x1, x2], dim=1) + return x + + +class LowPassFilter(nn.Module): + + def __init__(self): + super(LowPassFilter, self).__init__() + kernel_lpf = [[1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1]] + + kernel_lpf = torch.FloatTensor(kernel_lpf).unsqueeze(0).unsqueeze( + 0) / 49 + + self.weight_lpf = nn.Parameter(data=kernel_lpf, requires_grad=False) + + def forward(self, x): + x0 = x[:, 0] + x1 = x[:, 1] + y0 = F.conv2d(x0.unsqueeze(1), self.weight_lpf, padding=3) + y1 = F.conv2d(x1.unsqueeze(1), self.weight_lpf, padding=3) + + y = torch.cat([y0, y1], dim=1) + + return y + + +def backwarp(img, flow): + _, _, H, W = img.size() + + u = flow[:, 0, :, :] + v = flow[:, 1, :, :] + + gridX, gridY = np.meshgrid(np.arange(W), np.arange(H)) + + gridX = torch.tensor( + gridX, + requires_grad=False, + ).cuda() + gridY = torch.tensor( + gridY, + requires_grad=False, + ).cuda() + x = gridX.unsqueeze(0).expand_as(u).float() + u + y = gridY.unsqueeze(0).expand_as(v).float() + v + + x = 2 * (x / (W - 1) - 0.5) + y = 2 * (y / (H - 1) - 0.5) + + grid = torch.stack((x, y), dim=3) + + imgOut = torch.nn.functional.grid_sample( + img, grid, padding_mode='border', align_corners=True) + + return imgOut + + +class SmallMaskNet(nn.Module): + """A three-layer network for predicting mask""" + + def __init__(self, input, output): + super(SmallMaskNet, self).__init__() + self.conv1 = nn.Conv2d(input, 32, 5, padding=2) + self.conv2 = nn.Conv2d(32, 16, 3, padding=1) + self.conv3 = nn.Conv2d(16, output, 3, padding=1) + + def forward(self, x): + x = F.leaky_relu(self.conv1(x), negative_slope=0.1) + x = F.leaky_relu(self.conv2(x), negative_slope=0.1) + x = self.conv3(x) + return x + + +class StaticMaskNet(nn.Module): + """static mask""" + + def __init__(self, input, output): + super(StaticMaskNet, self).__init__() + + modules_body = [] + modules_body.append( + nn.Conv2d( + in_channels=input, + out_channels=32, + kernel_size=3, + stride=1, + padding=1)) + modules_body.append(nn.LeakyReLU(inplace=False, negative_slope=0.1)) + modules_body.append( + nn.Conv2d( + in_channels=32, + out_channels=32, + kernel_size=3, + stride=1, + padding=1)) + modules_body.append(nn.LeakyReLU(inplace=False, negative_slope=0.1)) + modules_body.append( + nn.Conv2d( + in_channels=32, + out_channels=16, + kernel_size=3, + stride=1, + padding=1)) + modules_body.append(nn.LeakyReLU(inplace=False, negative_slope=0.1)) + modules_body.append( + nn.Conv2d( + in_channels=16, + out_channels=16, + kernel_size=3, + stride=1, + padding=1)) + modules_body.append(nn.LeakyReLU(inplace=False, negative_slope=0.1)) + modules_body.append( + nn.Conv2d( + in_channels=16, + out_channels=output, + kernel_size=3, + stride=1, + padding=1)) + modules_body.append(nn.Sigmoid()) + + self.body = nn.Sequential(*modules_body) + + def forward(self, x): + y = self.body(x) + return y + + +def tensor_erode(bin_img, ksize=5): + B, C, H, W = bin_img.shape + pad = (ksize - 1) // 2 + bin_img = F.pad(bin_img, [pad, pad, pad, pad], mode='constant', value=0) + + patches = bin_img.unfold(dimension=2, size=ksize, step=1) + patches = patches.unfold(dimension=3, size=ksize, step=1) + + eroded, _ = patches.reshape(B, C, H, W, -1).max(dim=-1) + return eroded + + +class QVI_inter_Ds(nn.Module): + """Given flow, implement Quadratic Video Interpolation""" + + def __init__(self, debug_en=False, is_training=False): + super(QVI_inter_Ds, self).__init__() + self.acc = AcFusionLayer() + self.fwarp = FlowReversal() + self.refinenet = Small_UNet_Ds(20, 8) + self.masknet = SmallMaskNet(38, 1) + + self.staticnet = StaticMaskNet(56, 1) + self.lpfilter = LowPassFilter() + + self.get_grad = Get_gradient() + self.debug_en = debug_en + self.is_training = is_training + + def fill_flow_hole(self, ft, norm, ft_fill): + (N, C, H, W) = ft.shape + ft[norm == 0] = ft_fill[norm == 0] + + ft_1 = self.lpfilter(ft.clone()) + ft_ds = torch.nn.functional.interpolate( + input=ft_1, + size=(H // 4, W // 4), + mode='bilinear', + align_corners=False) + ft_up = torch.nn.functional.interpolate( + input=ft_ds, size=(H, W), mode='bilinear', align_corners=False) + + ft[norm == 0] = ft_up[norm == 0] + + return ft + + def forward(self, F10_Ds, F12_Ds, F21_Ds, F23_Ds, I1_Ds, I2_Ds, I1, I2, t): + if F12_Ds is None or F21_Ds is None: + return I1 + + if F10_Ds is not None and F23_Ds is not None: + F1t_Ds, F2t_Ds = self.acc(F10_Ds, F12_Ds, F21_Ds, F23_Ds, t) + + else: + F1t_Ds = t * F12_Ds + F2t_Ds = (1 - t) * F21_Ds + + # Flow Reversal + F1t_Ds2 = F.interpolate( + F1t_Ds, scale_factor=1.0 / 3, mode='nearest') / 3 + F2t_Ds2 = F.interpolate( + F2t_Ds, scale_factor=1.0 / 3, mode='nearest') / 3 + Ft1_Ds2, norm1_Ds2 = self.fwarp(F1t_Ds2, F1t_Ds2) + Ft1_Ds2 = -Ft1_Ds2 + Ft2_Ds2, norm2_Ds2 = self.fwarp(F2t_Ds2, F2t_Ds2) + Ft2_Ds2 = -Ft2_Ds2 + + Ft1_Ds2[norm1_Ds2 > 0] \ + = Ft1_Ds2[norm1_Ds2 > 0] / norm1_Ds2[norm1_Ds2 > 0].clone() + Ft2_Ds2[norm2_Ds2 > 0] \ + = Ft2_Ds2[norm2_Ds2 > 0] / norm2_Ds2[norm2_Ds2 > 0].clone() + if 1: + Ft1_Ds2_fill = -F1t_Ds2 + Ft2_Ds2_fill = -F2t_Ds2 + Ft1_Ds2 = self.fill_flow_hole(Ft1_Ds2, norm1_Ds2, Ft1_Ds2_fill) + Ft2_Ds2 = self.fill_flow_hole(Ft2_Ds2, norm2_Ds2, Ft2_Ds2_fill) + + Ft1_Ds = F.interpolate( + Ft1_Ds2, size=[F1t_Ds.size(2), F1t_Ds.size(3)], mode='nearest') * 3 + Ft2_Ds = F.interpolate( + Ft2_Ds2, size=[F2t_Ds.size(2), F2t_Ds.size(3)], mode='nearest') * 3 + + I1t_Ds = backwarp(I1_Ds, Ft1_Ds) + I2t_Ds = backwarp(I2_Ds, Ft2_Ds) + + output_Ds, feature_Ds = self.refinenet( + torch.cat( + [I1_Ds, I2_Ds, I1t_Ds, I2t_Ds, F12_Ds, F21_Ds, Ft1_Ds, Ft2_Ds], + dim=1)) + + # Adaptive filtering + Ft1r_Ds = backwarp( + Ft1_Ds, 10 * torch.tanh(output_Ds[:, 4:6])) + output_Ds[:, :2] + Ft2r_Ds = backwarp( + Ft2_Ds, 10 * torch.tanh(output_Ds[:, 6:8])) + output_Ds[:, 2:4] + + # warping and fusing + I1tf_Ds = backwarp(I1_Ds, Ft1r_Ds) + I2tf_Ds = backwarp(I2_Ds, Ft2r_Ds) + + G1_Ds = self.get_grad(I1_Ds) + G2_Ds = self.get_grad(I2_Ds) + G1tf_Ds = backwarp(G1_Ds, Ft1r_Ds) + G2tf_Ds = backwarp(G2_Ds, Ft2r_Ds) + + M_Ds = torch.sigmoid( + self.masknet(torch.cat([I1tf_Ds, I2tf_Ds, feature_Ds], + dim=1))).repeat(1, 3, 1, 1) + + Ft1r = F.interpolate( + Ft1r_Ds * 2, scale_factor=2, mode='bilinear', align_corners=False) + Ft2r = F.interpolate( + Ft2r_Ds * 2, scale_factor=2, mode='bilinear', align_corners=False) + + I1tf = backwarp(I1, Ft1r) + I2tf = backwarp(I2, Ft2r) + + M = F.interpolate( + M_Ds, scale_factor=2, mode='bilinear', align_corners=False) + + # fuse + It_warp = ((1 - t) * M * I1tf + t * (1 - M) * I2tf) \ + / ((1 - t) * M + t * (1 - M)).clone() + + # static blending + It_static = (1 - t) * I1 + t * I2 + tmp = torch.cat((I1tf_Ds, I2tf_Ds, G1tf_Ds, G2tf_Ds, I1_Ds, I2_Ds, + G1_Ds, G2_Ds, feature_Ds), + dim=1) + M_static_Ds = self.staticnet(tmp) + M_static_dilate = tensor_erode(M_static_Ds) + M_static_dilate = tensor_erode(M_static_dilate) + M_static = F.interpolate( + M_static_dilate, + scale_factor=2, + mode='bilinear', + align_corners=False) + + It_warp = (1 - M_static) * It_warp + M_static * It_static + + if self.is_training: + return It_warp, Ft1r, Ft2r + else: + if self.debug_en: + return It_warp, M, M_static, I1tf, I2tf, Ft1r, Ft2r + else: + return It_warp + + +class QVI_inter(nn.Module): + """Given flow, implement Quadratic Video Interpolation""" + + def __init__(self, debug_en=False, is_training=False): + super(QVI_inter, self).__init__() + self.acc = AcFusionLayer() + self.fwarp = FlowReversal() + self.refinenet = Small_UNet_Ds(20, 8) + self.masknet = SmallMaskNet(38, 1) + + self.staticnet = StaticMaskNet(56, 1) + self.lpfilter = LowPassFilter() + + self.get_grad = Get_gradient() + self.debug_en = debug_en + self.is_training = is_training + + def fill_flow_hole(self, ft, norm, ft_fill): + (N, C, H, W) = ft.shape + ft[norm == 0] = ft_fill[norm == 0] + + ft_1 = self.lpfilter(ft.clone()) + ft_ds = torch.nn.functional.interpolate( + input=ft_1, + size=(H // 4, W // 4), + mode='bilinear', + align_corners=False) + ft_up = torch.nn.functional.interpolate( + input=ft_ds, size=(H, W), mode='bilinear', align_corners=False) + + ft[norm == 0] = ft_up[norm == 0] + + return ft + + def forward(self, F10, F12, F21, F23, I1, I2, t): + if F12 is None or F21 is None: + return I1 + + if F10 is not None and F23 is not None: + F1t, F2t = self.acc(F10, F12, F21, F23, t) + + else: + F1t = t * F12 + F2t = (1 - t) * F21 + + # Flow Reversal + F1t_Ds = F.interpolate(F1t, scale_factor=1.0 / 3, mode='nearest') / 3 + F2t_Ds = F.interpolate(F2t, scale_factor=1.0 / 3, mode='nearest') / 3 + Ft1_Ds, norm1_Ds = self.fwarp(F1t_Ds, F1t_Ds) + Ft1_Ds = -Ft1_Ds + Ft2_Ds, norm2_Ds = self.fwarp(F2t_Ds, F2t_Ds) + Ft2_Ds = -Ft2_Ds + + Ft1_Ds[norm1_Ds > 0] \ + = Ft1_Ds[norm1_Ds > 0] / norm1_Ds[norm1_Ds > 0].clone() + Ft2_Ds[norm2_Ds > 0] \ + = Ft2_Ds[norm2_Ds > 0] / norm2_Ds[norm2_Ds > 0].clone() + if 1: + Ft1_fill = -F1t_Ds + Ft2_fill = -F2t_Ds + Ft1_Ds = self.fill_flow_hole(Ft1_Ds, norm1_Ds, Ft1_fill) + Ft2_Ds = self.fill_flow_hole(Ft2_Ds, norm2_Ds, Ft2_fill) + + Ft1 = F.interpolate( + Ft1_Ds, size=[F1t.size(2), F1t.size(3)], mode='nearest') * 3 + Ft2 = F.interpolate( + Ft2_Ds, size=[F2t.size(2), F2t.size(3)], mode='nearest') * 3 + + I1t = backwarp(I1, Ft1) + I2t = backwarp(I2, Ft2) + + output, feature = self.refinenet( + torch.cat([I1, I2, I1t, I2t, F12, F21, Ft1, Ft2], dim=1)) + + # Adaptive filtering + Ft1r = backwarp(Ft1, 10 * torch.tanh(output[:, 4:6])) + output[:, :2] + Ft2r = backwarp(Ft2, 10 * torch.tanh(output[:, 6:8])) + output[:, 2:4] + + # warping and fusing + I1tf = backwarp(I1, Ft1r) + I2tf = backwarp(I2, Ft2r) + + M = torch.sigmoid( + self.masknet(torch.cat([I1tf, I2tf, feature], + dim=1))).repeat(1, 3, 1, 1) + + It_warp = ((1 - t) * M * I1tf + t * (1 - M) * I2tf) \ + / ((1 - t) * M + t * (1 - M)).clone() + + G1 = self.get_grad(I1) + G2 = self.get_grad(I2) + G1tf = backwarp(G1, Ft1r) + G2tf = backwarp(G2, Ft2r) + + # static blending + It_static = (1 - t) * I1 + t * I2 + M_static = self.staticnet( + torch.cat([I1tf, I2tf, G1tf, G2tf, I1, I2, G1, G2, feature], + dim=1)) + M_static_dilate = tensor_erode(M_static) + M_static_dilate = tensor_erode(M_static_dilate) + It_warp = (1 - M_static_dilate) * It_warp + M_static_dilate * It_static + + if self.is_training: + return It_warp, Ft1r, Ft2r + else: + if self.debug_en: + return It_warp, M, M_static, I1tf, I2tf, Ft1r, Ft2r + else: + return It_warp + + +class InterpNetDs(nn.Module): + + def __init__(self, debug_en=False, is_training=False): + super(InterpNetDs, self).__init__() + self.ifnet = IFNet() + self.internet = QVI_inter_Ds( + debug_en=debug_en, is_training=is_training) + + def forward(self, + img1, + img2, + F10_up, + F12_up, + F21_up, + F23_up, + UHD=2, + timestep=0.5): + F12, F21 = self.ifnet(img1, img2, F12_up, F21_up, UHD) + It_warp = self.internet(F10_up, F12, F21, F23_up, img1, img2, timestep) + + return It_warp + + +class InterpNet(nn.Module): + + def __init__(self, debug_en=False, is_training=False): + super(InterpNet, self).__init__() + self.ifnet = IFNet() + self.internet = QVI_inter(debug_en=debug_en, is_training=is_training) + + def forward(self, + img1, + img2, + F10_up, + F12_up, + F21_up, + F23_up, + UHD=2, + timestep=0.5): + F12, F21 = self.ifnet(img1, img2, F12_up, F21_up, UHD) + It_warp = self.internet(F10_up, F12, F21, F23_up, img1, img2, timestep) + + return It_warp diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/transformer_layers.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/transformer_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..81ce114bded7056be2e2bfb1a4eb51f170740168 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/interp_model/transformer_layers.py @@ -0,0 +1,989 @@ +# The implementation is adopted from VFIformer, +# made publicly available at https://github.com/dvlab-research/VFIformer + +# ----------------------------------------------------------------------------------- +# modified from: +# SwinIR: Image Restoration Using Swin Transformer, https://github.com/JingyunLiang/SwinIR +# ----------------------------------------------------------------------------------- + +import functools +import math +import sys + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, + C) + windows = x.permute(0, 1, 3, 2, 4, + 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, + window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + r""" Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1).contiguous()) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C).contiguous() + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}' + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class WindowCrossAttention(nn.Module): + r""" Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table_x = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + self.relative_position_bias_table_y = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, + 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.merge1 = nn.Linear(dim * 2, dim) + self.merge2 = nn.Linear(dim, dim) + self.act = nn.LeakyReLU(negative_slope=0.2, inplace=True) + self.proj = nn.Linear(dim, dim) + + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table_x, std=.02) + trunc_normal_(self.relative_position_bias_table_y, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, y, mask_x=None, mask_y=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1).contiguous()) + + relative_position_bias = self.relative_position_bias_table_x[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask_x is not None: + nW = mask_x.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask_x.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C).contiguous() + + B_, N, C = y.shape + kv = self.kv(y).reshape(B_, N, 2, self.num_heads, + C // self.num_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[ + 1] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1).contiguous()) + + relative_position_bias = self.relative_position_bias_table_y[ + self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask_y is not None: + nW = mask_y.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask_y.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + y = (attn @ v).transpose(1, 2).reshape(B_, N, C).contiguous() + + x = self.merge2(self.act(self.merge1(torch.cat([x, y], dim=-1)))) + x + + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}' + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class TFL(nn.Module): + + def __init__(self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + use_crossattn=False): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.use_crossattn = use_crossattn + + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + if not use_crossattn: + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + else: + self.attn = WindowCrossAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + if self.shift_size > 0: + if not use_crossattn: + attn_mask = self.calculate_mask(self.input_resolution) + self.register_buffer('attn_mask', attn_mask) + else: + attn_mask_x = self.calculate_mask(self.input_resolution) + attn_mask_y = self.calculate_mask2(self.input_resolution) + self.register_buffer('attn_mask_x', attn_mask_x) + self.register_buffer('attn_mask_y', attn_mask_y) + + else: + if not use_crossattn: + attn_mask = None + self.register_buffer('attn_mask', attn_mask) + else: + attn_mask_x = None + attn_mask_y = None + self.register_buffer('attn_mask_x', attn_mask_x) + self.register_buffer('attn_mask_y', attn_mask_y) + + def calculate_mask(self, x_size): + # calculate attention mask for SW-MSA + H, W = x_size + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + return attn_mask + + def calculate_mask2(self, x_size): + # calculate attention mask for SW-MSA + H, W = x_size + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, + -self.shift_size), slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, + self.window_size * self.window_size) + + # downscale + img_mask_down = F.interpolate( + img_mask.permute(0, 3, 1, 2).contiguous(), + scale_factor=0.5, + mode='bilinear', + align_corners=False) + img_mask_down = F.pad( + img_mask_down, (self.window_size // 4, self.window_size // 4, + self.window_size // 4, self.window_size // 4), + mode='reflect') + mask_windows_down = F.unfold( + img_mask_down, + kernel_size=self.window_size, + dilation=1, + padding=0, + stride=self.window_size // 2) + mask_windows_down = mask_windows_down.view( + self.window_size * self.window_size, + -1).permute(1, 0).contiguous() # nW, window_size*window_size + + attn_mask = mask_windows_down.unsqueeze(1) - mask_windows.unsqueeze( + 2) # nW, window_size*window_size, window_size*window_size + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + + return attn_mask + + def forward(self, x, x_size): + H, W = x_size + B, L, C = x.shape + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll( + x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + if not self.use_crossattn: + if self.input_resolution == x_size: + attn_windows = self.attn( + x_windows, + mask=self.attn_mask) # nW*B, window_size*window_size, C + else: + attn_windows = self.attn( + x_windows, mask=self.calculate_mask(x_size).to(x.device)) + else: + shifted_x_down = F.interpolate( + shifted_x.permute(0, 3, 1, 2).contiguous(), + scale_factor=0.5, + mode='bilinear', + align_corners=False) + shifted_x_down = F.pad( + shifted_x_down, (self.window_size // 4, self.window_size // 4, + self.window_size // 4, self.window_size // 4), + mode='reflect') + x_windows_down = F.unfold( + shifted_x_down, + kernel_size=self.window_size, + dilation=1, + padding=0, + stride=self.window_size // 2) + x_windows_down = x_windows_down.view( + B, C, self.window_size * self.window_size, -1) + x_windows_down = x_windows_down.permute( + 0, 3, 2, + 1).contiguous().view(-1, self.window_size * self.window_size, + C) # nW*B, window_size*window_size, C + + if self.input_resolution == x_size: + attn_windows = self.attn( + x_windows, + x_windows_down, + mask_x=self.attn_mask_x, + mask_y=self.attn_mask_y + ) # nW*B, window_size*window_size, C + else: + attn_windows = self.attn( + x_windows, + x_windows_down, + mask_x=self.calculate_mask(x_size).to(x.device), + mask_y=self.calculate_mask2(x_size).to(x.device)) + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, + self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, + W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll( + shifted_x, + shifts=(self.shift_size, self.shift_size), + dims=(1, 2)) + else: + x = shifted_x + + x = x.view(B, H * W, C) + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, ' \ + f'window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}' + + def flops(self): + flops = 0 + H, W = self.input_resolution + # norm1 + flops += self.dim * H * W + # W-MSA/SW-MSA + nW = H * W / self.window_size / self.window_size + flops += nW * self.attn.flops(self.window_size * self.window_size) + # mlp + flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + return flops + + +class PatchMerging(nn.Module): + r""" Patch Merging Layer. + + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, 'input feature has wrong size' + assert H % 2 == 0 and W % 2 == 0, f'x size ({H}*{W}) are not even.' + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + def extra_repr(self) -> str: + return f'input_resolution={self.input_resolution}, dim={self.dim}' + + def flops(self): + H, W = self.input_resolution + flops = H * W * self.dim + flops += (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim + return flops + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + use_crossattn=None): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + if use_crossattn is None: + use_crossattn = [False for i in range(depth)] + + # build blocks + self.blocks = nn.ModuleList([ + TFL(dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + use_crossattn=use_crossattn[i]) for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample( + input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, x_size): + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, x_size) + else: + x = blk(x, x_size) + if self.downsample is not None: + x = self.downsample(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}' + + def flops(self): + flops = 0 + for blk in self.blocks: + flops += blk.flops() + if self.downsample is not None: + flops += self.downsample.flops() + return flops + + +class RTFL(nn.Module): + + def __init__(self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + img_size=224, + patch_size=4, + resi_connection='1conv', + use_crossattn=None): + super(RTFL, self).__init__() + + self.dim = dim + self.input_resolution = input_resolution + self.use_crossattn = use_crossattn + + self.residual_group = BasicLayer( + dim=dim, + input_resolution=input_resolution, + depth=depth, + num_heads=num_heads, + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path, + norm_layer=norm_layer, + downsample=downsample, + use_checkpoint=use_checkpoint, + use_crossattn=use_crossattn) + + if resi_connection == '1conv': + self.conv = nn.Conv2d(dim, dim, 3, 1, 1) + elif resi_connection == '3conv': + # to save parameters and memory + self.conv = nn.Sequential( + nn.Conv2d(dim, dim // 4, 3, 1, 1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(dim // 4, dim // 4, 1, 1, 0), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(dim // 4, dim, 3, 1, 1)) + + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=0, + embed_dim=dim, + norm_layer=None) + + self.patch_unembed = PatchUnEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=0, + embed_dim=dim, + norm_layer=None) + + def forward(self, x, x_size): + return self.patch_embed( + self.conv( + self.patch_unembed(self.residual_group(x, x_size), + x_size))) + x + + def flops(self): + flops = 0 + flops += self.residual_group.flops() + H, W = self.input_resolution + flops += H * W * self.dim * self.dim * 9 + flops += self.patch_embed.flops() + flops += self.patch_unembed.flops() + + return flops + + +class PatchEmbed(nn.Module): + r""" Image to Patch Embedding + + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + img_size[0] // patch_size[0], img_size[1] // patch_size[1] + ] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + x = x.flatten(2).transpose(1, 2).contiguous() # B Ph*Pw C + if self.norm is not None: + x = self.norm(x) + return x + + def flops(self): + flops = 0 + H, W = self.img_size + if self.norm is not None: + flops += H * W * self.embed_dim + return flops + + +class PatchUnEmbed(nn.Module): + r""" Image to Patch Unembedding + + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + norm_layer=None): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + img_size[0] // patch_size[0], img_size[1] // patch_size[1] + ] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + def forward(self, x, x_size): + B, HW, C = x.shape + x = x.transpose(1, 2).contiguous().view(B, self.embed_dim, x_size[0], + x_size[1]) # B Ph*Pw C + return x + + def flops(self): + flops = 0 + return flops + + +class Upsample(nn.Sequential): + """Upsample module. + + Args: + scale (int): Scale factor. Supported scales: 2^n and 3. + num_feat (int): Channel number of intermediate features. + """ + + def __init__(self, scale, num_feat): + m = [] + if (scale & (scale - 1)) == 0: # scale = 2^n + for _ in range(int(math.log(scale, 2))): + m.append(nn.Conv2d(num_feat, 4 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(2)) + elif scale == 3: + m.append(nn.Conv2d(num_feat, 9 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(3)) + else: + raise ValueError(f'scale {scale} is not supported. ' + 'Supported scales: 2^n and 3.') + super(Upsample, self).__init__(*m) + + +class UpsampleOneStep(nn.Sequential): + """UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle) + Used in lightweight SR to save parameters. + + Args: + scale (int): Scale factor. Supported scales: 2^n and 3. + num_feat (int): Channel number of intermediate features. + + """ + + def __init__(self, scale, num_feat, num_out_ch, input_resolution=None): + self.num_feat = num_feat + self.input_resolution = input_resolution + m = [] + m.append(nn.Conv2d(num_feat, (scale**2) * num_out_ch, 3, 1, 1)) + m.append(nn.PixelShuffle(scale)) + super(UpsampleOneStep, self).__init__(*m) + + def flops(self): + H, W = self.input_resolution + flops = H * W * self.num_feat * 3 * 9 + return flops diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/__init__.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/scene_change_detection.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/scene_change_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..379fe85562010ef63e67cbe3a175d96ff36950e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/scene_change_detection.py @@ -0,0 +1,97 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def calc_hist(img_tensor): + hist = torch.histc(img_tensor, bins=64, min=0, max=255) + return hist / img_tensor.numel() + + +def do_scene_detect(F01_tensor, F10_tensor, img0_tensor, img1_tensor): + device = img0_tensor.device + scene_change = False + img0_tensor = img0_tensor.clone() + img1_tensor = img1_tensor.clone() + + img0_gray = 0.299 * img0_tensor[:, 0: + 1] + 0.587 * img0_tensor[:, 1: + 2] + 0.114 * img0_tensor[:, + 2: + 3] + img1_gray = 0.299 * img1_tensor[:, 0: + 1] + 0.587 * img1_tensor[:, 1: + 2] + 0.114 * img1_tensor[:, + 2: + 3] + img0_gray = torch.clamp(img0_gray, 0, 255).byte().float().cpu() + img1_gray = torch.clamp(img1_gray, 0, 255).byte().float().cpu() + + hist0 = calc_hist(img0_gray) + hist1 = calc_hist(img1_gray) + diff = torch.abs(hist0 - hist1) + diff[diff < 0.01] = 0 + if torch.sum(diff) > 0.8 or diff.max() > 0.4: + return True + img0_gray = img0_gray.to(device) + img1_gray = img1_gray.to(device) + + # second stage: detect mv and pix mismatch + + (n, c, h, w) = F01_tensor.size() + scale_x = w / 1920 + scale_y = h / 1080 + + # compare mv + (y, x) = torch.meshgrid(torch.arange(h), torch.arange(w)) + (y_grid, x_grid) = torch.meshgrid( + torch.arange(64, h - 64, 8), torch.arange(64, w - 64, 8)) + x = x.to(device) + y = y.to(device) + y_grid = y_grid.to(device) + x_grid = x_grid.to(device) + fx = F01_tensor[0, 0] + fy = F01_tensor[0, 1] + x_ = x.float() + fx + y_ = y.float() + fy + x_ = torch.clamp(x_ + 0.5, 0, w - 1).long() + y_ = torch.clamp(y_ + 0.5, 0, h - 1).long() + + grid_fx = fx[y_grid, x_grid] + grid_fy = fy[y_grid, x_grid] + + x_grid_ = x_[y_grid, x_grid] + y_grid_ = y_[y_grid, x_grid] + + grid_fx_ = F10_tensor[0, 0, y_grid_, x_grid_] + grid_fy_ = F10_tensor[0, 1, y_grid_, x_grid_] + + sum_x = grid_fx + grid_fx_ + sum_y = grid_fy + grid_fy_ + distance = torch.sqrt(sum_x**2 + sum_y**2) + + fx_len = torch.abs(grid_fx) * scale_x + fy_len = torch.abs(grid_fy) * scale_y + ori_len = torch.where(fx_len > fy_len, fx_len, fy_len) + + thres = torch.clamp(0.1 * ori_len + 4, 5, 14) + + # compare pix diff + ori_img = img0_gray + ref_img = img1_gray[:, :, y_, x_] + + img_diff = ori_img.float() - ref_img.float() + img_diff = torch.abs(img_diff) + + kernel = np.ones([8, 8], float) / 64 + kernel = torch.FloatTensor(kernel).to(device).unsqueeze(0).unsqueeze(0) + diff = F.conv2d(img_diff, kernel, padding=4) + + diff = diff[0, 0, y_grid, x_grid] + + index = (distance > thres) * (diff > 5) + if index.sum().float() / distance.numel() > 0.5: + scene_change = True + return scene_change diff --git a/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/utils.py b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..68a8b99d155b166a32a17b8fb645b725b8f6a52d --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_frame_interpolation/utils/utils.py @@ -0,0 +1,96 @@ +# The implementation is adopted from RAFT, +# made publicly available under the BSD-3-Clause license at https://github.com/princeton-vl/RAFT + +import numpy as np +import torch +import torch.nn.functional as F +from scipy import interpolate + + +class InputPadder: + """ Pads images such that dimensions are divisible by 8 """ + + def __init__(self, dims, mode='sintel'): + self.ht, self.wd = dims[-2:] + pad_ht = (((self.ht // 8) + 1) * 8 - self.ht) % 8 + pad_wd = (((self.wd // 8) + 1) * 8 - self.wd) % 8 + if mode == 'sintel': + self._pad = [ + pad_wd // 2, pad_wd - pad_wd // 2, pad_ht // 2, + pad_ht - pad_ht // 2 + ] + else: + self._pad = [pad_wd // 2, pad_wd - pad_wd // 2, 0, pad_ht] + + def pad(self, *inputs): + return [F.pad(x, self._pad, mode='replicate') for x in inputs] + + def unpad(self, x): + ht, wd = x.shape[-2:] + c = [self._pad[2], ht - self._pad[3], self._pad[0], wd - self._pad[1]] + return x[..., c[0]:c[1], c[2]:c[3]] + + +def forward_interpolate(flow): + flow = flow.detach().cpu().numpy() + dx, dy = flow[0], flow[1] + + ht, wd = dx.shape + x0, y0 = np.meshgrid(np.arange(wd), np.arange(ht)) + + x1 = x0 + dx + y1 = y0 + dy + + x1 = x1.reshape(-1) + y1 = y1.reshape(-1) + dx = dx.reshape(-1) + dy = dy.reshape(-1) + + valid = (x1 > 0) & (x1 < wd) & (y1 > 0) & (y1 < ht) + x1 = x1[valid] + y1 = y1[valid] + dx = dx[valid] + dy = dy[valid] + + flow_x = interpolate.griddata((x1, y1), + dx, (x0, y0), + method='nearest', + fill_value=0) + + flow_y = interpolate.griddata((x1, y1), + dy, (x0, y0), + method='nearest', + fill_value=0) + + flow = np.stack([flow_x, flow_y], axis=0) + return torch.from_numpy(flow).float() + + +def bilinear_sampler(img, coords, mode='bilinear', mask=False): + """ Wrapper for grid_sample, uses pixel coordinates """ + H, W = img.shape[-2:] + xgrid, ygrid = coords.split([1, 1], dim=-1) + xgrid = 2 * xgrid / (W - 1) - 1 + ygrid = 2 * ygrid / (H - 1) - 1 + + grid = torch.cat([xgrid, ygrid], dim=-1) + img = F.grid_sample(img, grid, align_corners=True) + + if mask: + mask = (xgrid > -1) & (ygrid > -1) & (xgrid < 1) & (ygrid < 1) + return img, mask.float() + + return img + + +def coords_grid(batch, ht, wd, device): + coords = torch.meshgrid( + torch.arange(ht, device=device), torch.arange(wd, device=device)) + coords = torch.stack(coords[::-1], dim=0).float() + return coords[None].repeat(batch, 1, 1, 1) + + +def upflow8(flow, mode='bilinear'): + new_size = (8 * flow.shape[2], 8 * flow.shape[3]) + return 8 * F.interpolate( + flow, size=new_size, mode=mode, align_corners=True) diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/__init__.py b/runtime/Lib/site_packages/models/cv/video_human_matting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d47317c82c0be411a99380de64f0790aac1ed2b --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import VideoMattingNetwork + from .model import preprocess + +else: + _import_structure = {'model': ['VideoMattingNetwork', 'preprocess']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/model.py b/runtime/Lib/site_packages/models/cv/video_human_matting/model.py new file mode 100644 index 0000000000000000000000000000000000000000..9894805104f4b1a2449f2b04e3cd0df9b4250b67 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/model.py @@ -0,0 +1,38 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Optional + +import numpy as np +import torch +import torchvision +from torch.nn import functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.video_human_matting.models import MattingNetwork +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + + +@MODELS.register_module( + Tasks.video_human_matting, module_name=Models.video_human_matting) +class VideoMattingNetwork(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) + params = torch.load(model_path, map_location='cpu') + self.model = MattingNetwork() + if 'model_state_dict' in params.keys(): + params = params['model_state_dict'] + self.model.load_state_dict(params, strict=True) + self.model.eval() + + +def preprocess(image): + frame_np = np.float32(image) / 255.0 + frame_np = frame_np.transpose(2, 0, 1) + frame_tensor = torch.from_numpy(frame_np) + image_tensor = frame_tensor[None, :, :, :] + return image_tensor diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/__init__.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..471f0308677c4e49cd652273a7e1477c92cdb105 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/__init__.py @@ -0,0 +1 @@ +from .matting import MattingNetwork diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/decoder.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..ba82aa909a2f76a6906fb949aec53aeaedc9e361 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/decoder.py @@ -0,0 +1,330 @@ +""" +Part of the implementation is borrowed from paper RVM +paper publicly available at +""" +from typing import Optional + +import torch +from torch import Tensor, nn + + +class hswish(nn.Module): + + def forward(self, x): + return torch.nn.Hardswish(inplace=True)(x) + + +class scSEblock(nn.Module): + + def __init__(self, out): + super().__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(out, int(out / 2), 3, 1, 1), + nn.GroupNorm(out // 8, int(out / 2)), hswish()) + self.conv2 = nn.Sequential( + nn.Conv2d(int(out / 2), out, 1, 1, 0), + nn.GroupNorm(out // 4, out), + ) + self.avgpool = nn.AdaptiveAvgPool2d(1) + + def forward_single(self, x): + b, c, _, _ = x.size() + x2 = self.avgpool(x).view(b, c, 1, 1) + x2 = self.conv1(x2) + x2 = self.conv2(x2) + x2 = torch.sigmoid(x2) + out = x2 * x + return out + + def forward_time(self, x): + B, T, _, H, W = x.shape + x = x.flatten(0, 1) + out = self.forward_single(x) + out = out.unflatten(0, (B, T)) + return out + + def forward(self, x): + if x.ndim == 5: + return self.forward_time(x) + else: + return self.forward_single(x) + + +class RecurrentDecoder(nn.Module): + + def __init__(self, feature_channels, decoder_channels): + super().__init__() + self.avgpool = AvgPool() + self.decode4 = BottleneckBlock(feature_channels[3]) + self.decode3 = UpsamplingBlock(feature_channels[3], + feature_channels[2], 3, + decoder_channels[0]) + self.sc3 = scSEblock(decoder_channels[0]) + self.decode2 = UpsamplingBlock(decoder_channels[0], + feature_channels[1], 3, + decoder_channels[1]) + self.sc2 = scSEblock(decoder_channels[1]) + self.decode1 = UpsamplingBlock(decoder_channels[1], + feature_channels[0], 3, + decoder_channels[2]) + self.sc1 = scSEblock(decoder_channels[2]) + self.out0 = OutputBlock(decoder_channels[2], 3, decoder_channels[3]) + + self.crosslevel1 = crossfeature(feature_channels[3], + feature_channels[1]) + self.crosslevel2 = crossfeature(feature_channels[2], + feature_channels[0]) + + def forward(self, s0: Tensor, f1: Tensor, f2: Tensor, f3: Tensor, + f4: Tensor, r1: Optional[Tensor], r2: Optional[Tensor], + r3: Optional[Tensor], r4: Optional[Tensor]): + s2, s3, s4 = self.avgpool(s0) + x4, r4 = self.decode4(f4, r4) + x3, r3 = self.decode3(x4, f3, s4, r3) + x3 = self.sc3(x3) + f2 = self.crosslevel1(f4, f2) + x2, r2 = self.decode2(x3, f2, s3, r2) + x2 = self.sc2(x2) + f1 = self.crosslevel2(f3, f1) + x1, r1 = self.decode1(x2, f1, s2, r1) + x1 = self.sc1(x1) + out = self.out0(x1, s0) + return out, r1, r2, r3, r4 + + +class AvgPool(nn.Module): + + def __init__(self): + super().__init__() + self.avgpool = nn.AvgPool2d( + 2, 2, count_include_pad=False, ceil_mode=True) + + def forward_single_frame(self, s0): + s1 = self.avgpool(s0) + s2 = self.avgpool(s1) + s3 = self.avgpool(s2) + return s1, s2, s3 + + def forward_time_series(self, s0): + B, T = s0.shape[:2] + s0 = s0.flatten(0, 1) + s1, s2, s3 = self.forward_single_frame(s0) + s1 = s1.unflatten(0, (B, T)) + s2 = s2.unflatten(0, (B, T)) + s3 = s3.unflatten(0, (B, T)) + return s1, s2, s3 + + def forward(self, s0): + if s0.ndim == 5: + return self.forward_time_series(s0) + else: + return self.forward_single_frame(s0) + + +class crossfeature(nn.Module): + + def __init__(self, in_channels, out_channels): + super().__init__() + self.avg = nn.AdaptiveAvgPool2d(1) + self.conv = nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False) + + def forward_single_frame(self, x1, x2): + b, c, _, _ = x1.size() + x1 = self.avg(x1).view(b, c, 1, 1) + x1 = self.conv(x1) + x1 = torch.sigmoid(x1) + x2 = x1 * x2 + return x2 + + def forward_time_series(self, x1, x2): + b, t = x1.shape[:2] + x1 = x1.flatten(0, 1) + x2 = x2.flatten(0, 1) + x2 = self.forward_single_frame(x1, x2) + return x2.unflatten(0, (b, t)) + + def forward(self, x1, x2): + if x1.ndim == 5: + return self.forward_time_series(x1, x2) + else: + return self.forward_single_frame(x1, x2) + + +class BottleneckBlock(nn.Module): + + def __init__(self, channels): + super().__init__() + self.channels = channels + self.gru = GRU(channels // 2) + + def forward(self, x, r): + a, b = x.split(self.channels // 2, dim=-3) + b, r = self.gru(b, r) + x = torch.cat([a, b], dim=-3) + return x, r + + +class UpsamplingBlock(nn.Module): + + def __init__(self, in_channels, skip_channels, src_channels, out_channels): + super().__init__() + self.out_channels = out_channels + self.upsample = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=False) + self.shortcut = nn.Sequential( + nn.Conv2d(skip_channels, in_channels, 3, 1, 1, bias=False), + nn.GroupNorm(in_channels // 4, in_channels), hswish()) + self.att_skip = nn.Sequential( + nn.Conv2d(in_channels, in_channels, 1, 1, 0, bias=False), + nn.Sigmoid()) + self.conv = nn.Sequential( + nn.Conv2d( + in_channels + in_channels + src_channels, + out_channels, + 3, + 1, + 1, + bias=False), + nn.GroupNorm(out_channels // 4, out_channels), + hswish(), + ) + self.gru = GRU(out_channels // 2) + + def forward_single_frame(self, x, f, s, r: Optional[Tensor]): + x = self.upsample(x) + x = x[:, :, :s.size(2), :s.size(3)] + att = self.att_skip(x) + f = self.shortcut(f) + f = att * f + x = torch.cat([x, f, s], dim=1) + x = self.conv(x) + a, b = x.split(self.out_channels // 2, dim=1) + b, r = self.gru(b, r) + x = torch.cat([a, b], dim=1) + return x, r + + def forward_time_series(self, x, f, s, r: Optional[Tensor]): + B, T, _, H, W = s.shape + x = x.flatten(0, 1) + f = f.flatten(0, 1) + s = s.flatten(0, 1) + x = self.upsample(x) + x = x[:, :, :H, :W] + f = self.shortcut(f) + att = self.att_skip(x) + f = att * f + x = torch.cat([x, f, s], dim=1) + x = self.conv(x) + x = x.unflatten(0, (B, T)) + a, b = x.split(self.out_channels // 2, dim=2) + b, r = self.gru(b, r) + x = torch.cat([a, b], dim=2) + return x, r + + def forward(self, x, f, s, r: Optional[Tensor]): + if x.ndim == 5: + return self.forward_time_series(x, f, s, r) + else: + return self.forward_single_frame(x, f, s, r) + + +class OutputBlock(nn.Module): + + def __init__(self, in_channels, src_channels, out_channels): + super().__init__() + self.upsample = nn.Upsample( + scale_factor=2, mode='bilinear', align_corners=False) + self.conv = nn.Sequential( + nn.Conv2d( + in_channels + src_channels, out_channels, 3, 1, 1, bias=False), + nn.GroupNorm(out_channels // 2, out_channels), + hswish(), + nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False), + nn.GroupNorm(out_channels // 2, out_channels), + hswish(), + ) + + def forward_single_frame(self, x, s): + x = self.upsample(x) + x = x[:, :, :s.size(2), :s.size(3)] + x = torch.cat([x, s], dim=1) + x = self.conv(x) + return x + + def forward_time_series(self, x, s): + B, T, _, H, W = s.shape + x = x.flatten(0, 1) + s = s.flatten(0, 1) + x = self.upsample(x) + x = x[:, :, :H, :W] + x = torch.cat([x, s], dim=1) + x = self.conv(x) + x = x.unflatten(0, (B, T)) + return x + + def forward(self, x, s): + if x.ndim == 5: + return self.forward_time_series(x, s) + else: + return self.forward_single_frame(x, s) + + +class Projection(nn.Module): + + def __init__(self, in_channels, out_channels): + super().__init__() + self.conv = nn.Conv2d(in_channels, out_channels, 1) + + def forward_single_frame(self, x): + return self.conv(x) + + def forward_time_series(self, x): + B, T = x.shape[:2] + return self.conv(x.flatten(0, 1)).unflatten(0, (B, T)) + + def forward(self, x): + if x.ndim == 5: + return self.forward_time_series(x) + else: + return self.forward_single_frame(x) + + +class GRU(nn.Module): + + def __init__(self, channels, kernel_size=3, padding=1): + super().__init__() + self.channels = channels + self.ih = nn.Conv2d( + channels * 2, channels * 2, kernel_size, padding=padding) + self.act_ih = nn.Sigmoid() + self.hh = nn.Conv2d( + channels * 2, channels, kernel_size, padding=padding) + self.act_hh = nn.Tanh() + + def forward_single_frame(self, x, pre_fea): + fea_ih = self.ih(torch.cat([x, pre_fea], dim=1)) + r, z = self.act_ih(fea_ih).split(self.channels, dim=1) + fea_hh = self.hh(torch.cat([x, r * pre_fea], dim=1)) + c = self.act_hh(fea_hh) + fea_gru = (1 - z) * pre_fea + z * c + return fea_gru, fea_gru + + def forward_time_series(self, x, pre_fea): + o = [] + for xt in x.unbind(dim=1): + ot, pre_fea = self.forward_single_frame(xt, pre_fea) + o.append(ot) + o = torch.stack(o, dim=1) + return o, pre_fea + + def forward(self, x, pre_fea): + if pre_fea is None: + pre_fea = torch.zeros( + (x.size(0), x.size(-3), x.size(-2), x.size(-1)), + device=x.device, + dtype=x.dtype) + + if x.ndim == 5: + return self.forward_time_series(x, pre_fea) + else: + return self.forward_single_frame(x, pre_fea) diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/deep_guided_filter.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/deep_guided_filter.py new file mode 100644 index 0000000000000000000000000000000000000000..c0081026a36f558ee1c1ed34b09238b08d3dfe6c --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/deep_guided_filter.py @@ -0,0 +1,64 @@ +""" +Part of the implementation is borrowed and modified from DeepGuidedFilter +publicly available at +""" +import torch +from torch import nn +from torch.nn import functional as F + + +class DeepGuidedFilterRefiner(nn.Module): + + def __init__(self, hid_channels=16): + super().__init__() + self.box_filter = nn.Conv2d( + 4, 4, kernel_size=3, padding=1, bias=False, groups=4) + self.box_filter.weight.data[...] = 1 / 9 + self.conv = nn.Sequential( + nn.Conv2d( + 4 * 2 + hid_channels, hid_channels, kernel_size=1, bias=False), + nn.BatchNorm2d(hid_channels), nn.ReLU(True), + nn.Conv2d(hid_channels, hid_channels, kernel_size=1, bias=False), + nn.BatchNorm2d(hid_channels), nn.ReLU(True), + nn.Conv2d(hid_channels, 4, kernel_size=1, bias=True)) + + def forward_single_frame(self, fine_src, base_src, base_fgr, base_pha, + base_hid): + fine_x = torch.cat([fine_src, fine_src.mean(1, keepdim=True)], dim=1) + base_x = torch.cat([base_src, base_src.mean(1, keepdim=True)], dim=1) + base_y = torch.cat([base_fgr, base_pha], dim=1) + + mean_x = self.box_filter(base_x) + mean_y = self.box_filter(base_y) + cov_xy = self.box_filter(base_x * base_y) - mean_x * mean_y + var_x = self.box_filter(base_x * base_x) - mean_x * mean_x + + A = self.conv(torch.cat([cov_xy, var_x, base_hid], dim=1)) + b = mean_y - A * mean_x + + H, W = fine_src.shape[2:] + A = F.interpolate(A, (H, W), mode='bilinear', align_corners=False) + b = F.interpolate(b, (H, W), mode='bilinear', align_corners=False) + + out = A * fine_x + b + fgr, pha = out.split([3, 1], dim=1) + return fgr, pha + + def forward_time_series(self, fine_src, base_src, base_fgr, base_pha, + base_hid): + B, T = fine_src.shape[:2] + fgr, pha = self.forward_single_frame( + fine_src.flatten(0, 1), base_src.flatten(0, 1), + base_fgr.flatten(0, 1), base_pha.flatten(0, 1), + base_hid.flatten(0, 1)) + fgr = fgr.unflatten(0, (B, T)) + pha = pha.unflatten(0, (B, T)) + return fgr, pha + + def forward(self, fine_src, base_src, base_fgr, base_pha, base_hid): + if fine_src.ndim == 5: + return self.forward_time_series(fine_src, base_src, base_fgr, + base_pha, base_hid) + else: + return self.forward_single_frame(fine_src, base_src, base_fgr, + base_pha, base_hid) diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/effv2.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/effv2.py new file mode 100644 index 0000000000000000000000000000000000000000..8151e3b117710bc905758069ef32c1a17381f2eb --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/effv2.py @@ -0,0 +1,177 @@ +""" +Part of the implementation is borrowed and modified from EfficientNetV2 +publicly available at +""" + +import torch +import torch.nn.functional + + +class SiLU(torch.nn.Module): + """ + [https://arxiv.org/pdf/1710.05941.pdf] + """ + + def __init__(self, inplace: bool = False): + super().__init__() + self.silu = torch.nn.SiLU(inplace=inplace) + + def forward(self, x): + return self.silu(x) + + +class Conv(torch.nn.Module): + + def __init__(self, in_ch, out_ch, activation, k=1, s=1, g=1): + super().__init__() + self.conv = torch.nn.Conv2d( + in_ch, out_ch, k, s, k // 2, 1, g, bias=False) + self.norm = torch.nn.BatchNorm2d(out_ch, 0.001, 0.01) + self.silu = activation + + def forward(self, x): + return self.silu(self.norm(self.conv(x))) + + +class SE(torch.nn.Module): + """ + [https://arxiv.org/pdf/1709.01507.pdf] + """ + + def __init__(self, ch, r): + super().__init__() + self.se = torch.nn.Sequential( + torch.nn.Conv2d(ch, ch // (4 * r), 1), torch.nn.SiLU(), + torch.nn.Conv2d(ch // (4 * r), ch, 1), torch.nn.Sigmoid()) + + def forward(self, x): + return x * self.se(x.mean((2, 3), keepdim=True)) + + +class Residual(torch.nn.Module): + """ + [https://arxiv.org/pdf/1801.04381.pdf] + """ + + def __init__(self, in_ch, out_ch, s, r, fused=True): + super().__init__() + identity = torch.nn.Identity() + if fused: + if r == 1: + features = [Conv(in_ch, r * in_ch, torch.nn.SiLU(), 3, s)] + else: + features = [ + Conv(in_ch, r * in_ch, torch.nn.SiLU(), 3, s), + Conv(r * in_ch, out_ch, identity) + ] + else: + if r == 1: + features = [ + Conv(r * in_ch, r * in_ch, torch.nn.SiLU(), 3, s, + r * in_ch), + SE(r * in_ch, r), + Conv(r * in_ch, out_ch, identity) + ] + else: + features = [ + Conv(in_ch, r * in_ch, torch.nn.SiLU()), + Conv(r * in_ch, r * in_ch, torch.nn.SiLU(), 3, s, + r * in_ch), + SE(r * in_ch, r), + Conv(r * in_ch, out_ch, identity) + ] + self.add = s == 1 and in_ch == out_ch + self.res = torch.nn.Sequential(*features) + + def forward(self, x): + return x + self.res(x) if self.add else self.res(x) + + +class EfficientNet(torch.nn.Module): + + def __init__(self, pretrained: bool = False): + super().__init__() + gate_fn = [True, False] + filters = [24, 48, 64, 128, 160, 256] + feature = [Conv(3, filters[0], torch.nn.SiLU(), 3, 2)] + for i in range(2): + if i == 0: + feature.append( + Residual(filters[0], filters[0], 1, 1, gate_fn[0])) + else: + feature.append( + Residual(filters[0], filters[0], 1, 1, gate_fn[0])) + + for i in range(4): + if i == 0: + feature.append( + Residual(filters[0], filters[1], 2, 4, gate_fn[0])) + else: + feature.append( + Residual(filters[1], filters[1], 1, 4, gate_fn[0])) + + for i in range(4): + if i == 0: + feature.append( + Residual(filters[1], filters[2], 2, 4, gate_fn[0])) + else: + feature.append( + Residual(filters[2], filters[2], 1, 4, gate_fn[0])) + + for i in range(6): + if i == 0: + feature.append( + Residual(filters[2], filters[3], 2, 4, gate_fn[1])) + else: + feature.append( + Residual(filters[3], filters[3], 1, 4, gate_fn[1])) + + for i in range(9): + if i == 0: + feature.append( + Residual(filters[3], filters[4], 1, 6, gate_fn[1])) + else: + feature.append( + Residual(filters[4], filters[4], 1, 6, gate_fn[1])) + + self.feature = torch.nn.Sequential(*feature) + + def forward_single_frame(self, x): + x = self.feature[0](x) + x = self.feature[1](x) + x = self.feature[2](x) + f1 = x # 1/2 24 + for i in range(4): + x = self.feature[i + 3](x) + f2 = x # 1/4 48 + for i in range(4): + x = self.feature[i + 7](x) + f3 = x # 1/8 64 + for i in range(6): + x = self.feature[i + 11](x) + for i in range(9): + x = self.feature[i + 17](x) + f5 = x # 1/16 160 + return [f1, f2, f3, f5] + + def forward_time_series(self, x): + B, T = x.shape[:2] + features = self.forward_single_frame(x.flatten(0, 1)) + features = [f.unflatten(0, (B, T)) for f in features] + return features + + def forward(self, x): + if x.ndim == 5: + return self.forward_time_series(x) + else: + return self.forward_single_frame(x) + + def export(self): + for m in self.modules(): + if type(m) is Conv and hasattr(m, 'silu'): + if isinstance(m.silu, torch.nn.SiLU): + m.silu = SiLU() + if type(m) is SE: + if isinstance(m.se[1], torch.nn.SiLU): + m.se[1] = SiLU() + return self diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/lraspp.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/lraspp.py new file mode 100644 index 0000000000000000000000000000000000000000..234b81de32eb10d0b36e8d63c9c24fb4ad4abf04 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/lraspp.py @@ -0,0 +1,94 @@ +""" +Part of the implementation is borrowed and modified from Deeplab v3 +publicly available at +""" +import torch +from torch import nn + + +class ASP_OC_Module(nn.Module): + + def __init__(self, features, out_features=96, dilations=(2, 4, 8)): + super(ASP_OC_Module, self).__init__() + self.conv2 = nn.Sequential( + nn.Conv2d( + features, + out_features, + kernel_size=1, + padding=0, + dilation=1, + bias=False), nn.BatchNorm2d(out_features)) + self.conv3 = nn.Sequential( + nn.Conv2d( + features, + out_features, + kernel_size=3, + padding=dilations[0], + dilation=dilations[0], + bias=False), nn.BatchNorm2d(out_features)) + self.conv4 = nn.Sequential( + nn.Conv2d( + features, + out_features, + kernel_size=3, + padding=dilations[1], + dilation=dilations[1], + bias=False), nn.BatchNorm2d(out_features)) + self.conv5 = nn.Sequential( + nn.Conv2d( + features, + out_features, + kernel_size=3, + padding=dilations[2], + dilation=dilations[2], + bias=False), nn.BatchNorm2d(out_features)) + + self.conv_bn_dropout = nn.Sequential( + nn.Conv2d( + out_features * 4, + out_features * 2, + kernel_size=1, + padding=0, + dilation=1, + bias=False), nn.InstanceNorm2d(out_features * 2), + nn.Dropout2d(0.05)) + + def _cat_each(self, feat1, feat2, feat3, feat4, feat5): + assert (len(feat1) == len(feat2)) + z = [] + for i in range(len(feat1)): + z.append( + torch.cat((feat1[i], feat2[i], feat3[i], feat4[i], feat5[i]), + 1)) + return z + + def forward(self, x): + _, _, h, w = x.size() + feat2 = self.conv2(x) + feat3 = self.conv3(x) + feat4 = self.conv4(x) + feat5 = self.conv5(x) + out = torch.cat((feat2, feat3, feat4, feat5), 1) + output = self.conv_bn_dropout(out) + return output + + +class LRASPP(nn.Module): + + def __init__(self, in_channels, out_channels): + super().__init__() + self.aspp = ASP_OC_Module(in_channels, out_channels) + + def forward_single_frame(self, x): + return self.aspp(x) + + def forward_time_series(self, x): + B, T = x.shape[:2] + x = self.forward_single_frame(x.flatten(0, 1)).unflatten(0, (B, T)) + return x + + def forward(self, x): + if x.ndim == 5: + return self.forward_time_series(x) + else: + return self.forward_single_frame(x) diff --git a/runtime/Lib/site_packages/models/cv/video_human_matting/models/matting.py b/runtime/Lib/site_packages/models/cv/video_human_matting/models/matting.py new file mode 100644 index 0000000000000000000000000000000000000000..95cce15f8312c4611a08ac43b26ea52b48166af6 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_human_matting/models/matting.py @@ -0,0 +1,67 @@ +from typing import Optional + +import torch +from torch import Tensor +from torch.nn import functional as F + +from .decoder import Projection, RecurrentDecoder +from .deep_guided_filter import DeepGuidedFilterRefiner +from .effv2 import EfficientNet +from .lraspp import LRASPP + + +class MattingNetwork(torch.nn.Module): + + def __init__(self, pretrained_backbone: bool = False): + super().__init__() + self.backbone = EfficientNet(pretrained_backbone) + self.aspp = LRASPP(160, 64) + self.decoder = RecurrentDecoder([24, 48, 64, 128], [64, 32, 24, 16]) + self.project_mat = Projection(16, 4) + self.project_seg = Projection(16, 1) + self.refiner = DeepGuidedFilterRefiner() + + def forward(self, + src: Tensor, + r0: Optional[Tensor] = None, + r1: Optional[Tensor] = None, + r2: Optional[Tensor] = None, + r3: Optional[Tensor] = None, + downsample_ratio: float = 1, + segmentation_pass: bool = False): + + if downsample_ratio != 1: + src_sm = self._interpolate(src, scale_factor=downsample_ratio) + else: + src_sm = src + + f1, f2, f3, f4 = self.backbone(src_sm) + f4 = self.aspp(f4) + hid, *rec = self.decoder(src_sm, f1, f2, f3, f4, r0, r1, r2, r3) + + if not segmentation_pass: + fgr_residual, pha = self.project_mat(hid).split([3, 1], dim=-3) + if downsample_ratio != 1: + _, pha = self.refiner(src, src_sm, fgr_residual, pha, hid) + pha = pha.clamp(0., 1.) + return [pha, *rec] + else: + seg = self.project_seg(hid) + return [seg, *rec] + + def _interpolate(self, x: Tensor, scale_factor: float): + if x.ndim == 5: + B, T = x.shape[:2] + x = F.interpolate( + x.flatten(0, 1), + scale_factor=scale_factor, + mode='bilinear', + align_corners=False) + x = x.unflatten(0, (B, T)) + else: + x = F.interpolate( + x, + scale_factor=scale_factor, + mode='bilinear', + align_corners=False) + return x diff --git a/runtime/Lib/site_packages/models/cv/video_inpainting/__init__.py b/runtime/Lib/site_packages/models/cv/video_inpainting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f5489da96f9310063daaf2d0b2291694a1652c1f --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_inpainting/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .inpainting_model import VideoInpainting + +else: + _import_structure = {'inpainting_model': ['VideoInpainting']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting.py b/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting.py new file mode 100644 index 0000000000000000000000000000000000000000..7502bf38a3d969ee6100e940b3b857a250c51ffd --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting.py @@ -0,0 +1,299 @@ +""" VideoInpaintingProcess +The implementation here is modified based on STTN, +originally Apache 2.0 License and publicly available at https://github.com/researchmm/STTN +""" + +import os +import time + +import cv2 +import numpy as np +import torch +from PIL import Image +from torchvision import transforms + +torch.backends.cudnn.enabled = False + +w, h = 192, 96 +ref_length = 300 +neighbor_stride = 20 +default_fps = 24 +MAX_frame = 300 + + +def video_process(video_input_path): + video_input = cv2.VideoCapture(video_input_path) + success, frame = video_input.read() + if success is False: + decode_error = 'decode_error' + w, h, fps = 0, 0, 0 + else: + decode_error = None + h, w = frame.shape[0:2] + fps = video_input.get(cv2.CAP_PROP_FPS) + video_input.release() + + return decode_error, fps, w, h + + +class Stack(object): + + def __init__(self, roll=False): + self.roll = roll + + def __call__(self, img_group): + mode = img_group[0].mode + if mode == '1': + img_group = [img.convert('L') for img in img_group] + mode = 'L' + if mode == 'L': + return np.stack([np.expand_dims(x, 2) for x in img_group], axis=2) + elif mode == 'RGB': + if self.roll: + return np.stack([np.array(x)[:, :, ::-1] for x in img_group], + axis=2) + else: + return np.stack(img_group, axis=2) + else: + raise NotImplementedError(f'Image mode {mode}') + + +class ToTorchFormatTensor(object): + """ Converts a PIL.Image (RGB) or numpy.ndarray (H x W x C) in the range [0, 255] + to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] """ + + def __init__(self, div=True): + self.div = div + + def __call__(self, pic): + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic).permute(2, 3, 0, 1).contiguous() + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + img = img.view(pic.size[1], pic.size[0], len(pic.mode)) + img = img.transpose(0, 1).transpose(0, 2).contiguous() + img = img.float().div(255) if self.div else img.float() + return img + + +_to_tensors = transforms.Compose([Stack(), ToTorchFormatTensor()]) + + +def get_crop_mask_v1(mask): + orig_h, orig_w, _ = mask.shape + if (mask == 255).all(): + return mask, (0, int(orig_h), 0, + int(orig_w)), [0, int(orig_h), 0, + int(orig_w) + ], [0, int(orig_h), 0, + int(orig_w)] + + hs = np.min(np.where(mask == 0)[0]) + he = np.max(np.where(mask == 0)[0]) + ws = np.min(np.where(mask == 0)[1]) + we = np.max(np.where(mask == 0)[1]) + crop_box = [ws, hs, we, he] + + mask_h = round(int(orig_h / 2) / 4) * 4 + mask_w = round(int(orig_w / 2) / 4) * 4 + + if (hs < mask_h) and (he < mask_h) and (ws < mask_w) and (we < mask_w): + crop_mask = mask[:mask_h, :mask_w, :] + res_pix = (0, mask_h, 0, mask_w) + elif (hs < mask_h) and (he < mask_h) and (ws > mask_w) and (we > mask_w): + crop_mask = mask[:mask_h, orig_w - mask_w:orig_w, :] + res_pix = (0, mask_h, orig_w - mask_w, int(orig_w)) + elif (hs > mask_h) and (he > mask_h) and (ws < mask_w) and (we < mask_w): + crop_mask = mask[orig_h - mask_h:orig_h, :mask_w, :] + res_pix = (orig_h - mask_h, int(orig_h), 0, mask_w) + elif (hs > mask_h) and (he > mask_h) and (ws > mask_w) and (we > mask_w): + crop_mask = mask[orig_h - mask_h:orig_h, orig_w - mask_w:orig_w, :] + res_pix = (orig_h - mask_h, int(orig_h), orig_w - mask_w, int(orig_w)) + + elif (hs < mask_h) and (he < mask_h) and (ws < mask_w) and (we > mask_w): + crop_mask = mask[:mask_h, :, :] + res_pix = (0, mask_h, 0, int(orig_w)) + elif (hs < mask_h) and (he > mask_h) and (ws < mask_w) and (we < mask_w): + crop_mask = mask[:, :mask_w, :] + res_pix = (0, int(orig_h), 0, mask_w) + elif (hs > mask_h) and (he > mask_h) and (ws < mask_w) and (we > mask_w): + crop_mask = mask[orig_h - mask_h:orig_h, :, :] + res_pix = (orig_h - mask_h, int(orig_h), 0, int(orig_w)) + elif (hs < mask_h) and (he > mask_h) and (ws > mask_w) and (we > mask_w): + crop_mask = mask[:, orig_w - mask_w:orig_w, :] + res_pix = (0, int(orig_h), orig_w - mask_w, int(orig_w)) + else: + crop_mask = mask + res_pix = (0, int(orig_h), 0, int(orig_w)) + a = ws - res_pix[2] + b = hs - res_pix[0] + c = we - res_pix[2] + d = he - res_pix[0] + return crop_mask, res_pix, crop_box, [a, b, c, d] + + +def get_ref_index(neighbor_ids, length): + ref_index = [] + for i in range(0, length, ref_length): + if i not in neighbor_ids: + ref_index.append(i) + return ref_index + + +def read_mask_oneImage(mpath): + masks = [] + print('mask_path: {}'.format(mpath)) + start = int(mpath.split('/')[-1].split('mask_')[1].split('_')[0]) + end = int( + mpath.split('/')[-1].split('mask_')[1].split('_')[1].split('.')[0]) + m = Image.open(mpath) + m = np.array(m.convert('L')) + m = np.array(m > 0).astype(np.uint8) + m = 1 - m + for i in range(start - 1, end + 1): + masks.append(Image.fromarray(m * 255)) + return masks + + +def check_size(h, w): + is_resize = False + if h != 240: + h = 240 + is_resize = True + if w != 432: + w = 432 + is_resize = True + return is_resize + + +def get_mask_list(mask_path): + mask_names = os.listdir(mask_path) + mask_names.sort() + + abs_mask_path = [] + mask_list = [] + begin_list = [] + end_list = [] + + for mask_name in mask_names: + mask_name_tmp = mask_name.split('mask_')[1] + begin_list.append(int(mask_name_tmp.split('_')[0])) + end_list.append(int(mask_name_tmp.split('_')[1].split('.')[0])) + abs_mask_path.append(os.path.join(mask_path, mask_name)) + mask = cv2.imread(os.path.join(mask_path, mask_name)) + mask_list.append(mask) + return mask_list, begin_list, end_list, abs_mask_path + + +def inpainting_by_model_balance(model, video_inputPath, mask_path, + video_savePath, fps, w_ori, h_ori): + + video_ori = cv2.VideoCapture(video_inputPath) + + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + video_save = cv2.VideoWriter(video_savePath, fourcc, fps, (w_ori, h_ori)) + + mask_list, begin_list, end_list, abs_mask_path = get_mask_list(mask_path) + + img_npy = [] + + for index, mask in enumerate(mask_list): + + masks = read_mask_oneImage(abs_mask_path[index]) + + mask, res_pix, crop_for_oriimg, crop_for_inpimg = get_crop_mask_v1( + mask) + mask_h, mask_w = mask.shape[0:2] + is_resize = check_size(mask.shape[0], mask.shape[1]) + + begin = begin_list[index] + end = end_list[index] + print('begin: {}'.format(begin)) + print('end: {}'.format(end)) + + for i in range(begin, end + 1, MAX_frame): + begin_time = time.time() + if i + MAX_frame <= end: + video_length = MAX_frame + else: + video_length = end - i + 1 + + for frame_count in range(video_length): + _, frame = video_ori.read() + img_npy.append(frame) + frames_temp = [] + for f in img_npy: + f = Image.fromarray(f) + i_temp = f.crop( + (res_pix[2], res_pix[0], res_pix[3], res_pix[1])) + a = i_temp.resize((w, h), Image.NEAREST) + frames_temp.append(a) + feats_temp = _to_tensors(frames_temp).unsqueeze(0) * 2 - 1 + frames_temp = [np.array(f).astype(np.uint8) for f in frames_temp] + masks_temp = [] + for m in masks[i - begin:i + video_length - begin]: + + m_temp = m.crop( + (res_pix[2], res_pix[0], res_pix[3], res_pix[1])) + b = m_temp.resize((w, h), Image.NEAREST) + masks_temp.append(b) + binary_masks_temp = [ + np.expand_dims((np.array(m) != 0).astype(np.uint8), 2) + for m in masks_temp + ] + masks_temp = _to_tensors(masks_temp).unsqueeze(0) + if torch.cuda.is_available(): + feats_temp, masks_temp = feats_temp.cuda(), masks_temp.cuda() + comp_frames = [None] * video_length + model.eval() + with torch.no_grad(): + feats_out = feats_temp * (1 - masks_temp).float() + feats_out = feats_out.view(video_length, 3, h, w) + feats_out = model.model.encoder(feats_out) + _, c, feat_h, feat_w = feats_out.size() + feats_out = feats_out.view(1, video_length, c, feat_h, feat_w) + + for f in range(0, video_length, neighbor_stride): + neighbor_ids = [ + i for i in range( + max(0, f - neighbor_stride), + min(video_length, f + neighbor_stride + 1)) + ] + ref_ids = get_ref_index(neighbor_ids, video_length) + with torch.no_grad(): + pred_feat = model.model.infer( + feats_out[0, neighbor_ids + ref_ids, :, :, :], + masks_temp[0, neighbor_ids + ref_ids, :, :, :]) + pred_img = torch.tanh( + model.model.decoder( + pred_feat[:len(neighbor_ids), :, :, :])).detach() + pred_img = (pred_img + 1) / 2 + pred_img = pred_img.cpu().permute(0, 2, 3, 1).numpy() * 255 + for j in range(len(neighbor_ids)): + idx = neighbor_ids[j] + img = np.array(pred_img[j]).astype( + np.uint8) * binary_masks_temp[idx] + frames_temp[ + idx] * (1 - binary_masks_temp[idx]) + if comp_frames[idx] is None: + comp_frames[idx] = img + else: + comp_frames[idx] = comp_frames[idx].astype( + np.float32) * 0.5 + img.astype( + np.float32) * 0.5 + print('inpainting time:', time.time() - begin_time) + for f in range(video_length): + comp = np.array(comp_frames[f]).astype( + np.uint8) * binary_masks_temp[f] + frames_temp[f] * ( + 1 - binary_masks_temp[f]) + if is_resize: + comp = cv2.resize(comp, (mask_w, mask_h)) + complete_frame = img_npy[f] + a1, b1, c1, d1 = crop_for_oriimg + a2, b2, c2, d2 = crop_for_inpimg + complete_frame[b1:d1, a1:c1] = comp[b2:d2, a2:c2] + video_save.write(complete_frame) + + img_npy = [] + + video_ori.release() diff --git a/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting_model.py b/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting_model.py new file mode 100644 index 0000000000000000000000000000000000000000..168bc83e9729b6a7e292dc24b36a6d4b0b657114 --- /dev/null +++ b/runtime/Lib/site_packages/models/cv/video_inpainting/inpainting_model.py @@ -0,0 +1,381 @@ +""" VideoInpaintingProcess +The implementation here is modified based on STTN, + originally Apache 2.0 License and publicly available at https://github.com/researchmm/STTN +""" + +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.models as models + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class BaseNetwork(nn.Module): + + def __init__(self): + super(BaseNetwork, self).__init__() + + def print_network(self): + if isinstance(self, list): + self = self[0] + num_params = 0 + for param in self.parameters(): + num_params += param.numel() + print( + 'Network [%s] was created. Total number of parameters: %.1f million. ' + 'To see the architecture, do print(network).' % + (type(self).__name__, num_params / 1000000)) + + def init_weights(self, init_type='normal', gain=0.02): + ''' + initialize network's weights + init_type: normal | xavier | kaiming | orthogonal + https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/9451e70673400885567d08a9e97ade2524c700d0/models/networks.py#L39 + ''' + + def init_func(m): + classname = m.__class__.__name__ + if classname.find('InstanceNorm2d') != -1: + if hasattr(m, 'weight') and m.weight is not None: + nn.init.constant_(m.weight.data, 1.0) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias.data, 0.0) + elif hasattr(m, 'weight') and (classname.find('Conv') != -1 + or classname.find('Linear') != -1): + if init_type == 'normal': + nn.init.normal_(m.weight.data, 0.0, gain) + elif init_type == 'xavier': + nn.init.xavier_normal_(m.weight.data, gain=gain) + elif init_type == 'xavier_uniform': + nn.init.xavier_uniform_(m.weight.data, gain=1.0) + elif init_type == 'kaiming': + nn.init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + nn.init.orthogonal_(m.weight.data, gain=gain) + elif init_type == 'none': + m.reset_parameters() + else: + raise NotImplementedError( + 'initialization method [%s] is not implemented' + % init_type) + if hasattr(m, 'bias') and m.bias is not None: + nn.init.constant_(m.bias.data, 0.0) + + self.apply(init_func) + + for m in self.children(): + if hasattr(m, 'init_weights'): + m.init_weights(init_type, gain) + + +@MODELS.register_module( + Tasks.video_inpainting, module_name=Models.video_inpainting) +class VideoInpainting(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.model = InpaintGenerator() + if torch.cuda.is_available(): + device = 'cuda' + else: + device = 'cpu' + pretrained_params = torch.load( + '{}/{}'.format(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location=device) + self.model.load_state_dict(pretrained_params['netG']) + self.model.eval() + self.device_id = device_id + if self.device_id >= 0 and torch.cuda.is_available(): + self.model.to('cuda:{}'.format(self.device_id)) + logger.info('Use GPU: {}'.format(self.device_id)) + else: + self.device_id = -1 + logger.info('Use CPU for inference') + + +class InpaintGenerator(BaseNetwork): + + def __init__(self, init_weights=True): + super(InpaintGenerator, self).__init__() + channel = 256 + stack_num = 6 + patchsize = [(48, 24), (16, 8), (8, 4), (4, 2)] + blocks = [] + for _ in range(stack_num): + blocks.append(TransformerBlock(patchsize, hidden=channel)) + self.transformer = nn.Sequential(*blocks) + + self.encoder = nn.Sequential( + nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(128, channel, kernel_size=3, stride=1, padding=1), + nn.LeakyReLU(0.2, inplace=True), + ) + + self.decoder = nn.Sequential( + deconv(channel, 128, kernel_size=3, padding=1), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1), + nn.LeakyReLU(0.2, inplace=True), + deconv(64, 64, kernel_size=3, padding=1), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(64, 3, kernel_size=3, stride=1, padding=1)) + + if init_weights: + self.init_weights() + + def forward(self, masked_frames, masks): + b, t, c, h, w = masked_frames.size() + masks = masks.view(b * t, 1, h, w) + enc_feat = self.encoder(masked_frames.view(b * t, c, h, w)) + _, c, h, w = enc_feat.size() + masks = F.interpolate(masks, scale_factor=1.0 / 4) + enc_feat = self.transformer({ + 'x': enc_feat, + 'm': masks, + 'b': b, + 'c': c + })['x'] + output = self.decoder(enc_feat) + output = torch.tanh(output) + return output + + def infer(self, feat, masks): + t, c, h, w = masks.size() + masks = masks.view(t, c, h, w) + masks = F.interpolate(masks, scale_factor=1.0 / 4) + t, c, _, _ = feat.size() + enc_feat = self.transformer({ + 'x': feat, + 'm': masks, + 'b': 1, + 'c': c + })['x'] + return enc_feat + + +class deconv(nn.Module): + + def __init__(self, + input_channel, + output_channel, + kernel_size=3, + padding=0): + super().__init__() + self.conv = nn.Conv2d( + input_channel, + output_channel, + kernel_size=kernel_size, + stride=1, + padding=padding) + + def forward(self, x): + x = F.interpolate( + x, scale_factor=2, mode='bilinear', align_corners=True) + x = self.conv(x) + return x + + +class Attention(nn.Module): + """ + Compute 'Scaled Dot Product Attention + """ + + def forward(self, query, key, value, m): + scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt( + query.size(-1)) + scores.masked_fill(m, -1e9) + p_attn = F.softmax(scores, dim=-1) + p_val = torch.matmul(p_attn, value) + return p_val, p_attn + + +class MultiHeadedAttention(nn.Module): + """ + Take in model size and number of heads. + """ + + def __init__(self, patchsize, d_model): + super().__init__() + self.patchsize = patchsize + self.query_embedding = nn.Conv2d( + d_model, d_model, kernel_size=1, padding=0) + self.value_embedding = nn.Conv2d( + d_model, d_model, kernel_size=1, padding=0) + self.key_embedding = nn.Conv2d( + d_model, d_model, kernel_size=1, padding=0) + self.output_linear = nn.Sequential( + nn.Conv2d(d_model, d_model, kernel_size=3, padding=1), + nn.LeakyReLU(0.2, inplace=True)) + self.attention = Attention() + + def forward(self, x, m, b, c): + bt, _, h, w = x.size() + t = bt // b + d_k = c // len(self.patchsize) + output = [] + _query = self.query_embedding(x) + _key = self.key_embedding(x) + _value = self.value_embedding(x) + for (width, height), query, key, value in zip( + self.patchsize, + torch.chunk(_query, len(self.patchsize), dim=1), + torch.chunk(_key, len(self.patchsize), dim=1), + torch.chunk(_value, len(self.patchsize), dim=1)): + out_w, out_h = w // width, h // height + mm = m.view(b, t, 1, out_h, height, out_w, width) + mm = mm.permute(0, 1, 3, 5, 2, 4, + 6).contiguous().view(b, t * out_h * out_w, + height * width) + mm = (mm.mean(-1) > 0.5).unsqueeze(1).repeat( + 1, t * out_h * out_w, 1) + query = query.view(b, t, d_k, out_h, height, out_w, width) + query = query.permute(0, 1, 3, 5, 2, 4, + 6).contiguous().view(b, t * out_h * out_w, + d_k * height * width) + key = key.view(b, t, d_k, out_h, height, out_w, width) + key = key.permute(0, 1, 3, 5, 2, 4, + 6).contiguous().view(b, t * out_h * out_w, + d_k * height * width) + value = value.view(b, t, d_k, out_h, height, out_w, width) + value = value.permute(0, 1, 3, 5, 2, 4, + 6).contiguous().view(b, t * out_h * out_w, + d_k * height * width) + y, _ = self.attention(query, key, value, mm) + y = y.view(b, t, out_h, out_w, d_k, height, width) + y = y.permute(0, 1, 4, 2, 5, 3, 6).contiguous().view(bt, d_k, h, w) + output.append(y) + output = torch.cat(output, 1) + x = self.output_linear(output) + return x + + +class FeedForward(nn.Module): + + def __init__(self, d_model): + super(FeedForward, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(d_model, d_model, kernel_size=3, padding=2, dilation=2), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv2d(d_model, d_model, kernel_size=3, padding=1), + nn.LeakyReLU(0.2, inplace=True)) + + def forward(self, x): + x = self.conv(x) + return x + + +class TransformerBlock(nn.Module): + """ + Transformer = MultiHead_Attention + Feed_Forward with sublayer connection + """ + + def __init__(self, patchsize, hidden=128): # hidden=128 + super().__init__() + self.attention = MultiHeadedAttention(patchsize, d_model=hidden) + self.feed_forward = FeedForward(hidden) + + def forward(self, x): + x, m, b, c = x['x'], x['m'], x['b'], x['c'] + x = x + self.attention(x, m, b, c) + x = x + self.feed_forward(x) + return {'x': x, 'm': m, 'b': b, 'c': c} + + +class Discriminator(BaseNetwork): + + def __init__(self, + in_channels=3, + use_sigmoid=False, + use_spectral_norm=True, + init_weights=True): + super(Discriminator, self).__init__() + self.use_sigmoid = use_sigmoid + nf = 64 + + self.conv = nn.Sequential( + spectral_norm( + nn.Conv3d( + in_channels=in_channels, + out_channels=nf * 1, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=1, + bias=not use_spectral_norm), use_spectral_norm), + nn.LeakyReLU(0.2, inplace=True), + spectral_norm( + nn.Conv3d( + nf * 1, + nf * 2, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=(1, 2, 2), + bias=not use_spectral_norm), use_spectral_norm), + nn.LeakyReLU(0.2, inplace=True), + spectral_norm( + nn.Conv3d( + nf * 2, + nf * 4, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=(1, 2, 2), + bias=not use_spectral_norm), use_spectral_norm), + nn.LeakyReLU(0.2, inplace=True), + spectral_norm( + nn.Conv3d( + nf * 4, + nf * 4, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=(1, 2, 2), + bias=not use_spectral_norm), use_spectral_norm), + nn.LeakyReLU(0.2, inplace=True), + spectral_norm( + nn.Conv3d( + nf * 4, + nf * 4, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=(1, 2, 2), + bias=not use_spectral_norm), use_spectral_norm), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv3d( + nf * 4, + nf * 4, + kernel_size=(3, 5, 5), + stride=(1, 2, 2), + padding=(1, 2, 2))) + + if init_weights: + self.init_weights() + + def forward(self, xs): + xs_t = torch.transpose(xs, 0, 1) + xs_t = xs_t.unsqueeze(0) + feat = self.conv(xs_t) + if self.use_sigmoid: + feat = torch.sigmoid(feat) + out = torch.transpose(feat, 1, 2) + return out + + +def spectral_norm(module, mode=True): + if mode: + return _spectral_norm(module) + return module diff --git a/runtime/Lib/site_packages/models/multi_modal/__init__.py b/runtime/Lib/site_packages/models/multi_modal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9da84bef2304ddef34a0a5b0548c2a9503db51d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/__init__.py @@ -0,0 +1,59 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .clip import CLIPForMultiModalEmbedding + from .clip_interrogator import CLIP_Interrogator + from .diffusion import DiffusionForTextToImageSynthesis + from .efficient_diffusion_tuning import EfficientStableDiffusion + from .gemm import GEMMForMultiModalEmbedding + from .mmr import VideoCLIPForMultiModalEmbedding + from .mplug_for_all_tasks import HiTeAForAllTasks, MPlugForAllTasks + from .mplug_owl import MplugOwlForConditionalGeneration + from .multi_stage_diffusion import \ + MultiStageDiffusionForTextToImageSynthesis + from .ofa_for_all_tasks import OfaForAllTasks + from .ofa_for_text_to_image_synthesis_model import \ + OfaForTextToImageSynthesis + from .prost import ProSTForTVRetrieval + from .rleg import RLEGForMultiModalEmbedding + from .team import TEAMForMultiModalSimilarity + from .video_synthesis import TextToVideoSynthesis + from .vldoc import VLDocForDocVLEmbedding + from .videocomposer import VideoComposer + +else: + _import_structure = { + 'clip': ['CLIPForMultiModalEmbedding'], + 'diffusion': ['DiffusionForTextToImageSynthesis'], + 'gemm': ['GEMMForMultiModalEmbedding'], + 'rleg': ['RLEGForMultiModalEmbedding'], + 'team': ['TEAMForMultiModalSimilarity'], + 'mmr': ['VideoCLIPForMultiModalEmbedding'], + 'prost': ['ProSTForTVRetrieval'], + 'mplug_for_all_tasks': ['MPlugForAllTasks', 'HiTeAForAllTasks'], + 'ofa_for_all_tasks': ['OfaForAllTasks'], + 'ofa_for_text_to_image_synthesis_model': + ['OfaForTextToImageSynthesis'], + 'multi_stage_diffusion': + ['MultiStageDiffusionForTextToImageSynthesis'], + 'vldoc': ['VLDocForDocVLEmbedding'], + 'video_synthesis': ['TextToVideoSynthesis'], + 'efficient_diffusion_tuning': ['EfficientStableDiffusion'], + 'mplug_owl': ['MplugOwlForConditionalGeneration'], + 'clip_interrogator': ['CLIP_Interrogator'], + 'videocomposer': ['VideoComposer'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/clip/__init__.py b/runtime/Lib/site_packages/models/multi_modal/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e2e925ce6e1be03402bff9ab7f6fc6a00f8078f6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .model import CLIPForMultiModalEmbedding diff --git a/runtime/Lib/site_packages/models/multi_modal/clip/bert_tokenizer.py b/runtime/Lib/site_packages/models/multi_modal/clip/bert_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1ee715c91e14e5e44d8e8c77c41b72afa690fa08 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip/bert_tokenizer.py @@ -0,0 +1,422 @@ +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import, division, print_function +import collections +import os +import re +import unicodedata + +import six + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match('^.*?([A-Za-z0-9_-]+)/bert_model.ckpt', init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + 'uncased_L-24_H-1024_A-16', 'uncased_L-12_H-768_A-12', + 'multilingual_L-12_H-768_A-12', 'chinese_L-12_H-768_A-12' + ] + + cased_models = [ + 'cased_L-12_H-768_A-12', 'cased_L-24_H-1024_A-16', + 'multi_cased_L-12_H-768_A-12' + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = 'False' + case_name = 'lowercased' + opposite_flag = 'True' + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = 'True' + case_name = 'cased' + opposite_flag = 'False' + + if is_bad_config: + raise ValueError( + 'You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. ' + 'However, `%s` seems to be a %s model, so you ' + 'should pass in `--do_lower_case=%s` so that the fine-tuning matches ' + 'how the model was pre-training. If this error is wrong, please ' + 'just comment out this check.' % + (actual_flag, init_checkpoint, model_name, case_name, + opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode('utf-8', 'ignore') + elif isinstance(text, unicode): + return text + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + else: + raise ValueError('Not running on Python2 or Python 3?') + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode('utf-8') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + else: + raise ValueError('Not running on Python2 or Python 3?') + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, 'r', encoding='utf-8') as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + @staticmethod + def convert_tokens_to_string(tokens, clean_up_tokenization_spaces=True): + """ Converts a sequence of tokens (string) in a single string. """ + + def clean_up_tokenization(out_string): + """ Clean up a list of simple English tokenization artifacts + like spaces before punctuations and abreviated forms. + """ + out_string = ( + out_string.replace(' .', '.').replace(' ?', '?').replace( + ' !', '!').replace(' ,', ',').replace(" ' ", "'").replace( + " n't", "n't").replace(" 'm", "'m").replace( + " 's", "'s").replace(" 've", + "'ve").replace(" 're", "'re")) + return out_string + + text = ' '.join(tokens).replace(' ##', '').strip() + if clean_up_tokenization_spaces: + clean_text = clean_up_tokenization(text) + return clean_text + else: + return text + + def vocab_size(self): + return len(self.vocab) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(' '.join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize('NFD', text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == 'Mn': + continue + output.append(char) + return ''.join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return [''.join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(' ') + output.append(char) + output.append(' ') + else: + output.append(char) + return ''.join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F)): + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(' ') + else: + output.append(char) + return ''.join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token='[UNK]', max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = ''.join(chars[start:end]) + if start > 0: + substr = '##' + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == ' ' or char == '\t' or char == '\n' or char == '\r': + return True + cat = unicodedata.category(char) + if cat == 'Zs': + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == '\t' or char == '\n' or char == '\r': + return False + cat = unicodedata.category(char) + if cat in ('Cc', 'Cf'): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith('P'): + return True + return False diff --git a/runtime/Lib/site_packages/models/multi_modal/clip/configuration_bert.py b/runtime/Lib/site_packages/models/multi_modal/clip/configuration_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..b75f5db89104ab4d1f807ccef3ae45cb5c3bb273 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip/configuration_bert.py @@ -0,0 +1,82 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERT model configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import logging + +logger = logging.getLogger(__name__) + + +class BertConfig(object): + r""" + :class:`~transformers.BertConfig` is the configuration class to store the configuration of a + `BertModel`. + + + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + """ + + def __init__(self, + vocab_size_or_config_json_file=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + output_attentions=False, + output_hidden_states=False): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.output_attentions = output_attentions + self.output_hidden_states = output_hidden_states diff --git a/runtime/Lib/site_packages/models/multi_modal/clip/model.py b/runtime/Lib/site_packages/models/multi_modal/clip/model.py new file mode 100644 index 0000000000000000000000000000000000000000..03e95ea8d922c5ce44849ee62a109897ef3e3689 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip/model.py @@ -0,0 +1,608 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from collections import OrderedDict +from typing import Any, Dict, Tuple, Union + +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.clip.bert_tokenizer import FullTokenizer +from modelscope.models.multi_modal.clip.configuration_bert import BertConfig +from modelscope.models.multi_modal.clip.modeling_bert import BertModel +from modelscope.utils.constant import ModeKeys, ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['CLIPForMultiModalEmbedding'] + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class VisualTransformer(nn.Module): + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + x = torch.cat( + [ # noqa + self.class_embedding.to(x.dtype) + torch.zeros( # noqa + x.shape[0], + 1, + x.shape[-1], + dtype=x.dtype, + device=x.device), + x # noqa + ], + dim=1) # noqa shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + vocab_size: int, + text_attention_probs_dropout_prob: float, + text_hidden_act: str, + text_hidden_dropout_prob: float, + text_hidden_size: int, + text_initializer_range: float, + text_intermediate_size: int, + text_max_position_embeddings: int, + text_num_attention_heads: int, + text_num_hidden_layers: int, + text_type_vocab_size: int, + tokenizer: FullTokenizer, + # vision_head_width, added this param for ViT-H + vision_head_width: int = 64, + ): + super().__init__() + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // vision_head_width + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // vision_head_width + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim) + + self.bert_config = BertConfig( + vocab_size_or_config_json_file=vocab_size, + hidden_size=text_hidden_size, + num_hidden_layers=text_num_hidden_layers, + num_attention_heads=text_num_attention_heads, + intermediate_size=text_intermediate_size, + hidden_act=text_hidden_act, + hidden_dropout_prob=text_hidden_dropout_prob, + attention_probs_dropout_prob=text_attention_probs_dropout_prob, + max_position_embeddings=text_max_position_embeddings, + type_vocab_size=text_type_vocab_size, + initializer_range=text_initializer_range, + layer_norm_eps=1e-12, + ) + self.bert = BertModel(self.bert_config) + + self.text_projection = nn.Parameter( + torch.empty(text_hidden_size, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + self.tokenizer = tokenizer + + self.initialize_parameters() + + def initialize_parameters(self): + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.bert_config.hidden_size**-0.5) + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image): + return self.visual(image.type(self.dtype)) + + def encode_text(self, text): + pad_index = self.tokenizer.vocab['[PAD]'] + attn_mask = text.ne(pad_index).type(self.dtype) + x = self.bert( + text, attention_mask=attn_mask)[0].type( + self.dtype) # [batch_size, seq_length, hidden_size] + return x[:, 0, :] @ self.text_projection + + def forward(self, image, text): + assert image is not None or text is not None, 'text and image cannot both be None!' + + if image is None: + return self.encode_text(text) + elif text is None: + return self.encode_image(image) + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + return image_features, text_features, self.logit_scale.exp() + + def get_similarity(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=1, keepdim=True) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def convert_models_to_fp32(model): + for p in model.parameters(): + p.data = p.data.float() + if p.grad: + p.grad.data = p.grad.data.float() + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(module): + if isinstance(module, (nn.Conv1d, nn.Conv2d, nn.Linear)): + module.weight.data = module.weight.data.half() + if module.bias is not None: + module.bias.data = module.bias.data.half() + + if isinstance(module, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(module, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + if isinstance(module, BertModel): + module.to(torch.half) + + for name in ['text_projection', 'proj']: + if hasattr(module, name): + attr = getattr(module, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +@MODELS.register_module(Tasks.multi_modal_embedding, module_name=Models.clip) +class CLIPForMultiModalEmbedding(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir=model_dir, *args, **kwargs) + + # Initialize the model. + vision_model_config_file = '{}/vision_model_config.json'.format( + model_dir) + logger.info( + f'Loading vision model config from {vision_model_config_file}') + assert os.path.exists(vision_model_config_file) + + text_model_config_file = '{}/text_model_config.json'.format(model_dir) + logger.info(f'Loading text model config from {text_model_config_file}') + assert os.path.exists(text_model_config_file) + + with open( + vision_model_config_file, 'r', + encoding='utf-8') as fv,\ + open(text_model_config_file, 'r', encoding='utf-8') as ft: + self.model_info = json.load(fv) + for k, v in json.load(ft).items(): + self.model_info[k] = v + + vocab_file = f'{model_dir}/{ModelFile.VOCAB_FILE}' + self.tokenizer = FullTokenizer(vocab_file=vocab_file) + + # initialize the model + self.clip_model = CLIP(**self.model_info, tokenizer=self.tokenizer) + convert_weights(self.clip_model) + + # restore the pretrained weight + checkpoint = torch.load( + f'{model_dir}/{ModelFile.TORCH_MODEL_BIN_FILE}', 'cpu') + sd = checkpoint[ + 'state_dict'] if 'state_dict' in checkpoint else checkpoint + if next(iter(sd.items()))[0].startswith('module'): + sd = {k[len('module.'):]: v for k, v in sd.items()} + # support the finetuned model + if next(iter(sd.items()))[0].startswith('clip_model'): + sd = {k[len('clip_model.'):]: v for k, v in sd.items()} + self.clip_model.load_state_dict(sd) + self.clip_model.eval() + + # place the model + self.device = 'cuda:{}'.format(int(os.environ.get( + 'LOCAL_RANK', 0))) if torch.cuda.is_available() else 'cpu' + if torch.cuda.is_available(): + self.clip_model.to(self.device) + logger.info('Use GPU {} for finetuning & inference'.format( + int(os.environ.get('LOCAL_RANK', 0)))) + else: + self.clip_model.float() + logger.info('Use CPU for finetuning & inference') + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + from modelscope.outputs import OutputKeys + output = { + OutputKeys.IMG_EMBEDDING: None, + OutputKeys.TEXT_EMBEDDING: None + } + mode = input.get('mode', ModeKeys.INFERENCE) + + # encode the image + if 'img' in input and isinstance(input['img'], torch.Tensor): + image_tensor = input['img'].to(self.device) + if image_tensor.dim() == 5 and image_tensor.shape[1] == 1: + image_tensor = image_tensor.squeeze(1) + + with torch.autograd.set_grad_enabled(mode == ModeKeys.TRAIN): + image_features = self.clip_model.encode_image(image_tensor) + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) # l2-normalize + + output[OutputKeys.IMG_EMBEDDING] = image_features + + if 'text' in input and isinstance(input['text'], torch.Tensor): + text_tensor = input['text'].to(self.device) + if text_tensor.dim() == 3 and text_tensor.shape[1] == 1: + text_tensor = text_tensor.squeeze(1) + + with torch.autograd.set_grad_enabled(mode == ModeKeys.TRAIN): + text_features = self.clip_model.encode_text(text_tensor) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) # l2-normalize + output[OutputKeys.TEXT_EMBEDDING] = text_features + + if mode == ModeKeys.TRAIN: + output['logit_scale'] = (self.clip_model.logit_scale + * 1.0).exp().mean() + + return output + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs + + @property + def temperature(self): + return 1.0 / self.clip_model.logit_scale.exp() diff --git a/runtime/Lib/site_packages/models/multi_modal/clip/modeling_bert.py b/runtime/Lib/site_packages/models/multi_modal/clip/modeling_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..11c5c833811e467fca4df76bebe24e49847c76a1 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip/modeling_bert.py @@ -0,0 +1,506 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import logging +import math +import os +import sys +from io import open + +import json +import torch +from torch import nn + +from .configuration_bert import BertConfig + +logger = logging.getLogger(__name__) + + +def gelu(x): + """ Original Implementation of the gelu activation function in Google Bert repo when initially created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh( + math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = { + 'gelu': gelu, + 'relu': torch.nn.functional.relu, + 'swish': swish, + 'gelu_new': gelu_new +} + +BertLayerNorm = torch.nn.LayerNorm + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(BertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, config.hidden_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config): + super(BertSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if self.output_attentions else ( + context_layer, ) + return outputs + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super(BertSelfOutput, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config): + super(BertAttention, self).__init__() + self.self = BertSelfAttention(config) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def forward(self, input_tensor, attention_mask=None, head_mask=None): + self_outputs = self.self(input_tensor, attention_mask, head_mask) + attention_output = self.output(self_outputs[0], input_tensor) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super(BertIntermediate, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, + str) or (sys.version_info[0] == 2 + and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super(BertOutput, self).__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config): + super(BertLayer, self).__init__() + self.attention = BertAttention(config) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_outputs = self.attention(hidden_states, attention_mask, + head_mask) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + outputs = (layer_output, ) + attention_outputs[ + 1:] # add attentions if we output them + return outputs + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super(BertEncoder, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.layer = nn.ModuleList( + [BertLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + all_hidden_states = () + all_attentions = () + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_outputs = layer_module(hidden_states, attention_mask, + head_mask[i]) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1], ) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + outputs = (hidden_states, ) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states, ) + if self.output_attentions: + outputs = outputs + (all_attentions, ) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +class BertPooler(nn.Module): + + def __init__(self, config): + super(BertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super(BertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, + str) or (sys.version_info[0] == 2 + and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config): + super(BertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super(BertOnlyMLMHead, self).__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super(BertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config): + super(BertPreTrainingHeads, self).__init__() + self.predictions = BertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPreTrainedModel(nn.Module): + config_class = BertConfig + base_model_prefix = 'bert' + + def __init__(self, config): + super(BertPreTrainedModel, self).__init__() + self.config = config + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +class BertModel(BertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) + of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, + used to compute the weighted average in the self-attention heads. + + Examples: + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = BertModel.from_pretrained('bert-base-uncased') + >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + >>> outputs = model(input_ids) + >>> last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config): + super(BertModel, self).__init__(config) + + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder(config) + self.pooler = BertPooler(config) + + self.apply(self._init_weights) + + def forward(self, + input_ids, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze( + -1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, + -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze( + -1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters( + )).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings( + input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids) + encoder_outputs = self.encoder( + embedding_output, extended_attention_mask, head_mask=head_mask) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = ( + sequence_output, + pooled_output, + ) + encoder_outputs[ + 1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) diff --git a/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/__init__.py b/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..96fefbf6e9b21c2e16727f926f7a12977d66ffe8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/__init__.py @@ -0,0 +1 @@ +from .model import CLIP_Interrogator diff --git a/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/model.py b/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/model.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e27cbd0c5c66995b39c8e4ad5112eed1869dbb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/clip_interrogator/model.py @@ -0,0 +1,599 @@ +# This implementation is adopted from CLIP-Interrogator, made pubicly available under the MIT License at +# https://github.com/pharmapsychotic/clip-interrogator/blob/main/clip_interrogator/clip_interrogator.py + +import hashlib +import math +import os +import time +from dataclasses import dataclass +from typing import List, Optional + +import numpy as np +import open_clip +import requests +import torch +import torchvision.transforms as transforms +from PIL import Image +from safetensors.numpy import load_file, save_file +from tqdm import tqdm +from transformers import (AutoModelForCausalLM, AutoProcessor, + Blip2ForConditionalGeneration, + BlipForConditionalGeneration) + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['CLIP_Interrogator'] + +CAPTION_MODELS = { + 'blip-base': 'blip-image-captioning-base', + 'blip-large': 'blip-image-captioning-large', + 'blip2-2.7b': 'blip2-opt-2.7b', + 'blip2-flan-t5-xl': 'blip2-flan-t5-xl', + 'git-large-coco': 'git-large-coco', +} + + +@dataclass +class Config: + # models can optionally be passed in directly + caption_model = None + caption_processor = None + clip_model = None + clip_preprocess = None + + # blip settings + caption_max_length: int = 32 + caption_model_name: Optional[ + str] = 'blip-large' # use a key from CAPTION_MODELS or None + caption_offload: bool = False + + # clip settings + clip_model_name: str = 'ViT-L-14/openai' + clip_model_path: Optional[str] = None + clip_offload: bool = False + + # interrogator settings + cache_path: str = 'cache' # path to store cached text embeddings + download_cache: bool = False # when true, cached embeds are downloaded from huggingface + chunk_size: int = 2048 # batch size for CLIP, use smaller for lower VRAM + data_path: str = os.path.join(os.path.dirname(__file__), 'data') + device: str = ('cuda' if torch.cuda.is_available() else 'cpu') + flavor_intermediate_count: int = 2048 + quiet: bool = False # when quiet progress bars are not shown + + def apply_low_vram_defaults(self): + self.caption_model_name = 'blip-base' + self.caption_offload = True + self.clip_offload = True + self.chunk_size = 1024 + self.flavor_intermediate_count = 1024 + + +# CLIP-Interrogator utilize CLIP and BLIP to generate rich caption for images. +# CLIP is a zero-shot image classifier which can be used to generate image and text embeddings. +# BLIP is a new VLP framework which transfers flexibly to both vision-language understanding and generation tasks. +# BLIP effectively utilizes the noisy web data by bootstrapping the captions, where +# a captioner generates synthetic captions and a filter removes the noisy ones. +# Please infer to the paper CLIP: Learning Transferable Visual Models From Natural Language Supervision +# https://arxiv.org/abs/2103.00020 +# BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation +# https://arxiv.org/abs/2201.12086 + + +class Interrogator(): + + def __init__(self, config: Config): + self.config = config + self.device = config.device + self.dtype = torch.float16 if self.device == 'cuda' else torch.float32 + self.caption_offloaded = True + self.clip_offloaded = True + self.load_caption_model() + self.load_clip_model() + + def load_caption_model(self): + if self.config.caption_model is None and self.config.caption_model_name: + if not self.config.quiet: + print( + f'Loading caption model {self.config.caption_model_name}...' + ) + + model_path = CAPTION_MODELS[self.config.caption_model_name] + if self.config.caption_model_name.startswith('git-'): + caption_model = AutoModelForCausalLM.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=torch.float32) + elif self.config.caption_model_name.startswith('blip2-'): + caption_model = Blip2ForConditionalGeneration.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=self.dtype) + else: + caption_model = BlipForConditionalGeneration.from_pretrained( + os.path.join(self.config.cache_path, model_path), + torch_dtype=self.dtype) + self.caption_processor = AutoProcessor.from_pretrained( + os.path.join(self.config.cache_path, model_path)) + + caption_model.eval() + if not self.config.caption_offload: + caption_model = caption_model.to(self.config.device) + self.caption_model = caption_model + else: + self.caption_model = self.config.caption_model + self.caption_processor = self.config.caption_processor + + def load_clip_model(self): + start_time = time.time() + config = self.config + + clip_model_name, clip_model_pretrained_name = config.clip_model_name.split( + '/', 2) + + if config.clip_model is None: + if not config.quiet: + print(f'Loading CLIP model {config.clip_model_name}...') + + self.clip_model, _, self.clip_preprocess = open_clip.create_model_and_transforms( + clip_model_name, + pretrained=clip_model_pretrained_name, + precision='fp16' if config.device == 'cuda' else 'fp32', + device=config.device, + jit=False, + cache_dir=config.clip_model_path) + self.clip_model.eval() + else: + self.clip_model = config.clip_model + self.clip_preprocess = config.clip_preprocess + self.tokenize = open_clip.get_tokenizer(clip_model_name) + + sites = [ + 'Artstation', 'behance', 'cg society', 'cgsociety', 'deviantart', + 'dribbble', 'flickr', 'instagram', 'pexels', 'pinterest', + 'pixabay', 'pixiv', 'polycount', 'reddit', 'shutterstock', + 'tumblr', 'unsplash', 'zbrush central' + ] + trending_list = [site for site in sites] + trending_list.extend(['trending on ' + site for site in sites]) + trending_list.extend(['featured on ' + site for site in sites]) + trending_list.extend([site + ' contest winner' for site in sites]) + + raw_artists = load_list(config.data_path, 'artists.txt') + artists = [f'by {a}' for a in raw_artists] + artists.extend([f'inspired by {a}' for a in raw_artists]) + + self._prepare_clip() + self.artists = LabelTable(artists, 'artists', self) + self.flavors = LabelTable( + load_list(config.data_path, 'flavors.txt'), 'flavors', self) + self.mediums = LabelTable( + load_list(config.data_path, 'mediums.txt'), 'mediums', self) + self.movements = LabelTable( + load_list(config.data_path, 'movements.txt'), 'movements', self) + self.trendings = LabelTable(trending_list, 'trendings', self) + self.negative = LabelTable( + load_list(config.data_path, 'negative.txt'), 'negative', self) + + end_time = time.time() + if not config.quiet: + print( + f'Loaded CLIP model and data in {end_time-start_time:.2f} seconds.' + ) + + def chain(self, + image_features: torch.Tensor, + phrases: List[str], + best_prompt: str = '', + best_sim: float = 0, + min_count: int = 8, + max_count: int = 32, + desc='Chaining', + reverse: bool = False) -> str: + self._prepare_clip() + + phrases = set(phrases) + if not best_prompt: + best_prompt = self.rank_top( + image_features, [f for f in phrases], reverse=reverse) + best_sim = self.similarity(image_features, best_prompt) + phrases.remove(best_prompt) + curr_prompt, curr_sim = best_prompt, best_sim + + def check(addition: str, idx: int) -> bool: + nonlocal best_prompt, best_sim, curr_prompt, curr_sim + prompt = curr_prompt + ', ' + addition + sim = self.similarity(image_features, prompt) + if reverse: + sim = -sim + + if sim > best_sim: + best_prompt, best_sim = prompt, sim + if sim > curr_sim or idx < min_count: + curr_prompt, curr_sim = prompt, sim + return True + return False + + for idx in tqdm( + range(max_count), desc=desc, disable=self.config.quiet): + best = self.rank_top( + image_features, [f'{curr_prompt}, {f}' for f in phrases], + reverse=reverse) + flave = best[len(curr_prompt) + 2:] + if not check(flave, idx): + break + if _prompt_at_max_len(curr_prompt, self.tokenize): + break + phrases.remove(flave) + + return best_prompt + + def generate_caption(self, pil_image: Image) -> str: + assert self.caption_model is not None, 'No caption model loaded.' + self._prepare_caption() + inputs = self.caption_processor( + images=pil_image, return_tensors='pt').to(self.device) + if not self.config.caption_model_name.startswith('git-'): + inputs = inputs.to(self.dtype) + tokens = self.caption_model.generate( + **inputs, max_new_tokens=self.config.caption_max_length) + return self.caption_processor.batch_decode( + tokens, skip_special_tokens=True)[0].strip() + + def image_to_features(self, image: Image) -> torch.Tensor: + self._prepare_clip() + images = self.clip_preprocess(image).unsqueeze(0).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + image_features = self.clip_model.encode_image(images) + image_features /= image_features.norm(dim=-1, keepdim=True) + return image_features + + def interrogate_classic(self, + image: Image, + max_flavors: int = 3, + caption: Optional[str] = None) -> str: + """Classic mode creates a prompt in a standard format first describing the image, + then listing the artist, trending, movement, and flavor text modifiers.""" + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + + medium = self.mediums.rank(image_features, 1)[0] + artist = self.artists.rank(image_features, 1)[0] + trending = self.trendings.rank(image_features, 1)[0] + movement = self.movements.rank(image_features, 1)[0] + flaves = ', '.join(self.flavors.rank(image_features, max_flavors)) + + if caption.startswith(medium): + prompt = f'{caption} {artist}, {trending}, {movement}, {flaves}' + else: + prompt = f'{caption}, {medium} {artist}, {trending}, {movement}, {flaves}' + + return _truncate_to_fit(prompt, self.tokenize) + + def interrogate_fast(self, + image: Image, + max_flavors: int = 32, + caption: Optional[str] = None) -> str: + """Fast mode simply adds the top ranked terms after a caption. It generally results in + better similarity between generated prompt and image than classic mode, but the prompts + are less readable.""" + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + merged = _merge_tables([ + self.artists, self.flavors, self.mediums, self.movements, + self.trendings + ], self) + tops = merged.rank(image_features, max_flavors) + return _truncate_to_fit(caption + ', ' + ', '.join(tops), + self.tokenize) + + def interrogate_negative(self, image: Image, max_flavors: int = 32) -> str: + """Negative mode chains together the most dissimilar terms to the image. It can be used + to help build a negative prompt to pair with the regular positive prompt and often + improve the results of generated images particularly with Stable Diffusion 2.""" + image_features = self.image_to_features(image) + flaves = self.flavors.rank( + image_features, + self.config.flavor_intermediate_count, + reverse=True) + flaves = flaves + self.negative.labels + return self.chain( + image_features, + flaves, + max_count=max_flavors, + reverse=True, + desc='Negative chain') + + def interrogate(self, + image: Image, + min_flavors: int = 8, + max_flavors: int = 32, + caption: Optional[str] = None) -> str: + caption = caption or self.generate_caption(image) + image_features = self.image_to_features(image) + + merged = _merge_tables([ + self.artists, self.flavors, self.mediums, self.movements, + self.trendings + ], self) + flaves = merged.rank(image_features, + self.config.flavor_intermediate_count) + best_prompt, best_sim = caption, self.similarity( + image_features, caption) + best_prompt = self.chain( + image_features, + flaves, + best_prompt, + best_sim, + min_count=min_flavors, + max_count=max_flavors, + desc='Flavor chain') + + fast_prompt = self.interrogate_fast( + image, max_flavors, caption=caption) + classic_prompt = self.interrogate_classic( + image, max_flavors, caption=caption) + candidates = [caption, classic_prompt, fast_prompt, best_prompt] + return candidates[np.argmax( + self.similarities(image_features, candidates))] + + def rank_top(self, + image_features: torch.Tensor, + text_array: List[str], + reverse: bool = False) -> str: + self._prepare_clip() + text_tokens = self.tokenize([text + for text in text_array]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + if reverse: + similarity = -similarity + return text_array[similarity.argmax().item()] + + def similarity(self, image_features: torch.Tensor, text: str) -> float: + self._prepare_clip() + text_tokens = self.tokenize([text]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + return similarity[0][0].item() + + def similarities(self, image_features: torch.Tensor, + text_array: List[str]) -> List[float]: + self._prepare_clip() + text_tokens = self.tokenize([text + for text in text_array]).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = self.clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + similarity = text_features @ image_features.T + return similarity.T[0].tolist() + + def _prepare_caption(self): + if self.config.clip_offload and not self.clip_offloaded: + self.clip_model = self.clip_model.to('cpu') + self.clip_offloaded = True + if self.caption_offloaded: + self.caption_model = self.caption_model.to(self.device) + self.caption_offloaded = False + + def _prepare_clip(self): + if self.config.caption_offload and not self.caption_offloaded: + self.caption_model = self.caption_model.to('cpu') + self.caption_offloaded = True + if self.clip_offloaded: + self.clip_model = self.clip_model.to(self.device) + self.clip_offloaded = False + + +class LabelTable(): + + def __init__(self, labels: List[str], desc: str, ci: Interrogator): + clip_model, config = ci.clip_model, ci.config + self.chunk_size = config.chunk_size + self.config = config + self.device = config.device + self.embeds = [] + self.labels = labels + self.tokenize = ci.tokenize + + hash = hashlib.sha256(','.join(labels).encode()).hexdigest() + sanitized_name = self.config.clip_model_name.replace('/', '_').replace( + '@', '_') + self._load_cached(desc, hash, sanitized_name) + + if len(self.labels) != len(self.embeds): + self.embeds = [] + chunks = np.array_split( + self.labels, max(1, + len(self.labels) / config.chunk_size)) + for chunk in tqdm( + chunks, + desc=f'Preprocessing {desc}' if desc else None, + disable=self.config.quiet): + text_tokens = self.tokenize(chunk).to(self.device) + with torch.no_grad(), torch.cuda.amp.autocast(): + text_features = clip_model.encode_text(text_tokens) + text_features /= text_features.norm(dim=-1, keepdim=True) + text_features = text_features.half().cpu().numpy() + for i in range(text_features.shape[0]): + self.embeds.append(text_features[i]) + + if desc and self.config.cache_path: + os.makedirs(self.config.cache_path, exist_ok=True) + cache_filepath = os.path.join( + self.config.cache_path, + f'{sanitized_name}_{desc}.safetensors') + tensors = { + 'embeds': np.stack(self.embeds), + 'hash': np.array([ord(c) for c in hash], dtype=np.int8) + } + save_file(tensors, cache_filepath) + + if self.device == 'cpu' or self.device == torch.device('cpu'): + self.embeds = [e.astype(np.float32) for e in self.embeds] + + def _load_cached(self, desc: str, hash: str, sanitized_name: str) -> bool: + if self.config.cache_path is None or desc is None: + return False + + cached_safetensors = os.path.join( + self.config.cache_path, f'{sanitized_name}_{desc}.safetensors') + + if os.path.exists(cached_safetensors): + try: + tensors = load_file(cached_safetensors) + except Exception as e: + print(f'Failed to load {cached_safetensors}') + print(e) + return False + if 'hash' in tensors and 'embeds' in tensors: + if np.array_equal( + tensors['hash'], + np.array([ord(c) for c in hash], dtype=np.int8)): + self.embeds = tensors['embeds'] + if len(self.embeds.shape) == 2: + self.embeds = [ + self.embeds[i] for i in range(self.embeds.shape[0]) + ] + return True + + return False + + def _rank(self, + image_features: torch.Tensor, + text_embeds: torch.Tensor, + top_count: int = 1, + reverse: bool = False) -> str: + top_count = min(top_count, len(text_embeds)) + text_embeds = torch.stack([torch.from_numpy(t) + for t in text_embeds]).to(self.device) + with torch.cuda.amp.autocast(): + similarity = image_features @ text_embeds.T + if reverse: + similarity = -similarity + _, top_labels = similarity.float().cpu().topk(top_count, dim=-1) + return [top_labels[0][i].numpy() for i in range(top_count)] + + def rank(self, + image_features: torch.Tensor, + top_count: int = 1, + reverse: bool = False) -> List[str]: + if len(self.labels) <= self.chunk_size: + tops = self._rank( + image_features, + self.embeds, + top_count=top_count, + reverse=reverse) + return [self.labels[i] for i in tops] + + num_chunks = int(math.ceil(len(self.labels) / self.chunk_size)) + keep_per_chunk = int(self.chunk_size / num_chunks) + + top_labels, top_embeds = [], [] + for chunk_idx in tqdm(range(num_chunks), disable=self.config.quiet): + start = chunk_idx * self.chunk_size + stop = min(start + self.chunk_size, len(self.embeds)) + tops = self._rank( + image_features, + self.embeds[start:stop], + top_count=keep_per_chunk, + reverse=reverse) + top_labels.extend([self.labels[start + i] for i in tops]) + top_embeds.extend([self.embeds[start + i] for i in tops]) + + tops = self._rank(image_features, top_embeds, top_count=top_count) + return [top_labels[i] for i in tops] + + +def _download_file(url: str, + filepath: str, + chunk_size: int = 4 * 1024 * 1024, + quiet: bool = False): + r = requests.get(url, stream=True) + if r.status_code != 200: + return + + file_size = int(r.headers.get('Content-Length', 0)) + filename = url.split('/')[-1] + progress = tqdm( + total=file_size, + unit='B', + unit_scale=True, + desc=filename, + disable=quiet) + with open(filepath, 'wb') as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + progress.update(len(chunk)) + progress.close() + + +def _merge_tables(tables: List[LabelTable], ci: Interrogator) -> LabelTable: + m = LabelTable([], None, ci) + for table in tables: + m.labels.extend(table.labels) + m.embeds.extend(table.embeds) + return m + + +def _prompt_at_max_len(text: str, tokenize) -> bool: + tokens = tokenize([text]) + return tokens[0][-1] != 0 + + +def _truncate_to_fit(text: str, tokenize) -> str: + parts = text.split(', ') + new_text = parts[0] + for part in parts[1:]: + if _prompt_at_max_len(new_text + part, tokenize): + break + new_text += ', ' + part + return new_text + + +def list_caption_models() -> List[str]: + return list(CAPTION_MODELS.keys()) + + +def list_clip_models() -> List[str]: + return ['/'.join(x) for x in open_clip.list_pretrained()] + + +def load_list(data_path: str, filename: Optional[str] = None) -> List[str]: + """Load a list of strings from a file.""" + if filename is not None: + data_path = os.path.join(data_path, filename) + with open(data_path, 'r', encoding='utf-8', errors='replace') as f: + items = [line.strip() for line in f.readlines()] + return items + + +@MODELS.register_module( + Tasks.image_captioning, module_name=Models.clip_interrogator) +class CLIP_Interrogator(TorchModel): + + def __init__(self, model_dir, device='cuda', device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.device = device + self.dtype = torch.float16 if self.device == 'cuda' else torch.float32 + cf = Config(clip_model_name='ViT-L-14/openai') + cf.data_path = os.path.join(model_dir, 'data') + cf.clip_model_path = model_dir + cf.cache_path = model_dir + self.ci = Interrogator(cf) + + def forward(self, inputs): + image = transforms.ToPILImage()(inputs) + return {'caption': self.ci.interrogate(image)} diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/__init__.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7e374b6f6c327b53b89872b189372704611a25c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .model import DiffusionForTextToImageSynthesis diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/diffusion.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..04203e18a7155ed595b50296da37ea93118eaeae --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/diffusion.py @@ -0,0 +1,657 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch + +from modelscope.models.multi_modal.dpm_solver_pytorch import ( + DPM_Solver, NoiseScheduleVP, model_wrapper, model_wrapper_guided_diffusion) + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def kl_divergence(mu1, logvar1, mu2, logvar2): + a = -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + b = ((mu1 - mu2)**2) * torch.exp(-logvar2) + return 0.5 * (a + b) + + +def standard_normal_cdf(x): + return 0.5 * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x0, mean, log_scale): + assert x0.shape == mean.shape == log_scale.shape + cx = x0 - mean + inv_stdv = torch.exp(-log_scale) + cdf_plus = standard_normal_cdf(inv_stdv * (cx + 1.0 / 255.0)) + cdf_min = standard_normal_cdf(inv_stdv * (cx - 1.0 / 255.0)) + log_cdf_plus = torch.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = torch.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = torch.where( + x0 < -0.999, log_cdf_plus, + torch.where(x0 > 0.999, log_one_minus_cdf_min, + torch.log(cdf_delta.clamp(min=1e-12)))) + assert log_probs.shape == x0.shape + return log_probs + + +def _i(tensor, t, x): + tensor = tensor.to(x.device) + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +def cosine_fn(u): + return math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + betas.append(min(1.0 - cosine_fn(t2) / cosine_fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + noise = torch.randn_like(x0) if noise is None else noise + return _i(self.sqrt_alphas_cumprod, t, x0) * x0 + _i( + self.sqrt_one_minus_alphas_cumprod, t, x0) * noise + + def q_mean_variance(self, x0, t): + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + shape = (-1, ) + ((1, ) * (xt.ndim - 1)) + mask = t.ne(0).float().view(*shape) # no noise when t == 0 + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + assert self.mean_type == 'eps' + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + a = u_out[:, :3] + b = guide_scale * (y_out[:, :3] - u_out[:, :3]) + c = y_out[:, 3:] + out = torch.cat([a + b, c], dim=1) + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - _i( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + xt) * xt + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def dpm_solver_sample_loop(self, + noise, + model, + skip_type, + order, + method, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + dpm_solver_timesteps=20, + t_start=None, + t_end=None, + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpm_solver'): + r"""Sample using DPM-Solver-based method. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + Please check all the parameters in `dpm_solver.sample` before using. + """ + noise_schedule = NoiseScheduleVP( + schedule='discrete', betas=self.betas.float()) + model_fn = model_wrapper_guided_diffusion( + model=model, + noise_schedule=noise_schedule, + var_type=self.var_type, + mean_type=self.mean_type, + model_kwargs=model_kwargs, + clamp=clamp, + percentile=percentile, + rescale_timesteps=self.rescale_timesteps, + num_timesteps=self.num_timesteps, + guide_scale=guide_scale, + condition_fn=condition_fn, + ) + dpm_solver = DPM_Solver( + model_fn=model_fn, + noise_schedule=noise_schedule, + ) + xt = dpm_solver.sample( + noise, + steps=dpm_solver_timesteps, + order=order, + skip_type=skip_type, + method=method, + solver_type=solver_type, + t_start=t_start, + t_end=t_end, + lower_order_final=lower_order_final, + denoise_to_zero=denoise_to_zero) + return xt + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + a = (1 - alphas_prev) / (1 - alphas) + b = (1 - alphas / alphas_prev) + sigmas = eta * torch.sqrt(a * b) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b, c, h, w = x0.size() + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + - x0) / _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b, c, h, w = noise.size() + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def loss(self, x0, t, model, model_kwargs={}, noise=None): + noise = torch.randn_like(x0) if noise is None else noise + xt = self.q_sample(x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', 'rescaled_l1']: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = (out - target).pow(1 if self.loss_type.endswith('l1') else 2 + ).abs().flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) + kl = kl.flatten(1).mean(dim=1) / math.log(2.0) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) + nll = nll.flatten(1).mean(dim=1) / math.log(2.0) + + # NLL for p(x0 | x1) and KL otherwise + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # prepare input and output + b, c, h, w = x0.size() + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/model.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/model.py new file mode 100644 index 0000000000000000000000000000000000000000..34d8d3422f8600b5547fa69f4db08eff77f77717 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/model.py @@ -0,0 +1,349 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os.path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.diffusion.diffusion import ( + GaussianDiffusion, beta_schedule) +from modelscope.models.multi_modal.diffusion.structbert import (BertConfig, + BertModel) +from modelscope.models.multi_modal.diffusion.tokenizer import FullTokenizer +from modelscope.models.multi_modal.diffusion.unet_generator import \ + DiffusionGenerator +from modelscope.models.multi_modal.diffusion.unet_upsampler_256 import \ + SuperResUNet256 +from modelscope.models.multi_modal.diffusion.unet_upsampler_1024 import \ + SuperResUNet1024 +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['DiffusionForTextToImageSynthesis'] + + +def make_diffusion(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None, + var_type='fixed_small'): + betas = beta_schedule(schedule, num_timesteps, init_beta, last_beta) + diffusion = GaussianDiffusion(betas, var_type=var_type) + return diffusion + + +class Tokenizer(object): + + def __init__(self, vocab_file, seq_len=64): + self.vocab_file = vocab_file + self.seq_len = seq_len + self.tokenizer = FullTokenizer( + vocab_file=vocab_file, do_lower_case=True) + + def __call__(self, text): + # tokenization + tokens = self.tokenizer.tokenize(text) + tokens = ['[CLS]'] + tokens[:self.seq_len - 2] + ['[SEP]'] + input_ids = self.tokenizer.convert_tokens_to_ids(tokens) + input_mask = [1] * len(input_ids) + segment_ids = [0] * len(input_ids) + + # padding + input_ids += [0] * (self.seq_len - len(input_ids)) + input_mask += [0] * (self.seq_len - len(input_mask)) + segment_ids += [0] * (self.seq_len - len(segment_ids)) + assert len(input_ids) == len(input_mask) == len( + segment_ids) == self.seq_len + + # convert to tensors + input_ids = torch.LongTensor(input_ids) + input_mask = torch.LongTensor(input_mask) + segment_ids = torch.LongTensor(segment_ids) + return input_ids, segment_ids, input_mask + + +class DiffusionModel(nn.Module): + + def __init__(self, model_dir): + super(DiffusionModel, self).__init__() + # including text and generator config + model_config = json.load( + open('{}/model_config.json'.format(model_dir), encoding='utf-8')) + + # text encoder + text_config = model_config['text_config'] + self.text_encoder = BertModel(BertConfig.from_dict(text_config)) + + # generator (64x64) + generator_config = model_config['generator_config'] + self.unet_generator = DiffusionGenerator(**generator_config) + + # upsampler (256x256) + upsampler_256_config = model_config['upsampler_256_config'] + self.unet_upsampler_256 = SuperResUNet256(**upsampler_256_config) + + # upsampler (1024x1024) + upsampler_1024_config = model_config['upsampler_1024_config'] + self.unet_upsampler_1024 = SuperResUNet1024(**upsampler_1024_config) + + def forward(self, noise, timesteps, input_ids, token_type_ids, + attention_mask): + context, y = self.text_encoder( + input_ids=input_ids, + token_type_ids=token_type_ids, + attention_mask=attention_mask) + context = context[-1] + x = self.unet_generator(noise, timesteps, y, context, attention_mask) + x = self.unet_upsampler_256(noise, timesteps, x, + torch.zeros_like(timesteps), y, context, + attention_mask) + x = self.unet_upsampler_1024(x, t, x) + return x + + +@MODELS.register_module( + Tasks.text_to_image_synthesis, module_name=Models.diffusion) +class DiffusionForTextToImageSynthesis(Model): + + def __init__(self, model_dir, device='gpu', **kwargs): + device = 'gpu' if torch.cuda.is_available() else 'cpu' + super().__init__(model_dir=model_dir, device=device, **kwargs) + diffusion_model = DiffusionModel(model_dir=model_dir) + pretrained_params = torch.load( + osp.join(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), 'cpu') + diffusion_model.load_state_dict(pretrained_params) + diffusion_model.eval().to() + + self.device = create_device(device) + diffusion_model.to(self.device) + + # modules + self.text_encoder = diffusion_model.text_encoder + self.unet_generator = diffusion_model.unet_generator + self.unet_upsampler_256 = diffusion_model.unet_upsampler_256 + self.unet_upsampler_1024 = diffusion_model.unet_upsampler_1024 + + # text tokenizer + vocab_path = f'{model_dir}/{ModelFile.VOCAB_FILE}' + self.tokenizer = Tokenizer(vocab_file=vocab_path, seq_len=64) + + # diffusion process + diffusion_params = json.load( + open( + '{}/diffusion_config.json'.format(model_dir), + encoding='utf-8')) + self.diffusion_generator = make_diffusion( + **diffusion_params['generator_config']) + self.diffusion_upsampler_256 = make_diffusion( + **diffusion_params['upsampler_256_config']) + self.diffusion_upsampler_1024 = make_diffusion( + **diffusion_params['upsampler_1024_config']) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + if not all([key in input for key in ('text', 'noise', 'timesteps')]): + raise ValueError( + f'input should contains "text", "noise", and "timesteps", but got {input.keys()}' + ) + input_ids, token_type_ids, attention_mask = self.tokenizer( + input['text']) + input_ids = input_ids.to(self.device).unsqueeze(0) + token_type_ids = token_type_ids.to(self.device).unsqueeze(0) + attention_mask = attention_mask.to(self.device).unsqueeze(0) + context, y = self.text_encoder( + input_ids=input_ids, + token_type_ids=token_type_ids, + attention_mask=attention_mask) + context = context[-1] + x = self.unet_generator(noise, timesteps, y, context, attention_mask) + x = self.unet_upsampler_256(noise, timesteps, x, + torch.zeros_like(timesteps), y, context, + attention_mask) + x = self.unet_upsampler_1024(x, t, x) + img = x.clamp(-1, 1).add(1).mul(127.5) + img = img.squeeze(0).permute(1, 2, 0).cpu().numpy().astype(np.uint8) + return img + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs + + @torch.no_grad() + def generate(self, input: Dict[str, Any]) -> Dict[str, Any]: + if 'text' not in input: + raise ValueError( + f'input should contain "text", but got {input.keys()}') + + # encode text + input_ids, token_type_ids, attention_mask = self.tokenizer( + input['text']) + input_ids = input_ids.to(self.device).unsqueeze(0) + token_type_ids = token_type_ids.to(self.device).unsqueeze(0) + attention_mask = attention_mask.to(self.device).unsqueeze(0) + context, y = self.text_encoder( + input_ids=input_ids, + token_type_ids=token_type_ids, + attention_mask=attention_mask) + context = context[-1] + + # choose a proper solver + solver = input.get('solver', 'dpm-solver') + if solver == 'dpm-solver': + # generation + img = self.diffusion_generator.dpm_solver_sample_loop( + noise=torch.randn(1, 3, 64, 64).to(self.device), + model=self.unet_generator, + model_kwargs=[{ + 'y': y, + 'context': context, + 'mask': attention_mask + }, { + 'y': torch.zeros_like(y), + 'context': torch.zeros_like(context), + 'mask': attention_mask + }], + percentile=input.get('generator_percentile', 0.995), + guide_scale=input.get('generator_guide_scale', 5.0), + dpm_solver_timesteps=input.get('dpm_solver_timesteps', 20), + order=3, + skip_type='logSNR', + method='singlestep', + t_start=0.9946) + + # upsampling (64->256) + if not input.get('debug', False): + img = F.interpolate( + img, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + img = self.diffusion_upsampler_256.dpm_solver_sample_loop( + noise=torch.randn_like(img), + model=self.unet_upsampler_256, + model_kwargs=[{ + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': y, + 'context': context, + 'mask': attention_mask + }, { + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': torch.zeros_like(y), + 'context': torch.zeros_like(context), + 'mask': torch.zeros_like(attention_mask) + }], + percentile=input.get('upsampler_256_percentile', 0.995), + guide_scale=input.get('upsampler_256_guide_scale', 5.0), + dpm_solver_timesteps=input.get('dpm_solver_timesteps', 20), + order=3, + skip_type='logSNR', + method='singlestep', + t_start=0.9946) + + # upsampling (256->1024) + if not input.get('debug', False): + img = F.interpolate( + img, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + img = self.diffusion_upsampler_1024.dpm_solver_sample_loop( + noise=torch.randn_like(img), + model=self.unet_upsampler_256, + model_kwargs=[{ + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': y, + 'context': context, + 'mask': attention_mask + }, { + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': torch.zeros_like(y), + 'context': torch.zeros_like(context), + 'mask': torch.zeros_like(attention_mask) + }], + percentile=input.get('upsampler_256_percentile', 0.995), + guide_scale=input.get('upsampler_256_guide_scale', 5.0), + dpm_solver_timesteps=input.get('dpm_solver_timesteps', 10), + order=3, + skip_type='logSNR', + method='singlestep', + t_start=None) + elif solver == 'ddim': + # generation + img = self.diffusion_generator.ddim_sample_loop( + noise=torch.randn(1, 3, 64, 64).to(self.device), + model=self.unet_generator, + model_kwargs=[{ + 'y': y, + 'context': context, + 'mask': attention_mask + }, { + 'y': torch.zeros_like(y), + 'context': torch.zeros_like(context), + 'mask': attention_mask + }], + percentile=input.get('generator_percentile', 0.995), + guide_scale=input.get('generator_guide_scale', 5.0), + ddim_timesteps=input.get('generator_ddim_timesteps', 250), + eta=input.get('generator_ddim_eta', 0.0)) + + # upsampling (64->256) + if not input.get('debug', False): + img = F.interpolate( + img, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + img = self.diffusion_upsampler_256.ddim_sample_loop( + noise=torch.randn_like(img), + model=self.unet_upsampler_256, + model_kwargs=[{ + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': y, + 'context': context, + 'mask': attention_mask + }, { + 'lx': img, + 'lt': torch.zeros(1).to(self.device), + 'y': torch.zeros_like(y), + 'context': torch.zeros_like(context), + 'mask': torch.zeros_like(attention_mask) + }], + percentile=input.get('upsampler_256_percentile', 0.995), + guide_scale=input.get('upsampler_256_guide_scale', 5.0), + ddim_timesteps=input.get('upsampler_256_ddim_timesteps', 50), + eta=input.get('upsampler_256_ddim_eta', 0.0)) + + # upsampling (256->1024) + if not input.get('debug', False): + img = F.interpolate( + img, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + img = self.diffusion_upsampler_1024.ddim_sample_loop( + noise=torch.randn_like(img), + model=self.unet_upsampler_1024, + model_kwargs={'concat': img}, + percentile=input.get('upsampler_1024_percentile', 0.995), + ddim_timesteps=input.get('upsampler_1024_ddim_timesteps', 20), + eta=input.get('upsampler_1024_ddim_eta', 0.0)) + else: + raise ValueError( + 'currently only supports "ddim" and "dpm-solve" solvers') + + # output + img = img.clamp(-1, 1).add(1).mul(127.5).squeeze(0).permute( + 1, 2, 0).cpu().numpy().astype(np.uint8) + return img diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/structbert.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/structbert.py new file mode 100644 index 0000000000000000000000000000000000000000..0ca57fc4acb4811384868f97ac863414a04284ce --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/structbert.py @@ -0,0 +1,932 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team and Alibaba inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +from __future__ import absolute_import, division, print_function +import copy +import math + +import json +import numpy as np +import six +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +class BertConfig(object): + """Configuration class to store the configuration of a `BertModel`. + """ + + def __init__(self, + vocab_size, + hidden_size=768, + emb_size=-1, + num_hidden_layers=12, + transformer_type='original', + transition_function='linear', + weighted_transformer=0, + num_rolled_layers=3, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + attention_type='self', + rezero=False, + pre_ln=False, + squeeze_excitation=False, + transfer_matrix=False, + dim_dropout=False, + roberta_style=False, + set_mask_zero=False, + init_scale=False, + safer_fp16=False, + grad_checkpoint=False): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.emb_size = emb_size + self.num_hidden_layers = num_hidden_layers + self.transformer_type = transformer_type + self.transition_function = transition_function + self.weighted_transformer = weighted_transformer + self.num_rolled_layers = num_rolled_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.attention_type = attention_type + self.rezero = rezero + self.pre_ln = pre_ln + self.squeeze_excitation = squeeze_excitation + self.transfer_matrix = transfer_matrix + self.dim_dropout = dim_dropout + self.set_mask_zero = set_mask_zero + self.roberta_style = roberta_style + self.init_scale = init_scale + self.safer_fp16 = safer_fp16 + self.grad_checkpoint = grad_checkpoint + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' + + +class BERTLayerNorm(nn.Module): + + def __init__(self, config, variance_epsilon=1e-12, special_size=None): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(BERTLayerNorm, self).__init__() + self.config = config + hidden_size = special_size if special_size is not None else config.hidden_size + self.gamma = nn.Parameter(torch.ones(hidden_size)) + self.beta = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = variance_epsilon if not config.roberta_style else 1e-5 + + def forward(self, x): + previous_type = x.type() + if self.config.safer_fp16: + x = x.float() + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + if self.config.safer_fp16: + return (self.gamma * x + self.beta).type(previous_type) + else: + return self.gamma * x + self.beta + + +class BERTEmbeddings(nn.Module): + + def __init__(self, config): + super(BERTEmbeddings, self).__init__() + """Construct the embedding module from word, position and token_type embeddings. + """ + hidden_size = config.hidden_size if config.emb_size < 0 else config.emb_size + self.word_embeddings = nn.Embedding( + config.vocab_size, + hidden_size, + padding_idx=1 if config.roberta_style else None) + self.position_embeddings = nn.Embedding( + config.max_position_embeddings, + hidden_size, + padding_idx=1 if config.roberta_style else None) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + hidden_size) + self.config = config + self.proj = None if config.emb_size < 0 else nn.Linear( + config.emb_size, config.hidden_size) + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = BERTLayerNorm(config, special_size=hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, adv_embedding=None): + seq_length = input_ids.size(1) + if not self.config.roberta_style: + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + else: + mask = input_ids.ne(1).int() + position_ids = (torch.cumsum(mask, dim=1).type_as(mask) + * mask).long() + 1 + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings( + input_ids) if adv_embedding is None else adv_embedding + if self.config.set_mask_zero: + words_embeddings[input_ids == 103] = 0. + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + if not self.config.roberta_style: + embeddings = words_embeddings + position_embeddings + token_type_embeddings + else: + embeddings = words_embeddings + position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + if self.proj is not None: + embeddings = self.proj(embeddings) + embeddings = self.dropout(embeddings) + else: + return embeddings, words_embeddings + + +class BERTFactorizedAttention(nn.Module): + + def __init__(self, config): + super(BERTFactorizedAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, *size): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(size) + + def forward(self, hidden_states, attention_mask): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, 0, 2, 3, 1) + key_layer = self.transpose_for_scores(mixed_key_layer, 0, 2, 1, 3) + value_layer = self.transpose_for_scores(mixed_value_layer, 0, 2, 1, 3) + + s_attention_scores = query_layer + attention_mask + s_attention_probs = nn.Softmax(dim=-1)(s_attention_scores) + s_attention_probs = self.dropout(s_attention_probs) + + c_attention_probs = nn.Softmax(dim=-1)(key_layer) + s_context_layer = torch.matmul(s_attention_probs, value_layer) + context_layer = torch.matmul(c_attention_probs, s_context_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +def dim_dropout(x, p=0, dim=-1, training=False): + if not training or p == 0: + return x + a = (1 - p) + b = (x.data.new(x.size()).zero_() + 1) + dropout_mask = torch.bernoulli(a * b) + return dropout_mask * (dropout_mask.size(dim) / torch.sum( + dropout_mask, dim=dim, keepdim=True)) * x + + +class BERTSelfAttention(nn.Module): + + def __init__(self, config): + super(BERTSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.config = config + if config.pre_ln: + self.LayerNorm = BERTLayerNorm(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask, head_mask=None): + if self.config.pre_ln: + hidden_states = self.LayerNorm(hidden_states) + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + if head_mask is not None and not self.training: + for i, mask in enumerate(head_mask): + if head_mask[i] == 1: + attention_scores[:, i, :, :] = 0. + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + if not self.config.dim_dropout: + attention_probs = self.dropout(attention_probs) + else: + attention_probs = dim_dropout( + attention_probs, + p=self.config.attention_probs_dropout_prob, + dim=-1, + training=self.training) + + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +class BERTSelfOutput(nn.Module): + + def __init__(self, config): + super(BERTSelfOutput, self).__init__() + self.config = config + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if not config.pre_ln and not config.rezero: + self.LayerNorm = BERTLayerNorm(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + if config.rezero: + self.res_factor = nn.Parameter( + torch.Tensor(1).fill_(0.99).to( + dtype=next(self.parameters()).dtype)) + self.factor = nn.Parameter( + torch.ones(1).to(dtype=next(self.parameters()).dtype)) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + if not self.config.rezero and not self.config.pre_ln: + hidden_states = self.LayerNorm(hidden_states + input_tensor) + elif self.config.rezero: + hidden_states = hidden_states + self.factor * input_tensor + else: + pass + return hidden_states + + +class BERTAttention(nn.Module): + + def __init__(self, config): + super(BERTAttention, self).__init__() + if config.attention_type.lower() == 'self': + self.self = BERTSelfAttention(config) + elif config.attention_type.lower() == 'factorized': + self.self = BERTFactorizedAttention(config) + else: + raise ValueError( + 'Attention type must in [self, factorized], but got {}'.format( + config.attention_type)) + self.output = BERTSelfOutput(config) + + def forward(self, input_tensor, attention_mask, head_mask=None): + self_output = self.self(input_tensor, attention_mask, head_mask) + attention_output = self.output(self_output, input_tensor) + return attention_output + + +class DepthwiseSeparableConv1d(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + dilation=1, + bias=False): + super(DepthwiseSeparableConv1d, self).__init__() + padding = (kernel_size - 1) // 2 + self.depthwise = nn.Conv1d( + in_channels, + in_channels, + kernel_size, + stride, + padding, + dilation, + groups=in_channels, + bias=bias) + self.pointwise = nn.Conv1d( + in_channels, out_channels, 1, 1, 0, 1, 1, bias=bias) + + def forward(self, x): + x = self.depthwise(x) + x = self.pointwise(x) + return x + + +class BERTIntermediate(nn.Module): + + def __init__(self, config): + super(BERTIntermediate, self).__init__() + self.config = config + if self.config.pre_ln: + self.LayerNorm = BERTLayerNorm(config) + self.intermediate_act_fn = gelu + if config.transition_function.lower() == 'linear': + self.dense = nn.Linear(config.hidden_size, + config.intermediate_size) + elif config.transition_function.lower() == 'cnn': + self.cnn = DepthwiseSeparableConv1d( + config.hidden_size, 4 * config.hidden_size, kernel_size=7) + elif config.config.hidden_size.lower() == 'rnn': + raise NotImplementedError( + 'rnn transition function is not implemented yet') + else: + raise ValueError('Only support linear/cnn/rnn') + + def forward(self, hidden_states): + if self.config.pre_ln: + hidden_states = self.LayerNorm(hidden_states) + if self.config.transition_function.lower() == 'linear': + hidden_states = self.dense(hidden_states) + elif self.config.transition_function.lower() == 'cnn': + hidden_states = self.cnn(hidden_states.transpose(-1, + -2)).transpose( + -1, -2) + else: + pass + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class SqueezeExcitationBlock(nn.Module): + + def __init__(self, config): + super(SqueezeExcitationBlock, self).__init__() + self.down_sampling = nn.Linear(config.hidden_size, + config.hidden_size // 4) + self.up_sampling = nn.Linear(config.hidden_size // 4, + config.hidden_size) + + def forward(self, hidden_states): + squeeze = torch.mean(hidden_states, 1, keepdim=True) + excitation = torch.sigmoid( + self.up_sampling(gelu(self.down_sampling(squeeze)))) + return hidden_states * excitation + + +class BERTOutput(nn.Module): + + def __init__(self, config): + super(BERTOutput, self).__init__() + self.config = config + if config.transition_function.lower() == 'linear': + self.dense = nn.Linear(config.intermediate_size, + config.hidden_size) + elif config.transition_function.lower() == 'cnn': + self.cnn = DepthwiseSeparableConv1d( + 4 * config.hidden_size, config.hidden_size, kernel_size=7) + elif config.config.hidden_size.lower() == 'rnn': + raise NotImplementedError( + 'rnn transition function is not implemented yet') + else: + raise ValueError('Only support linear/cnn/rnn') + if not config.pre_ln and not config.rezero: + self.LayerNorm = BERTLayerNorm(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + if config.squeeze_excitation: + self.SEblock = SqueezeExcitationBlock(config) + if config.rezero: + self.res_factor = nn.Parameter( + torch.Tensor(1).fill_(0.99).to( + dtype=next(self.parameters()).dtype)) + self.factor = nn.Parameter( + torch.ones(1).to(dtype=next(self.parameters()).dtype)) + + def forward(self, hidden_states, input_tensor): + if self.config.transition_function.lower() == 'linear': + hidden_states = self.dense(hidden_states) + elif self.config.transition_function.lower() == 'cnn': + hidden_states = self.cnn(hidden_states.transpose(-1, + -2)).transpose( + -1, -2) + else: + pass + hidden_states = self.dropout(hidden_states) + if self.config.squeeze_excitation: + hidden_states = self.SEblock(hidden_states) + if not self.config.rezero and not self.config.pre_ln: + hidden_states = self.LayerNorm(hidden_states + input_tensor) + elif self.config.rezero: + hidden_states = hidden_states + self.factor * input_tensor + else: + pass + return hidden_states + + +class BERTLayer(nn.Module): + + def __init__(self, config): + super(BERTLayer, self).__init__() + self.attention = BERTAttention(config) + self.intermediate = BERTIntermediate(config) + self.output = BERTOutput(config) + + def forward(self, hidden_states, attention_mask, head_mask=None): + attention_output = self.attention(hidden_states, attention_mask, + head_mask) + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return attention_output, layer_output + + +class BERTWeightedLayer(nn.Module): + + def __init__(self, config): + super(BERTWeightedLayer, self).__init__() + self.config = config + self.self = BERTSelfAttention(config) + self.attention_head_size = self.self.attention_head_size + + self.w_o = nn.ModuleList([ + nn.Linear(self.attention_head_size, config.hidden_size) + for _ in range(config.num_attention_heads) + ]) + self.w_kp = torch.rand(config.num_attention_heads) + self.w_kp = nn.Parameter(self.w_kp / self.w_kp.sum()) + self.w_a = torch.rand(config.num_attention_heads) + self.w_a = nn.Parameter(self.w_a / self.w_a.sum()) + + self.intermediate = BERTIntermediate(config) + self.output = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = BERTLayerNorm(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, attention_mask): + self_output = self.self(hidden_states, attention_mask) + self_outputs = self_output.split(self.self.attention_head_size, dim=-1) + self_outputs = [ + self.w_o[i](self_outputs[i]) for i in range(len(self_outputs)) + ] + self_outputs = [ + self.dropout(self_outputs[i]) for i in range(len(self_outputs)) + ] + self_outputs = [ + kappa * output for kappa, output in zip(self.w_kp, self_outputs) + ] + self_outputs = [ + self.intermediate(self_outputs[i]) + for i in range(len(self_outputs)) + ] + self_outputs = [ + self.output(self_outputs[i]) for i in range(len(self_outputs)) + ] + self_outputs = [ + self.dropout(self_outputs[i]) for i in range(len(self_outputs)) + ] + self_outputs = [ + alpha * output for alpha, output in zip(self.w_a, self_outputs) + ] + output = sum(self_outputs) + return self.LayerNorm(hidden_states + output) + + +class BERTEncoder(nn.Module): + + def __init__(self, config): + super(BERTEncoder, self).__init__() + self.layer = nn.ModuleList() + for _ in range(config.num_hidden_layers): + if config.weighted_transformer: + self.layer.append(BERTWeightedLayer(config)) + else: + self.layer.append(BERTLayer(config)) + if config.rezero: + for index, layer in enumerate(self.layer): + layer.output.res_factor = nn.Parameter( + torch.Tensor(1).fill_(1.).to( + dtype=next(self.parameters()).dtype)) + layer.output.factor = nn.Parameter( + torch.Tensor(1).fill_(1).to( + dtype=next(self.parameters()).dtype)) + layer.attention.output.res_factor = layer.output.res_factor + layer.attention.output.factor = layer.output.factor + self.config = config + + def forward(self, + hidden_states, + attention_mask, + epoch_id=-1, + head_masks=None): + all_encoder_layers = [hidden_states] + if epoch_id != -1: + detach_index = int(len(self.layer) / 3) * (2 - epoch_id) - 1 + else: + detach_index = -1 + for index, layer_module in enumerate(self.layer): + if head_masks is None: + if not self.config.grad_checkpoint: + self_out, hidden_states = layer_module( + hidden_states, attention_mask, None) + else: + self_out, hidden_states = torch.utils.checkpoint.checkpoint( + layer_module, hidden_states, attention_mask, None) + else: + self_out, hidden_states = layer_module(hidden_states, + attention_mask, + head_masks[index]) + if detach_index == index: + hidden_states.detach_() + all_encoder_layers.append(self_out) + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BERTEncoderRolled(nn.Module): + + def __init__(self, config): + super(BERTEncoderRolled, self).__init__() + layer = BERTLayer(config) + self.config = config + self.layer = nn.ModuleList( + [copy.deepcopy(layer) for _ in range(config.num_rolled_layers)]) + + def forward(self, + hidden_states, + attention_mask, + epoch_id=-1, + head_masks=None): + all_encoder_layers = [hidden_states] + for i in range(self.config.num_hidden_layers): + if self.config.transformer_type.lower() == 'universal': + hidden_states = self.layer[i % self.config.num_rolled_layers]( + hidden_states, attention_mask) + elif self.config.transformer_type.lower() == 'albert': + a = i // ( + self.config.num_hidden_layers + // self.config.num_rolled_layers) + hidden_states = self.layer[a](hidden_states, attention_mask) + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BERTEncoderACT(nn.Module): + + def __init__(self, config): + super(BERTEncoderACT, self).__init__() + self.layer = BERTLayer(config) + p = nn.Linear(config.hidden_size, 1) + self.p = nn.ModuleList( + [copy.deepcopy(p) for _ in range(config.num_hidden_layers)]) + # Following act paper, set bias init ones + for module in self.p: + module.bias.data.fill_(1.) + self.config = config + self.act_max_steps = config.num_hidden_layers + self.threshold = 0.99 + + def should_continue(self, halting_probability, n_updates): + return (halting_probability.lt(self.threshold).__and__( + n_updates.lt(self.act_max_steps))).any() + + def forward(self, hidden_states, attention_mask): + all_encoder_layers = [hidden_states] + batch_size, seq_len, hdim = hidden_states.size() + halting_probability = torch.zeros(batch_size, seq_len).cuda() + remainders = torch.zeros(batch_size, seq_len).cuda() + n_updates = torch.zeros(batch_size, seq_len).cuda() + for i in range(self.act_max_steps): + p = torch.sigmoid(self.p[i](hidden_states).squeeze(2)) + still_running = halting_probability.lt(1.0).float() + new_halted = (halting_probability + p * still_running).gt( + self.threshold).float() * still_running + still_running = (halting_probability + p * still_running).le( + self.threshold).float() * still_running + halting_probability = halting_probability + p * still_running + remainders = remainders + new_halted * (1 - halting_probability) + halting_probability = halting_probability + new_halted * remainders + n_updates = n_updates + still_running + new_halted + update_weights = (p * still_running + + new_halted * remainders).unsqueeze(2) + transformed_states = self.layer(hidden_states, attention_mask) + hidden_states = transformed_states * update_weights + hidden_states * ( + 1 - update_weights) + all_encoder_layers.append(hidden_states) + if not self.should_continue(halting_probability, n_updates): + break + return all_encoder_layers, torch.mean(n_updates + remainders) + + +class BERTPooler(nn.Module): + + def __init__(self, config): + super(BERTPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertModel(nn.Module): + """BERT model ("Bidirectional Embedding Representations from a Transformer"). + + Example: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 2, 0]]) + + >>> config = modeling.BertConfig(vocab_size=32000, hidden_size=512, + >>> num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) + + >>> model = modeling.BertModel(config=config) + >>> all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config: BertConfig): + """Constructor for BertModel. + + Args: + config: `BertConfig` instance. + """ + super(BertModel, self).__init__() + self.config = config + self.embeddings = BERTEmbeddings(config) + if config.transformer_type.lower() == 'original': + self.encoder = BERTEncoder(config) + elif config.transformer_type.lower() == 'universal': + self.encoder = BERTEncoderRolled(config) + elif config.transformer_type.lower() == 'albert': + self.encoder = BERTEncoderRolled(config) + elif config.transformer_type.lower() == 'act': + self.encoder = BERTEncoderACT(config) + elif config.transformer_type.lower() == 'textnas': + from textnas_final import input_dict, op_dict, skip_dict + self.encoder = TextNASEncoder(config, op_dict, input_dict, + skip_dict) + else: + raise ValueError('Not support transformer type: {}'.format( + config.transformer_type.lower())) + self.pooler = BERTPooler(config) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + epoch_id=-1, + head_masks=None, + adv_embedding=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output, word_embeddings = self.embeddings( + input_ids, token_type_ids, adv_embedding) + if self.config.transformer_type.lower() == 'act': + all_encoder_layers, act_loss = self.encoder( + embedding_output, extended_attention_mask) + elif self.config.transformer_type.lower() == 'reformer': + sequence_output = self.encoder(embedding_output) + all_encoder_layers = [sequence_output, sequence_output] + else: + all_encoder_layers = self.encoder(embedding_output, + extended_attention_mask, + epoch_id, head_masks) + all_encoder_layers.insert(0, word_embeddings) + sequence_output = all_encoder_layers[-1] + if not self.config.safer_fp16: + pooled_output = self.pooler(sequence_output) + else: + pooled_output = sequence_output[:, 0] + return all_encoder_layers, pooled_output + + +class BertForSequenceClassificationMultiTask(nn.Module): + """BERT model for classification. + This module is composed of the BERT model with a linear layer on top of + the pooled output. + + Example: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 2, 0]]) + + >>> config = BertConfig(vocab_size=32000, hidden_size=512, + >>> num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) + + >>> num_labels = 2 + + >>> model = BertForSequenceClassification(config, num_labels) + >>> logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config, label_list, core_encoder): + super(BertForSequenceClassificationMultiTask, self).__init__() + if core_encoder.lower() == 'bert': + self.bert = BertModel(config) + elif core_encoder.lower() == 'lstm': + self.bert = LSTMModel(config) + else: + raise ValueError( + 'Only support lstm or bert, but got {}'.format(core_encoder)) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.ModuleList() + for label in label_list: + self.classifier.append(nn.Linear(config.hidden_size, len(label))) + self.label_list = label_list + + def init_weights(module): + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=config.initializer_range) + elif isinstance(module, BERTLayerNorm): + module.beta.data.normal_( + mean=0.0, std=config.initializer_range) + module.gamma.data.normal_( + mean=0.0, std=config.initializer_range) + if isinstance(module, nn.Linear): + module.bias.data.zero_() + + self.apply(init_weights) + + def forward(self, + input_ids, + token_type_ids, + attention_mask, + labels=None, + labels_index=None, + epoch_id=-1, + head_masks=None, + adv_embedding=None, + return_embedding=False, + loss_weight=None): + all_encoder_layers, pooled_output = self.bert(input_ids, + token_type_ids, + attention_mask, epoch_id, + head_masks, + adv_embedding) + pooled_output = self.dropout(pooled_output) + logits = [classifier(pooled_output) for classifier in self.classifier] + if labels is not None: + loss_fct = CrossEntropyLoss(reduction='none') + regression_loss_fct = nn.MSELoss(reduction='none') + labels_lst = torch.unbind(labels, 1) + loss_lst = [] + for index, (label, logit) in enumerate(zip(labels_lst, logits)): + if len(self.label_list[index]) != 1: + loss = loss_fct(logit, label.long()) + else: + loss = regression_loss_fct(logit.squeeze(-1), label) + labels_mask = (labels_index == index).to( + dtype=next(self.parameters()).dtype) + if loss_weight is not None: + loss = loss * loss_weight[index] + loss = torch.mean(loss * labels_mask) + loss_lst.append(loss) + if not return_embedding: + return sum(loss_lst), logits + else: + return sum(loss_lst), logits, all_encoder_layers[0] + else: + return logits diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/tokenizer.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..918498cd8e8c3e2369e8331171fcc281e2be44a5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/tokenizer.py @@ -0,0 +1,333 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team and Alibaba inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import, division, print_function +import collections +import unicodedata + +import six + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode('utf-8', 'ignore') + elif isinstance(text, unicode): + return text + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + else: + raise ValueError('Not running on Python2 or Python 3?') + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode('utf-8') + else: + raise ValueError('Unsupported string type: %s' % (type(text))) + else: + raise ValueError('Not running on Python2 or Python 3?') + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, 'r', encoding='utf-8') as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_tokens_to_ids(vocab, tokens): + """Converts a sequence of tokens into ids using the vocab.""" + ids = [] + for token in tokens: + ids.append(vocab[token]) + return ids + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_tokens_to_ids(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return [self.inv_vocab[i] for i in ids] + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(' '.join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize('NFD', text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == 'Mn': + continue + output.append(char) + return ''.join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return [''.join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(' ') + output.append(char) + output.append(' ') + else: + output.append(char) + return ''.join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF) + or (cp >= 0x20000 and cp <= 0x2A6DF) + or (cp >= 0x2A700 and cp <= 0x2B73F) + or (cp >= 0x2B740 and cp <= 0x2B81F) + or (cp >= 0x2B820 and cp <= 0x2CEAF) + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F)): + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(' ') + else: + output.append(char) + return ''.join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token='[UNK]', max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + >>> input = "unaffable" + >>> output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = ''.join(chars[start:end]) + if start > 0: + substr = '##' + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == ' ' or char == '\t' or char == '\n' or char == '\r': + return True + cat = unicodedata.category(char) + if cat == 'Zs': + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == '\t' or char == '\n' or char == '\r': + return False + cat = unicodedata.category(char) + if cat.startswith('C'): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith('P'): + return True + return False diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_generator.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..d6ab1068ecbc76853de98db8c445d6d35a25fc4f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_generator.py @@ -0,0 +1,322 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['DiffusionGenerator'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, scale_factor, use_conv=False): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.scale_factor = scale_factor + self.use_conv = use_conv + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(in_dim, out_dim, 3, padding=1) + if use_conv else nn.Identity()) + elif scale_factor == 0.5: + self.resample = nn.Conv2d( + in_dim, out_dim, 3, stride=2, + padding=1) if use_conv else nn.AvgPool2d( + kernel_size=2, stride=2) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + scale_factor=1.0, + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.scale_factor = scale_factor + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, scale_factor, use_conv=False) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e): + identity = self.resample(x) + x = self.layer1[-1](self.resample(self.layer1[:-1](x))) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None, mask=None): + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + if mask is not None: + assert context is not None + full_mask = x.new_ones((b, 1, q.size(-1), k.size(-1))) + full_mask[:, 0, :, :-q.size(-1)] = mask.unsqueeze(1) + attn = attn.masked_fill(full_mask == 0, float('-inf')) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class DiffusionGenerator(nn.Module): + + def __init__(self, + in_dim=3, + dim=512, + text_dim=1024, + context_dim=512, + out_dim=6, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + resblock_resample=True, + use_scale_shift_norm=True, + dropout=0.0): + embed_dim = dim * 4 + super(DiffusionGenerator, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.text_dim = text_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.resblock_resample = resblock_resample + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.pool_embedding = nn.Sequential( + nn.LayerNorm(text_dim), nn.Linear(text_dim, embed_dim)) + self.text_embedding = nn.Sequential( + nn.LayerNorm(text_dim), nn.Linear(text_dim, context_dim), + nn.SiLU(), nn.Linear(context_dim, context_dim)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList( + [ResidualBlock(in_dim, embed_dim, out_dim, dropout)]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + self.encoder.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample( + out_dim, out_dim, 0.5, use_conv=True) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + AttentionBlock(out_dim, context_dim, num_heads, head_dim), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample( + out_dim, out_dim, 2.0, use_conv=True) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y, context, mask=None): + # embeddings + e = self.time_embedding(sinusoidal_embedding( + t, self.dim)) + self.pool_embedding(y) + context = self.text_embedding(context) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, e, context, mask) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, e, context, mask) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, e, context, mask) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, e, context, mask): + if isinstance(module, ResidualBlock): + x = module(x, e) + elif isinstance(module, AttentionBlock): + x = module(x, context, mask) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context, mask) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_1024.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_1024.py new file mode 100644 index 0000000000000000000000000000000000000000..b566a92be692f055a4d0924a909968b8b3b6d018 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_1024.py @@ -0,0 +1,243 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['SuperResUNet1024'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, scale_factor, use_conv=False): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.scale_factor = scale_factor + self.use_conv = use_conv + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(in_dim, out_dim, 3, padding=1) + if use_conv else nn.Identity()) + elif scale_factor == 0.5: + self.resample = nn.Conv2d( + in_dim, out_dim, 3, stride=2, + padding=1) if use_conv else nn.AvgPool2d( + kernel_size=2, stride=2) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + scale_factor=1.0, + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.scale_factor = scale_factor + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, scale_factor, use_conv=False) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e): + identity = self.resample(x) + x = self.layer1[-1](self.resample(self.layer1[:-1](x))) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class SuperResUNet1024(nn.Module): + + def __init__(self, + in_dim=6, + dim=192, + out_dim=3, + dim_mult=[1, 1, 2, 2, 4, 4], + num_res_blocks=2, + resblock_resample=True, + use_scale_shift_norm=True, + dropout=0.0): + embed_dim = dim * 4 + super(SuperResUNet1024, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.resblock_resample = resblock_resample + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embedding + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual block + block = nn.ModuleList([ + ResidualBlock(in_dim, embed_dim, out_dim, + use_scale_shift_norm, 1.0, dropout) + ]) + shortcut_dims.append(out_dim) + in_dim = out_dim + self.encoder.append(block) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample( + out_dim, out_dim, 0.5, use_conv=True) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual block + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample( + out_dim, out_dim, 2.0, use_conv=True) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, concat): + # embedding + if concat is not None: + if concat.shape[-2:] != x.shape[-2:]: + concat = F.interpolate( + concat, x.shape[-2:], mode='bilinear', align_corners=False) + x = torch.cat([x, concat], dim=1) + e = self.time_embedding(sinusoidal_embedding(t, self.dim)) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, e) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, e) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, e) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, e): + if isinstance(module, ResidualBlock): + x = module(x, e) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_256.py b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_256.py new file mode 100644 index 0000000000000000000000000000000000000000..e8501122198387149ed9c5da698be127d1ee677e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/diffusion/unet_upsampler_256.py @@ -0,0 +1,340 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math +from functools import partial + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['SuperResUNet256'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, scale_factor, use_conv=False): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.scale_factor = scale_factor + self.use_conv = use_conv + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(in_dim, out_dim, 3, padding=1) + if use_conv else nn.Identity()) + elif scale_factor == 0.5: + self.resample = nn.Conv2d( + in_dim, out_dim, 3, stride=2, + padding=1) if use_conv else nn.AvgPool2d( + kernel_size=2, stride=2) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + scale_factor=1.0, + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.scale_factor = scale_factor + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample_x = Resample(in_dim, in_dim, scale_factor) + self.resample_i = Resample(in_dim, in_dim, scale_factor) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e): + identity = self.resample_i(x) + x = self.layer1[-1](self.resample_x(self.layer1[:-1](x))) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None, mask=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + mask: [B, L] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([k, ck], dim=-1) + v = torch.cat([v, cv], dim=-1) + + # compute attention + attn = torch.einsum('bndi,bndj->bnij', q * self.scale, k * self.scale) + if mask is not None: + pad_mask = mask.new_ones((b, 1, 1, h * w)) + mask = torch.cat((pad_mask, mask.unsqueeze(1).unsqueeze(1)), + dim=-1) + attn = attn.masked_fill(mask == 0, float('-inf')) + + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.einsum('bnij,bndj->bndi', attn, v) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class SuperResUNet256(nn.Module): + + def __init__(self, + in_dim=6, + out_dim=3, + dim=256, + text_dim=1024, + context_dim=512, + dim_mult=[1, 2, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=2, + attn_scales=[1 / 16], + resblock_resample=True, + use_conv=True, + use_scale_shift_norm=True, + dropout=0.1): + embed_dim = dim * 4 + super(SuperResUNet256, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.resblock_resample = resblock_resample + self.use_conv = use_conv + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.noise_time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.pool_embedding = nn.Sequential( + nn.LayerNorm(text_dim), nn.Linear(text_dim, embed_dim)) + self.text_embedding = nn.Sequential( + nn.LayerNorm(text_dim), nn.Linear(text_dim, context_dim), + nn.SiLU(), nn.Linear(context_dim, context_dim)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim, embed_dim, out_dim, + use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + shortcut_dims.append(out_dim) + in_dim = out_dim + self.encoder.append(block) + + # downsample + if i != len(dim_mult) - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample(out_dim, out_dim, 0.5, use_conv) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + AttentionBlock(out_dim, context_dim, num_heads, head_dim), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample(out_dim, out_dim, 2.0, use_conv) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, lx, lt, y, context, mask): + assert context.shape[:-1] == mask.shape + + # embeddings + t = self.time_embedding(sinusoidal_embedding(t, self.dim)) \ + + self.noise_time_embedding(sinusoidal_embedding(lt, self.dim)) \ + + self.pool_embedding(y) + + context = self.text_embedding(context) + + if lx.shape[-2:] != x.shape[-2:]: + lx = F.interpolate( + lx, x.shape[-2:], mode='bilinear', align_corners=False) + x = torch.cat([x, lx], dim=1) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, t, context, mask) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, t, context, mask) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, t, context, mask) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, t, context, mask): + if isinstance(module, ResidualBlock): + x = module(x, t) + elif isinstance(module, AttentionBlock): + x = module(x, context, mask) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, t, context, mask) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/dpm_solver_pytorch.py b/runtime/Lib/site_packages/models/multi_modal/dpm_solver_pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..1beb7a60a9d0db4fec04c60673c4cc48909856b4 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/dpm_solver_pytorch.py @@ -0,0 +1,1077 @@ +# The implementation is borrowed and modified from dpm-solver, +# publicly available at https://github.com/LuChengTHU/dpm-solver. +# Copyright LuChengTHU Authors. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. +# All rights reserved. +import math + +import torch +import torch.nn.functional as F + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +class NoiseScheduleVP: + + def __init__( + self, + schedule='discrete', + betas=None, + alphas_cumprod=None, + continuous_beta_0=0.1, + continuous_beta_1=20., + ): + if schedule not in ['discrete', 'linear', 'cosine']: + raise ValueError( + "Unsupported noise schedule {}. The schedule needs to be 'discrete' or 'linear' or 'cosine'" + .format(schedule)) + + self.schedule = schedule + if schedule == 'discrete': + if betas is not None: + log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0) + else: + assert alphas_cumprod is not None + log_alphas = 0.5 * torch.log(alphas_cumprod) + self.total_N = len(log_alphas) + self.T = 1. + self.t_array = torch.linspace(0., 1., + self.total_N + 1)[1:].reshape( + (1, -1)) + self.log_alpha_array = log_alphas.reshape(( + 1, + -1, + )) + else: + self.total_N = 1000 + self.beta_0 = continuous_beta_0 + self.beta_1 = continuous_beta_1 + self.cosine_s = 0.008 + self.cosine_beta_max = 999. + self.cosine_t_max = math.atan( + self.cosine_beta_max * (1. + self.cosine_s) / math.pi) * 2. * ( + 1. + self.cosine_s) / math.pi - self.cosine_s + self.cosine_log_alpha_0 = math.log( + math.cos(self.cosine_s / (1. + self.cosine_s) * math.pi / 2.)) + self.schedule = schedule + if schedule == 'cosine': + # For the cosine schedule, T = 1 will have numerical issues. So we manually set the ending time T. + # Note that T = 0.9946 may be not the optimal setting. However, we find it works well. + self.T = 0.9946 + else: + self.T = 1. + + def marginal_log_mean_coeff(self, t): + """ + Compute log(alpha_t) of a given continuous-time label t in [0, T]. + """ + if self.schedule == 'discrete': + return interpolate_fn( + t.reshape((-1, 1)), self.t_array.to(t.device), + self.log_alpha_array.to(t.device)).reshape((-1)) + elif self.schedule == 'linear': + return -0.25 * t**2 * (self.beta_1 + - self.beta_0) - 0.5 * t * self.beta_0 + elif self.schedule == 'cosine': + + def log_alpha_fn(s): + _a = (s + self.cosine_s) + _b = (1. + self.cosine_s) + _c = math.pi / 2. + return torch.log(torch.cos(_a / _b * _c)) + + log_alpha_t = log_alpha_fn(t) - self.cosine_log_alpha_0 + return log_alpha_t + + def marginal_alpha(self, t): + """ + Compute alpha_t of a given continuous-time label t in [0, T]. + """ + return torch.exp(self.marginal_log_mean_coeff(t)) + + def marginal_std(self, t): + """ + Compute sigma_t of a given continuous-time label t in [0, T]. + """ + return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t))) + + def marginal_lambda(self, t): + """ + Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T]. + """ + log_mean_coeff = self.marginal_log_mean_coeff(t) + log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff)) + return log_mean_coeff - log_std + + def inverse_lambda(self, lamb): + """ + Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t. + """ + if self.schedule == 'linear': + tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp( + -2. * lamb, + torch.zeros((1, )).to(lamb)) + Delta = self.beta_0**2 + tmp + return tmp / (torch.sqrt(Delta) + self.beta_0) / ( + self.beta_1 - self.beta_0) + elif self.schedule == 'discrete': + log_alpha = -0.5 * torch.logaddexp( + torch.zeros((1, )).to(lamb.device), -2. * lamb) + t = interpolate_fn( + log_alpha.reshape((-1, 1)), + torch.flip(self.log_alpha_array.to(lamb.device), [1]), + torch.flip(self.t_array.to(lamb.device), [1])) + return t.reshape((-1, )) + else: + log_alpha = -0.5 * torch.logaddexp(-2. * lamb, + torch.zeros((1, )).to(lamb)) + + def t_fn(log_alpha_t): + return torch.arccos( + torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2. * ( + 1. + self.cosine_s) / math.pi - self.cosine_s + + t = t_fn(log_alpha) + return t + + +def model_wrapper( + model, + noise_schedule, + model_type='noise', + model_kwargs={}, + guidance_type='uncond', + condition=None, + unconditional_condition=None, + guidance_scale=1., + classifier_fn=None, + classifier_kwargs={}, +): + + def get_model_input_time(t_continuous): + """ + Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time. + For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N]. + For continuous-time DPMs, we just use `t_continuous`. + """ + if noise_schedule.schedule == 'discrete': + return (t_continuous - 1. / noise_schedule.total_N) * 1000. + else: + return t_continuous + + def noise_pred_fn(x, t_continuous, cond=None): + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((x.shape[0])) + t_input = get_model_input_time(t_continuous) + if cond is None: + output = model(x, t_input, **model_kwargs) + else: + output = model(x, t_input, cond, **model_kwargs) + if model_type == 'noise': + return output + elif model_type == 'x_start': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + dims = x.dim() + return (x - expand_dims(alpha_t, dims) * output) / expand_dims( + sigma_t, dims) + elif model_type == 'v': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + dims = x.dim() + return expand_dims(alpha_t, dims) * output + expand_dims( + sigma_t, dims) * x + elif model_type == 'score': + sigma_t = noise_schedule.marginal_std(t_continuous) + dims = x.dim() + return -expand_dims(sigma_t, dims) * output + + def cond_grad_fn(x, t_input): + """ + Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t). + """ + with torch.enable_grad(): + x_in = x.detach().requires_grad_(True) + log_prob = classifier_fn(x_in, t_input, condition, + **classifier_kwargs) + return torch.autograd.grad(log_prob.sum(), x_in)[0] + + def model_fn(x, t_continuous): + """ + The noise predicition model function that is used for DPM-Solver. + """ + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((x.shape[0])) + if guidance_type == 'uncond': + return noise_pred_fn(x, t_continuous) + elif guidance_type == 'classifier': + assert classifier_fn is not None + t_input = get_model_input_time(t_continuous) + cond_grad = cond_grad_fn(x, t_input) + sigma_t = noise_schedule.marginal_std(t_continuous) + noise = noise_pred_fn(x, t_continuous) + return noise - guidance_scale * expand_dims( + sigma_t, dims=cond_grad.dim()) * cond_grad + elif guidance_type == 'classifier-free': + if guidance_scale == 1. or unconditional_condition is None: + return noise_pred_fn(x, t_continuous, cond=condition) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t_continuous] * 2) + c_in = torch.cat([unconditional_condition, condition]) + noise_uncond, noise = noise_pred_fn( + x_in, t_in, cond=c_in).chunk(2) + return noise_uncond + guidance_scale * (noise - noise_uncond) + + assert model_type in ['noise', 'x_start', 'v'] + assert guidance_type in ['uncond', 'classifier', 'classifier-free'] + return model_fn + + +def model_wrapper_guided_diffusion( + model, + noise_schedule, + var_type, + mean_type, + model_kwargs={}, + clamp=None, + percentile=None, + rescale_timesteps=False, + num_timesteps=1000, + guide_scale=None, + condition_fn=None, +): + + def _scale_timesteps(t): + if rescale_timesteps: + return t.float() * 1000.0 / num_timesteps + return t + + def get_model_input_time(t_continuous): + if noise_schedule.schedule == 'discrete': + return (t_continuous - 1. / noise_schedule.total_N) * 1000. + else: + return t_continuous + + def noise_pred_fn(xt, t_continuous): + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((xt.shape[0])) + t_input = get_model_input_time(_scale_timesteps(t_continuous)) + # predict distribution + if guide_scale is None: + out = model(xt, t_input, **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, t_input, **model_kwargs[0]) + u_out = model(xt, t_input, **model_kwargs[1]) + dim = y_out.size(1) if var_type.startswith( + 'fixed') else y_out.size(1) // 2 + _a = u_out[:, :dim] + _b = guide_scale * (y_out[:, :dim] - u_out[:, :dim]) + _c = [_a + _b, y_out[:, dim:]] + out = torch.cat(_c, dim=1) + if var_type == 'learned': + out, _ = out.chunk(2, dim=1) + elif var_type == 'learned_range': + out, _ = out.chunk(2, dim=1) + + if mean_type == 'eps': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + dims = xt.dim() + x0 = (xt - expand_dims(sigma_t, dims) * out) / expand_dims( + alpha_t, dims) + elif mean_type == 'x_{t-1}': + assert noise_schedule.schedule == 'discrete' + mu = out + posterior_mean_coef1 = None + posterior_mean_coef2 = None + x0 = expand_dims( + 1. / posterior_mean_coef1, dims) * mu - expand_dims( + posterior_mean_coef2 / posterior_mean_coef1, dims) * xt + elif mean_type == 'x0': + x0 = out + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + + if condition_fn is not None: + alpha_t = noise_schedule.marginal_alpha(t_continuous) + eps = (xt - expand_dims(alpha_t, dims) * x0) / expand_dims( + sigma_t, dims) + + eps = eps - (1 - alpha_t).sqrt() * condition_fn( + xt, t_input, **model_kwargs) + x0 = (xt - expand_dims(sigma_t, dims) * eps) / expand_dims( + alpha_t, dims) + + eps = (xt - expand_dims(alpha_t, dims) * x0) / expand_dims( + sigma_t, dims) + return eps + + def model_fn(x, t_continuous): + """ + The noise predicition model function that is used for DPM-Solver. + """ + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((x.shape[0])) + return noise_pred_fn(x, t_continuous) + + return model_fn + + +class DPM_Solver: + + def __init__(self, + model_fn, + noise_schedule, + predict_x0=False, + thresholding=False, + max_val=1.): + self.model = model_fn + self.noise_schedule = noise_schedule + self.predict_x0 = predict_x0 + self.thresholding = thresholding + self.max_val = max_val + + def noise_prediction_fn(self, x, t): + """ + Return the noise prediction model. + """ + return self.model(x, t) + + def data_prediction_fn(self, x, t): + """ + Return the data prediction model (with thresholding). + """ + noise = self.noise_prediction_fn(x, t) + dims = x.dim() + alpha_t, sigma_t = self.noise_schedule.marginal_alpha( + t), self.noise_schedule.marginal_std(t) + x0 = (x - expand_dims(sigma_t, dims) * noise) / expand_dims( + alpha_t, dims) + if self.thresholding: + p = 0.995 # A hyperparameter in the paper of "Imagen" [1]. + s = torch.quantile( + torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1) + s = expand_dims( + torch.maximum(s, + self.max_val * torch.ones_like(s).to(s.device)), + dims) + x0 = torch.clamp(x0, -s, s) / s + return x0 + + def model_fn(self, x, t): + """ + Convert the model to the noise prediction model or the data prediction model. + """ + if self.predict_x0: + return self.data_prediction_fn(x, t) + else: + return self.noise_prediction_fn(x, t) + + def get_time_steps(self, skip_type, t_T, t_0, N, device): + if skip_type == 'logSNR': + lambda_T = self.noise_schedule.marginal_lambda( + torch.tensor(t_T).to(device)) + lambda_0 = self.noise_schedule.marginal_lambda( + torch.tensor(t_0).to(device)) + logSNR_steps = torch.linspace(lambda_T.cpu().item(), + lambda_0.cpu().item(), + N + 1).to(device) + return self.noise_schedule.inverse_lambda(logSNR_steps) + elif skip_type == 'time_uniform': + return torch.linspace(t_T, t_0, N + 1).to(device) + elif skip_type == 'time_quadratic': + t_order = 2 + t = torch.linspace(t_T**(1. / t_order), t_0**(1. / t_order), + N + 1).pow(t_order).to(device) + return t + else: + raise ValueError( + "Unsupported skip_type {}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'" + .format(skip_type)) + + def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, + skip_type, t_T, t_0, + device): + if order == 3: + K = steps // 3 + 1 + if steps % 3 == 0: + orders = [ + 3, + ] * (K - 2) + [2, 1] + elif steps % 3 == 1: + orders = [ + 3, + ] * (K - 1) + [1] + else: + orders = [ + 3, + ] * (K - 1) + [2] + elif order == 2: + if steps % 2 == 0: + K = steps // 2 + orders = [ + 2, + ] * K + else: + K = steps // 2 + 1 + orders = [ + 2, + ] * (K - 1) + [1] + elif order == 1: + # Bug here. + # K = 1 + K = steps + orders = [ + 1, + ] * steps + else: + raise ValueError("'order' must be '1' or '2' or '3'.") + if skip_type == 'logSNR': + # To reproduce the results in DPM-Solver paper + timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, + device) + # TODO: bug here. + else: + timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, + device)[torch.cumsum( + torch.tensor([ + 0, + ] + orders)).to(device)] + return timesteps_outer, orders + + def denoise_to_zero_fn(self, x, s): + return self.data_prediction_fn(x, s) + + def dpm_solver_first_update(self, + x, + s, + t, + model_s=None, + return_intermediate=False): + ns = self.noise_schedule + dims = x.dim() + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + log_alpha_s, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_t = ns.marginal_std(s), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + if self.predict_x0: + phi_1 = torch.expm1(-h) + if model_s is None: + model_s = self.model_fn(x, s) + x_t = ( + expand_dims(sigma_t / sigma_s, dims) * x + - expand_dims(alpha_t * phi_1, dims) * model_s) + if return_intermediate: + return x_t, {'model_s': model_s} + else: + return x_t + else: + phi_1 = torch.expm1(h) + if model_s is None: + model_s = self.model_fn(x, s) + x_t = ( + expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x + - expand_dims(sigma_t * phi_1, dims) * model_s) + if return_intermediate: + return x_t, {'model_s': model_s} + else: + return x_t + + def singlestep_dpm_solver_second_update(self, + x, + s, + t, + r1=0.5, + model_s=None, + return_intermediate=False, + solver_type='dpm_solver'): + if solver_type not in ['dpm_solver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpm_solver' or 'taylor', got {}" + .format(solver_type)) + if r1 is None: + r1 = 0.5 + ns = self.noise_schedule + dims = x.dim() + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + lambda_s1 = lambda_s + r1 * h + s1 = ns.inverse_lambda(lambda_s1) + log_alpha_s, log_alpha_s1, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_s1, sigma_t = ns.marginal_std(s), ns.marginal_std( + s1), ns.marginal_std(t) + alpha_s1, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_t) + + if self.predict_x0: + phi_11 = torch.expm1(-r1 * h) + phi_1 = torch.expm1(-h) + + if model_s is None: + model_s = self.model_fn(x, s) + _a = expand_dims(sigma_s1 / sigma_s, dims) + _b = expand_dims(alpha_s1 * phi_11, dims) + x_s1 = (_a * x - _b * model_s) + model_s1 = self.model_fn(x_s1, s1) + if solver_type == 'dpm_solver': + _a = expand_dims(sigma_t / sigma_s, dims) + _b = expand_dims(alpha_t * phi_1, dims) + _c = expand_dims(alpha_t * phi_1, dims) + _d = (model_s1 - model_s) + x_t = (_a * x - _b * model_s - (0.5 / r1) * _c * _d) + elif solver_type == 'taylor': + _a = expand_dims(sigma_t / sigma_s, dims) + _b = expand_dims(alpha_t * phi_1, dims) + _c = expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), + dims) + _d = (model_s1 - model_s) + x_t = (_a * x - _b * model_s + (1. / r1) * _c * _d) + else: + phi_11 = torch.expm1(r1 * h) + phi_1 = torch.expm1(h) + + if model_s is None: + model_s = self.model_fn(x, s) + _a = expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) + _b = expand_dims(sigma_s1 * phi_11, dims) + x_s1 = (_a * x - _b * model_s) + model_s1 = self.model_fn(x_s1, s1) + if solver_type == 'dpm_solver': + _a = expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) + _b = expand_dims(sigma_t * phi_1, dims) + _c = expand_dims(sigma_t * phi_1, dims) + _d = (model_s1 - model_s) + x_t = (_a * x - _b * model_s - (0.5 / r1) * _c * _d) + elif solver_type == 'taylor': + _a = expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) + _b = expand_dims(sigma_t * phi_1, dims) + _c = expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), + dims) + _d = (model_s1 - model_s) + x_t = (_a * x - _b * model_s - (1. / r1) * _c * _d) + if return_intermediate: + return x_t, {'model_s': model_s, 'model_s1': model_s1} + else: + return x_t + + def singlestep_dpm_solver_third_update(self, + x, + s, + t, + r1=1. / 3., + r2=2. / 3., + model_s=None, + model_s1=None, + return_intermediate=False, + solver_type='dpm_solver'): + if solver_type not in ['dpm_solver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpm_solver' or 'taylor', got {}" + .format(solver_type)) + if r1 is None: + r1 = 1. / 3. + if r2 is None: + r2 = 2. / 3. + ns = self.noise_schedule + dims = x.dim() + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + lambda_s1 = lambda_s + r1 * h + lambda_s2 = lambda_s + r2 * h + s1 = ns.inverse_lambda(lambda_s1) + s2 = ns.inverse_lambda(lambda_s2) + log_alpha_s, log_alpha_s1, log_alpha_s2, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff( + s2), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_s1, sigma_s2, sigma_t = ns.marginal_std( + s), ns.marginal_std(s1), ns.marginal_std(s2), ns.marginal_std(t) + alpha_s1, alpha_s2, alpha_t = torch.exp(log_alpha_s1), torch.exp( + log_alpha_s2), torch.exp(log_alpha_t) + + if self.predict_x0: + phi_11 = torch.expm1(-r1 * h) + phi_12 = torch.expm1(-r2 * h) + phi_1 = torch.expm1(-h) + phi_22 = torch.expm1(-r2 * h) / (r2 * h) + 1. + phi_2 = phi_1 / h + 1. + phi_3 = phi_2 / h - 0.5 + + if model_s is None: + model_s = self.model_fn(x, s) + if model_s1 is None: + _a = expand_dims(sigma_s1 / sigma_s, dims) + _b = expand_dims(alpha_s1 * phi_11, dims) + x_s1 = (_a * x - _b * model_s) + model_s1 = self.model_fn(x_s1, s1) + _a = expand_dims(sigma_s2 / sigma_s, dims) + _b = expand_dims(alpha_s2 * phi_12, dims) + _c = expand_dims(alpha_s2 * phi_22, dims) + x_s2 = ( + _a * x - _b * model_s + r2 / r1 * _c * (model_s1 - model_s)) + model_s2 = self.model_fn(x_s2, s2) + if solver_type == 'dpm_solver': + _a = expand_dims(sigma_t / sigma_s, dims) + _b = expand_dims(alpha_t * phi_1, dims) + _c = expand_dims(alpha_t * phi_2, dims) + _d = (model_s2 - model_s) + x_t = (_a * x - _b * model_s + (1. / r2) * _c * _d) + elif solver_type == 'taylor': + D1_0 = (1. / r1) * (model_s1 - model_s) + D1_1 = (1. / r2) * (model_s2 - model_s) + D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1) + D2 = 2. * (D1_1 - D1_0) / (r2 - r1) + _a = expand_dims(sigma_t / sigma_s, dims) + _b = expand_dims(alpha_t * phi_1, dims) + _c = expand_dims(alpha_t * phi_2, dims) + _d = expand_dims(alpha_t * phi_3, dims) + x_t = (_a * x - _b * model_s + _c * D1 - _d * D2) + else: + phi_11 = torch.expm1(r1 * h) + phi_12 = torch.expm1(r2 * h) + phi_1 = torch.expm1(h) + phi_22 = torch.expm1(r2 * h) / (r2 * h) - 1. + phi_2 = phi_1 / h - 1. + phi_3 = phi_2 / h - 0.5 + + if model_s is None: + model_s = self.model_fn(x, s) + if model_s1 is None: + _a = expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) + _b = expand_dims(sigma_s1 * phi_11, dims) + x_s1 = (_a * x - _b * model_s) + model_s1 = self.model_fn(x_s1, s1) + _a = expand_dims(torch.exp(log_alpha_s2 - log_alpha_s), dims) + _b = expand_dims(sigma_s2 * phi_12, dims) + _c = expand_dims(sigma_s2 * phi_22, dims) + x_s2 = ( + _a * x - _b * model_s - r2 / r1 * _c * (model_s1 - model_s)) + model_s2 = self.model_fn(x_s2, s2) + if solver_type == 'dpm_solver': + _a = expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) + _b = expand_dims(sigma_t * phi_1, dims) + _c = expand_dims(sigma_t * phi_2, dims) + _d = (model_s2 - model_s) + x_t = (_a * x - _b * model_s - (1. / r2) * _c * _d) + elif solver_type == 'taylor': + D1_0 = (1. / r1) * (model_s1 - model_s) + D1_1 = (1. / r2) * (model_s2 - model_s) + D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1) + D2 = 2. * (D1_1 - D1_0) / (r2 - r1) + x_t = ( + expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x + - expand_dims(sigma_t * phi_1, dims) * model_s + - expand_dims(sigma_t * phi_2, dims) * D1 + - expand_dims(sigma_t * phi_3, dims) * D2) + + if return_intermediate: + return x_t, { + 'model_s': model_s, + 'model_s1': model_s1, + 'model_s2': model_s2 + } + else: + return x_t + + def multistep_dpm_solver_second_update(self, + x, + model_prev_list, + t_prev_list, + t, + solver_type='dpm_solver'): + if solver_type not in ['dpm_solver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpm_solver' or 'taylor', got {}" + .format(solver_type)) + ns = self.noise_schedule + dims = x.dim() + model_prev_1, model_prev_0 = model_prev_list[-2], model_prev_list[-1] + t_prev_1, t_prev_0 = t_prev_list[-2], t_prev_list[-1] + lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda( + t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t) + log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff( + t_prev_0), ns.marginal_log_mean_coeff(t) + sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + h_0 = lambda_prev_0 - lambda_prev_1 + h = lambda_t - lambda_prev_0 + r0 = h_0 / h + D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1) + if self.predict_x0: + if solver_type == 'dpm_solver': + _a = expand_dims(sigma_t / sigma_prev_0, dims) + _b = expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) + _c = expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) + x_t = (_a * x - _b * model_prev_0 - 0.5 * _c * D1_0) + elif solver_type == 'taylor': + _a = expand_dims(sigma_t / sigma_prev_0, dims) + _b = expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) + _c = expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), + dims) + x_t = (_a * x - _b * model_prev_0 + _c * D1_0) + else: + if solver_type == 'dpm_solver': + _a = expand_dims( + torch.exp(log_alpha_t - log_alpha_prev_0), dims) + _b = expand_dims(sigma_t * (torch.exp(h) - 1.), dims) + _c = expand_dims(sigma_t * (torch.exp(h) - 1.), dims) + x_t = (_a * x - _b * model_prev_0 - 0.5 * _c * D1_0) + elif solver_type == 'taylor': + _a = expand_dims( + torch.exp(log_alpha_t - log_alpha_prev_0), dims) + _b = expand_dims(sigma_t * (torch.exp(h) - 1.), dims) + _c = expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), + dims) + x_t = (_a * x - _b * model_prev_0 - _c * D1_0) + return x_t + + def multistep_dpm_solver_third_update(self, + x, + model_prev_list, + t_prev_list, + t, + solver_type='dpm_solver'): + ns = self.noise_schedule + dims = x.dim() + model_prev_2, model_prev_1, model_prev_0 = model_prev_list + t_prev_2, t_prev_1, t_prev_0 = t_prev_list + lambda_prev_2, lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda( + t_prev_2), ns.marginal_lambda(t_prev_1), ns.marginal_lambda( + t_prev_0), ns.marginal_lambda(t) + log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff( + t_prev_0), ns.marginal_log_mean_coeff(t) + sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + h_1 = lambda_prev_1 - lambda_prev_2 + h_0 = lambda_prev_0 - lambda_prev_1 + h = lambda_t - lambda_prev_0 + r0, r1 = h_0 / h, h_1 / h + D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1) + D1_1 = expand_dims(1. / r1, dims) * (model_prev_1 - model_prev_2) + D1 = D1_0 + expand_dims(r0 / (r0 + r1), dims) * (D1_0 - D1_1) + D2 = expand_dims(1. / (r0 + r1), dims) * (D1_0 - D1_1) + if self.predict_x0: + _a = expand_dims(sigma_t / sigma_prev_0, dims) + _b = expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) + _c = expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) + _d = expand_dims(alpha_t * ((torch.exp(-h) - 1. + h) / h**2 - 0.5), + dims) + x_t = (_a * x - _b * model_prev_0 + _c * D1 - _d * D2) + else: + _a = expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) + _b = expand_dims(sigma_t * (torch.exp(h) - 1.), dims) + _c = expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) + _d = expand_dims(sigma_t * ((torch.exp(h) - 1. - h) / h**2 - 0.5), + dims) + x_t = (_a * x - _b * model_prev_0 - _c * D1 - _d * D2) + return x_t + + def singlestep_dpm_solver_update(self, + x, + s, + t, + order, + return_intermediate=False, + solver_type='dpm_solver', + r1=None, + r2=None): + if order == 1: + return self.dpm_solver_first_update( + x, s, t, return_intermediate=return_intermediate) + elif order == 2: + return self.singlestep_dpm_solver_second_update( + x, + s, + t, + return_intermediate=return_intermediate, + solver_type=solver_type, + r1=r1) + elif order == 3: + return self.singlestep_dpm_solver_third_update( + x, + s, + t, + return_intermediate=return_intermediate, + solver_type=solver_type, + r1=r1, + r2=r2) + else: + raise ValueError( + 'Solver order must be 1 or 2 or 3, got {}'.format(order)) + + def multistep_dpm_solver_update(self, + x, + model_prev_list, + t_prev_list, + t, + order, + solver_type='dpm_solver'): + if order == 1: + return self.dpm_solver_first_update( + x, t_prev_list[-1], t, model_s=model_prev_list[-1]) + elif order == 2: + return self.multistep_dpm_solver_second_update( + x, model_prev_list, t_prev_list, t, solver_type=solver_type) + elif order == 3: + return self.multistep_dpm_solver_third_update( + x, model_prev_list, t_prev_list, t, solver_type=solver_type) + else: + raise ValueError( + 'Solver order must be 1 or 2 or 3, got {}'.format(order)) + + def dpm_solver_adaptive(self, + x, + order, + t_T, + t_0, + h_init=0.05, + atol=0.0078, + rtol=0.05, + theta=0.9, + t_err=1e-5, + solver_type='dpm_solver'): + ns = self.noise_schedule + s = t_T * torch.ones((x.shape[0], )).to(x) + lambda_s = ns.marginal_lambda(s) + lambda_0 = ns.marginal_lambda(t_0 * torch.ones_like(s).to(x)) + h = h_init * torch.ones_like(s).to(x) + x_prev = x + nfe = 0 + if order == 2: + r1 = 0.5 + + def lower_update(x, s, t): + return self.dpm_solver_first_update( + x, s, t, return_intermediate=True) + + def higher_update(x, s, t, **kwargs): + self.singlestep_dpm_solver_second_update( + x, s, t, r1=r1, solver_type=solver_type, **kwargs) + elif order == 3: + r1, r2 = 1. / 3., 2. / 3. + + def lower_update(x, s, t): + self.singlestep_dpm_solver_second_update( + x, + s, + t, + r1=r1, + return_intermediate=True, + solver_type=solver_type) + + def higher_update(x, s, t, **kwargs): + self.singlestep_dpm_solver_third_update( + x, s, t, r1=r1, r2=r2, solver_type=solver_type, **kwargs) + else: + raise ValueError( + 'For adaptive step size solver, order must be 2 or 3, got {}'. + format(order)) + while torch.abs((s - t_0)).mean() > t_err: + t = ns.inverse_lambda(lambda_s + h) + x_lower, lower_noise_kwargs = lower_update(x, s, t) + x_higher = higher_update(x, s, t, **lower_noise_kwargs) + delta = torch.max( + torch.ones_like(x).to(x) * atol, + rtol * torch.max(torch.abs(x_lower), torch.abs(x_prev))) + + def norm_fn(v): + return torch.sqrt( + torch.square(v.reshape( + (v.shape[0], -1))).mean(dim=-1, keepdim=True)) + + E = norm_fn((x_higher - x_lower) / delta).max() + if torch.all(E <= 1.): + x = x_higher + s = t + x_prev = x_lower + lambda_s = ns.marginal_lambda(s) + h = torch.min( + theta * h * torch.float_power(E, -1. / order).float(), + lambda_0 - lambda_s) + nfe += order + print('adaptive solver nfe', nfe) + return x + + def sample( + self, + x, + steps=20, + t_start=None, + t_end=None, + order=3, + skip_type='time_uniform', + method='singlestep', + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpm_solver', + atol=0.0078, + rtol=0.05, + ): + t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end + t_T = self.noise_schedule.T if t_start is None else t_start + device = x.device + if method == 'adaptive': + with torch.no_grad(): + x = self.dpm_solver_adaptive( + x, + order=order, + t_T=t_T, + t_0=t_0, + atol=atol, + rtol=rtol, + solver_type=solver_type) + elif method == 'multistep': + assert steps >= order + timesteps = self.get_time_steps( + skip_type=skip_type, t_T=t_T, t_0=t_0, N=steps, device=device) + assert timesteps.shape[0] - 1 == steps + with torch.no_grad(): + vec_t = timesteps[0].expand((x.shape[0])) + model_prev_list = [self.model_fn(x, vec_t)] + t_prev_list = [vec_t] + # Init the first `order` values by lower order multistep DPM-Solver. + for init_order in range(1, order): + vec_t = timesteps[init_order].expand(x.shape[0]) + x = self.multistep_dpm_solver_update( + x, + model_prev_list, + t_prev_list, + vec_t, + init_order, + solver_type=solver_type) + model_prev_list.append(self.model_fn(x, vec_t)) + t_prev_list.append(vec_t) + # Compute the remaining values by `order`-th order multistep DPM-Solver. + for step in range(order, steps + 1): + vec_t = timesteps[step].expand(x.shape[0]) + if lower_order_final and steps < 15: + step_order = min(order, steps + 1 - step) + else: + step_order = order + x = self.multistep_dpm_solver_update( + x, + model_prev_list, + t_prev_list, + vec_t, + step_order, + solver_type=solver_type) + for i in range(order - 1): + t_prev_list[i] = t_prev_list[i + 1] + model_prev_list[i] = model_prev_list[i + 1] + t_prev_list[-1] = vec_t + # We do not need to evaluate the final model value. + if step < steps: + model_prev_list[-1] = self.model_fn(x, vec_t) + elif method in ['singlestep', 'singlestep_fixed']: + if method == 'singlestep': + timesteps_outer, orders = self.get_orders_and_timesteps_for_singlestep_solver( + steps=steps, + order=order, + skip_type=skip_type, + t_T=t_T, + t_0=t_0, + device=device) + elif method == 'singlestep_fixed': + K = steps // order + orders = [ + order, + ] * K + timesteps_outer = self.get_time_steps( + skip_type=skip_type, t_T=t_T, t_0=t_0, N=K, device=device) + for i, order in enumerate(orders): + t_T_inner, t_0_inner = timesteps_outer[i], timesteps_outer[i + + 1] + timesteps_inner = self.get_time_steps( + skip_type=skip_type, + t_T=t_T_inner.item(), + t_0=t_0_inner.item(), + N=order, + device=device) + lambda_inner = self.noise_schedule.marginal_lambda( + timesteps_inner) + vec_s, vec_t = t_T_inner.tile(x.shape[0]), t_0_inner.tile( + x.shape[0]) + h = lambda_inner[-1] - lambda_inner[0] + r1 = None if order <= 1 else (lambda_inner[1] + - lambda_inner[0]) / h + r2 = None if order <= 2 else (lambda_inner[2] + - lambda_inner[0]) / h + x = self.singlestep_dpm_solver_update( + x, + vec_s, + vec_t, + order, + solver_type=solver_type, + r1=r1, + r2=r2) + if denoise_to_zero: + x = self.denoise_to_zero_fn( + x, + torch.ones((x.shape[0], )).to(device) * t_0) + return x + + +def interpolate_fn(x, xp, yp): + N, K = x.shape[0], xp.shape[1] + all_x = torch.cat( + [x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2) + sorted_all_x, x_indices = torch.sort(all_x, dim=2) + x_idx = torch.argmin(x_indices, dim=2) + cand_start_idx = x_idx - 1 + start_idx = torch.where( + torch.eq(x_idx, 0), + torch.tensor(1, device=x.device), + torch.where( + torch.eq(x_idx, K), + torch.tensor(K - 2, device=x.device), + cand_start_idx, + ), + ) + end_idx = torch.where( + torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1) + start_x = torch.gather( + sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2) + end_x = torch.gather( + sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2) + start_idx2 = torch.where( + torch.eq(x_idx, 0), + torch.tensor(0, device=x.device), + torch.where( + torch.eq(x_idx, K), + torch.tensor(K - 2, device=x.device), + cand_start_idx, + ), + ) + y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1) + start_y = torch.gather( + y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2) + end_y = torch.gather( + y_positions_expanded, dim=2, + index=(start_idx2 + 1).unsqueeze(2)).squeeze(2) + cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x) + return cand + + +def expand_dims(v, dims): + return v[(..., ) + (None, ) * (dims - 1)] diff --git a/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/__init__.py b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..42e2b7008dbc6cab3278d4fc42e4e89b4770b73d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .efficient_stable_diffusion import EfficientStableDiffusion + +else: + _import_structure = { + 'efficient_stable_diffusion': ['EfficientStableDiffusion'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa588d30e6629925068637aaa4bf555b49bae34 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py @@ -0,0 +1,917 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# The implementation is adopted from HighCWu, +# made pubicly available under the Apache License 2.0 License at https://github.com/HighCWu/ControlLoRA + +import os +from dataclasses import dataclass +from typing import List, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention_processor import Attention +from diffusers.models.lora import LoRALinearLayer +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.resnet import (Downsample2D, Upsample2D, downsample_2d, + partial, upsample_2d) +from diffusers.models.unet_2d_blocks import \ + get_down_block as get_down_block_default +from diffusers.utils.outputs import BaseOutput + +from .sd_lora import LoRACrossAttnProcessor + + +@dataclass +class ControlLoRAOutput(BaseOutput): + control_states: Tuple[torch.FloatTensor] + + +class ControlLoRATuner(ModelMixin, ConfigMixin): + """ The implementation of control lora module. + This module conduct encoding operation for control-condition and use lora to perform efficient tuning. + """ + + @staticmethod + def tune( + model: nn.Module, + tuner_config=None, + pretrained_tuner=None, + ): + tuner = ControlLoRATuner.from_config(tuner_config) + if pretrained_tuner is not None and os.path.exists(pretrained_tuner): + tuner.load_state_dict( + torch.load(pretrained_tuner, map_location='cpu'), strict=True) + + tune_layers_list = list( + [list(layer_list) for layer_list in tuner.lora_layers]) + + assert hasattr(model, 'unet') + unet = model.unet + tuner.to(unet.device) + tune_attn_procs = tuner.set_tune_layers(unet, tune_layers_list) + unet.set_attn_processor(tune_attn_procs) + return tuner + + def set_tune_layers(self, unet, tune_layers_list): + n_ch = len(unet.config.block_out_channels) + control_ids = [i for i in range(n_ch)] + tune_attn_procs = {} + + for name in unet.attn_processors.keys(): + if name.startswith('mid_block'): + control_id = control_ids[-1] + elif name.startswith('up_blocks'): + block_id = int(name[len('up_blocks.')]) + control_id = list(reversed(control_ids))[block_id] + elif name.startswith('down_blocks'): + block_id = int(name[len('down_blocks.')]) + control_id = control_ids[block_id] + + tune_layers = tune_layers_list[control_id] + if len(tune_layers) != 0: + tune_layer = tune_layers.pop(0) + tune_attn_procs[name] = tune_layer + return tune_attn_procs + + @register_to_config + def __init__(self, + in_channels: int = 3, + down_block_types: Tuple[str] = ( + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + ), + block_out_channels: Tuple[int] = (32, 64, 128, 256), + layers_per_block: int = 1, + act_fn: str = 'silu', + norm_num_groups: int = 32, + lora_pre_down_block_types: Tuple[str] = ( + None, + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + ), + lora_pre_down_layers_per_block: int = 1, + lora_pre_conv_skipped: bool = False, + lora_pre_conv_types: Tuple[str] = ( + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + 'SimpleDownEncoderBlock2D', + ), + lora_pre_conv_layers_per_block: int = 1, + lora_pre_conv_layers_kernel_size: int = 1, + lora_block_in_channels: Tuple[int] = (256, 256, 256, 256), + lora_block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + lora_cross_attention_dims: Tuple[List[int]] = ([ + None, 768, None, 768, None, 768, None, 768, None, 768 + ], [None, 768, None, 768, None, 768, None, 768, None, 768], [ + None, 768, None, 768, None, 768, None, 768, None, 768 + ], [None, 768]), + lora_rank: int = 4, + lora_control_rank: int = None, + lora_post_add: bool = False, + lora_concat_hidden: bool = False, + lora_control_channels: Tuple[int] = (None, None, None, None), + lora_control_self_add: bool = True, + lora_key_states_skipped: bool = False, + lora_value_states_skipped: bool = False, + lora_output_states_skipped: bool = False, + lora_control_version: int = 1): + """ Initialize a control lora module instance. + Args: + in_channels (`int`): The number of channels for input conditional data. + down_block_types (Tuple[str], *optional*): + The down block types for conditional data's downsample operation. + block_out_channels (Tuple[int], *optional*, defaults to (32, 64, 128, 256)): + The number of channels for every down-block. + layers_per_block (`int`, *optional*, defaults to 1): + The number of layers of every block. + act_fn (`str`, *optional*, defaults to silu): + The activation function. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for norm operation. + lora_pre_down_block_types (Tuple[str], *optional*): + The block'types for pre down-block. + lora_pre_down_layers_per_block (`int`, *optional*, defaults to 1) + The number of layers of every pre down-block block. + lora_pre_conv_skipped ('bool', *optional*, defaults to False ) + Set to True to skip conv in pre downsample. + lora_pre_conv_types (Tuple[str], *optional*): + The block'types for pre conv. + lora_pre_conv_layers_per_block (`int`, *optional*, defaults to 1) + The number of layers of every pre conv block. + lora_pre_conv_layers_kernel_size (`int`, *optional*, defaults to 1) + The conv kernel size of pre conv block. + lora_block_in_channels (Tuple[int], *optional*, defaults to (256, 256, 256, 256)): + The number of input channels for lora block. + lora_block_out_channels (Tuple[int], *optional*, defaults to (256, 256, 256, 256)): + The number of output channels for lora block. + lora_rank (int, *optional*, defaults to 4): + The rank of lora block. + lora_control_rank (int, *optional*, defaults to 4): + The rank of lora block. + lora_post_add (`bool`, *optional*, defaults to False): + Set to `True`, conduct weighted adding operation after lora. + lora_concat_hidden (`bool`, *optional*, defaults to False): + Set to `True`, conduct concat operation for hidden embedding. + lora_control_channels (Tuple[int], *optional*, defaults to (None, None, None, None)): + The number of control channels. + lora_control_self_add (`bool`, *optional*, defaults to True): + Set to `True` to perform self attn add. + lora_key_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on key value. + value_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on value. + output_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on output value. + lora_control_version (int, *optional*, defaults to 1): + Use lora attn version: ControlLoRACrossAttnProcessor vs ControlLoRACrossAttnProcessorV2. + """ + + super().__init__() + lora_control_cls = ControlLoRACrossAttnProcessor + if lora_control_version == 2: + lora_control_cls = ControlLoRACrossAttnProcessorV2 + + assert lora_block_in_channels[0] == block_out_channels[-1] + + if lora_pre_conv_skipped: + lora_control_channels = lora_block_in_channels + lora_control_self_add = False + + self.layers_per_block = layers_per_block + self.lora_pre_down_layers_per_block = lora_pre_down_layers_per_block + self.lora_pre_conv_layers_per_block = lora_pre_conv_layers_per_block + + self.conv_in = torch.nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1) + + self.down_blocks = nn.ModuleList([]) + self.pre_lora_layers = nn.ModuleList([]) + self.lora_layers = nn.ModuleList([]) + + # pre_down + pre_down_blocks = [] + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + pre_down_block = get_down_block( + down_block_type, + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attn_num_head_channels=None, + temb_channels=None, + ) + pre_down_blocks.append(pre_down_block) + self.down_blocks.append(nn.Sequential(*pre_down_blocks)) + self.pre_lora_layers.append( + get_down_block( + lora_pre_conv_types[0], + num_layers=self.lora_pre_conv_layers_per_block, + in_channels=lora_block_in_channels[0], + out_channels=( + lora_block_out_channels[0] if lora_control_channels[0] is + None else lora_control_channels[0]), + add_downsample=False, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attn_num_head_channels=None, + temb_channels=None, + resnet_kernel_size=lora_pre_conv_layers_kernel_size, + ) if not lora_pre_conv_skipped else nn.Identity()) + self.lora_layers.append( + nn.ModuleList([ + lora_control_cls( + lora_block_out_channels[0], + cross_attention_dim=cross_attention_dim, + rank=lora_rank, + control_rank=lora_control_rank, + post_add=lora_post_add, + concat_hidden=lora_concat_hidden, + control_channels=lora_control_channels[0], + control_self_add=lora_control_self_add, + key_states_skipped=lora_key_states_skipped, + value_states_skipped=lora_value_states_skipped, + output_states_skipped=lora_output_states_skipped) + for cross_attention_dim in lora_cross_attention_dims[0] + ])) + + # down + output_channel = lora_block_in_channels[0] + for i, down_block_type in enumerate(lora_pre_down_block_types): + if i == 0: + continue + input_channel = output_channel + output_channel = lora_block_in_channels[i] + + down_block = get_down_block( + down_block_type, + num_layers=self.lora_pre_down_layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=True, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attn_num_head_channels=None, + temb_channels=None, + ) + self.down_blocks.append(down_block) + + self.pre_lora_layers.append( + get_down_block( + lora_pre_conv_types[i], + num_layers=self.lora_pre_conv_layers_per_block, + in_channels=output_channel, + out_channels=( + lora_block_out_channels[i] if lora_control_channels[i] + is None else lora_control_channels[i]), + add_downsample=False, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attn_num_head_channels=None, + temb_channels=None, + resnet_kernel_size=lora_pre_conv_layers_kernel_size, + ) if not lora_pre_conv_skipped else nn.Identity()) + self.lora_layers.append( + nn.ModuleList([ + lora_control_cls( + lora_block_out_channels[i], + cross_attention_dim=cross_attention_dim, + rank=lora_rank, + control_rank=lora_control_rank, + post_add=lora_post_add, + concat_hidden=lora_concat_hidden, + control_channels=lora_control_channels[i], + control_self_add=lora_control_self_add, + key_states_skipped=lora_key_states_skipped, + value_states_skipped=lora_value_states_skipped, + output_states_skipped=lora_output_states_skipped) + for cross_attention_dim in lora_cross_attention_dims[i] + ])) + + def forward(self, + x: torch.FloatTensor, + return_dict: bool = True) -> Union[ControlLoRAOutput, Tuple]: + lora_layer: ControlLoRACrossAttnProcessor + + orig_dtype = x.dtype + dtype = self.conv_in.weight.dtype + + h = x.to(dtype) + h = self.conv_in(h) + control_states_list = [] + + # down + for down_block, pre_lora_layer, lora_layer_list in zip( + self.down_blocks, self.pre_lora_layers, self.lora_layers): + h = down_block(h) + control_states = pre_lora_layer(h) + if isinstance(control_states, tuple): + control_states = control_states[0] + control_states = control_states.to(orig_dtype) + for lora_layer in lora_layer_list: + lora_layer.inject_control_states(control_states) + control_states_list.append(control_states) + + if not return_dict: + return tuple(control_states_list) + + return ControlLoRAOutput(control_states=tuple(control_states_list)) + + +def get_down_block( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + attn_num_head_channels, + resnet_groups=None, + cross_attention_dim=None, + downsample_padding=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift='default', + resnet_kernel_size=3, +): + down_block_type = down_block_type[7:] if down_block_type.startswith( + 'UNetRes') else down_block_type + if down_block_type == 'SimpleDownEncoderBlock2D': + return SimpleDownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + add_downsample=add_downsample, + convnet_eps=resnet_eps, + convnet_act_fn=resnet_act_fn, + convnet_groups=resnet_groups, + downsample_padding=downsample_padding, + convnet_time_scale_shift=resnet_time_scale_shift, + convnet_kernel_size=resnet_kernel_size) + else: + return get_down_block_default( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + attn_num_head_channels, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + # resnet_kernel_size=resnet_kernel_size + ) + + +class ControlLoRACrossAttnProcessor(LoRACrossAttnProcessor): + + def __init__(self, + hidden_size, + cross_attention_dim=None, + rank=4, + control_rank=None, + post_add=False, + concat_hidden=False, + control_channels=None, + control_self_add=True, + key_states_skipped=False, + value_states_skipped=False, + output_states_skipped=False, + **kwargs): + super().__init__( + hidden_size, + cross_attention_dim, + rank, + post_add=post_add, + key_states_skipped=key_states_skipped, + value_states_skipped=value_states_skipped, + output_states_skipped=output_states_skipped) + + control_rank = rank if control_rank is None else control_rank + control_channels = hidden_size if control_channels is None else control_channels + self.concat_hidden = concat_hidden + self.control_self_add = control_self_add if control_channels is None else False + self.control_states: torch.Tensor = None + + self.to_control = LoRALinearLayer( + control_channels + (hidden_size if concat_hidden else 0), + hidden_size, control_rank) + self.pre_loras: List[LoRACrossAttnProcessor] = [] + self.post_loras: List[LoRACrossAttnProcessor] = [] + + def inject_pre_lora(self, lora_layer): + self.pre_loras.append(lora_layer) + + def inject_post_lora(self, lora_layer): + self.post_loras.append(lora_layer) + + def inject_control_states(self, control_states): + self.control_states = control_states + + def process_control_states(self, hidden_states, scale=1.0): + control_states = self.control_states.to(hidden_states.dtype) + if hidden_states.ndim == 3 and control_states.ndim == 4: + batch, _, height, width = control_states.shape + control_states = control_states.permute(0, 2, 3, 1).reshape( + batch, height * width, -1) + self.control_states = control_states + _control_states = control_states + if self.concat_hidden: + b1, b2 = control_states.shape[0], hidden_states.shape[0] + if b1 != b2: + control_states = control_states[:, None].repeat( + 1, b2 // b1, *([1] * (len(control_states.shape) - 1))) + control_states = control_states.view(-1, + *control_states.shape[2:]) + _control_states = torch.cat([hidden_states, control_states], -1) + _control_states = scale * self.to_control(_control_states) + if self.control_self_add: + control_states = control_states + _control_states + else: + control_states = _control_states + + return control_states + + def __call__(self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + scale=1.0): + pre_lora: LoRACrossAttnProcessor + post_lora: LoRACrossAttnProcessor + assert self.control_states is not None + + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) + query = attn.to_q(hidden_states) + for pre_lora in self.pre_loras: + lora_in = query if pre_lora.post_add else hidden_states + if isinstance(pre_lora, ControlLoRACrossAttnProcessor): + lora_in = lora_in + pre_lora.process_control_states( + hidden_states, scale) + query = query + scale * pre_lora.to_q_lora(lora_in) + query = query + scale * self.to_q_lora( + (query if self.post_add else hidden_states) + + self.process_control_states(hidden_states, scale)) + for post_lora in self.post_loras: + lora_in = query if post_lora.post_add else hidden_states + if isinstance(post_lora, ControlLoRACrossAttnProcessor): + lora_in = lora_in + post_lora.process_control_states( + hidden_states, scale) + query = query + scale * post_lora.to_q_lora(lora_in) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + + key = attn.to_k(encoder_hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.key_states_skipped: + key = key + scale * pre_lora.to_k_lora( + key if pre_lora.post_add else encoder_hidden_states) + if not self.key_states_skipped: + key = key + scale * self.to_k_lora( + key if self.post_add else encoder_hidden_states) + for post_lora in self.post_loras: + if not post_lora.key_states_skipped: + key = key + scale * post_lora.to_k_lora( + key if post_lora.post_add else encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.value_states_skipped: + value = value + pre_lora.to_v_lora( + value if pre_lora.post_add else encoder_hidden_states) + if not self.value_states_skipped: + value = value + scale * self.to_v_lora( + value if self.post_add else encoder_hidden_states) + for post_lora in self.post_loras: + if not post_lora.value_states_skipped: + value = value + post_lora.to_v_lora( + value if post_lora.post_add else encoder_hidden_states) + + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + out = attn.to_out[0](hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.output_states_skipped: + out = out + scale * pre_lora.to_out_lora( + out if pre_lora.post_add else hidden_states) + out = out + scale * self.to_out_lora( + out if self.post_add else hidden_states) + for post_lora in self.post_loras: + if not post_lora.output_states_skipped: + out = out + scale * post_lora.to_out_lora( + out if post_lora.post_add else hidden_states) + hidden_states = out + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class ControlLoRACrossAttnProcessorV2(LoRACrossAttnProcessor): + + def __init__(self, + hidden_size, + cross_attention_dim=None, + rank=4, + control_rank=None, + control_channels=None, + **kwargs): + super().__init__( + hidden_size, + cross_attention_dim, + rank, + post_add=False, + key_states_skipped=True, + value_states_skipped=True, + output_states_skipped=False) + + control_rank = rank if control_rank is None else control_rank + control_channels = hidden_size if control_channels is None else control_channels + self.concat_hidden = True + self.control_self_add = False + self.control_states: torch.Tensor = None + + self.to_control = LoRALinearLayer(hidden_size + control_channels, + hidden_size, control_rank) + self.to_control_out = LoRALinearLayer(hidden_size + control_channels, + hidden_size, control_rank) + self.pre_loras: List[LoRACrossAttnProcessor] = [] + self.post_loras: List[LoRACrossAttnProcessor] = [] + + def inject_pre_lora(self, lora_layer): + self.pre_loras.append(lora_layer) + + def inject_post_lora(self, lora_layer): + self.post_loras.append(lora_layer) + + def inject_control_states(self, control_states): + self.control_states = control_states + + def process_control_states(self, hidden_states, scale=1.0, is_out=False): + control_states = self.control_states.to(hidden_states.dtype) + if hidden_states.ndim == 3 and control_states.ndim == 4: + batch, _, height, width = control_states.shape + control_states = control_states.permute(0, 2, 3, 1).reshape( + batch, height * width, -1) + self.control_states = control_states + _control_states = control_states + if self.concat_hidden: + b1, b2 = control_states.shape[0], hidden_states.shape[0] + if b1 != b2: + control_states = control_states[:, None].repeat( + 1, b2 // b1, *([1] * (len(control_states.shape) - 1))) + control_states = control_states.view(-1, + *control_states.shape[2:]) + _control_states = torch.cat([hidden_states, control_states], -1) + _control_states = scale * (self.to_control_out + if is_out else self.to_control)( + _control_states) + if self.control_self_add: + control_states = control_states + _control_states + else: + control_states = _control_states + + return control_states + + def __call__(self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + scale=1.0): + pre_lora: LoRACrossAttnProcessor + post_lora: LoRACrossAttnProcessor + assert self.control_states is not None + + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) + for pre_lora in self.pre_loras: + if isinstance(pre_lora, ControlLoRACrossAttnProcessorV2): + hidden_states = hidden_states + pre_lora.process_control_states( + hidden_states, scale) + hidden_states = hidden_states + self.process_control_states( + hidden_states, scale) + for post_lora in self.post_loras: + if isinstance(post_lora, ControlLoRACrossAttnProcessorV2): + hidden_states = hidden_states + post_lora.process_control_states( + hidden_states, scale) + query = attn.to_q(hidden_states) + for pre_lora in self.pre_loras: + lora_in = query if pre_lora.post_add else hidden_states + query = query + scale * pre_lora.to_q_lora(lora_in) + query = query + scale * self.to_q_lora( + query if self.post_add else hidden_states) + for post_lora in self.post_loras: + lora_in = query if post_lora.post_add else hidden_states + query = query + scale * post_lora.to_q_lora(lora_in) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + + key = attn.to_k(encoder_hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.key_states_skipped: + key = key + scale * pre_lora.to_k_lora( + key if pre_lora.post_add else encoder_hidden_states) + if not self.key_states_skipped: + key = key + scale * self.to_k_lora( + key if self.post_add else encoder_hidden_states) + for post_lora in self.post_loras: + if not post_lora.key_states_skipped: + key = key + scale * post_lora.to_k_lora( + key if post_lora.post_add else encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.value_states_skipped: + value = value + pre_lora.to_v_lora( + value if pre_lora.post_add else encoder_hidden_states) + if not self.value_states_skipped: + value = value + scale * self.to_v_lora( + value if self.post_add else encoder_hidden_states) + for post_lora in self.post_loras: + if not post_lora.value_states_skipped: + value = value + post_lora.to_v_lora( + value if post_lora.post_add else encoder_hidden_states) + + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + for pre_lora in self.pre_loras: + if isinstance(pre_lora, ControlLoRACrossAttnProcessorV2): + hidden_states = hidden_states + pre_lora.process_control_states( + hidden_states, scale, is_out=True) + hidden_states = hidden_states + self.process_control_states( + hidden_states, scale, is_out=True) + for post_lora in self.post_loras: + if isinstance(post_lora, ControlLoRACrossAttnProcessorV2): + hidden_states = hidden_states + post_lora.process_control_states( + hidden_states, scale, is_out=True) + out = attn.to_out[0](hidden_states) + for pre_lora in self.pre_loras: + if not pre_lora.output_states_skipped: + out = out + scale * pre_lora.to_out_lora( + out if pre_lora.post_add else hidden_states) + out = out + scale * self.to_out_lora( + out if self.post_add else hidden_states) + for post_lora in self.post_loras: + if not post_lora.output_states_skipped: + out = out + scale * post_lora.to_out_lora( + out if post_lora.post_add else hidden_states) + hidden_states = out + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class ConvBlock2D(nn.Module): + + def __init__( + self, + *, + in_channels, + out_channels=None, + conv_kernel_size=3, + dropout=0.0, + temb_channels=512, + groups=32, + groups_out=None, + pre_norm=True, + eps=1e-6, + non_linearity='swish', + time_embedding_norm='default', + kernel=None, + output_scale_factor=1.0, + up=False, + down=False, + ): + super().__init__() + self.pre_norm = pre_norm + self.pre_norm = True + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.time_embedding_norm = time_embedding_norm + self.up = up + self.down = down + self.output_scale_factor = output_scale_factor + + if groups_out is None: + groups_out = groups + + self.norm1 = torch.nn.GroupNorm( + num_groups=groups, num_channels=in_channels, eps=eps, affine=True) + + self.conv1 = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=conv_kernel_size, + stride=1, + padding=conv_kernel_size // 2) + + if temb_channels is not None: + if self.time_embedding_norm == 'default': + time_emb_proj_out_channels = out_channels + elif self.time_embedding_norm == 'scale_shift': + time_emb_proj_out_channels = out_channels * 2 + else: + raise ValueError( + f'unknown time_embedding_norm : {self.time_embedding_norm} ' + ) + + self.time_emb_proj = torch.nn.Linear(temb_channels, + time_emb_proj_out_channels) + else: + self.time_emb_proj = None + + self.norm2 = torch.nn.GroupNorm( + num_groups=groups_out, + num_channels=out_channels, + eps=eps, + affine=True) + self.dropout = torch.nn.Dropout(dropout) + + if non_linearity == 'swish': + self.nonlinearity = lambda x: F.silu(x) + elif non_linearity == 'mish': + self.nonlinearity = nn.Mish() + elif non_linearity == 'silu': + self.nonlinearity = nn.SiLU() + + self.upsample = self.downsample = None + if self.up: + if kernel == 'fir': + fir_kernel = (1, 3, 3, 1) + self.upsample = lambda x: upsample_2d(x, kernel=fir_kernel) + elif kernel == 'sde_vp': + self.upsample = partial( + F.interpolate, scale_factor=2.0, mode='nearest') + else: + self.upsample = Upsample2D(in_channels, use_conv=False) + elif self.down: + if kernel == 'fir': + fir_kernel = (1, 3, 3, 1) + self.downsample = lambda x: downsample_2d(x, kernel=fir_kernel) + elif kernel == 'sde_vp': + self.downsample = partial( + F.avg_pool2d, kernel_size=2, stride=2) + else: + self.downsample = Downsample2D( + in_channels, use_conv=False, padding=1, name='op') + + def forward(self, input_tensor, temb): + hidden_states = input_tensor + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + + if self.upsample is not None: + # upsample_nearest_nhwc fails with large batch sizes. + # see https://github.com/huggingface/diffusers/issues/984 + if hidden_states.shape[0] >= 64: + input_tensor = input_tensor.contiguous() + hidden_states = hidden_states.contiguous() + _ = self.upsample(input_tensor) + hidden_states = self.upsample(hidden_states) + elif self.downsample is not None: + _ = self.downsample(input_tensor) + hidden_states = self.downsample(hidden_states) + + hidden_states = self.conv1(hidden_states) + + if temb is not None: + temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, + None] + + if temb is not None and self.time_embedding_norm == 'default': + hidden_states = hidden_states + temb + + hidden_states = self.norm2(hidden_states) + + if temb is not None and self.time_embedding_norm == 'scale_shift': + scale, shift = torch.chunk(temb, 2, dim=1) + hidden_states = hidden_states * (1 + scale) + shift + + hidden_states = self.nonlinearity(hidden_states) + + output_tensor = self.dropout(hidden_states) + + return output_tensor + + +class SimpleDownEncoderBlock2D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + convnet_eps: float = 1e-6, + convnet_time_scale_shift: str = 'default', + convnet_act_fn: str = 'swish', + convnet_groups: int = 32, + convnet_pre_norm: bool = True, + convnet_kernel_size: int = 3, + output_scale_factor=1.0, + add_downsample=True, + downsample_padding=1, + ): + super().__init__() + convnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + convnets.append( + ConvBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=convnet_eps, + groups=convnet_groups, + dropout=dropout, + time_embedding_norm=convnet_time_scale_shift, + non_linearity=convnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=convnet_pre_norm, + conv_kernel_size=convnet_kernel_size, + )) + in_channels = in_channels if num_layers == 0 else out_channels + + self.convnets = nn.ModuleList(convnets) + + if add_downsample: + self.downsamplers = nn.ModuleList([ + Downsample2D( + in_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name='op') + ]) + else: + self.downsamplers = None + + def forward(self, hidden_states): + for convnet in self.convnets: + hidden_states = convnet(hidden_states, temb=None) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states diff --git a/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..79ac2c33bb145cf2f37ad744ea9175acaad8ed11 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -0,0 +1,327 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# The implementation is adopted from HighCWu, +# made pubicly available under the Apache License 2.0 License at https://github.com/HighCWu/ControlLoRA +import os +import os.path as osp +from functools import partial +from typing import Any, Callable, List, Mapping, Optional, Union + +import torch +import torch.nn.functional as F +from diffusers import (AutoencoderKL, DDPMScheduler, DiffusionPipeline, + DPMSolverMultistepScheduler, UNet2DConditionModel, + utils) +from diffusers.models import attention +from diffusers.utils import deprecation_utils +from transformers import CLIPTextModel, CLIPTokenizer + +from modelscope import snapshot_download +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.efficient_diffusion_tuning.sd_lora import \ + LoRATuner +from modelscope.outputs import OutputKeys +from modelscope.utils.checkpoint import save_checkpoint, save_configuration +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.import_utils import is_swift_available +from .control_sd_lora import ControlLoRATuner + +utils.deprecate = lambda *arg, **kwargs: None +deprecation_utils.deprecate = lambda *arg, **kwargs: None +attention.deprecate = lambda *arg, **kwargs: None + +__tuner_MAP__ = {'lora': LoRATuner, 'control_lora': ControlLoRATuner} + +if is_swift_available(): + from swift import AdapterConfig, LoRAConfig, PromptConfig, Swift + + +@MODELS.register_module( + Tasks.efficient_diffusion_tuning, + module_name=Models.efficient_diffusion_tuning) +class EfficientStableDiffusion(TorchModel): + """ The implementation of efficient diffusion tuning model based on TorchModel. + + This model is constructed with the implementation of stable diffusion model. If you want to + finetune lightweight parameters on your own dataset, you can define you own tuner module + and load in this cls. + """ + + def __init__(self, model_dir, *args, **kwargs): + """ Initialize a vision efficient diffusion tuning model. + + Args: + model_dir: model id or path, where model_dir/pytorch_model.bin + """ + super().__init__(model_dir, *args, **kwargs) + tuner_name = kwargs.pop('tuner_name', 'lora') + pretrained_model_name_or_path = kwargs.pop( + 'pretrained_model_name_or_path', + 'AI-ModelScope/stable-diffusion-v1-5') + pretrained_model_name_or_path = snapshot_download( + pretrained_model_name_or_path) + tuner_config = kwargs.pop('tuner_config', None) + pretrained_tuner = kwargs.pop('pretrained_tuner', None) + revision = kwargs.pop('revision', None) + inference = kwargs.pop('inference', True) + + if pretrained_tuner is not None: + pretrained_tuner = osp.join(model_dir, pretrained_tuner) + + self.weight_dtype = torch.float32 + self.inference = inference + + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + if self.inference: + self.pipe = DiffusionPipeline.from_pretrained( + pretrained_model_name_or_path, + revision=revision, + torch_dtype=self.weight_dtype, + safety_checker=None) + self.pipe.scheduler = DPMSolverMultistepScheduler.from_config( + self.pipe.scheduler.config) + self.pipe = self.pipe.to(self.device) + self.unet = self.pipe.unet + self.text_encoder = self.pipe.text_encoder + self.vae = self.pipe.vae + else: + # Load scheduler, tokenizer and models. + self.noise_scheduler = DDPMScheduler.from_pretrained( + pretrained_model_name_or_path, subfolder='scheduler') + self.tokenizer = CLIPTokenizer.from_pretrained( + pretrained_model_name_or_path, + subfolder='tokenizer', + revision=revision) + self.text_encoder = CLIPTextModel.from_pretrained( + pretrained_model_name_or_path, + subfolder='text_encoder', + revision=revision) + self.vae = AutoencoderKL.from_pretrained( + pretrained_model_name_or_path, + subfolder='vae', + revision=revision) + self.unet = UNet2DConditionModel.from_pretrained( + pretrained_model_name_or_path, + subfolder='unet', + revision=revision) + self.unet.requires_grad_(False) + self.vae.requires_grad_(False) + self.text_encoder.requires_grad_(False) + self.is_control = tuner_name.startswith('control_') + self.tuner_name = tuner_name + + if tuner_name == 'swift-lora': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) + rank = tuner_config[ + 'rank'] if tuner_config and 'rank' in tuner_config else 4 + lora_config = LoRAConfig( + r=rank, + target_modules=['to_q', 'to_k', 'to_v', 'to_out.0'], + merge_weights=False, + use_merged_linear=False) + self.unet = Swift.prepare_model(self.unet, lora_config) + elif tuner_name == 'swift-adapter': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) + adapter_length = tuner_config[ + 'adapter_length'] if tuner_config and 'adapter_length' in tuner_config else 10 + adapter_config_dict = {} + dim_list = [320, 640, 1280] + target_modules_list = [ + r'(down_blocks.0.*ff\.net\.2$)|(up_blocks.3.*ff\.net\.2$)', + r'(down_blocks.1.*ff\.net\.2$)|(up_blocks.2.*ff\.net\.2$)', + r'(down_blocks.2.*ff\.net\.2$)|(up_blocks.1.*ff\.net\.2$)|(mid_block.*ff\.net\.2$)' + ] + for dim, target_modules in zip(dim_list, target_modules_list): + adapter_config = AdapterConfig( + dim=dim, + hidden_pos=0, + target_modules=target_modules, + adapter_length=adapter_length) + adapter_config_dict[f'adapter_{dim}'] = adapter_config + self.unet = Swift.prepare_model(self.unet, adapter_config_dict) + elif tuner_name == 'swift-prompt': + if not is_swift_available(): + raise ValueError( + 'Please install swift by `pip install ms-swift` to use swift tuners.' + ) + prompt_length = tuner_config[ + 'prompt_length'] if tuner_config and 'prompt_length' in tuner_config else 10 + prompt_config = PromptConfig( + dim=[ + 320, 320, 640, 640, 1280, 1280, 1280, 1280, 1280, 640, 640, + 640, 320, 320, 320 + ], + target_modules= + r'.*[down_blocks|up_blocks|mid_block]\.\d+\.attentions\.\d+\.transformer_blocks\.\d+$', + embedding_pos=0, + prompt_length=prompt_length, + attach_front=False, + extract_embedding=True) + self.unet = Swift.prepare_model(self.unet, prompt_config) + elif tuner_name in ('lora', 'control_lora'): + # if not set the config of control-tuner, we add the lora tuner directly to the original framework, + # otherwise the control side network is also added. + tuner_cls = __tuner_MAP__[tuner_name] + tuner = tuner_cls.tune( + self, + tuner_config=osp.join(model_dir, tuner_config), + pretrained_tuner=pretrained_tuner) + self.tuner = tuner + + def train(self, mode: bool = True): + self.training = mode + if hasattr(self, 'tuner'): + self.tuner.train(mode=mode) + else: + super().train(mode=mode) + + def load_state_dict(self, + state_dict: Mapping[str, Any], + strict: bool = True): + if hasattr(self, 'tuner'): + self.tuner.load_state_dict(state_dict=state_dict, strict=strict) + else: + super().load_state_dict(state_dict=state_dict, strict=strict) + + def state_dict(self, *arg, **kwargs): + if hasattr(self, 'tuner'): + return self.tuner.state_dict(*arg, **kwargs) + elif self.tuner_name.startswith('swift-'): + return self.unet.state_dict(*arg, **kwargs) + else: + return super().state_dict(*arg, **kwargs) + + def tokenize_caption(self, captions): + """ Convert caption text to token data. + + Args: + captions: a batch of texts. + Returns: token's data as tensor. + """ + inputs = self.tokenizer( + captions, + max_length=self.tokenizer.model_max_length, + padding='max_length', + truncation=True, + return_tensors='pt') + return inputs.input_ids + + def forward(self, prompt, cond=None, target=None, **args): + if self.inference: + if 'generator_seed' in args and isinstance(args['generator_seed'], + int): + generator = torch.Generator(device=self.device).manual_seed( + args['generator_seed']) + else: + generator = None + num_inference_steps = args.get('num_inference_steps', 30) + guidance_scale = args.get('guidance_scale', 7.5) + if self.is_control: + _ = self.tuner(cond.to(self.device)).control_states + images = self.pipe( + prompt, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + generator=generator).images + return images + else: + with torch.no_grad(): + latents = self.vae.encode( + target.to(dtype=self.weight_dtype)).latent_dist.sample() + latents = latents * self.vae.config.scaling_factor + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.num_train_timesteps, (bsz, ), + device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = self.noise_scheduler.add_noise( + latents, noise, timesteps) + + input_ids = self.tokenize_caption(prompt).to(self.device) + + # Get the text embedding for conditioning + # with torch.no_grad(): + encoder_hidden_states = self.text_encoder(input_ids)[0] + + # Inject control states to unet + if self.is_control: + _ = self.tuner(cond.to(dtype=self.weight_dtype)).control_states + # else: + # tune_weights_list = self.tuner() + + # Get the target for loss depending on the prediction type + if self.noise_scheduler.config.prediction_type == 'epsilon': + target = noise + elif self.noise_scheduler.config.prediction_type == 'v_prediction': + target = self.noise_scheduler.get_velocity( + latents, noise, timesteps) + else: + raise ValueError( + f'Unknown prediction type {self.noise_scheduler.config.prediction_type}' + ) + + # Predict the noise residual and compute loss + model_pred = self.unet(noisy_latents, timesteps, + encoder_hidden_states).sample + loss = F.mse_loss( + model_pred.float(), target.float(), reduction='mean') + output = {OutputKeys.LOSS: loss} + return output + + def parameters(self, recurse: bool = True): + if hasattr(self, 'tuner'): + return self.tuner.parameters(recurse=recurse) + else: + return super().parameters(recurse=recurse) + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = partial( + save_checkpoint, with_meta=False), + config: Optional[dict] = None, + save_config_function: Callable = save_configuration, + **kwargs): + + if config is None and hasattr(self, 'cfg'): + config = self.cfg + + config['model']['inference'] = True + config['model']['pretrained_tuner'] = 'pytorch_model.bin' + super().save_pretrained(target_folder, save_checkpoint_names, + save_function, config, save_config_function, + **kwargs) + + @classmethod + def _instantiate(cls, model_dir, **kwargs): + config = Config.from_file(osp.join(model_dir, ModelFile.CONFIGURATION)) + for k, v in kwargs.items(): + config.model[k] = v + + model = EfficientStableDiffusion( + model_dir, + pretrained_model_name_or_path=config.model. + pretrained_model_name_or_path, + tuner_name=config.model.tuner_name, + tuner_config=config.model.tuner_config, + pretrained_tuner=config.model.get('pretrained_tuner', None), + inference=config.model.get('inference', False)) + model.config = config + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/sd_lora.py b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/sd_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..306ca2b0c7280ff9cc391bf583a898b4c0182a44 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/efficient_diffusion_tuning/sd_lora.py @@ -0,0 +1,219 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# The implementation is adopted from HighCWu, +# made pubicly available under the Apache License 2.0 License at https://github.com/HighCWu/ControlLoRA +import os +from dataclasses import dataclass +from typing import List, Tuple, Union + +import torch +import torch.nn as nn +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention_processor import Attention +from diffusers.models.lora import LoRALinearLayer +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils.outputs import BaseOutput + + +@dataclass +class TunerOutput(BaseOutput): + lora_states: Tuple[torch.FloatTensor] + + +class LoRACrossAttnProcessor(nn.Module): + """ The implementation of lora attention module. + """ + + def __init__(self, + hidden_size, + cross_attention_dim=None, + rank=4, + post_add=False, + key_states_skipped=False, + value_states_skipped=False, + output_states_skipped=False): + """ Initialize a lora attn instance. + Args: + hidden_size (`int`): The number of channels in embedding. + cross_attention_dim (`int`, *optional*): + The number of channels in the hidden_states. If not given, defaults to `hidden_size`. + rank (`int`, *optional*, defaults to 4): The number of rank of lora. + post_add (`bool`, *optional*, defaults to False): Set to `True`, conduct weighted + adding operation after lora. + key_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on key value. + value_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on value. + output_states_skipped (`bool`, *optional*, defaults to False): + Set to `True` for skip to perform lora on output value. + """ + super().__init__() + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.rank = rank + self.post_add = post_add + + self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank) + if not key_states_skipped: + self.to_k_lora = LoRALinearLayer( + hidden_size if post_add else + (cross_attention_dim or hidden_size), hidden_size, rank) + if not value_states_skipped: + self.to_v_lora = LoRALinearLayer( + hidden_size if post_add else + (cross_attention_dim or hidden_size), hidden_size, rank) + if not output_states_skipped: + self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank) + + self.key_states_skipped: bool = key_states_skipped + self.value_states_skipped: bool = value_states_skipped + self.output_states_skipped: bool = output_states_skipped + + def skip_key_states(self, is_skipped: bool = True): + if not is_skipped: + assert hasattr(self, 'to_k_lora') + self.key_states_skipped = is_skipped + + def skip_value_states(self, is_skipped: bool = True): + if not is_skipped: + assert hasattr(self, 'to_q_lora') + self.value_states_skipped = is_skipped + + def skip_output_states(self, is_skipped: bool = True): + if not is_skipped: + assert hasattr(self, 'to_out_lora') + self.output_states_skipped = is_skipped + + def __call__(self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + scale=1.0): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, + target_length=sequence_length, + batch_size=batch_size) + + query = attn.to_q(hidden_states) + query = query + scale * self.to_q_lora( + query if self.post_add else hidden_states) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + + key = attn.to_k(encoder_hidden_states) + if not self.key_states_skipped: + key = key + scale * self.to_k_lora( + key if self.post_add else encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + if not self.value_states_skipped: + value = value + scale * self.to_v_lora( + value if self.post_add else encoder_hidden_states) + + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + out = attn.to_out[0](hidden_states) + if not self.output_states_skipped: + out = out + scale * self.to_out_lora( + out if self.post_add else hidden_states) + hidden_states = out + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class LoRATuner(ModelMixin, ConfigMixin): + + @staticmethod + def tune( + model: nn.Module, + tuner_config=None, + pretrained_tuner=None, + ): + tuner = LoRATuner.from_config(tuner_config) + if pretrained_tuner is not None and os.path.exists(pretrained_tuner): + tuner.load_state_dict( + torch.load(pretrained_tuner, map_location='cpu'), strict=True) + tune_layers_list = list( + [list(layer_list) for layer_list in tuner.lora_layers]) + assert hasattr(model, 'unet') + unet = model.unet + tuner.to(unet.device) + tune_attn_procs = tuner.set_tune_layers(unet, tune_layers_list) + unet.set_attn_processor(tune_attn_procs) + return tuner + + def set_tune_layers(self, unet, tune_layers_list): + n_ch = len(unet.config.block_out_channels) + control_ids = [i for i in range(n_ch)] + tune_attn_procs = {} + + for name in unet.attn_processors.keys(): + if name.startswith('mid_block'): + control_id = control_ids[-1] + elif name.startswith('up_blocks'): + block_id = int(name[len('up_blocks.')]) + control_id = list(reversed(control_ids))[block_id] + elif name.startswith('down_blocks'): + block_id = int(name[len('down_blocks.')]) + control_id = control_ids[block_id] + + tune_layers = tune_layers_list[control_id] + if len(tune_layers) != 0: + tune_layer = tune_layers.pop(0) + tune_attn_procs[name] = tune_layer + return tune_attn_procs + + @register_to_config + def __init__( + self, + lora_block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + lora_cross_attention_dims: Tuple[List[int]] = ([ + None, 768, None, 768, None, 768, None, 768, None, 768 + ], [None, 768, None, 768, None, 768, None, 768, None, + 768], [None, 768, None, 768, None, 768, None, 768, None, + 768], [None, 768]), + lora_rank: int = 4, + lora_post_add: bool = False, + lora_key_states_skipped: bool = False, + lora_value_states_skipped: bool = False, + lora_output_states_skipped: bool = False, + ): + super().__init__() + + lora_cls = LoRACrossAttnProcessor + + self.lora_layers = nn.ModuleList([]) + + for i, lora_cross_attention_dim in enumerate( + lora_cross_attention_dims): + self.lora_layers.append( + nn.ModuleList([ + lora_cls( + lora_block_out_channels[i], + cross_attention_dim=cross_attention_dim, + rank=lora_rank, + post_add=lora_post_add, + key_states_skipped=lora_key_states_skipped, + value_states_skipped=lora_value_states_skipped, + output_states_skipped=lora_output_states_skipped) + for cross_attention_dim in lora_cross_attention_dim + ])) + + def forward(self) -> Union[TunerOutput, Tuple]: + lora_states_list = [] + tune_layers_list = list( + [list(layer_list) for layer_list in self.lora_layers]) + for tune_list in tune_layers_list: + for tune_layer in tune_list: + lora_states_list.append(tune_layer.to_q_lora.down.weight) + return TunerOutput(lora_states=tuple(lora_states_list)) diff --git a/runtime/Lib/site_packages/models/multi_modal/freeu/__init__.py b/runtime/Lib/site_packages/models/multi_modal/freeu/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3cd55cf30c665bce0ecf6a9a0f2b51c1e5f23814 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/freeu/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .free_lunch_utils import register_free_upblock2d, register_free_crossattn_upblock2d +else: + _import_structure = { + 'free_lunch_utils': + ['register_free_upblock2d', 'register_free_crossattn_upblock2d'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/freeu/free_lunch_utils.py b/runtime/Lib/site_packages/models/multi_modal/freeu/free_lunch_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eb5d191fffcfc6c0edcd1eb5f176fa4b0aaac3cd --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/freeu/free_lunch_utils.py @@ -0,0 +1,331 @@ +# ------------------------------------------------------------------------ +# Modified from https://github.com/ChenyangSi/FreeU/blob/main/demo/free_lunch_utils.py +# Copyright (c) 2023 TencentARC. All Rights Reserved. +# ------------------------------------------------------------------------ + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.fft as fft +from diffusers.utils import is_torch_version + + +def isinstance_str(x: object, cls_name: str): + """ + Checks whether x has any class *named* cls_name in its ancestry. + Doesn't require access to the class's implementation. + + Useful for patching! + """ + + for _cls in x.__class__.__mro__: + if _cls.__name__ == cls_name: + return True + + return False + + +def Fourier_filter(x, threshold, scale): + dtype = x.dtype + x = x.type(torch.float32) + # FFT + x_freq = fft.fftn(x, dim=(-2, -1)) + x_freq = fft.fftshift(x_freq, dim=(-2, -1)) + + B, C, H, W = x_freq.shape + mask = torch.ones((B, C, H, W)).cuda() + + crow, ccol = H // 2, W // 2 + mask[..., crow - threshold:crow + threshold, + ccol - threshold:ccol + threshold] = scale + x_freq = x_freq * mask + + # IFFT + x_freq = fft.ifftshift(x_freq, dim=(-2, -1)) + x_filtered = fft.ifftn(x_freq, dim=(-2, -1)).real + + x_filtered = x_filtered.type(dtype) + return x_filtered + + +def register_upblock2d(model): + + def up_forward(self): + + def forward(hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, 'UpBlock2D'): + upsample_block.forward = up_forward(upsample_block) + + +def register_free_upblock2d(model, b1=1.2, b2=1.4, s1=0.9, s2=0.2): + + def up_forward(self): + + def forward(hidden_states, + res_hidden_states_tuple, + temb=None, + upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # --------------- FreeU code ----------------------- + # Only operate on the first two stages + if hidden_states.shape[1] == 1280: + hidden_states[:, :640] = hidden_states[:, :640] * self.b1 + res_hidden_states = Fourier_filter( + res_hidden_states, threshold=1, scale=self.s1) + if hidden_states.shape[1] == 640: + hidden_states[:, :320] = hidden_states[:, :320] * self.b2 + res_hidden_states = Fourier_filter( + res_hidden_states, threshold=1, scale=self.s2) + # --------------------------------------------------------- + + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version('>=', '1.11.0'): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, 'UpBlock2D'): + upsample_block.forward = up_forward(upsample_block) + setattr(upsample_block, 'b1', b1) + setattr(upsample_block, 'b2', b2) + setattr(upsample_block, 's1', s1) + setattr(upsample_block, 's2', s2) + + +def register_crossattn_upblock2d(model): + + def up_forward(self): + + def forward( + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, 'CrossAttnUpBlock2D'): + upsample_block.forward = up_forward(upsample_block) + + +def register_free_crossattn_upblock2d(model, b1=1.2, b2=1.4, s1=0.9, s2=0.2): + + def up_forward(self): + + def forward( + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # --------------- FreeU code ----------------------- + # Only operate on the first two stages + if hidden_states.shape[1] == 1280: + hidden_states[:, :640] = hidden_states[:, :640] * self.b1 + res_hidden_states = Fourier_filter( + res_hidden_states, threshold=1, scale=self.s1) + if hidden_states.shape[1] == 640: + hidden_states[:, :320] = hidden_states[:, :320] * self.b2 + res_hidden_states = Fourier_filter( + res_hidden_states, threshold=1, scale=self.s2) + # --------------------------------------------------------- + + hidden_states = torch.cat([hidden_states, res_hidden_states], + dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = { + 'use_reentrant': False + } if is_torch_version('>=', '1.11.0') else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, 'CrossAttnUpBlock2D'): + upsample_block.forward = up_forward(upsample_block) + setattr(upsample_block, 'b1', b1) + setattr(upsample_block, 'b2', b2) + setattr(upsample_block, 's1', s1) + setattr(upsample_block, 's2', s2) diff --git a/runtime/Lib/site_packages/models/multi_modal/gemm/__init__.py b/runtime/Lib/site_packages/models/multi_modal/gemm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fe5df1fe031daa87c62392222f2fd9ad0aa37588 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/gemm/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .gemm_model import GEMMForMultiModalEmbedding diff --git a/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_base.py b/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_base.py new file mode 100644 index 0000000000000000000000000000000000000000..c77a682ab674b998e42f931c4a3ad1aaacca81c9 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_base.py @@ -0,0 +1,558 @@ +# Copyright 2021 The OpenAI Team Authors. +# Copyright 2022 Phil Wang. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# +# The implementation here is modified based on OpenAI CLIP, +# originally MIT License, Copyright (c) 2021 OpenAI, +# and publicly available at https://github.com/openai/CLIP/. +# The implementation here is modified based on Coca-pytorch, +# originally MIT License, Copyright (c) 2022 Phil Wang, +# and publicly available at https://github.com/lucidrains/CoCa-pytorch/, +""" Generative Multimodal Model Architecture.""" + +import os +from collections import OrderedDict +from typing import Tuple, Union + +import json +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import LayerNorm + +from modelscope.models.multi_modal.gemm.tokenizer import (SimpleTokenizer, + clip_tokenize) + + +class Bottleneck(nn.Module): + """ ResNet style bottleneck module + From https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + if stride > 1 or inplanes != planes * Bottleneck.expansion: + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + if self.downsample is not None: + identity = self.downsample(x) + out += identity + out = self.relu(out) + return out + + +class QuickGELU(nn.Module): + """ A quick version of GELU module + From https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + """ Multihead attention block with residual link + Adapted from https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + attn_mask = self.attn_mask + if attn_mask is not None and attn_mask.shape[0] > x.shape[0]: + attn_mask = self.attn_mask[:x.shape[0], :x.shape[0]] + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + """ Transformer encoder module + Adapted from https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None, + use_gc: bool = False): + super().__init__() + self.use_gc = use_gc + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class AttentionPool2d(nn.Module): + """ Pool layer with attention module + Adapted from https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, 1) + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) + x = x + self.positional_embedding[:, None, :].to(x.dtype) + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + return x.permute(1, 0, 2).contiguous() + + +class CrossAttention(nn.Module): + """ Cross attention module with query and context as input + Adapted from https://github.com/lucidrains/CoCa-pytorch/blob/main/coca_pytorch/coca_pytorch.py + """ + + def __init__(self, + dim, + *, + context_dim=None, + dim_head=64, + heads=8, + parallel_ff=False, + ff_mult=4, + norm_context=False): + super().__init__() + self.heads = heads + self.scale = dim_head**-0.5 + inner_dim = heads * dim_head + context_dim = dim if context_dim is None else context_dim + self.norm = LayerNorm(dim) + self.context_norm = LayerNorm( + context_dim) if norm_context else nn.Identity() + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(context_dim, dim_head * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + ff_inner_dim = ff_mult * dim + self.ff = nn.Sequential( + nn.Linear(dim, ff_inner_dim * 2, bias=False), SwiGLU(), + nn.Linear(ff_inner_dim, dim, bias=False)) if parallel_ff else None + + def forward(self, x, context): + """ + einstein notation + b - batch + h - heads + n, i, j - sequence length (base sequence length, source, target) + d - feature dimension + """ + + x = self.norm(x) + context = self.context_norm(context) + + q = self.to_q(x) + q = q.view(q.shape[0], q.shape[1], self.heads, + -1).permute(0, 2, 1, 3).contiguous() + q = q * self.scale + k, v = self.to_kv(context).chunk(2, dim=-1) + sim = torch.einsum('b h i d, b j d -> b h i j', q, k) + sim = sim - sim.amax(dim=-1, keepdim=True) + attn = sim.softmax(dim=-1) + out = torch.einsum('b h i j, b j d -> b h i d', attn, v) + out = out.permute(0, 2, 1, + 3).contiguous().reshape(out.shape[0], out.shape[2], + -1) + out = self.to_out(out) + if self.ff is not None: + out = out + self.ff(x) + return out + + +class ModifiedResNet(nn.Module): + """ Modified ResNet backbone + From https://github.com/openai/CLIP/blob/main/clip/model.py + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + self._inplanes = width + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class VisualTransformer(nn.Module): + """ ViT transformer backbone + From https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int, use_gc: bool): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + self.transformer = Transformer(width, layers, heads, use_gc=use_gc) + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) + x = x.reshape(x.shape[0], x.shape[1], -1) + x = x.permute(0, 2, 1) + z = torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([self.class_embedding.to(x.dtype) + z, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + x = x.permute(1, 0, 2) + x = self.transformer(x) + x = x.permute(1, 0, 2) + x = self.ln_post(x) + if self.proj is not None: + x = x @ self.proj + return x + + +class GEVL(nn.Module): + """ Generative vision-language model + Support learning from both generative and contrastive loss. + Given image and text input, it could output the features of + image and text respectively. Furthermore, caption could also + be produced when image input is available. + """ + + def __init__(self, embed_dim: int, image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], + int], vision_width: int, + vision_patch_size: int, context_length: int, vocab_size: int, + transformer_width: int, transformer_heads: int, + transformer_layers: int, use_gc: bool, tokenizer): + nn.Module.__init__(self) + self.context_length = context_length + self.vis_token_size = context_length + self.tokenizer = tokenizer + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim, + use_gc=use_gc) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask(), + use_gc=use_gc) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.vis_token_projection = nn.Parameter( + torch.empty(embed_dim, transformer_width)) + nn.init.normal_( + self.vis_token_projection, std=self.transformer.width**-0.5) + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + self.decoder = Transformer( + width=transformer_width, + layers=4, + heads=transformer_heads, + attn_mask=self.build_attention_mask( + self.vis_token_size + self.context_length, + self.vis_token_size), + use_gc=use_gc) + self.to_logits = nn.Sequential( + LayerNorm(transformer_width), + nn.Linear(transformer_width, transformer_width), + nn.Linear(transformer_width, vocab_size, bias=False)) + self.gen_logit_scale = nn.Parameter( + torch.ones([]) * np.log(np.log(vocab_size))) + self.bias = nn.Parameter(torch.ones(vocab_size)) + self.to_logits[-1].weight = self.token_embedding.weight + self.to_logits[-1].bias = self.bias + self.img_queries = nn.Parameter( + torch.randn(self.vis_token_size, transformer_width)) + self.img_attn_pool = CrossAttention( + dim=transformer_width, norm_context=True) + self.img_attn_pool_norm = LayerNorm(transformer_width) + + def build_attention_mask(self, seq_length=None, prefix_length=0): + seq_length = self.context_length if seq_length is None else seq_length + mask = torch.empty(seq_length, seq_length) + mask.fill_(torch.tensor(torch.finfo(torch.float16).min)) + mask.triu_(1) + if prefix_length > 0: + mask[:prefix_length, :prefix_length] = 0 + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image, return_tokens=False): + image_outputs = self.visual(image) + image_features = image_outputs[:, 0, :] + image_features = image_features / image_features.norm( + dim=-1, p=2, keepdim=True) + if return_tokens: + image_tokens = image_outputs[:, 1:, :] @ self.vis_token_projection + return image_features, image_tokens + else: + return image_features + + def encode_text(self, text, return_tokens=False): + x = self.token_embedding(text) + x = x + self.positional_embedding[:x.shape[1], :] + x = x.permute(1, 0, 2) + x = self.transformer(x) + x = x.permute(1, 0, 2) + x = self.ln_final(x) + text_features = x[torch.arange(x.shape[0]), + text.argmax(dim=-1), ...] @ self.text_projection + text_features = text_features / text_features.norm( + dim=-1, p=2, keepdim=True) + if return_tokens: + text_tokens = x + return text_features, text_tokens + else: + return text_features + + def image_to_text(self, image): + image_features, image_tokens = self.encode_image( + image, return_tokens=True) + img_queries = self.img_queries.expand(image_tokens.shape[0], -1, -1) + img_token_features = self.img_attn_pool(img_queries, image_tokens) + img_token_features = self.img_attn_pool_norm(img_token_features) + sot_token = self.tokenizer.encoder['<|startoftext|>'] + eot_token = self.tokenizer.encoder['<|endoftext|>'] + text_input = image.new_ones( + image.shape[0], 1, dtype=torch.long) * sot_token + input_tokens = img_token_features + pred_tokens = [] + for text_idx in range(self.context_length): + text_features, text_tokens = self.encode_text( + text_input, return_tokens=True) + input_tokens = torch.cat([img_token_features, text_tokens], axis=1) + out_embs = self.decoder(input_tokens.permute(1, 0, 2).contiguous()) + gen_logits = self.to_logits(out_embs[-1:, ...]) + probs = F.softmax(self.gen_logit_scale.exp() * gen_logits, dim=-1) + pred = torch.argmax( + probs * (2.0 + torch.rand_like(probs)), axis=-1) + if int(pred) >= eot_token or int(pred) <= 0: + break + pred_tokens.append(pred) + text_input = torch.cat( + [text_input, pred.permute(1, 0).contiguous()], axis=1) + pred_text_tokens = torch.cat(pred_tokens, axis=0).permute(1, 0) + text_list = [] + for out_tokens in pred_text_tokens: + tokens = [] + for x in out_tokens: + tokens.append(int(x)) + out_text = self.tokenizer.decode(tokens) + out_text = out_text.strip() + text_list.append(out_text) + return image_features, text_list[0] + + +class GEMMModel(nn.Module): + """ Generative multi-modal model, wrapper of GEVL module. + It takes image or text or both of them as input, and output + features of input or caption when image input is available. + """ + + def __init__(self, model_dir): + super().__init__() + with open( + '{}/encoder_config.json'.format(model_dir), 'r', + encoding='utf-8') as f: + model_config = json.loads(f.read()) + model_name = list(model_config.keys())[0] + config_args = model_config[model_name] + bpe_path = os.path.join(model_dir, 'bpe_vocab_16e6.txt.gz') + self.tokenizer = SimpleTokenizer(bpe_path) + self.model = GEVL(*config_args, self.tokenizer) + + def tokenize(self, text_str): + text_tensor = clip_tokenize(self.tokenizer, [text_str])[0] + return text_tensor + + def parse_feat(self, feat): + out = feat.cpu().numpy() + return out + + @torch.no_grad() + def forward(self, image=None, text=None, captioning=True): + img_feature, text_feature, caption = None, None, None + if captioning and image is not None: + img_feature, caption = self.model.image_to_text(image) + img_feature = self.parse_feat(img_feature) + elif image is not None: + img_feature = self.parse_feat(self.model.encode_image(image)) + if text is not None: + text_feature = self.parse_feat(self.model.encode_text(text)) + out = { + 'image_feature': img_feature, + 'text_feature': text_feature, + 'caption': caption, + } + return out diff --git a/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_model.py b/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c90b35d401c84c0077ca7148b9368b5d34b9c80a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/gemm/gemm_model.py @@ -0,0 +1,94 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +""" Generative Multimodal Model Wrapper.""" +import os.path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image +from torchvision import transforms as T + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.gemm.gemm_base import GEMMModel +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['GEMMForMultiModalEmbedding'] + + +@MODELS.register_module( + Tasks.generative_multi_modal_embedding, module_name=Models.gemm) +class GEMMForMultiModalEmbedding(TorchModel): + """ Generative multi-modal model for multi-modal embedding + Inputs could be image or text or both of them. + Outputs could be features of input image or text, + image caption could also be produced when image is available. + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.gemm_model = GEMMModel(model_dir=model_dir) + pretrained_params = torch.load('{}/{}'.format( + model_dir, ModelFile.TORCH_MODEL_BIN_FILE)) + self.gemm_model.load_state_dict(pretrained_params) + self.gemm_model.eval() + self.device_id = device_id + if self.device_id >= 0 and torch.cuda.is_available(): + self.gemm_model.to('cuda:{}'.format(self.device_id)) + logger.info('Use GPU: {}'.format(self.device_id)) + else: + self.device_id = -1 + logger.info('Use CPU for inference') + self.img_preprocessor = T.Compose([ + T.Resize(224), + T.CenterCrop(224), + T.ToTensor(), + T.Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)) + ]) + + def parse_image(self, input_img): + if input_img is None: + return None + input_img = LoadImage.convert_to_img(input_img) + img_tensor = self.img_preprocessor(input_img)[None, ...] + if self.device_id >= 0: + img_tensor = img_tensor.to('cuda:{}'.format(self.device_id)) + return img_tensor + + def parse_text(self, text_str): + if text_str is None or len(text_str) == 0: + return None + if isinstance(text_str, str): + text_ids_tensor = self.gemm_model.tokenize(text_str) + else: + raise TypeError(f'text should be str, but got {type(text_str)}') + if self.device_id >= 0: + text_ids_tensor = text_ids_tensor.to('cuda:{}'.format( + self.device_id)) + return text_ids_tensor.view(1, -1) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + image_input = input.get('image', input.get('img', None)) + text_input = input.get('text', input.get('txt', None)) + captioning_input = input.get('captioning', None) + image = self.parse_image(image_input) + text = self.parse_text(text_input) + captioning = captioning_input is True or text_input == '' + out = self.gemm_model(image, text, captioning) + output = { + OutputKeys.IMG_EMBEDDING: out.get('image_feature', None), + OutputKeys.TEXT_EMBEDDING: out.get('text_feature', None), + OutputKeys.CAPTION: out.get('caption', None) + } + return output diff --git a/runtime/Lib/site_packages/models/multi_modal/gemm/tokenizer.py b/runtime/Lib/site_packages/models/multi_modal/gemm/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..8b7cc094751f8ed9b51e30dbfed6eb9bb830068c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/gemm/tokenizer.py @@ -0,0 +1,201 @@ +# Copyright 2021 The OpenAI Team Authors. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# +# The implementation here is modified based on OpenAI CLIP, +# originally MIT License, Copyright (c) 2021 OpenAI, +# and publicly available at https://github.com/openai/CLIP/. +""" CLIP Tokenizer.""" + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re +import torch + + +@lru_cache() +def default_bpe(): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'bpe_simple_vocab_16e6.txt.gz') + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + error_list = [] + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception as err: + error_list.append(err) + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + if len(error_list) > 100: + print(error_list[-1]) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + +def clip_tokenize(tokenizer, texts, context_length=77, truncate=True): + """ + Returns the tokenized representation of given input string(s) + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + context_length : int + The context length to use; all CLIP models use 77 as the context length + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]. + We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long. + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = tokenizer.encoder['<|startoftext|>'] + eot_token = tokenizer.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + tokenizer.encode(text) + [eot_token] + for text in texts] + result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError( + f'Input {texts[i]} is too long for context length {context_length}' + ) + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/__init__.py b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..93d0ca51488754d2ee89476c5340cfafa3dacc29 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .unet import HFUNetModel + from .script import create_diffusion +else: + _import_structure = { + 'unet': ['HFUNetModel'], + 'script': ['create_diffusion'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/gaussian_diffusion.py b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/gaussian_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..430aa3786f111573a1f1efbcc7705c9ed2d1ad9a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/gaussian_diffusion.py @@ -0,0 +1,930 @@ +# This code is borrowed and modified from Guided Diffusion Model, +# made publicly available under MIT license +# at https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/disco_project + +import enum +import math + +import numpy as np +import torch as th + + +def get_named_beta_schedule(schedule_name, num_diffusion_timesteps): + """ + Get a pre-defined beta schedule for the given name. + + The beta schedule library consists of beta schedules which remain similar + in the limit of num_diffusion_timesteps. + Beta schedules may be added, but should not be removed or changed once + they are committed to maintain backwards compatibility. + """ + if schedule_name == 'linear': + # Linear schedule from Ho et al, extended to work for any number of + # diffusion steps. + scale = 1000 / num_diffusion_timesteps + beta_start = scale * 0.0001 + beta_end = scale * 0.02 + return np.linspace( + beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64) + elif schedule_name == 'cosine': + return betas_for_alpha_bar( + num_diffusion_timesteps, + lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2)**2, + ) + else: + raise NotImplementedError(f'unknown beta schedule: {schedule_name}') + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + + +class ModelMeanType(enum.Enum): + """ + Which type of output the model predicts. + """ + + PREVIOUS_X = enum.auto() # the model predicts x_{t-1} + START_X = enum.auto() # the model predicts x_0 + EPSILON = enum.auto() # the model predicts epsilon + + +class ModelVarType(enum.Enum): + """ + What is used as the model's output variance. + + The LEARNED_RANGE option has been added to allow the model to predict + values between FIXED_SMALL and FIXED_LARGE, making its job easier. + """ + + LEARNED = enum.auto() + FIXED_SMALL = enum.auto() + FIXED_LARGE = enum.auto() + LEARNED_RANGE = enum.auto() + + +class LossType(enum.Enum): + MSE = enum.auto() # use raw MSE loss (and KL when learning variances) + RESCALED_MSE = ( + enum.auto() + ) # use raw MSE loss (with RESCALED_KL when learning variances) + KL = enum.auto() # use the variational lower-bound + RESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB + + def is_vb(self): + return self == LossType.KL or self == LossType.RESCALED_KL + + +class GaussianDiffusion: + """ + Utilities for training and sampling diffusion models. + + Ported directly from here, and then adapted over time to further experimentation. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py#L42 + + :param betas: a 1-D numpy array of betas for each diffusion timestep, + starting at T and going to 1. + :param model_mean_type: a ModelMeanType determining what the model outputs. + :param model_var_type: a ModelVarType determining how variance is output. + :param loss_type: a LossType determining the loss function to use. + :param rescale_timesteps: if True, pass floating point timesteps into the + model so that they are always scaled like in the + original paper (0 to 1000). + """ + + def __init__( + self, + *, + betas, + model_mean_type, + model_var_type, + loss_type, + rescale_timesteps=False, + ): + self.model_mean_type = model_mean_type + self.model_var_type = model_var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # Use float64 for accuracy. + betas = np.array(betas, dtype=np.float64) + self.betas = betas + assert len(betas.shape) == 1, 'betas must be 1-D' + assert (betas > 0).all() and (betas <= 1).all() + + self.num_timesteps = int(betas.shape[0]) + + alphas = 1.0 - betas + self.alphas_cumprod = np.cumprod(alphas, axis=0) + self.alphas_cumprod_prev = np.append(1.0, self.alphas_cumprod[:-1]) + self.alphas_cumprod_next = np.append(self.alphas_cumprod[1:], 0.0) + assert self.alphas_cumprod_prev.shape == (self.num_timesteps, ) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.sqrt_alphas_cumprod = np.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = np.sqrt(1.0 - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = np.log(1.0 - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod + - 1) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + v1 = betas * (1.0 - self.alphas_cumprod_prev) + v2 = 1.0 - self.alphas_cumprod + self.posterior_variance = v1 / v2 + # log calculation clipped because the posterior variance is 0 at the + # beginning of the diffusion chain. + self.posterior_log_variance_clipped = np.log( + np.append(self.posterior_variance[1], self.posterior_variance[1:])) + + v1 = betas * np.sqrt(self.alphas_cumprod_prev) + v2 = 1.0 - self.alphas_cumprod + self.posterior_mean_coef1 = v1 / v2 + + v1 = (1.0 - self.alphas_cumprod_prev) * np.sqrt(alphas) + v2 = 1.0 - self.alphas_cumprod + self.posterior_mean_coef2 = v1 / v2 + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start) + variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, + x_start.shape) + log_variance = _extract_into_tensor(self.log_one_minus_alphas_cumprod, + t, x_start.shape) + return mean, variance, log_variance + + def q_sample(self, x_start, t, noise=None): + """ + Diffuse the data for a given number of diffusion steps. + + In other words, sample from q(x_t | x_0). + + :param x_start: the initial data batch. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :param noise: if specified, the split-out normal noise. + :return: A noisy version of x_start. + """ + if noise is None: + noise = th.randn_like(x_start) + assert noise.shape == x_start.shape + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + _extract_into_tensor( + self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise) + + def q_posterior_mean_variance(self, x_start, x_t, t): + """ + Compute the mean and variance of the diffusion posterior: + + q(x_{t-1} | x_t, x_0) + + """ + assert x_start.shape == x_t.shape + posterior_mean = ( + _extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) + * x_start + + _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) + * x_t) + posterior_variance = _extract_into_tensor(self.posterior_variance, t, + x_t.shape) + posterior_log_variance_clipped = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape) + assert posterior_mean.shape[0] == posterior_variance.shape[0] + assert posterior_mean.shape[0] == posterior_log_variance_clipped.shape[ + 0] + assert posterior_mean.shape[0] == x_start.shape[0] + + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def p_mean_variance(self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + model_kwargs=None): + """ + Apply the model to get p(x_{t-1} | x_t), as well as a prediction of + the initial x, x_0. + + :param model: the model, which takes a signal and a batch of timesteps + as input. + :param x: the [N x C x ...] tensor at time t. + :param t: a 1-D Tensor of timesteps. + :param clip_denoised: if True, clip the denoised signal into [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. Applies before + clip_denoised. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict with the following keys: + - 'mean': the model mean output. + - 'variance': the model variance output. + - 'log_variance': the log of 'variance'. + - 'pred_xstart': the prediction for x_0. + """ + if model_kwargs is None: + model_kwargs = {} + + B, C = x.shape[:2] + assert t.shape == (B, ) + model_output = model(x, self._scale_timesteps(t), **model_kwargs) + + if self.model_var_type in [ + ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE + ]: + assert model_output.shape == (B, C * 2, *x.shape[2:]) + model_output, model_var_values = th.split(model_output, C, dim=1) + if self.model_var_type == ModelVarType.LEARNED: + model_log_variance = model_var_values + model_variance = th.exp(model_log_variance) + else: + min_log = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x.shape) + max_log = _extract_into_tensor(np.log(self.betas), t, x.shape) + # The model_var_values is [-1, 1] for [min_var, max_var]. + frac = (model_var_values + 1) / 2 + model_log_variance = frac * max_log + (1 - frac) * min_log + model_variance = th.exp(model_log_variance) + else: + model_variance, model_log_variance = { + # for fixedlarge, we set the initial (log-)variance like so + # to get a better decoder log likelihood. + ModelVarType.FIXED_LARGE: ( + np.append(self.posterior_variance[1], self.betas[1:]), + np.log( + np.append(self.posterior_variance[1], self.betas[1:])), + ), + ModelVarType.FIXED_SMALL: ( + self.posterior_variance, + self.posterior_log_variance_clipped, + ), + }[self.model_var_type] + model_variance = _extract_into_tensor(model_variance, t, x.shape) + model_log_variance = _extract_into_tensor(model_log_variance, t, + x.shape) + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + if self.model_mean_type == ModelMeanType.PREVIOUS_X: + pred_xstart = process_xstart( + self._predict_xstart_from_xprev( + x_t=x, t=t, xprev=model_output)) + model_mean = model_output + elif self.model_mean_type in [ + ModelMeanType.START_X, ModelMeanType.EPSILON + ]: + if self.model_mean_type == ModelMeanType.START_X: + pred_xstart = process_xstart(model_output) + else: + pred_xstart = process_xstart( + self._predict_xstart_from_eps( + x_t=x, t=t, eps=model_output)) + model_mean, _, _ = self.q_posterior_mean_variance( + x_start=pred_xstart, x_t=x, t=t) + else: + raise NotImplementedError(self.model_mean_type) + + return { + 'mean': model_mean, + 'variance': model_variance, + 'log_variance': model_log_variance, + 'pred_xstart': pred_xstart, + } + + def _predict_xstart_from_eps(self, x_t, t, eps): + assert x_t.shape == eps.shape + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, + x_t.shape) * eps) + + def _predict_xstart_from_xprev(self, x_t, t, xprev): + assert x_t.shape == xprev.shape + return ( # (xprev - coef2*x_t) / coef1 + _extract_into_tensor(1.0 / self.posterior_mean_coef1, t, x_t.shape) + * xprev - _extract_into_tensor( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + x_t.shape) * x_t) + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t - pred_xstart) / _extract_into_tensor( + self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * (1000.0 / self.num_timesteps) + return t + + def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, self._scale_timesteps(t), **model_kwargs) + new_mean = ( + p_mean_var['mean'].float() + + p_mean_var['variance'] * gradient.float()) + return new_mean + + def condition_mean_with_grad(self, + cond_fn, + p_mean_var, + x, + t, + model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, t, p_mean_var, **model_kwargs) + new_mean = ( + p_mean_var['mean'].float() + + p_mean_var['variance'] * gradient.float()) + return new_mean + + def condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute what the p_mean_variance output would have been, should the + model's score function be conditioned by cond_fn. + + See condition_mean() for details on cond_fn. + + Unlike condition_mean(), this instead uses the conditioning strategy + from Song et al (2020). + """ + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + + eps = self._predict_eps_from_xstart(x, t, p_mean_var['pred_xstart']) + eps = eps - (1 - alpha_bar).sqrt() * cond_fn( + x, self._scale_timesteps(t), **model_kwargs) + + out = p_mean_var.copy() + out['pred_xstart'] = self._predict_xstart_from_eps(x, t, eps) + out['mean'], _, _ = self.q_posterior_mean_variance( + x_start=out['pred_xstart'], x_t=x, t=t) + return out + + def condition_score_with_grad(self, + cond_fn, + p_mean_var, + x, + t, + model_kwargs=None): + """ + Compute what the p_mean_variance output would have been, should the + model's score function be conditioned by cond_fn. + + See condition_mean() for details on cond_fn. + + Unlike condition_mean(), this instead uses the conditioning strategy + from Song et al (2020). + """ + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + + eps = self._predict_eps_from_xstart(x, t, p_mean_var['pred_xstart']) + + grad = cond_fn(x, t, p_mean_var, **model_kwargs) + eps = eps - (1 - alpha_bar).sqrt() * grad + + out = p_mean_var.copy() + out['pred_xstart'] = self._predict_xstart_from_eps(x, t, eps) + out['mean'], _, _ = self.q_posterior_mean_variance( + x_start=out['pred_xstart'], x_t=x, t=t) + return out + + def p_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + noise = th.randn_like(x) + nonzero_mask = ((t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out['mean'] = self.condition_mean( + cond_fn, out, x, t, model_kwargs=model_kwargs) + sample = out['mean'] + nonzero_mask * th.exp( + 0.5 * out['log_variance']) * noise + return {'sample': sample, 'pred_xstart': out['pred_xstart']} + + def p_sample_with_grad( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + with th.enable_grad(): + x = x.detach().requires_grad_() + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + noise = th.randn_like(x) + nonzero_mask = ((t != 0).float().view(-1, + *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out['mean'] = self.condition_mean_with_grad( + cond_fn, out, x, t, model_kwargs=model_kwargs) + sample = out['mean'] + nonzero_mask * th.exp( + 0.5 * out['log_variance']) * noise + return {'sample': sample, 'pred_xstart': out['pred_xstart'].detach()} + + def p_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + ): + """ + Generate samples from the model. + + :param model: the model module. + :param shape: the shape of the samples, (N, C, H, W). + :param noise: if specified, the noise from the encoder to sample. + Should be of the same shape as `shape`. + :param clip_denoised: if True, clip x_start predictions to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param device: if specified, the device to create the samples on. + If not specified, use a model parameter's device. + :param progress: if True, show a tqdm progress bar. + :return: a non-differentiable batch of samples. + """ + final = None + for sample in self.p_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + skip_timesteps=skip_timesteps, + init_image=init_image, + randomize_class=randomize_class, + cond_fn_with_grad=cond_fn_with_grad, + ): + final = sample + return final['sample'] + + def p_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + ): + """ + Generate samples from the model and yield intermediate samples from + each timestep of diffusion. + + Arguments are the same as p_sample_loop(). + Returns a generator over dicts, where each dict is the return value of + p_sample(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + + if skip_timesteps and init_image is None: + init_image = th.zeros_like(img) + + indices = list(range(self.num_timesteps - skip_timesteps))[::-1] + + if init_image is not None: + my_t = th.ones([shape[0]], device=device, + dtype=th.long) * indices[0] + img = self.q_sample(init_image, my_t, img) + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, desc='Steps') + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + if randomize_class and 'y' in model_kwargs: + model_kwargs['y'] = th.randint( + low=0, + high=model.num_classes, + size=model_kwargs['y'].shape, + device=model_kwargs['y'].device) + with th.no_grad(): + sample_fn = self.p_sample_with_grad if cond_fn_with_grad else self.p_sample + out = sample_fn( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + yield out + img = out['sample'] + + def ddim_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + inpainting_mode=False, + orig_img=None, + mask_inpaint=None, + ): + """ + Sample x_{t-1} from the model using DDIM. + + Same usage as p_sample(). + """ + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + if inpainting_mode: + noised_orig_img = th.sqrt(alpha_bar) * orig_img + \ + th.sqrt(1 - alpha_bar) * th.randn_like(x) + # noised_orig_img_pil = TF.to_pil_image(noised_orig_img[0].add(1).div(2).clamp(0, 1)) + # noised_orig_img_pil.save(f'/content/drive/MyDrive/AI/Disco_Diffusion/images_out/InpaintingTest/inpainting_dump/noised_orig_{t[0].item()}.png') + x = (1 - mask_inpaint) * noised_orig_img + mask_inpaint * x + # mixed_x = TF.to_pil_image(x[0].add(1).div(2).clamp(0, 1)) + # mixed_x.save(f'/content/drive/MyDrive/AI/Disco_Diffusion/images_out/InpaintingTest/inpainting_dump/mixed_x_{t[0].item()}.png') + + out_orig = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score( + cond_fn, out_orig, x, t, model_kwargs=model_kwargs) + else: + out = out_orig + + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = self._predict_eps_from_xstart(x, t, out['pred_xstart']) + + alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, + x.shape) + + v1 = eta * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) + v2 = th.sqrt(1 - alpha_bar / alpha_bar_prev) + sigma = v1 * v2 + + # Equation 12. + noise = th.randn_like(x) + mean_pred = ( + out['pred_xstart'] * th.sqrt(alpha_bar_prev) + + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps) + nonzero_mask = ((t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + sample = mean_pred + nonzero_mask * sigma * noise + return {'sample': sample, 'pred_xstart': out_orig['pred_xstart']} + + def ddim_sample_with_grad( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + ): + """ + Sample x_{t-1} from the model using DDIM. + + Same usage as p_sample(). + """ + with th.enable_grad(): + x = x.detach().requires_grad_() + out_orig = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score_with_grad( + cond_fn, out_orig, x, t, model_kwargs=model_kwargs) + else: + out = out_orig + + out['pred_xstart'] = out['pred_xstart'].detach() + + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = self._predict_eps_from_xstart(x, t, out['pred_xstart']) + + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, + x.shape) + + v1 = eta * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) + v2 = th.sqrt(1 - alpha_bar / alpha_bar_prev) + sigma = v1 * v2 + + # Equation 12. + noise = th.randn_like(x) + mean_pred = ( + out['pred_xstart'] * th.sqrt(alpha_bar_prev) + + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps) + nonzero_mask = ((t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + sample = mean_pred + nonzero_mask * sigma * noise + return { + 'sample': sample, + 'pred_xstart': out_orig['pred_xstart'].detach() + } + + def ddim_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + eta=0.0, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + ): + """ + Generate samples from the model using DDIM. + + Same usage as p_sample_loop(). + """ + final = None + for sample in self.ddim_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + eta=eta, + skip_timesteps=skip_timesteps, + init_image=init_image, + randomize_class=randomize_class, + cond_fn_with_grad=cond_fn_with_grad, + ): + final = sample + return final['sample'] + + def ddim_sample_loop_progressive(self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + eta=0.0, + skip_timesteps=0, + init_image=None, + randomize_class=False, + cond_fn_with_grad=False, + transformation_fn=None, + transformation_percent=[], + inpainting_mode=False, + mask_inpaint=None, + skip_timesteps_orig=None): + """ + Use DDIM to sample from the model and yield intermediate samples from + each timestep of DDIM. + + Same usage as p_sample_loop_progressive(). + """ + + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + + if skip_timesteps and init_image is None: + init_image = th.zeros_like(img) + + indices = list(range(self.num_timesteps - skip_timesteps))[::-1] + transformation_steps = [ + int(len(indices) * (1 - i)) for i in transformation_percent + ] + + if init_image is not None: + my_t = th.ones([shape[0]], device=device, + dtype=th.long) * indices[0] + img = self.q_sample(init_image, my_t, img) + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + indices = tqdm(indices, desc='Steps') + + if inpainting_mode and skip_timesteps_orig is None: + skip_timesteps_orig = self.num_timesteps + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + if randomize_class and 'y' in model_kwargs: + model_kwargs['y'] = th.randint( + low=0, + high=model.num_classes, + size=model_kwargs['y'].shape, + device=model_kwargs['y'].device) + with th.no_grad(): + if i in transformation_steps and transformation_fn is not None: + img = transformation_fn(img) + sample_fn = self.ddim_sample_with_grad if cond_fn_with_grad else self.ddim_sample + if inpainting_mode \ + and i >= self.num_timesteps - skip_timesteps_orig \ + and not cond_fn_with_grad: + out = sample_fn( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + eta=eta, + inpainting_mode=inpainting_mode, + orig_img=init_image, + mask_inpaint=mask_inpaint, + ) + else: + out = sample_fn( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + eta=eta, + ) + yield out + img = out['sample'] + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + """ + Extract values from a 1-D numpy array for a batch of indices. + + :param arr: the 1-D numpy array. + :param timesteps: a tensor of indices into the array to extract. + :param broadcast_shape: a larger shape of K dimensions with the batch + dimension equal to the length of timesteps. + :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims. + """ + res = th.from_numpy(arr).to(device=timesteps.device)[timesteps].float() + while len(res.shape) < len(broadcast_shape): + res = res[..., None] + return res.expand(broadcast_shape) diff --git a/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/respace.py b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/respace.py new file mode 100644 index 0000000000000000000000000000000000000000..b179aae1891c2e232e6c2f98194af937fab3c050 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/respace.py @@ -0,0 +1,78 @@ +# This code is borrowed and modified from Guided Diffusion Model, +# made publicly available under MIT license +# at https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/disco_project + +import numpy as np +import torch as th + +from .gaussian_diffusion import GaussianDiffusion + + +class SpacedDiffusion(GaussianDiffusion): + """ + A diffusion process which can skip steps in a base diffusion process. + + :param use_timesteps: a collection (sequence or set) of timesteps from the + original diffusion process to retain. + :param kwargs: the kwargs to create the base diffusion process. + """ + + def __init__(self, use_timesteps, **kwargs): + self.use_timesteps = set(use_timesteps) + self.timestep_map = [] + self.original_num_steps = len(kwargs['betas']) + + base_diffusion = GaussianDiffusion(**kwargs) # pylint: disable=missing-kwoa + last_alpha_cumprod = 1.0 + new_betas = [] + for i, alpha_cumprod in enumerate(base_diffusion.alphas_cumprod): + if i in self.use_timesteps: + new_betas.append(1 - alpha_cumprod / last_alpha_cumprod) + last_alpha_cumprod = alpha_cumprod + self.timestep_map.append(i) + kwargs['betas'] = np.array(new_betas) + super().__init__(**kwargs) + + def p_mean_variance(self, model, *args, **kwargs): # pylint: disable=signature-differs + return super().p_mean_variance( + self._wrap_model(model), *args, **kwargs) + + def training_losses(self, model, *args, **kwargs): # pylint: disable=signature-differs + return super().training_losses( + self._wrap_model(model), *args, **kwargs) + + def condition_mean(self, cond_fn, *args, **kwargs): + return super().condition_mean( + self._wrap_model(cond_fn), *args, **kwargs) + + def condition_score(self, cond_fn, *args, **kwargs): + return super().condition_score( + self._wrap_model(cond_fn), *args, **kwargs) + + def _wrap_model(self, model): + if isinstance(model, _WrappedModel): + return model + return _WrappedModel(model, self.timestep_map, self.rescale_timesteps, + self.original_num_steps) + + def _scale_timesteps(self, t): + # Scaling is done by the wrapped model. + return t + + +class _WrappedModel: + + def __init__(self, model, timestep_map, rescale_timesteps, + original_num_steps): + self.model = model + self.timestep_map = timestep_map + self.rescale_timesteps = rescale_timesteps + self.original_num_steps = original_num_steps + + def __call__(self, x, ts, **kwargs): + map_tensor = th.tensor( + self.timestep_map, device=ts.device, dtype=ts.dtype) + new_ts = map_tensor[ts] + if self.rescale_timesteps: + new_ts = new_ts.float() * (1000.0 / self.original_num_steps) + return self.model(x, new_ts, **kwargs) diff --git a/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/script.py b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/script.py new file mode 100644 index 0000000000000000000000000000000000000000..83193379d2090ebe44e99c6ce56326ab735e1c1d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/script.py @@ -0,0 +1,39 @@ +# This code is borrowed and modified from Guided Diffusion Model, +# made publicly available under MIT license +# at https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/disco_project + +from modelscope.models.cv.motion_generation.modules.respace import \ + space_timesteps +from . import gaussian_diffusion as gd +from .respace import SpacedDiffusion + + +def create_diffusion(diffusion_config): + predict_xstart = False + sigma_small = False + learn_sigma = True + + steps = diffusion_config['steps'] + timestep_respacing = f'ddim{steps}' + diffusion_steps = 1000 + + rescale_timesteps = True + + betas = gd.get_named_beta_schedule('linear', diffusion_steps) + loss_type = gd.LossType.MSE + + if not timestep_respacing: + timestep_respacing = [diffusion_steps] + + diffusion = SpacedDiffusion( + use_timesteps=space_timesteps(diffusion_steps, timestep_respacing), + betas=betas, + model_mean_type=(gd.ModelMeanType.EPSILON + if not predict_xstart else gd.ModelMeanType.START_X), + model_var_type=((gd.ModelVarType.FIXED_LARGE + if not sigma_small else gd.ModelVarType.FIXED_SMALL) + if not learn_sigma else gd.ModelVarType.LEARNED_RANGE), + loss_type=loss_type, + rescale_timesteps=rescale_timesteps) + + return diffusion diff --git a/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/unet.py b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..946a4179814aefe632be6ec519e6c5fae9234f0e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/guided_diffusion/unet.py @@ -0,0 +1,1046 @@ +# This code is borrowed and modified from Guided Diffusion Model, +# made publicly available under MIT license at +# https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/disco_project + +import math +from abc import abstractmethod + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig, PreTrainedModel + + +class GroupNorm(nn.GroupNorm): + + def forward(self, x): + return super(GroupNorm, self).forward(x.float()).type(x.dtype) + + +def timestep_embedding(timesteps, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + + :param timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an [N x dim] Tensor of positional embeddings. + """ + half = dim // 2 + freqs = th.exp(-math.log(max_period) + * th.arange(start=0, end=half, dtype=th.float32) + / half).to(device=timesteps.device) + args = timesteps[:, None].float() * freqs[None] + embedding = th.cat([th.cos(args), th.sin(args)], dim=-1) + if dim % 2: + embedding = th.cat( + [embedding, th.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + +def convert_module_to_f16(ll): + """ + Convert primitive modules to float16. + """ + if isinstance(ll, (nn.Conv1d, nn.Conv2d, nn.Conv3d)): + ll.weight.data = ll.weight.data.half() + if ll.bias is not None: + ll.bias.data = ll.bias.data.half() + + +def convert_module_to_f32(ll): + """ + Convert primitive modules to float32, undoing convert_module_to_f16(). + """ + if isinstance(ll, (nn.Conv1d, nn.Conv2d, nn.Conv3d)): + ll.weight.data = ll.weight.data.float() + if ll.bias is not None: + ll.bias.data = ll.bias.data.float() + + +def conv_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f'unsupported dimensions: {dims}') + + +def checkpoint(func, inputs, params, flag): + """ + Evaluate a function without caching intermediate activations, allowing for + reduced memory at the expense of extra compute in the backward pass. + :param func: the function to evaluate. + :param inputs: the argument sequence to pass to `func`. + :param params: a sequence of parameters `func` depends on but does not + explicitly take as arguments. + :param flag: if False, disable gradient checkpointing. + """ + if flag: + args = tuple(inputs) + tuple(params) + return CheckpointFunction.apply(func, len(inputs), *args) + else: + return func(*inputs) + + +class AttentionPool2d(nn.Module): + """ + Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__( + self, + spacial_dim: int, + embed_dim: int, + num_heads_channels: int, + output_dim: int = None, + ): + super().__init__() + self.positional_embedding = nn.Parameter( + th.randn(embed_dim, spacial_dim**2 + 1) / embed_dim**0.5) + self.qkv_proj = conv_nd(1, embed_dim, 3 * embed_dim, 1) + self.c_proj = conv_nd(1, embed_dim, output_dim or embed_dim, 1) + self.num_heads = embed_dim // num_heads_channels + self.attention = QKVAttention(self.num_heads) + + def forward(self, x): + b, c, *_spatial = x.shape + x = x.reshape(b, c, -1) # NC(HW) + x = th.cat([x.mean(dim=-1, keepdim=True), x], dim=-1) # NC(HW+1) + x = x + self.positional_embedding[None, :, :].to(x.dtype) # NC(HW+1) + x = self.qkv_proj(x) + x = self.attention(x) + x = self.c_proj(x) + return x[:, :, 0] + + +class TimestepBlock(nn.Module): + """ + Any module where forward() takes timestep embeddings as a second argument. + """ + + @abstractmethod + def forward(self, x, emb): + """ + Apply the module to `x` given `emb` timestep embeddings. + """ + + +class TimestepEmbedSequential(nn.Sequential, TimestepBlock): + """ + A sequential module that passes timestep embeddings to the children that + support it as an extra input. + """ + + def forward(self, x, emb): + for layer in self: + if isinstance(layer, TimestepBlock): + x = layer(x, emb) + else: + x = layer(x) + return x + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, channels, use_conv, dims=2, out_channels=None): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = conv_nd( + dims, self.channels, self.out_channels, 3, padding=1) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + if self.use_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, channels, use_conv, dims=2, out_channels=None): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = conv_nd( + dims, + self.channels, + self.out_channels, + 3, + stride=stride, + padding=1) + else: + assert self.channels == self.out_channels + self.op = nn.AvgPool2d(kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class ResBlock(TimestepBlock): + """ + A residual block that can optionally change the number of channels. + + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + use_checkpoint=False, + up=False, + down=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_checkpoint = use_checkpoint + self.use_scale_shift_norm = use_scale_shift_norm + + self.in_layers = nn.Sequential( + GroupNorm(32, channels), + nn.SiLU(), + conv_nd(dims, channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + nn.Linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + GroupNorm(32, self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1), + ) + + nn.init.zeros_(self.out_layers[-1].weight) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = conv_nd(dims, channels, self.out_channels, + 1) + + def forward(self, x, emb): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return checkpoint(self._forward, (x, emb), self.parameters(), + self.use_checkpoint) + + def _forward(self, x, emb): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + return self.skip_connection(x) + h + + +class AttentionBlock(nn.Module): + """ + An attention block that allows spatial positions to attend to each other. + + Originally ported from here, but adapted to the N-d case. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66. + """ + + def __init__( + self, + channels, + num_heads=1, + num_head_channels=-1, + use_checkpoint=False, + use_new_attention_order=False, + ): + super().__init__() + self.channels = channels + if num_head_channels == -1: + self.num_heads = num_heads + else: + assert ( + channels % num_head_channels == 0 + ), f'q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}' + self.num_heads = channels // num_head_channels + self.use_checkpoint = use_checkpoint + self.norm = GroupNorm(32, channels) + self.qkv = conv_nd(1, channels, channels * 3, 1) + if use_new_attention_order: + # split qkv before split heads + self.attention = QKVAttention(self.num_heads) + else: + # split heads before split qkv + self.attention = QKVAttentionLegacy(self.num_heads) + + self.proj_out = conv_nd(1, channels, channels, 1) + + nn.init.zeros_(self.proj_out.weight) + + def forward(self, x): + return checkpoint(self._forward, (x, ), self.parameters(), + self.use_checkpoint) + + def _forward(self, x): + b, c, *spatial = x.shape + x = x.reshape(b, c, -1) + qkv = self.qkv(self.norm(x)) + h = self.attention(qkv) + h = self.proj_out(h) + return (x + h).reshape(b, c, *spatial) + + +def count_flops_attn(model, _x, y): + """ + A counter for the `thop` package to count the operations in an + attention operation. + Meant to be used like: + macs, params = thop.profile( + model, + inputs=(inputs, timestamps), + custom_ops={QKVAttention: QKVAttention.count_flops}, + ) + """ + b, c, *spatial = y[0].shape + num_spatial = int(np.prod(spatial)) + # We perform two matmuls with the same number of ops. + # The first computes the weight matrix, the second computes + # the combination of the value vectors. + matmul_ops = 2 * b * (num_spatial**2) * c + model.total_ops += th.DoubleTensor([matmul_ops]) + + +class QKVAttentionLegacy(nn.Module): + """ + A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + + :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.reshape(bs * self.n_heads, ch * 3, length).split( + ch, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + 'bct,bcs->bts', q * scale, + k * scale) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum('bts,bcs->bct', weight, v) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class QKVAttention(nn.Module): + """ + A module which performs QKV attention and splits in a different order. + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + + :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.chunk(3, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + 'bct,bcs->bts', + (q * scale).view(bs * self.n_heads, ch, length), + (k * scale).view(bs * self.n_heads, ch, length), + ) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum('bts,bcs->bct', weight, + v.reshape(bs * self.n_heads, ch, length)) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class UNetModel(nn.Module): + """ + The full UNet model with attention and timestep embedding. + + :param in_channels: channels in the input Tensor. + :param model_channels: base channel count for the model. + :param out_channels: channels in the output Tensor. + :param num_res_blocks: number of residual blocks per downsample. + :param attention_resolutions: a collection of downsample rates at which + attention will take place. May be a set, list, or tuple. + For example, if this contains 4, then at 4x downsampling, attention + will be used. + :param dropout: the dropout probability. + :param channel_mult: channel multiplier for each level of the UNet. + :param conv_resample: if True, use learned convolutions for upsampling and + downsampling. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param num_classes: if specified (as an int), then this model will be + class-conditional with `num_classes` classes. + :param use_checkpoint: use gradient checkpointing to reduce memory usage. + :param num_heads: the number of attention heads in each attention layer. + :param num_head_channels: if specified, ignore num_heads and instead use + a fixed channel width per attention head. + :param num_heads_upsample: works with num_heads to set a different number + of heads for upsampling. Deprecated. + :param use_scale_shift_norm: use a FiLM-like conditioning mechanism. + :param resblock_updown: use residual blocks for up/downsampling. + :param use_new_attention_order: use a different attention pattern for potentially + increased efficiency. + """ + + def __init__( + self, + image_size, + in_channels, + model_channels, + out_channels, + num_res_blocks, + attention_resolutions, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + num_classes=None, + use_checkpoint=False, + use_fp16=False, + num_heads=1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + ): + super().__init__() + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + self.image_size = image_size + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + self.num_res_blocks = num_res_blocks + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + nn.Linear(model_channels, time_embed_dim), + nn.SiLU(), + nn.Linear(time_embed_dim, time_embed_dim), + ) + + if self.num_classes is not None: + self.label_emb = nn.Embedding(num_classes, time_embed_dim) + + ch = input_ch = int(channel_mult[0] * model_channels) + self.input_blocks = nn.ModuleList([ + TimestepEmbedSequential( + conv_nd(dims, in_channels, ch, 3, padding=1)) + ]) + self._feature_size = ch + input_block_chans = [ch] + ds = 1 + for level, mult in enumerate(channel_mult): + for _ in range(num_res_blocks): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=int(mult * model_channels), + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = int(mult * model_channels) + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + )) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) if resblock_updown else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + self.middle_block = TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + ) + self._feature_size += ch + + self.output_blocks = nn.ModuleList([]) + for level, mult in list(enumerate(channel_mult))[::-1]: + for i in range(num_res_blocks + 1): + ich = input_block_chans.pop() + layers = [ + ResBlock( + ch + ich, + time_embed_dim, + dropout, + out_channels=int(model_channels * mult), + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = int(model_channels * mult) + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads_upsample, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + )) + if level and i == num_res_blocks: + out_ch = ch + layers.append( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + up=True, + ) if resblock_updown else Upsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ds //= 2 + self.output_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + + self.out = nn.Sequential( + GroupNorm(32, ch), + nn.SiLU(), + conv_nd(dims, input_ch, out_channels, 3, padding=1), + ) + + nn.init.zeros_(self.out[-1].weight) + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + self.output_blocks.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + self.output_blocks.apply(convert_module_to_f32) + + def forward(self, x, timesteps, y=None): + """ + Apply the model to an input batch. + + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :param y: an [N] Tensor of labels, if class-conditional. + :return: an [N x C x ...] Tensor of outputs. + """ + assert (y is not None) == ( + self.num_classes is not None + ), 'must specify y if and only if the model is class-conditional' + + hs = [] + emb = self.time_embed( + timestep_embedding(timesteps, self.model_channels)) + + if self.num_classes is not None: + assert y.shape == (x.shape[0], ) + emb = emb + self.label_emb(y) + + h = x.type(self.dtype) + for module in self.input_blocks: + h = module(h, emb) + hs.append(h) + h = self.middle_block(h, emb) + for module in self.output_blocks: + h = th.cat([h, hs.pop()], dim=1) + h = module(h, emb) + h = h.type(x.dtype) + return self.out(h) + + +class SuperResModel(UNetModel): + """ + A UNetModel that performs super-resolution. + + Expects an extra kwarg `low_res` to condition on a low-resolution image. + """ + + def __init__(self, image_size, in_channels, *args, **kwargs): + super().__init__(image_size, in_channels * 2, *args, **kwargs) + + def forward(self, x, timesteps, low_res=None, **kwargs): + _, _, new_height, new_width = x.shape + upsampled = F.interpolate( + low_res, (new_height, new_width), mode='bilinear') + x = th.cat([x, upsampled], dim=1) + return super().forward(x, timesteps, **kwargs) + + +class EncoderUNetModel(nn.Module): + """ + The half UNet model with attention and timestep embedding. + + For usage, see UNet. + """ + + def __init__( + self, + image_size, + in_channels, + model_channels, + out_channels, + num_res_blocks, + attention_resolutions, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + use_checkpoint=False, + use_fp16=False, + num_heads=1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + pool='adaptive', + ): + super().__init__() + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + self.num_res_blocks = num_res_blocks + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.use_checkpoint = use_checkpoint + self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + nn.Linear(model_channels, time_embed_dim), + nn.SiLU(), + nn.Linear(time_embed_dim, time_embed_dim), + ) + + ch = int(channel_mult[0] * model_channels) + self.input_blocks = nn.ModuleList([ + TimestepEmbedSequential( + conv_nd(dims, in_channels, ch, 3, padding=1)) + ]) + self._feature_size = ch + input_block_chans = [ch] + ds = 1 + for level, mult in enumerate(channel_mult): + for _ in range(num_res_blocks): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=int(mult * model_channels), + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = int(mult * model_channels) + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + )) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) if resblock_updown else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch)) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + self.middle_block = TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + ) + self._feature_size += ch + self.pool = pool + if pool == 'adaptive': + self.out = nn.Sequential( + GroupNorm(32, ch), + nn.SiLU(), + nn.AdaptiveAvgPool2d((1, 1)), + conv_nd(dims, ch, out_channels, 1), + nn.Flatten(), + ) + nn.init.zeros_(self.out[-1].weight) + elif pool == 'attention': + assert num_head_channels != -1 + self.out = nn.Sequential( + GroupNorm(32, ch), + nn.SiLU(), + AttentionPool2d((image_size // ds), ch, num_head_channels, + out_channels), + ) + elif pool == 'spatial': + self.out = nn.Sequential( + nn.Linear(self._feature_size, 2048), + nn.ReLU(), + nn.Linear(2048, self.out_channels), + ) + elif pool == 'spatial_v2': + self.out = nn.Sequential( + nn.Linear(self._feature_size, 2048), + GroupNorm(32, 2048), + nn.SiLU(), + nn.Linear(2048, self.out_channels), + ) + else: + raise NotImplementedError(f'Unexpected {pool} pooling') + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + + def forward(self, x, timesteps): + """ + Apply the model to an input batch. + + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :return: an [N x K] Tensor of outputs. + """ + emb = self.time_embed( + timestep_embedding(timesteps, self.model_channels)) + + results = [] + h = x.type(self.dtype) + for module in self.input_blocks: + h = module(h, emb) + if self.pool.startswith('spatial'): + results.append(h.type(x.dtype).mean(dim=(2, 3))) + h = self.middle_block(h, emb) + if self.pool.startswith('spatial'): + results.append(h.type(x.dtype).mean(dim=(2, 3))) + h = th.cat(results, axis=-1) + return self.out(h) + else: + h = h.type(x.dtype) + return self.out(h) + + +class UNetConfig(PretrainedConfig): + + def __init__(self, + image_size=512, + in_channels=3, + model_channels=256, + out_channels=6, + num_res_blocks=2, + attention_resolutions=[16, 32, 64], + dropout=0.0, + channel_mult=(0.5, 1, 1, 2, 2, 4, 4), + num_classes=None, + use_checkpoint=False, + use_fp16=True, + num_heads=4, + num_head_channels=64, + num_heads_upsample=-1, + use_scale_shift_norm=True, + resblock_updown=True, + use_new_attention_order=False, + **kwargs): + self.image_size = image_size + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + self.num_res_blocks = num_res_blocks + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.use_fp16 = use_fp16 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + self.use_scale_shift_norm = use_scale_shift_norm + self.resblock_updown = resblock_updown + self.use_new_attention_order = use_new_attention_order + super().__init__(**kwargs) + + +class HFUNetModel(PreTrainedModel): + config_class = UNetConfig + + def __init__(self, config): + super().__init__(config) + self.model = UNetModel( + image_size=config.image_size, + in_channels=config.in_channels, + model_channels=config.model_channels, + out_channels=config.out_channels, + num_res_blocks=config.num_res_blocks, + attention_resolutions=config.attention_resolutions, + dropout=config.dropout, + channel_mult=config.channel_mult, + num_classes=config.num_classes, + use_checkpoint=config.use_checkpoint, + use_fp16=config.use_fp16, + num_heads=config.num_heads, + num_head_channels=config.num_head_channels, + num_heads_upsample=config.num_heads_upsample, + use_scale_shift_norm=config.use_scale_shift_norm, + resblock_updown=config.resblock_updown, + use_new_attention_order=config.use_new_attention_order, + ) + + def forward(self, x, timesteps, y=None): + return self.model.forward(x, timesteps, y) + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.model.input_blocks.apply(convert_module_to_f16) + self.model.middle_block.apply(convert_module_to_f16) + self.model.output_blocks.apply(convert_module_to_f16) diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/__init__.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e78958d193b9e4ebd136defac2024040b7f25b07 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .image_to_video_model import ImageToVideo + +else: + _import_structure = { + 'image_to_video_model': ['ImageToVideo'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/image_to_video_model.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/image_to_video_model.py new file mode 100644 index 0000000000000000000000000000000000000000..498ad1eca45525b57e0cacc3006cf0aac2b2b966 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/image_to_video_model.py @@ -0,0 +1,217 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import os.path as osp +import random +from copy import copy +from typing import Any, Dict + +import torch +import torch.cuda.amp as amp + +import modelscope.models.multi_modal.image_to_video.utils.transforms as data +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.image_to_video.modules import * +from modelscope.models.multi_modal.image_to_video.modules import ( + AutoencoderKL, FrozenOpenCLIPVisualEmbedder, Img2VidSDUNet) +from modelscope.models.multi_modal.image_to_video.utils.config import cfg +from modelscope.models.multi_modal.image_to_video.utils.diffusion import \ + GaussianDiffusion +from modelscope.models.multi_modal.image_to_video.utils.seed import setup_seed +from modelscope.models.multi_modal.image_to_video.utils.shedule import \ + beta_schedule +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +__all__ = ['ImageToVideo'] + +logger = get_logger() + + +@MODELS.register_module( + Tasks.image_to_video, module_name=Models.image_to_video_model) +class ImageToVideo(TorchModel): + r""" + Image2Video aims to solve the task of generating high-definition videos based on input images. + Image2Video is a video generation basic model developed by Alibaba Cloud, with a parameter size + of approximately 2 billion. It has been pre trained on large-scale video and image data and + fine-tuned on a small amount of high-quality data. The data is widely distributed and diverse + in categories, and the model has good generalization ability for different types of data + + Paper link: https://arxiv.org/abs/2306.02018 + + Attributes: + diffusion: diffusion model for DDIM. + autoencoder: decode the latent representation into visual space. + clip_encoder: encode the image into image embedding. + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + + self.config = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + + # assign default value + cfg.batch_size = self.config.model.model_cfg.batch_size + cfg.target_fps = self.config.model.model_cfg.target_fps + cfg.max_frames = self.config.model.model_cfg.max_frames + cfg.latent_hei = self.config.model.model_cfg.latent_hei + cfg.latent_wid = self.config.model.model_cfg.latent_wid + cfg.model_path = osp.join(model_dir, + self.config.model.model_args.ckpt_unet) + + required_device = kwargs.pop('device', 'gpu') + self.device = create_device(required_device) + + if 'seed' in self.config.model.model_args.keys(): + cfg.seed = self.config.model.model_args.seed + else: + cfg.seed = random.randint(0, 99999) + setup_seed(cfg.seed) + + # transform + vid_trans = data.Compose([ + data.CenterCropWide(size=(cfg.resolution[0], cfg.resolution[0])), + data.Resize(cfg.vit_resolution), + data.ToTensor(), + data.Normalize(mean=cfg.vit_mean, std=cfg.vit_std) + ]) + self.vid_trans = vid_trans + + cfg.embedder.pretrained = osp.join( + model_dir, self.config.model.model_args.ckpt_clip) + clip_encoder = FrozenOpenCLIPVisualEmbedder( + device=self.device, **cfg.embedder) + clip_encoder.model.to(self.device) + self.clip_encoder = clip_encoder + logger.info(f'Build encoder with {cfg.embedder.type}') + + # [unet] + generator = Img2VidSDUNet(**cfg.UNet) + generator = generator.to(self.device) + generator.eval() + load_dict = torch.load(cfg.model_path, map_location='cpu') + ret = generator.load_state_dict(load_dict['state_dict'], strict=True) + self.generator = generator + logger.info('Load model {} path {}, with local status {}'.format( + cfg.UNet.type, cfg.model_path, ret)) + + # [diffusion] + betas = beta_schedule( + 'linear_sd', + cfg.num_timesteps, + init_beta=0.00085, + last_beta=0.0120) + diffusion = GaussianDiffusion( + betas=betas, + mean_type=cfg.mean_type, + var_type=cfg.var_type, + loss_type=cfg.loss_type, + rescale_timesteps=False, + noise_strength=getattr(cfg, 'noise_strength', 0)) + self.diffusion = diffusion + logger.info('Build diffusion with type of GaussianDiffusion') + + # [auotoencoder] + cfg.auto_encoder.pretrained = osp.join( + model_dir, self.config.model.model_args.ckpt_autoencoder) + autoencoder = AutoencoderKL(**cfg.auto_encoder) + autoencoder.eval() + for param in autoencoder.parameters(): + param.requires_grad = False + autoencoder.to(self.device) + self.autoencoder = autoencoder + torch.cuda.empty_cache() + + zero_feature = torch.zeros(1, 1, cfg.UNet.input_dim).to(self.device) + self.zero_feature = zero_feature + self.fps_tensor = torch.tensor([cfg.target_fps], + dtype=torch.long, + device=self.device) + self.cfg = cfg + + def forward(self, input: Dict[str, Any]): + r""" + The entry function of image to video task. + 1. Using diffusion model to generate the video's latent representation. + 2. Using autoencoder to decode the video's latent representation to visual space. + + Args: + input (`Dict[Str, Any]`): + The input of the task + Returns: + A generated video (as pytorch tensor). + """ + + vit_frame = input['vit_frame'] + cfg = self.cfg + + img_embedding = self.clip_encoder(vit_frame).unsqueeze(1) + + noise = self.build_noise() + zero_feature = copy(self.zero_feature) + with torch.no_grad(): + with amp.autocast(enabled=cfg.use_fp16): + model_kwargs = [{ + 'y': img_embedding, + 'fps': self.fps_tensor + }, { + 'y': zero_feature.repeat(cfg.batch_size, 1, 1), + 'fps': self.fps_tensor + }] + gen_video = self.diffusion.ddim_sample_loop( + noise=noise, + model=self.generator, + model_kwargs=model_kwargs, + guide_scale=cfg.guide_scale, + ddim_timesteps=cfg.ddim_timesteps, + eta=0.0) + + gen_video = 1. / cfg.scale_factor * gen_video + gen_video = rearrange(gen_video, 'b c f h w -> (b f) c h w') + chunk_size = min(cfg.decoder_bs, gen_video.shape[0]) + gen_video_list = torch.chunk( + gen_video, gen_video.shape[0] // chunk_size, dim=0) + decode_generator = [] + for vd_data in gen_video_list: + gen_frames = self.autoencoder.decode(vd_data) + decode_generator.append(gen_frames) + + gen_video = torch.cat(decode_generator, dim=0) + gen_video = rearrange( + gen_video, '(b f) c h w -> b c f h w', b=cfg.batch_size) + + return gen_video.type(torch.float32).cpu() + + def build_noise(self): + cfg = self.cfg + noise = torch.randn( + [1, 4, cfg.max_frames, cfg.latent_hei, + cfg.latent_wid]).to(self.device) + if cfg.noise_strength > 0: + b, c, f, *_ = noise.shape + offset_noise = torch.randn(b, c, f, 1, 1, device=noise.device) + noise = noise + cfg.noise_strength * offset_noise + return noise.contiguous() diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/__init__.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc69102be02565f5d544ed2e56060bd4658d7504 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .autoencoder import * +from .embedder import * +from .unet_i2v import * diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/autoencoder.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..34e8b7b0fcf7ba5e4f9ed86d366aa7a7433e8934 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/autoencoder.py @@ -0,0 +1,576 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import collections + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm( + num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +class DiagonalGaussianDistribution(object): + + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean).to(device=self.parameters.device) + + def sample(self): + x = self.mean + self.std * torch.randn( + self.mean.shape).to(device=self.parameters.device) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3]) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var - 1.0 - self.logvar + other.logvar, + dim=[1, 2, 3]) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + + torch.pow(sample - self.mean, 2) / self.var, + dim=dims) + + def mode(self): + return self.mean + + +class ResnetBlock(nn.Module): + + def __init__(self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) + k = k.reshape(b, c, h * w) + w_ = torch.bmm(q, k) + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) + h_ = torch.bmm(v, w_) + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class Upsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x = torch.nn.functional.interpolate( + x, scale_factor=2.0, mode='nearest') + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class Encoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + use_linear_attn=False, + attn_type='vanilla', + **ignore_kwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type='vanilla', + **ignorekwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2**(self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + logger.info('Working with z of shape {} = {} dimensions.'.format( + self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class AutoencoderKL(nn.Module): + + def __init__(self, + ddconfig, + embed_dim, + pretrained=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + ema_decay=None, + learn_logvar=False, + **kwargs): + super().__init__() + self.learn_logvar = learn_logvar + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d(2 * ddconfig['z_channels'], + 2 * embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + + self.use_ema = ema_decay is not None + + if pretrained is not None: + self.init_from_ckpt(pretrained, ignore_keys=ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + sd_new = collections.OrderedDict() + for k in keys: + if k.find('first_stage_model') >= 0: + k_new = k.split('first_stage_model.')[-1] + sd_new[k_new] = sd[k] + self.load_state_dict(sd_new, strict=True) + logger.info(f'Restored from {path}') + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + return x + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, log_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + if log_ema or self.use_ema: + with self.ema_scope(): + xrec_ema, posterior_ema = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec_ema.shape[1] > 3 + xrec_ema = self.to_rgb(xrec_ema) + log['samples_ema'] = self.decode( + torch.randn_like(posterior_ema.sample())) + log['reconstructions_ema'] = xrec_ema + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class IdentityFirstStage(torch.nn.Module): + + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/embedder.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..69445667fd4486c2304700fd751dcd5459094ca2 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/embedder.py @@ -0,0 +1,81 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import numpy as np +import open_clip +import torch +import torch.nn as nn +import torchvision.transforms as T + + +class FrozenOpenCLIPVisualEmbedder(nn.Module): + """ + Uses the OpenCLIP transformer encoder for text + """ + LAYERS = ['last', 'penultimate'] + + def __init__(self, + pretrained, + vit_resolution=(224, 224), + arch='ViT-H-14', + device='cuda', + max_length=77, + freeze=True, + layer='last', + **kwargs): + super().__init__() + assert layer in self.LAYERS + model, _, preprocess = open_clip.create_model_and_transforms( + arch, device=torch.device('cpu'), pretrained=pretrained) + + del model.transformer + self.model = model + data_white = np.ones( + (vit_resolution[0], vit_resolution[1], 3), dtype=np.uint8) * 255 + self.white_image = preprocess(T.ToPILImage()(data_white)).unsqueeze(0) + + self.device = device + self.max_length = max_length + if freeze: + self.freeze() + self.layer = layer + if self.layer == 'last': + self.layer_idx = 0 + elif self.layer == 'penultimate': + self.layer_idx = 1 + else: + raise NotImplementedError() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, image): + z = self.model.encode_image(image.to(self.device)) + return z + + def encode_with_transformer(self, text): + x = self.model.token_embedding(text) + x = x + self.model.positional_embedding + x = x.permute(1, 0, 2) + x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) + x = x.permute(1, 0, 2) + x = self.model.ln_final(x) + + return x + + def text_transformer_forward(self, x: torch.Tensor, attn_mask=None): + for i, r in enumerate(self.model.transformer.resblocks): + if i == len(self.model.transformer.resblocks) - self.layer_idx: + break + if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting( + ): + x = checkpoint(r, x, attn_mask) + else: + x = r(x, attn_mask=attn_mask) + return x + + def encode(self, text): + return self(text) diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/unet_i2v.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/unet_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..dae226a5a4f70f66e2915cd855fdbd8ed0a2d0fd --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/modules/unet_i2v.py @@ -0,0 +1,1504 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import xformers +import xformers.ops +from einops import rearrange +from fairscale.nn.checkpoint import checkpoint_wrapper +from rotary_embedding_torch import RotaryEmbedding + +USE_TEMPORAL_TRANSFORMER = True + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if callable(d) else d + + +def prob_mask_like(shape, prob, device): + if prob == 1: + return torch.ones(shape, device=device, dtype=torch.bool) + elif prob == 0: + return torch.zeros(shape, device=device, dtype=torch.bool) + else: + mask = torch.zeros(shape, device=device).float().uniform_(0, 1) < prob + # aviod mask all, which will cause find_unused_parameters error + if mask.all(): + mask[0] = False + return mask + + +class MemoryEfficientCrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.0): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.heads = heads + self.dim_head = dim_head + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + self.attention_op: Optional[Any] = None + + def forward(self, x, context=None, mask=None): + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + b, _, _ = q.shape + q, k, v = map( + lambda t: t.unsqueeze(3).reshape(b, t.shape[ + 1], self.heads, self.dim_head).permute(0, 2, 1, 3).reshape( + b * self.heads, t.shape[1], self.dim_head).contiguous(), + (q, k, v), + ) + + out = xformers.ops.memory_efficient_attention( + q, k, v, attn_bias=None, op=self.attention_op) + + if exists(mask): + raise NotImplementedError + out = ( + out.unsqueeze(0).reshape( + b, self.heads, out.shape[1], + self.dim_head).permute(0, 2, 1, + 3).reshape(b, out.shape[1], + self.heads * self.dim_head)) + return self.to_out(out) + + +class RelativePositionBias(nn.Module): + + def __init__(self, heads=8, num_buckets=32, max_distance=128): + super().__init__() + self.num_buckets = num_buckets + self.max_distance = max_distance + self.relative_attention_bias = nn.Embedding(num_buckets, heads) + + @staticmethod + def _relative_position_bucket(relative_position, + num_buckets=32, + max_distance=128): + ret = 0 + n = -relative_position + + num_buckets //= 2 + ret += (n < 0).long() * num_buckets + n = torch.abs(n) + + max_exact = num_buckets // 2 + is_small = n < max_exact + + val_if_large = max_exact + ( + torch.log(n.float() / max_exact) + / math.log(max_distance / max_exact) * # noqa + (num_buckets - max_exact)).long() + val_if_large = torch.min( + val_if_large, torch.full_like(val_if_large, num_buckets - 1)) + + ret += torch.where(is_small, n, val_if_large) + return ret + + def forward(self, n, device): + q_pos = torch.arange(n, dtype=torch.long, device=device) + k_pos = torch.arange(n, dtype=torch.long, device=device) + rel_pos = rearrange(k_pos, 'j -> 1 j') - rearrange(q_pos, 'i -> i 1') + rp_bucket = self._relative_position_bucket( + rel_pos, + num_buckets=self.num_buckets, + max_distance=self.max_distance) + values = self.relative_attention_bias(rp_bucket) + return rearrange(values, 'i j h -> h i j') + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + NEW: use_linear for more efficiency instead of the 1x1 convs + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True): + super().__init__() + if exists(context_dim) and not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + disable_self_attn=disable_self_attn, + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if not isinstance(context, list): + context = [context] + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + if not self.use_linear: + x = self.proj_in(x) + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + if self.use_linear: + x = self.proj_in(x) + for i, block in enumerate(self.transformer_blocks): + x = block(x, context=context[i]) + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = self.proj_out(x) + return x + x_in + + +_ATTN_PRECISION = os.environ.get('ATTN_PRECISION', 'fp32') + + +class CrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + + # force cast to fp32 to avoid overflowing + if _ATTN_PRECISION == 'fp32': + with torch.autocast(enabled=False, device_type='cuda'): + q, k = q.float(), k.float() + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + else: + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + + del q, k + + if exists(mask): + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + sim = sim.softmax(dim=-1) + + out = torch.einsum('b i j, b j d -> b i d', sim, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + return self.to_out(out) + + +class BasicTransformerBlock(nn.Module): + + def __init__(self, + dim, + n_heads, + d_head, + dropout=0., + context_dim=None, + gated_ff=True, + checkpoint=True, + disable_self_attn=False): + super().__init__() + + attn_cls = MemoryEfficientCrossAttention + self.disable_self_attn = disable_self_attn + self.attn1 = attn_cls( + query_dim=dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + context_dim=context_dim if self.disable_self_attn else None) + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = attn_cls( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout) + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward_(self, x, context=None): + return checkpoint(self._forward, (x, context), self.parameters(), + self.checkpoint) + + def forward(self, x, context=None): + x = self.attn1( + self.norm1(x), + context=context if self.disable_self_attn else None) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +# feedforward +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = nn.Conv2d( + self.channels, self.out_channels, 3, padding=padding) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + if self.use_conv: + x = self.conv(x) + return x + + +class ResBlock(nn.Module): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + up=False, + down=False, + use_temporal_conv=True, + use_image_dataset=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_scale_shift_norm = use_scale_shift_norm + self.use_temporal_conv = use_temporal_conv + + self.in_layers = nn.Sequential( + nn.GroupNorm(32, channels), + nn.SiLU(), + nn.Conv2d(channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + nn.Linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + nn.GroupNorm(32, self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + nn.Conv2d(self.out_channels, self.out_channels, 3, padding=1)), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = nn.Conv2d(channels, self.out_channels, 1) + + if self.use_temporal_conv: + self.temopral_conv = TemporalConvBlock_v2( + self.out_channels, + self.out_channels, + dropout=0.1, + use_image_dataset=use_image_dataset) + + def forward(self, x, emb, batch_size): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return self._forward(x, emb, batch_size) + + def _forward(self, x, emb, batch_size): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + h = self.skip_connection(x) + h + + if self.use_temporal_conv: + h = rearrange(h, '(b f) c h w -> b c f h w', b=batch_size) + h = self.temopral_conv(h) + h = rearrange(h, 'b c f h w -> (b f) c h w') + return h + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = nn.Conv2d( + self.channels, + self.out_channels, + 3, + stride=stride, + padding=padding) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, mode): + assert mode in ['none', 'upsample', 'downsample'] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.mode = mode + + def forward(self, x, reference=None): + if self.mode == 'upsample': + assert reference is not None + x = F.interpolate(x, size=reference.shape[-2:], mode='nearest') + elif self.mode == 'downsample': + x = F.adaptive_avg_pool2d( + x, output_size=tuple(u // 2 for u in x.shape[-2:])) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + mode='none', + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.mode = mode + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, mode) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e, reference=None): + identity = self.resample(x, reference) + x = self.layer1[-1](self.resample(self.layer1[:-1](x), reference)) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class TemporalAttentionBlock(nn.Module): + + def __init__(self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False): + super().__init__() + # consider num_heads first, as pos_bias needs fixed num_heads + dim_head = dim // heads + assert heads * dim_head == dim + self.use_image_dataset = use_image_dataset + self.use_sim_mask = use_sim_mask + + self.scale = dim_head**-0.5 + self.heads = heads + hidden_dim = dim_head * heads + + self.norm = nn.GroupNorm(32, dim) + self.rotary_emb = rotary_emb + self.to_qkv = nn.Linear(dim, hidden_dim * 3) + self.to_out = nn.Linear(hidden_dim, dim) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + + identity = x + n, height, device = x.shape[2], x.shape[-2], x.device + + x = self.norm(x) + x = rearrange(x, 'b c f h w -> b (h w) f c') + + qkv = self.to_qkv(x).chunk(3, dim=-1) + + if exists(focus_present_mask) and focus_present_mask.all(): + # if all batch samples are focusing on present + # it would be equivalent to passing that token's values (v=qkv[-1]) through to the output + values = qkv[-1] + out = self.to_out(values) + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + return out + identity + + # split out heads + # shape [b (hw) h n c/h], n=f + q = rearrange(qkv[0], '... n (h d) -> ... h n d', h=self.heads) + k = rearrange(qkv[1], '... n (h d) -> ... h n d', h=self.heads) + v = rearrange(qkv[2], '... n (h d) -> ... h n d', h=self.heads) + + # scale + + q = q * self.scale + + # rotate positions into queries and keys for time attention + if exists(self.rotary_emb): + q = self.rotary_emb.rotate_queries_or_keys(q) + k = self.rotary_emb.rotate_queries_or_keys(k) + + # similarity + # shape [b (hw) h n n], n=f + sim = torch.einsum('... h i d, ... h j d -> ... h i j', q, k) + + # relative positional bias + + if exists(pos_bias): + sim = sim + pos_bias + + if (focus_present_mask is None and video_mask is not None): + # video_mask: [B, n] + mask = video_mask[:, None, :] * video_mask[:, :, None] + mask = mask.unsqueeze(1).unsqueeze(1) + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + elif exists(focus_present_mask) and not (~focus_present_mask).all(): + attend_all_mask = torch.ones((n, n), + device=device, + dtype=torch.bool) + attend_self_mask = torch.eye(n, device=device, dtype=torch.bool) + + mask = torch.where( + rearrange(focus_present_mask, 'b -> b 1 1 1 1'), + rearrange(attend_self_mask, 'i j -> 1 1 1 i j'), + rearrange(attend_all_mask, 'i j -> 1 1 1 i j'), + ) + + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + + if self.use_sim_mask: + sim_mask = torch.tril( + torch.ones((n, n), device=device, dtype=torch.bool), + diagonal=0) + sim = sim.masked_fill(~sim_mask, -torch.finfo(sim.dtype).max) + + # numerical stability + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + # aggregate values + + out = torch.einsum('... h i j, ... h j d -> ... h i d', attn, v) + out = rearrange(out, '... h n d -> ... n (h d)') + out = self.to_out(out) + + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + if self.use_image_dataset: + out = identity + 0 * out + else: + out = identity + out + return out + + +class TemporalTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True, + only_self_att=True, + multiply_zero=False): + super().__init__() + self.multiply_zero = multiply_zero + self.only_self_att = only_self_att + self.use_adaptor = False + if self.only_self_att: + context_dim = None + if not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv1d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + if self.use_adaptor: + self.adaptor_in = nn.Linear(frames, frames) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv1d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + if self.use_adaptor: + self.adaptor_out = nn.Linear(frames, frames) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if self.only_self_att: + context = None + if not isinstance(context, list): + context = [context] + b, c, f, h, w = x.shape + x_in = x + x = self.norm(x) + + if not self.use_linear: + x = rearrange(x, 'b c f h w -> (b h w) c f').contiguous() + x = self.proj_in(x) + # [16384, 16, 320] + if self.use_linear: + x = rearrange( + x, '(b f) c h w -> b (h w) f c', f=self.frames).contiguous() + x = self.proj_in(x) + + if self.only_self_att: + x = rearrange(x, 'bhw c f -> bhw f c').contiguous() + for i, block in enumerate(self.transformer_blocks): + x = block(x) + x = rearrange(x, '(b hw) f c -> b hw f c', b=b).contiguous() + else: + x = rearrange(x, '(b hw) c f -> b hw f c', b=b).contiguous() + for i, block in enumerate(self.transformer_blocks): + context[i] = rearrange( + context[i], '(b f) l con -> b f l con', + f=self.frames).contiguous() + # calculate each batch one by one (since number in shape could not greater then 65,535 for some package) + for j in range(b): + context_i_j = repeat( + context[i][j], + 'f l con -> (f r) l con', + r=(h * w) // self.frames, + f=self.frames).contiguous() + x[j] = block(x[j], context=context_i_j) + + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) f c -> b f c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = rearrange(x, 'b hw f c -> (b hw) c f').contiguous() + x = self.proj_out(x) + x = rearrange( + x, '(b h w) c f -> b c f h w', b=b, h=h, w=w).contiguous() + + if self.multiply_zero: + x = 0.0 * x + x_in + else: + x = x + x_in + return x + + +class TemporalAttentionMultiBlock(nn.Module): + + def __init__( + self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False, + temporal_attn_times=1, + ): + super().__init__() + self.att_layers = nn.ModuleList([ + TemporalAttentionBlock(dim, heads, dim_head, rotary_emb, + use_image_dataset, use_sim_mask) + for _ in range(temporal_attn_times) + ]) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + for layer in self.att_layers: + x = layer(x, pos_bias, focus_present_mask, video_mask) + return x + + +class InitTemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(InitTemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv[-1].weight) + nn.init.zeros_(self.conv[-1].bias) + + def forward(self, x): + identity = x + x = self.conv(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv2[-1].weight) + nn.init.zeros_(self.conv2[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock_v2(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock_v2, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv3 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv4 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv4[-1].weight) + nn.init.zeros_(self.conv4[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + + if self.use_image_dataset: + x = identity + 0.0 * x + else: + x = identity + x + return x + + +class Img2VidSDUNet(nn.Module): + + def __init__(self, + in_dim=7, + dim=512, + y_dim=512, + num_tokens=4, + context_dim=512, + out_dim=6, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + use_scale_shift_norm=True, + dropout=0.1, + default_fps=8, + temporal_attn_times=1, + temporal_attention=True, + use_checkpoint=False, + use_image_dataset=False, + use_sim_mask=False, + training=True, + inpainting=True, + **kwargs): + embed_dim = dim * 4 + num_heads = num_heads if num_heads else dim // 32 + super(Img2VidSDUNet, self).__init__() + self.in_dim = in_dim + self.num_tokens = num_tokens + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + # for temporal attention + self.num_heads = num_heads + # for spatial attention + self.default_fps = default_fps + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.use_scale_shift_norm = use_scale_shift_norm + self.temporal_attn_times = temporal_attn_times + self.temporal_attention = temporal_attention + self.use_checkpoint = use_checkpoint + self.use_image_dataset = use_image_dataset + self.use_sim_mask = use_sim_mask + self.training = training + self.inpainting = inpainting + + use_linear_in_temporal = False + transformer_depth = 1 + disabled_sa = False + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embed = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + + self.context_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, context_dim * self.num_tokens)) + + self.fps_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + nn.init.zeros_(self.fps_embedding[-1].weight) + nn.init.zeros_(self.fps_embedding[-1].bias) + + if temporal_attention and not USE_TEMPORAL_TRANSFORMER: + self.rotary_emb = RotaryEmbedding(min(32, head_dim)) + self.time_rel_pos_bias = RelativePositionBias( + heads=num_heads, max_distance=32) + + # encoder + self.input_blocks = nn.ModuleList() + init_block = nn.ModuleList([nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + # need an initial temporal attention? + if temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + init_block.append( + TemporalTransformer( + dim, + num_heads, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + init_block.append( + TemporalAttentionMultiBlock( + dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + temporal_attn_times=temporal_attn_times, + use_image_dataset=use_image_dataset)) + + self.input_blocks.append(init_block) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + block = nn.ModuleList([ + ResBlock( + in_dim, + embed_dim, + dropout, + out_channels=out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + self.input_blocks.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + downsample = Downsample( + out_dim, True, dims=2, out_channels=out_dim) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.input_blocks.append(downsample) + + self.middle_block = nn.ModuleList([ + ResBlock( + out_dim, + embed_dim, + dropout, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ), + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True) + ]) + + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + self.middle_block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset, + )) + else: + self.middle_block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + + self.middle_block.append( + ResBlock(out_dim, embed_dim, dropout, use_scale_shift_norm=False)) + + # decoder + self.output_blocks = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + block = nn.ModuleList([ + ResBlock( + in_dim + shortcut_dims.pop(), + embed_dim, + dropout, + out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=1024, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + upsample = Upsample( + out_dim, True, dims=2.0, out_channels=out_dim) + scale *= 2.0 + block.append(upsample) + self.output_blocks.append(block) + + # head + self.out = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.out[-1].weight) + + def forward(self, + x, + t, + y, + fps=None, + video_mask=None, + focus_present_mask=None, + prob_focus_present=0., + mask_last_frame_num=0, + **kwargs): + + batch, c, f, h, w = x.shape + device = x.device + self.batch = batch + if fps is None: + fps = torch.tensor( + [cfg.default_fps] * batch, dtype=torch.long, device=device) + + # image and video joint training, if mask_last_frame_num is set, prob_focus_present will be ignored + if mask_last_frame_num > 0: + focus_present_mask = None + video_mask[-mask_last_frame_num:] = False + else: + focus_present_mask = default( + focus_present_mask, lambda: prob_mask_like( + (batch, ), prob_focus_present, device=device)) + + if self.temporal_attention and not USE_TEMPORAL_TRANSFORMER: + time_rel_pos_bias = self.time_rel_pos_bias( + x.shape[2], device=x.device) + else: + time_rel_pos_bias = None + + # embeddings + embeddings = self.time_embed(sinusoidal_embedding( + t, self.dim)) + self.fps_embedding( + sinusoidal_embedding(fps, self.dim)) + + context = self.context_embedding(y) + context = context.view(-1, self.num_tokens, self.context_dim) + + # repeat f times for spatial e and context + embeddings = embeddings.repeat_interleave(repeats=f, dim=0) + context = context.repeat_interleave(repeats=f, dim=0) + + # always in shape (b f) c h w, except for temporal layer + x = rearrange(x, 'b c f h w -> (b f) c h w') + # encoder + xs = [] + for block in self.input_blocks: + x = self._forward_single(block, x, embeddings, context, + time_rel_pos_bias, focus_present_mask, + video_mask) + xs.append(x) + + # middle + for block in self.middle_block: + x = self._forward_single(block, x, embeddings, context, + time_rel_pos_bias, focus_present_mask, + video_mask) + + # decoder + for block in self.output_blocks: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single( + block, + x, + embeddings, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=xs[-1] if len(xs) > 0 else None) + + # head + x = self.out(x) + + # reshape back to (b c f h w) + x = rearrange(x, '(b f) c h w -> b c f h w', b=batch) + return x + + def _forward_single(self, + module, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=None): + if isinstance(module, ResidualBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, reference) + elif isinstance(module, ResBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, self.batch) + elif isinstance(module, SpatialTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, TemporalTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, context) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, CrossAttention): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, MemoryEfficientCrossAttention): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, BasicTransformerBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, FeedForward): + x = module(x, context) + elif isinstance(module, Upsample): + x = module(x) + elif isinstance(module, Downsample): + x = module(x) + elif isinstance(module, Resample): + x = module(x, reference) + elif isinstance(module, TemporalAttentionBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalAttentionMultiBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, InitTemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context, + time_rel_pos_bias, focus_present_mask, + video_mask, reference) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/__init__.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..92654e519e5750b9df8b976efbc8ecec635495e3 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/config.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..f5e71cdea0db74bece6d54c272363dc6d595fb98 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/config.py @@ -0,0 +1,161 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import logging +import os +import os.path as osp +from datetime import datetime + +import torch +from easydict import EasyDict + +cfg = EasyDict(__name__='Config: VideoLDM Decoder') + +# ---------------------------work dir-------------------------- +cfg.work_dir = 'workspace/' + +# ---------------------------Global Variable----------------------------------- +cfg.resolution = [448, 256] +# ----------------------------------------------------------------------------- + +# ---------------------------Dataset Parameter--------------------------------- +cfg.mean = [0.5, 0.5, 0.5] +cfg.std = [0.5, 0.5, 0.5] +cfg.max_words = 1000 + +# PlaceHolder +cfg.vit_out_dim = 1024 +cfg.vit_resolution = [224, 224] +cfg.depth_clamp = 10.0 +cfg.misc_size = 384 +cfg.depth_std = 20.0 + +cfg.frame_lens = 32 +cfg.sample_fps = 8 + +cfg.batch_sizes = 1 +# ----------------------------------------------------------------------------- + +# ---------------------------Mode Parameters----------------------------------- +# Diffusion +cfg.schedule = 'cosine' +cfg.num_timesteps = 1000 +cfg.mean_type = 'v' +cfg.var_type = 'fixed_small' +cfg.loss_type = 'mse' +cfg.ddim_timesteps = 50 +cfg.ddim_eta = 0.0 +cfg.clamp = 1.0 +cfg.share_noise = False +cfg.use_div_loss = False +cfg.noise_strength = 0.1 + +# classifier-free guidance +cfg.p_zero = 0.1 +cfg.guide_scale = 3.0 + +# clip vision encoder +cfg.vit_mean = [0.48145466, 0.4578275, 0.40821073] +cfg.vit_std = [0.26862954, 0.26130258, 0.27577711] + +# Model +cfg.scale_factor = 0.18215 +cfg.use_fp16 = True +cfg.temporal_attention = True +cfg.decoder_bs = 8 + +cfg.UNet = { + 'type': 'Img2VidSDUNet', + 'in_dim': 4, + 'dim': 320, + 'y_dim': cfg.vit_out_dim, + 'context_dim': 1024, + 'out_dim': 8 if cfg.var_type.startswith('learned') else 4, + 'dim_mult': [1, 2, 4, 4], + 'num_heads': 8, + 'head_dim': 64, + 'num_res_blocks': 2, + 'attn_scales': [1 / 1, 1 / 2, 1 / 4], + 'dropout': 0.1, + 'temporal_attention': cfg.temporal_attention, + 'temporal_attn_times': 1, + 'use_checkpoint': False, + 'use_fps_condition': False, + 'use_sim_mask': False, + 'num_tokens': 4, + 'default_fps': 8, + 'input_dim': 1024 +} + +cfg.guidances = [] + +# auotoencoder from stabel diffusion +cfg.auto_encoder = { + 'type': 'AutoencoderKL', + 'ddconfig': { + 'double_z': True, + 'z_channels': 4, + 'resolution': 256, + 'in_channels': 3, + 'out_ch': 3, + 'ch': 128, + 'ch_mult': [1, 2, 4, 4], + 'num_res_blocks': 2, + 'attn_resolutions': [], + 'dropout': 0.0 + }, + 'embed_dim': 4, + 'pretrained': 'v2-1_512-ema-pruned.ckpt' +} +# clip embedder +cfg.embedder = { + 'type': 'FrozenOpenCLIPVisualEmbedder', + 'layer': 'penultimate', + 'vit_resolution': [224, 224], + 'pretrained': 'open_clip_pytorch_model.bin' +} +# ----------------------------------------------------------------------------- + +# ---------------------------Training Settings--------------------------------- +# training and optimizer +cfg.ema_decay = 0.9999 +cfg.num_steps = 600000 +cfg.lr = 5e-5 +cfg.weight_decay = 0.0 +cfg.betas = (0.9, 0.999) +cfg.eps = 1.0e-8 +cfg.chunk_size = 16 +cfg.alpha = 0.7 +cfg.save_ckp_interval = 1000 +# ----------------------------------------------------------------------------- + +# ----------------------------Pretrain Settings--------------------------------- +# Default: load 2d pretrain +cfg.fix_weight = False +cfg.load_match = False +cfg.pretrained_checkpoint = 'v2-1_512-ema-pruned.ckpt' +cfg.pretrained_image_keys = 'stable_diffusion_image_key_temporal_attention_x1.json' +cfg.resume_checkpoint = 'img2video_ldm_0779000.pth' +# ----------------------------------------------------------------------------- + +# -----------------------------Visual------------------------------------------- +# Visual videos +cfg.viz_interval = 1000 +cfg.visual_train = { + 'type': 'VisualVideoTextDuringTrain', +} +cfg.visual_inference = { + 'type': 'VisualGeneratedVideos', +} +cfg.inference_list_path = '' + +# logging +cfg.log_interval = 100 + +# Default log_dir +cfg.log_dir = 'workspace/output_data' +# ----------------------------------------------------------------------------- + +# ---------------------------Others-------------------------------------------- +# seed +cfg.seed = 8888 +# ----------------------------------------------------------------------------- diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/diffusion.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..fc4b96cf58f8018c05a0a783050ebdf442d22b1e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/diffusion.py @@ -0,0 +1,511 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + if tensor.device != x.device: + tensor = tensor.to(x.device) + return tensor[t].view(shape).to(x) + + +def fn(u): + return math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + betas.append(min(1.0 - fn(t2) / fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + epsilon=1e-12, + rescale_timesteps=False, + noise_strength=0.0): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps', 'v'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1', + 'charbonnier' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.epsilon = epsilon + self.rescale_timesteps = rescale_timesteps + self.noise_strength = noise_strength + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def sample_loss(self, x0, noise=None): + if noise is None: + noise = torch.randn_like(x0) + if self.noise_strength > 0: + b, c, f, _, _ = x0.shape + offset_noise = torch.randn(b, c, f, 1, 1, device=x0.device) + noise = noise + self.noise_strength * offset_noise + return noise + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + # noise = torch.randn_like(x0) if noise is None else noise + noise = self.sample_loss(x0, noise) + return _i(self.sqrt_alphas_cumprod, t, x0) * x0 + ( + _i(self.sqrt_one_minus_alphas_cumprod, t, x0) * noise) + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + dim = y_out.size(1) if self.var_type.startswith( + 'fixed') else y_out.size(1) // 2 + out = torch.cat( + [ + u_out[:, :dim] + guide_scale * # noqa + (y_out[:, :dim] - u_out[:, :dim]), + y_out[:, dim:] + ], + dim=1) + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - ( + _i(self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + xt) * xt) + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * out) + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'v': + x0 = _i(self.sqrt_alphas_cumprod, t, xt) * xt - ( + _i(self.sqrt_one_minus_alphas_cumprod, t, xt) * out) + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + r"""Sample from p(x_{t-1} | x_t) using DDIM. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt)) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt)) + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + sigmas = eta * torch.sqrt((1 - alphas_prev) / (1 - alphas) * # noqa + (1 - alphas / alphas_prev)) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt)) + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b = x0.size(0) + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt)) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps) + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt)) + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - ( + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps) + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/seed.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/seed.py new file mode 100644 index 0000000000000000000000000000000000000000..df3c9c50e72e69ca6c93cb74bff272d041fdd78f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/seed.py @@ -0,0 +1,14 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import random + +import numpy as np +import torch + + +def setup_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/shedule.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/shedule.py new file mode 100644 index 0000000000000000000000000000000000000000..a255be15d4fd9e480c799b41070bea4cefaf9f2f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/shedule.py @@ -0,0 +1,60 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch + + +def fn(u): + return math.cos((u + 0.008) / 1.008 * math.pi / 2)**2 + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + ''' + This code defines a function beta_schedule that generates a sequence of beta values based on the given input + parameters. These beta values can be used in video diffusion processes. The function has the following parameters: + schedule(str): Determines the type of beta schedule to be generated. It can be 'linear', 'linear_sd', + 'quadratic', or 'cosine'. + num_timesteps(int, optional): The number of timesteps for the generated beta schedule. Default is 1000. + init_beta(float, optional): The initial beta value. If not provided, a default value is used based on the + chosen schedule. + last_beta(float, optional): The final beta value. If not provided, a default value is used based on the + chosen schedule. + The function returns a PyTorch tensor containing the generated beta values. + The beta schedule is determined by the schedule parameter: + 1.Linear: Generates a linear sequence of beta values betweeninit_betaandlast_beta. + 2.Linear_sd: Generates a linear sequence of beta values between the square root of init_beta and the square root + oflast_beta, and then squares the result. + 3.Quadratic: Similar to the 'linear_sd' schedule, but with different default values forinit_betaandlast_beta. + 4.Cosine: Generates a sequence of beta values based on a cosine function, ensuring the values are between 0 + and 0.999. + If an unsupported schedule is provided, a ValueError is raised with a message indicating the issue. + ''' + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'linear_sd': + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + betas.append(min(1.0 - fn(t2) / fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') diff --git a/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/transforms.py b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..3663620f0a646d7574d4d1852a0acce86eace2a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/image_to_video/utils/transforms.py @@ -0,0 +1,404 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import random + +import numpy as np +import torch +import torchvision.transforms.functional as F +from PIL import Image, ImageFilter + +__all__ = [ + 'Compose', 'Resize', 'Rescale', 'CenterCrop', 'CenterCropV2', + 'CenterCropWide', 'RandomCrop', 'RandomCropV2', 'RandomHFlip', + 'GaussianBlur', 'ColorJitter', 'RandomGray', 'ToTensor', 'Normalize', + 'ResizeRandomCrop', 'ExtractResizeRandomCrop', 'ExtractResizeAssignCrop' +] + + +class Compose(object): + + def __init__(self, transforms): + self.transforms = transforms + + def __getitem__(self, index): + if isinstance(index, slice): + return Compose(self.transforms[index]) + else: + return self.transforms[index] + + def __len__(self): + return len(self.transforms) + + def __call__(self, rgb): + for t in self.transforms: + rgb = t(rgb) + return rgb + + +class Resize(object): + + def __init__(self, size=256): + if isinstance(size, int): + size = (size, size) + self.size = size + + def __call__(self, rgb): + if isinstance(rgb, list): + rgb = [u.resize(self.size, Image.BILINEAR) for u in rgb] + else: + rgb = rgb.resize(self.size, Image.BILINEAR) + return rgb + + +class Rescale(object): + + def __init__(self, size=256, interpolation=Image.BILINEAR): + self.size = size + self.interpolation = interpolation + + def __call__(self, rgb): + w, h = rgb[0].size + scale = self.size / min(w, h) + out_w, out_h = int(round(w * scale)), int(round(h * scale)) + rgb = [u.resize((out_w, out_h), self.interpolation) for u in rgb] + return rgb + + +class CenterCrop(object): + + def __init__(self, size=224): + self.size = size + + def __call__(self, rgb): + w, h = rgb[0].size + assert min(w, h) >= self.size + x1 = (w - self.size) // 2 + y1 = (h - self.size) // 2 + rgb = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in rgb] + return rgb + + +class ResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + return rgb + + +class ExtractResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + wh = [x1, y1, x1 + out_w, y1 + out_h] + return rgb, wh + + +class ExtractResizeAssignCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb, wh): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + + rgb = [u.crop(wh) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class CenterCropV2(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + # fast resize + while min(img[0].size) >= 2 * self.size: + img = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in img + ] + scale = self.size / min(img[0].size) + img = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in img + ] + + # center crop + x1 = (img[0].width - self.size) // 2 + y1 = (img[0].height - self.size) // 2 + img = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in img] + return img + + +class CenterCropWide(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + if isinstance(img, list): + scale = min(img[0].size[0] / self.size[0], + img[0].size[1] / self.size[1]) + img = [ + u.resize((round(u.width // scale), round(u.height // scale)), + resample=Image.BOX) for u in img + ] + + # center crop + x1 = (img[0].width - self.size[0]) // 2 + y1 = (img[0].height - self.size[1]) // 2 + img = [ + u.crop((x1, y1, x1 + self.size[0], y1 + self.size[1])) + for u in img + ] + return img + else: + scale = min(img.size[0] / self.size[0], img.size[1] / self.size[1]) + img = img.resize( + (round(img.width // scale), round(img.height // scale)), + resample=Image.BOX) + x1 = (img.width - self.size[0]) // 2 + y1 = (img.height - self.size[1]) // 2 + img = img.crop((x1, y1, x1 + self.size[0], y1 + self.size[1])) + return img + + +class RandomCrop(object): + + def __init__(self, size=224, min_area=0.4): + self.size = size + self.min_area = min_area + + def __call__(self, rgb): + + # consistent crop between rgb and m + w, h = rgb[0].size + area = w * h + out_w, out_h = float('inf'), float('inf') + while out_w > w or out_h > h: + target_area = random.uniform(self.min_area, 1.0) * area + aspect_ratio = random.uniform(3. / 4., 4. / 3.) + out_w = int(round(math.sqrt(target_area * aspect_ratio))) + out_h = int(round(math.sqrt(target_area / aspect_ratio))) + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class RandomCropV2(object): + + def __init__(self, size=224, min_area=0.4, ratio=(3. / 4., 4. / 3.)): + if isinstance(size, (tuple, list)): + self.size = size + else: + self.size = (size, size) + self.min_area = min_area + self.ratio = ratio + + def _get_params(self, img): + width, height = img.size + area = height * width + + for _ in range(10): + target_area = random.uniform(self.min_area, 1.0) * area + log_ratio = (math.log(self.ratio[0]), math.log(self.ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if 0 < w <= width and 0 < h <= height: + i = random.randint(0, height - h) + j = random.randint(0, width - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = float(width) / float(height) + if (in_ratio < min(self.ratio)): + w = width + h = int(round(w / min(self.ratio))) + elif (in_ratio > max(self.ratio)): + h = height + w = int(round(h * max(self.ratio))) + else: + w = width + h = height + i = (height - h) // 2 + j = (width - w) // 2 + return i, j, h, w + + def __call__(self, rgb): + i, j, h, w = self._get_params(rgb[0]) + rgb = [F.resized_crop(u, i, j, h, w, self.size) for u in rgb] + return rgb + + +class RandomHFlip(object): + + def __init__(self, p=0.5): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.transpose(Image.FLIP_LEFT_RIGHT) for u in rgb] + return rgb + + +class GaussianBlur(object): + + def __init__(self, sigmas=[0.1, 2.0], p=0.5): + self.sigmas = sigmas + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + sigma = random.uniform(*self.sigmas) + rgb = [ + u.filter(ImageFilter.GaussianBlur(radius=sigma)) for u in rgb + ] + return rgb + + +class ColorJitter(object): + + def __init__(self, + brightness=0.4, + contrast=0.4, + saturation=0.4, + hue=0.1, + p=0.5): + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + brightness, contrast, saturation, hue = self._random_params() + transforms = [ + lambda f: F.adjust_brightness(f, brightness), + lambda f: F.adjust_contrast(f, contrast), + lambda f: F.adjust_saturation(f, saturation), + lambda f: F.adjust_hue(f, hue) + ] + random.shuffle(transforms) + for t in transforms: + rgb = [t(u) for u in rgb] + + return rgb + + def _random_params(self): + brightness = random.uniform( + max(0, 1 - self.brightness), 1 + self.brightness) + contrast = random.uniform(max(0, 1 - self.contrast), 1 + self.contrast) + saturation = random.uniform( + max(0, 1 - self.saturation), 1 + self.saturation) + hue = random.uniform(-self.hue, self.hue) + return brightness, contrast, saturation, hue + + +class RandomGray(object): + + def __init__(self, p=0.2): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.convert('L').convert('RGB') for u in rgb] + return rgb + + +class ToTensor(object): + + def __call__(self, rgb): + if isinstance(rgb, list): + rgb = torch.stack([F.to_tensor(u) for u in rgb], dim=0) + else: + rgb = F.to_tensor(rgb) + + return rgb + + +class Normalize(object): + + def __init__(self, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]): + self.mean = mean + self.std = std + + def __call__(self, rgb): + rgb = rgb.clone() + rgb.clamp_(0, 1) + if not isinstance(self.mean, torch.Tensor): + self.mean = rgb.new_tensor(self.mean).view(-1) + if not isinstance(self.std, torch.Tensor): + self.std = rgb.new_tensor(self.std).view(-1) + if rgb.dim() == 4: + rgb.sub_(self.mean.view(1, -1, 1, + 1)).div_(self.std.view(1, -1, 1, 1)) + elif rgb.dim() == 3: + rgb.sub_(self.mean.view(-1, 1, 1)).div_(self.std.view(-1, 1, 1)) + return rgb diff --git a/runtime/Lib/site_packages/models/multi_modal/mgeo/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mgeo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c311e8a67117dc79cbdee2f5bd50f5a5f9221007 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mgeo/__init__.py @@ -0,0 +1,41 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import (MGeo, MGeoPreTrainedModel) + from .text_classification import MGeoForSequenceClassification + from .token_classification import MGeoForTokenClassification + from .text_ranking import MGeoForTextRanking +else: + _import_structure = { + 'backbone': ['MGeo', 'MGeoPreTrainedModel'], + 'text_classification': ['MGeoForSequenceClassification'], + 'token_classification': ['MGeoForTokenClassification'], + 'text_ranking': ['MGeoForTextRanking'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/mgeo/backbone.py b/runtime/Lib/site_packages/models/multi_modal/mgeo/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..bed46af08d647e06201febb86eb370876d793f3a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mgeo/backbone.py @@ -0,0 +1,2507 @@ +import math +import os +import random +import warnings +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from torch import Tensor, device, dtype, nn +from torch.nn import CrossEntropyLoss, MSELoss +from transformers.activations import ACT2FN +from transformers.file_utils import (ModelOutput, add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings) +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions, MaskedLMOutput, + MultipleChoiceModelOutput, NextSentencePredictorOutput, + QuestionAnsweringModelOutput, SequenceClassifierOutput, + TokenClassifierOutput) +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.models.bert.configuration_bert import BertConfig +from transformers.utils import logging + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.nlp.utils import parse_labels_in_order + +transformers.logging.set_verbosity_error() + +logger = logging.get_logger() + +_CONFIG_FOR_DOC = 'BertConfig' +_TOKENIZER_FOR_DOC = 'BertTokenizer' + + +def load_tf_weights_in_bert(model, config, tf_checkpoint_path): + """Load tf checkpoints in a pytorch model.""" + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + 'Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see ' + 'https://www.tensorflow.org/install/ for installation instructions.' + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info('Converting TensorFlow checkpoint from {}'.format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info('Loading TF weight {} with shape {}'.format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to + # calculated m and v which are not required for using pretrained model + if any(n in [ + 'adam_v', 'adam_m', 'AdamWeightDecayOptimizer', + 'AdamWeightDecayOptimizer_1', 'global_step' + ] for n in name): + logger.info('Skipping {}'.format('/'.join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + scope_names = re.split(r'_(\d+)', m_name) + else: + scope_names = [m_name] + if scope_names[0] == 'kernel' or scope_names[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'output_bias' or scope_names[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif scope_names[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info('Skipping {}'.format('/'.join(name))) + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f'Pointer shape {pointer.shape} and array shape {array.shape} mismatched' + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info('Initialize PyTorch weight {}'.format(name)) + pointer.data = torch.from_numpy(array) + return model + + +class GisEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, config.hidden_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding( + config.type_vocab_size, config.hidden_size, padding_idx=0) + self.rel_type_embeddings = nn.Embedding( + config.rel_type_vocab_size, config.hidden_size, padding_idx=0) + self.absolute_x_embeddings = nn.Embedding( + config.absolute_x_vocab_size, config.hidden_size, padding_idx=0) + self.absolute_y_embeddings = nn.Embedding( + config.absolute_y_vocab_size, config.hidden_size, padding_idx=0) + self.relative_x_embeddings = nn.Embedding( + config.relative_x_vocab_size, config.hidden_size, padding_idx=0) + self.relative_y_embeddings = nn.Embedding( + config.relative_y_vocab_size, config.hidden_size, padding_idx=0) + if hasattr(config, 'prov_vocab_size'): + self.prov_embeddings = nn.Embedding( + config.prov_vocab_size, config.hidden_size, padding_idx=0) + self.city_embeddings = nn.Embedding( + config.city_vocab_size, config.hidden_size, padding_idx=0) + self.dist_embeddings = nn.Embedding( + config.dist_vocab_size, config.hidden_size, padding_idx=0) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + + self.config = config + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + rel_type_ids=None, + absolute_position_ids=None, + relative_position_ids=None, + prov_ids=None, + city_ids=None, + dist_ids=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings += self.rel_type_embeddings(rel_type_ids) + embeddings += self.absolute_x_embeddings(absolute_position_ids[:, :, + 0]) + embeddings += self.absolute_y_embeddings(absolute_position_ids[:, :, + 1]) + embeddings += self.absolute_x_embeddings(absolute_position_ids[:, :, + 2]) + embeddings += self.absolute_y_embeddings(absolute_position_ids[:, :, + 3]) + embeddings += self.relative_x_embeddings(relative_position_ids[:, :, + 0]) + embeddings += self.relative_y_embeddings(relative_position_ids[:, :, + 1]) + embeddings += self.relative_x_embeddings(relative_position_ids[:, :, + 2]) + embeddings += self.relative_y_embeddings(relative_position_ids[:, :, + 3]) + if prov_ids is not None: + embeddings += self.prov_embeddings(prov_ids) + embeddings += self.city_embeddings(city_ids) + embeddings += self.dist_embeddings(dist_ids) + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type + embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + + self.config = config + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + rel_type_ids=None, + absolute_position_ids=None, + relative_position_ids=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config, is_cross_attention): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + self.key = nn.Linear(config.encoder_width, self.all_head_size) + self.value = nn.Linear(config.encoder_width, self.all_head_size) + else: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if is_cross_attention and self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + outputs = outputs + (past_key_value, ) + return outputs + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config, is_cross_attention=False): + super().__init__() + self.self = BertSelfAttention(config, is_cross_attention) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config, layer_num): + super().__init__() + self.config = config + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BertAttention(config) + + self.has_cross_attention = (layer_num >= config.fusion_layer) + if self.has_cross_attention: + self.layer_num = layer_num + self.crossattention = BertAttention( + config, is_cross_attention=True) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at + # positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + + if self.has_cross_attention: + assert encoder_hidden_states is not None, 'encoder_hidden_states must be given for cross-attention layers' + + if type(encoder_hidden_states) == list: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states[(self.layer_num + - self.config.fusion_layer) + % len(encoder_hidden_states)], + encoder_attention_mask[(self.layer_num + - self.config.fusion_layer) + % len(encoder_hidden_states)], + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:-1] + + else: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1: + -1] # add cross attentions if we output attention weights + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [BertLayer(config, i) for i in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + mode='multi_modal', + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + + if mode == 'text': + start_layer = 0 + output_layer = self.config.fusion_layer + + elif mode == 'query': + start_layer = 0 + output_layer = self.config.num_hidden_layers + + elif mode == 'fusion': + start_layer = self.config.fusion_layer + output_layer = self.config.num_hidden_layers + + elif mode == 'multi_modal': + start_layer = 0 + output_layer = self.config.num_hidden_layers + + for i in range(start_layer, output_layer): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + + if use_cache: + logger.warn( + '`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting ' + '`use_cache=False`...') + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class BertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface + for downloading and loading pretrained models. + """ + + config_class = BertConfig + load_tf_weights = load_tf_weights_in_bert + base_model_prefix = 'bert' + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +@dataclass +class BertForPreTrainingOutput(ModelOutput): + """ + Output type of :class:`~transformers.BertForPreTraining`. Args: + loss (`optional`, returned when ``labels`` is provided, + ``torch.FloatTensor`` of shape :obj:`(1,)`): + Total loss as the sum of the masked language modeling loss and the + next sequence prediction (classification) loss. + prediction_logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, + sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each + vocabulary token before SoftMax). + seq_relationship_logits (:obj:`torch.FloatTensor` of shape + :obj:`(batch_size, 2)`): + Prediction scores of the next sequence prediction (classification) + head (scores of True/False continuation before SoftMax). + hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned + when ``output_hidden_states=True`` is passed or when + ``config.output_hidden_states=True``): + Tuple of :obj:`torch.FloatTensor` (one for the output of the + embeddings + one for the output of each layer) of shape + :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of + the model at the output of each layer plus the initial embedding + outputs. + attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when + ``output_attentions=True`` is passed or when + ``config.output_attentions=True``): + Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape + :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. + Attentions weights after the attention softmax, used to compute the + weighted average in the self-attention heads. + """ + + loss: Optional[torch.FloatTensor] = None + prediction_logits: torch.FloatTensor = None + seq_relationship_logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +class BertModel(BertPreTrainedModel): + """ + Noted that the bert model here is slightly updated from original bert, so we + maintian the code here independently. The Bert Model transformer outputting + raw hidden-states without any specific head on top. + + This model inherits from [`PreTrainedModel`]. Check the superclass + documentation for the generic methods the library implements for all its + model (such as downloading or saving, resizing the input embeddings, pruning + heads etc.) + + This model is also a PyTorch + [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch + documentation for all matter related to general usage and behavior. + + Parameters: + config ([`BertConfig`]): Model configuration class with all the + parameters of the model. + Initializing with a config file does not load the weights associated + with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a + decoder, in which case a layer of cross-attention is added between the + self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam + Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz + Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the + `is_decoder` argument of the configuration set to `True`. To be used in a + Seq2Seq model, the model needs to initialized with both `is_decoder` + argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` + is then expected as an input to the forward pass. + + + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + if config.gis_embedding == 0: + self.embeddings = BertEmbeddings(config) + else: + self.embeddings = GisEmbeddings(config) + + self.encoder = BertEncoder(config) + + self.pooler = BertPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def get_extended_attention_mask(self, attention_mask: Tensor, + input_shape: Tuple[int], device: device, + is_decoder: bool) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked + tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens + to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, with a the same + dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, + # from_seq_length, to_seq_length] ourselves in which case we just need + # to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to + # the padding mask + # - if the model is an encoder, make the mask broadcastable to + # [batch_size, num_heads, seq_length, seq_length] + if is_decoder: + batch_size, seq_length = input_shape + seq_ids = torch.arange(seq_length, device=device) + causal_mask = seq_ids[None, None, :].repeat( + batch_size, seq_length, 1) <= seq_ids[None, :, None] + # in case past_key_values are used we need to add a prefix ones + # mask to the causal mask causal and attention masks must have + # same type with pytorch version < 1.3 + causal_mask = causal_mask.to(attention_mask.dtype) + + if causal_mask.shape[1] < attention_mask.shape[1]: + prefix_seq_len = attention_mask.shape[ + 1] - causal_mask.shape[1] + causal_mask = torch.cat( + [ + torch.ones( + (batch_size, seq_length, prefix_seq_len), + device=device, + dtype=causal_mask.dtype), + causal_mask, + ], + axis=-1, + ) + + extended_attention_mask = causal_mask[:, + None, :, :] * attention_mask[:, + None, + None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + 'Wrong shape for input_ids (shape {}) or attention_mask (shape {})' + .format(input_shape, attention_mask.shape)) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 + # for masked positions, this operation will create a tensor which is 0.0 + # for positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multi_modal', + rel_type_ids=None, + absolute_position_ids=None, + relative_position_ids=None, + prov_ids=None, + city_ids=None, + dist_ids=None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`BertTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `((batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the + inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position + embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, + num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask + values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length, hidden_size)`, + *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a + plain tuple. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, + sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the + encoder. Used in the cross-attention if the model is configured as a + decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, + sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of + the encoder input. This mask is used in the cross-attention if the + model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length + `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only + the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead + of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned + and can be used to speed up decoding (see `past_key_values`). + Others (**kwargs) + some additional parameters might passed in from upstream pipeline, + which not influence the results. + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = inputs_embeds.device + elif encoder_embeds is not None: + input_shape = encoder_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = encoder_embeds.device + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds or encoder_embeds' + ) + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, + # from_seq_length, to_seq_length] ourselves in which case we just need + # to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device, is_decoder) + + # If a 2D or 3D attention mask is provided for the cross-attention we + # need to make broadcastable to [batch_size, num_heads, seq_length, + # seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[ + 0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [ + self.invert_attention_mask(mask) + for mask in encoder_attention_mask + ] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + if encoder_embeds is None: + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + rel_type_ids=rel_type_ids, + absolute_position_ids=absolute_position_ids, + relative_position_ids=relative_position_ids, + prov_ids=prov_ids, + city_ids=city_ids, + dist_ids=dist_ids, + ) + else: + embedding_output = encoder_embeds + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + mode=mode, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +class BertForPreTraining(BertPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.cls = BertPreTrainingHeads(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + next_sentence_label=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape ``(batch_size, sequence_length)``, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + next_sentence_label (``torch.LongTensor`` of shape ``(batch_size,)``, `optional`): + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair + (see :obj:`input_ids` docstring) Indices should be in ``[0, 1]``: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. + kwargs (:obj:`Dict[str, any]`, optional, defaults to `{}`): + Used to hide legacy arguments that have been deprecated. + Returns: + Example: + >>> from transformers import BertTokenizer, BertForPreTraining + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = BertForPreTraining.from_pretrained('bert-base-uncased') + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.prediction_logits + >>> seq_relationship_logits = outputs.seq_relationship_logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output, pooled_output = outputs[:2] + prediction_scores, seq_relationship_score = self.cls( + sequence_output, pooled_output) + + total_loss = None + if labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + next_sentence_loss = loss_fct( + seq_relationship_score.view(-1, 2), + next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + + if not return_dict: + output = (prediction_scores, seq_relationship_score) + outputs[2:] + return ((total_loss, ) + + output) if total_loss is not None else output + + return BertForPreTrainingOutput( + loss=total_loss, + prediction_logits=prediction_scores, + seq_relationship_logits=seq_relationship_score, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class BertLMHeadModel(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=True, + reduction='mean', + mode='multi_modal', + soft_labels=None, + alpha=0, + return_logits=False, + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape + :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the + encoder. Used in the cross-attention if the model is configured as a + decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape + :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of + the encoder input. This mask is used in the cross-attention if the + model is configured as a decoder. Mask values selected in ``[0, + 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that + are **masked**. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Labels for computing the left-to-right language modeling loss (next + word prediction). Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with + indices set to ``-100`` are ignored (masked), the loss is only + computed for the tokens with labels n ``[0, ..., + config.vocab_size]`` + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length + :obj:`config.n_layers` with each tuple having 4 tensors of shape + :obj:`(batch_size, num_heads, sequence_length - 1, + embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. If :obj:`past_key_values` + are used, the user can optionally input only the last + :obj:`decoder_input_ids` (those that don't have their past key value + states given to this model) of shape :obj:`(batch_size, 1)` instead + of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, + sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are + returned and can be used to speed up decoding (see + :obj:`past_key_values`). + + Returns: + + Example: + >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased') + >>> config = BertConfig.from_pretrained("bert-base-cased") + >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config) + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores[:, :-1, :].contiguous() + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, : + -1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss(reduction=reduction) + lm_loss = loss_fct( + shifted_prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + lm_loss = lm_loss.view(prediction_scores.size(0), -1).sum(1) + + if soft_labels is not None: + loss_distill = -torch.sum( + F.log_softmax(shifted_prediction_scores, dim=-1) * soft_labels, + dim=-1) + loss_distill = (loss_distill * (labels != -100)).sum(1) + lm_loss = (1 - alpha) * lm_loss + alpha * loss_distill + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((lm_loss, ) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, + input_ids, + past=None, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + + return { + 'input_ids': + input_ids, + 'attention_mask': + attention_mask, + 'past_key_values': + past, + 'encoder_hidden_states': + model_kwargs.get('encoder_hidden_states', None), + 'encoder_attention_mask': + model_kwargs.get('encoder_attention_mask', None), + 'is_decoder': + True, + } + + def _reorder_cache(self, past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple( + past_state.index_select(0, beam_idx) + for past_state in layer_past), ) + return reordered_past + + +class GisBertLMPredictionHead(nn.Module): + + def __init__(self, config, vocab_size): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.hidden_size, vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BertForGisMaskedLM(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config): + super().__init__(config) + self.bert = BertModel(config, add_pooling_layer=False) + self.cls_geom_id = GisBertLMPredictionHead(config, config.vocab_size) + self.cls_geom_type = GisBertLMPredictionHead(config, + config.type_vocab_size) + self.cls_rel_type = GisBertLMPredictionHead(config, + config.rel_type_vocab_size) + self.cls_absolute_position_x1 = GisBertLMPredictionHead( + config, config.absolute_x_vocab_size) + self.cls_absolute_position_x2 = GisBertLMPredictionHead( + config, config.absolute_x_vocab_size) + self.cls_absolute_position_y1 = GisBertLMPredictionHead( + config, config.absolute_y_vocab_size) + self.cls_absolute_position_y2 = GisBertLMPredictionHead( + config, config.absolute_y_vocab_size) + self.cls_relative_position_x1 = GisBertLMPredictionHead( + config, config.relative_x_vocab_size) + self.cls_relative_position_x2 = GisBertLMPredictionHead( + config, config.relative_x_vocab_size) + self.cls_relative_position_y1 = GisBertLMPredictionHead( + config, config.relative_y_vocab_size) + self.cls_relative_position_y2 = GisBertLMPredictionHead( + config, config.relative_y_vocab_size) + self.config = config + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multi_modal', + soft_labels=None, + alpha=0, + return_logits=False, + rel_type_ids=None, + absolute_position_ids=None, + relative_position_ids=None, + token_type_ids_label=None, + rel_type_ids_label=None, + absolute_position_ids_label=None, + relative_position_ids_label=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices + should be in ``[-100, 0, ..., config.vocab_size]`` (see + ``input_ids`` docstring) Tokens with indices set to ``-100`` are + ignored (masked), the loss is only computed for the tokens with + labels in ``[0, ..., config.vocab_size]`` + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_embeds=encoder_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + rel_type_ids=rel_type_ids, + absolute_position_ids=absolute_position_ids, + relative_position_ids=relative_position_ids, + ) + + sequence_output = outputs[0] + + prediction_scores = self.cls_geom_id(sequence_output) + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + positions_cls = [ + self.cls_geom_type, self.cls_rel_type, + self.cls_absolute_position_x1, self.cls_absolute_position_x2, + self.cls_absolute_position_y1, self.cls_absolute_position_y2, + self.cls_relative_position_x1, self.cls_relative_position_x2, + self.cls_relative_position_y1, self.cls_relative_position_y2 + ] + positions_label = [ + token_type_ids_label, rel_type_ids_label, + absolute_position_ids_label[:, :, + 0], absolute_position_ids_label[:, :, + 2], + absolute_position_ids_label[:, :, + 1], absolute_position_ids_label[:, :, + 3], + relative_position_ids_label[:, :, + 0], relative_position_ids_label[:, :, + 2], + relative_position_ids_label[:, :, + 1], relative_position_ids_label[:, :, + 3] + ] + positions_size = [ + self.config.type_vocab_size, self.config.rel_type_vocab_size, + self.config.absolute_x_vocab_size, + self.config.absolute_x_vocab_size, + self.config.absolute_y_vocab_size, + self.config.absolute_y_vocab_size, + self.config.relative_x_vocab_size, + self.config.relative_x_vocab_size, + self.config.relative_y_vocab_size, + self.config.relative_y_vocab_size + ] + for mycls, mylabels, mysize in zip(positions_cls, positions_label, + positions_size): + if mylabels is not None: + myprediction_scores = mycls(sequence_output) + masked_lm_loss += loss_fct( + myprediction_scores.view(-1, mysize), mylabels.view(-1)) + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation(self, + input_ids, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + assert self.config.pad_token_id is not None, 'The PAD token should be defined for generation' + padding_mask = attention_mask.new_zeros((attention_mask.shape[0], 1)) + attention_mask = torch.cat([attention_mask, padding_mask], dim=-1) + dummy_token = torch.full((effective_batch_size, 1), + self.config.pad_token_id, + dtype=torch.long, + device=input_ids.device) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {'input_ids': input_ids, 'attention_mask': attention_mask} + + +class BertForMaskedLM(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multi_modal', + soft_labels=None, + alpha=0, + return_logits=False, + rel_type_ids=None, + absolute_position_ids=None, + relative_position_ids=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_embeds=encoder_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + rel_type_ids=rel_type_ids, + absolute_position_ids=absolute_position_ids, + relative_position_ids=relative_position_ids, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + if soft_labels is not None: + loss_distill = -torch.sum( + F.log_softmax(prediction_scores, dim=-1) * soft_labels, dim=-1) + loss_distill = loss_distill[labels != -100].mean() + masked_lm_loss = (1 + - alpha) * masked_lm_loss + alpha * loss_distill + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation(self, + input_ids, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + assert self.config.pad_token_id is not None, 'The PAD token should be defined for generation' + + padding_mask = attention_mask.new_zeros((attention_mask.shape[0], 1)) + attention_mask = torch.cat([attention_mask, padding_mask], dim=-1) + dummy_token = torch.full((effective_batch_size, 1), + self.config.pad_token_id, + dtype=torch.long, + device=input_ids.device) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {'input_ids': input_ids, 'attention_mask': attention_mask} + + +class BertForNextSentencePrediction(BertPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.cls = BertOnlyNSPHead(config) + + self.init_weights() + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the next sequenåce prediction (classification) loss. Input should be a sequence pair + (see ``input_ids`` docstring). Indices should be in ``[0, 1]``: + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. + Returns: + + Example: + >>> from transformers import BertTokenizer, BertForNextSentencePrediction + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + >>> model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') + >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." + >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." + >>> encoding = tokenizer(prompt, next_sentence, return_tensors='pt') + >>> outputs = model(**encoding, labels=torch.LongTensor([1])) + >>> logits = outputs.logits + >>> assert logits[0, 0] < logits[0, 1] # next sentence was random + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + seq_relationship_scores = self.cls(pooled_output) + + next_sentence_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + next_sentence_loss = loss_fct( + seq_relationship_scores.view(-1, 2), labels.view(-1)) + + if not return_dict: + output = (seq_relationship_scores, ) + outputs[2:] + return ((next_sentence_loss, ) + + output) if next_sentence_loss is not None else output + + return NextSentencePredictorOutput( + loss=next_sentence_loss, + logits=seq_relationship_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class BertForSequenceClassification(BertPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class BertForMultipleChoice(BertPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, + `optional`): + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices-1]`` where + :obj:`num_choices` is the size of the second dimension of the input + tensors. (See :obj:`input_ids` above) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + num_choices = input_ids.shape[ + 1] if input_ids is not None else inputs_embeds.shape[1] + + input_ids = input_ids.view( + -1, input_ids.size(-1)) if input_ids is not None else None + attention_mask = attention_mask.view( + -1, + attention_mask.size(-1)) if attention_mask is not None else None + token_type_ids = token_type_ids.view( + -1, + token_type_ids.size(-1)) if token_type_ids is not None else None + position_ids = position_ids.view( + -1, position_ids.size(-1)) if position_ids is not None else None + inputs_embeds = ( + inputs_embeds.view(-1, inputs_embeds.size(-2), + inputs_embeds.size(-1)) + if inputs_embeds is not None else None) + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + + if not return_dict: + output = (reshaped_logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return MultipleChoiceModelOutput( + loss=loss, + logits=reshaped_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class BertForTokenClassification(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config, add_pooling_layer=False) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Labels for computing the token classification loss. Indices should + be in ``[0, ..., config.num_labels - 1]``. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class BertForQuestionAnswering(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.bert = BertModel(config, add_pooling_layer=False) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + start_positions=None, + end_positions=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, + `optional`): + Labels for position (index) of the start of the labelled span for + computing the token classification loss. Positions are clamped to + the length of the sequence (:obj:`sequence_length`). Position + outside of the sequence are not taken into account for computing the + loss. + end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, + `optional`): + Labels for position (index) of the end of the labelled span for + computing the token classification loss. Positions are clamped to + the length of the sequence (:obj:`sequence_length`). Position + outside of the sequence are not taken into account for computing the + loss. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if not return_dict: + output = (start_logits, end_logits) + outputs[2:] + return ((total_loss, ) + + output) if total_loss is not None else output + + return QuestionAnsweringModelOutput( + loss=total_loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class MGeoPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface + for downloading and loading pretrained models. + """ + + config_class = BertConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, BertEncoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the + label information. num_labels: An optional arg to tell the + model how many classes to initialize. + Method will call utils.parse_label_mapping + if num_labels not supplied. If num_labels is + not found, the model will use the default + setting (2 classes). + + Returns: + The loaded model, which is initialized by + transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + if model_dir is None: + config = BertConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + model.model_dir = model_dir + return model + + +@MODELS.register_module(Tasks.backbone, module_name=Models.mgeo) +class MGeo(MGeoPreTrainedModel): + + def __init__(self, + config: BertConfig, + finetune_mode: str = 'single-modal', + gis_num: int = 1, + add_pooling_layer=False, + **kwargs): + super().__init__(config) + + self.finetune_mode = finetune_mode + + self.config = config + self.text_encoder = BertModel( + config, add_pooling_layer=add_pooling_layer) + + if self.finetune_mode == 'multi-modal': + gis_config = BertConfig.from_dict(config.gis_encoder) + self.gis_encoder = BertModel(gis_config, add_pooling_layer=False) + for param in self.gis_encoder.parameters(): + param.requires_grad = False + self.gis2text = nn.Linear(gis_config.hidden_size, + self.config.hidden_size) + self.gis_type = nn.Embedding(gis_num, gis_config.hidden_size) + + self.init_weights() + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='single-modal', + gis_list=None, + gis_tp=None, + use_token_type=False): + if self.finetune_mode == 'multi-modal' and gis_list is not None and len( + gis_list) > 0: + gis_embs = [] + gis_atts = [] + for gis in gis_list: + gis_embs.append( + self.gis_encoder(return_dict=True, mode='text', + **gis).last_hidden_state) + gis_atts.append(gis['attention_mask']) + if use_token_type: + embedding_output = self.text_encoder.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + ) + else: + embedding_output = self.text_encoder.embeddings( + input_ids=input_ids, ) + + if self.finetune_mode == 'multi-modal' and gis_list is not None and len( + gis_list) > 0: + embs = [embedding_output] + atts = [attention_mask] + tp_emb = [self.gis_type(gtp) for gtp in gis_tp] + for ge, ga, gt in zip(gis_embs, gis_atts, tp_emb): + embs.append(self.gis2text(ge + gt)) + atts.append(ga) + merge_emb = torch.cat(embs, dim=1) + merge_attention = torch.cat(atts, dim=-1) + else: + merge_emb = embedding_output + merge_attention = attention_mask + encoder_outputs = self.text_encoder( + attention_mask=merge_attention, + encoder_embeds=merge_emb, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + mode='text') + + if not return_dict: + return encoder_outputs + + return AttentionBackboneModelOutput( + last_hidden_state=encoder_outputs.last_hidden_state, + pooler_output=encoder_outputs.pooler_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + return output + + def extract_sequence_outputs(self, outputs): + return outputs['last_hidden_state'] + + def extract_pooled_outputs(self, outputs): + return outputs['pooler_output'] diff --git a/runtime/Lib/site_packages/models/multi_modal/mgeo/text_classification.py b/runtime/Lib/site_packages/models/multi_modal/mgeo/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..a9c90c7c48e250d5fad2176b48f052eec9f7a0eb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mgeo/text_classification.py @@ -0,0 +1,187 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import MGeo, MGeoPreTrainedModel + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.text_classification, module_name=Models.mgeo) +@MODELS.register_module(Tasks.nli, module_name=Models.mgeo) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.mgeo) +@MODELS.register_module(Tasks.sentence_similarity, module_name=Models.mgeo) +@MODELS.register_module( + Tasks.zero_shot_classification, module_name=Models.mgeo) +class MGeoForSequenceClassification(MGeoPreTrainedModel): + + def __init__(self, config, finetune_mode: str = 'single-modal', **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + + setattr(self, self.base_model_prefix, + MGeo(config, finetune_mode, add_pooling_layer=True)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + gis_list=None, + gis_tp=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Returns: + Returns `modelscope.outputs.AttentionTextClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.base_model.forward( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + gis_list=gis_list, + gis_tp=gis_tp, + use_token_type=True, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = 'regression' + elif self.num_labels > 1 and (labels.dtype == torch.long + or labels.dtype == torch.int): + self.config.problem_type = 'single_label_classification' + else: + self.config.problem_type = 'multi_label_classification' + + if self.config.problem_type == 'regression': + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == 'single_label_classification': + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == 'multi_label_classification': + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/mgeo/text_ranking.py b/runtime/Lib/site_packages/models/multi_modal/mgeo/text_ranking.py new file mode 100644 index 0000000000000000000000000000000000000000..c3e2959bfda1a9810b03acc7e8b7560bc28698c6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mgeo/text_ranking.py @@ -0,0 +1,88 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import MGeo, MGeoPreTrainedModel + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.text_ranking, module_name=Models.mgeo) +class MGeoForTextRanking(MGeoPreTrainedModel): + + def __init__(self, + config, + finetune_mode: str = 'single-modal', + gis_num: int = 1, + *args, + **kwargs): + super().__init__(config) + neg_sample = kwargs.get('neg_sample', 8) + eval_neg_sample = kwargs.get('eval_neg_sample', 8) + self.neg_sample = neg_sample + self.eval_neg_sample = eval_neg_sample + setattr( + self, self.base_model_prefix, + MGeo(self.config, finetune_mode, gis_num, add_pooling_layer=True)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, 1) + + # Initialize weights and apply final processing + self.post_init() + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + gis_list=None, + gis_tp=None, + *args, + **kwargs) -> AttentionTextClassificationModelOutput: + outputs = self.base_model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + gis_list=gis_list, + gis_tp=gis_tp, + ) + + # backbone model should return pooled_output as its second output + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + if self.base_model.training: + scores = logits.view(-1, self.neg_sample + 1) + batch_size = scores.size(0) + loss_fct = torch.nn.CrossEntropyLoss() + target_label = torch.zeros( + batch_size, dtype=torch.long, device=scores.device) + loss = loss_fct(scores, target_label) + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + ) + return AttentionTextClassificationModelOutput(logits=logits, ) diff --git a/runtime/Lib/site_packages/models/multi_modal/mgeo/token_classification.py b/runtime/Lib/site_packages/models/multi_modal/mgeo/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..7a665a61cbb078cc3cb43d18e1ee73820a682438 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mgeo/token_classification.py @@ -0,0 +1,230 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTokenClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import MGeo, MGeoPreTrainedModel + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.token_classification, module_name=Models.mgeo) +@MODELS.register_module(Tasks.part_of_speech, module_name=Models.mgeo) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.mgeo) +class MGeoForTokenClassification(MGeoPreTrainedModel): + r"""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks, word-segmentation. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the fill_mask model of Bert, the preprocessor of this model + is `modelscope.preprocessors.TokenClassificationTransformersPreprocessor`. + + Trainer: + This model is a normal PyTorch model, and can be trained by variable trainers, like EpochBasedTrainer, + NlpEpochBasedTrainer, or trainers from other frameworks. + The preferred trainer in ModelScope is NlpEpochBasedTrainer. + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config, finetune_mode: str = 'single-modal', **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + + setattr(self, self.base_model_prefix, + MGeo(config, finetune_mode, add_pooling_layer=False)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + offset_mapping=None, + label_mask=None, + gis_list=None, + gis_tp=None, + ): + r""" + Args: input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using + :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and + :meth:`transformers.PreTrainedTokenizer.__call__` for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask + values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the + inputs. Indices are selected in ``[0, 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position + embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or + :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask + values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, + sequence_length, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert :obj:`input_ids` indices into + associated vectors than the model's internal embedding lookup + matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention + layers. See ``attentions`` under returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See + ``hidden_states`` under returned tensors for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` + instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, + `optional`): + Labels for computing the sequence classification/regression loss. + Indices should be in :obj:`[0, ..., config.num_labels - 1]`. If + :obj:`config.num_labels == 1` a regression loss is computed + (Mean-Square loss), If :obj:`config.num_labels > 1` a classification + loss is computed (Cross-Entropy). + offset_mapping (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the sentence. + Selected in the range ``[0, sequence_length - 1]``. + label_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, + sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask + values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + Returns `modelscope.outputs.AttentionTokenClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_bert_word-segmentation_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_bert_word-segmentation_chinese-base') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + gis_list=gis_list, + gis_tp=gis_tp, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + offset_mapping=offset_mapping, + label_mask=label_mask, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mmr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9dac8409c82c441aaeb62ce050b67997bd9ac08d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .models import VideoCLIPForMultiModalEmbedding diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/dataloaders/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mmr/dataloaders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/dataloaders/rawvideo_util.py b/runtime/Lib/site_packages/models/multi_modal/mmr/dataloaders/rawvideo_util.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ac3f94719f96e26aa2db4a2228bf6c5aa47282 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/dataloaders/rawvideo_util.py @@ -0,0 +1,117 @@ +# The implementation is adopted from Huaishao Luo, +# made pubicly available under the MIT License at https://github.com/ArrowLuo/CLIP4Clip + +import cv2 +import numpy as np +import torch as th +from PIL import Image +from torchvision.transforms import (CenterCrop, Compose, InterpolationMode, + Normalize, Resize, ToTensor) + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class RawVideoExtractorCV2(): + + def __init__( + self, + centercrop=False, + size=224, + frame_rate=-1, + ): + self.centercrop = centercrop + self.size = size + self.framerate = frame_rate + self.transform = self._transform(self.size) + + def _transform(self, n_px): + return Compose([ + Resize(n_px, interpolation=InterpolationMode.BICUBIC), + CenterCrop(n_px), + lambda image: image.convert('RGB'), + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)), + ]) + + def video_to_tensor(self, + video_file, + preprocess, + sample_fp=0, + start_time=None, + end_time=None): + if start_time is not None or end_time is not None: + assert isinstance(start_time, int) and isinstance(end_time, int) \ + and start_time > -1 and end_time > start_time + assert sample_fp > -1 + + # Samples a frame sample_fp X frames. + cap = cv2.VideoCapture(video_file) + frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = int(cap.get(cv2.CAP_PROP_FPS)) + + if fps == 0: + logger.info(f'{video_file} with fps 0!!!') + total_duration = (frameCount + fps - 1) // fps + start_sec, end_sec = 0, total_duration + + if start_time is not None: + start_sec, end_sec = start_time, end_time if end_time <= total_duration else total_duration + cap.set(cv2.CAP_PROP_POS_FRAMES, int(start_time * fps)) + + interval = 1 + if sample_fp > 0: + interval = fps // sample_fp + else: + sample_fp = fps + if interval == 0: + interval = 1 + + inds = [ind for ind in np.arange(0, fps, interval)] + assert len(inds) >= sample_fp + inds = inds[:sample_fp] + + ret = True + images = [] + + for sec in np.arange(start_sec, end_sec + 1): + if not ret: + break + sec_base = int(sec * fps) + for ind in inds: + cap.set(cv2.CAP_PROP_POS_FRAMES, sec_base + ind) + ret, frame = cap.read() + if not ret: + break + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + images.append( + preprocess(Image.fromarray(frame_rgb).convert('RGB'))) + + cap.release() + + if len(images) > 0: + video_data = th.tensor(np.stack(images)) + else: + video_data = th.zeros(1) + return {'video': video_data} + + def get_video_data(self, video_path, start_time=None, end_time=None): + image_input = self.video_to_tensor( + video_path, + self.transform, + sample_fp=self.framerate, + start_time=start_time, + end_time=end_time) + return image_input + + def process_raw_data(self, raw_video_data): + tensor_size = raw_video_data.size() + tensor = raw_video_data.view(-1, 1, tensor_size[-3], tensor_size[-2], + tensor_size[-1]) + return tensor + + +# An ordinary video frame extractor based CV2 +RawVideoExtractor = RawVideoExtractorCV2 diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..da832719557ee069034013f32c16b83a2b6cc3e8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .clip_for_mm_video_embedding import VideoCLIPForMultiModalEmbedding diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..743c049adf61c589f0b86adcedd815edc11f1835 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/clip_for_mm_video_embedding.py @@ -0,0 +1,248 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import os +import random +import uuid +from os.path import exists +from tempfile import TemporaryDirectory +from typing import Any, Dict +from urllib.parse import urlparse + +import json +import numpy as np +import torch +from decord import VideoReader, cpu +from PIL import Image + +from modelscope.hub.file_download import http_get_file +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from ..dataloaders.rawvideo_util import RawVideoExtractor +from .modeling import CLIP4Clip +from .tokenization_clip import SimpleTokenizer as ClipTokenizer + +logger = get_logger() + + +@MODELS.register_module( + Tasks.video_multi_modal_embedding, module_name=Models.video_clip) +class VideoCLIPForMultiModalEmbedding(TorchModel): + + def __init__(self, model_dir, **kwargs): + super().__init__(model_dir=model_dir, **kwargs) + # model config parameters + with open( + f'{model_dir}/{ModelFile.CONFIGURATION}', 'r', + encoding='utf-8') as json_file: + model_config = json.load(json_file) + model_config = model_config['paras'] + model_config['model_dir'] = model_dir + self.SPECIAL_TOKEN = { + 'CLS_TOKEN': '<|startoftext|>', + 'SEP_TOKEN': '<|endoftext|>', + 'MASK_TOKEN': '[MASK]', + 'UNK_TOKEN': '[UNK]', + 'PAD_TOKEN': '[PAD]' + } + self.max_words = model_config['max_words'] + self.max_frames = model_config['max_frames'] + self.feature_framerate = model_config['feature_framerate'] + self.image_resolution = 224 + if torch.cuda.is_available(): + self.device = model_config['device'] + else: + self.device = 'cpu' + self.init_model = f'{model_dir}/{ModelFile.TORCH_MODEL_BIN_FILE}' + + self.tokenizer = ClipTokenizer(model_dir) + self.rawVideoExtractor = RawVideoExtractor( + frame_rate=self.feature_framerate, size=self.image_resolution) + self.local_transform = self.rawVideoExtractor.transform + + self.model = CLIP4Clip(model_config) + if hasattr(self.model, 'module'): + self.model = self.model.module.to(self.device) + else: + self.model = self.model.to(self.device) + if self.init_model: + assert exists(self.init_model) + model_state_dict = torch.load(self.init_model, map_location='cpu') + self.model.load_state_dict(model_state_dict, strict=False) + self.model.to(self.device) + + def _get_text(self, caption, tokenizer, enable_zh=False): + + if type(caption) is str: + _caption_text, s, e = caption, None, None + elif type(caption) is tuple: + if len(caption) == 3: + _caption_text, s, e = caption + elif len(caption) == 4: + _caption_text, s, e, pos = caption + else: + NotImplementedError + + if isinstance(_caption_text, list): + caption_text = random.choice(_caption_text) + else: + caption_text = _caption_text + if enable_zh: + _token = tokenizer.encode(caption_text) + input_ids = _token.ids + input_mask = _token.attention_mask + segment_ids = _token.type_ids + else: + words = tokenizer.tokenize(caption_text) + + words = [self.SPECIAL_TOKEN['CLS_TOKEN']] + words + total_length_with_CLS = self.max_words - 1 + if len(words) > total_length_with_CLS: + words = words[:total_length_with_CLS] + words = words + [self.SPECIAL_TOKEN['SEP_TOKEN']] + + input_ids = tokenizer.convert_tokens_to_ids(words) + input_mask = [1] * len(input_ids) + segment_ids = [0] * len(input_ids) + + while len(input_ids) < self.max_words: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + assert len(input_ids) == self.max_words + assert len(input_mask) == self.max_words + assert len(segment_ids) == self.max_words + + pairs_text = np.array(input_ids) + pairs_mask = np.array(input_mask) + pairs_segment = np.array(segment_ids) + + return pairs_text, pairs_mask, pairs_segment, s, e + + def _get_rawvideo_dec(self, + video_path, + rawVideoExtractor, + local_transform, + s=None, + e=None): + video_mask = np.zeros(self.max_frames, dtype=int) + max_video_length = 0 + + # T x 3 x H x W + video = np.zeros((self.max_frames, 3, rawVideoExtractor.size, + rawVideoExtractor.size), + dtype=float) + + if s is None: + start_time, end_time = None, None + else: + start_time = int(s) + end_time = int(e) + start_time = start_time if start_time >= 0. else 0. + end_time = end_time if end_time >= 0. else 0. + if start_time > end_time: + start_time, end_time = end_time, start_time + elif start_time == end_time: + end_time = end_time + 1 + + url_parsed = urlparse(video_path) + if url_parsed.scheme in ('file', '') and exists( + url_parsed.path): # Possibly a local file + vreader = VideoReader(video_path, ctx=cpu(0)) + else: + try: + with TemporaryDirectory() as temporary_cache_dir: + random_str = uuid.uuid4().hex + http_get_file( + url=video_path, + local_dir=temporary_cache_dir, + file_name=random_str, + cookies=None) + temp_file_path = os.path.join(temporary_cache_dir, + random_str) + vreader = VideoReader(temp_file_path, ctx=cpu(0)) + except Exception as ex: + logger.error('non video input, output is {}!!!'.format(ex)) + return video, video_mask + + fps = vreader.get_avg_fps() + f_start = 0 if start_time is None else int(start_time * fps) + f_end = int( + min(1000000000 if end_time is None else end_time * fps, + len(vreader) - 1)) + num_frames = f_end - f_start + 1 + if num_frames > 0: + # L x T x 3 x H x W + sample_fps = int(self.feature_framerate) + t_stride = int(round(float(fps) / sample_fps)) + + all_pos = list(range(f_start, f_end + 1, t_stride)) + if len(all_pos) > self.max_frames: + sample_pos = [ + all_pos[_] for _ in np.linspace( + 0, len(all_pos) - 1, num=self.max_frames, dtype=int) + ] + else: + sample_pos = all_pos + patch_images = [ + Image.fromarray(f) + for f in vreader.get_batch(sample_pos).asnumpy() + ] + patch_images = torch.stack( + [local_transform(img) for img in patch_images]) + slice_len = patch_images.shape[0] + max_video_length = max_video_length if max_video_length > slice_len else slice_len + if slice_len < 1: + pass + else: + video[:slice_len, ...] = patch_images + else: + logger.error('video path: {} error. video id: {}'.format( + video_path, video_id)) + + video_mask[:max_video_length] = [1] * max_video_length + + return video, video_mask + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + + from modelscope.outputs import OutputKeys + output = {} + + if 'video' in input and input['video'] is not None: + video_path = input['video'] + video, video_mask = self._get_rawvideo_dec(video_path, + self.rawVideoExtractor, + self.local_transform) + video = torch.unsqueeze( + torch.from_numpy(video), dim=0).to(self.device) + video_mask = torch.unsqueeze( + torch.from_numpy(video_mask), dim=0).to(self.device) + + if 'text' in input and input['text'] is not None: + caption = input['text'] + pairs_text, pairs_mask, pairs_segment, s, e = self._get_text( + caption, self.tokenizer, enable_zh=False) + input_ids = torch.unsqueeze( + torch.from_numpy(pairs_text), dim=0).to(self.device) + input_mask = torch.unsqueeze( + torch.from_numpy(pairs_mask), dim=0).to(self.device) + segment_ids = torch.unsqueeze( + torch.from_numpy(pairs_segment), dim=0).to(self.device) + + sequence_output, visual_output = self.model.get_sequence_visual_output( + input_ids, segment_ids, input_mask, video, video_mask) + logger.info('text feature: {}'.format(sequence_output[0][0][0])) + logger.info('video feature: {}'.format(visual_output[0][0][0])) + + output[ + OutputKeys.VIDEO_EMBEDDING] = visual_output.cpu().detach().numpy() + output[OutputKeys.TEXT_EMBEDDING] = sequence_output.cpu().detach( + ).numpy() + return output + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/dynamic_inverted_softmax.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/dynamic_inverted_softmax.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d96275d31b872bd482d735e794519d9d7cdf19 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/dynamic_inverted_softmax.py @@ -0,0 +1,45 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import numpy as np + + +def get_retrieved_videos(sims, k): + """ + Returns list of retrieved top k videos based on the sims matrix + Args: + sims: similar matrix. + K: top k number of videos + """ + argm = np.argsort(-sims, axis=1) + topk = argm[:, :k].reshape(-1) + retrieved_videos = np.unique(topk) + return retrieved_videos + + +def get_index_to_normalize(sims, videos): + """ + Returns list of indices to normalize from sims based on videos + Args: + sims: similar matrix. + videos: video array. + """ + argm = np.argsort(-sims, axis=1)[:, 0] + result = np.array(list(map(lambda x: x in videos, argm))) + result = np.nonzero(result) + return result + + +def qb_norm(train_test, test_test, args): + k = args.get('k', 1) + beta = args.get('beta', 20) + retrieved_videos = get_retrieved_videos(train_test, k) + test_test_normalized = test_test + train_test = np.exp(train_test * beta) + test_test = np.exp(test_test * beta) + + normalizing_sum = np.sum(train_test, axis=0) + index_for_normalizing = get_index_to_normalize(test_test, retrieved_videos) + test_test_normalized[index_for_normalizing, :] = \ + np.divide(test_test[index_for_normalizing, :], normalizing_sum) + return test_test_normalized diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/modeling.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..dc6510bf5e3b2445b356e11fb439cef85b5921d2 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/modeling.py @@ -0,0 +1,507 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os +import platform +from collections import OrderedDict +from types import SimpleNamespace + +import torch +from torch import nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + +from modelscope.models.multi_modal.mmr.models.module_clip import ( + _PT_NAME, CLIP, QuickGELU, convert_weights) +from modelscope.models.multi_modal.mmr.models.module_cross import \ + Transformer as TransformerClip +from modelscope.models.multi_modal.mmr.models.until_module import (AllGather, + CrossEn, + LayerNorm) +from modelscope.utils.logger import get_logger + +allgather = AllGather.apply + +logger = get_logger() +__all__ = ['CLIP4Clip'] + + +class CLIP4Clip(nn.Module): + + def __init__(self, config): + super(CLIP4Clip, self).__init__() + + self.config = config + self.loose_type = config['loose_type'] + self.sim_header = config['sim_header'] + if self.sim_header in [ + 'tightTransf', 'tightFc1', 'tightFc2', 'tightFc3', 'tightFc4', + 'tightMean', 'tightFc5' + ]: + assert self.loose_type is False + + backbone = config['pretrained_clip_name'] + + # fix backbone without downlond + model_path = '{}/ViT-B-16.pt'.format(config['model_dir']) + if not os.path.exists(model_path): + logger.info('no model loaded!!!') + + try: + # loading JIT archive + model = torch.jit.load(model_path, map_location='cpu').eval() + state_dict = model.state_dict() + except RuntimeError: + state_dict = torch.load(model_path, map_location='cpu') + + vision_width = state_dict['visual.conv1.weight'].shape[0] + vision_layers = len([ + k for k in state_dict.keys() + if k.startswith('visual.') and k.endswith('.attn.in_proj_weight') + ]) + vision_patch_size = state_dict['visual.conv1.weight'].shape[-1] + grid_size = round( + (state_dict['visual.positional_embedding'].shape[0] - 1)**0.5) + image_resolution = vision_patch_size * grid_size + + embed_dim = state_dict['text_projection'].shape[1] + context_length = state_dict['positional_embedding'].shape[0] + vocab_size = state_dict['token_embedding.weight'].shape[0] + transformer_width = state_dict['ln_final.weight'].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split('.')[2] for k in state_dict + if k.startswith('transformer.resblocks'))) + + cut_top_layer = 0 + self.clip = CLIP( + embed_dim, + image_resolution, + vision_layers - cut_top_layer, + vision_width, + vision_patch_size, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers - cut_top_layer, + linear_patch=config['linear_patch'], + use_gc=config['use_gc']).float() + + if backbone in ['ViT-B/32', 'ViT-B/16']: + cross_config = SimpleNamespace(**{ + 'hidden_size': 512, + 'max_position_embeddings': 128, + }) + elif backbone in ['ViT-L/14', 'ViT-B/14-336px']: + cross_config = SimpleNamespace(**{ + 'hidden_size': 768, + 'max_position_embeddings': 128, + }) + else: + raise ValueError + + cross_config.max_position_embeddings = context_length + self.cross_config = cross_config + + self.text_weight_fc = nn.Sequential( + nn.Linear(transformer_width, transformer_width), + nn.ReLU(inplace=True), nn.Linear(transformer_width, 1)) + self.video_weight_fc = nn.Sequential( + nn.Linear(transformer_width, transformer_width), + nn.ReLU(inplace=True), nn.Linear(transformer_width, 1)) + + if self.loose_type is False: + raise NotImplementedError + + if self.sim_header in ['seqLSTM', 'seqTransf', 'tightFc1']: + self.frame_position_embeddings = nn.Embedding( + cross_config.max_position_embeddings, cross_config.hidden_size) + if self.sim_header in ['seqTransf', 'tightFc1']: + self.transformerClip = TransformerClip( + width=transformer_width, + layers=config['cross_num_hidden_layers'], + heads=transformer_heads, + ) + if self.sim_header == 'seqLSTM': + self.lstm_visual = nn.LSTM( + input_size=cross_config.hidden_size, + hidden_size=cross_config.hidden_size, + batch_first=True, + bidirectional=False, + num_layers=1) + + self.loss_fct = CrossEn(config) + + self.apply(self.init_weights) + self.clip.load_state_dict(state_dict, strict=False) + + # ===> Initialization trick [HARD CODE] + if backbone not in _PT_NAME: + raise NotImplementedError + # reload + else: + if config['linear_patch'] == '3d': + raise NotImplementedError + + new_state_dict = OrderedDict() + if self.sim_header == 'tightTransf': + raise NotImplementedError + + if self.sim_header in ['seqLSTM', 'seqTransf', 'seqFc1']: + contain_frame_position = False + for key in state_dict.keys(): + if key.find('frame_position_embeddings') > -1: + contain_frame_position = True + break + if contain_frame_position is False: + for key, val in state_dict.items(): + if key == 'positional_embedding': + new_state_dict[ + 'frame_position_embeddings.weight'] = val.clone() + continue + if self.sim_header in [ + 'seqTransf', 'seqFc1' + ] and key.find('transformer.resblocks') == 0: + num_layer = int(key.split('.')[2]) + # cut from beginning + if num_layer < config['cross_num_hidden_layers']: + new_state_dict[key.replace( + 'transformer.', + 'transformerClip.')] = val.clone() + continue + # <=== End of initialization trick + + self.load_state_dict( + new_state_dict, strict=False + ) # only update new state (seqTransf/seqLSTM/tightTransf) + if self.sim_header == 'tightFc5': + raise ValueError + + def forward(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask=None): + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + # B x T x 3 x H x W - > (B x T) x 3 x H x W + video = torch.as_tensor(video).float() + if len(video.shape) == 6: # image + b, bs, ts, channel, h, w = video.shape + b = b * bs + else: # video + b, ts, channel, h, w = video.shape + video = video.view(b * ts, channel, h, w) + + sequence_output, visual_output = self.get_sequence_visual_output( + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=True) + + if self.training: + loss = 0. + sim_matrix1, sim_matrix2, barlow_loss = self.get_similarity_logits( + sequence_output, + visual_output, + attention_mask, + video_mask, + shaped=True, + loose_type=self.loose_type) + sim_loss = (self.loss_fct(sim_matrix1) + + self.loss_fct(sim_matrix2)) / 2 + loss += sim_loss + barlow_loss * self.config.cdcr_lambda + + return loss + else: + return None + + def get_sequence_output(self, + input_ids, + token_type_ids, + attention_mask, + shaped=False): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + + bs_pair = input_ids.size(0) + sequence_hidden = self.clip.encode_text( + input_ids, return_hidden=True, prompt=None)[1].float() + sequence_hidden = sequence_hidden.view(bs_pair, -1, + sequence_hidden.size(-1)) + + return sequence_hidden + + def get_visual_output(self, video, video_mask, shaped=False): + if shaped is False: + video_mask = video_mask.view(-1, video_mask.shape[-1]) + video = torch.as_tensor(video).float() + b, ts, channel, h, w = video.shape + video = video.view(b * ts, channel, h, w) + + bs_pair = video_mask.size(0) + visual_hidden = self.clip.encode_image(video).float() + visual_hidden = visual_hidden.float().view(bs_pair, -1, + visual_hidden.size(-1)) + + return visual_hidden + + def get_sequence_visual_output(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=False): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + video = torch.as_tensor(video).float() + if len(video.shape) == 6: # image + b, bs, ts, channel, h, w = video.shape + b = b * bs + else: # video + b, ts, channel, h, w = video.shape + video = video.view(b * ts, channel, h, w) + + sequence_output = self.get_sequence_output( + input_ids, token_type_ids, attention_mask, shaped=True) + visual_output = self.get_visual_output(video, video_mask, shaped=True) + + return sequence_output, visual_output + + def agg_video_feat(self, visual_output, video_mask, sim_header='meanP'): + if self.config.max_sum == 0: + raise ValueError + + if sim_header == 'meanP': + # Default: Parameter-free type + pass + elif sim_header == 'seqLSTM': + # Sequential type: LSTM + visual_output_original = visual_output + visual_output = pack_padded_sequence( + visual_output, + torch.sum(video_mask, dim=-1).cpu(), + batch_first=True, + enforce_sorted=False) + visual_output, _ = self.lstm_visual(visual_output) + if self.training: + self.lstm_visual.flatten_parameters() + visual_output, _ = pad_packed_sequence( + visual_output, batch_first=True) + visual_output = torch.cat( + (visual_output, visual_output_original[:, + visual_output.size(1):, + ...].contiguous()), + dim=1) + visual_output = visual_output + visual_output_original + elif sim_header == 'seqTransf': + # Sequential type: Transformer Encoder + visual_output_original = visual_output + seq_length = visual_output.size(1) + position_ids = torch.arange( + seq_length, dtype=torch.long, device=visual_output.device) + position_ids = position_ids.unsqueeze(0).expand( + visual_output.size(0), -1) + frame_position_embeddings = self.frame_position_embeddings( + position_ids) + visual_output = visual_output + frame_position_embeddings + + extended_video_mask = (1.0 - video_mask.unsqueeze(1)) * -1000000.0 + extended_video_mask = extended_video_mask.expand( + -1, video_mask.size(1), -1) + visual_output = visual_output.permute(1, 0, 2) # NLD -> LND + visual_output = self.transformerClip(visual_output, + extended_video_mask) + visual_output = visual_output.permute(1, 0, 2) # LND -> NLD + visual_output = visual_output + visual_output_original + + return visual_output + + def wti_interaction(self, text_feat, video_feat, text_mask, video_mask): + text_weight = self.text_weight_fc(text_feat).squeeze( + 2) # B x N_t x D -> B x N_t + text_weight.masked_fill_( + torch.tensor((1 - text_mask), dtype=torch.bool), float('-inf')) + text_weight = torch.softmax(text_weight, dim=-1) # B x N_t + + video_weight = self.video_weight_fc(video_feat).squeeze( + 2) # B x N_v x D -> B x N_v + video_weight.masked_fill_( + torch.tensor((1 - video_mask), dtype=torch.bool), float('-inf')) + video_weight = torch.softmax(video_weight, dim=-1) # B x N_v + + text_feat = text_feat / text_feat.norm(dim=-1, keepdim=True) + video_feat = video_feat / video_feat.norm(dim=-1, keepdim=True) + + retrieve_logits = torch.einsum('atd,bvd->abtv', + [text_feat, video_feat]) + retrieve_logits = torch.einsum('abtv,at->abtv', + [retrieve_logits, text_mask]) + retrieve_logits = torch.einsum('abtv,bv->abtv', + [retrieve_logits, video_mask]) + + t2v_logits, max_idx1 = retrieve_logits.max(dim=-1) # abtv -> abt + t2v_logits = torch.einsum('abt,at->ab', [t2v_logits, text_weight]) + + v2t_logits, max_idx2 = retrieve_logits.max(dim=-2) # abtv -> abv + v2t_logits = torch.einsum('abv,bv->ab', [v2t_logits, video_weight]) + retrieve_logits = (t2v_logits + v2t_logits) / 2.0 + + if self.training: + logit_scale = self.clip.logit_scale.exp() + retrieve_logits = logit_scale * retrieve_logits + + # selecet max + max_idx1 = max_idx1[torch.arange(max_idx1.shape[0]), + torch.arange(max_idx1.shape[1])] + max_idx2 = max_idx2[torch.arange(max_idx2.shape[0]), + torch.arange(max_idx2.shape[1])] + + max_t_feat = text_feat[torch.arange(max_idx2.shape[0]). + repeat_interleave(max_idx2.shape[1]), + max_idx2.flatten()].squeeze(1) + max_v_feat = video_feat[torch.arange(max_idx1.shape[0]). + repeat_interleave(max_idx1.shape[1]), + max_idx1.flatten()].squeeze(1) + + t_feat = text_feat.reshape(-1, text_feat.shape[-1]) + t_mask = text_mask.flatten().type(torch.bool) + v_feat = video_feat.reshape(-1, video_feat.shape[-1]) + v_mask = video_mask.flatten().type(torch.bool) + t_feat = t_feat[t_mask] + v_feat = v_feat[v_mask] + max_t_feat = max_t_feat[v_mask] + max_v_feat = max_v_feat[t_mask] + text_weight = text_weight.flatten()[t_mask] + video_weight = video_weight.flatten()[v_mask] + + z_a_norm = (t_feat - t_feat.mean(0)) / t_feat.std(0) # (BxN_t)xD + z_b_norm = (max_v_feat - max_v_feat.mean(0)) / max_v_feat.std( + 0) # (BxN_t)xD + + x_a_norm = (v_feat - v_feat.mean(0)) / v_feat.std(0) # (BxN_v)xD + x_b_norm = (max_t_feat - max_t_feat.mean(0)) / max_t_feat.std( + 0) # (BxN_v)xD + + # cross-correlation matrix + N, D = z_a_norm.shape + B = text_feat.shape[0] + c1 = torch.einsum('acd,a->cd', + torch.einsum('ac,ad->acd', z_a_norm, z_b_norm), + text_weight) / B # DxD + c2 = torch.einsum('acd,a->cd', + torch.einsum('ac,ad->acd', x_a_norm, x_b_norm), + video_weight) / B # DxD + c = (c1 + c2) / 2.0 + # loss + on_diag = torch.diagonal(c).add_(-1).pow_(2).sum() + off_diag = c.flatten()[1:].view(D - 1, D + 1)[:, :-1].pow_(2).sum() + cdcr_loss = ( + on_diag * self.config.cdcr_alpha1 + + off_diag * self.config.cdcr_alpha2) + return retrieve_logits, retrieve_logits.T, cdcr_loss + else: + return retrieve_logits, retrieve_logits.T + + def _loose_similarity(self, + sequence_output, + visual_output, + attention_mask, + video_mask, + sim_header='seqTransf'): + sequence_output, visual_output = sequence_output.contiguous( + ), visual_output.contiguous() + + visual_output = self.agg_video_feat(visual_output, video_mask, + sim_header) + + if self.training: # batch merge here + visual_output = allgather(visual_output, self.config) + attention_mask = allgather(attention_mask, self.config) + video_mask = allgather(video_mask, self.config) + sequence_output = allgather(sequence_output, self.config) + torch.distributed.barrier() # force sync + + return self.wti_interaction(sequence_output, visual_output, + attention_mask, video_mask) + + def get_similarity_logits(self, + sequence_output, + visual_output, + attention_mask, + video_mask, + shaped=False, + loose_type=False): + if shaped is False: + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + if loose_type: + assert self.sim_header in ['meanP', 'seqLSTM', 'seqTransf'] + + if self.training: + retrieve_logits1, retrieve_logits2, barlow_loss = self._loose_similarity( + sequence_output, + visual_output, + attention_mask, + video_mask, + sim_header=self.sim_header) + return retrieve_logits1, retrieve_logits2, barlow_loss + else: + retrieve_logits1, retrieve_logits2 = self._loose_similarity( + sequence_output, + visual_output, + attention_mask, + video_mask, + sim_header=self.sim_header) + return retrieve_logits1, retrieve_logits2 + else: + raise NotImplementedError + + @property + def dtype(self): + """ + :obj:`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + try: + return next(self.parameters()).dtype + except StopIteration: + # For nn.DataParallel compatibility in PyTorch 1.5 + def find_tensor_attributes(module: nn.Module): + tuples = [(k, v) for k, v in module.__dict__.items() + if torch.is_tensor(v)] + return tuples + + gen = self._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + def init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, LayerNorm): + if 'beta' in dir(module) and 'gamma' in dir(module): + module.beta.data.zero_() + module.gamma.data.fill_(1.0) + else: + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_clip.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..5350172034365110e0d11b2ba9a373cc13186c8c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_clip.py @@ -0,0 +1,527 @@ +# The implementation is adopated from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import hashlib +import os +import urllib +import warnings +from collections import OrderedDict +from typing import Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import nn +from tqdm import tqdm + +_MODELS = {} +_PT_NAME = {'ViT-B/16': 'ViT-B-16.pt'} + + +def available_models(): + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super(Bottleneck, self).__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super(AttentionPool2d, self).__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super(ModifiedResNet, self).__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int, attn_mask=None): + super(ResidualAttentionBlock, self).__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + attn_mask_ = self.attn_mask + if self.attn_mask is not None and hasattr(self.attn_mask, '__call__'): + attn_mask_ = self.attn_mask(x.size(0)) # LND + + attn_mask_ = attn_mask_.to( + dtype=x.dtype, device=x.device) if attn_mask_ is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, x): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask=None, + use_gc=0): + super(Transformer, self).__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + self.use_gc = use_gc + + def forward(self, x: torch.Tensor): + if self.use_gc > 0: + for blk in self.resblocks: + x = checkpoint.checkpoint(blk, x) + return x + else: + return self.resblocks(x) + + +class VisualTransformer(nn.Module): + + def __init__(self, + input_resolution: int, + patch_size: int, + width: int, + layers: int, + heads: int, + output_dim: int, + linear_patch: str = '2d', + use_gc: int = 0): + super(VisualTransformer, self).__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads, use_gc=use_gc) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + # For 3D + assert linear_patch in ['2d', '3d'] + self.linear_patch = linear_patch + if self.linear_patch == '3d': + self.conv2 = nn.Conv3d( + in_channels=3, + out_channels=width, + kernel_size=(3, patch_size, patch_size), + stride=(1, patch_size, patch_size), + padding=(1, 0, 0), + bias=False) + + def forward(self, x: torch.Tensor, video_frame=-1): + + if self.linear_patch == '3d': + assert video_frame != -1 + x_3d = x.reshape(-1, video_frame, x.shape[-3], x.shape[-2], + x.shape[-1]) + x_3d = x_3d.permute(0, 2, 1, 3, 4) + x_3d = self.conv2(x_3d) # shape = [*, width, frame, grid, grid] + x_3d = x_3d.permute(0, 2, 1, 3, + 4) # shape = [*, frame, width, grid, grid] + x = x_3d.reshape( + -1, x_3d.shape[-3], x_3d.shape[-2], + x_3d.shape[-1]).contiguous() # shape = [*, width, grid, grid] + else: + x = self.conv1(x) # shape = [*, width, grid, grid] + + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + _x = self.class_embedding.to(x.dtype) + torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([_x, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + # vision linear of patch + linear_patch: str = '2d', + use_gc: int = 0): + super(CLIP, self).__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim, + linear_patch=linear_patch, + use_gc=use_gc) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([])) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self, context_length): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.zeros(context_length, context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image, return_hidden=False): + hidden = self.visual(image.type(self.dtype)) + hidden = self.visual.ln_post(hidden) @ self.visual.proj + + x = hidden[:, 0, :] + + if return_hidden: + return x, hidden + + return x + + def encode_text(self, text, return_hidden=False, prompt=None): + x = self.token_embedding(text).type( + self.dtype) # [batch_size, n_ctx, d_model] + if prompt: + x = prompt(x) + + pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype) + x = x + pos_emd + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + hidden = self.ln_final(x).type(self.dtype) @ self.text_projection + + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)] + + if return_hidden: + return x, hidden + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logit_scale * text_features @ image_features.t() + + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(lay): + # l = lay + if isinstance(lay, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear)): + lay.weight.data = lay.weight.data.half() + if lay.bias is not None: + lay.bias.data = lay.bias.data.half() + + if isinstance(lay, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(lay, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ['text_projection', 'proj']: + if hasattr(lay, name): + attr = getattr(lay, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_cross.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_cross.py new file mode 100644 index 0000000000000000000000000000000000000000..b958d5bca87032ca0f3b06741f860c6ac609f05a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/module_cross.py @@ -0,0 +1,103 @@ +# The implementation is adopated from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +from __future__ import absolute_import, division, print_function +import logging +from collections import OrderedDict + +import json +import torch +from torch import nn + +from .until_module import ACT2FN, LayerNorm + +logger = logging.getLogger(__name__) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.n_head = n_head + + def attention(self, x: torch.Tensor, attn_mask: torch.Tensor): + attn_mask_ = attn_mask.repeat_interleave(self.n_head, dim=0) + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, para_tuple: tuple): + # x: torch.Tensor, attn_mask: torch.Tensor + x, attn_mask = para_tuple + x = x + self.attention(self.ln_1(x), attn_mask) + x = x + self.mlp(self.ln_2(x)) + return (x, attn_mask) + + +class Transformer(nn.Module): + + def __init__(self, width: int, layers: int, heads: int): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential( + *[ResidualAttentionBlock(width, heads) for _ in range(layers)]) + + def forward(self, x: torch.Tensor, attn_mask: torch.Tensor): + return self.resblocks((x, attn_mask))[0] + + +class CrossEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(CrossEmbeddings, self).__init__() + + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, concat_embeddings, concat_type=None): + + _, seq_length = concat_embeddings.size(0), concat_embeddings.size(1) + position_ids = torch.arange( + seq_length, dtype=torch.long, device=concat_embeddings.device) + position_ids = position_ids.unsqueeze(0).expand( + concat_embeddings.size(0), -1) + + position_embeddings = self.position_embeddings(position_ids) + + embeddings = concat_embeddings + position_embeddings # + token_type_embeddings + embeddings = self.dropout(embeddings) + return embeddings + + +class CrossPooler(nn.Module): + + def __init__(self, config): + super(CrossPooler, self).__init__() + self.ln_pool = LayerNorm(config.hidden_size) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = QuickGELU() + + def forward(self, hidden_states, hidden_mask): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + hidden_states = self.ln_pool(hidden_states) + pooled_output = hidden_states[:, 0] + pooled_output = self.dense(pooled_output) + pooled_output = self.activation(pooled_output) + return pooled_output diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/tokenization_clip.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/tokenization_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..97ee7156a75d53d4787b01cc018d23eefc244161 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/tokenization_clip.py @@ -0,0 +1,161 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, model_dir): + bpe_path = '{}/bpe_simple_vocab_16e6.txt.gz'.format(model_dir) + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + self.vocab = self.encoder + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + def tokenize(self, text): + tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + tokens.extend( + bpe_token for bpe_token in self.bpe(token).split(' ')) + return tokens + + def convert_tokens_to_ids(self, tokens): + return [self.encoder[bpe_token] for bpe_token in tokens] diff --git a/runtime/Lib/site_packages/models/multi_modal/mmr/models/until_module.py b/runtime/Lib/site_packages/models/multi_modal/mmr/models/until_module.py new file mode 100644 index 0000000000000000000000000000000000000000..24e886b0f11d651fc57f0f1182b3cf734be0a122 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mmr/models/until_module.py @@ -0,0 +1,120 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +import logging +import math + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +logger = logging.getLogger(__name__) + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class LayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class CrossEn(nn.Module): + + def __init__(self, config=None): + super(CrossEn, self).__init__() + + def forward(self, sim_matrix): + logpt = F.log_softmax(sim_matrix, dim=-1) + logpt = torch.diag(logpt) + nce_loss = -logpt + sim_loss = nce_loss.mean() + return sim_loss + + +class AllGather(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + return ( + grad_output[ctx.batch_size * ctx.rank:ctx.batch_size + * (ctx.rank + 1)], + None, + ) + + +class AllGather2(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + # https://github.com/PyTorchLightning/lightning-bolts/blob/8d3fbf7782e3d3937ab8a1775a7092d7567f2933/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20 + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + grad_input = grad_output.clone() + torch.distributed.all_reduce( + grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False) + return (grad_input[ctx.rank * ctx.batch_size:(ctx.rank + 1) + * ctx.batch_size], None) diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mplug/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67b0a4261800e209bf526552efe95c9af4607cad --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .configuration_mplug import HiTeAConfig, MPlugConfig +from .modeling_mplug import CONFIG_NAME, HiTeA, MPlug diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/clip/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mplug/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e6007a04e11317eb4d8e65a4c61b79f31112e9a3 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/clip/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .clip import load_from_config diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/clip/clip.py b/runtime/Lib/site_packages/models/multi_modal/mplug/clip/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..aa56e39bb93307bf34bf6ebd8fa67970906138e7 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/clip/clip.py @@ -0,0 +1,461 @@ +# Copyright 2021 The OpenAI CLIP Authors. All rights reserved. + +from collections import OrderedDict +from typing import Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import nn + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None, + use_grad_ckp: bool = True): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + self.use_grad_ckp = use_grad_ckp + + def forward(self, x: torch.Tensor): + if self.use_grad_ckp: + for each_block in self.resblocks: + x = checkpoint.checkpoint(each_block, x) + return x + else: + return self.resblocks(x) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + if self.training: + dropout = 0.1 + else: + dropout = 0.0 + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=dropout, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x, skip_last_layer=False): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + if not skip_last_layer: + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x) + return ret.type(orig_type) + + +class VisualTransformer(nn.Module): + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.heads = heads + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, + x: torch.Tensor, + skip_last_layer=False, + text_embedding=None, + text_mask=None): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + cls_emb = self.class_embedding.to(x.dtype) + x_zeros = torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([cls_emb + x_zeros, x], + dim=1) # shape = [*, grid ** 2 + 1, width] + + x = x + self.positional_embedding.to(x.dtype)[:x.size(1), :] + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + + x = self.transformer(x) + + x = x.permute(1, 0, 2) # LND -> NLD + + if skip_last_layer: + x = self.ln_post(x) + # x = x @ self.proj + else: + x = x @ self.proj + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int): + super().__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask()) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([])) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image): + return self.visual(image.type(self.dtype)) + + def encode_text(self, text): + x = self.token_embedding(text).type( + self.dtype) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding.type(self.dtype) + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = x[torch.arange(x.shape[0]), + text.argmax(dim=-1)] @ self.text_projection + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logit_scale * text_features @ image_features.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def load_from_config(config): + return CLIP(config.clip_embed_dim, config.clip_image_resolution, + config.clip_vision_layers, config.clip_vision_width, + config.clip_vision_patch_size, config.clip_context_length, + config.clip_vocab_size, config.clip_transformer_width, + config.clip_transformer_heads, config.clip_transformer_layers) diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/configuration_mplug.py b/runtime/Lib/site_packages/models/multi_modal/mplug/configuration_mplug.py new file mode 100644 index 0000000000000000000000000000000000000000..dcbb0270e4261ebf2acf02ffc3010645f7fb48fb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/configuration_mplug.py @@ -0,0 +1,180 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MPLUG model configuration """ +import os +from typing import Any, Dict, Union + +import yaml +from transformers import PretrainedConfig +from transformers.utils import logging + +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +class MPlugConfig(PretrainedConfig): + + model_type = 'mplug' + + def __init__( + self, + task=Tasks.visual_question_answering, + bert_config='config_bert.json', + image_res=504, + batch_size_train=128, + vision_width=1024, + distill=True, + clip_name='ViT-L-14', # ViT-B-16 | ViT-L-14 + batch_size_test=64, + k_test=128, + alpha=0.4, + warm_up=True, + eos='[SEP]', + optimizer=None, + schedular=None, + min_length=1, + max_length=10, + beam_size=5, + add_ocr=False, + add_object=False, + text_encoder='bert-base-uncased', + text_decoder='bert-base-uncased', + # clip + clip_embed_dim=768, + clip_image_resolution=224, + clip_vision_layers=24, + clip_vision_width=1024, + clip_vision_patch_size=14, + clip_context_length=77, + clip_vocab_size=49408, + clip_transformer_width=768, + clip_transformer_heads=12, + clip_transformer_layers=12, + # retrieval + queue_size=65536, + embed_dim=256, + temp=0.07, + **kwargs): + + super().__init__(**kwargs) + self.task = task + self.bert_config = bert_config + self.image_res = image_res + self.batch_size_train = batch_size_train + self.vision_width = vision_width + self.distill = distill + self.clip_name = clip_name + self.batch_size_test = batch_size_test + self.k_test = k_test + self.alpha = alpha + self.warm_up = warm_up + self.eos = eos + self.optimizer = optimizer + self.schedular = schedular + self.min_length = min_length + self.max_length = max_length + self.beam_size = beam_size + self.add_ocr = add_ocr + self.add_object = add_object + self.text_encoder = text_encoder + self.text_decoder = text_decoder + # clip + self.clip_embed_dim = clip_embed_dim + self.clip_image_resolution = clip_image_resolution + self.clip_vision_layers = clip_vision_layers + self.clip_vision_width = clip_vision_width + self.clip_vision_patch_size = clip_vision_patch_size + self.clip_context_length = clip_context_length + self.clip_vocab_size = clip_vocab_size + self.clip_transformer_width = clip_transformer_width + self.clip_transformer_heads = clip_transformer_heads + self.clip_transformer_layers = clip_transformer_layers + # retrieval + self.queue_size = queue_size + self.embed_dim = embed_dim + self.temp = temp + + @classmethod + def from_yaml_file(cls, yaml_file: Union[str, + os.PathLike]) -> Dict[str, Any]: + with open(yaml_file, 'r', encoding='utf-8') as reader: + config_dict = yaml.load(reader, Loader=yaml.Loader) + return cls(**config_dict) + + +class HiTeAConfig(PretrainedConfig): + + model_type = 'hitea' + + def __init__( + self, + task=Tasks.video_question_answering, + bert_config='config_bert.json', + image_res=224, + num_frames=16, + batch_size_train=32, + vision_width=768, + distill=True, + batch_size_test=64, + k_test=128, + alpha=0.4, + warm_up=True, + eos='[SEP]', + optimizer=None, + schedular=None, + min_length=1, + max_length=10, + beam_size=5, + text_encoder='bert-base-uncased', + text_decoder='bert-base-uncased', + # retrieval + queue_size=65536, + embed_dim=256, + temp=0.07, + **kwargs): + + super().__init__(**kwargs) + self.task = task + self.bert_config = bert_config + self.image_res = image_res + self.num_frames = num_frames + self.batch_size_train = batch_size_train + self.vision_width = vision_width + self.distill = distill + self.batch_size_test = batch_size_test + self.k_test = k_test + self.alpha = alpha + self.warm_up = warm_up + self.eos = eos + self.optimizer = optimizer + self.schedular = schedular + self.min_length = min_length + self.max_length = max_length + self.beam_size = beam_size + self.text_encoder = text_encoder + self.text_decoder = text_decoder + # retrieval + self.queue_size = queue_size + self.embed_dim = embed_dim + self.temp = temp + + @classmethod + def from_yaml_file(cls, yaml_file: Union[str, + os.PathLike]) -> Dict[str, Any]: + with open(yaml_file, 'r', encoding='utf-8') as reader: + config_dict = yaml.load(reader, Loader=yaml.Loader) + return cls(**config_dict) diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/modeling_mplug.py b/runtime/Lib/site_packages/models/multi_modal/mplug/modeling_mplug.py new file mode 100644 index 0000000000000000000000000000000000000000..5ebf4704880f2a7c35ec5f324da85538e16e495c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/modeling_mplug.py @@ -0,0 +1,2807 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch MPLUG model. """ + +import math +import os +from typing import Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from torch import Tensor, device, nn +from torch.nn import CrossEntropyLoss +from transformers import BertConfig, BertTokenizer +from transformers.activations import ACT2FN +from transformers.file_utils import (add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings) +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions) +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.utils import logging + +from modelscope.models.multi_modal.mplug.configuration_mplug import ( + HiTeAConfig, MPlugConfig) +from modelscope.models.multi_modal.mplug.mvit import MViTv2, MViTv2_Base_config +from modelscope.models.multi_modal.mplug.predictor import TextGenerator +from modelscope.utils.constant import ModelFile + +transformers.logging.set_verbosity_error() + +logger = logging.get_logger() + +CONFIG_NAME = 'config.yaml' + +_CONFIG_FOR_DOC = 'BertConfig' +_TOKENIZER_FOR_DOC = 'BertTokenizer' + + +def load_tf_weights_in_bert(model, config, tf_checkpoint_path): + """Load tf checkpoints in a pytorch model.""" + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + 'Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see ' + 'https://www.tensorflow.org/install/ for installation instructions.' + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info('Converting TensorFlow checkpoint from {}'.format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info('Loading TF weight {} with shape {}'.format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in [ + 'adam_v', 'adam_m', 'AdamWeightDecayOptimizer', + 'AdamWeightDecayOptimizer_1', 'global_step' + ] for n in name): + logger.info('Skipping {}'.format('/'.join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + scope_names = re.split(r'_(\d+)', m_name) + else: + scope_names = [m_name] + if scope_names[0] == 'kernel' or scope_names[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'output_bias' or scope_names[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif scope_names[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info('Skipping {}'.format('/'.join(name))) + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f'Pointer shape {pointer.shape} and array shape {array.shape} mismatched' + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info('Initialize PyTorch weight {}'.format(name)) + pointer.data = torch.from_numpy(array) + return model + + +def clamp_inf(tensor): + if tensor.dtype == torch.float16 and torch.isinf(tensor).any(): + clamp_value = torch.finfo(tensor.dtype).max - 1000 + tensor = torch.clamp(tensor, min=-clamp_value, max=clamp_value) + return tensor + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + + self.config = config + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config, is_cross_attention): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + self.key = nn.Linear(config.encoder_width, self.all_head_size) + self.value = nn.Linear(config.encoder_width, self.all_head_size) + else: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = clamp_inf(attention_scores) + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if is_cross_attention and self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + outputs = outputs + (past_key_value, ) + return outputs + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config, is_cross_attention=False): + super().__init__() + self.self = BertSelfAttention(config, is_cross_attention) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = clamp_inf(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = clamp_inf(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class FusionLayer(nn.Module): + + def __init__(self, config, layer_num): + super().__init__() + self.config = config + self.stride_layer = getattr(self.config, 'stride_layer', 100) + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BertAttention(config) + + self.crossattention = BertAttention(config, is_cross_attention=True) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + layer_nums=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + if layer_nums == 0 or layer_nums % self.stride_layer != 0: + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + assert encoder_hidden_states is not None, 'encoder_hidden_states must be given for cross-attention layers' + + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + elif layer_nums != 0 and layer_nums % self.stride_layer == 0: + self_attention_outputs = self.attention( + torch.cat([encoder_hidden_states, hidden_states], 1), + torch.cat([encoder_attention_mask, attention_mask], 3), + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + outputs = outputs + (present_key_value[0], present_key_value[1]) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertLayer(nn.Module): + + def __init__(self, config, layer_num): + super().__init__() + self.config = config + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BertAttention(config) + + self.has_cross_attention = getattr(self.config, 'add_cross_attention', + False) + if self.has_cross_attention: + self.crossattention = BertAttention( + config, is_cross_attention=True) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + + if self.has_cross_attention: + assert encoder_hidden_states is not None, 'encoder_hidden_states must be given for cross-attention layers' + + if type(encoder_hidden_states) == list: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states[(self.layer_num + - self.config.fusion_layer) + % len(encoder_hidden_states)], + encoder_attention_mask[(self.layer_num + - self.config.fusion_layer) + % len(encoder_hidden_states)], + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:-1] + + else: + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1: + -1] # add cross attentions if we output attention weights + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + outputs = outputs + (present_key_value[0], present_key_value[1]) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class FusionEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [FusionLayer(config, i) for i in range(config.num_hidden_layers)]) + self.start_layer = max(0, + config.num_hidden_layers - config.fusion_layers) + + def forward(self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + + next_decoder_cache = () if use_cache else None + + self.stride_layer = getattr(self.config, 'stride_layer', 100) + image_length = encoder_hidden_states.shape[1] + text_length = hidden_states.shape[1] + + for i in range(self.start_layer, len(self.layer)): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting ' + '`use_cache=False`...') + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return tuple( + module(*inputs, past_key_value, output_attentions)) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + i - self.start_layer, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + i - self.start_layer, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if hidden_states.shape[1] == (image_length + text_length): + encoder_hidden_states_new, hidden_states = torch.split( + hidden_states, (image_length, text_length), 1) + encoder_hidden_states += encoder_hidden_states_new + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + return [encoder_hidden_states, hidden_states] + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [BertLayer(config, i) for i in range(config.num_hidden_layers)]) + + def forward(self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + + for i in range(len(self.layer)): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting ' + '`use_cache=False`...') + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return tuple( + module(*inputs, past_key_value, output_attentions)) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), hidden_states, + attention_mask, layer_head_mask, encoder_hidden_states, + encoder_attention_mask) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class BertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class BertPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = BertConfig + load_tf_weights = load_tf_weights_in_bert + base_model_prefix = 'bert' + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +BERT_START_DOCSTRING = r""" + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + Parameters: + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): + Indices of input sequence tokens in the vocabulary. + Indices can be obtained using :class:`~transformers.BertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + `What are input IDs? <../glossary.html#input-ids>`__ + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + `What are attention masks? <../glossary.html#attention-mask>`__ + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + `What are position IDs? <../glossary.html#position-ids>`_ + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. +""" + + +@add_start_docstrings( + 'The bare Bert Model transformer outputting raw hidden-states without any specific head on top.', + BERT_START_DOCSTRING, +) +class BertModel(BertPreTrainedModel): + """ + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = BertEmbeddings(config) + + self.encoder = BertEncoder(config) + + self.pooler = BertPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + @add_start_docstrings_to_model_forward( + BERT_INPUTS_DOCSTRING.format('batch_size, sequence_length')) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint='bert-base-uncased', + output_type=BaseModelOutputWithPoolingAndCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def get_extended_attention_mask(self, attention_mask: Tensor, + input_shape: Tuple[int], device: device, + is_decoder: bool) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to + # [batch_size, num_heads, seq_length, seq_length] + if is_decoder: + batch_size, seq_length = input_shape + seq_ids = torch.arange(seq_length, device=device) + causal_mask = seq_ids[None, None, :].repeat( + batch_size, seq_length, 1) <= seq_ids[None, :, None] + # in case past_key_values are used we need to add a prefix ones mask to the causal mask + # causal and attention masks must have same type with pytorch version < 1.3 + causal_mask = causal_mask.to(attention_mask.dtype) + + if causal_mask.shape[1] < attention_mask.shape[1]: + prefix_seq_len = attention_mask.shape[ + 1] - causal_mask.shape[1] + causal_mask = torch.cat( + [ + torch.ones( + (batch_size, seq_length, prefix_seq_len), + device=device, + dtype=causal_mask.dtype), + causal_mask, + ], + axis=-1, + ) + + extended_attention_mask = causal_mask[:, + None, :, :] * attention_mask[:, + None, + None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + 'Wrong shape for input_ids (shape {}) or attention_mask (shape {})' + .format(input_shape, attention_mask.shape)) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + ): + r""" + encoder_hidden_states + (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values + (:obj:`tuple(tuple(torch.FloatTensor))` of length + :obj:`config.n_layers` with each tuple having 4 tensors of shape + :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = inputs_embeds.device + elif encoder_embeds is not None: + input_shape = encoder_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = encoder_embeds.device + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds or encoder_embeds' + ) + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device, is_decoder) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[ + 0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [ + self.invert_attention_mask(mask) + for mask in encoder_attention_mask + ] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + if encoder_embeds is None: + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + else: + embedding_output = encoder_embeds + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +class FusionModel(BertPreTrainedModel): + """ + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + self.encoder = FusionEncoder(config) + self.pooler = BertPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + @add_start_docstrings_to_model_forward( + BERT_INPUTS_DOCSTRING.format('batch_size, sequence_length')) + @add_code_sample_docstrings( + # tokenizer_class=_TOKENIZER_FOR_DOC, + processor_class=_TOKENIZER_FOR_DOC, + checkpoint='bert-base-uncased', + output_type=BaseModelOutputWithPoolingAndCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def get_extended_attention_mask(self, attention_mask: Tensor, + input_shape: Tuple[int], device: device, + is_decoder: bool) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to + # [batch_size, num_heads, seq_length, seq_length] + if is_decoder: + batch_size, seq_length = input_shape + seq_ids = torch.arange(seq_length, device=device) + causal_mask = seq_ids[None, None, :].repeat( + batch_size, seq_length, 1) <= seq_ids[None, :, None] + # in case past_key_values are used we need to add a prefix ones mask to the causal mask + # causal and attention masks must have same type with pytorch version < 1.3 + causal_mask = causal_mask.to(attention_mask.dtype) + + if causal_mask.shape[1] < attention_mask.shape[1]: + prefix_seq_len = attention_mask.shape[ + 1] - causal_mask.shape[1] + causal_mask = torch.cat( + [ + torch.ones( + (batch_size, seq_length, prefix_seq_len), + device=device, + dtype=causal_mask.dtype), + causal_mask, + ], + axis=-1, + ) + + extended_attention_mask = causal_mask[:, + None, :, :] * attention_mask[:, + None, + None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + 'Wrong shape for input_ids (shape {}) or attention_mask (shape {})' + .format(input_shape, attention_mask.shape)) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False): + r""" + encoder_hidden_states + (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values + (:obj:`tuple(tuple(torch.FloatTensor))` of length + :obj:`config.n_layers` with each tuple having 4 tensors of shape + :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = inputs_embeds.device + elif encoder_embeds is not None: + input_shape = encoder_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = encoder_embeds.device + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds or encoder_embeds' + ) + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device, is_decoder) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[ + 0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [ + self.invert_attention_mask(mask) + for mask in encoder_attention_mask + ] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + if encoder_embeds is None: + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + else: + embedding_output = encoder_embeds + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + encoder_hidden_states, sequence_output = encoder_outputs + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return [encoder_hidden_states, sequence_output] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +@add_start_docstrings( + """Bert Model with a `language modeling` head on top for CLM fine-tuning. """, + BERT_START_DOCSTRING) +class BertLMHeadModel(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + @add_start_docstrings_to_model_forward( + BERT_INPUTS_DOCSTRING.format('batch_size, sequence_length')) + @replace_return_docstrings( + output_type=CausalLMOutputWithCrossAttentions, + config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=True, + reduction='mean', + soft_labels=None, + alpha=0, + return_logits=False, + ): + r""" + encoder_hidden_states + (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are + ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` + past_key_values + (:obj:`tuple(tuple(torch.FloatTensor))` of length + :obj:`config.n_layers` with each tuple having 4 tensors of shape + :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + Returns: + + Example: + >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig + >>> import torch + >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased') + >>> config = BertConfig.from_pretrained("bert-base-cased") + >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config) + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores[:, :-1, :].contiguous() + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, : + -1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss(reduction=reduction) + lm_loss = loss_fct( + shifted_prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + lm_loss = lm_loss.view(prediction_scores.size(0), -1).sum(1) + + if soft_labels is not None: + loss_distill = -torch.sum( + F.log_softmax(shifted_prediction_scores, dim=1) * soft_labels, + dim=-1) + loss_distill = (loss_distill * (labels != -100)).sum(1) + lm_loss = (1 - alpha) * lm_loss + alpha * loss_distill + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((lm_loss, ) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, + input_ids, + past=None, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + + return { + 'input_ids': + input_ids, + 'attention_mask': + attention_mask, + 'past_key_values': + past, + 'encoder_hidden_states': + model_kwargs.get('encoder_hidden_states', None), + 'encoder_attention_mask': + model_kwargs.get('encoder_attention_mask', None), + 'is_decoder': + True, + } + + def _reorder_cache(self, past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple( + past_state.index_select(0, beam_idx) + for past_state in layer_past), ) + return reordered_past + + +class BertPrefixModel(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config): + super().__init__(config) + + self.bert = BertModel(config, add_pooling_layer=False) + self.cls = BertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + @add_start_docstrings_to_model_forward( + BERT_INPUTS_DOCSTRING.format('batch_size, sequence_length')) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint='bert-base-uncased', + output_type=CausalLMOutputWithCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=True, + reduction='mean', + soft_labels=None, + alpha=0, + return_logits=False, + ): + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores[:, :-1, :].contiguous() + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, : + -1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss() + lm_loss = loss_fct( + shifted_prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + if soft_labels is not None: + loss_distill = -torch.sum( + F.log_softmax(shifted_prediction_scores, dim=1) * soft_labels, + dim=-1) + loss_distill = loss_distill[labels != -100].mean() + lm_loss = (1 - alpha) * lm_loss + alpha * loss_distill + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((lm_loss, ) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + +class MPlug(PreTrainedModel): + config_class = MPlugConfig + + def __init__(self, config): + super().__init__(config) + self.config = config + self.tokenizer = BertTokenizer.from_pretrained( + os.path.join(config.model_dir, ModelFile.VOCAB_FILE)) + self.module_setting(config) + self.visual_encoder = self._initialize_clip(config) + self.text_encoder = BertModel( + self.config_encoder, add_pooling_layer=False) + self.fusion_encoder = FusionModel( + self.config_fusion, add_pooling_layer=False) + + @classmethod + def from_pretrained(cls, model_dir, task=None, load_checkpoint=True): + from modelscope.utils.constant import Tasks + + task_mapping = { + Tasks.visual_question_answering: MPlugForVisualQuestionAnswering, + Tasks.image_captioning: MPlugForImageCaption, + Tasks.image_text_retrieval: MPlugForImageTextRetrieval, + } + config = cls.config_class.from_yaml_file( + os.path.join(model_dir, CONFIG_NAME)) + config.model_dir = model_dir + if task is None: + task = config.task + model = task_mapping[task](config) + if load_checkpoint: + checkpoint_path = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + checkpoint = torch.load(checkpoint_path, map_location='cpu') + if 'model' in checkpoint: + checkpoint = checkpoint['model'] + if 'module' in checkpoint: + checkpoint = checkpoint['module'] + checkpoint = { + k.replace('model.', ''): v + for k, v in checkpoint.items() + } + + msg = model.load_state_dict(checkpoint, strict=False) + print('load checkpoint from %s' % checkpoint_path) + print(msg) + return model + + @staticmethod + def _initialize_clip(config, num_patches=240): + + def resize_pos_embed(posemb, posemb_new): + # Rescale the grid of position embeddings when loading from state_dict. Adapted from + # https://github.com/google-research/vision_transformer/blob/00883dd691c63a6830751563748663526e811cee/vit_jax/checkpoint.py#L224 + ntok_new = posemb_new.shape[1] + + posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:] + ntok_new -= 1 + + gs_old = int(math.sqrt(len(posemb_grid))) + gs_new = int(math.sqrt(ntok_new)) + # _logger.info('Position embedding grid-size from %s to %s', gs_old, gs_new) + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, + -1).permute(0, 3, 1, 2) + orig = posemb_grid.dtype + posemb_grid = F.interpolate( + posemb_grid.float(), size=(gs_new, gs_new), mode='bilinear') + posemb_grid = posemb_grid.to(orig) + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape( + 1, gs_new * gs_new, -1) + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + return posemb + + from .clip import clip + clip_model = clip.load_from_config(config) + if 'ViT-B-16' in config.clip_name: + num_patches = int(config.image_res * config.image_res / (16 * 16)) + pos_embed = nn.Parameter(torch.zeros(num_patches + 1, 768).float()) + else: + num_patches = int(config.image_res * config.image_res / (14 * 14)) + pos_embed = nn.Parameter( + torch.zeros(num_patches + 1, 1024).float()) + pos_embed.weight = resize_pos_embed( + clip_model.visual.positional_embedding.unsqueeze(0), + pos_embed.unsqueeze(0)) + clip_model.visual.positional_embedding = pos_embed + return clip_model + + def init_distill(self, config): + self.distill = config.distill + if self.distill: + self.visual_encoder_m = self._initialize_clip(config) + self.text_encoder_m = BertModel( + self.config_encoder, add_pooling_layer=False) + self.fusion_encoder_m = FusionModel( + self.config_fusion, add_pooling_layer=False) + self.text_decoder_m = BertLMHeadModel(self.config_decoder) + self.model_pairs = [ + [self.visual_encoder, self.visual_encoder_m], + [self.text_encoder, self.text_encoder_m], + [self.text_decoder, self.text_decoder_m], + ] + if self.config_encoder.hidden_size != config.vision_width: + self.visn_fc_m = nn.Linear(config.vision_width, + self.config_encoder.hidden_size) + self.visn_layer_norm_m = nn.LayerNorm( + self.config_encoder.hidden_size, eps=1e-12) + self.dropout_m = nn.Dropout( + self.config_encoder.hidden_dropout_prob) + self.model_pairs.extend( + [[self.visn_fc, self.visn_fc_m], + [self.visn_layer_norm, self.visn_layer_norm_m]]) + self.copy_params() + self.momentum = 0.995 + + def forward(self, *args, **kwargs): + raise NotImplementedError + + def module_setting(self, config): + bert_config_path = os.path.join(config.model_dir, config.bert_config) + self.config_encoder = BertConfig.from_json_file(bert_config_path) + self.config_encoder.num_hidden_layers = self.config_encoder.text_encoder_layers + self.config_fusion = BertConfig.from_json_file(bert_config_path) + self.config_decoder = BertConfig.from_json_file(bert_config_path) + self.config_decoder.add_cross_attention = True + self.config_decoder.num_hidden_layers = self.config_decoder.text_decode_layers + self.large = False + if self.config_encoder.hidden_size != config.vision_width: + self.visn_fc = nn.Linear(config.vision_width, + self.config_encoder.hidden_size) + self.visn_layer_norm = nn.LayerNorm( + self.config_encoder.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(self.config_encoder.hidden_dropout_prob) + self.large = True + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), + model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), + model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * ( + 1. - self.momentum) + + def generation(self, question_states, question_atts, out_size=1): + encoder_inputs = [question_states, question_atts] + topk_ids, topk_scores = self.beam_generator.translate_batch( + encoder_inputs, out_size=out_size) + return topk_ids, topk_scores + + @staticmethod + def _tile(x, dim, n_tile): + import numpy as np + init_dim = x.size(dim) + repeat_idx = [1] * x.dim() + repeat_idx[dim] = n_tile + x = x.repeat(*(repeat_idx)) + order_index = torch.LongTensor( + np.concatenate( + [init_dim * np.arange(n_tile) + i for i in range(init_dim)])) + return torch.index_select(x, dim, order_index.to(x.device)) + + +class MPlugForVisualQuestionAnswering(MPlug): + + def __init__(self, config): + super().__init__(config) + self.text_decoder = BertLMHeadModel(self.config_decoder) + self.beam_generator = TextGenerator(config, self.text_decoder) + self.init_distill(config) + + def forward(self, + image, + question, + answer=None, + alpha=0, + k=None, + weights=None, + train=True): + image = image.to(dtype=next(self.parameters()).dtype) + image_embeds = self.visual_encoder.visual(image, skip_last_layer=True) + if self.large: + image_embeds = self.dropout( + self.visn_layer_norm(self.visn_fc(image_embeds))) + image_atts = torch.ones( + image_embeds.size()[:-1], dtype=torch.long).to(image.device) + + if train: + ''' + k: number of answers for each question + weights: weight for each answer + ''' + answer_targets = answer.input_ids.masked_fill( + answer.input_ids == self.tokenizer.pad_token_id, -100) + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=False) + + image_output, question_output = fusion_output + + question_output = torch.cat([image_output, question_output], 1) + merge_text_attention = torch.cat( + [image_atts, question.attention_mask], 1) + + if k is None: + k = [1] * question_output.shape[0] + question_states = [] + question_atts = [] + for b, n in enumerate(k): + question_states += [question_output[b]] * n + question_atts += [merge_text_attention[b]] * n + question_states = torch.stack(question_states, 0) + question_atts = torch.stack(question_atts, 0) + + if self.distill: + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m.visual( + image, skip_last_layer=True) + if self.large: + image_embeds_m = self.dropout_m( + self.visn_layer_norm_m( + self.visn_fc_m(image_embeds_m))) + text_output_m = self.text_encoder_m( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds_m = text_output_m.last_hidden_state + fusion_output_m = self.fusion_encoder_m( + encoder_embeds=text_embeds_m, + attention_mask=question.attention_mask, + encoder_hidden_states=image_embeds_m, + encoder_attention_mask=image_atts, + return_dict=False) + + image_output_m, question_output_m = fusion_output_m + question_output_m = torch.cat( + [image_output_m, question_output_m], 1) + + question_states_m = [] + for b, n in enumerate(k): + question_states_m += [question_output_m[b]] * n + question_states_m = torch.stack(question_states_m, 0) + + logits_m = self.text_decoder_m( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states_m, + encoder_attention_mask=question_atts, + return_logits=True, + ) + + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states, + encoder_attention_mask=question_atts, + labels=answer_targets, + return_dict=True, + soft_labels=F.softmax(logits_m, dim=-1), + reduction='none', + ) + else: + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states, + encoder_attention_mask=question_atts, + labels=answer_targets, + return_dict=True, + reduction='none', + ) + if weights is None: + weights = 1 + loss = weights * answer_output.loss + loss = loss.sum() / image.size(0) + + return loss + + else: + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=False) + image_output, question_output = fusion_output + question_output = torch.cat([image_output, question_output], 1) + merge_text_attention = torch.cat( + [image_atts, question.attention_mask], 1) + topk_ids, topk_probs = self.generation(question_output, + merge_text_attention) + return topk_ids, topk_probs + + +class MPlugForImageCaption(MPlug): + + def __init__(self, config): + super().__init__(config) + self.text_decoder = BertPrefixModel(self.config_decoder) + self.beam_generator = TextGenerator(config, self.text_decoder) + + def beam_search(self, + image, + question, + answer=None, + train=True, + out_size=5): + image_embeds = self.visual_encoder.visual(image, skip_last_layer=True) + if self.large: + image_embeds = self.dropout( + self.visn_layer_norm(self.visn_fc(image_embeds))) + image_atts = torch.ones( + image_embeds.size()[:-1], dtype=torch.long).to(image.device) + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=False) + image_output, question_output = fusion_output + question_output = torch.cat([image_output, question_output], 1) + merge_text_attention = torch.cat([image_atts, question.attention_mask], + 1) + topk_ids, topk_probs = self.generation( + question_output, merge_text_attention, out_size=out_size) + return topk_ids, topk_probs + + def forward(self, + image, + question, + answer=None, + train=True, + out_size=5, + scst=False): + if (scst): + return self.beam_search( + image, question, answer, train=True, out_size=out_size) + image = image.to(dtype=next(self.parameters()).dtype) + image_embeds = self.visual_encoder.visual(image, skip_last_layer=True) + if self.large: + image_embeds = self.dropout( + self.visn_layer_norm(self.visn_fc(image_embeds))) + image_atts = torch.ones( + image_embeds.size()[:-1], dtype=torch.long).to(image.device) + + if train: + answer_targets = answer.input_ids.masked_fill( + answer.input_ids == self.tokenizer.pad_token_id, -100) + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + labels=answer_targets, + return_dict=True, + reduction='none') + loss = answer_output.loss + + return loss + else: + topk_ids, topk_probs = self.generation(image_embeds, image_atts) + return topk_ids, topk_probs + + +class MPlugForImageTextRetrieval(MPlug): + + def __init__(self, config): + super().__init__(config) + self.embed_dim = config.embed_dim + self.temp = nn.Parameter(torch.ones([]) * config.temp) + self.queue_size = config.queue_size + self.momentum = config.momentum + self.alpha = config.alpha + + self.queue_size = config.queue_size + self.text_width = self.config_encoder.hidden_size + self.embed_dim = config.embed_dim + + self.vision_proj = nn.Linear(self.text_width, self.embed_dim) + self.text_proj = nn.Linear(self.text_width, self.embed_dim) + self.itm_head = nn.Linear(self.text_width, 2) + + self.register_buffer('image_queue', + torch.randn(self.embed_dim, self.queue_size)) + self.register_buffer('text_queue', + torch.randn(self.embed_dim, self.queue_size)) + self.register_buffer('idx_queue', torch.full((1, self.queue_size), + -100)) + self.register_buffer('queue_ptr', torch.zeros(1, dtype=torch.long)) + + self.image_queue = F.normalize(self.image_queue, dim=0) + self.text_queue = F.normalize(self.text_queue, dim=0) + self.init_distill(config) + + def init_distill(self, config): + self.distill = config.distill + if self.distill: + self.visual_encoder_m = self._initialize_clip(config) + self.text_encoder_m = BertModel( + self.config_encoder, add_pooling_layer=False) + self.fusion_encoder_m = FusionModel( + self.config_fusion, add_pooling_layer=False) + self.vision_proj_m = nn.Linear(self.text_width, self.embed_dim) + self.text_proj_m = nn.Linear(self.text_width, self.embed_dim) + self.model_pairs = [ + [self.visual_encoder, self.visual_encoder_m], + [self.text_encoder, self.text_encoder_m], + [self.text_proj, self.text_proj_m], + [self.vision_proj, self.vision_proj_m], + ] + if self.config_encoder.hidden_size != config.vision_width: + self.visn_fc_m = nn.Linear(config.vision_width, + self.config_encoder.hidden_size) + self.visn_layer_norm_m = nn.LayerNorm( + self.config_encoder.hidden_size, eps=1e-12) + self.dropout_m = nn.Dropout( + self.config_encoder.hidden_dropout_prob) + self.model_pairs.extend( + [[self.visn_fc, self.visn_fc_m], + [self.visn_layer_norm, self.visn_layer_norm_m]]) + self.copy_params() + self.momentum = 0.995 + + @torch.no_grad() + def _dequeue_and_enqueue(self, image_feat, text_feat, idx): + + def concat_all_gather(tensor): + """ + Performs all_gather operation on the provided tensors. + *** Warning ***: torch.distributed.all_gather has no gradient. + """ + if not torch.distributed.is_initialized(): + return tensor + tensors_gather = [ + torch.ones_like(tensor) + for _ in range(torch.distributed.get_world_size()) + ] + torch.distributed.all_gather( + tensors_gather, tensor, async_op=False) + + output = torch.cat(tensors_gather, dim=0) + return output + + # gather keys before updating queue + image_feats = concat_all_gather(image_feat) + text_feats = concat_all_gather(text_feat) + idxs = concat_all_gather(idx) + + batch_size = image_feats.shape[0] + + ptr = int(self.queue_ptr) + # assert self.queue_size % batch_size == 0 # for simplicity + + # replace the keys at ptr (dequeue and enqueue) + self.image_queue[:, ptr:ptr + batch_size] = image_feats.T + self.text_queue[:, ptr:ptr + batch_size] = text_feats.T + self.idx_queue[:, ptr:ptr + batch_size] = idxs.T + ptr = (ptr + batch_size) % self.queue_size # move pointer + + self.queue_ptr[0] = ptr + + def forward(self, image, text, idx=None, train=True): + if train: + image_embeds = self.visual_encoder.visual( + image, skip_last_layer=True) + if self.large: + image_embeds = self.dropout( + self.visn_layer_norm(self.visn_fc(image_embeds))) + image_atts = torch.ones( + image_embeds.size()[:-1], dtype=torch.long).to(image.device) + + image_feat = F.normalize( + self.vision_proj(image_embeds[:, 0, :]), dim=-1) + text_output = self.text_encoder( + text.input_ids, + attention_mask=text.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + text_feat = F.normalize( + self.text_proj(text_embeds[:, 0, :]), dim=-1) + + idx = idx.view(-1, 1) + idx_all = torch.cat( + [idx.t(), self.idx_queue.clone().detach()], dim=1) + pos_idx = torch.eq(idx, idx_all).float() + sim_targets = pos_idx / pos_idx.sum(1, keepdim=True) + + with torch.no_grad(): + self._momentum_update() + image_embeds_m = self.visual_encoder_m.visual( + image, skip_last_layer=True) + if self.large: + image_embeds_m = self.dropout_m( + self.visn_layer_norm_m(self.visn_fc_m(image_embeds_m))) + image_feat_m = F.normalize( + self.vision_proj_m(image_embeds_m[:, 0, :]), dim=-1) + image_feat_all = torch.cat( + [image_feat_m.t(), + self.image_queue.clone().detach()], + dim=1) + text_output_m = self.text_encoder_m( + text.input_ids, + attention_mask=text.attention_mask, + return_dict=True) + text_feat_m = F.normalize( + self.text_proj_m(text_output_m.last_hidden_state[:, 0, :]), + dim=-1) + text_feat_all = torch.cat( + [text_feat_m.t(), + self.text_queue.clone().detach()], dim=1) + + if self.distill: + sim_i2t_m = image_feat_m @ text_feat_all / self.temp + sim_t2i_m = text_feat_m @ image_feat_all / self.temp + + sim_i2t_targets = self.alpha * F.softmax( + sim_i2t_m, dim=1) + (1 - self.alpha) * sim_targets + sim_t2i_targets = self.alpha * F.softmax( + sim_t2i_m, dim=1) + (1 - self.alpha) * sim_targets + + sim_i2t = image_feat @ text_feat_all / self.temp + sim_t2i = text_feat @ image_feat_all / self.temp + + if self.distill: + loss_i2t = -torch.sum( + F.log_softmax(sim_i2t, dim=1) * sim_i2t_targets, + dim=1).mean() + loss_t2i = -torch.sum( + F.log_softmax(sim_t2i, dim=1) * sim_t2i_targets, + dim=1).mean() + else: + loss_i2t = -torch.sum( + F.log_softmax(sim_i2t, dim=1) * sim_targets, dim=1).mean() + loss_t2i = -torch.sum( + F.log_softmax(sim_t2i, dim=1) * sim_targets, dim=1).mean() + + loss_ita = (loss_i2t + loss_t2i) / 2 + + self._dequeue_and_enqueue(image_feat_m, text_feat_m, idx) + + # forward the positve image-text pair + _, output_pos = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=text.attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=False, + ) + with torch.no_grad(): + bs = image.size(0) + weights_i2t = F.softmax(sim_i2t[:, :bs], dim=1) + weights_t2i = F.softmax(sim_t2i[:, :bs], dim=1) + + mask = torch.eq(idx, idx.T) + weights_i2t.masked_fill_(mask, 0) + weights_t2i.masked_fill_(mask, 0) + + # select a negative image for each text + image_embeds_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_t2i[b], 1).item() + image_embeds_neg.append(image_embeds[neg_idx]) + image_embeds_neg = torch.stack(image_embeds_neg, dim=0) + + # select a negative text for each image + text_embeds_neg = [] + text_atts_neg = [] + for b in range(bs): + neg_idx = torch.multinomial(weights_i2t[b], 1).item() + text_embeds_neg.append(text_embeds[neg_idx]) + text_atts_neg.append(text.attention_mask[neg_idx]) + text_embeds_neg = torch.stack(text_embeds_neg, dim=0) + text_atts_neg = torch.stack(text_atts_neg, dim=0) + + text_embeds_all = torch.cat([text_embeds, text_embeds_neg], dim=0) + text_atts_all = torch.cat([text.attention_mask, text_atts_neg], + dim=0) + + image_embeds_all = torch.cat([image_embeds_neg, image_embeds], + dim=0) + image_atts_all = torch.cat([image_atts, image_atts], dim=0) + + _, output_neg = self.fusion_encoder( + encoder_embeds=text_embeds_all, + attention_mask=text_atts_all, + encoder_hidden_states=image_embeds_all, + encoder_attention_mask=image_atts_all, + return_dict=False, + ) + + vl_embeddings = torch.cat( + [output_pos[:, 0, :], output_neg[:, 0, :]], dim=0) + vl_output = self.itm_head(vl_embeddings) + + ones_tmp = torch.ones(bs, dtype=torch.long) + zeros_tmp = torch.zeros(2 * bs, dtype=torch.long) + itm_labels = torch.cat([ones_tmp, zeros_tmp], + dim=0).to(image.device) + loss_itm = F.cross_entropy(vl_output, itm_labels) + + return loss_ita + loss_itm + else: + text_output = self.text_encoder( + text.input_ids, attention_mask=text.attention_mask) + text_feat = text_output.last_hidden_state + image_feat = self.visual_encoder.visual( + image, skip_last_layer=True) + image_feat = self.visn_layer_norm(self.visn_fc(image_feat)) + image_att = torch.ones( + image_feat.size()[:-1], + dtype=torch.long, + device=image_feat.device) + _, output = self.fusion_encoder( + encoder_embeds=text_feat, + attention_mask=text.attention_mask, + encoder_hidden_states=image_feat, + encoder_attention_mask=image_att, + return_dict=False, + ) + scores = self.itm_head(output[:, 0, :]) + scores = F.softmax(scores, dim=-1) + + return scores + + +class HiTeA(PreTrainedModel): + config_class = HiTeAConfig + + def __init__(self, config): + super().__init__(config) + self.config = config + self.tokenizer = BertTokenizer.from_pretrained( + os.path.join(config.model_dir, ModelFile.VOCAB_FILE)) + self.module_setting(config) + self.visual_encoder = MViTv2( + img_size=config.image_res, + config=MViTv2_Base_config, + num_frames=config.num_frames) + self.text_encoder = BertModel( + self.config_encoder, add_pooling_layer=False) + self.fusion_encoder = FusionModel( + self.config_fusion, add_pooling_layer=False) + + @classmethod + def from_pretrained(cls, model_dir, load_checkpoint=True): + from modelscope.utils.constant import Tasks + + task_mapping = { + Tasks.video_question_answering: HiTeAForVideoQuestionAnswering, + Tasks.video_captioning: HiTeAForVideoCaption, + } + config = cls.config_class.from_yaml_file( + os.path.join(model_dir, CONFIG_NAME)) + config.model_dir = model_dir + model = task_mapping[config.task](config) + if load_checkpoint: + checkpoint_path = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + checkpoint = torch.load(checkpoint_path, map_location='cpu') + if 'model' in checkpoint: + checkpoint = checkpoint['model'] + if 'module' in checkpoint: + checkpoint = checkpoint['module'] + checkpoint = { + k.replace('model.', ''): v + for k, v in checkpoint.items() + } + + model.load_state_dict(checkpoint, strict=False) + return model + + def init_distill(self, config): + self.distill = config.distill + if self.distill: + self.visual_encoder_m = MViTv2( + img_size=config.image_res, + config=MViTv2_Base_config, + num_frames=config.num_frames) + self.text_encoder_m = BertModel( + self.config_encoder, add_pooling_layer=False) + self.fusion_encoder_m = FusionModel( + self.config_fusion, add_pooling_layer=False) + self.text_decoder_m = BertLMHeadModel(self.config_decoder) + self.model_pairs = [ + [self.visual_encoder, self.visual_encoder_m], + [self.text_encoder, self.text_encoder_m], + [self.text_decoder, self.text_decoder_m], + ] + self.copy_params() + self.momentum = 0.995 + + def forward(self, *args, **kwargs): + raise NotImplementedError + + def module_setting(self, config): + bert_config_path = os.path.join(config.model_dir, config.bert_config) + self.config_encoder = BertConfig.from_json_file(bert_config_path) + self.config_encoder.num_hidden_layers = self.config_encoder.text_encoder_layers + self.config_fusion = BertConfig.from_json_file(bert_config_path) + self.config_decoder = BertConfig.from_json_file(bert_config_path) + self.config_decoder.add_cross_attention = True + self.config_decoder.num_hidden_layers = self.config_decoder.text_decode_layers + + @torch.no_grad() + def copy_params(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), + model_pair[1].parameters()): + param_m.data.copy_(param.data) # initialize + param_m.requires_grad = False # not update by gradient + + @torch.no_grad() + def _momentum_update(self): + for model_pair in self.model_pairs: + for param, param_m in zip(model_pair[0].parameters(), + model_pair[1].parameters()): + param_m.data = param_m.data * self.momentum + param.data * ( + 1. - self.momentum) + + def generation(self, question_states, question_atts, out_size=1): + encoder_inputs = [question_states, question_atts] + topk_ids, topk_scores = self.beam_generator.translate_batch( + encoder_inputs, out_size=out_size) + return topk_ids, topk_scores + + @staticmethod + def _tile(x, dim, n_tile): + import numpy as np + init_dim = x.size(dim) + repeat_idx = [1] * x.dim() + repeat_idx[dim] = n_tile + x = x.repeat(*(repeat_idx)) + order_index = torch.LongTensor( + np.concatenate( + [init_dim * np.arange(n_tile) + i for i in range(init_dim)])) + return torch.index_select(x, dim, order_index.to(x.device)) + + +class HiTeAForVideoQuestionAnswering(HiTeA): + + def __init__(self, config): + super().__init__(config) + self.text_decoder = BertLMHeadModel(self.config_decoder) + self.beam_generator = TextGenerator(config, self.text_decoder) + self.init_distill(config) + + def forward(self, + video, + question, + answer=None, + alpha=0, + k=None, + weights=None, + train=True): + video = video.to(dtype=next(self.parameters()).dtype) + video_embeds = self.visual_encoder(video) + video_atts = torch.ones( + video_embeds.size()[:-1], dtype=torch.long).to(video.device) + + if train: + ''' + k: number of answers for each question + weights: weight for each answer + ''' + answer_targets = answer.input_ids.masked_fill( + answer.input_ids == self.tokenizer.pad_token_id, -100) + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=video_embeds, + encoder_attention_mask=video_atts, + return_dict=False) + + video_output, question_output = fusion_output + + question_output = torch.cat([video_output, question_output], 1) + merge_text_attention = torch.cat( + [video_atts, question.attention_mask], 1) + + if k is None: + k = [1] * question_output.shape[0] + question_states = [] + question_atts = [] + for b, n in enumerate(k): + question_states += [question_output[b]] * n + question_atts += [merge_text_attention[b]] * n + question_states = torch.stack(question_states, 0) + question_atts = torch.stack(question_atts, 0) + + if self.distill: + with torch.no_grad(): + self._momentum_update() + video_embeds_m = self.visual_encoder_m(video) + text_output_m = self.text_encoder_m( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds_m = text_output_m.last_hidden_state + fusion_output_m = self.fusion_encoder_m( + encoder_embeds=text_embeds_m, + attention_mask=question.attention_mask, + encoder_hidden_states=video_embeds_m, + encoder_attention_mask=video_atts, + return_dict=False) + + image_output_m, question_output_m = fusion_output_m + question_output_m = torch.cat( + [image_output_m, question_output_m], 1) + + question_states_m = [] + for b, n in enumerate(k): + question_states_m += [question_output_m[b]] * n + question_states_m = torch.stack(question_states_m, 0) + + logits_m = self.text_decoder_m( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states_m, + encoder_attention_mask=question_atts, + return_logits=True, + ) + + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states, + encoder_attention_mask=question_atts, + labels=answer_targets, + return_dict=True, + soft_labels=F.softmax(logits_m, dim=-1), + reduction='none', + ) + else: + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=question_states, + encoder_attention_mask=question_atts, + labels=answer_targets, + return_dict=True, + reduction='none', + ) + if weights is None: + weights = 1 + loss = weights * answer_output.loss + loss = loss.sum() / video.size(0) + + return loss + + else: + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=video_embeds, + encoder_attention_mask=video_atts, + return_dict=False) + video_output, question_output = fusion_output + question_output = torch.cat([video_output, question_output], 1) + merge_text_attention = torch.cat( + [video_atts, question.attention_mask], 1) + topk_ids, topk_probs = self.generation(question_output, + merge_text_attention) + return topk_ids, topk_probs + + +class HiTeAForVideoCaption(HiTeA): + + def __init__(self, config): + super().__init__(config) + self.text_decoder = BertPrefixModel(self.config_decoder) + self.beam_generator = TextGenerator(config, self.text_decoder) + + def beam_search(self, + video, + question, + answer=None, + train=True, + out_size=5): + video_embeds = self.visual_encoder(video) + video_atts = torch.ones( + video_embeds.size()[:-1], dtype=torch.long).to(video.device) + text_output = self.text_encoder( + question.input_ids, + attention_mask=question.attention_mask, + return_dict=True) + text_embeds = text_output.last_hidden_state + fusion_output = self.fusion_encoder( + encoder_embeds=text_embeds, + attention_mask=question.attention_mask, + encoder_hidden_states=video_embeds, + encoder_attention_mask=video_atts, + return_dict=False) + video_output, question_output = fusion_output + question_output = torch.cat([video_output, question_output], 1) + merge_text_attention = torch.cat([video_atts, question.attention_mask], + 1) + topk_ids, topk_probs = self.generation( + question_output, merge_text_attention, out_size=out_size) + return topk_ids, topk_probs + + def forward(self, + video, + question, + answer=None, + train=True, + out_size=5, + scst=False): + if (scst): + return self.beam_search( + video, question, answer, train=True, out_size=out_size) + video = video.to(dtype=next(self.parameters()).dtype) + video_embeds = self.visual_encoder(video) + video_atts = torch.ones( + video_embeds.size()[:-1], dtype=torch.long).to(video.device) + + if train: + answer_targets = answer.input_ids.masked_fill( + answer.input_ids == self.tokenizer.pad_token_id, -100) + answer_output = self.text_decoder( + answer.input_ids, + attention_mask=answer.attention_mask, + encoder_hidden_states=video_embeds, + encoder_attention_mask=video_atts, + labels=answer_targets, + return_dict=True, + reduction='none') + loss = answer_output.loss + + return loss + else: + topk_ids, topk_probs = self.generation(video_embeds, video_atts) + return topk_ids, topk_probs diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/mvit.py b/runtime/Lib/site_packages/models/multi_modal/mplug/mvit.py new file mode 100644 index 0000000000000000000000000000000000000000..f3140ce46ebf2365b1864b5cde2981e11e91ad68 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/mvit.py @@ -0,0 +1,1007 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. All Rights Reserved. + +from collections import OrderedDict +from functools import partial + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import trunc_normal_ + +try: + from fairscale.nn.checkpoint import checkpoint_wrapper +except ImportError: + checkpoint_wrapper = None + +MViTv2_Base_config = { + 'depth': + 24, + 'dim_mul': [[2, 2.0], [5, 2.0], [21, 2.0]], + 'head_mul': [[2, 2.0], [5, 2.0], [21, 2.0]], + 'pool_q_stride': + [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], + [5, 1, 2, 2], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], + [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], + [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], + [20, 1, 1, 1], [21, 1, 2, 2], [22, 1, 1, 1], [23, 1, 1, 1]], + 'pool_kvq_kernel': [3, 3, 3], + 'pool_kv_stride_adaptive': [1, 4, 4], +} + + +def interpolate_rel_pos_embed(state_dict_origin, + state_dict_model, + temporal=True, + verbose=False): + rel_pos_embed_types = ['rel_pos_h', 'rel_pos_w'] + if temporal: + rel_pos_embed_types += ['rel_pos_t'] + + state_dict_inflated = state_dict_origin.copy() + for k, v2d in state_dict_origin.items(): + if any([x in k for x in rel_pos_embed_types]): + v3d = state_dict_model[k] + if v2d.shape[0] != v3d.shape[0]: + rel_pos_resized = F.interpolate( + v2d.reshape(1, v2d.shape[0], -1).permute(0, 2, 1), + size=v3d.shape[0], + mode='linear', + ) + v3d = rel_pos_resized.reshape(-1, v3d.shape[0]).permute(1, 0) + if verbose: + print('Inflate {}: {} -> {}: {}'.format( + k, v2d.shape, k, v3d.shape)) + else: + v3d = v2d + state_dict_inflated[k] = v3d.clone() + return state_dict_inflated + + +def _prepare_mvit_configs(cfg): + depth = cfg['depth'] + dim_mul, head_mul = torch.ones(depth + 1), torch.ones(depth + 1) + for i in range(len(cfg['dim_mul'])): + dim_mul[cfg['dim_mul'][i][0]] = cfg['dim_mul'][i][1] + for i in range(len(cfg['head_mul'])): + head_mul[cfg['head_mul'][i][0]] = cfg['head_mul'][i][1] + + pool_q = [[] for i in range(depth)] + pool_kv = [[] for i in range(depth)] + stride_q = [[] for i in range(depth)] + stride_kv = [[] for i in range(depth)] + + for i in range(len(cfg['pool_q_stride'])): + stride_q[cfg['pool_q_stride'][i][0]] = cfg['pool_q_stride'][i][1:] + pool_q[cfg['pool_q_stride'][i][0]] = cfg['pool_kvq_kernel'] + + if cfg['pool_kv_stride_adaptive'] is not None: + _stride_kv = cfg['pool_kv_stride_adaptive'] + cfg['pool_kv_stride'] = [] + for i in range(cfg['depth']): + if len(stride_q[i]) > 0: + _stride_kv = [ + max(_stride_kv[d] // stride_q[i][d], 1) + for d in range(len(_stride_kv)) + ] + cfg['pool_kv_stride'].append([i] + _stride_kv) + + for i in range(len(cfg['pool_kv_stride'])): + stride_kv[cfg['pool_kv_stride'][i][0]] = cfg['pool_kv_stride'][i][1:] + pool_kv[cfg['pool_kv_stride'][i][0]] = cfg['pool_kvq_kernel'] + + return dim_mul, head_mul, pool_q, pool_kv, stride_q, stride_kv + + +class Mlp(nn.Module): + + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop_rate=0.0, + ): + super().__init__() + self.drop_rate = drop_rate + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + if self.drop_rate > 0.0: + self.drop = nn.Dropout(drop_rate) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + if self.drop_rate > 0.0: + x = self.drop(x) + x = self.fc2(x) + if self.drop_rate > 0.0: + x = self.drop(x) + return x + + +class Permute(nn.Module): + + def __init__(self, dims): + super().__init__() + self.dims = dims + + def forward(self, x): + return x.permute(*self.dims) + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + """ + Stochastic Depth per sample. + """ + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + mask = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + mask.floor_() # binarize + output = x.div(keep_prob) * mask + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +def round_width(width, multiplier, min_width=1, divisor=1, verbose=False): + if not multiplier: + return width + width *= multiplier + min_width = min_width or divisor + if verbose: + print(f'min width {min_width}') + print(f'width {width} divisor {divisor}') + print(f'other {int(width + divisor / 2) // divisor * divisor}') + + width_out = max(min_width, int(width + divisor / 2) // divisor * divisor) + if width_out < 0.9 * width: + width_out += divisor + return int(width_out) + + +class PatchEmbed(nn.Module): + """ + PatchEmbed. + """ + + def __init__( + self, + dim_in=3, + dim_out=768, + kernel=(7, 7), + stride=(4, 4), + padding=(3, 3), + conv2d=False, + ): + super().__init__() + + if conv2d: + conv_function = nn.Conv2d + else: + conv_function = nn.Conv3d + + self.proj = conv_function( + dim_in, + dim_out, + kernel_size=kernel, + stride=stride, + padding=padding, + ) + + def forward(self, x): + x = self.proj(x) + # B C H W -> B HW C + return x.flatten(2).transpose(1, 2), x.shape + + +def attention_pool(tensor, pool, thw_shape, has_cls_embed=True, norm=None): + if pool is None: + return tensor, thw_shape + tensor_dim = tensor.ndim + if tensor_dim == 4: + pass + elif tensor_dim == 3: + tensor = tensor.unsqueeze(1) + else: + raise NotImplementedError( + f'Unsupported input dimension {tensor.shape}') + + if has_cls_embed: + cls_tok, tensor = tensor[:, :, :1, :], tensor[:, :, 1:, :] + + B, N, L, C = tensor.shape + T, H, W = thw_shape + tensor = ( + tensor.reshape(B * N, T, H, W, C).permute(0, 4, 1, 2, 3).contiguous()) + + tensor = pool(tensor) + + thw_shape = [tensor.shape[2], tensor.shape[3], tensor.shape[4]] + L_pooled = tensor.shape[2] * tensor.shape[3] * tensor.shape[4] + tensor = tensor.reshape(B, N, C, L_pooled).transpose(2, 3) + if has_cls_embed: + tensor = torch.cat((cls_tok, tensor), dim=2) + if norm is not None: + tensor = norm(tensor) + # Assert tensor_dim in [3, 4] + if tensor_dim == 4: + pass + else: # tensor_dim == 3: + tensor = tensor.squeeze(1) + return tensor, thw_shape + + +def get_rel_pos(rel_pos, d): + if isinstance(d, int): + ori_d = rel_pos.shape[0] + if ori_d == d: + return rel_pos + else: + # Interpolate rel pos. + new_pos_embed = F.interpolate( + rel_pos.reshape(1, ori_d, -1).permute(0, 2, 1), + size=d, + mode='linear', + ) + + return new_pos_embed.reshape(-1, d).permute(1, 0) + + +def cal_rel_pos_spatial(attn, q, k, has_cls_embed, q_shape, k_shape, rel_pos_h, + rel_pos_w): + """ + Decomposed Spatial Relative Positional Embeddings. + """ + sp_idx = 1 if has_cls_embed else 0 + q_t, q_h, q_w = q_shape + k_t, k_h, k_w = k_shape + dh = int(2 * max(q_h, k_h) - 1) + dw = int(2 * max(q_w, k_w) - 1) + + # Scale up rel pos if shapes for q and k are different. + q_h_ratio = max(k_h / q_h, 1.0) + k_h_ratio = max(q_h / k_h, 1.0) + dist_h = ( + torch.arange(q_h)[:, None] * q_h_ratio + - torch.arange(k_h)[None, :] * k_h_ratio) + dist_h += (k_h - 1) * k_h_ratio + q_w_ratio = max(k_w / q_w, 1.0) + k_w_ratio = max(q_w / k_w, 1.0) + dist_w = ( + torch.arange(q_w)[:, None] * q_w_ratio + - torch.arange(k_w)[None, :] * k_w_ratio) + dist_w += (k_w - 1) * k_w_ratio + + # Intepolate rel pos if needed. + rel_pos_h = get_rel_pos(rel_pos_h, dh) + rel_pos_w = get_rel_pos(rel_pos_w, dw) + Rh = rel_pos_h[dist_h.long()] + Rw = rel_pos_w[dist_w.long()] + + B, n_head, q_N, dim = q.shape + + r_q = q[:, :, sp_idx:].reshape(B, n_head, q_t, q_h, q_w, dim) + rel_h_q = torch.einsum('bythwc,hkc->bythwk', r_q, + Rh) # [B, H, q_t, qh, qw, k_h] + rel_w_q = torch.einsum('bythwc,wkc->bythwk', r_q, + Rw) # [B, H, q_t, qh, qw, k_w] + + attn[:, :, sp_idx:, sp_idx:] = ( + attn[:, :, sp_idx:, sp_idx:].view(B, -1, q_t, q_h, q_w, k_t, k_h, k_w) + + rel_h_q[:, :, :, :, :, None, :, None] + + rel_w_q[:, :, :, :, :, None, None, :]).view(B, -1, q_t * q_h * q_w, + k_t * k_h * k_w) + + return attn + + +def cal_rel_pos_temporal(attn, q, has_cls_embed, q_shape, k_shape, rel_pos_t): + """ + Temporal Relative Positional Embeddings. + """ + sp_idx = 1 if has_cls_embed else 0 + q_t, q_h, q_w = q_shape + k_t, k_h, k_w = k_shape + dt = int(2 * max(q_t, k_t) - 1) + # Intepolate rel pos if needed. + rel_pos_t = get_rel_pos(rel_pos_t, dt) + + # Scale up rel pos if shapes for q and k are different. + q_t_ratio = max(k_t / q_t, 1.0) + k_t_ratio = max(q_t / k_t, 1.0) + dist_t = ( + torch.arange(q_t)[:, None] * q_t_ratio + - torch.arange(k_t)[None, :] * k_t_ratio) + dist_t += (k_t - 1) * k_t_ratio + Rt = rel_pos_t[dist_t.long()] + + B, n_head, q_N, dim = q.shape + + r_q = q[:, :, sp_idx:].reshape(B, n_head, q_t, q_h, q_w, dim) + # [B, H, q_t, q_h, q_w, dim] -> [q_t, B, H, q_h, q_w, dim] -> [q_t, B*H*q_h*q_w, dim] + r_q = r_q.permute(2, 0, 1, 3, 4, 5).reshape(q_t, B * n_head * q_h * q_w, + dim) + + # [q_t, B*H*q_h*q_w, dim] * [q_t, dim, k_t] = [q_t, B*H*q_h*q_w, k_t] -> [B*H*q_h*q_w, q_t, k_t] + rel = torch.matmul(r_q, Rt.transpose(1, 2)).transpose(0, 1) + # [B*H*q_h*q_w, q_t, k_t] -> [B, H, q_t, q_h, q_w, k_t] + rel = rel.view(B, n_head, q_h, q_w, q_t, k_t).permute(0, 1, 4, 2, 3, 5) + + attn[:, :, sp_idx:, sp_idx:] = ( + attn[:, :, sp_idx:, sp_idx:].view(B, -1, q_t, q_h, q_w, k_t, k_h, k_w) + + rel[:, :, :, :, :, :, None, None]).view(B, -1, q_t * q_h * q_w, + k_t * k_h * k_w) + + return attn + + +class MultiScaleAttention(nn.Module): + + def __init__( + self, + dim, + dim_out, + input_size, + num_heads=8, + qkv_bias=False, + drop_rate=0.0, + kernel_q=(1, 1, 1), + kernel_kv=(1, 1, 1), + stride_q=(1, 1, 1), + stride_kv=(1, 1, 1), + norm_layer=nn.LayerNorm, + has_cls_embed=True, + # Options include `conv`, `avg`, and `max`. + mode='conv', + # If True, perform pool before projection. + pool_first=False, + rel_pos_spatial=False, + rel_pos_temporal=False, + rel_pos_zero_init=False, + residual_pooling=True, + separate_qkv=False, + ): + super().__init__() + self.pool_first = pool_first + self.separate_qkv = separate_qkv + self.drop_rate = drop_rate + self.num_heads = num_heads + self.dim_out = dim_out + head_dim = dim_out // num_heads + self.scale = head_dim**-0.5 + self.has_cls_embed = has_cls_embed + padding_q = [int(q // 2) for q in kernel_q] + padding_kv = [int(kv // 2) for kv in kernel_kv] + + if pool_first or separate_qkv: + self.q = nn.Linear(dim, dim_out, bias=qkv_bias) + self.k = nn.Linear(dim, dim_out, bias=qkv_bias) + self.v = nn.Linear(dim, dim_out, bias=qkv_bias) + else: + self.qkv = nn.Linear(dim, dim_out * 3, bias=qkv_bias) + + self.proj = nn.Linear(dim_out, dim_out) + if drop_rate > 0.0: + self.proj_drop = nn.Dropout(drop_rate) + + # Skip pooling with kernel and stride size of (1, 1, 1). + if np.prod(kernel_q) == 1 and np.prod(stride_q) == 1: + kernel_q = () + if np.prod(kernel_kv) == 1 and np.prod(stride_kv) == 1: + kernel_kv = () + self.mode = mode + + if mode in ('avg', 'max'): + pool_op = nn.MaxPool3d if mode == 'max' else nn.AvgPool3d + self.pool_q = ( + pool_op(kernel_q, stride_q, padding_q, ceil_mode=False) + if len(kernel_q) > 0 else None) + self.pool_k = ( + pool_op(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 else None) + self.pool_v = ( + pool_op(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 else None) + elif mode == 'conv' or mode == 'conv_unshared': + if pool_first: + dim_conv = dim // num_heads if mode == 'conv' else dim + else: + dim_conv = dim_out // num_heads if mode == 'conv' else dim_out + self.pool_q = ( + nn.Conv3d( + dim_conv, + dim_conv, + kernel_q, + stride=stride_q, + padding=padding_q, + groups=dim_conv, + bias=False, + ) if len(kernel_q) > 0 else None) + self.norm_q = norm_layer(dim_conv) if len(kernel_q) > 0 else None + self.pool_k = ( + nn.Conv3d( + dim_conv, + dim_conv, + kernel_kv, + stride=stride_kv, + padding=padding_kv, + groups=dim_conv, + bias=False, + ) if len(kernel_kv) > 0 else None) + self.norm_k = norm_layer(dim_conv) if len(kernel_kv) > 0 else None + self.pool_v = ( + nn.Conv3d( + dim_conv, + dim_conv, + kernel_kv, + stride=stride_kv, + padding=padding_kv, + groups=dim_conv, + bias=False, + ) if len(kernel_kv) > 0 else None) + self.norm_v = norm_layer(dim_conv) if len(kernel_kv) > 0 else None + else: + raise NotImplementedError(f'Unsupported model {mode}') + + self.rel_pos_spatial = rel_pos_spatial + self.rel_pos_temporal = rel_pos_temporal + if self.rel_pos_spatial: + assert input_size[1] == input_size[2] + size = input_size[1] + q_size = size // stride_q[1] if len(stride_q) > 0 else size + kv_size = size // stride_kv[1] if len(stride_kv) > 0 else size + rel_sp_dim = 2 * max(q_size, kv_size) - 1 + + self.rel_pos_h = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(rel_sp_dim, head_dim)) + if not rel_pos_zero_init: + trunc_normal_(self.rel_pos_h, std=0.02) + trunc_normal_(self.rel_pos_w, std=0.02) + if self.rel_pos_temporal: + self.rel_pos_t = nn.Parameter( + torch.zeros(2 * input_size[0] - 1, head_dim)) + # if not rel_pos_zero_init: + # trunc_normal_(self.rel_pos_t, std=0.02) + + self.residual_pooling = residual_pooling + + def forward(self, x, thw_shape): + B, N, _ = x.shape + + if self.pool_first: + if self.mode == 'conv_unshared': + fold_dim = 1 + else: + fold_dim = self.num_heads + x = x.reshape(B, N, fold_dim, -1).permute(0, 2, 1, 3) + q = k = v = x + else: + assert self.mode != 'conv_unshared' + if not self.separate_qkv: + qkv = ( + self.qkv(x).reshape(B, N, 3, self.num_heads, + -1).permute(2, 0, 3, 1, 4)) + q, k, v = qkv[0], qkv[1], qkv[2] + else: + q = k = v = x + q = ( + self.q(q).reshape(B, N, self.num_heads, + -1).permute(0, 2, 1, 3)) + k = ( + self.k(k).reshape(B, N, self.num_heads, + -1).permute(0, 2, 1, 3)) + v = ( + self.v(v).reshape(B, N, self.num_heads, + -1).permute(0, 2, 1, 3)) + + q, q_shape = attention_pool( + q, + self.pool_q, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_q if hasattr(self, 'norm_q') else None, + ) + k, k_shape = attention_pool( + k, + self.pool_k, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_k if hasattr(self, 'norm_k') else None, + ) + v, v_shape = attention_pool( + v, + self.pool_v, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_v if hasattr(self, 'norm_v') else None, + ) + + if self.pool_first: + q_N = ( + np.prod(q_shape) + + 1 if self.has_cls_embed else np.prod(q_shape)) + k_N = ( + np.prod(k_shape) + + 1 if self.has_cls_embed else np.prod(k_shape)) + v_N = ( + np.prod(v_shape) + + 1 if self.has_cls_embed else np.prod(v_shape)) + + q = q.permute(0, 2, 1, 3).reshape(B, q_N, -1) + q = ( + self.q(q).reshape(B, q_N, self.num_heads, + -1).permute(0, 2, 1, 3)) + + v = v.permute(0, 2, 1, 3).reshape(B, v_N, -1) + v = ( + self.v(v).reshape(B, v_N, self.num_heads, + -1).permute(0, 2, 1, 3)) + + k = k.permute(0, 2, 1, 3).reshape(B, k_N, -1) + k = ( + self.k(k).reshape(B, k_N, self.num_heads, + -1).permute(0, 2, 1, 3)) + + N = q.shape[2] + attn = (q * self.scale) @ k.transpose(-2, -1) + if self.rel_pos_spatial: + attn = cal_rel_pos_spatial( + attn, + q, + k, + self.has_cls_embed, + q_shape, + k_shape, + self.rel_pos_h, + self.rel_pos_w, + ) + + if self.rel_pos_temporal: + attn = cal_rel_pos_temporal( + attn, + q, + self.has_cls_embed, + q_shape, + k_shape, + self.rel_pos_t, + ) + attn = attn.softmax(dim=-1) + + x = attn @ v + + if self.residual_pooling: + # Minor Difference + if self.has_cls_embed: + x[:, :, 1:, :] += q[:, :, 1:, :] + else: + x = x + q + + x = x.transpose(1, 2).reshape(B, -1, self.dim_out) + x = self.proj(x) + + if self.drop_rate > 0.0: + x = self.proj_drop(x) + return x, q_shape + + +class MultiScaleBlock(nn.Module): + + def __init__( + self, + dim, + dim_out, + num_heads, + input_size, + mlp_ratio=4.0, + qkv_bias=False, + qk_scale=None, + drop_rate=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + up_rate=None, + kernel_q=(1, 1, 1), + kernel_kv=(1, 1, 1), + stride_q=(1, 1, 1), + stride_kv=(1, 1, 1), + mode='conv', + has_cls_embed=True, + pool_first=False, + rel_pos_spatial=False, + rel_pos_temporal=False, + rel_pos_zero_init=False, + residual_pooling=True, + dim_mul_in_att=False, + separate_qkv=False, + use_grad_checkpoint=False, + ): + super().__init__() + self.dim = dim + self.dim_out = dim_out + self.norm1 = norm_layer(dim) + self.dim_mul_in_att = dim_mul_in_att + kernel_skip = [s + 1 if s > 1 else s for s in stride_q] + stride_skip = stride_q + padding_skip = [int(skip // 2) for skip in kernel_skip] + att_dim = dim_out if dim_mul_in_att else dim + + self.use_grad_checkpoint = use_grad_checkpoint + + self.attn = MultiScaleAttention( + dim, + att_dim, + num_heads=num_heads, + input_size=input_size, + qkv_bias=qkv_bias, + drop_rate=drop_rate, + kernel_q=kernel_q, + kernel_kv=kernel_kv, + stride_q=stride_q, + stride_kv=stride_kv, + norm_layer=norm_layer, + has_cls_embed=has_cls_embed, + mode=mode, + pool_first=pool_first, + rel_pos_spatial=rel_pos_spatial, + rel_pos_temporal=rel_pos_temporal, + rel_pos_zero_init=rel_pos_zero_init, + residual_pooling=residual_pooling, + separate_qkv=separate_qkv, + ) + self.drop_path = ( + DropPath(drop_path) if drop_path > 0.0 else nn.Identity()) + self.norm2 = norm_layer(att_dim) + mlp_hidden_dim = int(att_dim * mlp_ratio) + self.has_cls_embed = has_cls_embed + # TODO: check the use case for up_rate, and merge the following lines + if up_rate is not None and up_rate > 1: + mlp_dim_out = dim * up_rate + else: + mlp_dim_out = dim_out + self.mlp = Mlp( + in_features=att_dim, + hidden_features=mlp_hidden_dim, + out_features=mlp_dim_out, + act_layer=act_layer, + drop_rate=drop_rate, + ) + if dim != dim_out: + self.proj = nn.Linear(dim, dim_out) + + self.pool_skip = ( + nn.MaxPool3d( + kernel_skip, stride_skip, padding_skip, ceil_mode=False) + if len(kernel_skip) > 0 else None) + + def forward(self, x, thw_shape): + x_norm = self.norm1(x) + if self.use_grad_checkpoint: + x_block, thw_shape_new = checkpoint.checkpoint( + self.attn, x_norm, thw_shape) + else: + x_block, thw_shape_new = self.attn(x_norm, thw_shape) + + if self.dim_mul_in_att and self.dim != self.dim_out: + x = self.proj(x_norm) + x_res, _ = attention_pool( + x, self.pool_skip, thw_shape, has_cls_embed=self.has_cls_embed) + x = x_res + self.drop_path(x_block) + x_norm = self.norm2(x) + if self.use_grad_checkpoint: + x_mlp = checkpoint.checkpoint(self.mlp, x_norm) + else: + x_mlp = self.mlp(x_norm) + + if not self.dim_mul_in_att and self.dim != self.dim_out: + x = self.proj(x_norm) + x = x + self.drop_path(x_mlp) + return x, thw_shape_new + + +class MViTv2(nn.Module): + """ + Improved Multiscale Vision Transformers for Classification and Detection + Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik, + Christoph Feichtenhofer* + https://arxiv.org/abs/2112.01526 + Multiscale Vision Transformers + Haoqi Fan*, Bo Xiong*, Karttikeya Mangalam*, Yanghao Li*, Zhicheng Yan, Jitendra Malik, + Christoph Feichtenhofer* + https://arxiv.org/abs/2104.11227 + """ + + def __init__( + self, + img_size=224, + embed_dim=96, + num_classes=1000, + num_frames=4, + num_heads=1, + depth=24, + patch_kernel=[3, 7, 7], + patch_stride=[2, 4, 4], + patch_padding=[1, 3, 3], + config=None, + dropout_rate=0., + drop_path_rate=0., + mlp_ratio=4., + qkv_bias=True, + mode='conv', + cls_embed_on=True, + use_abs_pos=False, + rel_pos_spatial=True, + rel_pos_temporal=True, + rel_pos_zero_init=False, + residual_pooling=True, + dim_mul_in_att=True, + pool_first=False, + zero_decay_pos_cls=False, + separate_qkv=False, + norm_stem=False, + sep_pos_embed=False, + use_grad_checkpoint=True, + ): + super().__init__() + # Prepare input. + in_chans = 3 + self.img_size = img_size + # Prepare output. + self.num_classes = num_classes + self.embed_dim = embed_dim + # MViT params. + self.num_heads = num_heads + self.depth = depth + self.cls_embed_on = cls_embed_on + self.use_abs_pos = use_abs_pos + self.zero_decay_pos_cls = zero_decay_pos_cls + self.use_grad_checkpoint = use_grad_checkpoint + self.sep_pos_embed = sep_pos_embed + self.drop_rate = dropout_rate + + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + if use_grad_checkpoint: + self.patch_embed = checkpoint_wrapper( + PatchEmbed( + dim_in=in_chans, + dim_out=embed_dim, + kernel=patch_kernel, + stride=patch_stride, + padding=patch_padding, + )) + else: + self.patch_embed = PatchEmbed( + dim_in=in_chans, + dim_out=embed_dim, + kernel=patch_kernel, + stride=patch_stride, + padding=patch_padding, + ) + + patch_dims = [ + num_frames // patch_stride[0], + img_size // patch_stride[1], + img_size // patch_stride[2], + ] + num_patches = np.prod(patch_dims) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + + if self.cls_embed_on: + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + pos_embed_dim = num_patches + 1 + else: + pos_embed_dim = num_patches + + if self.use_abs_pos: + self.pos_embed = nn.Parameter( + torch.zeros(1, pos_embed_dim, embed_dim)) + + if self.use_abs_pos: + if self.sep_pos_embed: + self.pos_embed_spatial = nn.Parameter( + torch.zeros(1, self.patch_dims[1] * self.patch_dims[2], + embed_dim)) + self.pos_embed_temporal = nn.Parameter( + torch.zeros(1, self.patch_dims[0], embed_dim)) + if self.cls_embed_on: + self.pos_embed_class = nn.Parameter( + torch.zeros(1, 1, embed_dim)) + else: + self.pos_embed = nn.Parameter( + torch.zeros(1, pos_embed_dim, embed_dim)) + + assert config is not None + # MViT backbone configs + dim_mul, head_mul, pool_q, pool_kv, stride_q, stride_kv = _prepare_mvit_configs( + config) + input_size = patch_dims + + self.norm_stem = norm_layer(embed_dim) if norm_stem else None + + self.blocks = nn.ModuleList() + for i in range(depth): + num_heads = round_width(num_heads, head_mul[i]) + if dim_mul_in_att: + dim_out = round_width( + embed_dim, + dim_mul[i], + divisor=round_width(num_heads, head_mul[i]), + ) + else: + dim_out = round_width( + embed_dim, + dim_mul[i + 1], + divisor=round_width(num_heads, head_mul[i + 1]), + ) + attention_block = MultiScaleBlock( + dim=embed_dim, + dim_out=dim_out, + num_heads=num_heads, + input_size=input_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_rate=self.drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + kernel_q=pool_q[i] if len(pool_q) > i else [], + kernel_kv=pool_kv[i] if len(pool_kv) > i else [], + stride_q=stride_q[i] if len(stride_q) > i else [], + stride_kv=stride_kv[i] if len(stride_kv) > i else [], + mode=mode, + has_cls_embed=self.cls_embed_on, + pool_first=pool_first, + rel_pos_spatial=rel_pos_spatial, + rel_pos_temporal=rel_pos_temporal, + rel_pos_zero_init=rel_pos_zero_init, + residual_pooling=residual_pooling, + dim_mul_in_att=dim_mul_in_att, + separate_qkv=separate_qkv, + use_grad_checkpoint=False) + if use_grad_checkpoint: + attention_block = checkpoint_wrapper( + attention_block, offload_to_cpu=False) + self.blocks.append(attention_block) + + if len(stride_q[i]) > 0: + input_size = [ + size // stride + for size, stride in zip(input_size, stride_q[i]) + ] + embed_dim = dim_out + + self.norm = norm_layer(embed_dim) + + self.head = nn.Identity() + + if self.use_abs_pos: + if self.sep_pos_embed: + trunc_normal_(self.pos_embed_spatial, std=0.02) + trunc_normal_(self.pos_embed_temporal, std=0.02) + if self.cls_embed_on: + trunc_normal_(self.pos_embed_class, std=0.02) + else: + trunc_normal_(self.pos_embed, std=0.02) + if self.cls_embed_on: + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + names = [] + if self.zero_decay_pos_cls: + if self.use_abs_pos: + if self.sep_pos_embed: + names.extend([ + 'pos_embed_spatial', + 'pos_embed_temporal', + 'pos_embed_class', + ]) + else: + names.append(['pos_embed']) + if self.rel_pos_spatial: + names.extend(['rel_pos_h', 'rel_pos_w', 'rel_pos_hw']) + if self.rel_pos_temporal: + names.extend(['rel_pos_t']) + if self.cls_embed_on: + names.append('cls_token') + + return names + + def _get_pos_embed(self, pos_embed, bcthw): + t, h, w = bcthw[-3], bcthw[-2], bcthw[-1] + if self.cls_embed_on: + cls_pos_embed = pos_embed[:, 0:1, :] + pos_embed = pos_embed[:, 1:] + txy_num = pos_embed.shape[1] + p_t, p_h, p_w = self.patch_dims + assert p_t * p_h * p_w == txy_num + + if (p_t, p_h, p_w) != (t, h, w): + new_pos_embed = F.interpolate( + pos_embed[:, :, :].reshape(1, p_t, p_h, p_w, + -1).permute(0, 4, 1, 2, 3), + size=(t, h, w), + mode='trilinear', + ) + pos_embed = new_pos_embed.reshape(1, -1, + t * h * w).permute(0, 2, 1) + + if self.cls_embed_on: + pos_embed = torch.cat((cls_pos_embed, pos_embed), dim=1) + + return pos_embed + + def forward_features(self, x): + x = x.permute(0, 2, 1, 3, 4) + x, bcthw = self.patch_embed(x) + + T, H, W = bcthw[-3], bcthw[-2], bcthw[-1] + B, N, C = x.shape + + if self.cls_embed_on: + cls_tokens = self.cls_token.expand( + B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + if self.use_abs_pos: + if self.sep_pos_embed: + pos_embed = self.pos_embed_spatial.repeat( + 1, self.patch_dims[0], 1) + torch.repeat_interleave( + self.pos_embed_temporal, + self.patch_dims[1] * self.patch_dims[2], + dim=1) + if self.cls_embed_on: + pos_embed = torch.cat([self.pos_embed_class, pos_embed], 1) + pos_embed = self._get_pos_embed(pos_embed, bcthw) + x = x + pos_embed + else: + pos_embed = self._get_pos_embed(self.pos_embed, bcthw) + x = x + pos_embed + + if self.drop_rate: + x = self.pos_drop(x) + + if self.norm_stem: + x = self.norm_stem(x) + + thw = [T, H, W] + for blk in self.blocks: + x, thw = blk(x, thw) + + x = self.norm(x) + return x + + def forward(self, x): + x = self.forward_features(x) + x = self.head(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug/predictor.py b/runtime/Lib/site_packages/models/multi_modal/mplug/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..6375d1d7eeac4edd98ed7b60df932bf0db59a40c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug/predictor.py @@ -0,0 +1,551 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import torch +import torch.nn.functional as F + + +def build_predictor(args, tokenizer, symbols, model, logger=None): + scorer = None + + translator = TextGenerator( + args, model, tokenizer, symbols, global_scorer=scorer, logger=logger) + return translator + + +class TextGenerator(object): + """ + Uses a model to translate a batch of sentences. + + + Args: + model (:obj:`onmt.modules.NMTModel`): + NMT model to use for translation + fields (dict of Fields): data fields + beam_size (int): size of beam to use + n_best (int): number of translations produced + max_length (int): maximum length output to produce + global_scores (:obj:`GlobalScorer`): + object to rescore final translations + copy_attn (bool): use copy attention during translation + cuda (bool): use cuda + beam_trace (bool): trace beam search for debugging + logger(logging.Logger): logger. + """ + + def __init__(self, + args, + model, + vocab=None, + symbols=None, + global_scorer=None, + logger=None, + dump_beam=''): + self.alpha = 0.6 + + self.logger = logger + self.cuda = (torch.cuda.device_count() > 0) + + self.args = args + self.model = model + + self.vocab = vocab + self.symbols = symbols + self.start_token = 101 # ['[PAD]'] + self.end_token = 102 # ['[PAD]'] + + self.global_scorer = global_scorer + self.beam_size = args.beam_size + self.min_length = args.min_length + self.max_length = args.max_length + + self.dump_beam = dump_beam + + # for debugging + self.beam_trace = self.dump_beam != '' + self.beam_accum = None + + if self.beam_trace: + self.beam_accum = { + 'predicted_ids': [], + 'beam_parent_ids': [], + 'scores': [], + 'log_probs': [] + } + + def _build_target_tokens(self, pred): + tokens = [] + for tok in pred: + tok = int(tok) + tokens.append(tok) + if tokens[-1] == self.end_token: + tokens = tokens[:-1] + break + tokens = [t for t in tokens if t < len(self.vocab)] + tokens = self.vocab.DecodeIds(tokens).split(' ') + return tokens + + def translate_batch(self, encoder_inputs, do_sample=False, out_size=1): + """ + Translate a batch of sentences. + + Mostly a wrapper around :obj:`Beam`. + + Args: + batch (:obj:`Batch`): a batch from a dataset object + data (:obj:`Dataset`): the dataset object + fast (bool): enables fast beam search (may not support all features) + + Todo: + Shouldn't need the original dataset. + """ + if do_sample: + return self._fast_translate_batch( + encoder_inputs, + self.max_length, + min_length=self.min_length, + do_sample=do_sample, + out_size=out_size) + else: + with torch.no_grad(): + return self._fast_translate_batch( + encoder_inputs, + self.max_length, + min_length=self.min_length, + do_sample=do_sample, + out_size=out_size) + + def translate_batch_scst(self, + encoder_inputs, + do_sample=False, + out_size=1): + return self._fast_translate_batch( + encoder_inputs, + self.max_length, + min_length=self.min_length, + do_sample=do_sample, + out_size=out_size) + + def _fast_translate_batch(self, + encoder_inputs, + max_length, + min_length=0, + do_sample=False, + out_size=1): + + assert not self.dump_beam + if do_sample: + beam_size = 1 + else: + beam_size = self.beam_size + if len(encoder_inputs) == 3: + src_features, padding_mask, input_ids = encoder_inputs + elif len(encoder_inputs) == 2: + src_features, padding_mask = encoder_inputs + input_ids = None + + device = src_features.device + + # Tile states and memory beam_size times. + batch_size = src_features.size(0) + src_features = tile(src_features, beam_size, dim=0) + attention_mask = tile(padding_mask, beam_size, dim=0) + + batch_offset = torch.arange( + batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, + batch_size * beam_size, + step=beam_size, + dtype=torch.long, + device=device) + if input_ids is not None: + alive_seq = tile(input_ids, beam_size, dim=0) + else: + alive_seq = torch.full([batch_size * beam_size, 1], + self.start_token, + dtype=torch.long, + device=device) + + # Give full probability to the first beam on the first step. + topk_log_probs = ( + torch.tensor( + [0.0] + [float('-inf')] * (beam_size - 1), + device=device).repeat(batch_size)) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results['predictions'] = [[] for _ in range(batch_size)] # noqa: F812 + results['scores'] = [[] for _ in range(batch_size)] # noqa: F812 + results['gold_score'] = [0] * batch_size + results['batch'] = [] + + for step in range(max_length): + dec_feat_seq = self.model( + alive_seq, + encoder_hidden_states=src_features, + encoder_attention_mask=attention_mask, + return_dict=True, + reduction='none') + + dec_feat_seq = dec_feat_seq.logits[:, -1, :] + vocab_size = dec_feat_seq.size(-1) + log_probs = torch.log( + torch.softmax(dec_feat_seq.view(-1, vocab_size), dim=-1)) + if step < min_length: + log_probs[:, self.end_token] = -1e20 + alpha = self.alpha + if do_sample: + length_penalty = 1.0 + else: + length_penalty = ((5.0 + (step + 1)) / 6.0)**alpha + + if do_sample: + _scores = log_probs / self.args.temperature + _scores = top_k_top_p_filtering( + _scores, + top_k=self.args.top_k, + top_p=self.args.top_p, + min_tokens_to_keep=1 + ) # (batch_size * num_beams, vocab_size) + # Sample 2 next words for each beam + # (so we have some spare tokens and match output of greedy beam search) + topk_ids = torch.multinomial( + F.softmax(_scores, dim=-1), + num_samples=1) # (batch_size * num_beams, 2) + # Compute next scores + _scores = F.log_softmax( + _scores, dim=1) # (batch_size * num_beams, vocab_size) + + _scores += topk_log_probs.view(-1).unsqueeze(1) + topk_scores = torch.gather( + _scores, -1, topk_ids) # (batch_size * num_beams, 2) + # log_probs += # (batch_size * num_beams, 2) + # Match shape of greedy beam search + topk_ids = topk_ids.view( + -1, beam_size) # (batch_size, 2 * num_beams) + topk_scores = topk_scores.view( + -1, beam_size) # (batch_size, 2 * num_beams) + else: + log_probs += topk_log_probs.view(-1).unsqueeze(1) + curr_scores = log_probs / length_penalty + + curr_scores = curr_scores.reshape(-1, beam_size * vocab_size) + topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1) + topk_log_probs = topk_scores * length_penalty + + # Resolve beam origin and true word ids. + # topk_beam_index = topk_ids.div(vocab_size) + topk_beam_index = torch.div( + topk_ids, vocab_size, rounding_mode='floor') + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = ( + topk_beam_index + + beam_offset[:topk_beam_index.size(0)].unsqueeze(1)) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat([ + alive_seq.index_select(0, select_indices), + topk_ids.view(-1, 1) + ], -1) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(1) # self.end_token) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) # self.end_token) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(1) # self.end_token) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append( + (topk_scores[i, j], predictions[i, j, 0:])) + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted( + hypotheses[b], key=lambda x: x[0], reverse=True) + + for each in best_hyp[:beam_size]: + score, pred = each + results['scores'][b].append(score) + results['predictions'][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished) \ + .view(-1, alive_seq.size(-1)) + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + attention_mask = attention_mask.index_select(0, select_indices) + pred_ids = [] + scores = [] + # print (pred_ids, scores) + for each in results['scores']: + scores.append(each[:out_size]) + for each in results['predictions']: + pred_ids.append(each[:out_size]) + return pred_ids, scores + + def _generate_no_beam_search( + self, + input_ids, + cur_len, + max_length, + do_sample, + temperature, + top_k, + top_p, + repetition_penalty, + pad_token_id, + eos_token_ids, + batch_size, + ): + """ Generate sequences for each example without beam search (num_beams == 1). + All returned sequence are generated independantly. + """ + assert self.num_keep_best == 1, 'cannot generate >1 sentences in greedy search' + # current position / max lengths / length of generated sentences / unfinished sentences + unfinished_sents = [] + cur_unfinished = input_ids.new(batch_size).fill_(1) + + # log of scores for each sentence in the batch + logprobs = [] + + past = None + + while cur_len < max_length: + model_inputs = self.prepare_inputs_for_generation( + input_ids, past=past) + outputs = self(**model_inputs) + if cur_len == 1: + token_len = 2 + self.od_labels_len + next_token_idx = 1 + else: + assert cur_len > 1 + if not self._do_output_past(outputs): + token_len = cur_len + 1 + self.od_labels_len + next_token_idx = cur_len + else: + token_len = 2 + next_token_idx = 1 + assert outputs[0].shape[1] == token_len + + next_token_logits = outputs[0][:, next_token_idx, :] + + # if model has past, then set the past variable to speed up decoding + if self._do_output_past(outputs): + past = outputs[1] + + # repetition penalty from CTRL paper (https://arxiv.org/abs/1909.05858) + if repetition_penalty != 1.0: + for i in range(batch_size): + for previous_token in set(input_ids[i].tolist()): + # if score < 0 then repetition penalty has to multiplied + # to reduce the previous token probability + if next_token_logits[i, previous_token] < 0: + next_token_logits[ + i, previous_token] *= repetition_penalty + else: + next_token_logits[ + i, previous_token] /= repetition_penalty + + if do_sample: + # Temperature (higher temperature => more likely to sample low probability tokens) + if temperature != 1.0: + next_token_logits = next_token_logits / temperature + # Top-p/top-k filtering + next_token_logits = top_k_top_p_filtering( + next_token_logits, top_k=top_k, top_p=top_p) + # Sample + next_token = torch.multinomial( + F.softmax(next_token_logits, dim=-1), + num_samples=1).squeeze(1) + else: + # Greedy decoding + next_token = torch.argmax(next_token_logits, dim=-1) + + # Compute scores + _scores = F.log_softmax( + next_token_logits, dim=-1) # (batch_size, vocab_size) + _scores = torch.gather(_scores, -1, + next_token.unsqueeze(-1)) # (batch_size, 1) + logprobs.append(_scores) # (batch_size, 1) + unfinished_sents.append(cur_unfinished) + + # update generations and finished sentences + tokens_to_add = next_token * cur_unfinished + pad_token_id * ( + 1 - cur_unfinished) + input_ids = torch.cat( + [input_ids, tokens_to_add.unsqueeze(-1)], dim=-1) + + for eos_token_id in eos_token_ids: + cur_unfinished = cur_unfinished.mul( + tokens_to_add.ne(eos_token_id).long()) + cur_len = cur_len + 1 + + # stop when there is a in each sentence, or if we exceed the maximul length + if cur_unfinished.max() == 0: + break + + # add eos_token_ids to unfinished sentences + if cur_len == max_length: + input_ids[:, -1].masked_fill_( + cur_unfinished.to(dtype=torch.bool), eos_token_ids[0]) + + logprobs = torch.cat(logprobs, dim=1) + unfinished_sents = torch.stack(unfinished_sents, dim=1).float() + sum_logprobs = (logprobs * unfinished_sents).sum(dim=1) + # return logprobs to keep consistent with beam search output + logprobs = sum_logprobs / unfinished_sents.sum(dim=1) + + # pad to the same length, otherwise DataParallel will give error + pad_len = max_length - input_ids.shape[1] + if pad_len > 0: + padding_ids = input_ids.new(batch_size, + pad_len).fill_(pad_token_id) + input_ids = torch.cat([input_ids, padding_ids], dim=1) + + # (batch_size, n_best, max_len), (batch_size, n_best) + return input_ids.unsqueeze(1), logprobs.unsqueeze(1) + + +def top_k_top_p_filtering(logits, + top_k=10, + top_p=1.0, + filter_value=-float('Inf'), + min_tokens_to_keep=1): + + if top_k > 0: + top_k = min(max(top_k, min_tokens_to_keep), + logits.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = filter_value + return logits + + +class Translation(object): + """ + Container for a translated sentence. + + Attributes: + src (`LongTensor`): src word ids + src_raw ([str]): raw src words + + pred_sents ([[str]]): words from the n-best translations + pred_scores ([[float]]): log-probs of n-best translations + attns ([`FloatTensor`]) : attention dist for each translation + gold_sent ([str]): words from gold translation + gold_score ([float]): log-prob of gold translation + + """ + + def __init__(self, fname, src, src_raw, pred_sents, attn, pred_scores, + tgt_sent, gold_score): + self.fname = fname + self.src = src + self.src_raw = src_raw + self.pred_sents = pred_sents + self.attns = attn + self.pred_scores = pred_scores + self.gold_sent = tgt_sent + self.gold_score = gold_score + + def log(self, sent_number): + """ + Log translation. + """ + + output = '\nSENT {}: {}\n'.format(sent_number, self.src_raw) + + best_pred = self.pred_sents[0] + best_score = self.pred_scores[0] + pred_sent = ' '.join(best_pred) + output += 'PRED {}: {}\n'.format(sent_number, pred_sent) + output += 'PRED SCORE: {:.4f}\n'.format(best_score) + + if self.gold_sent is not None: + tgt_sent = ' '.join(self.gold_sent) + output += 'GOLD {}: {}\n'.format(sent_number, tgt_sent) + output += ('GOLD SCORE: {:.4f}\n'.format(self.gold_score)) + if len(self.pred_sents) > 1: + output += '\nBEST HYP:\n' + for score, sent in zip(self.pred_scores, self.pred_sents): + output += '[{:.4f}] {}\n'.format(score, sent) + + return output + + +def tile(x, count, dim=0): + """ + Tiles x on dimension dim count times. + """ + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = x.view(batch, -1) \ + .transpose(0, 1) \ + .repeat(count, 1) \ + .transpose(0, 1) \ + .contiguous() \ + .view(*out_size) + if dim != 0: + x = x.permute(perm).contiguous() + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug_for_all_tasks.py b/runtime/Lib/site_packages/models/multi_modal/mplug_for_all_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..f15b69d2c8e739560c09cb3dd0ba56516f26449d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug_for_all_tasks.py @@ -0,0 +1,149 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp +from typing import Dict, List + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['MPlugForAllTasks', 'HiTeAForAllTasks'] + + +@MODELS.register_module( + Tasks.visual_question_answering, module_name=Models.mplug) +@MODELS.register_module(Tasks.image_captioning, module_name=Models.mplug) +@MODELS.register_module(Tasks.image_text_retrieval, module_name=Models.mplug) +class MPlugForAllTasks(TorchModel): + + def __init__(self, model_dir: str, task=None, *args, **kwargs): + """initialize the mplug model from the `model_dir` path. + Args: + model_dir (str): the model path. + """ + + super().__init__(model_dir, *args, **kwargs) + from modelscope.models.multi_modal.mplug import MPlug + self.model = MPlug.from_pretrained(model_dir, task=task) + self.tokenizer = self.model.tokenizer + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'predictions': Tensor([[1377, 4959, 2785, 6392...])]), + } + """ + + # get task from config file + task = Config.from_file( + osp.join(self.model_dir, ModelFile.CONFIGURATION)).task + + # inference + if not self.training and 'question' in input: + output = self.model(input['image'], input['question'], train=False) + if task == Tasks.image_text_retrieval: + return {OutputKeys.SCORES: output[0].tolist()} + topk_ids, _ = output + pred_string: List[str] = \ + self.tokenizer.decode(topk_ids[0][0], skip_special_tokens=True) + output_key = OutputKeys.CAPTION \ + if task == Tasks.image_captioning else OutputKeys.TEXT + return {output_key: pred_string} + + # train and evaluate + import addict + image = input['image'] + answer = addict.Dict( + input_ids=input['answer_input_ids'], + attention_mask=input['answer_attention_mask']) + if 'index' not in input: + question = addict.Dict( + input_ids=input['question_input_ids'], + attention_mask=input['question_attention_mask']) + output = self.model(image, question, answer, train=self.training) + else: + index = input['index'] + output = self.model(image, answer, index, train=self.training) + if self.training: + return {OutputKeys.LOSS: output} + + # evaluate + topk_ids, _ = output + return {'sequences': [list_tensor[0] for list_tensor in topk_ids]} + + +@MODELS.register_module( + Tasks.video_question_answering, module_name=Models.hitea) +@MODELS.register_module(Tasks.video_captioning, module_name=Models.hitea) +class HiTeAForAllTasks(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the hitea model from the `model_dir` path. + Args: + model_dir (str): the model path. + """ + + super().__init__(model_dir, *args, **kwargs) + from modelscope.models.multi_modal.mplug import HiTeA + self.model = HiTeA.from_pretrained(model_dir) + self.tokenizer = self.model.tokenizer + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'predictions': Tensor([[1377, 4959, 2785, 6392...])]), + } + """ + + # get task from config file + task = Config.from_file( + osp.join(self.model_dir, ModelFile.CONFIGURATION)).task + + # inference + if not self.training and 'question' in input: + output = self.model(input['video'], input['question'], train=False) + topk_ids, _ = output + pred_string: List[str] = \ + self.tokenizer.decode(topk_ids[0][0], skip_special_tokens=True) + output_key = OutputKeys.CAPTION \ + if task == Tasks.video_captioning else OutputKeys.TEXT + return {output_key: pred_string} + + # train and evaluate + import addict + video = input['video'] + answer = addict.Dict( + input_ids=input['answer_input_ids'], + attention_mask=input['answer_attention_mask']) + if 'index' not in input: + question = addict.Dict( + input_ids=input['question_input_ids'], + attention_mask=input['question_attention_mask']) + output = self.model(video, question, answer, train=self.training) + else: + index = input['index'] + output = self.model(video, answer, index, train=self.training) + if self.training: + return {OutputKeys.LOSS: output} + + # evaluate + topk_ids, _ = output + return {'sequences': [list_tensor[0] for list_tensor in topk_ids]} diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug_owl/__init__.py b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..76ccfb5a46ec62108823307069374d4185e96d88 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .configuration_mplug_owl import (MplugOwlConfig, MplugOwlVisionConfig, + MplugOwlVisualAbstractorConfig) +from .modeling_mplug_owl import MplugOwlForConditionalGeneration diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug_owl/configuration_mplug_owl.py b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/configuration_mplug_owl.py new file mode 100644 index 0000000000000000000000000000000000000000..f211a1368510530f5d5efbbc51984c9ffd9b5b78 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/configuration_mplug_owl.py @@ -0,0 +1,261 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MPLUG OWL model configuration """ +import copy +import os +from typing import Union + +from transformers import PretrainedConfig +from transformers.models.auto import CONFIG_MAPPING +from transformers.utils import logging + +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +class MplugOwlVisionConfig(PretrainedConfig): + r""" + Args: + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 32): + The size (resolution) of each patch. + hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-5): + The epsilon used by the layer normalization layers. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + ```""" + + model_type = 'mplug_owl_vision_model' + + def __init__( + self, + hidden_size=1024, + intermediate_size=4096, + projection_dim=768, + num_hidden_layers=24, + num_attention_heads=16, + num_channels=3, + image_size=224, + patch_size=14, + hidden_act='quick_gelu', + layer_norm_eps=1e-6, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=1.0, + use_flash_attn=False, + use_fp32_layernorm=True, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.use_flash_attn = use_flash_attn + self.use_fp32_layernorm = use_fp32_layernorm + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, + os.PathLike], + **kwargs) -> 'PretrainedConfig': + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs) + + # get the vision config dict if we are loading from MplugOwlConfig + if config_dict.get('model_type') == 'mplug_owl': + config_dict = config_dict['vision_config'] + + if 'model_type' in config_dict and hasattr( + cls, + 'model_type') and config_dict['model_type'] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' + ) + + return cls.from_dict(config_dict, **kwargs) + + +class MplugOwlVisualAbstractorConfig(PretrainedConfig): + + model_type = 'MPlugOwlVisualAbstractor' + + def __init__( + self, + hidden_size=1024, + num_hidden_layers=6, + num_attention_heads=16, + intermediate_size=4096, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + layer_norm_eps=1e-6, + encoder_hidden_size=1024, + use_fp32_layernorm=True, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.encoder_hidden_size = encoder_hidden_size + self.use_fp32_layernorm = use_fp32_layernorm + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, + os.PathLike], + **kwargs) -> 'PretrainedConfig': + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs) + + # get the qformer config dict if we are loading from MplugOwlConfig + if config_dict.get('model_type') == 'mplug_owl': + config_dict = config_dict['abstractor_config'] + + if 'model_type' in config_dict and hasattr( + cls, + 'model_type') and config_dict['model_type'] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' + ) + + return cls.from_dict(config_dict, **kwargs) + + +class MplugOwlConfig(PretrainedConfig): + r""" + Args: + vision_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`MplugOwlVisionConfig`]. + qformer_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`MplugOwlVisualAbstractorConfig`]. + text_config (`dict`, *optional*): + Dictionary of configuration options used to initialize any [`PretrainedConfig`]. + num_query_tokens (`int`, *optional*, defaults to 32): + The number of query tokens passed through the Transformer. + + kwargs (*optional*): + Dictionary of keyword arguments. + """ + + model_type = 'mplug_owl' + is_composition = True + + def __init__(self, + task=Tasks.multimodal_dialogue, + vision_config=None, + visual_abstractor_config=None, + text_config=None, + num_query_tokens=64, + **kwargs): + + super().__init__(**kwargs) + self.task = task + if vision_config is None: + vision_config = MplugOwlVisionConfig().to_dict() + logger.info('vision_config is None.') + + if visual_abstractor_config is None: + visual_abstractor_config = {} + logger.info('abstractor_config is None. ') + + if text_config is None: + # we use LLAMA 7b by default + from transformers.models.llama.configuration_llama import \ + LlamaConfig + text_config = LlamaConfig(pad_token_id=2).to_dict() + logger.info('text_config is None.') + + self.vision_config = MplugOwlVisionConfig(**vision_config) + self.visual_abstractor_config = MplugOwlVisualAbstractorConfig( + **visual_abstractor_config) + text_model_type = text_config[ + 'model_type'] if 'model_type' in text_config else 'llama' + self.text_config = CONFIG_MAPPING[text_model_type](**text_config) + + self.tie_word_embeddings = self.text_config.tie_word_embeddings + + self.num_query_tokens = num_query_tokens + self.initializer_factor = 1.0 + self.initializer_range = 0.02 + + @classmethod + def from_vision_abstractor_text_configs( + cls, + vision_config: MplugOwlVisionConfig, + visual_abstractor_config: MplugOwlVisualAbstractorConfig, + text_config: PretrainedConfig, + **kwargs, + ): + r""" + Returns: + [`MplugOwlConfig`]: An instance of a configuration object + """ + + return cls( + vision_config=vision_config.to_dict(), + visual_abstractor_config=visual_abstractor_config.to_dict(), + text_config=text_config.to_dict(), + **kwargs, + ) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output['vision_config'] = self.vision_config.to_dict() + tmp = self.visual_abstractor_config.to_dict() + output['visual_abstractor_config'] = tmp + output['text_config'] = self.text_config.to_dict() + output['model_type'] = self.__class__.model_type + return output diff --git a/runtime/Lib/site_packages/models/multi_modal/mplug_owl/modeling_mplug_owl.py b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/modeling_mplug_owl.py new file mode 100644 index 0000000000000000000000000000000000000000..99b9365a8dfda2fc231079ec0e93d5315c26c852 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/mplug_owl/modeling_mplug_owl.py @@ -0,0 +1,1706 @@ +# Copyright 2021-2023 The Alibaba DAMO mPLUG Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch MPLUG OWL model. """ + +import copy +import logging +import math +import os +import os.path as osp +import random +from dataclasses import dataclass +from io import BytesIO +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN +from transformers.modeling_outputs import ( + BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions) +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.models.auto import AutoModelForCausalLM +from transformers.utils import ModelOutput + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.mplug_owl.configuration_mplug_owl import ( + MplugOwlConfig, MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig) +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['MplugOwlForConditionalGeneration'] + + +@dataclass +class MplugOwlForConditionalGenerationModelOutput(ModelOutput): + """ + Class defining the outputs of [`MPlugOwlForConditionalGeneration`]. + + Args: + loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): + Language modeling loss from the language model. + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head of the language model. + vision_outputs (`BaseModelOutputWithPooling`): + Outputs of the vision encoder. + + language_model_outputs (`CausalLMOutputWithPast`): + Outputs of the language model. + """ + + loss: Optional[Tuple[torch.FloatTensor]] = None + logits: Optional[Tuple[torch.FloatTensor]] = None + vision_outputs: Optional[torch.FloatTensor] = None + language_model_outputs: Optional[Tuple[torch.FloatTensor]] = None + + def to_tuple(self) -> Tuple[Any]: + return tuple( + self[k] if k not in ['vision_outputs', 'language_model_outputs' + ] else getattr(self, k).to_tuple() + for k in self.keys()) + + +# Hack for bloomz +def bloom_forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], + ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, +) -> Union[Tuple[torch.Tensor, ...], + BaseModelOutputWithPastAndCrossAttentions]: + if len(deprecated_arguments) > 0: + raise ValueError(f'Got unexpected arguments: {deprecated_arguments}') + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + if past_key_values is None: + past_key_values = tuple([None] * len(self.h)) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape batch_size x num_heads x N x N + # head_mask has shape n_layer x batch x num_heads x N x N + head_mask = self.get_head_mask(head_mask, self.config.n_layer) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + inputs_embeds = self.word_embeddings_layernorm(inputs_embeds) + + hidden_states = inputs_embeds + + presents = () if use_cache else None + all_self_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + + # Compute alibi tensor: check build_alibi_tensor documentation + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values[0] is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + if attention_mask is None: + attention_mask = torch.ones((batch_size, seq_length_with_past), + device=hidden_states.device) + else: + attention_mask = attention_mask.to(hidden_states.device) + + alibi = self.build_alibi_tensor( + attention_mask, self.num_heads, dtype=hidden_states.dtype) + + causal_mask = self._prepare_attn_mask( + attention_mask, + input_shape=(batch_size, seq_length), + past_key_values_length=past_key_values_length, + ) + + for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + # None for past_key_value + return module( + *inputs, + use_cache=use_cache, + output_attentions=output_attentions) + + return custom_forward + + outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + alibi, + causal_mask, + layer_past, + head_mask[i], + ) + else: + outputs = block( + hidden_states, + layer_past=layer_past, + attention_mask=causal_mask, + head_mask=head_mask[i], + use_cache=use_cache, + output_attentions=output_attentions, + alibi=alibi, + ) + + hidden_states = outputs[0] + if use_cache is True: + presents = presents + (outputs[1], ) + + if output_attentions: + all_self_attentions = all_self_attentions + ( + outputs[2 if use_cache else 1], ) + + # Add last hidden state + hidden_states = self.ln_f(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple( + v for v in + [hidden_states, presents, all_hidden_states, all_self_attentions] + if v is not None) + + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +def get_ltor_masks_and_position_ids_from_embeddings(data): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size()[:2] + + # Attention mask (lower triangular). + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), + device=data.device)).view(att_mask_batch, 1, seq_length, + seq_length) + + # Loss mask. + loss_mask = torch.ones( + data.size()[:2], dtype=torch.float, device=data.device) + + # Position ids. + position_ids = torch.arange( + seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data[..., 0]) + + # Convert attention mask to binary: + attention_mask = (attention_mask < 0.5) + + return attention_mask, loss_mask, position_ids + + +class MplugOwlVisionEmbeddings(nn.Module): + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.cls_token = nn.Parameter(torch.randn(1, 1, self.hidden_size)) + + self.patch_embed = nn.Conv2d( + in_channels=3, + out_channels=self.hidden_size, + kernel_size=self.patch_size, + stride=self.patch_size, + bias=False) + + self.num_patches = (self.image_size // self.patch_size)**2 + + self.position_embedding = nn.Parameter( + torch.randn(1, self.num_patches + 1, self.hidden_size)) + + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.pre_layernorm = layernorm_func( + self.hidden_size, eps=config.layer_norm_eps) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + batch_size = pixel_values.size(0) + image_embeds = self.patch_embed(pixel_values) + image_embeds = image_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.cls_token.expand(batch_size, 1, + -1).to(image_embeds.dtype) + embeddings = torch.cat([class_embeds, image_embeds], dim=1) + embeddings = embeddings + \ + self.position_embedding[:, : embeddings.size(1)].to( + image_embeds.dtype) + embeddings = self.pre_layernorm(embeddings) + return embeddings + + +class LayerNormFp32(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back).""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def forward(self, x: torch.Tensor): + output = torch.nn.functional.layer_norm( + x.float(), + self.normalized_shape, + self.weight.float() if self.weight is not None else None, + self.bias.float() if self.bias is not None else None, + self.eps, + ) + return output.type_as(x) + + +class MplugOwlVisionAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:' + f' {self.num_heads}).') + self.scale = self.head_dim**-0.5 + self.dropout = nn.Dropout(config.attention_dropout) + + self.query_key_value = nn.Linear(self.hidden_size, + 3 * self.hidden_size) + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, + self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], + Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, seq_len, embed_dim = hidden_states.size() + + mixed_qkv = self.query_key_value(hidden_states) + + mixed_qkv = mixed_qkv.reshape(bsz, seq_len, self.num_heads, 3, + embed_dim // self.num_heads).permute( + 3, 0, 2, 1, 4) # [3, b, np, sq, hn] + query_states, key_states, value_states = ( + mixed_qkv[0], + mixed_qkv[1], + mixed_qkv[2], + ) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_states, + key_states.transpose(-1, -2)) + + attention_scores = attention_scores * self.scale + + # Normalize the attention scores to probabilities. + attention_probs = torch.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, + value_states).permute(0, 2, 1, 3) + + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size, ) + context_layer = context_layer.reshape(new_context_layer_shape) + + output = self.dense(context_layer) + + outputs = (output, attention_probs) if output_attentions else (output, + None) + + return outputs + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class MplugOwlMLP(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = QuickGELU() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +class MplugOwlVisionEncoderLayer(nn.Module): + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = MplugOwlVisionAttention(config) + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.input_layernorm = layernorm_func( + self.hidden_size, eps=config.layer_norm_eps) + self.mlp = MplugOwlMLP(config) + self.post_attention_layernorm = layernorm_func( + self.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + `(config.encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + head_mask=attention_mask, + output_attentions=output_attentions, + ) + hidden_states = hidden_states + residual + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + + hidden_states = hidden_states + residual + + outputs = (hidden_states, ) + + if output_attentions: + outputs += (attn_weights, ) + + return outputs + + +class MplugOwlPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = MplugOwlConfig + base_model_prefix = 'mplug_owl' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [ + r'position_ids', + r'language_model.encoder.embed_tokens.weight', + r'language_model.decoder.embed_tokens.weight', + r'language_model.lm_head.weight', + ] + _no_split_modules = ['MplugOwlAttention'] + _keep_in_fp32_modules = ['wo'] + + def _init_weights(self, module): + """Initialize the weights""" + factor = self.config.initializer_range + if isinstance(module, nn.Conv2d) or isinstance( + module, nn.Embedding) or isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=factor) + if hasattr(module, 'bias') and module.bias is not None: + module.bias.data.zero_() + + if isinstance(module, MplugOwlVisionEmbeddings): + if hasattr(self.config, 'vision_config'): + factor = self.config.vision_config.initializer_range + nn.init.trunc_normal_( + module.position_embedding, mean=0.0, std=factor) + nn.init.trunc_normal_(module.cls_token, mean=0.0, std=factor) + + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + elif isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Parameter): + nn.init.trunc_normal_(module.data, mean=0.0, std=factor) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, MplugOwlVisionEncoder): + module.gradient_checkpointing = value + + +MPLUG_OWL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MplugOwlConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +MPLUG_OWL_VISION_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`MplugOwlPreprocessor`]. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +MPLUG_OWL_TEXT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +MPLUG_OWL_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`MplugOwlPreprocessor`]. + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be + provided to serve as text prompt, which the language model can continue. + + Indices can be obtained using [`MplugOwlPreprocessor`]. See [`MplugOwlPreprocessor.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an + encoder-decoder language model (like T5) is used. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) + + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + Only relevant in case an encoder-decoder language model (like T5) is used. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +class MplugOwlVisionEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`MplugOwlVisionEncoderLayer`]. + + Args: + config (`MplugOwlVisionConfig`): + The corresponding vision configuration for the `MplugOwlEncoder`. + """ + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + MplugOwlVisionEncoderLayer(config) + for _ in range(config.num_hidden_layers) + ]) + self.gradient_checkpointing = False + + def forward( + self, + inputs_embeds, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Embedded representation of the inputs. Should be float, not int tokens. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + hidden_states = inputs_embeds + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(encoder_layer), + hidden_states, + attention_mask, + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1], ) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states, ) + + if not return_dict: + return tuple( + v for v in [hidden_states, encoder_states, all_attentions] + if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, + hidden_states=encoder_states, + attentions=all_attentions) + + +class MplugOwlVisionModel(MplugOwlPreTrainedModel): + main_input_name = 'pixel_values' + config_class = MplugOwlVisionConfig + + def __init__(self, config: MplugOwlVisionConfig): + super().__init__(config) + self.config = config + self.hidden_size = config.hidden_size + + self.embeddings = MplugOwlVisionEmbeddings(config) + self.encoder = MplugOwlVisionEncoder(config) + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.post_layernorm = layernorm_func( + self.hidden_size, eps=config.layer_norm_eps) + + self.post_init() + + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError('You have to specify pixel_values') + + hidden_states = self.embeddings(pixel_values) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.post_layernorm(last_hidden_state) + + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + def get_input_embeddings(self): + return self.embeddings + + +class MplugOwlVisualAbstractorMLP(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.config = config + in_features = config.hidden_size + hidden_features = config.intermediate_size + hidden_features = int(2 * hidden_features / 3) + multiple_of = 256 + hidden_features = multiple_of * \ + ((hidden_features + multiple_of - 1) // multiple_of) + self.act = nn.SiLU() + + self.w1 = nn.Linear(in_features, hidden_features) + self.w2 = nn.Linear(hidden_features, in_features) + self.w3 = nn.Linear(in_features, hidden_features) + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.ffn_ln = layernorm_func( + hidden_features, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.act( + self.w1(hidden_states)) * self.w3(hidden_states) + hidden_states = self.ffn_ln(hidden_states) + hidden_states = self.w2(hidden_states) + return hidden_states + + +class MplugOwlVisualAbstractorMultiHeadAttention(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention heads (%d)' + % (config.hidden_size, config.num_attention_heads)) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.encoder_hidden_size, self.all_head_size) + self.value = nn.Linear(config.encoder_hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + + mixed_query_layer = self.query(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + attention_scores = attention_scores / \ + math.sqrt(self.attention_head_size) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + outputs = outputs + (past_key_value, ) + return outputs + + +class MplugOwlVisualAbstractorCrossOutput(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + dim = config.hidden_size + self.out_proj = nn.Linear(dim, dim, bias=True) + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.norm2 = layernorm_func(dim) + self.mlp = MplugOwlVisualAbstractorMLP(config) + + def forward(self, hidden_states: torch.Tensor, + input_tensor: torch.Tensor) -> torch.Tensor: + input_tensor = input_tensor + self.out_proj(hidden_states) + input_tensor = input_tensor + self.mlp(self.norm2(input_tensor)) + return input_tensor + + +class MplugOwlVisualAbstractorAttention(nn.Module): + + def __init__(self, config: MplugOwlVisualAbstractorConfig): + super().__init__() + self.attention = MplugOwlVisualAbstractorMultiHeadAttention(config) + self.output = MplugOwlVisualAbstractorCrossOutput(config) + self.pruned_heads = set() + layernorm_func = LayerNormFp32 if config.use_fp32_layernorm else nn.LayerNorm + self.norm1 = layernorm_func(config.hidden_size) + self.normk = layernorm_func(config.hidden_size) + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.attention.num_attention_heads, + self.attention.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.attention.query = prune_linear_layer(self.attention.query, index) + self.attention.key = prune_linear_layer(self.attention.key, index) + self.attention.value = prune_linear_layer(self.attention.value, index) + self.output.dense = prune_linear_layer( + self.output.out_proj, index, dim=1) + + # Update hyper params and store pruned heads + self.attention.num_attention_heads = self.attention.num_attention_heads - \ + len(heads) + self.attention.all_head_size = self.attention.attention_head_size * \ + self.attention.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + # HACK we apply norm on q and k + hidden_states = self.norm1(hidden_states) + encoder_hidden_states = self.normk(encoder_hidden_states) + encoder_hidden_states = torch.cat( + [hidden_states, encoder_hidden_states], dim=1) + encoder_attention_mask = torch.cat( + [attention_mask, encoder_attention_mask], dim=-1) + self_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + # add attentions if we output them + outputs = (attention_output, ) + self_outputs[1:] + return outputs + + +class MplugOwlVisualAbstractorLayer(nn.Module): + + def __init__(self, config, layer_idx): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + + self.layer_idx = layer_idx + + self.crossattention = MplugOwlVisualAbstractorAttention(config) + self.has_cross_attention = True + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + ): + if encoder_hidden_states is None: + raise ValueError( + 'encoder_hidden_states must be given for cross-attention layers' + ) + cross_attention_outputs = self.crossattention( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + query_attention_output = cross_attention_outputs[0] + + outputs = (query_attention_output, ) + return outputs + + +class MplugOwlVisualAbstractorEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layers = nn.ModuleList([ + MplugOwlVisualAbstractorLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + + for i in range(self.config.num_hidden_layers): + layer_module = self.layers[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + + hidden_states = layer_outputs[0] + + return BaseModelOutput(last_hidden_state=hidden_states, ) + + +class MplugOwlVisualAbstractorModel(MplugOwlPreTrainedModel): + + def __init__(self, config: MplugOwlVisualAbstractorConfig, + language_hidden_size): + super().__init__(config) + self.config = config + + self.encoder = MplugOwlVisualAbstractorEncoder(config) + self.visual_fc = torch.nn.Linear(config.hidden_size, + language_hidden_size) + self.vit_eos = torch.nn.Parameter( + torch.randn(1, 1, language_hidden_size)) + self.post_init() + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def get_extended_attention_mask( + self, + attention_mask: torch.Tensor, + input_shape: Tuple[int], + device: torch.device, + ) -> torch.Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (`Tuple[int]`): + The shape of the input to the model. + device: (`torch.device`): + The device of the input to the model. + + Returns: + `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + 'Wrong shape for input_ids (shape {}) or attention_mask (shape {})' + .format(input_shape, attention_mask.shape)) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + query_embeds, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors: + shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and + value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are + used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape + `(batch_size, sequence_length)`. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + embedding_output = query_embeds + input_shape = embedding_output.size()[:-1] + batch_size, seq_length = input_shape + device = embedding_output.device + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask is None: + attention_mask = torch.ones( + (query_embeds.shape[0], query_embeds.shape[1]), + dtype=torch.long, + device=query_embeds.device) + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[ + 0].size() + else: + ( + encoder_batch_size, + encoder_sequence_length, + _, + ) = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [ + self.invert_attention_mask(mask) + for mask in encoder_attention_mask + ] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = sequence_output[:, 0, :] + + sequence_output = self.visual_fc(sequence_output) + eos_repeat = self.vit_eos.repeat(sequence_output.shape[0], 1, 1) + sequence_output = torch.cat([sequence_output, eos_repeat], dim=1) + + return BaseModelOutputWithPooling( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + ) + + +class MplugOwlModel(MplugOwlPreTrainedModel): + r"""The mPLUG-Owl model is a multi-modal conversation model that support various modalities as input. + mPLUG-Owl consists a visual encoder, a visual abstrator module and a language decoder model, which enables + both image and text input. + This model is implemented base on mPLUG-Owl: Modularization Empowers Large Language Models with Multimodality. + `Paper `. + """ + config_class = MplugOwlConfig + main_input_name = 'pixel_values' + + def __init__(self, config: MplugOwlConfig): + super().__init__(config) + + self.vision_model = MplugOwlVisionModel(config.vision_config) + + self.query_tokens = nn.Parameter( + torch.zeros(1, config.num_query_tokens, + config.visual_abstractor_config.hidden_size)) + self.abstractor = MplugOwlVisualAbstractorModel( + config.visual_abstractor_config, config.text_config.hidden_size) + + # if config.use_decoder_only_language_model: + language_model = AutoModelForCausalLM.from_config(config.text_config) + self.language_model = language_model + + if config.text_config.model_type == 'bloom': + bound_method = bloom_forward.__get__( + self.language_model.transformer, + self.language_model.transformer.__class__) + setattr(self.language_model.transformer, 'forward', bound_method) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def get_output_embeddings(self) -> nn.Module: + return self.language_model.get_output_embeddings() + + def get_encoder(self): + return self.language_model.get_encoder() + + def get_decoder(self): + return self.language_model.get_decoder() + + def _tie_weights(self): + if not self.config.use_decoder_only_language_model: + self.language_model.encoder.embed_tokens = self.language_model.shared + self.language_model.decoder.embed_tokens = self.language_model.shared + + def get_text_features( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.Tensor] = None, + decoder_attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.use_decoder_only_language_model: + text_outputs = self.language_model( + input_ids=input_ids, + attention_mask=attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + else: + inputs_embeds = self.language_model.get_input_embeddings()( + input_ids) + + text_outputs = self.language_model( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + labels=labels, + ) + + return text_outputs + + def get_image_features( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + return vision_outputs + + +def get_media_indices(my_list): + if isinstance(my_list, torch.Tensor): + my_list = my_list.cpu().tolist() + result = [] + for i in range(len(my_list)): + if i == 0 and my_list[i] < 0: + result.append(i) + elif my_list[i] != my_list[i - 1] and my_list[i] < 0: + result.append(i) + return result + + +class MplugOwlForConditionalGenerationHF(MplugOwlPreTrainedModel): + config_class = MplugOwlConfig + main_input_name = 'pixel_values' + + def __init__(self, config: MplugOwlConfig, **kwargs): + super().__init__(config) + + self.vision_model = MplugOwlVisionModel(config.vision_config) + + self.query_tokens = nn.Parameter( + torch.zeros(1, config.num_query_tokens, + config.visual_abstractor_config.hidden_size)) + self.abstractor = MplugOwlVisualAbstractorModel( + config.visual_abstractor_config, config.text_config.hidden_size) + + # if config.use_decoder_only_language_model: + language_model = AutoModelForCausalLM.from_config(config.text_config) + self.language_model = language_model + + # Initialize weights and apply final processing + self.post_init() + self.main_input_name = 'input_ids' + + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def get_output_embeddings(self) -> nn.Module: + return self.language_model.get_output_embeddings() + + def get_encoder(self): + return self.language_model.get_encoder() + + def get_decoder(self): + return self.language_model.get_decoder() + + def _tie_weights(self): + if not self.config.use_decoder_only_language_model: + self.language_model.encoder.embed_tokens = self.language_model.shared + self.language_model.decoder.embed_tokens = self.language_model.shared + + def _preprocess_accelerate(self): + r""" + Some pre-processing hacks to make the model `accelerate` compatible. Check + https://github.com/huggingface/transformers/pull/21707 for more details. + """ + hf_device_map = self.hf_device_map + + if len( + hf_device_map + ) > 1 and 'language_model' not in hf_device_map and torch.cuda.device_count( + ) > 1: + # warn users about unexpected behavior when using multi-GPU + mPLUG-Owl + `accelerate`. + logger.warning( + 'The `language_model` is not in the `hf_device_map` dictionary and you are running your script' + ' in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`.' + ' Please pass a `device_map` that contains `language_model` to remove this warning.' + ' Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for' + ' more details on creating a `device_map` for large models.', ) + + if hasattr(self.language_model, '_hf_hook'): + self.language_model._hf_hook.io_same_device = True # For `generate` compatibility + + def forward( + self, + pixel_values: torch.FloatTensor, + input_ids: torch.FloatTensor, + num_images, + non_padding_mask: Optional[torch.LongTensor] = None, + non_media_mask: Optional[torch.LongTensor] = None, + prompt_mask: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, MplugOwlForConditionalGenerationModelOutput]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # get text embedding + text_tokens_ = input_ids + batch_size = input_ids.shape[0] + + media_token_indices = [ + # [:-1] since we would not use the last token for embedding + get_media_indices(text_tokens_[i][:-1]) for i in range(batch_size) + ] + text_tokens_[text_tokens_ < 0] = 1 # Not used + text_embeds = self.get_input_embeddings()( + text_tokens_) # Temporally Embedding + + if pixel_values is not None: + pixel_values = pixel_values.half() + image_embeds = self.vision_model( + pixel_values, return_dict=True).last_hidden_state + + image_attention_mask = torch.ones( + image_embeds.size()[:-1], + dtype=torch.long, + device=image_embeds.device) + query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, + -1) + + query_features = self.abstractor( + query_embeds=query_tokens, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_attention_mask, + )['last_hidden_state'] + img_seq_length = query_features.shape[1] + + num_images_per_sample = num_images.long().cpu().tolist() + + text_chunk_embeds = [] + img_idx = 0 + for b in range(batch_size): + start = 0 + result = [] + if len(media_token_indices[b]) > 0: + for i, pos in enumerate(media_token_indices[b]): + if pos > start: + result.append(text_embeds[b, start:pos]) + result.append(query_features[img_idx + i]) + start = pos + img_seq_length + if start < text_embeds.shape[1]: + result.append(text_embeds[b, start:]) + + img_idx += num_images_per_sample[b] + text_chunk_embeds.append(torch.cat(result, dim=0)) + + # Actual Input Embeddings + input_embeds = torch.stack(text_chunk_embeds, dim=0) + + # Create causal mask and position ids + _, loss_mask, position_ids = \ + get_ltor_masks_and_position_ids_from_embeddings(input_embeds) + + # Calculate the loss_mask + non_padding_mask = non_padding_mask.long() + non_media_mask = non_media_mask.long() + prompt_mask = prompt_mask.long() # TODO How to deal with prompt mask + loss_mask = loss_mask[:, :-1] + + loss_mask = loss_mask * non_padding_mask * non_media_mask * prompt_mask + + # Forward into GPT + outputs = self.language_model( + inputs_embeds=input_embeds, + attention_mask=attention_mask, + labels=labels, + ) + outputs.loss = (outputs.loss + * loss_mask.view(-1)).sum() / loss_mask.sum() + return outputs + + @torch.no_grad() + def generate( + self, + pixel_values: torch.FloatTensor, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + **generate_kwargs, + ) -> torch.LongTensor: + """ + Overrides `generate` function to be able to use the model as a conditional generator. + + Args: + pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)): + Input images to be processed. + input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + The sequence used as a prompt for the generation. + attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): + Mask to avoid performing attention on padding token indices + + Returns: + captions (list): A list of strings of length batch_size * num_captions. + """ + + if input_ids is not None: + batch_size = input_ids.size(0) + media_token_indices = [ + get_media_indices(input_ids[i]) for i in range(batch_size) + ] + num_images_per_sample = [len(x) for x in media_token_indices] + input_ids = input_ids.clone() + input_ids[input_ids < 0] = 0 # Not used + + if attention_mask is None: + attention_mask = torch.ones_like(input_ids).long().to( + input_ids.device) + + if hasattr(self, 'hf_device_map'): + # preprocess for `accelerate` + self._preprocess_accelerate() + batch_size = input_ids.shape[0] + # get text embedding + inputs_embeds = self.get_input_embeddings()(input_ids) + # get visual embedding + if pixel_values is not None: + pixel_values = pixel_values.half() + pixel_values = pixel_values.to(input_ids.device) + with torch.no_grad(): + image_embeds = self.vision_model( + pixel_values, return_dict=True).last_hidden_state + image_attention_mask = torch.ones( + image_embeds.size()[:-1], + dtype=torch.long, + device=image_embeds.device) + query_tokens = self.query_tokens.expand( + image_embeds.shape[0], -1, -1) + query_outputs = self.abstractor( + query_embeds=query_tokens, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_attention_mask, + return_dict=True, + ) + query_output = query_outputs['last_hidden_state'] + image_embeds = query_output + img_seq_length = image_embeds.shape[1] + + # =================== + # Get actual input embeddings + # =================== + text_chunk_embeds = [] + text_chunk_attns = [] + img_idx = 0 + + for b in range(batch_size): + start = 0 + result = [] + result_attn = [] + for i, pos in enumerate(media_token_indices[b]): + if pos > start: + result.append(inputs_embeds[b, start:pos]) + result_attn.append(attention_mask[b, start:pos]) + result.append(image_embeds[img_idx + i]) + result_attn.append( + torch.ones( + image_embeds[img_idx + i].shape[0], + device=inputs_embeds.device)) + start = pos + img_seq_length + if start < inputs_embeds.shape[1]: + result.append(inputs_embeds[b, start:]) + result_attn.append(attention_mask[b, start:]) + + img_idx += num_images_per_sample[b] + text_chunk_embeds.append(torch.cat(result, dim=0)) + text_chunk_attns.append(torch.cat(result_attn, dim=0)) + inputs_embeds = torch.stack(text_chunk_embeds, dim=0) + attention_mask = torch.stack(text_chunk_attns, dim=0) + + outputs = self.language_model.generate( + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + **generate_kwargs, + ) + + return outputs + + +@MODELS.register_module( + Tasks.multimodal_dialogue, module_name=Models.mplug_owl) +class MplugOwlForConditionalGeneration(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the mPLUG-Owl model from the `model_dir` path. + Args: + model_dir (str): the model path. + """ + + super().__init__(model_dir, *args, **kwargs) + self.model = MplugOwlForConditionalGenerationHF.from_pretrained( + model_dir, + torch_dtype=torch.half, + ) + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + output = self.model.generate(**input) + return output diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/__init__.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1b3f445b13a037c6ddbe7f616236de1860756662 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .model import MultiStageDiffusionForTextToImageSynthesis diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/clip.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..594e3c73714011bb141d5805c87163c93ab94548 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/clip.py @@ -0,0 +1,319 @@ +# Part of the implementation is borrowed and modified from CLIP, publicly available at https://github.com/openai/CLIP. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['CLIP'] + + +def to_fp16(m): + if isinstance(m, (nn.Linear, nn.Conv2d)): + m.weight.data = m.weight.data.half() + if m.bias is not None: + m.bias.data = m.bias.data.half() + elif hasattr(m, 'head'): + p = getattr(m, 'head') + p.data = p.data.half() + + +class QuickGELU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class LayerNorm(nn.LayerNorm): + r"""Subclass of nn.LayerNorm to handle fp16. + """ + + def forward(self, x): + return super(LayerNorm, self).forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.attn_dropout = nn.Dropout(attn_dropout) + self.proj = nn.Linear(dim, dim) + self.proj_dropout = nn.Dropout(proj_dropout) + + def forward(self, x, mask=None): + r"""x: [B, L, C]. + mask: [*, L, L]. + """ + b, l, _, n = *x.size(), self.num_heads + + # compute query, key, and value + q, k, v = self.to_qkv(x.transpose(0, 1)).chunk(3, dim=-1) + q = q.reshape(l, b * n, -1).transpose(0, 1) + k = k.reshape(l, b * n, -1).transpose(0, 1) + v = v.reshape(l, b * n, -1).transpose(0, 1) + + # compute attention + attn = self.scale * torch.bmm(q, k.transpose(1, 2)) + if mask is not None: + attn = attn.masked_fill(mask[:, :l, :l] == 0, float('-inf')) + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + attn = self.attn_dropout(attn) + + # gather context + x = torch.bmm(attn, v) + x = x.view(b, n, l, -1).transpose(1, 2).reshape(b, l, -1) + + # output + x = self.proj(x) + x = self.proj_dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads, attn_dropout, proj_dropout) + self.norm2 = LayerNorm(dim) + self.mlp = nn.Sequential( + nn.Linear(dim, dim * 4), QuickGELU(), nn.Linear(dim * 4, dim), + nn.Dropout(proj_dropout)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.mlp(self.norm2(x)) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=224, + patch_size=16, + dim=768, + out_dim=512, + num_heads=12, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + assert image_size % patch_size == 0 + super(VisionTransformer, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + gain = 1.0 / math.sqrt(dim) + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.pos_embedding = nn.Parameter( + gain * torch.randn(1, self.num_patches + 1, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.pre_norm = LayerNorm(dim) + self.transformer = nn.Sequential(*[ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.post_norm = LayerNorm(dim) + + # head + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + def forward(self, x): + b, dtype = x.size(0), self.head.dtype + x = x.type(dtype) + + # patch-embedding + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) # [b, n, c] + x = torch.cat([self.cls_embedding.repeat(b, 1, 1).type(dtype), x], + dim=1) + x = self.dropout(x + self.pos_embedding.type(dtype)) + x = self.pre_norm(x) + + # transformer + x = self.transformer(x) + + # head + x = self.post_norm(x) + x = torch.mm(x[:, 0, :], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class TextTransformer(nn.Module): + + def __init__(self, + vocab_size, + text_len, + dim=512, + out_dim=512, + num_heads=8, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(TextTransformer, self).__init__() + self.vocab_size = vocab_size + self.text_len = text_len + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + + # embeddings + self.token_embedding = nn.Embedding(vocab_size, dim) + self.pos_embedding = nn.Parameter(0.01 * torch.randn(1, text_len, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.transformer = nn.ModuleList([ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.norm = LayerNorm(dim) + + # head + gain = 1.0 / math.sqrt(dim) + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + # causal attention mask + self.register_buffer('attn_mask', + torch.tril(torch.ones(1, text_len, text_len))) + + def forward(self, x): + eot, dtype = x.argmax(dim=-1), self.head.dtype + + # embeddings + x = self.dropout( + self.token_embedding(x).type(dtype) + + self.pos_embedding.type(dtype)) + + # transformer + for block in self.transformer: + x = block(x, self.attn_mask) + + # head + x = self.norm(x) + x = torch.mm(x[torch.arange(x.size(0)), eot], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class CLIP(nn.Module): + + def __init__(self, + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(CLIP, self).__init__() + self.embed_dim = embed_dim + self.image_size = image_size + self.patch_size = patch_size + self.vision_dim = vision_dim + self.vision_heads = vision_heads + self.vision_layers = vision_layers + self.vocab_size = vocab_size + self.text_len = text_len + self.text_dim = text_dim + self.text_heads = text_heads + self.text_layers = text_layers + + # models + self.visual = VisionTransformer( + image_size=image_size, + patch_size=patch_size, + dim=vision_dim, + out_dim=embed_dim, + num_heads=vision_heads, + num_layers=vision_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.textual = TextTransformer( + vocab_size=vocab_size, + text_len=text_len, + dim=text_dim, + out_dim=embed_dim, + num_heads=text_heads, + num_layers=text_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.log_scale = nn.Parameter(math.log(1 / 0.07) * torch.ones([])) + + def forward(self, imgs, txt_tokens): + r"""imgs: [B, C, H, W] of torch.float32. + txt_tokens: [B, T] of torch.long. + """ + xi = self.visual(imgs) + xt = self.textual(txt_tokens) + + # normalize features + xi = F.normalize(xi, p=2, dim=1) + xt = F.normalize(xt, p=2, dim=1) + + # logits + scale = self.log_scale.exp() + logits_i2t = scale * torch.mm(xi, xt.t()) + logits_t2i = scale * torch.mm(xt, xi.t()) + return logits_i2t, logits_t2i + + def init_weights(self): + # embeddings + nn.init.normal_(self.textual.token_embedding.weight, std=0.02) + nn.init.normal_(self.visual.patch_embedding.weight, tsd=0.1) + + # attentions + for modality in ['visual', 'textual']: + dim = self.vision_dim if modality == 'visual' else 'textual' + transformer = getattr(self, modality).transformer + proj_gain = (1.0 / math.sqrt(dim)) * ( + 1.0 / math.sqrt(2 * transformer.num_layers)) + attn_gain = 1.0 / math.sqrt(dim) + mlp_gain = 1.0 / math.sqrt(2.0 * dim) + for block in transformer.layers: + nn.init.normal_(block.attn.to_qkv.weight, std=attn_gain) + nn.init.normal_(block.attn.proj.weight, std=proj_gain) + nn.init.normal_(block.mlp[0].weight, std=mlp_gain) + nn.init.normal_(block.mlp[2].weight, std=proj_gain) + + def fp16(self): + return self.apply(to_fp16) diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/decoder.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..eb52a48be3764b3f82294d61e006c1e587485e4e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/decoder.py @@ -0,0 +1,322 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['Decoder'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, scale_factor, use_conv=False): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.scale_factor = scale_factor + self.use_conv = use_conv + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(in_dim, out_dim, 3, padding=1) + if use_conv else nn.Identity()) + elif scale_factor == 0.5: + self.resample = nn.Conv2d( + in_dim, out_dim, 3, stride=2, + padding=1) if use_conv else nn.AvgPool2d( + kernel_size=2, stride=2) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + scale_factor=1.0, + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.scale_factor = scale_factor + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, scale_factor, use_conv=False) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e): + identity = self.resample(x) + x = self.layer1[-1](self.resample(self.layer1[:-1](x))) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class Decoder(nn.Module): + + def __init__(self, + in_dim=3, + dim=512, + y_dim=512, + context_dim=512, + out_dim=6, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + resblock_resample=True, + use_scale_shift_norm=True, + dropout=0.1): + embed_dim = dim * 4 + super(Decoder, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.resblock_resample = resblock_resample + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.y_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.context_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, context_dim * 4)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim, embed_dim, out_dim, + use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + self.encoder.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample( + out_dim, out_dim, 0.5, use_conv=True) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + AttentionBlock(out_dim, context_dim, num_heads, head_dim), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample( + out_dim, out_dim, 2.0, use_conv=True) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y): + # embeddings + e = self.time_embedding(sinusoidal_embedding( + t, self.dim)) + self.y_embedding(y) + context = self.context_embedding(y).view(-1, 4, self.context_dim) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, e, context) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, e, context) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, e, context) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, e, context): + if isinstance(module, ResidualBlock): + x = module(x, e) + elif isinstance(module, AttentionBlock): + x = module(x, context) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/gaussian_diffusion.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/gaussian_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..12a14d769d256afbca2fe807a90415abf39e0e9e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/gaussian_diffusion.py @@ -0,0 +1,700 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly available at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch + +from modelscope.models.multi_modal.dpm_solver_pytorch import ( + DPM_Solver, NoiseScheduleVP, model_wrapper, model_wrapper_guided_diffusion) + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def kl_divergence(mu1, logvar1, mu2, logvar2): + u1 = -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + u2 = ((mu1 - mu2)**2) * torch.exp(-logvar2) + return 0.5 * (u1 + u2) + + +def standard_normal_cdf(x): + r"""A fast approximation of the cumulative distribution function of the standard normal. + """ + return 0.5 * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x0, mean, log_scale): + assert x0.shape == mean.shape == log_scale.shape + cx = x0 - mean + inv_stdv = torch.exp(-log_scale) + cdf_plus = standard_normal_cdf(inv_stdv * (cx + 1.0 / 255.0)) + cdf_min = standard_normal_cdf(inv_stdv * (cx - 1.0 / 255.0)) + log_cdf_plus = torch.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = torch.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = torch.where( + x0 < -0.999, log_cdf_plus, + torch.where(x0 > 0.999, log_one_minus_cdf_min, + torch.log(cdf_delta.clamp(min=1e-12)))) + assert log_probs.shape == x0.shape + return log_probs + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + fn_t1 = math.cos((t1 + 0.008) / 1.008 * math.pi / 2)**2 + fn_t2 = math.cos((t2 + 0.008) / 1.008 * math.pi / 2)**2 + betas.append(min(1.0 - fn_t2 / fn_t1, 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + noise = torch.randn_like(x0) if noise is None else noise + u1 = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + u2 = _i(self.sqrt_one_minus_alphas_cumprod, t, x0) * noise + return u1 + u2 + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + shape = (-1, *((1, ) * (xt.ndim - 1))) + mask = t.ne(0).float().view(shape) # no noise when t == 0 + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + cond = self.var_type.startswith('fixed') + dim = y_out.size(1) if cond else y_out.size(1) // 2 + u1 = u_out[:, :dim] + u2 = guide_scale * (y_out[:, :dim] - u_out[:, :dim]) + out = torch.cat([u1 + u2, y_out[:, dim:]], dim=1) + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + u1 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu + u2 = _i(self.posterior_mean_coef2 / self.posterior_mean_coef1, t, + xt) * xt + x0 = u1 - u2 + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + u1 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * out + x0 = u1 - u2 + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def dpm_solver_sample_loop(self, + noise, + model, + skip_type, + order, + method, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + dpm_solver_timesteps=20, + t_start=None, + t_end=None, + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpm_solver'): + r"""Sample using DPM-Solver-based method. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + Please check all the parameters in `dpm_solver.sample` before using. + """ + noise_schedule = NoiseScheduleVP( + schedule='discrete', betas=self.betas.float()) + model_fn = model_wrapper_guided_diffusion( + model=model, + noise_schedule=noise_schedule, + var_type=self.var_type, + mean_type=self.mean_type, + model_kwargs=model_kwargs, + clamp=clamp, + percentile=percentile, + rescale_timesteps=self.rescale_timesteps, + num_timesteps=self.num_timesteps, + guide_scale=guide_scale, + condition_fn=condition_fn, + ) + dpm_solver = DPM_Solver( + model_fn=model_fn, + noise_schedule=noise_schedule, + ) + xt = dpm_solver.sample( + noise, + steps=dpm_solver_timesteps, + order=order, + skip_type=skip_type, + method=method, + solver_type=solver_type, + t_start=t_start, + t_end=t_end, + lower_order_final=lower_order_final, + denoise_to_zero=denoise_to_zero) + return xt + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + r"""Sample from p(x_{t-1} | x_t) using DDIM. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + u1 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + x0 = u1 - u2 + + # derive variables + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + u1 = (1 - alphas_prev) / (1 - alphas) + u2 = (1 - alphas / alphas_prev) + sigmas = eta * torch.sqrt(u1 * u2) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b = x0.size(0) + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + u1 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + x0 = u1 - u2 + + # derive eps + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + u1 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + x0 = u1 - u2 + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + # mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def loss(self, x0, t, model, model_kwargs={}, noise=None, input_x0=None): + noise = torch.randn_like(x0) if noise is None else noise + input_x0 = x0 if input_x0 is None else input_x0 + xt = self.q_sample(input_x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', 'rescaled_l1']: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = (out - target).pow(1 if self.loss_type.endswith('l1') else 2 + ).abs().flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) + kl = kl.flatten(1).mean(dim=1) / math.log(2.0) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) + nll = nll.flatten(1).mean(dim=1) / math.log(2.0) + + # NLL for p(x0 | x1) and KL otherwise + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + r"""Compute the entire variational lower bound, measured in bits-per-dim. + """ + # prepare input and output + b = x0.size(0) + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + u1 = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) + u2 = _i(self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = u1 / u2 + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/model.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5912df7b91d2abd659245333a05504de5688a51e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/model.py @@ -0,0 +1,352 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math +import os.path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.multi_stage_diffusion.clip import CLIP +from modelscope.models.multi_modal.multi_stage_diffusion.decoder import Decoder +from modelscope.models.multi_modal.multi_stage_diffusion.gaussian_diffusion import ( + GaussianDiffusion, beta_schedule) +from modelscope.models.multi_modal.multi_stage_diffusion.prior import Prior +from modelscope.models.multi_modal.multi_stage_diffusion.tokenizer import ( + CLIPTokenizer, XGLMTokenizer) +from modelscope.models.multi_modal.multi_stage_diffusion.upsampler import ( + Upsampler256, Upsampler1024) +from modelscope.models.multi_modal.multi_stage_diffusion.xglm import XGLM +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['MultiStageDiffusionForTextToImageSynthesis'] + + +def make_diffusion(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None, + mean_type='eps', + var_type='fixed_small'): + betas = beta_schedule(schedule, num_timesteps, init_beta, last_beta) + diffusion = GaussianDiffusion( + betas, mean_type=mean_type, var_type=var_type) + return diffusion + + +class UnCLIP(nn.Module): + + def __init__(self, model_dir): + super(UnCLIP, self).__init__() + self.model_dir = model_dir + self.config = json.load( + open(f'{model_dir}/{ModelFile.CONFIGURATION}', encoding='utf-8')) + + # modules + self.clip = CLIP(**self.config['clip']).fp16() + self.xglm = XGLM(**self.config['xglm']) + self.prior = Prior(**self.config['prior']) + self.decoder = Decoder(**self.config['decoder']) + self.upsampler256 = Upsampler256(**self.config['upsampler256']) + self.upsampler1024 = Upsampler1024(**self.config['upsampler1024']) + + # diffusions + self.prior_diffusion = make_diffusion(**self.config['prior_diffusion']) + self.decoder_diffusion = make_diffusion( + **self.config['decoder_diffusion']) + self.upsampler256_diffusion = make_diffusion( + **self.config['upsampler256_diffusion']) + self.upsampler1024_diffusion = make_diffusion( + **self.config['upsampler1024_diffusion']) + + # tokenizers + self.clip_tokenizer = CLIPTokenizer( + bpe_path=f'{model_dir}/bpe_simple_vocab_16e6.txt.gz') + self.xglm_tokenizer = XGLMTokenizer(model_dir=model_dir) + + def forward(self, *args, **kwargs): + raise NotImplementedError( + '"forward" is not implemented. Use "synthesis" instead.') + + @torch.no_grad() + def synthesis(self, + text='A photo of a confused grizzly bear in calculus class.', + tokenizer='clip', + batch_size=4, + timesteps_prior=100, + timesteps_64=50, + timesteps_256=20, + timesteps_1024=20, + guide_prior=3.0, + guide_64=7.0, + guide_256=3.0, + guide_1024=3.0, + eta_prior=0.0, + eta_64=0.0, + eta_256=0.0, + eta_1024=0.0, + solver='dpm-solver'): + device = next(self.parameters()).device + + # check params + assert all([ + t > 0 and t <= 1000 for t in + [timesteps_prior, timesteps_64, timesteps_256, timesteps_1024] + ]) + assert all([ + g > 1 and g < 15 + for g in [guide_prior, guide_64, guide_256, guide_1024] + ]) + assert all([ + e >= 0 and e <= 1.0 + for e in [eta_prior, eta_64, eta_256, eta_1024] + ]) + assert batch_size >= 1 and batch_size <= 16 + + # tokenize the text + if tokenizer == 'clip': + y = F.normalize( + self.clip.textual(self.clip_tokenizer([text]).to(device)), + p=2, + dim=1) + zero_y = F.normalize( + self.clip.textual(self.clip_tokenizer(['']).to(device)), + p=2, + dim=1) + elif tokenizer == 'xglm': + y = F.normalize( + self.xglm(*to_device(self.xglm_tokenizer([text]), device)), + p=2, + dim=1) + zero_y = F.normalize( + self.xglm(*to_device(self.xglm_tokenizer(['']), device)), + p=2, + dim=1) + else: + raise ValueError( + f'Expected tokenizer to be one of "clip" or "xglm", but got {tokenizer}' + ) + y = math.sqrt(y.size(1)) * y.repeat(batch_size, 1) + zero_y = math.sqrt(zero_y.size(1)) * zero_y.repeat(batch_size, 1) + + # synthesis + with amp.autocast(enabled=True): + # choose a proper solver + if solver == 'dpm-solver': + # prior + x0 = self.prior_diffusion.dpm_solver_sample_loop( + noise=torch.randn_like(y), + model=self.prior, + model_kwargs=[{ + 'y': y + }, { + 'y': zero_y + }], + guide_scale=guide_prior, + dpm_solver_timesteps=timesteps_prior, + order=3, + skip_type='logSNR', + method='singlestep', + t_start=0.9946) + + # decoder + imgs64 = self.decoder_diffusion.dpm_solver_sample_loop( + noise=torch.randn(batch_size, 3, 64, 64).to(device), + model=self.decoder, + model_kwargs=[{ + 'y': x0 + }, { + 'y': torch.zeros_like(x0) + }], + guide_scale=guide_64, + percentile=0.995, + dpm_solver_timesteps=timesteps_64, + order=3, + skip_type='logSNR', + method='singlestep', + t_start=0.9946).clamp_(-1, 1) + + # upsampler256 + imgs256 = F.interpolate( + imgs64, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + imgs256 = self.upsampler256_diffusion.dpm_solver_sample_loop( + noise=torch.randn_like(imgs256), + model=self.upsampler256, + model_kwargs=[{ + 'y': y, + 'concat': imgs256 + }, { + 'y': zero_y, + 'concat': imgs256 + }], + guide_scale=guide_256, + percentile=0.995, + dpm_solver_timesteps=timesteps_256, + order=3, + skip_type='logSNR', + method='singlestep', + t_start=0.9946).clamp_(-1, 1) + + # upsampler1024 + imgs1024 = F.interpolate( + imgs256, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + imgs1024 = self.upsampler1024_diffusion.dpm_solver_sample_loop( + noise=torch.randn_like(imgs1024), + model=self.upsampler1024, + model_kwargs=[{ + 'y': y, + 'concat': imgs1024 + }, { + 'y': zero_y, + 'concat': imgs1024 + }], + guide_scale=guide_1024, + percentile=0.995, + dpm_solver_timesteps=timesteps_1024, + order=3, + skip_type='logSNR', + method='singlestep', + t_start=None).clamp_(-1, 1) + elif solver == 'ddim': + # prior + x0 = self.prior_diffusion.ddim_sample_loop( + noise=torch.randn_like(y), + model=self.prior, + model_kwargs=[{ + 'y': y + }, { + 'y': zero_y + }], + guide_scale=guide_prior, + ddim_timesteps=timesteps_prior, + eta=eta_prior) + + # decoder + imgs64 = self.decoder_diffusion.ddim_sample_loop( + noise=torch.randn(batch_size, 3, 64, 64).to(device), + model=self.decoder, + model_kwargs=[{ + 'y': x0 + }, { + 'y': torch.zeros_like(x0) + }], + guide_scale=guide_64, + percentile=0.995, + ddim_timesteps=timesteps_64, + eta=eta_64).clamp_(-1, 1) + + # upsampler256 + imgs256 = F.interpolate( + imgs64, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + imgs256 = self.upsampler256_diffusion.ddim_sample_loop( + noise=torch.randn_like(imgs256), + model=self.upsampler256, + model_kwargs=[{ + 'y': y, + 'concat': imgs256 + }, { + 'y': zero_y, + 'concat': imgs256 + }], + guide_scale=guide_256, + percentile=0.995, + ddim_timesteps=timesteps_256, + eta=eta_256).clamp_(-1, 1) + + # upsampler1024 + imgs1024 = F.interpolate( + imgs256, + scale_factor=4.0, + mode='bilinear', + align_corners=False) + imgs1024 = self.upsampler1024_diffusion.ddim_sample_loop( + noise=torch.randn_like(imgs1024), + model=self.upsampler1024, + model_kwargs=[{ + 'y': y, + 'concat': imgs1024 + }, { + 'y': zero_y, + 'concat': imgs1024 + }], + guide_scale=guide_1024, + percentile=0.995, + ddim_timesteps=timesteps_1024, + eta=eta_1024).clamp_(-1, 1) + else: + raise ValueError( + 'currently only supports "ddim" and "dpm-solve" solvers') + + # output ([B, C, H, W] within range [0, 1]) + imgs1024 = imgs1024.add_(1).mul_(255 / 2.0).permute(0, 2, 3, 1).cpu() + imgs1024 = [ + Image.fromarray(np.array(u, dtype=np.uint8)) for u in imgs1024 + ] + return imgs1024 + + +@MODELS.register_module( + Tasks.text_to_image_synthesis, module_name=Models.multi_stage_diffusion) +class MultiStageDiffusionForTextToImageSynthesis(TorchModel): + + def __init__(self, model_dir, device='gpu'): + device = 'gpu' if torch.cuda.is_available() else 'cpu' + super().__init__(model_dir=model_dir, device=device) + model = UnCLIP(model_dir=model_dir) + pretrained_params = torch.load( + osp.join(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), 'cpu') + model.load_state_dict(pretrained_params) + model.eval() + + self.device = create_device(device) + self.model = model.to(self.device) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(input, dict): + raise ValueError( + f'Expected the input to be a dictionary, but got {type(input)}' + ) + if 'text' not in input: + raise ValueError('input should contain "text", but not found') + + # sampling + imgs = self.model.synthesis( + text=input.get('text'), + tokenizer=input.get('tokenizer', 'clip'), + batch_size=input.get('batch_size', 4), + timesteps_prior=input.get('timesteps_prior', 100), + timesteps_64=input.get('timesteps_64', 50), + timesteps_256=input.get('timesteps_256', 20), + timesteps_1024=input.get('timesteps_1024', 20), + guide_prior=input.get('guide_prior', 3.0), + guide_64=input.get('guide_64', 7.0), + guide_256=input.get('guide_256', 3.0), + guide_1024=input.get('guide_1024', 3.0), + eta_prior=input.get('eta_prior', 0.0), + eta_64=input.get('eta_64', 0.0), + eta_256=input.get('eta_256', 0.0), + eta_1024=input.get('eta_1024', 0.0), + solver=input.get('solver', 'dpm-solver')) + imgs = [np.array(u)[..., ::-1] for u in imgs] + return imgs diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/prior.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/prior.py new file mode 100644 index 0000000000000000000000000000000000000000..9f4ef2d5971d59c294c3dba61193adb0982d28c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/prior.py @@ -0,0 +1,170 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['Prior'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = math.pow(self.head_dim, -0.25) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.proj = nn.Linear(dim, dim) + + def forward(self, x, mask): + b, l, n, c = *x.shape[:2], self.num_heads, self.head_dim + + # compute query, key, value + q, k, v = self.to_qkv(x).view(b, l, n * 3, c).chunk(3, dim=2) + + # compute attention + attn = torch.einsum('binc,bjnc->bnij', q * self.scale, k * self.scale) + if mask is not None: + attn = attn.masked_fill(mask[:, :, :l, :l] == 0, float('-inf')) + attn = F.softmax(attn.float(), dim=-1).type(attn.dtype) + + # gather context + x = torch.einsum('bnij,bjnc->binc', attn, v) + x = x.reshape(b, l, -1) + + # output + x = self.proj(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = nn.LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads) + self.norm2 = nn.LayerNorm(dim) + self.ffn = nn.Sequential( + nn.Linear(dim, dim * 4), nn.GELU(), nn.Linear(dim * 4, dim)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.ffn(self.norm2(x)) + return x + + +class Prior(nn.Module): + + def __init__(self, dim=2048, clip_dim=768, num_heads=32, num_layers=24): + super(Prior, self).__init__() + self.dim = dim + self.clip_dim = clip_dim + self.num_heads = num_heads + self.num_layers = num_layers + + # embeddings + self.text_embedding = nn.Sequential( + nn.Linear(clip_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) + self.time_embedding = nn.Sequential( + nn.Linear(dim, dim), nn.SiLU(), nn.Linear(dim, dim)) + self.vision_embedding = nn.Sequential( + nn.Linear(clip_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) + self.eos_embedding = nn.Parameter(torch.zeros(1, 1, dim)) + self.pos_embedding = nn.Parameter(torch.zeros(1, 4, dim)) + + # transformer + self.blocks = nn.ModuleList( + [AttentionBlock(dim, num_heads) for _ in range(num_layers)]) + self.norm = nn.LayerNorm(dim) + + # head + self.head = nn.Linear(dim, clip_dim) + + # causal attention mask + self.register_buffer('attn_mask', torch.tril(torch.ones(1, 1, 4, 4))) + + # initialize weights + self.init_weights() + + def forward(self, x, t, y): + r"""x: [B, C]. + t: [B]. + y: [B, C]. + """ + b = x.size(0) + + # embeddings of shape [B, L + 4, C] + u1 = sinusoidal_embedding(t, self.dim) + u2 = [ + self.text_embedding(y).unsqueeze(1), + self.time_embedding(u1).unsqueeze(1), + self.vision_embedding(x).unsqueeze(1), + self.eos_embedding.repeat(b, 1, 1) + ] + x = self.pos_embedding + torch.cat(u2, dim=1) + + # transformer + for block in self.blocks: + x = block(x, self.attn_mask) + x = self.norm(x) + + # head + x = self.head(x[:, -1]) + return x + + def init_weights(self): + std = 0.02 / math.sqrt(2.0 * self.num_layers) + for name, m in self.named_modules(): + if name.endswith('attn.proj') or name.endswith('ffn.2'): + # smaller std for output layers + nn.init.normal_(m.weight, std=std) + nn.init.zeros_(m.bias) + elif isinstance(m, (nn.Linear, nn.Embedding)): + nn.init.normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.LayerNorm): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + + def param_groups(self): + groups = [{ + 'params': [ + p for n, p in self.named_parameters() + if 'norm' in n or n.endswith('bias') + ], + 'weight_decay': + 0.0 + }, { + 'params': [ + p for n, p in self.named_parameters() + if not ('norm' in n or n.endswith('bias')) + ] + }] + return groups diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/tokenizer.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..f8c9775ba9b2f49a8ec8905317db6468625983d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/tokenizer.py @@ -0,0 +1,200 @@ +# Part of the implementation is borrowed and modified from CLIP, publicly available at https://github.com/openai/CLIP. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import gzip +import html +from functools import lru_cache + +import ftfy +import regex as re +import torch +from transformers import AutoTokenizer + +__all__ = ['CLIPTokenizer', 'XGLMTokenizer'] + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + +class CLIPTokenizer(object): + r"""CLIP tokenizer, adapted from https://github.com/openai/CLIP. + """ + + def __init__(self, bpe_path, length=77): + self.bpe_path = bpe_path + self.length = length + + # init tokenizer + self.tokenizer = SimpleTokenizer(bpe_path=bpe_path) + self.sos_token = self.tokenizer.encoder['<|startoftext|>'] + self.eos_token = self.tokenizer.encoder['<|endoftext|>'] + self.vocab_size = len(self.tokenizer.encoder) + + def __call__(self, sequence): + if isinstance(sequence, str): + return torch.LongTensor(self._tokenizer(sequence)) + elif isinstance(sequence, list): + return torch.LongTensor([self._tokenizer(u) for u in sequence]) + else: + raise TypeError( + f'Expected the "sequence" to be a string or a list, but got {type(sequence)}' + ) + + def _tokenizer(self, text): + tokens = self.tokenizer.encode(text)[:self.length - 2] + tokens = [self.sos_token] + tokens + [self.eos_token] + tokens = tokens + [0] * (self.length - len(tokens)) + return tokens + + +class XGLMTokenizer(object): + r"""A wrapper of HuggingFace's XGLM tokenizer. + """ + + def __init__(self, model_dir, length=77, **kwargs): + self.length = length + self.tokenizer = AutoTokenizer.from_pretrained(model_dir, **kwargs) + self.vocab_size = self.tokenizer.vocab_size + + def __call__(self, sequence, **kwargs): + _kwargs = { + 'return_tensors': 'pt', + 'padding': 'max_length', + 'truncation': True, + 'max_length': self.length + } + _kwargs.update(**kwargs) + tokens = self.tokenizer(sequence, **_kwargs) + return tokens.input_ids, tokens.attention_mask diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/upsampler.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/upsampler.py new file mode 100644 index 0000000000000000000000000000000000000000..a292edae27672c11bddb9c59fc69cdecc8a58c87 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/upsampler.py @@ -0,0 +1,466 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['Upsampler256', 'Upsampler1024'] + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, scale_factor, use_conv=False): + assert scale_factor in [0.5, 1.0, 2.0] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.scale_factor = scale_factor + self.use_conv = use_conv + + # layers + if scale_factor == 2.0: + self.resample = nn.Sequential( + nn.Upsample(scale_factor=scale_factor, mode='nearest'), + nn.Conv2d(in_dim, out_dim, 3, padding=1) + if use_conv else nn.Identity()) + elif scale_factor == 0.5: + self.resample = nn.Conv2d( + in_dim, out_dim, 3, stride=2, + padding=1) if use_conv else nn.AvgPool2d( + kernel_size=2, stride=2) + else: + self.resample = nn.Identity() + + def forward(self, x): + return self.resample(x) + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + scale_factor=1.0, + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.scale_factor = scale_factor + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, scale_factor, use_conv=False) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e): + identity = self.resample(x) + x = self.layer1[-1](self.resample(self.layer1[:-1](x))) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class Upsampler256(nn.Module): + + def __init__(self, + in_dim=6, + dim=320, + y_dim=768, + context_dim=512, + out_dim=3, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 8], + resblock_resample=True, + use_scale_shift_norm=True, + dropout=0.1): + embed_dim = dim * 4 + super(Upsampler256, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.resblock_resample = resblock_resample + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.y_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.context_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, context_dim * 4)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim, embed_dim, out_dim, + use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + self.encoder.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample( + out_dim, out_dim, 0.5, use_conv=True) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + AttentionBlock(out_dim, context_dim, num_heads, head_dim), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + if scale in attn_scales: + block.append( + AttentionBlock(out_dim, context_dim, num_heads, + head_dim)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample( + out_dim, out_dim, 2.0, use_conv=True) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y, concat): + # embeddings + x = torch.cat([x, concat], dim=1) + e = self.time_embedding(sinusoidal_embedding( + t, self.dim)) + self.y_embedding(y) + context = self.context_embedding(y).view(-1, 4, self.context_dim) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, e, context) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, e, context) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, e, context) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, e, context): + if isinstance(module, ResidualBlock): + x = module(x, e) + elif isinstance(module, AttentionBlock): + x = module(x, context) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context) + else: + x = module(x) + return x + + +class Upsampler1024(nn.Module): + + def __init__(self, + in_dim=6, + dim=192, + y_dim=768, + out_dim=3, + dim_mult=[1, 1, 2, 2, 4, 4], + num_res_blocks=2, + resblock_resample=True, + use_scale_shift_norm=True, + dropout=0.0): + embed_dim = dim * 4 + super(Upsampler1024, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.resblock_resample = resblock_resample + self.use_scale_shift_norm = use_scale_shift_norm + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embedding + self.time_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.y_embedding = nn.Sequential( + nn.Linear(y_dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + + # encoder + self.encoder = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual block + block = nn.ModuleList([ + ResidualBlock(in_dim, embed_dim, out_dim, + use_scale_shift_norm, 1.0, dropout) + ]) + shortcut_dims.append(out_dim) + in_dim = out_dim + self.encoder.append(block) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + if resblock_resample: + downsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 0.5, + dropout) + else: + downsample = Resample( + out_dim, out_dim, 0.5, use_conv=True) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.encoder.append(downsample) + + # middle + self.middle = nn.ModuleList([ + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout), + ResidualBlock(out_dim, embed_dim, out_dim, use_scale_shift_norm, + 1.0, dropout) + ]) + + # decoder + self.decoder = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual block + block = nn.ModuleList([ + ResidualBlock(in_dim + shortcut_dims.pop(), embed_dim, + out_dim, use_scale_shift_norm, 1.0, dropout) + ]) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + if resblock_resample: + upsample = ResidualBlock(out_dim, embed_dim, out_dim, + use_scale_shift_norm, 2.0, + dropout) + else: + upsample = Resample( + out_dim, out_dim, 2.0, use_conv=True) + scale *= 2.0 + block.append(upsample) + self.decoder.append(block) + + # head + self.head = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.head[-1].weight) + + def forward(self, x, t, y, concat): + # embedding + x = torch.cat([x, concat], dim=1) + e = self.time_embedding(sinusoidal_embedding( + t, self.dim)) + self.y_embedding(y) + + # encoder + xs = [] + for block in self.encoder: + x = self._forward_single(block, x, e) + xs.append(x) + + # middle + for block in self.middle: + x = self._forward_single(block, x, e) + + # decoder + for block in self.decoder: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single(block, x, e) + + # head + x = self.head(x) + return x + + def _forward_single(self, module, x, e): + if isinstance(module, ResidualBlock): + x = module(x, e) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/xglm.py b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/xglm.py new file mode 100644 index 0000000000000000000000000000000000000000..8ff39d99e17c16c927165254a869e2930cbf2ba8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/multi_stage_diffusion/xglm.py @@ -0,0 +1,206 @@ +# Part of the implementation is borrowed and modified from HuggingFace XGLM, +# publicly available at https://github.com/huggingface/transformers. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['XGLM'] + + +def sinusoidal_embedding(seq_len, dim, pad_token=None): + half = dim // 2 + sinusoid = torch.outer( + torch.arange(seq_len, dtype=torch.float32), + torch.pow(10000, + -torch.arange(half, dtype=torch.float32).div(half - 1))) + x = torch.cat([torch.sin(sinusoid), torch.cos(sinusoid)], dim=1) + if dim % 2 == 1: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + if pad_token is not None: + x[pad_token, :] = 0 + return x + + +class SinusoidalEmbedding(nn.Module): + + def __init__(self, seq_len, dim, pad_token): + super(SinusoidalEmbedding, self).__init__() + self.seq_len = seq_len + self.dim = dim + self.pad_token = pad_token + self.register_buffer('weight', + sinusoidal_embedding(seq_len + 2, dim, pad_token)) + + def forward(self, tokens): + mask = tokens.ne(self.pad_token).long() + indices = torch.cumsum(mask, dim=1) * mask + self.pad_token + pos_embeds = self.weight.index_select(0, indices.view(-1)).view( + *tokens.shape, -1) + return pos_embeds + + +class GELU(nn.Module): + + def forward(self, x): + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, dropout=0.1): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.q = nn.Linear(dim, dim) + self.k = nn.Linear(dim, dim) + self.v = nn.Linear(dim, dim) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, mask=None): + r"""x: [B, L, C]. + mask: [B, *, L, L] or None. + """ + b, l, n, c = *x.shape[:2], self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, l, n, c) + k = self.k(x).view(b, l, n, c) + v = self.v(x).view(b, l, n, c) + + # compute attention + attn = self.scale * torch.einsum('binc,bjnc->bnij', q, k) + if mask is not None: + attn = attn.masked_fill(mask == 0, float('-inf')) + attn = F.softmax(attn, dim=-1) + attn = self.dropout(attn) + + # gather context + x = torch.einsum('bnij,bjnc->binc', attn, v) + x = x.reshape(b, l, -1) + + # output + x = self.o(x) + x = self.dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, ffn_dim, ffn_act, num_heads, dropout=0.1): + assert ffn_act in ['gelu', 'relu'] + super(AttentionBlock, self).__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.ffn_act = ffn_act + self.num_heads = num_heads + + # layers + self.norm1 = nn.LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads, dropout) + self.norm2 = nn.LayerNorm(dim) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), + GELU() if ffn_act == 'gelu' else nn.ReLU(inplace=True), + nn.Linear(ffn_dim, dim), nn.Dropout(dropout)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.ffn(self.norm2(x)) + return x + + +class XGLM(nn.Module): + r"""A multilingual GPT model with an embedding head. + """ + + def __init__(self, + vocab_size=256008, + max_seq_len=2048, + dim=1024, + ffn_dim=4096, + ffn_act='gelu', + embed_dim=768, + num_heads=16, + num_layers=24, + pad_token=1, + dropout=0.1): + super(XGLM, self).__init__() + self.vocab_size = vocab_size + self.max_seq_len = max_seq_len + self.dim = dim + self.ffn_dim = ffn_dim + self.ffn_act = ffn_act + self.embed_dim = embed_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.pad_token = pad_token + self.scale = math.sqrt(dim) # rescale token embedings + + # layers + self.token_embedding = nn.Embedding(vocab_size, dim, pad_token) + self.pos_embedding = SinusoidalEmbedding(max_seq_len, dim, pad_token) + self.eos_embedding = nn.Parameter(torch.randn(1, 1, dim)) + self.dropout = nn.Dropout(dropout) + self.blocks = nn.ModuleList([ + AttentionBlock(dim, ffn_dim, ffn_act, num_heads, dropout) + for _ in range(num_layers) + ]) + self.norm = nn.LayerNorm(dim) + self.head = nn.Linear(dim, embed_dim, bias=False) + + # causal attention mask + self.register_buffer( + 'attn_mask', + torch.tril(torch.ones(1, 1, 1 + max_seq_len, 1 + max_seq_len))) + + # init weights + self.apply(self.init_weights) + + def forward(self, tokens, mask=None): + r"""tokens: [B, L]. + mask: [B, L]. + """ + b, seq_len = tokens.size(0), 1 + tokens.size(1) + + # embeddings + x = self.scale * self.token_embedding(tokens) + x = torch.cat([x, self.eos_embedding.repeat(b, 1, 1)], dim=1) + # x = x + self.pos_embedding(tokens) + x = self.dropout(x) + + # attention mask + if mask is None: + mask = self.attn_mask[:, :, :seq_len, :seq_len].repeat(b, 1, 1, 1) + else: + mask = self.attn_mask[:, :, :seq_len, :seq_len] * torch.cat( + [mask, torch.zeros_like(mask[:, :1])], dim=1).view( + b, 1, 1, seq_len) + + # transformer + for block in self.blocks: + x = block(x, mask) + x = self.norm(x) + + # head + logits = self.head(x[:, -1]) + return logits + + def init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Embedding): + nn.init.normal_(m.weight, std=0.02) + if m.padding_idx is not None: + nn.init.zeros_(m.weight[m.padding_idx]) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/__init__.py b/runtime/Lib/site_packages/models/multi_modal/ofa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..da2d09fb0a9c0619d6829e94aa39e842bed3c447 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .modeling_mmspeech import MMSpeechModel +from .modeling_ofa import OFADecoder, OFAEncoder, OFAModel, OFAPreTrainedModel +from .tokenization_ofa import OFATokenizer, OFATokenizerZH +from .tokenization_ofa_fast import OFATokenizerFast, OFATokenizerZHFast diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/adaptor/__init__.py b/runtime/Lib/site_packages/models/multi_modal/ofa/adaptor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_mmspeech.py b/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_mmspeech.py new file mode 100644 index 0000000000000000000000000000000000000000..48240877d2a57aff47edc39c70d3f3cc9a3b9138 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_mmspeech.py @@ -0,0 +1,349 @@ +# Copyright 2022 Alibaba Group and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MMSpeech model configuration""" +import warnings + +from transformers import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger() + + +class MMSpeechConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`~OFAModel`]. It is used to instantiate an OFA + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the OFA [ofa-base](https://huggingface.co/ofa-base) + architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50265): + Vocabulary size of the OFA model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`~OFAModel`] or [`~TFOFAModel`]. + d_model (`int`, *optional*, defaults to 1024): + Dimension of the layers and the pooler layer. + encoder_layers (`int`, *optional*, defaults to 12): + Number of encoder layers. + decoder_layers (`int`, *optional*, defaults to 12): + Number of decoder layers. + encoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + encoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + dropout (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + classifier_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for classifier. + max_position_embeddings (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + init_std (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + encoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + is_encoder_decoder (`bool`, *optional*, defaults to `True`): + Whether or not the model is used as an encoder/decoder. + scale_embedding (`bool`, *optional*, defaults to `False`): + Whether or not to scale the embedding. If True, embedding = Math.qrt(d_model) * embedding + pad_token_id (`int`, *optional*, defaults to `1`): + The id of the _padding_ token. + bos_token_id (`int`, *optional*, defaults to `0`): + The id of the _beginning-of-stream_ token. + decoder_start_token_id (`int`, *optional*, defaults to `0`): + If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token. + eos_token_id (`int`, *optional*, defaults to `2`): + The id of the _end-of-stream_ token. + forced_eos_token_id (`int`, *optional*, defaults to `2`): + The id of the token to force as the last generated token when `max_length` is reached. + encoder_normalize_before (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization before the self attention and fc layer within encoder layer + decoder_normalize_before (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization before the self attention and fc layer within decoder layer + normformer (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization between the self attention layers and fc layer within + encoder&decoder layer + encoder_drop_path_rate (`float`, *optional*, defaults to `0.0`): + The drop path rate using in the encoder. see more about drop path [drop path](https://arxiv.org/abs/1605.07648) + decoder_drop_path_rate (`float`, *optional*, defaults to `0.0`): + The drop path rate using in the decoder. see more about drop path [drop path](https://arxiv.org/abs/1605.07648) + layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization for text input embedding in encoder and decoder. + patch_layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization for image patch input embedding in encoder and decoder. + entangle_position_embedding (`bool`, *optional*, defaults to `False`): + Whether or not to entangle position embedding to input embedding. + resnet_type (`str`, *optional*, defaults to `"resnet101"`): + The image encoder's type in OFA, only works when use_ofasys=False. `"resnet18"`, `"resnet34"`, + `"resnet50"`, `"resnet101"` and `"resnet152"` are supported. + resnet_model_path (`str`, *optional*, defaults to `None`): + The path where can load resnet model. If None, will use random initialized weights. + resnet_drop_path_rate + The drop path rate using in resnet for image encoding, see more about drop path + [drop path](https://arxiv.org/abs/1605.07648) + token_bucket_size (`int`, *optional*, defaults to `256`): + The number of token buckets to use for each attention layer. + image_bucket_size (`int`, *optional*, defaults to `42`): + The number of image buckets to use for each attention layer. + add_type_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to add type embedding to the input while encoding. So far, type means the type of modality, + and only Text&Image modalities is supported, `0`=Text, `1`=Image + share_decoder_input_output_embed (`bool`, *optional*, defaults to `True`): + Whether or not to share the input embedding table as the weights the output projection in decoder. If False, + using a new linear projection. + attn_scale_factor (`float`, *optional*, defaults to `2.0`): + The position embedding scaling factor. If it works, + position_embedding = position_embedding * float(d_model / num_attention_heads * attn_scale_factor)**-0.5 + code_layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to user layer normalization for code generation + code_image_size (`int`, *optional*, defaults to `128`): + Image size of generated images. Also used in calculating the image's position id for attention bias. + interpolate_position (`bool`, *optional*, defaults to `False`): + Deprecated now, will be deleted in next version. + orig_patch_image_size (`int`, *optional*, defaults to `224`): + Deprecated now, will be deleted in next version. + share_attn_bias (`bool`, *optional*, defaults to `False`): + Whether or not to share attn_bias cross transformer layers + use_image_feature (`bool`, *optional*, defaults to `True`): + Whether or not the model have image modality. + disable_entangle (`bool`, *optional*, defaults to `False`): + Whether or not to disable the entangle relative configs. + use_ofasys (`bool`, *optional*, defaults to `False`): + Whether or not the model is come from OFA-Sys. If True, the model structure will be some differences from OFA + vit_type (`str`, *optional*, defaults to `"vit_base"`): + The image encoder's type in OFA-Sys, only works when use_ofasys=True. `"vit_base"`, `"vit_large"`, + `"vit_large_336"` and `"vit_huge"` are supported. + vit_drop_path_rate + The drop path rate using the image encoder vit. see more about drop path + [drop path](https://arxiv.org/abs/1605.07648) + """ # noqa + + model_type = 'ofa' + keys_to_ignore_at_inference = ['past_key_values'] + + attribute_map = { + 'num_attention_heads': 'encoder_attention_heads', + 'hidden_size': 'd_model' + } + + def __init__(self, + vocab_size=59457, + max_position_embeddings=1024, + encoder_layers=4, + encoder_ffn_dim=512 * 4, + encoder_attention_heads=8, + decoder_layers=4, + decoder_ffn_dim=512 * 4, + decoder_attention_heads=8, + encoder_layerdrop=0.0, + decoder_layerdrop=0.0, + use_cache=True, + is_encoder_decoder=True, + activation_function='gelu', + d_model=512, + dropout=0.1, + attention_dropout=0.0, + activation_dropout=0.0, + init_std=0.02, + classifier_dropout=0.0, + scale_embedding=False, + pad_token_id=1, + bos_token_id=0, + decoder_start_token_id=0, + eos_token_id=2, + forced_eos_token_id=2, + encoder_normalize_before=True, + decoder_normalize_before=True, + normformer=True, + encoder_drop_path_rate=0.0, + decoder_drop_path_rate=0.0, + layernorm_embedding=True, + patch_layernorm_embedding=True, + resnet_type='resnet101', + resnet_model_path=None, + resnet_drop_path_rate=0.0, + token_bucket_size=256, + image_bucket_size=42, + add_type_embedding=True, + share_decoder_input_output_embed=True, + attn_scale_factor=2., + code_layernorm_embedding=False, + code_image_size=128, + entangle_position_embedding=False, + interpolate_position=False, + orig_patch_image_size=224, + share_attn_bias=False, + use_image_feature=True, + disable_entangle=False, + use_ofasys=False, + vit_type='vit_base', + vit_drop_path_rate=0.0, + use_gamma_feature=False, + gamma=1.0, + exclude_mlp=True, + temperature_init_value=None, + remove_decoder_type_embedding=False, + mlp_dim=512, + required_seq_len_multiple=2, + encoder_pos_conv_depth=5, + encoder_conv_pos=95, + encoder_conv_pos_groups=16, + encoder_max_positions=100000, + phone_vocab_size=141, + audio_mask_prob=0.65, + audio_mask_selection='static', + audio_mask_other=0, + audio_mask_length=10, + audio_no_mask_overlap=False, + audio_mask_min_space=1, + audio_mask_channel_prob=0.0, + audio_mask_channel_before=False, + audio_mask_channel_selection='static', + audio_mask_channel_other=0, + audio_mask_channel_length=10, + audio_no_mask_channel_overlap=False, + audio_mask_channel_min_space=1, + encoder_dropout_input=0.0, + encoder_dropout_features=0.0, + phone_dict_size=124, + **kwargs): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.d_model = d_model + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.decoder_ffn_dim = decoder_ffn_dim + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.classifier_dropout = classifier_dropout + self.use_cache = use_cache + self.num_hidden_layers = encoder_layers + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.encoder_normalize_before = encoder_normalize_before + self.decoder_normalize_before = decoder_normalize_before + self.normformer = normformer + self.encoder_drop_path_rate = encoder_drop_path_rate + self.decoder_drop_path_rate = decoder_drop_path_rate + self.layernorm_embedding = layernorm_embedding + self.patch_layernorm_embedding = patch_layernorm_embedding + self.resnet_type = resnet_type + self.resnet_model_path = resnet_model_path + self.resnet_drop_path_rate = resnet_drop_path_rate + self.token_bucket_size = token_bucket_size + self.image_bucket_size = image_bucket_size + self.add_type_embedding = add_type_embedding + self.share_decoder_input_output_embed = share_decoder_input_output_embed + self.attn_scale_factor = attn_scale_factor + self.code_layernorm_embedding = code_layernorm_embedding + self.code_image_size = code_image_size + self.entangle_position_embedding = entangle_position_embedding + self.interpolate_position = interpolate_position + self.orig_patch_image_size = orig_patch_image_size + + self.share_attn_bias = share_attn_bias + self.use_image_feature = use_image_feature + self.disable_entangle = disable_entangle + self.use_ofasys = use_ofasys + self.vit_type = vit_type + self.vit_drop_path_rate = vit_drop_path_rate + self.use_gamma_feature = use_gamma_feature + + # add some new features from ofa + self.use_gamma_feature = use_gamma_feature + self.gamma = gamma + self.exclude_mlp = exclude_mlp + self.temperature_init_value = temperature_init_value + self.remove_decoder_type_embedding = remove_decoder_type_embedding + self.mlp_dim = mlp_dim + + # FP16 optimization + self.required_seq_len_multiple = required_seq_len_multiple + + # encoder_pos_conv + self.encoder_pos_conv_depth = encoder_pos_conv_depth + self.encoder_conv_pos = encoder_conv_pos + self.encoder_conv_pos_groups = encoder_conv_pos_groups + self.encoder_max_positions = encoder_max_positions + + # phone + self.phone_vocab_size = phone_vocab_size + + # audio_mask + self.audio_mask_prob = audio_mask_prob + self.audio_mask_selection = audio_mask_selection + self.audio_mask_other = audio_mask_other + self.audio_mask_length = audio_mask_length + self.audio_no_mask_overlap = audio_no_mask_overlap + self.audio_mask_min_space = audio_mask_min_space + + self.audio_mask_channel_prob = audio_mask_channel_prob + self.audio_mask_channel_before = audio_mask_channel_before + self.audio_mask_channel_selection = audio_mask_channel_selection + self.audio_mask_channel_other = audio_mask_channel_other + self.audio_mask_channel_length = audio_mask_channel_length + self.audio_no_mask_channel_overlap = audio_no_mask_channel_overlap + self.audio_mask_channel_min_space = audio_mask_channel_min_space + + # audio encoder + self.encoder_dropout_input = encoder_dropout_input + self.encoder_dropout_features = encoder_dropout_features + + self.phone_dict_size = phone_dict_size + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + decoder_start_token_id=decoder_start_token_id, + forced_eos_token_id=forced_eos_token_id, + **kwargs, + ) + + # ensure backward compatibility for BART CNN models + if self.forced_bos_token_id is None and kwargs.get( + 'force_bos_token_to_be_generated', False): + self.forced_bos_token_id = self.bos_token_id + warnings.warn( + f'Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. ' + 'The config can simply be saved and uploaded again to be fixed.' + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_ofa.py b/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_ofa.py new file mode 100644 index 0000000000000000000000000000000000000000..c520db3408e4a394867e17cbb5e7f283ae4647fe --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/configuration_ofa.py @@ -0,0 +1,299 @@ +# Copyright 2022 Alibaba Group and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" OFA model configuration""" +import warnings + +from transformers import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger() + +OFA_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'ofa-medium': 'https://huggingface.co/ofa-base/resolve/main/config.json', + # OFA models are implemeted to be compatible with both huggingface + # and modelscope frameworks. For all OFA models available on huggingface, + # please refer to https://huggingface.co/models?filter=ofa +} + + +class OFAConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`~OFAModel`]. It is used to instantiate an OFA + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the OFA [ofa-base](https://huggingface.co/ofa-base) + architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50265): + Vocabulary size of the OFA model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`~OFAModel`] or [`~TFOFAModel`]. + d_model (`int`, *optional*, defaults to 1024): + Dimension of the layers and the pooler layer. + encoder_layers (`int`, *optional*, defaults to 12): + Number of encoder layers. + decoder_layers (`int`, *optional*, defaults to 12): + Number of decoder layers. + encoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + encoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimension of the "intermediate" (often named feed-forward) layer in decoder. + activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + dropout (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + classifier_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for classifier. + max_position_embeddings (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + init_std (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + encoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + is_encoder_decoder (`bool`, *optional*, defaults to `True`): + Whether or not the model is used as an encoder/decoder. + scale_embedding (`bool`, *optional*, defaults to `False`): + Whether or not to scale the embedding. If True, embedding = Math.qrt(d_model) * embedding + pad_token_id (`int`, *optional*, defaults to `1`): + The id of the _padding_ token. + bos_token_id (`int`, *optional*, defaults to `0`): + The id of the _beginning-of-stream_ token. + decoder_start_token_id (`int`, *optional*, defaults to `0`): + If an encoder-decoder model starts decoding with a different token than _bos_, the id of that token. + eos_token_id (`int`, *optional*, defaults to `2`): + The id of the _end-of-stream_ token. + forced_eos_token_id (`int`, *optional*, defaults to `2`): + The id of the token to force as the last generated token when `max_length` is reached. + encoder_normalize_before (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization before the self attention and fc layer within encoder layer + decoder_normalize_before (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization before the self attention and fc layer within decoder layer + normformer (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization between the self attention layers and fc layer within + encoder&decoder layer + encoder_drop_path_rate (`float`, *optional*, defaults to `0.0`): + The drop path rate using in the encoder. see more about drop path [drop path](https://arxiv.org/abs/1605.07648) + decoder_drop_path_rate (`float`, *optional*, defaults to `0.0`): + The drop path rate using in the decoder. see more about drop path [drop path](https://arxiv.org/abs/1605.07648) + layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization for text input embedding in encoder and decoder. + patch_layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to use layer normalization for image patch input embedding in encoder and decoder. + entangle_position_embedding (`bool`, *optional*, defaults to `False`): + Whether or not to entangle position embedding to input embedding. + resnet_type (`str`, *optional*, defaults to `"resnet101"`): + The image encoder's type in OFA, only works when use_ofasys=False. `"resnet18"`, `"resnet34"`, + `"resnet50"`, `"resnet101"` and `"resnet152"` are supported. + resnet_model_path (`str`, *optional*, defaults to `None`): + The path where can load resnet model. If None, will use random initialized weights. + resnet_drop_path_rate + The drop path rate using in resnet for image encoding, see more about drop path + [drop path](https://arxiv.org/abs/1605.07648) + token_bucket_size (`int`, *optional*, defaults to `256`): + The number of token buckets to use for each attention layer. + image_bucket_size (`int`, *optional*, defaults to `42`): + The number of image buckets to use for each attention layer. + add_type_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to add type embedding to the input while encoding. So far, type means the type of modality, + and only Text&Image modalities is supported, `0`=Text, `1`=Image + share_decoder_input_output_embed (`bool`, *optional*, defaults to `True`): + Whether or not to share the input embedding table as the weights the output projection in decoder. If False, + using a new linear projection. + attn_scale_factor (`float`, *optional*, defaults to `2.0`): + The position embedding scaling factor. If it works, + position_embedding = position_embedding * float(d_model / num_attention_heads * attn_scale_factor)**-0.5 + code_layernorm_embedding (`bool`, *optional*, defaults to `True`): + Whether or not to user layer normalization for code generation + code_image_size (`int`, *optional*, defaults to `128`): + Image size of generated images. Also used in calculating the image's position id for attention bias. + interpolate_position (`bool`, *optional*, defaults to `False`): + Deprecated now, will be deleted in next version. + orig_patch_image_size (`int`, *optional*, defaults to `224`): + Deprecated now, will be deleted in next version. + share_attn_bias (`bool`, *optional*, defaults to `False`): + Whether or not to share attn_bias cross transformer layers + use_image_feature (`bool`, *optional*, defaults to `True`): + Whether or not the model have image modality. + disable_entangle (`bool`, *optional*, defaults to `False`): + Whether or not to disable the entangle relative configs. + use_ofasys (`bool`, *optional*, defaults to `False`): + Whether or not the model is come from OFA-Sys. If True, the model structure will be some differences from OFA + vit_type (`str`, *optional*, defaults to `"vit_base"`): + The image encoder's type in OFA-Sys, only works when use_ofasys=True. `"vit_base"`, `"vit_large"`, + `"vit_large_336"` and `"vit_huge"` are supported. + vit_drop_path_rate + The drop path rate using the image encoder vit. see more about drop path + [drop path](https://arxiv.org/abs/1605.07648) + """ # noqa + + model_type = 'ofa' + keys_to_ignore_at_inference = ['past_key_values'] + + attribute_map = { + 'num_attention_heads': 'encoder_attention_heads', + 'hidden_size': 'd_model' + } + + def __init__(self, + vocab_size=59457, + max_position_embeddings=1024, + encoder_layers=4, + encoder_ffn_dim=512 * 4, + encoder_attention_heads=8, + decoder_layers=4, + decoder_ffn_dim=512 * 4, + decoder_attention_heads=8, + encoder_layerdrop=0.0, + decoder_layerdrop=0.0, + use_cache=True, + is_encoder_decoder=True, + activation_function='gelu', + d_model=512, + dropout=0.1, + attention_dropout=0.0, + activation_dropout=0.0, + init_std=0.02, + classifier_dropout=0.0, + scale_embedding=False, + pad_token_id=1, + bos_token_id=0, + decoder_start_token_id=0, + eos_token_id=2, + forced_eos_token_id=2, + encoder_normalize_before=True, + decoder_normalize_before=True, + normformer=True, + encoder_drop_path_rate=0.0, + decoder_drop_path_rate=0.0, + layernorm_embedding=True, + patch_layernorm_embedding=True, + resnet_type='resnet101', + resnet_model_path=None, + resnet_drop_path_rate=0.0, + token_bucket_size=256, + image_bucket_size=42, + add_type_embedding=True, + share_decoder_input_output_embed=True, + attn_scale_factor=2., + code_layernorm_embedding=True, + code_image_size=128, + entangle_position_embedding=False, + interpolate_position=False, + orig_patch_image_size=224, + share_attn_bias=False, + use_image_feature=True, + disable_entangle=False, + use_ofasys=False, + vit_type='vit_base', + vit_drop_path_rate=0.0, + use_gamma_feature=False, + gamma=1.0, + exclude_mlp=True, + temperature_init_value=None, + remove_decoder_type_embedding=False, + mlp_dim=512, + **kwargs): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.d_model = d_model + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.decoder_ffn_dim = decoder_ffn_dim + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.classifier_dropout = classifier_dropout + self.use_cache = use_cache + self.num_hidden_layers = encoder_layers + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.encoder_normalize_before = encoder_normalize_before + self.decoder_normalize_before = decoder_normalize_before + self.normformer = normformer + self.encoder_drop_path_rate = encoder_drop_path_rate + self.decoder_drop_path_rate = decoder_drop_path_rate + self.layernorm_embedding = layernorm_embedding + self.patch_layernorm_embedding = patch_layernorm_embedding + self.resnet_type = resnet_type + self.resnet_model_path = resnet_model_path + self.resnet_drop_path_rate = resnet_drop_path_rate + self.token_bucket_size = token_bucket_size + self.image_bucket_size = image_bucket_size + self.add_type_embedding = add_type_embedding + self.share_decoder_input_output_embed = share_decoder_input_output_embed + self.attn_scale_factor = attn_scale_factor + self.code_layernorm_embedding = code_layernorm_embedding + self.code_image_size = code_image_size + self.entangle_position_embedding = entangle_position_embedding + self.interpolate_position = interpolate_position + self.orig_patch_image_size = orig_patch_image_size + + self.share_attn_bias = share_attn_bias + self.use_image_feature = use_image_feature + self.disable_entangle = disable_entangle + self.use_ofasys = use_ofasys + self.vit_type = vit_type + self.vit_drop_path_rate = vit_drop_path_rate + + # add some new features from ofa + self.use_gamma_feature = use_gamma_feature + self.gamma = gamma + self.exclude_mlp = exclude_mlp + self.temperature_init_value = temperature_init_value + self.remove_decoder_type_embedding = remove_decoder_type_embedding + self.mlp_dim = mlp_dim + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + decoder_start_token_id=decoder_start_token_id, + forced_eos_token_id=forced_eos_token_id, + **kwargs, + ) + + # ensure backward compatibility for BART CNN models + if self.forced_bos_token_id is None and kwargs.get( + 'force_bos_token_to_be_generated', False): + self.forced_bos_token_id = self.bos_token_id + warnings.warn( + f'Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. ' + 'The config can simply be saved and uploaded again to be fixed.' + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/__init__.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/incremental_decoding_utils.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/incremental_decoding_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..db0df9b27f833be26d704fdc628ab60c3a62a372 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/incremental_decoding_utils.py @@ -0,0 +1,51 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/facebookresearch/fairseq/blob/main/LICENSE + +import uuid +from typing import Dict, Optional + +from torch import Tensor + + +class FairseqIncrementalState(object): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.init_incremental_state() + + def init_incremental_state(self): + self._incremental_state_id = str(uuid.uuid4()) + + def _get_full_incremental_state_key(self, key: str) -> str: + return '{}.{}'.format(self._incremental_state_id, key) + + def get_incremental_state( + self, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]], + key: str, + ) -> Optional[Dict[str, Optional[Tensor]]]: + """Helper for getting incremental state for an nn.Module.""" + full_key = self._get_full_incremental_state_key(key) + if incremental_state is None or full_key not in incremental_state: + return None + return incremental_state[full_key] + + def set_incremental_state( + self, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]], + key: str, + value: Dict[str, Optional[Tensor]], + ) -> Optional[Dict[str, Dict[str, Optional[Tensor]]]]: + """Helper for setting incremental state for an nn.Module.""" + if incremental_state is not None: + full_key = self._get_full_incremental_state_key(key) + incremental_state[full_key] = value + return incremental_state + + +def with_incremental_state(cls): + cls.__bases__ = (FairseqIncrementalState, ) + tuple( + b for b in cls.__bases__ if b != FairseqIncrementalState) + return cls diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/multihead_attention.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/multihead_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..9101d52d8b824899930515d6fe2125e61bd49922 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/multihead_attention.py @@ -0,0 +1,510 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/facebookresearch/fairseq/blob/main/LICENSE + +import math +from typing import Dict, Optional, Tuple + +import torch +import torch.nn.functional as F +from fairseq import utils +from fairseq.incremental_decoding_utils import with_incremental_state +from fairseq.modules.fairseq_dropout import FairseqDropout +from fairseq.modules.quant_noise import quant_noise +from torch import Tensor, nn +from torch.nn import Parameter + + +@with_incremental_state +class MultiheadAttention(nn.Module): + """Multi-headed attention. + + See "Attention Is All You Need" for more details. + """ + + def __init__( + self, + embed_dim, + num_heads, + kdim=None, + vdim=None, + dropout=0.0, + bias=True, + add_bias_kv=False, + add_zero_attn=False, + self_attention=False, + encoder_decoder_attention=False, + q_noise=0.0, + qn_block_size=8, + ): + super().__init__() + self.embed_dim = embed_dim + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim + + self.num_heads = num_heads + self.dropout_module = FairseqDropout( + dropout, module_name=self.__class__.__name__) + + self.head_dim = embed_dim // num_heads + assert (self.head_dim * num_heads == self.embed_dim + ), 'embed_dim must be divisible by num_heads' + self.scaling = self.head_dim**-0.5 + + self.self_attention = self_attention + self.encoder_decoder_attention = encoder_decoder_attention + + assert not self.self_attention or self.qkv_same_dim, ( + 'Self-attention requires query, key and ' + 'value to be of the same size') + + self.k_proj = quant_noise( + nn.Linear(self.kdim, embed_dim, bias=bias), q_noise, qn_block_size) + self.v_proj = quant_noise( + nn.Linear(self.vdim, embed_dim, bias=bias), q_noise, qn_block_size) + self.q_proj = quant_noise( + nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size) + + self.out_proj = quant_noise( + nn.Linear(embed_dim, embed_dim, bias=bias), q_noise, qn_block_size) + + if add_bias_kv: + self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim)) + self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim)) + else: + self.bias_k = self.bias_v = None + + self.add_zero_attn = add_zero_attn + + self.reset_parameters() + + self.onnx_trace = False + + def prepare_for_onnx_export_(self): + self.onnx_trace = True + + def reset_parameters(self): + if self.qkv_same_dim: + # Empirically observed the convergence to be much better with + # the scaled initialization + nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2)) + else: + nn.init.xavier_uniform_(self.k_proj.weight) + nn.init.xavier_uniform_(self.v_proj.weight) + nn.init.xavier_uniform_(self.q_proj.weight) + + nn.init.xavier_uniform_(self.out_proj.weight) + if self.out_proj.bias is not None: + nn.init.constant_(self.out_proj.bias, 0.0) + if self.bias_k is not None: + nn.init.xavier_normal_(self.bias_k) + if self.bias_v is not None: + nn.init.xavier_normal_(self.bias_v) + + def forward( + self, + query, + key: Optional[Tensor], + value: Optional[Tensor], + key_padding_mask: Optional[Tensor] = None, + incremental_state: Optional[Dict[str, Dict[str, + Optional[Tensor]]]] = None, + need_weights: bool = True, + static_kv: bool = False, + attn_mask: Optional[Tensor] = None, + before_softmax: bool = False, + need_head_weights: bool = False, + ) -> Tuple[Tensor, Optional[Tensor]]: + """Input shape: Time x Batch x Channel + + Args: + key_padding_mask (ByteTensor, optional): mask to exclude + keys that are pads, of shape `(batch, src_len)`, where + padding elements are indicated by 1s. + need_weights (bool, optional): return the attention weights, + averaged over heads (default: False). + attn_mask (ByteTensor, optional): typically used to + implement causal attention, where the mask prevents the + attention from looking forward in time (default: None). + before_softmax (bool, optional): return the raw attention + weights and values before the attention softmax. + need_head_weights (bool, optional): return the attention + weights for each head. Implies *need_weights*. Default: + return the average attention weights over all heads. + """ + if need_head_weights: + need_weights = True + + is_tpu = query.device.type == 'xla' + + tgt_len, bsz, embed_dim = query.size() + src_len = tgt_len + assert embed_dim == self.embed_dim, f'query dim {embed_dim} != {self.embed_dim}' + assert list(query.size()) == [tgt_len, bsz, embed_dim] + if key is not None: + src_len, key_bsz, _ = key.size() + if not torch.jit.is_scripting(): + assert key_bsz == bsz + assert value is not None + assert src_len, bsz == value.shape[:2] + + if (not self.onnx_trace + and not is_tpu # don't use PyTorch version on TPUs + and incremental_state is None and not static_kv + # A workaround for quantization to work. Otherwise JIT compilation + # treats bias in linear module as method. + and not torch.jit.is_scripting()): + assert key is not None and value is not None + return F.multi_head_attention_forward( + query, + key, + value, + self.embed_dim, + self.num_heads, + torch.empty([0]), + torch.cat( + (self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), + self.bias_k, + self.bias_v, + self.add_zero_attn, + self.dropout_module.p, + self.out_proj.weight, + self.out_proj.bias, + self.training or self.dropout_module.apply_during_inference, + key_padding_mask, + need_weights, + attn_mask, + use_separate_proj_weight=True, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + ) + + if incremental_state is not None: + saved_state = self._get_input_buffer(incremental_state) + if saved_state is not None and 'prev_key' in saved_state: + # previous time steps are cached - no need to recompute + # key and value if they are static + if static_kv: + assert self.encoder_decoder_attention and not self.self_attention + key = value = None + else: + saved_state = None + + if self.self_attention: + q = self.q_proj(query) + k = self.k_proj(query) + v = self.v_proj(query) + elif self.encoder_decoder_attention: + # encoder-decoder attention + q = self.q_proj(query) + if key is None: + assert value is None + k = v = None + else: + k = self.k_proj(key) + v = self.v_proj(key) + + else: + assert key is not None and value is not None + q = self.q_proj(query) + k = self.k_proj(key) + v = self.v_proj(value) + q *= self.scaling + + if self.bias_k is not None: + assert self.bias_v is not None + k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)]) + v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)]) + if attn_mask is not None: + attn_mask = torch.cat( + [attn_mask, + attn_mask.new_zeros(attn_mask.size(0), 1)], + dim=1) + if key_padding_mask is not None: + key_padding_mask = torch.cat( + [ + key_padding_mask, + key_padding_mask.new_zeros( + key_padding_mask.size(0), 1), + ], + dim=1, + ) + + q = ( + q.contiguous().view(tgt_len, bsz * self.num_heads, + self.head_dim).transpose(0, 1)) + if k is not None: + k = ( + k.contiguous().view(-1, bsz * self.num_heads, + self.head_dim).transpose(0, 1)) + if v is not None: + v = ( + v.contiguous().view(-1, bsz * self.num_heads, + self.head_dim).transpose(0, 1)) + + if saved_state is not None: + # saved states are stored with shape (bsz, num_heads, seq_len, head_dim) + if 'prev_key' in saved_state: + _prev_key = saved_state['prev_key'] + assert _prev_key is not None + prev_key = _prev_key.view(bsz * self.num_heads, -1, + self.head_dim) + if static_kv: + k = prev_key + else: + assert k is not None + k = torch.cat([prev_key, k], dim=1) + src_len = k.size(1) + if 'prev_value' in saved_state: + _prev_value = saved_state['prev_value'] + assert _prev_value is not None + prev_value = _prev_value.view(bsz * self.num_heads, -1, + self.head_dim) + if static_kv: + v = prev_value + else: + assert v is not None + v = torch.cat([prev_value, v], dim=1) + prev_key_padding_mask: Optional[Tensor] = None + if 'prev_key_padding_mask' in saved_state: + prev_key_padding_mask = saved_state['prev_key_padding_mask'] + assert k is not None and v is not None + key_padding_mask = MultiheadAttention._append_prev_key_padding_mask( + key_padding_mask=key_padding_mask, + prev_key_padding_mask=prev_key_padding_mask, + batch_size=bsz, + src_len=k.size(1), + static_kv=static_kv, + ) + + saved_state['prev_key'] = k.view(bsz, self.num_heads, -1, + self.head_dim) + saved_state['prev_value'] = v.view(bsz, self.num_heads, -1, + self.head_dim) + saved_state['prev_key_padding_mask'] = key_padding_mask + # In this branch incremental_state is never None + assert incremental_state is not None + incremental_state = self._set_input_buffer(incremental_state, + saved_state) + assert k is not None + assert k.size(1) == src_len + + # This is part of a workaround to get around fork/join parallelism + # not supporting Optional types. + if key_padding_mask is not None and key_padding_mask.dim() == 0: + key_padding_mask = None + + if key_padding_mask is not None: + assert key_padding_mask.size(0) == bsz + assert key_padding_mask.size(1) == src_len + + if self.add_zero_attn: + assert v is not None + src_len += 1 + k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], + dim=1) + v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], + dim=1) + if attn_mask is not None: + attn_mask = torch.cat( + [attn_mask, + attn_mask.new_zeros(attn_mask.size(0), 1)], + dim=1) + if key_padding_mask is not None: + key_padding_mask = torch.cat( + [ + key_padding_mask, + torch.zeros(key_padding_mask.size(0), + 1).type_as(key_padding_mask), + ], + dim=1, + ) + + attn_weights = torch.bmm(q, k.transpose(1, 2)) + attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, + bsz) + + assert list( + attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len] + + if attn_mask is not None: + attn_mask = attn_mask.unsqueeze(0) + if self.onnx_trace: + attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1) + attn_weights += attn_mask + + if key_padding_mask is not None: + # don't attend to padding symbols + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, + src_len) + if not is_tpu: + attn_weights = attn_weights.masked_fill( + key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), + float('-inf'), + ) + else: + attn_weights = attn_weights.transpose(0, 2) + attn_weights = attn_weights.masked_fill( + key_padding_mask, float('-inf')) + attn_weights = attn_weights.transpose(0, 2) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, + src_len) + + if before_softmax: + return attn_weights, v + + attn_weights_float = utils.softmax( + attn_weights, dim=-1, onnx_trace=self.onnx_trace) + attn_weights = attn_weights_float.type_as(attn_weights) + attn_probs = self.dropout_module(attn_weights) + + assert v is not None + attn = torch.bmm(attn_probs, v) + assert list( + attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim] + if self.onnx_trace and attn.size(1) == 1: + # when ONNX tracing a single decoder step (sequence length == 1) + # the transpose is a no-op copy before view, thus unnecessary + attn = attn.contiguous().view(tgt_len, bsz, embed_dim) + else: + attn = attn.transpose(0, + 1).contiguous().view(tgt_len, bsz, embed_dim) + attn = self.out_proj(attn) + attn_weights: Optional[Tensor] = None + if need_weights: + attn_weights = attn_weights_float.view(bsz, self.num_heads, + tgt_len, + src_len).transpose(1, 0) + if not need_head_weights: + # average attention weights over heads + attn_weights = attn_weights.mean(dim=0) + + return attn, attn_weights + + @staticmethod + def _append_prev_key_padding_mask( + key_padding_mask: Optional[Tensor], + prev_key_padding_mask: Optional[Tensor], + batch_size: int, + src_len: int, + static_kv: bool, + ) -> Optional[Tensor]: + # saved key padding masks have shape (bsz, seq_len) + if prev_key_padding_mask is not None and static_kv: + new_key_padding_mask = prev_key_padding_mask + elif prev_key_padding_mask is not None and key_padding_mask is not None: + new_key_padding_mask = torch.cat( + [prev_key_padding_mask.float(), + key_padding_mask.float()], + dim=1) + # During incremental decoding, as the padding token enters and + # leaves the frame, there will be a time when prev or current + # is None + elif prev_key_padding_mask is not None: + if src_len > prev_key_padding_mask.size(1): + filler = torch.zeros( + (batch_size, src_len - prev_key_padding_mask.size(1)), + device=prev_key_padding_mask.device, + ) + new_key_padding_mask = torch.cat( + [prev_key_padding_mask.float(), + filler.float()], dim=1) + else: + new_key_padding_mask = prev_key_padding_mask.float() + elif key_padding_mask is not None: + if src_len > key_padding_mask.size(1): + filler = torch.zeros( + (batch_size, src_len - key_padding_mask.size(1)), + device=key_padding_mask.device, + ) + new_key_padding_mask = torch.cat( + [filler.float(), key_padding_mask.float()], dim=1) + else: + new_key_padding_mask = key_padding_mask.float() + else: + new_key_padding_mask = prev_key_padding_mask + return new_key_padding_mask + + @torch.jit.export + def reorder_incremental_state( + self, + incremental_state: Dict[str, Dict[str, Optional[Tensor]]], + new_order: Tensor, + ): + """Reorder buffered internal state (for incremental generation).""" + input_buffer = self._get_input_buffer(incremental_state) + if input_buffer is not None: + for k in input_buffer.keys(): + input_buffer_k = input_buffer[k] + if input_buffer_k is not None: + if self.encoder_decoder_attention and input_buffer_k.size( + 0) == new_order.size(0): + break + input_buffer[k] = input_buffer_k.index_select(0, new_order) + incremental_state = self._set_input_buffer(incremental_state, + input_buffer) + return incremental_state + + def _get_input_buffer( + self, incremental_state: Optional[Dict[str, Dict[str, + Optional[Tensor]]]] + ) -> Dict[str, Optional[Tensor]]: + result = self.get_incremental_state(incremental_state, 'attn_state') + if result is not None: + return result + else: + empty_result: Dict[str, Optional[Tensor]] = {} + return empty_result + + def _set_input_buffer( + self, + incremental_state: Dict[str, Dict[str, Optional[Tensor]]], + buffer: Dict[str, Optional[Tensor]], + ): + return self.set_incremental_state(incremental_state, 'attn_state', + buffer) + + def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, + bsz: int): + return attn_weights + + def upgrade_state_dict_named(self, state_dict, name): + prefix = name + '.' if name != '' else '' + items_to_add = {} + keys_to_remove = [] + for k in state_dict.keys(): + if k.endswith(prefix + 'in_proj_weight'): + # in_proj_weight used to be q + k + v with same dimensions + dim = int(state_dict[k].shape[0] / 3) + items_to_add[prefix + 'q_proj.weight'] = state_dict[k][:dim] + items_to_add[prefix + 'k_proj.weight'] = state_dict[k][dim:2 + * dim] + items_to_add[prefix + 'v_proj.weight'] = state_dict[k][2 + * dim:] + + keys_to_remove.append(k) + + k_bias = prefix + 'in_proj_bias' + if k_bias in state_dict.keys(): + dim = int(state_dict[k].shape[0] / 3) + items_to_add[prefix + + 'q_proj.bias'] = state_dict[k_bias][:dim] + items_to_add[prefix + + 'k_proj.bias'] = state_dict[k_bias][dim:2 + * dim] + items_to_add[prefix + + 'v_proj.bias'] = state_dict[k_bias][2 + * dim:] + + keys_to_remove.append(prefix + 'in_proj_bias') + + for k in keys_to_remove: + del state_dict[k] + + for key, value in items_to_add.items(): + state_dict[key] = value diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/ngram_repeat_block.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/ngram_repeat_block.py new file mode 100644 index 0000000000000000000000000000000000000000..4bccfa7602c71a460c3f6e9294310c5db488bacb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/ngram_repeat_block.py @@ -0,0 +1,155 @@ +# Originally from Microsoft Corporation. +# Licensed under the MIT License. +""" Wrapper for ngram_repeat_block cuda extension """ +import math +import warnings +from typing import Dict, List + +import torch +from torch import nn + +try: + from fairseq import ngram_repeat_block_cuda + + EXTENSION_BUILT = True +except ImportError: + EXTENSION_BUILT = False + + +def is_cuda_extension_usable() -> bool: + """Check whether ngram_repeat_block_cuda is built properly""" + if not EXTENSION_BUILT or not torch.cuda.is_available(): + return False + bsz = 2 + tokens = torch.tensor([[4, 4, 3, 2], [1, 2, 3, 4]], + dtype=torch.long, + device='cuda') + lprobs = torch.rand((8, 12), device='cuda') + try: + outputs = ngram_repeat_block_cuda.forward(tokens, lprobs, bsz, 3, 4, 3) + outputs = outputs + 4 # This line breaks if the extension is built incorrectly. + return True + except RuntimeError: + warnings.warn( + 'NGramRepeatBlock extension must be rebuilt.' + 'Run TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0" python setup.py build_ext --inplace' + ) + return False + + +class NGramRepeatBlock(nn.Module): + """ Wrapper class for calling ngram_repeat_block cuda extension """ + + def __init__(self, no_repeat_ngram_size: int, use_extension: bool = True): + super().__init__() + self.use_extension = is_cuda_extension_usable( + ) if use_extension else False + self.no_repeat_ngram_size = no_repeat_ngram_size + + def reset_parameters(self): + pass + + @torch.jit.unused + def call_cuda_extension( + self, + tokens, + lprobs, + bsz: int, + beam_size: int, + step: int, + ): + return ngram_repeat_block_cuda.forward(tokens, lprobs, bsz, step, + beam_size, + self.no_repeat_ngram_size) + + def forward( + self, + tokens, + lprobs, + bsz: int, + beam_size: int, + step: int, + ): + """ + Args: + tokens(Tensor): Input tokens(Bsz*beam, seq_len) + lprobs(Tensor): likelihood probability, + Expected to be updated in place.(Bsz*beam, vocab_size) + bsz(int): batch size + step(int): current step + beam_size(int): beam size + no_repeat_ngram_size(int): Ngram size + """ + msg = f'expected {bsz * beam_size} got' + assert tokens.size(0) == bsz * beam_size, f'{msg} {tokens.size(0)}' + assert lprobs.size(0) == bsz * beam_size, f'{msg} {lprobs.size(0)}' + if self.use_extension: + return self.call_cuda_extension(tokens, lprobs, bsz, beam_size, + step) + + else: + return self._no_repeat_ngram( + tokens, + lprobs, + bsz, + beam_size, + step, + ) + + def _no_repeat_ngram(self, tokens, lprobs, bsz: int, beam_size: int, + step: int): + """For each hypothesis generate a list of previous ngrams and set associated lprobs to -inf""" + gen_ngrams: List[Dict[str, List[int]]] = [ + torch.jit.annotate(Dict[str, List[int]], {}) + for bbsz_idx in range(bsz * beam_size) + ] + cpu_tokens = tokens.cpu() + for bbsz_idx in range(bsz * beam_size): + gen_tokens: List[int] = cpu_tokens[bbsz_idx].tolist() + for ngram in self.transpose_list([ + gen_tokens[i:] for i in range(self.no_repeat_ngram_size) + ]): # noqa + key = ','.join([str(x) for x in ngram[:-1]]) + gen_ngrams[bbsz_idx][key] = gen_ngrams[bbsz_idx].get( + key, torch.jit.annotate(List[int], [])) + [ngram[-1]] + if step + 2 - self.no_repeat_ngram_size >= 0: + # no banned tokens if we haven't generated no_repeat_ngram_size tokens yet + banned_tokens = [ + self.calculate_banned_tokens(tokens, step, gen_ngrams, + self.no_repeat_ngram_size, + bbsz_idx) + for bbsz_idx in range(bsz * beam_size) + ] + else: + banned_tokens = [ + torch.jit.annotate(List[int], []) + for bbsz_idx in range(bsz * beam_size) + ] + for bbsz_idx in range(bsz * beam_size): + lprobs[bbsz_idx][torch.tensor( + banned_tokens[bbsz_idx], + dtype=torch.int64)] = torch.tensor(-math.inf).to(lprobs) + return lprobs + + @staticmethod + def calculate_banned_tokens( + tokens, + step: int, + gen_ngrams: List[Dict[str, List[int]]], + no_repeat_ngram_size: int, + bbsz_idx: int, + ): + tokens_list: List[int] = tokens[bbsz_idx, + step + 2 - no_repeat_ngram_size:step + + 1].tolist() # noqa + # before decoding the next token, prevent decoding of ngrams that have already appeared + ngram_index = ','.join([str(x) for x in tokens_list]) + return gen_ngrams[bbsz_idx].get(ngram_index, + torch.jit.annotate(List[int], [])) + + @staticmethod + def transpose_list(l: List[List[int]]): # noqa + # GeneratorExp aren't supported in TS so ignoring the lint + min_len = min([len(x) for x in l]) # noqa + l2 = [[row[i] for row in l] for i in range(min_len)] + return l2 diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/search.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/search.py new file mode 100644 index 0000000000000000000000000000000000000000..f37585a1e5a2eda15f9fbd1d5d75d011ceda7fcf --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/search.py @@ -0,0 +1,1047 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/facebookresearch/fairseq/blob/main/LICENSE + +import math +from typing import List, Optional + +import torch +import torch.nn as nn +from torch import Tensor + +from .token_generation_constraints import (ConstraintState, + OrderedConstraintState, + UnorderedConstraintState) + + +class Search(nn.Module): + + def __init__(self, tokenizer): + super().__init__() + self.pad = tokenizer.pad_token_id + self.unk = tokenizer.unk_token_id + self.eos = tokenizer.eos_token_id + tgt_dict = {value: key for key, value in tokenizer.get_vocab().items()} + added = { + value: key + for key, value in tokenizer.get_added_vocab().items() + } + tgt_dict.update(added) + self.vocab_size = len(tgt_dict) + self.src_lengths = torch.tensor(-1) + self.supports_constraints = False + self.stop_on_max_len = False + + def step(self, + step, + lprobs, + scores, + prev_output_tokens=None, + original_batch_idxs=None): + """Take a single search step. + + Args: + step: the current search step, starting at 0 + lprobs: (bsz x input_beam_size x vocab_size) + the model's log-probabilities over the vocabulary at the current step + scores: (bsz x input_beam_size x step) + the historical model scores of each hypothesis up to this point + prev_output_tokens: (bsz x step) + the previously generated oputput tokens + original_batch_idxs: (bsz) + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Return: A tuple of (scores, indices, beams) where: + scores: (bsz x output_beam_size) + the scores of the chosen elements; output_beam_size can be + larger than input_beam_size, e.g., we may return + 2*input_beam_size to account for EOS + indices: (bsz x output_beam_size) + the indices of the chosen elements + beams: (bsz x output_beam_size) + the hypothesis ids of the chosen elements, in the range [0, input_beam_size) + """ + raise NotImplementedError + + @torch.jit.export + def set_src_lengths(self, src_lengths): + self.src_lengths = src_lengths + + @torch.jit.export + def init_constraints(self, batch_constraints: Optional[Tensor], + beam_size: int): + """Initialize constraint states for constrained decoding (if supported). + + Args: + batch_constraints: (torch.Tensor, optional) + the list of constraints, in packed form + beam_size: (int) + the beam size + Returns: + *encoder_out* rearranged according to *new_order* + """ + pass + + def prune_sentences(self, batch_idxs: Tensor): + """ + Removes constraint states for completed sentences (if supported). + This is called from sequence_generator._generate() when sentences are + deleted from the batch. + + Args: + batch_idxs: Indices of *sentences* whose constraint state should be *kept*. + """ + pass + + def update_constraints(self, active_hypos: Tensor): + """ + Updates the constraint states by selecting the beam items that are retained. + This is called at each time step of sequence_generator._generate() when + the set of 2 * {beam_size} candidate hypotheses are reduced to the beam size. + + Args: + active_hypos: (batch size, beam size) + list of integers denoting, for each sentence, which beam candidate items + should be kept. + """ + pass + + +class BeamSearch(Search): + r""" + Beam search strategy. + + step 1. Calculate top k candidates in model's log-probability under descending order. While k is the minor of + `beam_size * 2` and `beam_size * vocabulary_size`. + step 2. Modify hypothesis score, relative indices, beam indices for the final result. + """ + + def __init__(self, tgt_dict): + super().__init__(tgt_dict) + self.constraint_states = None + + @torch.jit.export + def step( + self, + step: int, + lprobs, + scores: Optional[Tensor], + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + scores_buf (`Tensor` with size `(bsz, output_beam_size)`): + The model's log-probabilities over the elements selected to sample from. + `output_beam_size` is the minor of `2 * input_beam_size` and `vocab_size - 1`. + which cumulates the score before. + indices_buf (`Tensor` with size `(bsz, output_beam_size)`): + The indices of chosen elements. + beams_buf (`Tensor` with size `(bsz, output_beam_size)`): + The indices of each beam. + """ + bsz, beam_size, vocab_size = lprobs.size() + + if step == 0: + # at the first step all hypotheses are equally likely, so use + # only the first beam + lprobs = lprobs[:, ::beam_size, :].contiguous() + else: + # make probs contain cumulative scores for each hypothesis + assert scores is not None + lprobs = lprobs + scores[:, :, step - 1].unsqueeze(-1) + + top_prediction = torch.topk( + lprobs.view(bsz, -1), + k=min( + # Take the best 2 x beam_size predictions. We'll choose the first + # beam_size of these which don't predict eos to continue with. + beam_size * 2, + lprobs.view(bsz, -1).size(1) - 1, # -1 so we never select pad + ), + ) + scores_buf = top_prediction[0] + indices_buf = top_prediction[1] + # Project back into relative indices and beams + beams_buf = torch.div(indices_buf, vocab_size, rounding_mode='floor') + indices_buf = indices_buf.fmod(vocab_size) + + # At this point, beams_buf and indices_buf are single-dim and contain relative indices + return scores_buf, indices_buf, beams_buf + + +class PrefixConstrainedBeamSearch(Search): + r""" + Prefix constrained beam search. + + step 1. Calculate a mask according to a `prefix_allowed_tokens_fn` + function with input of previous hypothesis tokens and indices. + step 2. Calculate a candidate set of `lprobs` with `lprobs` and mask produced in step 1. + step 3. Just like beam search strategy to generate the hypothesis token. + And the difference is the k in top k function is the minor of `beam_size` and `vocab_size -1` + """ + + def __init__(self, tgt_dict, prefix_allowed_tokens_fn): + super().__init__(tgt_dict) + self.prefix_allowed_tokens_fn = prefix_allowed_tokens_fn + self.stop_on_max_len = True + + @torch.jit.export + def apply_mask(self, x, prev_output_tokens, original_batch_idxs): + beam_size = x.shape[0] // original_batch_idxs.shape[0] + original_batch_idxs = ( + original_batch_idxs.unsqueeze(-1).repeat( + (1, beam_size)).flatten().tolist()) + + mask = torch.full_like(x, -math.inf) + for sent_i, (sent, batch_i) in enumerate( + zip(prev_output_tokens, original_batch_idxs)): + mask[sent_i, :, self.prefix_allowed_tokens_fn(batch_i, sent)] = 0 + + return mask + + @torch.jit.export + def step( + self, + step: int, + lprobs: Tensor, + scores: Tensor, + prev_output_tokens: Tensor, + original_batch_idxs: Tensor, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + scores_buf (`Tensor` with size `(bsz, input_beam_size)`): + The model's log-probabilities over the elements selected to sample from. + which cumulates the score before. + indices_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of chosen elements. + beams_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of each beam. + """ + bsz, beam_size, vocab_size = lprobs.size() + + lprobs += self.apply_mask( + lprobs.view(bsz * beam_size, 1, vocab_size), + prev_output_tokens, + original_batch_idxs, + ).view(bsz, beam_size, vocab_size) + + if step == 0: + # at the first step all hypotheses are equally likely, so use + # only the first beam + lprobs = lprobs[:, ::beam_size, :].contiguous() + else: + # make probs contain cumulative scores for each hypothesis + assert scores is not None + lprobs = lprobs + scores[:, :, step - 1].unsqueeze(-1) + + top_prediction = torch.topk( + lprobs.view(bsz, -1), + k=min( + # Take the best beam_size predictions. We'll choose the first + # beam_size of these which don't predict eos to continue with. + beam_size, + lprobs.view(bsz, -1).size(1) - 1, # -1 so we never select pad + ), + ) + scores_buf = top_prediction[0] + indices_buf = top_prediction[1] + beams_buf = indices_buf // vocab_size + indices_buf = indices_buf.fmod(vocab_size) + return scores_buf, indices_buf, beams_buf + + +class LexicallyConstrainedBeamSearch(Search): + """Implements lexically constrained beam search as described in + + Fast Lexically Constrained Decoding with Dynamic Beam + Allocation for Neural Machine Translation. Post & Vilar, + NAACL 2018. https://www.aclweb.org/anthology/N18-1119/ + + and + + Improved Lexically Constrained Decoding for Translation and + Monolingual Rewriting. Hu et al, NAACL + 2019. https://www.aclweb.org/anthology/N19-1090/ + + This is accomplished by maintaining, for each beam hypothesis, a + ConstraintState object (see constraints.py) that tracks which + constraints have been generated and using this information to + shape the beam for each input sentence. + """ + + def __init__(self, tokenizer, representation): + super().__init__(tokenizer) + self.representation = representation + tgt_dict = {value: key for key, value in tokenizer.get_vocab().items()} + added = { + value: key + for key, value in tokenizer.get_added_vocab().items() + } + tgt_dict.update(added) + self.vocab_size = len(tgt_dict) + self.num_cands = 0 + self.supports_constraints = True + + @torch.jit.export + def init_constraints(self, batch_constraints: Optional[Tensor], + beam_size: int): + self.constraint_states = [] + for constraint_tensor in batch_constraints: + if self.representation == 'ordered': + constraint_state = OrderedConstraintState.create( + constraint_tensor) + elif self.representation == 'unordered': + constraint_state = UnorderedConstraintState.create( + constraint_tensor) + + self.constraint_states.append( + [constraint_state for i in range(beam_size)]) + + @torch.jit.export + def prune_sentences(self, batch_idxs: Tensor): + self.constraint_states = [ + self.constraint_states[i] for i in batch_idxs.tolist() + ] + + @torch.jit.export + def update_constraints(self, active_hypos: Tensor): + if self.constraint_states: + batch_size = active_hypos.size(0) + for sentid in range(batch_size): + self.constraint_states[sentid] = [ + self.constraint_states[sentid][i] + for i in active_hypos[sentid] + ] + + @torch.jit.export + def step( + self, + step: int, + lprobs: Tensor, + scores: Optional[Tensor], + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + """ + A constrained step builds a large candidates list from the following: + - the top 2 * {beam_size} items over the whole beam + - for each item in the beam + - the top {each_k} (default 1) + - all next constraints + We then compute the constrained state of each beam item, and assign + stripe codes: 0 to the best in each bank, 1 to the 2nd-best, and so + on. We then sort by (stripe, score), and truncate the list at + 2 * beam size. + + Args: + step: the decoder step + lprobs: (batch size, beam size, target vocab) + the target-vocab distributions for each item in the beam. + Retrun: A tuple of (scores, indices, beams, constraints) where: + scores: (batch, output beam size) + the scores of the chosen elements + indices: (batch, output beam size) + the target vocab indices of the chosen elements + beams: (batch, output beam size) + the 0-indexed hypothesis ids of the chosen elements + constraints: (batch, output beam size) + the new constraint states + """ + each_k = 1 + device = lprobs.device + + batch_size, beam_size, vocab_size = lprobs.size() + + self.num_cands = min( + # Just take the k-best. We'll get another k from the 1-best from each + # row, plus more from the constraints + beam_size * 2, + lprobs.view(batch_size, -1).size(1) + - 1, # -1 so we never select pad + ) + + # STEP 0: Preliminary. Prevent EOS for unfinished hyps across all batch items + constraint_states = self.constraint_states + if constraint_states and step > 0: + not_finished_indices = [] + for sentno, sent_constraints in enumerate(constraint_states): + for beamno, state in enumerate(sent_constraints): + index = sentno * beam_size + beamno + if not state.finished: + not_finished_indices.append(index) + not_finished_indices = torch.tensor(not_finished_indices) + if not_finished_indices.numel() > 0: + lprobs.view(batch_size * beam_size, -1)[not_finished_indices, + self.eos] = -math.inf + + if step == 0: + # at the first step all hypotheses are equally likely, so use + # only the first beam entry for each batch item + lprobs = lprobs[:, ::beam_size, :].contiguous() + else: + # make probs contain cumulative scores for each hypothesis + assert scores is not None + lprobs = lprobs + scores[:, :, step - 1].unsqueeze(-1) + + top_prediction = torch.topk( + lprobs.view(batch_size, -1), + self.num_cands, + ) + scores_buf, indices_buf = top_prediction + # Project back into relative indices and beams + beams_buf = indices_buf // vocab_size + indices_buf = indices_buf.fmod(vocab_size) + + # Short circuit if there are no constraints in this batch + if not constraint_states: + return scores_buf, indices_buf, beams_buf + + # STEP 1: get top-1 from each hypothesis across all sentences in the batch + if step > 0: + top_scores, top_indices = torch.topk( + lprobs.view(batch_size * beam_size, -1), + k=each_k, + dim=1, + ) + top_scores = top_scores.view(batch_size, -1) + top_indices = top_indices.view(batch_size, -1) + scores_buf = torch.cat((scores_buf, top_scores), dim=1) + indices_buf = torch.cat((indices_buf, top_indices), dim=1) + new_beams = torch.arange( + 0, beam_size, device=device).repeat(batch_size, 1) + beams_buf = torch.cat((beams_buf, new_beams), dim=1) + + # Now, process sentences in the batch one by one. + new_scores_buf = torch.zeros((batch_size, 2 * beam_size), + device=device) + new_indices_buf = torch.zeros((batch_size, 2 * beam_size), + device=device).long() + new_beams_buf = torch.zeros((batch_size, 2 * beam_size), + device=device).long() + for sentno, states in enumerate(constraint_states): + scores, indices, beams, new_states = self.step_sentence( + step, + sentno, + lprobs[sentno], + constraint_states[sentno], + beams_buf[sentno].clone(), + indices_buf[sentno].clone(), + scores_buf[sentno].clone(), + ) + new_scores_buf[sentno] = scores + new_indices_buf[sentno] = indices + new_beams_buf[sentno] = beams + self.constraint_states[sentno] = new_states + + return new_scores_buf, new_indices_buf, new_beams_buf + + @torch.jit.export + def step_sentence( + self, + step: int, + sentno: int, + lprobs: Tensor, + constraint_states: List[List[ConstraintState]], + beams_buf: Tensor, + indices_buf: Tensor, + scores_buf: Tensor, + ): + """Does per-sentence processing. Adds all constraints for each + hypothesis to the list of candidates; then removes duplicates, + sorts, and dynamically stripes across the banks. All tensor inputs + are collapsed to those pertaining to a single input sentence. + """ + device = lprobs.device + + # STEP 2: Add all constraints for each beam item + for beamno, state in enumerate(constraint_states): + next_tokens = torch.tensor( + list(state.next_tokens()), device=device).long() + if next_tokens.numel() != 0: + indices_buf = torch.cat((indices_buf, next_tokens)) + next_beams = ( + torch.tensor(beamno, device=device).repeat( + next_tokens.size(0)).long()) + beams_buf = torch.cat((beams_buf, next_beams)) + next_values = lprobs[beamno].take(next_tokens.view(-1)) + scores_buf = torch.cat((scores_buf, next_values)) + + # At the 0th time step, there is just one beam item + if step == 0: + break + + # STEP 3: Compute the "bank" for each candidate. This is the + # number of constraints it's generated. We need this so that + # we can do round-robin allocation of the beam across these + # banks. If C is the number of constraints, we select the best + # item in bank C, then the best in bank C-1, etc, followed by + # the 2nd-best in bank C, the 2nd-best in bank C-1, etc, and so + # on, until the maximum beam size. We accomplish this by + # creating a sort key and striping across the banks. + + # Compute the new states for all candidates + cands_size = indices_buf.size(0) + constraint_states = [ + constraint_states[beams_buf[i]].advance(indices_buf[i]) + for i in range(cands_size) + ] + + banks = torch.tensor([state.bank for state in constraint_states], + device=device) + + # STEP 4: Sort + num_constraint_tokens = len(state.tokens) + + # Sort by keys (bank, score) (i.e., sort banks together, and scores + # within banks). AFAIK pytorch doesn't support either stable sort or + # multi-key sorting, so we have to hack this. + MAX_SCORE = -100 + sort_key = (num_constraint_tokens - banks) * MAX_SCORE + scores_buf + sort_values, sort_indices = sort_key.sort(dim=0, descending=True) + scores_buf = scores_buf[sort_indices] + indices_buf = indices_buf[sort_indices] + beams_buf = beams_buf[sort_indices] + banks = banks[sort_indices] + + # Sort the constraints to follow suit + constraint_states = [constraint_states[i] for i in sort_indices] + + # STEP 5: Remove duplicates. The topk calls (overall and + # per-row) plus the per-row generation of constraints will + # produce duplicates. Here we remove them. + + def roll(t): + """Rolls a 1d tensor left by 1. + + [0, 1, 2, 3, 4] becomes [4, 0, 1, 2, 3] + """ + return torch.cat((t[-1].unsqueeze(0), t[0:-1]), dim=0) + + # We map candidates (beam, token_id) to a single dimension. + # This is then shifted by 1. We can then easily identify + # duplicates and create a mask that identifies unique + # extensions. + uniques_mask = beams_buf * (self.vocab_size + 1) + indices_buf + uniques_mask = roll(uniques_mask) != uniques_mask + + # Use the mask to pare down the data structures + scores_buf = torch.masked_select(scores_buf, uniques_mask) + indices_buf = torch.masked_select(indices_buf, uniques_mask) + beams_buf = torch.masked_select(beams_buf, uniques_mask) + banks = torch.masked_select(banks, uniques_mask) + i = 1 + for mask in uniques_mask[1:]: + if not mask: + constraint_states.pop(i) + i += mask + + # STEP 6: Assign IDs round-robin across banks, sort, and + # truncate. Now that the candidates are sorted by (bank, + # score) and uniqed, we dynamically allocate the {beam_size} + # beam by striping across the candidates. These stripes will + # be used as sort keys to do round-robin selection. This is + # accomplished in a single pass with offsets. Sorting by + # highest-banks (furthest-along hypotheses) first ensures + # progress through the constraints. + # + # e.g., BANKS: 3 3 3 2 2 2 2 1 1 1 0 0 + # OLD STRIPES: 0 1 2 0 1 2 3 0 1 2 0 1 + # NEW STRIPES: 0 1+4 2+8 0+1 1+5 2+9 3+11 0+2 1+6 2+10 0+3 1+7 + # = 0 5 10 1 6 11 13 2 7 12 3 8 + # + # Sorting by this then gives the following banks: + # + # 3 2 1 0 3 2 1 0 3 2 1 2 + # + # We'll take the top {beam_size} of these. + stripe_offsets = [ + offset * (len(banks) + 1) for offset in range(len(banks) + 1) + ] + stripes = torch.zeros_like(banks) + cur_bank_count = -1 + cur_bank = banks[0] + for i, bank in enumerate(banks): + if bank != cur_bank: + cur_bank_count = 0 + cur_bank = bank + else: + cur_bank_count += 1 + stripes[i] = num_constraint_tokens - bank + stripe_offsets[ + cur_bank_count] + + # STEP 7: Sort by the stripes values + sort_values, sort_indices = stripes.sort(dim=0) + scores_buf = scores_buf[sort_indices] + indices_buf = indices_buf[sort_indices] + beams_buf = beams_buf[sort_indices] + constraint_states = [constraint_states[i] for i in sort_indices] + + # STEP 8: Truncate to the candidates size! + scores_buf = scores_buf[:self.num_cands] + indices_buf = indices_buf[:self.num_cands] + beams_buf = beams_buf[:self.num_cands] + + return scores_buf, indices_buf, beams_buf, constraint_states + + +class LengthConstrainedBeamSearch(Search): + r""" + Length constrained beam search for generation. + + step 1. Build length constraints in model's log-probability. If `min_lens` > `step`, + set eos token's score to `-math.inf`, so the generation will not be easily stopped. + Otherwise, `max_lens` <= `step`, set eos token's score to `0`, so the generation will + be easily stopped. + step 2. Using beam search to generate the hypothesis tokens with scores. + """ + + def __init__(self, tgt_dict, min_len_a, min_len_b, max_len_a, max_len_b): + super().__init__(tgt_dict) + self.min_len_a = min_len_a + self.min_len_b = min_len_b + self.max_len_a = max_len_a + self.max_len_b = max_len_b + self.beam = BeamSearch(tgt_dict) + self.needs_src_lengths = True + + def step( + self, + step: int, + lprobs, + scores, + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + scores_buf (`Tensor` with size `(bsz, output_beam_size)`): + The model's log-probabilities over the elements selected to sample from. + `output_beam_size` is the minor of `2 * input_beam_size` and `vocab_size - 1`. + which cumulates the score before. + indices_buf (`Tensor` with size `(bsz, output_beam_size)`): + The indices of chosen elements. + beams_buf (`Tensor` with size `(bsz, output_beam_size)`): + The indices of each beam. + """ + min_lens = self.min_len_a * self.src_lengths + self.min_len_b + max_lens = self.max_len_a * self.src_lengths + self.max_len_b + # There seems to be a bug here. Should be right like: + # lprobs[[step < min_lens] * len(lprobs), :, self.eos] = -math.inf + # lprobs[[step >= max_lens] * len(lprobs), :, self.eos] = 0 + lprobs[step < min_lens, :, self.eos] = -math.inf + lprobs[step >= max_lens, :, self.eos] = 0 + return self.beam.step(step, lprobs, scores) + + +class DiverseBeamSearch(Search): + """Diverse Beam Search. + + See "Diverse Beam Search: Decoding Diverse Solutions from Neural Sequence + Models" for details. + + We only implement the Hamming Diversity penalty here, which performed best + in the original paper. + """ + + def __init__(self, tgt_dict, num_groups, diversity_strength): + super().__init__(tgt_dict) + self.num_groups = num_groups + self.diversity_strength = -diversity_strength + self.beam = BeamSearch(tgt_dict) + + @torch.jit.export + def step( + self, + step: int, + lprobs, + scores, + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + scores_buf (`Tensor` with size `(bsz, input_beam_size)`): + The model's log-probabilities over the elements selected to sample from, + which cumulates the score before. + indices_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of chosen elements. + beams_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of each beam. + """ + bsz, beam_size, vocab_size = lprobs.size() + if beam_size % self.num_groups != 0: + raise ValueError( + 'DiverseBeamSearch requires --beam to be divisible by the number of groups' + ) + + # initialize diversity penalty + diversity_buf = torch.zeros(lprobs[:, 0, :].size()).to(lprobs) + + scores_G, indices_G, beams_G = [], [], [] + for g in range(self.num_groups): + lprobs_g = lprobs[:, g::self.num_groups, :] + scores_g = scores[:, g::self.num_groups, :] if step > 0 else None + + # apply diversity penalty + if g > 0: + lprobs_g = torch.add( + lprobs_g, + other=diversity_buf.unsqueeze(1), + alpha=self.diversity_strength, + ) + else: + lprobs_g = lprobs_g.contiguous() + + scores_buf, indices_buf, beams_buf = self.beam.step( + step, lprobs_g, scores_g) + beams_buf.mul_(self.num_groups).add_(g) + + scores_G.append(scores_buf.clone()) + indices_G.append(indices_buf.clone()) + beams_G.append(beams_buf.clone()) + + # update diversity penalty + diversity_buf.scatter_add_( + 1, indices_buf, + torch.ones(indices_buf.size()).to(diversity_buf)) + + # interleave results from different groups + scores_buf = torch.stack(scores_G, dim=2).view(bsz, -1) + indices_buf = torch.stack(indices_G, dim=2).view(bsz, -1) + beams_buf = torch.stack(beams_G, dim=2).view(bsz, -1) + return scores_buf, indices_buf, beams_buf + + +class Sampling(Search): + r""" + Sampling search for generation. + + 1. Calculate the sample set. + 1.1 If `sampling_topk` is not None, chose the candidates which cumulative sum of model's + log-probability under descending order is less than `sampling_topk`. + 1.2 If `sampling_topp` is not None, chose the top k candidates by model's log-probability under + the descending order. + 1.3 Chose the whole input set as sampling set. + 2. Using multinomial sample strategy to sample candidates from sample set as hypothesis. + 3. Modify hypothesis score, relative indices, beam indices for the final result. + + Attributes: + sampling_topk (`int`, **optional**, default to `-1`): + The value of k in the sampling strategy of top k. + sampling_topp (`float`, **optional**, default to '-1.0'): + The value of p The sampling strategy of top p. + """ + sampling_topk: int + sampling_topp: float + + def __init__(self, tgt_dict, sampling_topk=-1, sampling_topp=-1.0): + super().__init__(tgt_dict) + self.sampling_topk = sampling_topk + self.sampling_topp = sampling_topp + + def _sample_topp(self, lprobs): + """Sample among the smallest set of elements whose cumulative probability mass exceeds p. + + See `"The Curious Case of Neural Text Degeneration" + (Holtzman et al., 2019) `_. + + Args: + lprobs: (bsz x input_beam_size x vocab_size) + the model's log-probabilities over the vocabulary at the current step + + Return: A tuple of (trimed_probs, truncated_indices) where: + trimed_probs: (bsz x input_beam_size x ?) + the model's probabilities over the elements selected to sample from. The + width of the third dimension is determined by top-P. + truncated_indices: (bsz x input_beam_size x ?) + the indices of the chosen elements. + """ + probs = lprobs.exp_() + + # sort the last dimension (vocab dimension) in descending order + sorted_probs, sorted_indices = probs.sort(descending=True) + + # compute a mask to indicate the words to be included in the top-P set. + cumsum_probs = sorted_probs.cumsum(dim=2) + mask = cumsum_probs.lt(self.sampling_topp) + + # note that mask was computed by 'lt'. One more word needs to be included + # so that the cumulative probability mass can exceed p. + cumsum_mask = mask.cumsum(dim=2) + last_included = cumsum_mask[:, :, -1:] + last_included.clamp_(0, mask.size()[2] - 1) + mask = mask.scatter_(2, last_included, 1) + + # truncate unnecessary dims. + max_dim = last_included.max() + truncated_mask = mask[:, :, :max_dim + 1] + truncated_probs = sorted_probs[:, :, :max_dim + 1] + truncated_indices = sorted_indices[:, :, :max_dim + 1] + + # trim the words that are not in top-P by setting their probabilities + # to 0, so that they would not be sampled later. + trim_mask = ~truncated_mask + trimed_probs = truncated_probs.masked_fill_(trim_mask, 0) + return trimed_probs, truncated_indices + + @torch.jit.export + def step( + self, + step: int, + lprobs, + scores, + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + scores_buf (`Tensor` with size `(bsz, input_beam_size)`): + The model's log-probabilities over the elements selected to sample from. + which cumulates the score before. + indices_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of chosen elements. + beams_buf (`Tensor` with size `(bsz, input_beam_size)`): + The indices of each beam. + """ + bsz, beam_size, vocab_size = lprobs.size() + + if step == 0: + # at the first step all hypotheses are equally likely, so use + # only the first beam + lprobs = lprobs[:, ::beam_size, :].contiguous() + + if self.sampling_topp > 0: + # only sample from the smallest set of words whose cumulative probability mass exceeds p + probs, top_indices = self._sample_topp(lprobs) + elif self.sampling_topk > 0: + # only sample from top-k candidates + lprobs, top_indices = lprobs.topk(self.sampling_topk) + probs = lprobs.exp_() + else: + probs = lprobs.exp_() + + # dummy data to be consistent with true branch for type check + top_indices = torch.empty(0).to(probs) + # sample + if step == 0: + indices_buf = torch.multinomial( + probs.view(bsz, -1), + beam_size, + replacement=True, + ).view(bsz, beam_size) + else: + indices_buf = torch.multinomial( + probs.view(bsz * beam_size, -1), + 1, + replacement=True, + ).view(bsz, beam_size) + + if step == 0: + # expand to beam size + probs = probs.expand(bsz, beam_size, -1) + + # gather scores + scores_buf = torch.gather( + probs, dim=2, index=indices_buf.unsqueeze(-1)) + scores_buf = scores_buf.log_().view(bsz, -1) + + # remap indices if using top-k or top-P sampling + if self.sampling_topk > 0 or self.sampling_topp > 0: + indices_buf = torch.gather( + top_indices.expand(bsz, beam_size, -1), + dim=2, + index=indices_buf.unsqueeze(-1), + ).squeeze(2) + + if step == 0: + beams_buf = indices_buf.new_zeros(bsz, beam_size) + else: + beams_buf = torch.arange(0, + beam_size).to(indices_buf).repeat(bsz, 1) + # make scores cumulative + scores_buf.add_( + torch.gather(scores[:, :, step - 1], dim=1, index=beams_buf)) + + return scores_buf, indices_buf, beams_buf + + +class DiverseSiblingsSearch(Search): + """ + Beam search with diverse siblings. + + See "A Simple, Fast Diverse Decoding Algorithm for Neural Generation" for details. + https://arxiv.org/abs/1611.08562 + + 1/ Calculate hypotheses for each beam + 2/ Intra-sibling ordering + 3/ Rewrite scores + 4/ Choose top K hypotheses + + if diversity_rate == 0 is equivalent to BeamSearch + """ + + def __init__(self, tgt_dict, diversity_rate): + super().__init__(tgt_dict) + self.diversity_rate = diversity_rate + self.beam = BeamSearch(tgt_dict) + + def step( + self, + step: int, + lprobs, + scores, + prev_output_tokens: Optional[Tensor] = None, + original_batch_idxs: Optional[Tensor] = None, + ): + r""" + Take a single search step. + + Args: + step (`int`): Current step, start with 0. + lprobs (`Tensor` with size `(bsz, input_beam_size, vocab_size)`): + the model's log-probabilities over the vocabulary at the current step. + scores (`Tensor` with size `(bsz, input_beam_size, step - 1)`): + Previous sampling scores for each beam. + prev_output_tokens (`Tensor`, **optional**. default to `None`): + Previous output tokens, no usage in this function, will be deprecated in next version. + original_batch_idxs (`Tensor`, **optional**, default to `None`): + the tensor with the batch indices, in the range [0, bsz) + this is useful in case there has been applied a re-ordering + and we need to know the original indices + + Returns: A tuple of (scores_buf, indices_buf, beams_buf), where: + final_scores (`Tensor` with size `(bsz, output_beam_size)`): + The model's log-probabilities over the elements selected to sample from, + which cumulates the score before. `output_beam_size` is the minor of + `2 * input_beam_size` and `vocab_size - 1`. + final_indices (`Tensor` with size `(bsz, output_beam_size)`): + The indices of chosen elements. + final_beams (`Tensor` with size `(bsz, ourput_beam_size)`): + The indices of each beam. + """ + bsz, beam_size, vocab_size = lprobs.size() + k = min( + # Take the best 2 x beam_size predictions. We'll choose the first + # beam_size of these which don't predict eos to continue with. + beam_size * 2, + lprobs.view(bsz, -1).size(1) - 1, # -1 so we never select pad + ) + s_list: List[Tensor] + i_list: List[Tensor] + s_list = [torch.empty(0).to(lprobs) for i in range(beam_size)] + i_list = [ + torch.LongTensor().to(device=lprobs.device) + for i in range(beam_size) + ] + sibling_score = torch.arange(1, k + 1).to(lprobs) * self.diversity_rate + + if step == 0: + return self.beam.step(step, lprobs, scores) + lprobs.add_(scores[:, :, step - 1].unsqueeze(-1)) + + # 1/ Calculate hypotheses for each beam + for i in range(beam_size): + torch.topk( + lprobs[:, i, :].view(bsz, -1), k, out=(s_list[i], i_list[i])) + i_list[i].fmod_(vocab_size) + + # 2/ Intra-sibling ordering by default from topk + 3/ Rewrite scores + s_list[i].sub_(sibling_score) + + # 4/ Choose top K hypotheses + indices = torch.stack(i_list, dim=1).view(bsz, -1) + + final_scores = torch.empty(0).to(lprobs) + final_indices = torch.LongTensor().to(device=lprobs.device) + final_beams = torch.LongTensor().to(device=lprobs.device) + (final_scores, final_indices) = torch.topk( + torch.stack(s_list, dim=1).view(bsz, -1), + k, + ) + + final_beams = final_indices // k + + for i in range(bsz): + final_indices[i] = indices[i][final_indices[i]] + + return final_scores, final_indices, final_beams diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/sequence_generator.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/sequence_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c86f171e4eab69fd055def3877f9cefe978e6f0b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/sequence_generator.py @@ -0,0 +1,975 @@ +# Copyright 2022 The OFA-Sys Team. +# All rights reserved. +# This source code is licensed under the Apache 2.0 license +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 + +import math +import sys +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +from torch import Tensor + +from modelscope.models.multi_modal.ofa.generate import search +from modelscope.models.multi_modal.ofa.generate.ngram_repeat_block import \ + NGramRepeatBlock + + +def _expand_mask(mask: torch.Tensor, + dtype: torch.dtype, + tgt_len: Optional[int] = None): + r""" + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, + src_len).to(dtype) + return expanded_mask.masked_fill(expanded_mask.bool(), + torch.finfo(dtype).min) + + +class SequenceGenerator(nn.Module): + + def __init__(self, + tokenizer, + beam_size=1, + max_len_a=0, + max_len_b=200, + max_len=0, + min_len=1, + normalize_scores=True, + len_penalty=1.0, + unk_penalty=0.0, + temperature=1.0, + match_source_len=False, + no_repeat_ngram_size=0, + search_strategy=None, + eos=None, + symbols_to_strip_from_output=None, + lm_model=None, + lm_weight=1.0, + constraint_trie=None, + constraint_range=None, + gen_code=False, + gen_box=False, + ignore_eos=False, + zero_shot=False): + """Generates translations of a given source sentence. + + Args: + models (List[~fairseq.models.FairseqModel]): ensemble of models, + currently support fairseq.models.TransformerModel for scripting + beam_size (int, optional): beam width (default: 1) + max_len_a/b (int, optional): generate sequences of maximum length + ax + b, where x is the source length + max_len (int, optional): the maximum length of the generated output + (not including end-of-sentence) + min_len (int, optional): the minimum length of the generated output + (not including end-of-sentence) + normalize_scores (bool, optional): normalize scores by the length + of the output (default: True) + len_penalty (float, optional): length penalty, where <1.0 favors + shorter, >1.0 favors longer sentences (default: 1.0) + unk_penalty (float, optional): unknown word penalty, where <0 + produces more unks, >0 produces fewer (default: 0.0) + temperature (float, optional): temperature, where values + >1.0 produce more uniform samples and values <1.0 produce + sharper samples (default: 1.0) + match_source_len (bool, optional): outputs should match the source + length (default: False) + """ + super().__init__() + self.gen_code = gen_code + self.gen_box = gen_box + self.ignore_eos = ignore_eos + self.tokenizer = tokenizer + self.tgt_dict = { + value: key + for key, value in tokenizer.get_vocab().items() + } + added = { + value: key + for key, value in tokenizer.get_added_vocab().items() + } + self.tgt_dict.update(added) + self.pad = tokenizer.pad_token_id + self.unk = tokenizer.unk_token_id + self.bos = tokenizer.bos_token_id + self.eos = tokenizer.eos_token_id + self.symbols_to_strip_from_output = ( + symbols_to_strip_from_output.union({self.eos}) if + symbols_to_strip_from_output is not None else {self.bos, self.eos}) + self.vocab_size = len(self.tgt_dict) + self.beam_size = beam_size + # the max beam size is the dictionary size - 1, since we never select pad + self.beam_size = min(beam_size, self.vocab_size - 1) + self.max_len_a = max_len_a + self.max_len_b = max_len_b + self.min_len = min_len + self.max_len = max_len + + self.normalize_scores = normalize_scores + self.len_penalty = len_penalty + self.unk_penalty = unk_penalty + self.temperature = temperature + self.match_source_len = match_source_len + self.zero_shot = zero_shot + + if no_repeat_ngram_size > 0: + self.repeat_ngram_blocker = NGramRepeatBlock(no_repeat_ngram_size) + else: + self.repeat_ngram_blocker = None + + assert temperature > 0, '--temperature must be greater than 0' + + self.search = ( + search.BeamSearch(self.tokenizer) + if search_strategy is None else search_strategy) + # We only need to set src_lengths in LengthConstrainedBeamSearch. + # As a module attribute, setting it would break in multithread + # settings when the model is shared. + self.should_set_src_lengths = ( + hasattr(self.search, 'needs_src_lengths') + and self.search.needs_src_lengths) + + self.lm_model = lm_model + self.lm_weight = lm_weight + if self.lm_model is not None: + self.lm_model.eval() + + self.constraint_trie = constraint_trie + + self.constraint_start = None + self.constraint_end = None + if constraint_range is not None: + constraint_start, constraint_end = constraint_range.split(',') + self.constraint_start = int(constraint_start) + self.constraint_end = int(constraint_end) + + @torch.no_grad() + def forward( + self, + sample: Dict[str, Dict[str, Tensor]], + prefix_tokens: Optional[Tensor] = None, + bos_token: Optional[int] = None, + ): + """Generate a batch of translations. + + Args: + sample (dict): batch + prefix_tokens (torch.LongTensor, optional): force decoder to begin + with these tokens + bos_token (int, optional): beginning of sentence token + (default: self.eos) + """ + return self._generate(sample, prefix_tokens, bos_token=bos_token) + + @torch.no_grad() + def generate(self, models, sample: Dict[str, Dict[str, Tensor]], + **kwargs) -> List[List[Dict[str, Tensor]]]: + """Generate translations. Match the api of other fairseq generators. + + Args: + models (List[~fairseq.models.FairseqModel]): ensemble of models + sample (dict): batch + prefix_tokens (torch.LongTensor, optional): force decoder to begin + with these tokens + constraints (torch.LongTensor, optional): force decoder to include + the list of constraints + bos_token (int, optional): beginning of sentence token + (default: self.eos) + """ + return self._generate(models, sample, **kwargs) + + def _generate( + self, + models, + sample: Dict[str, Dict[str, Tensor]], + prefix_tokens: Optional[Tensor] = None, + constraints: Optional[Tensor] = None, + bos_token: Optional[int] = None, + ): + model = EnsembleModel(models) + incremental_states = torch.jit.annotate( + List[Tuple[Tuple[torch.Tensor]]], + [ + torch.jit.annotate(Tuple[Tuple[torch.Tensor]], {}) + for i in range(model.models_size) + ], + ) + net_input = sample['net_input'] + + if 'src_tokens' in net_input: + src_tokens = net_input['src_tokens'] + # length of the source text being the character length except EndOfSentence and pad + src_lengths = ((src_tokens.ne(self.eos) + & src_tokens.ne(self.pad)).long().sum(dim=1)) + elif 'input_ids' in net_input: + src_tokens = net_input['input_ids'] + # length of the source text being the character length except EndOfSentence and pad + src_lengths = ((src_tokens.ne(self.eos) + & src_tokens.ne(self.pad)).long().sum(dim=1)) + elif 'source' in net_input: + src_tokens = net_input['source'] + src_lengths = ( + net_input['padding_mask'].size(-1) + - net_input['padding_mask'].sum(-1) + if net_input['padding_mask'] is not None else torch.tensor( + src_tokens.size(-1)).to(src_tokens)) + elif 'features' in net_input: + src_tokens = net_input['features'] + src_lengths = ( + net_input['padding_mask'].size(-1) + - net_input['padding_mask'].sum(-1) + if net_input['padding_mask'] is not None else torch.tensor( + src_tokens.size(-1)).to(src_tokens)) + elif 'fbank' in net_input: + src_tokens = net_input['fbank'] + src_lengths = net_input['fbank_length'] + else: + raise Exception( + 'expected src_tokens or source in net input. input keys: ' + + str(net_input.keys())) + + # bsz: total number of sentences in beam + # Note that src_tokens may have more than 2 dimensions (i.e. audio features) + bsz, src_len = src_tokens.size()[:2] + beam_size = self.beam_size + + if constraints is not None and not self.search.supports_constraints: + raise NotImplementedError( + "Target-side constraints were provided, but search method doesn't support them" + ) + + # Initialize constraints, when active + self.search.init_constraints(constraints, beam_size) + + max_len: int = -1 + if self.match_source_len: + max_len = src_lengths.max().item() + else: + max_len = int(self.max_len_a * src_len + self.max_len_b) + assert ( + self.min_len <= max_len + ), 'min_len cannot be larger than max_len, please adjust these!' + # compute the encoder output for each beam + with torch.autograd.profiler.record_function( + 'EnsembleModel: forward_encoder'): + encoder_outs = model.forward_encoder(net_input) + + # placeholder of indices for bsz * beam_size to hold tokens and accumulative scores + new_order = torch.arange(bsz).view(-1, 1).repeat(1, beam_size).view(-1) + new_order = new_order.to(src_tokens.device).long() + encoder_outs = model.reorder_encoder_out(encoder_outs, new_order) + # ensure encoder_outs is a List. + assert encoder_outs is not None + + # initialize buffers + scores = (torch.zeros(bsz * beam_size, + max_len + 1).to(src_tokens).float() + ) # +1 for eos; pad is never chosen for scoring + tokens = (torch.zeros(bsz * beam_size, + max_len + 2).to(src_tokens).long().fill_( + self.pad)) # +2 for eos and pad + tokens[:, 0] = self.bos + attn: Optional[Tensor] = None + + # A list that indicates candidates that should be ignored. + # For example, suppose we're sampling and have already finalized 2/5 + # samples. Then cands_to_ignore would mark 2 positions as being ignored, + # so that we only finalize the remaining 3 samples. + cands_to_ignore = (torch.zeros(bsz, beam_size).to(src_tokens).eq(-1) + ) # forward and backward-compatible False mask + + # list of completed sentences + finalized = torch.jit.annotate( + List[List[Dict[str, Tensor]]], + [ + torch.jit.annotate(List[Dict[str, Tensor]], []) + for i in range(bsz) + ], + ) # contains lists of dictionaries of infomation about the hypothesis being finalized at each step + + # a boolean array indicating if the sentence at the index is finished or not + finished = [False for i in range(bsz)] + num_remaining_sent = bsz # number of sentences remaining + + # number of candidate hypos per step + cand_size = 2 * beam_size # 2 x beam size in case half are EOS + + # offset arrays for converting between different indexing schemes + bbsz_offsets = ((torch.arange(0, bsz) + * beam_size).unsqueeze(1).type_as(tokens).to( + src_tokens.device)) + cand_offsets = torch.arange(0, cand_size).type_as(tokens).to( + src_tokens.device) + + reorder_state: Optional[Tensor] = None + batch_idxs: Optional[Tensor] = None + + original_batch_idxs: Optional[Tensor] = None + if 'id' in sample and isinstance(sample['id'], Tensor): + original_batch_idxs = sample['id'] + else: + original_batch_idxs = torch.arange(0, bsz).type_as(tokens) + + for step in range(max_len + 1): # one extra step for EOS marker + # reorder decoder internal states based on the prev choice of beams + if reorder_state is not None: + if batch_idxs is not None: + # update beam indices to take into account removed sentences + corr = batch_idxs - torch.arange( + batch_idxs.numel()).type_as(batch_idxs) + reorder_state.view(-1, beam_size).add_( + corr.unsqueeze(-1) * beam_size) + original_batch_idxs = original_batch_idxs[batch_idxs] + model.reorder_incremental_state(incremental_states, + reorder_state) + encoder_outs = model.reorder_encoder_out( + encoder_outs, reorder_state) + + with torch.autograd.profiler.record_function( + 'EnsembleModel: forward_decoder'): + lprobs, avg_attn_scores = model.forward_decoder( + tokens[:, :step + 1], + encoder_outs, + incremental_states, + self.temperature, + constraint_trie=self.constraint_trie, + constraint_start=self.constraint_start, + constraint_end=self.constraint_end, + gen_code=self.gen_code, + zero_shot=self.zero_shot, + prefix_tokens=prefix_tokens) + + if self.lm_model is not None: + lm_out = self.lm_model(tokens[:, :step + 1]) + probs = self.lm_model.get_normalized_probs( + lm_out, log_probs=True, sample=None) + probs = probs[:, -1, :] * self.lm_weight + lprobs += probs + # handle prefix tokens (possibly with different lengths) + if (prefix_tokens is not None and step < prefix_tokens.size(1) + and step < max_len): + lprobs, tokens, scores = self._prefix_tokens( + step, lprobs, scores, tokens, prefix_tokens, beam_size) + elif step < self.min_len: + # minimum length constraint (does not apply if using prefix_tokens) + lprobs[:, self.eos] = -math.inf + + lprobs[lprobs != lprobs] = torch.tensor(-math.inf).to(lprobs) + + lprobs[:, self.pad] = -math.inf # never select pad + lprobs[:, self.unk] -= self.unk_penalty # apply unk penalty + + if (self.gen_code or self.gen_box) and step < max_len: + lprobs[:, :4] = -math.inf + if self.gen_box: + lprobs[:, -1] = -math.inf + if (step + 1) % 5 == 0: + lprobs[:, self.constraint_start:59457] = -math.inf + else: + lprobs[:, 59457:] = -math.inf + + # handle max length constraint + if step >= max_len: + lprobs[:, :self.eos] = -math.inf + lprobs[:, self.eos + 1:] = -math.inf + if self.ignore_eos: + lprobs[:, self.eos] = 1 + + # Record attention scores, only support avg_attn_scores is a Tensor + if avg_attn_scores is not None: + if attn is None: + attn = torch.empty(bsz * beam_size, + avg_attn_scores.size(1), + max_len + 2).to(scores) + attn[:, :, step + 1].copy_(avg_attn_scores) + + scores = scores.type_as(lprobs) + eos_bbsz_idx = torch.empty(0).to( + tokens + ) # indices of hypothesis ending with eos (finished sentences) + eos_scores = torch.empty(0).to( + scores + ) # scores of hypothesis ending with eos (finished sentences) + + if self.should_set_src_lengths: + self.search.set_src_lengths(src_lengths) + + if self.repeat_ngram_blocker is not None: + # process prefix_tokens + p_toks_len = prefix_tokens.ne(self.pad).sum( + dim=1) if prefix_tokens is not None else None + if p_toks_len is not None: + p_toks_len_beam = p_toks_len.unsqueeze(-1).repeat( + 1, beam_size).view(-1) + no_repeat_ngram_size = self.repeat_ngram_blocker.no_repeat_ngram_size + out_prefix = p_toks_len_beam < ( + step + no_repeat_ngram_size - 1) + else: + out_prefix = torch.ones(bsz * beam_size).bool() + ngram_blocker_tokens = tokens[out_prefix] + ngram_blocker_lprobs = lprobs[out_prefix] + ngram_blocker_bsz = torch.div( + out_prefix.sum(), beam_size, rounding_mode='trunc') + + lprobs[out_prefix] = self.repeat_ngram_blocker( + tokens=ngram_blocker_tokens, + lprobs=ngram_blocker_lprobs, + bsz=ngram_blocker_bsz, + beam_size=beam_size, + step=step) + + # Shape: (batch, cand_size) + cand_scores, cand_indices, cand_beams = self.search.step( + step, + lprobs.view(bsz, -1, self.vocab_size), + scores.view(bsz, beam_size, -1)[:, :, :step], + tokens[:, :step + 1], + original_batch_idxs, + ) + # cand_bbsz_idx contains beam indices for the top candidate + # hypotheses, with a range of values: [0, bsz*beam_size), + # and dimensions: [bsz, cand_size] + cand_bbsz_idx = cand_beams.add(bbsz_offsets) + + # finalize hypotheses that end in eos + # Shape of eos_mask: (batch size, beam size) + eos_mask = cand_indices.eq(self.eos) & cand_scores.ne(-math.inf) + eos_mask[:, :beam_size][cands_to_ignore] = torch.tensor(0).to( + eos_mask) + + # only consider eos when it's among the top beam_size indices + # Now we know what beam item(s) to finish + # Shape: 1d list of absolute-numbered + eos_bbsz_idx = torch.masked_select( + cand_bbsz_idx[:, :beam_size], mask=eos_mask[:, :beam_size]) + + finalized_sents: List[int] = [] + if eos_bbsz_idx.numel() > 0: + eos_scores = torch.masked_select( + cand_scores[:, :beam_size], mask=eos_mask[:, :beam_size]) + + finalized_sents = self.finalize_hypos( + step, + eos_bbsz_idx, + eos_scores, + tokens, + scores, + finalized, + finished, + beam_size, + attn, + src_lengths, + max_len, + ) + num_remaining_sent -= len(finalized_sents) + + assert num_remaining_sent >= 0 + if num_remaining_sent == 0: + break + if self.search.stop_on_max_len and step >= max_len: + break + assert step < max_len, f'{step} < {max_len}' + + # Remove finalized sentences (ones for which {beam_size} + # finished hypotheses have been generated) from the batch. + if len(finalized_sents) > 0: + new_bsz = bsz - len(finalized_sents) + + # construct batch_idxs which holds indices of batches to keep for the next pass + batch_mask = torch.ones( + bsz, dtype=torch.bool, device=cand_indices.device) + batch_mask[finalized_sents] = False + batch_idxs = torch.arange( + bsz, device=cand_indices.device).masked_select(batch_mask) + + # Choose the subset of the hypothesized constraints that will continue + self.search.prune_sentences(batch_idxs) + + eos_mask = eos_mask[batch_idxs] + cand_beams = cand_beams[batch_idxs] + bbsz_offsets.resize_(new_bsz, 1) + cand_bbsz_idx = cand_beams.add(bbsz_offsets) + cand_scores = cand_scores[batch_idxs] + cand_indices = cand_indices[batch_idxs] + + if prefix_tokens is not None: + prefix_tokens = prefix_tokens[batch_idxs] + src_lengths = src_lengths[batch_idxs] + cands_to_ignore = cands_to_ignore[batch_idxs] + + scores = scores.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, -1) + tokens = tokens.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, -1) + if attn is not None: + attn = attn.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, attn.size(1), -1) + bsz = new_bsz + else: + batch_idxs = None + + # Set active_mask so that values > cand_size indicate eos hypos + # and values < cand_size indicate candidate active hypos. + # After, the min values per row are the top candidate active hypos + + # Rewrite the operator since the element wise or is not supported in torchscript. + + eos_mask[:, :beam_size] = ~( # noqa + (~cands_to_ignore) & (~eos_mask[:, :beam_size])) # noqa + active_mask = torch.add( + eos_mask.type_as(cand_offsets) * cand_size, + cand_offsets[:eos_mask.size(1)], + ) + + # get the top beam_size active hypotheses, which are just + # the hypos with the smallest values in active_mask. + # {active_hypos} indicates which {beam_size} hypotheses + # from the list of {2 * beam_size} candidates were + # selected. Shapes: (batch size, beam size) + new_cands_to_ignore, active_hypos = torch.topk( + active_mask, k=beam_size, dim=1, largest=False) + + # update cands_to_ignore to ignore any finalized hypos. + cands_to_ignore = new_cands_to_ignore.ge(cand_size)[:, :beam_size] + # Make sure there is at least one active item for each sentence in the batch. + assert (~cands_to_ignore).any(dim=1).all() + + # update cands_to_ignore to ignore any finalized hypos + + # {active_bbsz_idx} denotes which beam number is continued for each new hypothesis (a beam + # can be selected more than once). + active_bbsz_idx = torch.gather( + cand_bbsz_idx, dim=1, index=active_hypos) + active_scores = torch.gather( + cand_scores, dim=1, index=active_hypos) + + active_bbsz_idx = active_bbsz_idx.view(-1) + active_scores = active_scores.view(-1) + + # copy tokens and scores for active hypotheses + + # Set the tokens for each beam (can select the same row more than once) + tokens[:, :step + 1] = torch.index_select( + tokens[:, :step + 1], dim=0, index=active_bbsz_idx) + # Select the next token for each of them + tokens.view(bsz, beam_size, -1)[:, :, step + 1] = torch.gather( + cand_indices, dim=1, index=active_hypos) + if step > 0: + scores[:, :step] = torch.index_select( + scores[:, :step], dim=0, index=active_bbsz_idx) + scores.view(bsz, beam_size, -1)[:, :, step] = torch.gather( + cand_scores, dim=1, index=active_hypos) + + # Update constraints based on which candidates were selected for the next beam + self.search.update_constraints(active_hypos) + + # copy attention for active hypotheses + if attn is not None: + attn[:, :, :step + 2] = torch.index_select( + attn[:, :, :step + 2], dim=0, index=active_bbsz_idx) + + # reorder incremental state in decoder + reorder_state = active_bbsz_idx + + # sort by score descending + for sent in range(len(finalized)): + scores = torch.tensor( + [float(elem['score'].item()) for elem in finalized[sent]]) + _, sorted_scores_indices = torch.sort(scores, descending=True) + finalized[sent] = [ + finalized[sent][ssi] for ssi in sorted_scores_indices + ] + finalized[sent] = torch.jit.annotate(List[Dict[str, Tensor]], + finalized[sent]) + return finalized + + def _prefix_tokens(self, step: int, lprobs, scores, tokens, prefix_tokens, + beam_size: int): + """Handle prefix tokens""" + prefix_toks = prefix_tokens[:, step].unsqueeze(-1).repeat( + 1, beam_size).view(-1) + prefix_lprobs = lprobs.gather(-1, prefix_toks.unsqueeze(-1)) + prefix_mask = prefix_toks.ne(self.pad) + if self.constraint_trie is None: + lprobs[prefix_mask] = torch.min(prefix_lprobs) - 1 + else: + lprobs[prefix_mask] = -math.inf + lprobs[prefix_mask] = lprobs[prefix_mask].scatter( + -1, prefix_toks[prefix_mask].unsqueeze(-1), + prefix_lprobs[prefix_mask]) + # if prefix includes eos, then we should make sure tokens and + # scores are the same across all beams + eos_mask = prefix_toks.eq(self.eos) + if eos_mask.any(): + # validate that the first beam matches the prefix + first_beam = tokens[eos_mask].view(-1, beam_size, + tokens.size(-1))[:, 0, + 1:step + 1] + eos_mask_batch_dim = eos_mask.view(-1, beam_size)[:, 0] + target_prefix = prefix_tokens[eos_mask_batch_dim][:, :step] + assert (first_beam == target_prefix).all() + + # copy tokens, scores and lprobs from the first beam to all beams + tokens = self.replicate_first_beam(tokens, eos_mask_batch_dim, + beam_size) + scores = self.replicate_first_beam(scores, eos_mask_batch_dim, + beam_size) + lprobs = self.replicate_first_beam(lprobs, eos_mask_batch_dim, + beam_size) + return lprobs, tokens, scores + + def replicate_first_beam(self, tensor, mask, beam_size: int): + tensor = tensor.view(-1, beam_size, tensor.size(-1)) + tensor[mask] = tensor[mask][:, :1, :] + return tensor.view(-1, tensor.size(-1)) + + def finalize_hypos( + self, + step: int, + bbsz_idx, + eos_scores, + tokens, + scores, + finalized: List[List[Dict[str, Tensor]]], + finished: List[bool], + beam_size: int, + attn: Optional[Tensor], + src_lengths, + max_len: int, + ): + """Finalize hypothesis, store finalized information in `finalized`, and change `finished` accordingly. + A sentence is finalized when {beam_size} finished items have been collected for it. + + Returns number of sentences (not beam items) being finalized. + These will be removed from the batch and not processed further. + Args: + bbsz_idx (Tensor): + """ + assert bbsz_idx.numel() == eos_scores.numel() + + # clone relevant token and attention tensors. + # tokens is (batch * beam, max_len). So the index_select + # gets the newly EOS rows, then selects cols 1..{step + 2} + tokens_clone = tokens.index_select( + 0, bbsz_idx)[:, 1:step + 2] # skip the first index, which is EOS + + tokens_clone[:, step] = self.eos + attn_clone = ( + attn.index_select(0, bbsz_idx)[:, :, 1:step + + 2] if attn is not None else None) + + # compute scores per token position + pos_scores = scores.index_select(0, bbsz_idx)[:, :step + 1] + pos_scores[:, step] = eos_scores + # convert from cumulative to per-position scores + pos_scores[:, 1:] = pos_scores[:, 1:] - pos_scores[:, :-1] + + # normalize sentence-level scores + if self.normalize_scores: + eos_scores /= (step + 1)**self.len_penalty + + # cum_unfin records which sentences in the batch are finished. + # It helps match indexing between (a) the original sentences + # in the batch and (b) the current, possibly-reduced set of + # sentences. + cum_unfin: List[int] = [] + prev = 0 + for f in finished: + if f: + prev += 1 + else: + cum_unfin.append(prev) + cum_fin_tensor = torch.tensor(cum_unfin, dtype=torch.int).to(bbsz_idx) + + unfin_idx = torch.div(bbsz_idx, beam_size, rounding_mode='floor') + sent = unfin_idx + torch.index_select(cum_fin_tensor, 0, unfin_idx) + + # Create a set of "{sent}{unfin_idx}", where + # "unfin_idx" is the index in the current (possibly reduced) + # list of sentences, and "sent" is the index in the original, + # unreduced batch + # For every finished beam item + # sentence index in the current (possibly reduced) batch + seen = (sent << 32) + unfin_idx + unique_seen: List[int] = torch.unique(seen).tolist() + + if self.match_source_len: + condition = step > torch.index_select(src_lengths, 0, unfin_idx) + eos_scores = torch.where(condition, torch.tensor(-math.inf), + eos_scores) + sent_list: List[int] = sent.tolist() + for i in range(bbsz_idx.size()[0]): + # An input sentence (among those in a batch) is finished when + # beam_size hypotheses have been collected for it + if len(finalized[sent_list[i]]) < beam_size: + if attn_clone is not None: + # remove padding tokens from attn scores + hypo_attn = attn_clone[i] + else: + hypo_attn = torch.empty(0) + + finalized[sent_list[i]].append({ + 'tokens': + tokens_clone[i], + 'score': + eos_scores[i], + 'attention': + hypo_attn, # src_len x tgt_len + 'alignment': + torch.empty(0), + 'positional_scores': + pos_scores[i], + }) + + newly_finished: List[int] = [] + for unique_s in unique_seen: + # check termination conditions for this sentence + unique_sent: int = unique_s >> 32 + unique_unfin_idx: int = unique_s - (unique_sent << 32) + + if not finished[unique_sent] and self.is_finished( + step, unique_unfin_idx, max_len, len( + finalized[unique_sent]), beam_size): + finished[unique_sent] = True + newly_finished.append(unique_unfin_idx) + + return newly_finished + + def is_finished( + self, + step: int, + unfin_idx: int, + max_len: int, + finalized_sent_len: int, + beam_size: int, + ): + """ + Check whether decoding for a sentence is finished, which + occurs when the list of finalized sentences has reached the + beam size, or when we reach the maximum length. + """ + assert finalized_sent_len <= beam_size + if finalized_sent_len == beam_size or step == max_len: + return True + return False + + +class EnsembleModel(nn.Module): + """A wrapper around an ensemble of models.""" + + def __init__(self, models): + super().__init__() + self.models_size = len(models) + # method '__len__' is not supported in ModuleList for torch script + self.single_model = models[0] + self.models = nn.ModuleList(models) + + # self.has_incremental: bool = False + # if all( + # hasattr(m, "decoder") and isinstance(m.decoder, FairseqIncrementalDecoder) + # for m in models + # ): + # self.has_incremental = True + + self.has_incremental = True + + def forward(self): + pass + + def has_encoder(self): + return hasattr(self.single_model, 'encoder') + + def has_incremental_states(self): + return self.has_incremental + + def max_decoder_positions(self): + return min([ + m.max_decoder_positions() + for m in self.models if hasattr(m, 'max_decoder_positions') + ] + [sys.maxsize]) # + + @torch.jit.export + def forward_encoder(self, net_input: Dict[str, Tensor]): + if not self.has_encoder(): + return None + encoder_input = { + k: v + for k, v in net_input.items() if k != 'decoder_input_ids' + } + encoder_input['output_hidden_states'] = True + return [ + model.encoder.forward(**encoder_input) for model in self.models + ] + + @torch.jit.export + def forward_decoder(self, + tokens, + encoder_outs: List[Dict[str, List[Tensor]]], + incremental_states: List[Optional[torch.Tensor]], + temperature: float = 1.0, + constraint_trie=None, + constraint_start=None, + constraint_end=None, + gen_code=False, + zero_shot=False, + prefix_tokens=None): + log_probs = [] + avg_attn: Optional[Tensor] = None + encoder_out: Optional[Dict[str, List[Tensor]]] = None + code_mask = (tokens.new_ones(tokens.size(0)) * gen_code).bool() + + for i, model in enumerate(self.models): + if self.has_encoder(): + encoder_out = encoder_outs[i] + encoder_hidden_states = encoder_out.last_hidden_state + encoder_attention_mask = _expand_mask( + encoder_out.padding_mask, encoder_hidden_states.dtype, + tokens.shape[-1]) + src_pos_embed = encoder_out.position_embedding + + # if tokens.eq(self.single_model.config.pad_token_id).any(): + attention_mask = tokens.eq(self.single_model.padding_idx) + + # decode each model + if self.has_incremental_states(): + decoder_out = model.decoder.forward( + input_ids=tokens, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + code_masks=code_mask, + src_pos_embed=src_pos_embed, + past_key_values=incremental_states[i], + use_cache=True, + output_attentions=True) + else: + if hasattr(model, 'decoder'): + # decoder_out = model.decoder.forward(tokens, code_masks=code_mask, encoder_out=encoder_out) + decoder_out = model.decoder.forward( + input_ids=tokens, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + code_masks=code_mask, + src_pos_embed=src_pos_embed) + else: + decoder_out = model.forward(tokens) + + attn: Optional[Tensor] = None + decoder_len = len(decoder_out) + + if 'cross_attentions' in decoder_out: + attn = decoder_out['cross_attentions'][-1].transpose(1, 0) + attn = attn.mean(dim=0) # (B, tgt_len, src_len) + if attn is not None: + attn = attn[:, -1, :] + + decoder_out_tuple = ( + decoder_out[0][:, -1:, :].div_(temperature), + None if decoder_len <= 1 else attn, + ) + + beam_size = decoder_out_tuple[0].size(0) // prefix_tokens.size( + 0) if prefix_tokens is not None else 0 + if constraint_trie is not None and not zero_shot: + assert constraint_start is None and constraint_end is None + constraint_masks = decoder_out_tuple[0].new_zeros( + decoder_out_tuple[0].size()).bool() + constraint_prefix_tokens = tokens.tolist() + for token_index, constraint_prefix_token in enumerate( + constraint_prefix_tokens): + prefix_len = prefix_tokens[token_index // beam_size].ne( + 1).sum().item() if prefix_tokens is not None else 0 + if len(constraint_prefix_token) > prefix_len: + constraint_prefix_token = [ + 0 + ] + constraint_prefix_token[prefix_len + 1:] + constraint_nodes = constraint_trie.get_next_layer( + constraint_prefix_token) + constraint_masks[token_index][:, + constraint_nodes] = True + else: + constraint_masks[token_index] = True + decoder_out_tuple[0].masked_fill_(~constraint_masks, -math.inf) + if constraint_start is not None and constraint_end is not None and not zero_shot: + assert constraint_trie is None + decoder_out_tuple[0][:, :, 4:constraint_start] = -math.inf + decoder_out_tuple[0][:, :, constraint_end:] = -math.inf + + probs = model.get_normalized_probs( + decoder_out_tuple, log_probs=True, sample=None) + if constraint_trie is not None and zero_shot: + assert constraint_start is None and constraint_end is None + constraint_masks = decoder_out_tuple[0].new_zeros( + decoder_out_tuple[0].size()).bool() + constraint_prefix_tokens = tokens.tolist() + for token_index, constraint_prefix_token in enumerate( + constraint_prefix_tokens): + constraint_nodes = constraint_trie.get_next_layer( + constraint_prefix_token) + constraint_masks[token_index][:, constraint_nodes] = True + probs.masked_fill_(~constraint_masks, -math.inf) + if constraint_start is not None and constraint_end is not None and zero_shot: + assert constraint_trie is None + probs[:, :, 4:constraint_start] = -math.inf + probs[:, :, constraint_end:] = -math.inf + probs = probs[:, -1, :] + if self.models_size == 1: + return probs, attn + + log_probs.append(probs) + if attn is not None: + if avg_attn is None: + avg_attn = attn + else: + avg_attn.add_(attn) + + avg_probs = torch.logsumexp( + torch.stack(log_probs, dim=0), dim=0) - math.log(self.models_size) + + if avg_attn is not None: + avg_attn.div_(self.models_size) + return avg_probs, avg_attn + + @torch.jit.export + def reorder_encoder_out(self, + encoder_outs: Optional[List[Dict[str, + List[Tensor]]]], + new_order): + """ + Reorder encoder output according to *new_order*. + + Args: + encoder_out: output from the ``forward()`` method + new_order (LongTensor): desired order + + Returns: + *encoder_out* rearranged according to *new_order* + """ + new_outs: List[Dict[str, List[Tensor]]] = [] + if not self.has_encoder(): + return new_outs + for i, model in enumerate(self.models): + assert encoder_outs is not None + new_outs.append( + model.encoder.reorder_encoder_out(encoder_outs[i], new_order)) + return new_outs + + @torch.jit.export + def reorder_incremental_state( + self, + incremental_states: List[Optional[torch.Tensor]], + new_order, + ): + if not self.has_incremental_states(): + return + for i, model in enumerate(self.models): + model.decoder.reorder_incremental_state_scripting( + incremental_states[i], new_order) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/token_generation_constraints.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/token_generation_constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..13fb3fcf653492bfc8dac8b2e24bb0e55d822ddb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/token_generation_constraints.py @@ -0,0 +1,512 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/facebookresearch/fairseq/blob/main/LICENSE +"""Implements tracking of constraints for a beam item. + +A list of constraints is given as a list of one or more token +sequences, each of length at least one token. For example, for an input sentence + +> Die maschinelle Übersetzung ist schwer zu kontrollieren. + +We could have the constraints: +* to influence +* hard + +There are two implementations: +* OrderedConstraintState: Tracks progress through an ordered list of multitoken constraints. +* UnorderedConstraintState: Tracks progress through an unordered list of multitoken constraints. + +The difference is that in the first, the constraints are assumed to be +in order; the algorithm will permit zero or more tokens between them. +In the second, the constraints are not ordered, so many orderings will +be explored. + +The same sequence can be present any number of times, and will appear +that many times in the output. +""" + +from collections import Counter +from typing import List, Set + +import torch + + +class ConstraintState: + + def __init__(self): + pass + + +def pack_constraints( + batch_constraints: List[List[torch.Tensor]]) -> torch.Tensor: + """Takes a list of list of constraints in tensor form (a list of + tensor constraints for each sentence) and transforms it into a + packed Tensor. For example, here is a batch of size 3 with 3, 0, + and 1 constraints: + + [ [ [3 1 2], [3], [4 5 6 7], ] + [], + [ [1 8 9 10 1 4 11 12], ] + ] + + Its corresponding packed structure is: + + [ [ 3 3 1 2 0 3 0 4 5 6 7 0], + [ 0 0 0 0 0 0 0 0 0 0 0 0], + [ 1 1 8 9 10 1 4 11 12 0 0 0] ] + + The packed tensor has shape (batch size, maxlen), where + maxlen is defined below. Each row contains concatenated + constraint tokens for that sentence, with 0 appended after + each constraint. The first item in each row is the number + of constraints for that sentence. So maxlen is the maximum + of + + (number of constraints) + (sum length of constraints) + 1. + + across all sentences in the batch. + """ + # The maximum word length of concatenated constraints for any sentence + max_constraints_len = 1 + for sentence_constraints in batch_constraints: + if len(sentence_constraints): + # number of constraints, plus sum of constrain lens, plus a zero after each + constraints_len = (1 + + sum([c.size(0) for c in sentence_constraints]) + + len(sentence_constraints)) + max_constraints_len = max(max_constraints_len, constraints_len) + + batch_size = len(batch_constraints) + constraints_tensor = torch.zeros((batch_size, max_constraints_len)).long() + for i, sentence_constraints in enumerate(batch_constraints): + constraints_tensor[i, 0] = len(sentence_constraints) + offset = 1 + for j, constraint in enumerate(sentence_constraints): + this_len = constraint.size(0) + constraints_tensor[i, offset:offset + this_len] = constraint + offset += this_len + 1 + + return constraints_tensor.long() + + +def unpack_constraints(constraint_tensor: torch.Tensor) -> List[torch.Tensor]: + """ + Transforms *one row* of a packed constraint tensor (e.g., for one + sentence in the batch) into a list of constraint tensors. + """ + constraint_list = [] + num_constraints = constraint_tensor[0] + constraints = constraint_tensor.tolist() + offset = 1 + for i in range(num_constraints): + where = constraints.index(0, offset) + constraint_list.append(constraint_tensor[offset:where]) + offset = where + 1 + + return constraint_list + + +class ConstraintNode: + """ + Represents a node in a trie managing unordered constraints. + """ + + def __init__(self, token: int = None, parent=None): + # The token associate with this node (None for the root) + self.token = int(token) if token is not None else None + # The parent (None at the root) + self.parent = parent + # Whether this node is a completed constraint + self.terminal = 0 + # List of child nodes + self.children = {} + + # The cumulative number of constraints from this point in the + # trie forward + self.num_constraints = 0 + + @property + def id(self): + return self.token + + def __str__(self): + term = self.terminal != 0 + return f'[{self.token}].{term}#{self.num_constraints}' + + def __getitem__(self, key: int): + return self.children.get(key, None) + + def next_tokens(self) -> Set[int]: + """The set of child labels.""" + return set(self.children.keys()) + + @staticmethod + def create(constraints: List[List[int]]): + root = ConstraintNode() + for sequence in constraints: + root.add_sequence(sequence) + + return root + + @staticmethod + def print_graph(node: 'ConstraintNode'): + if len(node.children) == 0: + return str(node) + else: + s = f'({node}' + for child in node.children.values(): + s += ' ' + ConstraintNode.print_graph(child) + s += ')' + return s + + def token_counts(self) -> Counter: + """Returns a counter of the number of times each token is used + in a constraint. + """ + token_counts = Counter() + kids = list(self.children.values()) + while len(kids) > 0: + kid = kids.pop() + token_counts[kid.id] += kid.num_constraints + kids += list(kid.children.values()) + + return token_counts + + def tokens(self) -> Set[int]: + """Returns the set of tokens in constraints.""" + return set(self.token_counts().keys()) + + def add_sequence(self, sequence: List[int]): + """Adds a constraint, represented as a list of integers, to + the trie.""" + assert len(sequence) > 0 + + token = int(sequence[0]) + if token not in self.children: + self.children[token] = ConstraintNode(token, parent=self) + + node = self.children[token] + if len(sequence) == 1: + node.terminal += 1 + node.num_constraints += 1 + parent = node.parent + while parent is not None: + parent.num_constraints += 1 + parent = parent.parent + else: + node.add_sequence(sequence[1:]) + + +class UnorderedConstraintState(ConstraintState): + """ + Records progress through the set of constraints for each item in the beam + using a trie. + """ + + def __init__(self, + node: ConstraintNode, + copy_from: 'ConstraintState' = None): + self.node = node + + if copy_from is None: + # The root node + self.root = node + # The set of states in the graph that have been completed + self.completed = Counter() + # The... + self.generated = Counter() + # The list of tokens we need to generate + self.needed_tokens = self.root.tokens() + else: + self.completed = Counter(copy_from.completed) + self.generated = Counter(copy_from.generated) + self.root = copy_from.root + + # Mark the node as generated + if self.node != self.root: + self.generated[node] += 1 + + @staticmethod + def create(constraint_tensor: torch.Tensor): + constraint_list = unpack_constraints(constraint_tensor) + constraint_trie_root = ConstraintNode.create(constraint_list) + return UnorderedConstraintState(constraint_trie_root) + + def __str__(self): + gen_str = ','.join([str(node) for node in self.generated]) + return f'{self.name}/{self.bank}({gen_str})x{self.num_completed}' + + def __copy__(self): + copied_state = UnorderedConstraintState(self.node, copy_from=self) + return copied_state + + def copy(self): + return self.__copy__() + + @property + def name(self): + if self.node.id is None: + return 'ROOT' + else: + return str(self.node.id) + + @property + def is_root(self): + return self.node == self.root + + @property + def bank(self): + return sum(self.generated.values()) + + @property + def num_completed(self): + """The number of constraints (not constraint tokens) that are completed. + In addition to the already-completed states, we need to account for the + current state, which might get marked as completed when another token + is generated. + """ + in_final = self.node.terminal and self.completed[ + self.node] < self.node.terminal + return sum(self.completed.values()) + in_final + + @property + def finished(self): + return self.root.num_constraints - self.num_completed == 0 + + @property + def token_counts(self): + return self.root.token_counts() + + @property + def tokens(self): + return self.root.tokens() + + @property + def num_constraint_tokens(self): + return sum(self.token_counts.values()) + + def next_tokens(self) -> Set[int]: + """Returns the list of tokens that could come next. + These are (a) all tokens extending the root state and, for + non-root states, additionally all tokens extending the current + state.""" + + if self.node != self.root: + return self.root.next_tokens().union(self.node.next_tokens()) + else: + return self.root.next_tokens() + + def advance(self, token: int): + """Reads in a token and advances the state. Here's how it works. + + We can advance to the next state if: + - there is a matching child + - its path isn't blocked + + A path is blocked when all constraints that are descendants of + that node have already been generated, in the current state. + + If we are not able to advance from the current state, we "fall + off the graph" and return to the root state. There, we again + try to advance, checking the same criteria. + + In any case, when falling off the graph, we need to do some + bookkeeping. We: + - check whether any constraints were met (all prefixes of + current state) + - if one is found, mark it as completed + - adjust visited nodes accordingly + """ + token = int(token) + + next_state = None + child = self.node[token] + if child is not None and self.generated[child] < child.num_constraints: + next_state = UnorderedConstraintState(child, copy_from=self) + + def rewind(): + """If we're mid-trie and an "illegal" token is chosen next, we need + to reset our state to the root state. However, along the way, we need + to check whether a prefix of the current trie state represents a state + we could mark as completed. + """ + node = self.node + while node != self.root: + if node.terminal and self.completed[node] < node.terminal: + next_state.completed[node] += 1 + return + + next_state.generated[node] -= 1 + node = node.parent + + # Fall off the graph, check the root + if next_state is None and token in self.root.next_tokens(): + child = self.root[token] + # We can only traverse this edge if it's not saturated + if self.generated[child] < child.num_constraints: + next_state = UnorderedConstraintState(child, copy_from=self) + else: + next_state = UnorderedConstraintState( + self.root, copy_from=self) + + # Rewind + rewind() + + elif next_state is None: + next_state = UnorderedConstraintState(self.root, copy_from=self) + # Rewind + rewind() + + return next_state + + +class ConstraintSequence: + + def __init__(self, sequences: List[List[int]]): + """Represents a set of possibly multitoken constraints by + concatenating them and internally recording the end points. + """ + self.sequences = [] + self.endpoints = [] + self.num_tokens = 0 + self.tokens = set() + for sequence in sequences: + for token in sequence: + self.tokens.add(token) + self.num_tokens += len(sequence) + self.endpoints += [False + for x in range(len(sequence) - 1)] + [True] + self.sequences += sequence + + def __getitem__(self, key: int): + return self.sequences[key] + + def __len__(self): + return len(self.sequences) + + def __str__(self): + return str(self.sequences) + + +class OrderedConstraintState(ConstraintState): + """ + Records progress through the set of linear nonbranching constraints with gaps. + """ + + def __init__(self, sequence: ConstraintSequence, state: int = -1): + self.sequence = sequence + self.state = state + + @staticmethod + def create(constraint_tensor: torch.Tensor): + constraint_list = unpack_constraints(constraint_tensor) + return OrderedConstraintState(ConstraintSequence(constraint_list), -1) + + def __str__(self): + return f'{self.state}/{self.bank}x{self.num_completed}' + + def __copy__(self): + return OrderedConstraintState(self.sequence, self.state) + + def copy(self): + return self.__copy__() + + @property + def num_completed(self): + if self.state == -1: + return 0 + count = len( + list( + filter(lambda x: x, + self.sequence.endpoints[0:self.state + 1]))) + return count + + @property + def is_root(self): + return self.state == -1 + + @property + def name(self): + if self.state == -1: + return 'ROOT' + else: + return str(self.sequence[self.state]) + + @property + def bank(self) -> int: + return self.state + 1 + + @property + def finished(self): + return self.state + 1 == len(self.sequence) + + @property + def token_counts(self): + return self.sequence.token_counts() + + @property + def tokens(self): + return self.sequence.tokens + + @property + def num_constraint_tokens(self): + return sum(self.token_counts.values()) + + def next_tokens(self) -> Set[int]: + """Returns the list of tokens that could come next. + These are (a) all tokens extending the root state and, for + non-root states, additionally all tokens extending the current + state.""" + + tokens = set() + if self.state > 0: + tokens.add(self.sequence[0]) + if not self.finished: + tokens.add(self.sequence[self.state + 1]) + return tokens + + def advance(self, token: int): + """Reads in a token and advances the state. Here's how it works. + + We can advance to the next state if: + - there is a matching child + - its path isn't blocked + + A path is blocked when all constraints that are descendants of + that node have already been generated, in the current state. + + If we are not able to advance from the current state, we "fall + off the graph" and return to the root state. There, we again + try to advance, checking the same criteria. + + In any case, when falling off the graph, we need to do some + bookkeeping. We: + - check whether any constraints were met (all prefixes of + current state) + - if one is found, mark it as completed + - adjust visited nodes accordingly + """ + token = int(token) + # print(f"{self} ADVANCE({token}) {self.sequence} -> ", end="") + + if self.finished: + # Accept anything + next_state = self.copy() + + elif self.sequence[self.state + 1] == token: + # Advance to the next token + next_state = OrderedConstraintState(self.sequence, self.state + 1) + + elif self.sequence.endpoints[self.state]: + # Accept anything between constraints (*) + next_state = self.copy() + + elif token == self.sequence[0]: + # Start over having generated the first token + next_state = OrderedConstraintState(self.sequence, 0) + else: + # Start over from the root + next_state = OrderedConstraintState(self.sequence, -1) + + return next_state diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/generate/utils.py b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5a5a3eb5af9f3a18f9cc242f746faed4701888da --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/generate/utils.py @@ -0,0 +1,155 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/facebookresearch/fairseq/blob/main/LICENSE + +import collections +from collections import abc +from itertools import accumulate + +import torch +import torch.nn.functional as F + +try: + from amp_C import multi_tensor_l2norm + + multi_tensor_l2norm_available = True +except ImportError: + multi_tensor_l2norm_available = False + +try: + import torch_xla.core.xla_model as xm +except ImportError: + xm = None + +MANIFOLD_PATH_SEP = '|' + + +def apply_to_sample(f, sample): + r""" + Apply some function to the sample. The f function will effect on the `Tensor` object, otherwise do nothing. + """ + if hasattr(sample, '__len__') and len(sample) == 0: + return {} + + def _apply(x): + if torch.is_tensor(x): + return f(x) + elif isinstance(x, collections.OrderedDict): + # OrderedDict has attributes that needs to be preserved + od = collections.OrderedDict( + (key, _apply(value)) for key, value in x.items()) + od.__dict__ = x.__dict__ + return od + elif isinstance(x, dict): + return {key: _apply(value) for key, value in x.items()} + elif isinstance(x, list): + return [_apply(x) for x in x] + elif isinstance(x, tuple): + return tuple(_apply(x) for x in x) + elif isinstance(x, set): + return {_apply(x) for x in x} + else: + return x + + return _apply(sample) + + +def move_to_device(batch, device): + r""" + Puts each data field to the device + """ + if isinstance(batch, torch.Tensor): + return batch.to(device) + elif isinstance(batch, (list, tuple)): + return tuple(move_to_device(item, device) for item in batch) + elif isinstance(batch, abc.Mapping): + return { + key: move_to_device(value, device) + for key, value in batch.items() + } + else: + return batch + + +def strip_pad(tensor, pad): + r""" + Get the non pad value from input tensor + """ + return tensor[tensor.ne(pad)] + + +def get_token_to_word_mapping(tokens, exclude_list): + r""" + Get the token to word mapping. The token indicates the original token index, while word indicates the token index + excluding the `exclude_list`. + + >>> import torch + >>> all_tokens = torch.arange(4) + >>> exclude_tokens = [1] + >>> get_token_to_word_mapping(all_tokens, exclude_tokens) # {0: 1, 1: 1, 2: 2, 3: 3} + """ + n = len(tokens) + word_start = [int(token not in exclude_list) for token in tokens] + word_idx = list(accumulate(word_start)) + token_to_word = {i: word_idx[i] for i in range(n)} + return token_to_word + + +def extract_hard_alignment(attn, src_sent, tgt_sent, pad, eos): + r""" + @deprecated + There is no usage in this project, should be removed. + """ + tgt_valid = (((tgt_sent != pad) & # noqa + (tgt_sent != eos)).nonzero(as_tuple=False).squeeze(dim=-1)) + src_invalid = (((src_sent == pad) | # noqa + (src_sent == eos)).nonzero(as_tuple=False).squeeze(dim=-1)) + src_token_to_word = get_token_to_word_mapping(src_sent, [eos, pad]) + tgt_token_to_word = get_token_to_word_mapping(tgt_sent, [eos, pad]) + alignment = [] + if len(tgt_valid) != 0 and len(src_invalid) < len(src_sent): + attn_valid = attn[tgt_valid] + attn_valid[:, src_invalid] = float('-inf') + _, src_indices = attn_valid.max(dim=1) + for tgt_idx, src_idx in zip(tgt_valid, src_indices): + alignment.append(( + src_token_to_word[src_idx.item()] - 1, + tgt_token_to_word[tgt_idx.item()] - 1, + )) + return alignment + + +def softmax(x, dim: int, onnx_trace: bool = False): + r""" + softmax function. Using `torch.nn.functional.softmax` + """ + if onnx_trace: + return F.softmax(x.float(), dim=dim) + else: + return F.softmax(x, dim=dim, dtype=torch.float32) + + +def log_softmax(x, dim: int, onnx_trace: bool = False): + r""" + log softmax function. Using `torch.nn.functional.log_softmax` + """ + if onnx_trace: + return F.log_softmax(x.float(), dim=dim) + else: + return F.log_softmax(x, dim=dim, dtype=torch.float32) + + +def extract_soft_alignment(attn, src_sent, tgt_sent, pad, eos): + r""" + @deprecated + There is no usage in this project, should be removed. + """ + tgt_valid = (tgt_sent != pad).nonzero(as_tuple=False) + src_valid = (src_sent != pad).nonzero(as_tuple=False).squeeze(dim=-1) + alignment = [] + if len(tgt_valid) != 0 and len(src_valid) != 0: + attn_valid = attn[tgt_valid, src_valid] + alignment = [['{:.6f}'.format(p) for p in src_probs.tolist()] + for src_probs in attn_valid] + return alignment diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_mmspeech.py b/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_mmspeech.py new file mode 100644 index 0000000000000000000000000000000000000000..c71e23d06bac2ee627e7ae1b401b256459459f99 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_mmspeech.py @@ -0,0 +1,1069 @@ +# Copyright 2022 OFA-Sys Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OFA-MMSpeech model.""" + +import math +from dataclasses import dataclass +from typing import Optional, Tuple + +import numpy as np +import torch +import torch.distributed as dist +from fairseq.data.data_utils import compute_mask_indices +from fairseq.models.wav2vec.wav2vec2 import TransformerSentenceEncoderLayer +from fairseq.modules import LayerNorm, SamePad, TransposeLast +from fairseq.modules.transformer_sentence_encoder import init_bert_params +from fairseq.utils import index_put +from packaging import version +from torch import nn +from torch.nn import functional as F +from transformers.file_utils import (ModelOutput, add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward) +from transformers.utils import logging + +from .configuration_mmspeech import MMSpeechConfig +from .generate import utils +from .modeling_ofa import (Embedding, OFADecoder, OFAModel, OFAPreTrainedModel, + _expand_mask) + +logger = logging.get_logger() + +_CHECKPOINT_FOR_DOC = 'mmspeech-base' +_CONFIG_FOR_DOC = 'MMSpeechConfig' +_TOKENIZER_FOR_DOC = 'OFATokenizer' +TORCH_VERSION = version.parse(torch.__version__) +TORCH_MESH_GRID_WARNING_VERSION = version.parse('1.9.1') + +DEFAULT_MAX_SOURCE_POSITIONS = 1024 +DEFAULT_MAX_TARGET_POSITIONS = 1024 + +DEFAULT_MIN_PARAMS_TO_WRAP = int(1e8) + +OFA_PRETRAINED_MODEL_ARCHIVE_LIST = ['mmspeech-base', 'mmspeech-large'] + +try: + from apex.normalization import FusedLayerNorm as _FusedLayerNorm + + has_fused_layernorm = True + + class FusedLayerNorm(_FusedLayerNorm): + + @torch.jit.unused + def forward(self, x): + if not x.is_cuda: + return super().forward(x) + else: + with torch.cuda.device(x.device): + return super().forward(x) + +except ImportError: + has_fused_layernorm = False + + +class MMSpeechPreTrainedModel(OFAPreTrainedModel): + r""" + Base class OFA + """ + + config_class = MMSpeechConfig + + def _set_gradient_checkpointing(self, module, value=False): + r""" + Turn on the switch of gradient checkpointing. + """ + if isinstance(module, (OFADecoder, MMSpeechEncoder)): + module.gradient_checkpointing = value + + +@dataclass +class MMSpeechEncoderOutput(ModelOutput): + r""" + Base class for OFA's outputs. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): + Sequence of hidden-states at the output of the last layer of the model. + + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed + or when `config.output_hidden_states=True`): + + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of + shape `(bsz, seq_len, hidden)`. + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed + or when `config.output_attentions=True`): + + Tuple of `torch.FloatTensor` (one for each layer) of shape `(bsz, num_heads, seq_len, seq_len)`. + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + + position_embedding (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): + postional embeddings of the inputs. + """ + + phone_distribution: torch.Tensor = None + last_hidden_state: torch.Tensor = None + padding_mask: torch.Tensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + position_embedding: Optional[torch.FloatTensor] = None + kl_loss: Optional[torch.Tensor] = None + + +@dataclass +class MMSpeechModelOutput(ModelOutput): + """ + Base class for sequence-to-sequence language models outputs. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss. + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, + returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, + returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + Hidden-states of the decoder at the output of each layer plus the initial embedding outputs. + decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, + returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + Attentions weights of the decoder, after the attention softmax, + used to compute the weighted average in the + self-attention heads. + cross_attentions (`tuple(torch.FloatTensor)`, *optional*, + returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + Attentions weights of the decoder's cross-attention layer, + after the attention softmax, used to compute the + weighted average in the cross-attention heads. + encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, + *optional*): + Sequence of hidden-states at the output of the last layer of the encoder of the model. + encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, + returned when `output_hidden_states=True` is passed + or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the encoder at the output of each layer plus the initial embedding outputs. + encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, + returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the + self-attention heads. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None + decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None + decoder_attentions: Optional[Tuple[torch.FloatTensor]] = None + cross_attentions: Optional[Tuple[torch.FloatTensor]] = None + encoder_last_hidden_state: Optional[torch.FloatTensor] = None + encoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None + encoder_attentions: Optional[Tuple[torch.FloatTensor]] = None + encoder_padding_mask: Optional[torch.Tensor] = None + phone_distribution: Optional[torch.Tensor] = None + kl_loss: Optional[torch.Tensor] = None + + +MMSPEECH_START_DOCSTRING = r""" + This model inherits from [`OFAModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`~MMSpeechConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +MMSPEECH_GENERATION_EXAMPLE = r""" + Image captioning example: + + ```python + >>> import soundfile as sf + >>> import torchaudio + >>> import torchaudio.compliance.kaldi as ta_kaldi + >>> wav, sr = sf.read(data[self.column_map['wav']]) + >>> wav = torchaudio.sox_effects.apply_effects_tensor( + >>> wav, sr, + >>> [['speed', '1.0'], ['rate', '16000'], ['gain', '-n'], ['channels', '1']])) + >>> wav = wav * (2**15) + >>> wav = torch.from_numpy(wav.numpy()) + >>> fbank = ta_kaldi.fbank( + waveform, num_mel_bins=n_bins, sample_frequency=sample_rate) + >>> fbank_mask = torch.tensor([True]) + >>> model = MMSpeechModel.from_pretrained(ckpt_dir) + >>> tokenizer = OFATokenizerZH.from_pretrained(ckpt_dir) + + >>> gen = model.generate(fbank=fbank, fbank_mask=fbank_mask, num_beams=4) + >>> print(tokenizer.decode(gen, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` +""" + +MMSPEECH_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): + indices of input sequence tokens in the vocabular, and padding will be ignored by default; + + indices can be obtained using [`~OFATokenizer`]. + + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the resized image, which are transformed by the default operations. + patch_images_2 (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the second (if it exists) image. + patch_masks (`torch.BoolTensor`): the patches to be masked. + token_embeddings (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): token embeddings. + sample_patch_num (`int`): the number of patches to sample. + fbank (`torch.Tensor`): fbank feature of audio. + fbank_length (`torch.Tensor`): fbank length of audio. + fbank_masks (`torch.BoolTensor`): whether to have fbank feature. + phone_items (`torch.Tensor`): phoneme sequence. + phone_masks (`torch.BoolTensor`): whether to have phoneme feature. + features_only (`torch.BoolTensor`): whether to return encoder features only. + mask (`torch.BoolTensor`): whether to mask fbank feature. + mask_prob (`torch.Tensor`): the prob of mask fbank feature. + layer (`int`): the number of layer to cache hidden state. + decoder_input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the sequence in the vocabulary. + code_masks (`torch.Tensor` of shape `(bsz, seq_len)`): masks only for code generation. + attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): attention mask for decoding. + encoder_outputs (`OFAEncoderOutput`): + encoder outputs with hidden states, positional embeddings, and padding masks. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(bsz, num_heads, tgt_len, head_size)`) and 2 additional tensors of + shape `(bsz, num_heads, src_len, head_size)`. + use_cache (`bool`): whether to use cache for faster inference. + output_attentions (`bool`): whether to output attention weights. + output_hidden_states (`bool`): whether to output hidden states. + return_dict (`bool`): unused. Keep it for generation only. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. +""" + + +class Conv2dSubsampling4(nn.Module): + """Convolutional 2D subsampling (to 1/4 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, idim: int, odim: int): + """Construct an Conv2dSubsampling4 object.""" + super().__init__() + self.conv = torch.nn.Sequential( + torch.nn.Conv2d(1, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 3, 2), + torch.nn.ReLU(), + ) + self.out = torch.nn.Sequential( + torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim)) + # The right context for every conv layer is computed by: + # (kernel_size - 1) * frame_rate_of_this_layer + self.subsampling_rate = 4 + # 6 = (3 - 1) * 1 + (3 - 1) * 2 + self.right_context = 6 + + def get_out_seq_lens_tensor(self, in_seq_lens_tensor): + out = in_seq_lens_tensor.clone() + for _ in range(2): + out = ((out.float() - 1) // 2 + 1).floor().long() + return out + + def forward(self, x: torch.Tensor, + x_length: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 4. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 4. + + """ + x = x.unsqueeze(1) # (b, c=1, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + + return x, self.get_out_seq_lens_tensor(x_length) + + +class TransformerEncoder(nn.Module): + + def build_encoder_layer(self, args: MMSpeechConfig): + layer = TransformerSentenceEncoderLayer( + embedding_dim=self.embedding_dim, + ffn_embedding_dim=args.encoder_ffn_dim, + num_attention_heads=args.encoder_attention_heads, + dropout=self.dropout, + attention_dropout=args.attention_dropout, + activation_dropout=args.activation_dropout, + activation_fn=args.activation_function, + layer_norm_first=args.encoder_normalize_before, + ) + return layer + + def __init__(self, args: MMSpeechConfig): + super().__init__() + + self.dropout = args.dropout + self.embedding_dim = args.d_model + self.required_seq_len_multiple = args.required_seq_len_multiple + + pos_conv_depth = args.encoder_pos_conv_depth + if pos_conv_depth > 1: + num_layers = args.encoder_pos_conv_depth + k = max(3, args.encoder_conv_pos // num_layers) + + def make_conv_block(e, k, g, la): + return nn.Sequential(*[ + nn.Sequential( + nn.Conv1d( + e, + e, + kernel_size=k, + padding=k // 2, + groups=g, + ), + SamePad(k), + TransposeLast(), + LayerNorm(e, elementwise_affine=False), + TransposeLast(), + nn.GELU(), + ) for _ in range(la) + ]) + + self.pos_conv = make_conv_block(self.embedding_dim, k, + args.encoder_conv_pos_groups, + num_layers) + self.phone_pos_conv = make_conv_block(self.embedding_dim, k, + args.encoder_conv_pos_groups, + num_layers) + + else: + + def make_conv_pos(e, k, g): + pos_conv = nn.Conv1d( + e, + e, + kernel_size=k, + padding=k // 2, + groups=g, + ) + dropout = 0 + std = math.sqrt((4 * (1.0 - dropout)) / (k * e)) + nn.init.normal_(pos_conv.weight, mean=0, std=std) + nn.init.constant_(pos_conv.bias, 0) + + pos_conv = nn.utils.weight_norm(pos_conv, name='weight', dim=2) + pos_conv = nn.Sequential(pos_conv, SamePad(k), nn.GELU()) + + return pos_conv + + self.pos_conv = make_conv_pos( + self.embedding_dim, + args.encoder_conv_pos, + args.encoder_conv_pos_groups, + ) + self.phone_pos_conv = make_conv_pos( + self.embedding_dim, + args.encoder_conv_pos, + args.encoder_conv_pos_groups, + ) + + self.layers = nn.ModuleList([ + self.build_encoder_layer(args) for _ in range(args.encoder_layers) + ]) + self.layer_norm_first = args.encoder_normalize_before + + self.layer_norm = LayerNorm(self.embedding_dim) + self.phone_layer_norm = LayerNorm(self.embedding_dim) + + self.layerdrop = args.encoder_layerdrop + + self.apply(init_bert_params) + + def forward(self, + x, + padding_mask=None, + phone_x=None, + phone_padding_mask=None, + layer=None, + context_layer=None): + x, layer_results, x_conv, pre_padding_mask = self.extract_features( + x, + padding_mask, + phone_x, + phone_padding_mask, + layer, + context_layer=context_layer) + + if self.layer_norm_first and layer is None: + x = self.layer_norm(x) + + return x, layer_results, x_conv, pre_padding_mask + + def extract_features( + self, + x, + padding_mask=None, + phone_x=None, + phone_padding_mask=None, + tgt_layer=None, + min_layer=0, + context_layer=None, + ): + + if padding_mask is not None: + x = index_put(x, padding_mask, 0) + + x_conv = self.pos_conv(x.transpose(1, 2)) + x_conv = x_conv.transpose(1, 2) + x = x + x_conv + + if not self.layer_norm_first: + x = self.layer_norm(x) + + if phone_x is not None: + if phone_padding_mask is not None: + phone_x = index_put(phone_x, phone_padding_mask, 0) + + phone_x_conv = self.phone_pos_conv(phone_x.transpose(1, 2)) + phone_x_conv = phone_x_conv.transpose(1, 2) + phone_x = phone_x + phone_x_conv + + if not self.layer_norm_first: + # to fix + phone_x = self.layer_norm(phone_x) + + pre_padding_mask = padding_mask.clone() + + x = F.dropout(x, p=self.dropout, training=self.training) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + layer_results = [] + r = None + for i, layer in enumerate(self.layers): + + if i < context_layer and (~padding_mask).any() is False: + continue + + if i == context_layer and phone_x is not None and phone_x_conv is not None: + x = x.transpose(0, 1) + x = torch.cat([x, phone_x], dim=1) + padding_mask = torch.cat([padding_mask, phone_padding_mask], + dim=1) + pre_padding_mask = padding_mask.clone() + x_conv = torch.cat([x_conv, phone_x_conv], dim=1) + x = x.transpose(0, 1) + + dropout_probability = np.random.random( + ) if self.layerdrop > 0 else 1 + if not self.training or (dropout_probability > self.layerdrop): + x, (z, lr) = layer( + x, self_attn_padding_mask=padding_mask, need_weights=False) + if i >= min_layer: + layer_results.append((x, z, lr)) + if i == tgt_layer: + r = x + break + + if r is not None: + x = r + + # T x B x C -> B x T x C + x = x.transpose(0, 1) + + return x, layer_results, x_conv, pre_padding_mask + + def max_positions(self): + """Maximum output length supported by the encoder.""" + return self.args.encoder_max_positions + + def upgrade_state_dict_named(self, state_dict, name): + """Upgrade a (possibly old) state dict for new versions of fairseq.""" + return state_dict + + +class MMSpeechEncoder(MMSpeechPreTrainedModel): + + def __init__(self, + cfg: MMSpeechConfig, + embed_tokens: Optional[nn.Embedding] = None): + + super().__init__(cfg) + + self.cfg = cfg + + self.embed = cfg.d_model + + # fbank encoder + self.subsample = Conv2dSubsampling4(80 * 1, cfg.d_model) + self.post_subsample_proj = nn.Linear(cfg.d_model, cfg.d_model) + + # phone and text encoder + self.padding_idx = embed_tokens.padding_idx + self.phone_padding_idx = self.padding_idx + self.phone_item_embedding = Embedding(cfg.phone_vocab_size, self.embed, + self.phone_padding_idx) + + # mask + self.mask_prob = cfg.audio_mask_prob + self.mask_selection = cfg.audio_mask_selection + self.mask_other = cfg.audio_mask_other + self.mask_length = cfg.audio_mask_length + self.no_mask_overlap = cfg.audio_no_mask_overlap + self.mask_min_space = cfg.audio_mask_min_space + + self.mask_channel_prob = cfg.audio_mask_channel_prob + self.mask_channel_before = cfg.audio_mask_channel_before + self.mask_channel_selection = cfg.audio_mask_channel_selection + self.mask_channel_other = cfg.audio_mask_channel_other + self.mask_channel_length = cfg.audio_mask_channel_length + self.no_mask_channel_overlap = cfg.audio_no_mask_channel_overlap + self.mask_channel_min_space = cfg.audio_mask_channel_min_space + + self.dropout_input = nn.Dropout(cfg.encoder_dropout_input) + self.dropout_features = nn.Dropout(cfg.encoder_dropout_features) + + self.mask_emb = nn.Parameter(torch.FloatTensor(cfg.d_model).uniform_()) + + self.encoder = TransformerEncoder(cfg) + + self.final_proj = nn.Linear(self.embed, self.embed) + + self.num_updates = 0 + + def get_input_embeddings(self): + r""" + Get the embedding weight. + """ + return self.embed_tokens + + def set_input_embeddings(self, value): + r""" + Set the weight of embedding with the given tensor. + """ + self.embed_tokens = value + + def apply_mask(self, + x, + padding_mask, + mask_indices=None, + mask_channel_indices=None, + mask_prob=None): + B, T, C = x.shape + + if self.mask_channel_prob > 0 and self.mask_channel_before: + mask_channel_indices = compute_mask_indices( + (B, C), + None, + self.mask_channel_prob, + self.mask_channel_length, + self.mask_channel_selection, + self.mask_channel_other, + no_overlap=self.no_mask_channel_overlap, + min_space=self.mask_channel_min_space, + ) + mask_channel_indices = ( + torch.from_numpy(mask_channel_indices).to( + x.device).unsqueeze(1).expand(-1, T, -1)) + x[mask_channel_indices] = 0 + + if self.mask_prob > 0 or mask_prob is not None: + if mask_indices is None: + if mask_prob is None: + mask_prob = self.mask_prob + mask_indices = compute_mask_indices( + (B, T), + padding_mask, + mask_prob, + self.mask_length, + self.mask_selection, + self.mask_other, + min_masks=1, + no_overlap=self.no_mask_overlap, + min_space=self.mask_min_space, + require_same_masks=self.cfg.require_same_masks, + mask_dropout=self.cfg.mask_dropout, + ) + mask_indices = torch.from_numpy(mask_indices).to(x.device) + x = index_put(x, mask_indices, self.mask_emb) + else: + mask_indices = None + + if self.mask_channel_prob > 0 and not self.mask_channel_before: + if mask_channel_indices is None: + mask_channel_indices = compute_mask_indices( + (B, C), + None, + self.mask_channel_prob, + self.mask_channel_length, + self.mask_channel_selection, + self.mask_channel_other, + no_overlap=self.no_mask_channel_overlap, + min_space=self.mask_channel_min_space, + ) + mask_channel_indices = ( + torch.from_numpy(mask_channel_indices).to( + x.device).unsqueeze(1).expand(-1, T, -1)) + x = index_put(x, mask_channel_indices, 0) + + return x, mask_indices + + def _get_feat_extract_output_lengths(self, + input_lengths: torch.LongTensor): + """ + Computes the output length of the convolutional layers + """ + + def _conv_out_length(input_length, kernel_size, stride): + return torch.floor((input_length - kernel_size) / stride + 1) + + conv_cfg_list = eval(self.cfg.conv_feature_layers) + + for i in range(len(conv_cfg_list)): + input_lengths = _conv_out_length(input_lengths, + conv_cfg_list[i][1], + conv_cfg_list[i][2]) + + return input_lengths.to(torch.long) + + def forward(self, + fbank: Optional[torch.Tensor] = None, + fbank_length: Optional[torch.Tensor] = None, + fbank_masks: Optional[torch.Tensor] = None, + phone_items: Optional[torch.Tensor] = None, + phone_masks: Optional[torch.Tensor] = None, + features_only: Optional[torch.Tensor] = True, + mask: Optional[torch.Tensor] = False, + mask_prob: Optional[torch.Tensor] = None, + layer=None, + output_hidden_states=False): + + features, fbank_feature_length = self.subsample(fbank, fbank_length) + + if self.post_subsample_proj is not None: + features = self.post_subsample_proj(features) + + padding_mask = ( + torch.BoolTensor(features.shape[:2]).fill_(False) + # if self.pad_audio else None + ).to(features.device) + for i, l in enumerate(fbank_feature_length): + diff = l - padding_mask.shape[-1] + if diff < 0: + padding_mask[i, diff:] = True + + pre_encoder_features = features.clone() + features = self.dropout_input(features) + + if mask: + x, mask_indices = self.apply_mask( + features, padding_mask, mask_prob=mask_prob) + else: + x = features + mask_indices = None + + padding_mask[~fbank_masks] = True + + phone_x = None + phone_padding_mask = None + if phone_items is not None: + phone_x = self.phone_item_embedding(phone_items) + phone_padding_mask = phone_items.eq(self.phone_padding_idx) + phone_padding_mask[~phone_masks] = True + if mask_indices is not None: + phone_mask_indices = phone_padding_mask.new_zeros( + phone_padding_mask.size()).bool() + mask_indices = torch.cat([mask_indices, phone_mask_indices], + dim=1) + + pre_padding_mask = padding_mask.clone() + x, layer_results, pos_embed, padding_mask = self.encoder( + x, + padding_mask=padding_mask, + phone_x=phone_x, + phone_padding_mask=phone_padding_mask, + layer=layer, + context_layer=6) + + emb_weight = self.phone_item_embedding.weight[ + 3:self.cfg.phone_dict_size, :] + if features_only is False: # no gradient for embedding here + emb_weight = emb_weight.detach() + + phone_distribution = F.linear(x, emb_weight, None) + + if features_only: + return MMSpeechEncoderOutput( + phone_distribution=phone_distribution.transpose(0, 1), + last_hidden_state=x, + padding_mask=padding_mask, + position_embedding=pos_embed) + + result = { + 'losses': {}, + } + + with torch.no_grad(): + self.encoder.eval() + y, y_layer_results, _, _ = self.encoder.extract_features( + pre_encoder_features, + padding_mask=pre_padding_mask, + phone_x=phone_x, + phone_padding_mask=phone_padding_mask, + min_layer= + 0, # self.cfg.encoder_layers - self.average_top_k_layers, + context_layer=6) + y = { + 'x': y, + 'padding_mask': padding_mask, + 'layer_results': y_layer_results, + } + + emb_weight = self.phone_item_embedding.weight[ + 3:self.cfg.phone_dict_size, :] + + y = F.linear(y['x'], emb_weight, None) + y = y[mask_indices] + self.encoder.train() + + y_student = phone_distribution[mask_indices] + + def _kl_loss(p, q): + loss = F.kl_div( + utils.log_softmax(p, dim=-1), + utils.softmax(q, dim=-1), + reduction='sum') + return loss + + y = y + kl_loss = _kl_loss(y_student.float(), y.float()) + + with torch.no_grad(): + result['target_var'] = self.compute_var(y) + result['pred_var'] = self.compute_var(y_student.float()) + + if self.num_updates > 5000 and result[ + 'target_var'] < self.cfg.min_target_var: + logger.error( + f"target var is {result['target_var'].item()} < {self.cfg.min_target_var}, exiting" + ) + raise Exception( + f"target var is {result['target_var'].item()} < {self.cfg.min_target_var}, exiting" + ) + if self.num_updates > 5000 and result[ + 'pred_var'] < self.cfg.min_pred_var: + logger.error( + f"pred var is {result['pred_var'].item()} < {self.cfg.min_pred_var}, exiting" + ) + raise Exception( + f"pred var is {result['pred_var'].item()} < {self.cfg.min_pred_var}, exiting" + ) + + return MMSpeechEncoderOutput( + phone_distribution=phone_distribution.transpose(0, 1), + last_hidden_state=x, + padding_mask=padding_mask, + position_embedding=pos_embed, + kl_loss=kl_loss) + + def reorder_encoder_out(self, encoder_out, new_order): + """ + Reorder encoder output according to *new_order*. + + Args: + encoder_out: output from the ``forward()`` method + new_order (LongTensor): desired order + + Returns: + *encoder_out* rearranged according to *new_order* + """ + # if encoder_out["last_hidden_state"] is None: + if 'last_hidden_state' not in encoder_out: + new_encoder_out = None + else: + new_encoder_out = encoder_out['last_hidden_state'].index_select( + 0, new_order) + # if encoder_out["padding_mask"] is None: + if 'padding_mask' not in encoder_out: + new_encoder_padding_mask = None + else: + new_encoder_padding_mask = encoder_out[ + 'padding_mask'].index_select(0, new_order) + + # if encoder_out["position_embedding"] is None: + if 'position_embedding' not in encoder_out: + new_position_embeddings = None + else: + new_position_embeddings = encoder_out[ + 'position_embedding'].index_select(0, new_order) + + if 'hidden_states' not in encoder_out: + new_encoer_states = None + else: + encoder_states = encoder_out['hidden_states'] + new_encoer_states = () + if len(encoder_states) > 0: + for idx, state in enumerate(encoder_states): + new_encoer_states += (state.index_select(0, new_order), ) + + if 'attentions' not in encoder_out: + attentions = None + else: + attentions = encoder_out['attentions'] + + new_kl_loss = None + if 'kl_loss' in encoder_out: + new_kl_loss = encoder_out['kl_loss'] + + if len(encoder_out['phone_distribution']) == 0: + new_phone_distribution = None + else: + new_phone_distribution = encoder_out[ + 'phone_distribution'].index_select(1, new_order) + + return MMSpeechEncoderOutput( + phone_distribution=new_phone_distribution, + last_hidden_state=new_encoder_out, # B x T x C + padding_mask=new_encoder_padding_mask, # B x T + hidden_states=new_encoer_states, # List[T x B x C] + attentions=attentions, + position_embedding=new_position_embeddings, # B x T x C + kl_loss=new_kl_loss) + + @staticmethod + def compute_var(y): + y = y.view(-1, y.size(-1)) + if dist.is_initialized(): + zc = torch.tensor(y.size(0)).cuda() + zs = y.sum(dim=0) + zss = (y**2).sum(dim=0) + + dist.all_reduce(zc) + dist.all_reduce(zs) + dist.all_reduce(zss) + + var = zss / (zc - 1) - (zs**2) / (zc * (zc - 1)) + return torch.sqrt(var + 1e-6).mean() + else: + return torch.sqrt(y.var(dim=0) + 1e-6).mean() + + +@add_start_docstrings( + 'The bare OFA Model outputting raw hidden-states without any specific head on top.', + MMSPEECH_START_DOCSTRING, +) +class MMSpeechModel(OFAModel): + r""" + The OFA model built with an encoder and a decoder only, without any classification head. + + Args: + config (MMSpeechConfig): OFA configuration. + """ + + config_class = MMSpeechConfig + + def __init__(self, config: MMSpeechConfig, **kwargs): + super().__init__(config) + self.disable_entangle = getattr(kwargs, 'disable_entangle', False) + + self.padding_idx, vocab_size = config.pad_token_id, config.vocab_size + shared = nn.Embedding(vocab_size, config.d_model, self.padding_idx) + + self.encoder = MMSpeechEncoder(config, shared) + self.decoder = OFADecoder(config, shared) + self.use_ofasys = config.use_ofasys + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(MMSPEECH_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=MMSpeechModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + def get_encoder_normalized_probs(self, net_output, log_probs, **kwargs): + """Get normalized probabilities (or log probs) from a net's output.""" + logits = net_output['phone_distribution'] + if log_probs: + return utils.log_softmax(logits.float(), dim=-1) + else: + return utils.softmax(logits.float(), dim=-1) + + def forward(self, + input_ids=None, + patch_images=None, + patch_images_2=None, + patch_masks=None, + token_embeddings=None, + sample_patch_num=None, + fbank=None, + fbank_length=None, + fbank_masks=None, + phone_items=None, + phone_masks=None, + features_only=True, + mask=False, + mask_prob=None, + layer=None, + decoder_input_ids=None, + code_masks=None, + attention_mask=None, + encoder_outputs=None, + past_key_values=None, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=False): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): + indices of input sequence tokens in the vocabular, and padding will be ignored by default; + + indices can be obtained using [`~OFATokenizer`]. + + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the resized image, which are transformed by the default operations. + patch_images_2 (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the second (if it exists) image. + patch_masks (`torch.BoolTensor`): the patches to be masked. + token_embeddings (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): token embeddings. + sample_patch_num (`int`): the number of patches to sample. + decoder_input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the sequence in the vocabulary. + code_masks (`torch.Tensor` of shape `(bsz, seq_len)`): masks only for code generation. + attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): attention mask for decoding. + encoder_outputs (`OFAEncoderOutput`): + encoder outputs with hidden states, positional embeddings, and padding masks. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(bsz, num_heads, tgt_len, head_size)`) and 2 additional tensors of + shape `(bsz, num_heads, src_len, head_size)`. + use_cache (`bool`): whether to use cache for faster inference. + output_attentions (`bool`): whether to output attention weights. + output_hidden_states (`bool`): whether to output hidden states. + return_dict (`bool`): unused. Keep it for generation only. + + Returns: + OFASpeechOutput: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): the last decoder hidden states. + past_key_values (`tuple(tuple(torch.FloatTensor)): past keys and values for faster inference. + decoder_hidden_states (`tuple(torch.FloatTensor)`): the decoder hidden states of all layers. + decoder_attentions (`tuple(torch.FloatTensor)): the decoder self attention weights of all layers. + cross_attentions (`tuple(torch.FloatTensor)): cross attention weights of all layers. + encoder_last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the encoder last hidden state. + encoder_hidden_states (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the encoder states of all layers including the embeddings. + encoder_attentions (`torch.FloatTensor` of shape `(bsz, num_heads, seq_len, seq_len)`): + the encoder attention weights of all layers. + """ # noqa + + output_attentions = output_attentions if output_attentions else self.config.output_attentions + output_hidden_states = ( + output_hidden_states + if output_hidden_states else self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if encoder_outputs is None: + encoder_outputs = self.encoder( + fbank=fbank, + fbank_length=fbank_length, + fbank_masks=fbank_masks, + phone_items=phone_items, + phone_masks=phone_masks, + features_only=features_only, + mask=mask, + mask_prob=mask_prob, + layer=layer) + + if decoder_input_ids.eq(self.config.pad_token_id).any(): + attention_mask = decoder_input_ids.eq(self.padding_idx) + + encoder_hidden_states = encoder_outputs.last_hidden_state + encoder_attention_mask = _expand_mask(encoder_outputs.padding_mask, + encoder_hidden_states.dtype, + decoder_input_ids.shape[-1]) + src_pos_embed = encoder_outputs.position_embedding + + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + code_masks=code_masks, + src_pos_embed=src_pos_embed, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + + return MMSpeechModelOutput( + logits=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + encoder_padding_mask=encoder_outputs.padding_mask, + phone_distribution=encoder_outputs.phone_distribution, + kl_loss=encoder_outputs.kl_loss) + + def _set_gradient_checkpointing(self, module, value=False): + r""" + Turn on the switch of gradient checkpointing. + """ + if isinstance(module, (OFADecoder, MMSpeechEncoder)): + module.gradient_checkpointing = value diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_ofa.py b/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_ofa.py new file mode 100644 index 0000000000000000000000000000000000000000..14aba8ca8e2d37994b7ead3d65a404782d4d2ce6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/modeling_ofa.py @@ -0,0 +1,2311 @@ +# Copyright 2022 OFA-Sys Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OFA model.""" + +import math +import random +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch +from packaging import version +from torch import Tensor, nn +from torch.nn import functional as F +from transformers.activations import ACT2FN +from transformers.file_utils import (ModelOutput, add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward) +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, Seq2SeqLMOutput, + Seq2SeqModelOutput) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import logging + +from .configuration_ofa import OFAConfig +from .generate import utils +from .resnet import ResNet +from .utils.utils import DropPath +from .vit import vit_base, vit_huge, vit_large, vit_large_336 + +logger = logging.get_logger() + +_CHECKPOINT_FOR_DOC = 'ofa-base' +_CONFIG_FOR_DOC = 'OFAConfig' +_TOKENIZER_FOR_DOC = 'OFATokenizer' +TORCH_VERSION = version.parse(torch.__version__) +TORCH_MESH_GRID_WARNING_VERSION = version.parse('1.9.1') + +DEFAULT_MAX_SOURCE_POSITIONS = 1024 +DEFAULT_MAX_TARGET_POSITIONS = 1024 + +DEFAULT_MIN_PARAMS_TO_WRAP = int(1e8) + +OFA_PRETRAINED_MODEL_ARCHIVE_LIST = [ + 'ofa-tiny', + 'ofa-medium', + 'ofa-base', + 'ofa-large', + 'ofa-huge', +] + +try: + from apex.normalization import FusedLayerNorm as _FusedLayerNorm + + has_fused_layernorm = True + + class FusedLayerNorm(_FusedLayerNorm): + + @torch.jit.unused + def forward(self, x): + if not x.is_cuda: + return super().forward(x) + else: + with torch.cuda.device(x.device): + return super().forward(x) + +except ImportError: + has_fused_layernorm = False + + +def LayerNorm(normalized_shape, + eps=1e-5, + elementwise_affine=True, + export=False): + r""" + Layer normalization. + If apex is available, use `FusedLayerNorm` instead. + """ + if torch.jit.is_scripting(): + export = True + if not export and torch.cuda.is_available() and has_fused_layernorm: + return FusedLayerNorm(normalized_shape, eps, elementwise_affine) + return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) + + +def make_token_bucket_position(bucket_size, + max_position=DEFAULT_MAX_SOURCE_POSITIONS): + r""" + Make relative position indices for the text. + """ + context_pos = torch.arange(max_position, dtype=torch.long)[:, None] + memory_pos = torch.arange(max_position, dtype=torch.long)[None, :] + relative_pos = context_pos - memory_pos + sign = torch.sign(relative_pos) + mid = bucket_size // 2 + abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), + mid - 1, torch.abs(relative_pos)) + log_pos = torch.ceil( # noqa + torch.log(abs_pos / mid) / math.log((max_position - 1) / mid) * # noqa + (mid - 1)) + mid # noqa + log_pos = log_pos.int() + bucket_pos = torch.where(abs_pos.le(mid), relative_pos, + log_pos * sign).long() + return bucket_pos + bucket_size - 1 + + +def make_image_bucket_position(bucket_size, num_relative_distance): + r""" + Make relative position indices for the image. + """ + coords_h = torch.arange(bucket_size) + coords_w = torch.arange(bucket_size) + if TORCH_VERSION > TORCH_MESH_GRID_WARNING_VERSION: + coords = torch.stack( + torch.meshgrid([coords_h, coords_w], indexing='ij')) # 2, Wh, Ww + else: + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - \ + coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute( + 1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += bucket_size - 1 # shift to start from 0 + relative_coords[:, :, 1] += bucket_size - 1 + relative_coords[:, :, 0] *= 2 * bucket_size - 1 + relative_position_index = torch.zeros( + size=(bucket_size * bucket_size + 1, ) * 2, + dtype=relative_coords.dtype) + relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + relative_position_index[0, 0:] = num_relative_distance - 3 + relative_position_index[0:, 0] = num_relative_distance - 2 + relative_position_index[0, 0] = num_relative_distance - 1 + return relative_position_index + + +def new_arange(x, *size): + r""" + Return a Tensor of `size` filled with a range function on the device of x. + If size is empty, using the size of the variable x. + """ + if len(size) == 0: + size = x.size() + return torch.arange(size[-1], device=x.device).expand(*size).contiguous() + + +def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, + decoder_start_token_id: int): + r""" + Shift input ids one token to the right. + """ + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[:, 1:] = input_ids[:, :-1].clone() + shifted_input_ids[:, 0] = decoder_start_token_id + + assert pad_token_id is not None, 'self.model.config.pad_token_id has to be defined.' + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +def _make_causal_mask(input_ids_shape: torch.Size, + dtype: torch.dtype, + past_key_values_length: int = 0): + r""" + Make causal mask used for uni-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), float('-inf')) + mask_cond = torch.arange(mask.size(-1)) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat( + [torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], + dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, + tgt_len + past_key_values_length) + + +def _expand_mask(mask: torch.Tensor, + dtype: torch.dtype, + tgt_len: Optional[int] = None): + r""" + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, + src_len).to(dtype) + return expanded_mask.masked_fill(expanded_mask.bool(), + torch.finfo(dtype).min) + + +def Embedding(num_embeddings, + embedding_dim, + padding_idx=None, + zero_init=False): + r""" + Embedding for tokens + """ + m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx) + nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5) + if padding_idx is not None: + nn.init.constant_(m.weight[padding_idx], 0) + if zero_init: + nn.init.constant_(m.weight, 0) + return m + + +def Linear(in_features, out_features, bias=True): + r""" + Implementation of linear projection with xavier initialization + """ + m = nn.Linear(in_features, out_features, bias) + nn.init.xavier_uniform_(m.weight) + if bias: + nn.init.constant_(m.bias, 0.0) + return m + + +class LayerDropModuleList(nn.ModuleList): + r""" + A LayerDrop implementation based on :class:`torch.nn.ModuleList`. + + Args: + p (float): probability of dropping out each layer + modules (iterable, optional): an iterable of modules to add + """ + + def __init__(self, p, modules=None): + super().__init__(modules) + self.p = p + + def __iter__(self): + dropout_probs = torch.empty(len(self)).uniform_() + for i, m in enumerate(super().__iter__()): + if not self.training or (dropout_probs[i] > self.p): + yield m + + +class OFAAttention(nn.Module): + r""" + Multi-headed attention, with additional implementation for NormFormer. + + Args: + embed_dim (`int`): embedding dimension. + num_heads (`int`): the number of attention heads. + dropout (`float32`): the ratio for dropout. + is_decoder (`bool`): whether or not decoder attention. + bias (`bool`): whether to add bias. + scale_heads (`bool`): whether to learn scaling heads, only for Normformer. + scale_factor (`float32`, *optional*, defaults to `2.0`): + The position embedding scaling factor. If it works, + self.scaling = float(self.head_dim * scale_factor)**-0.5 + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = True, + scale_heads: bool = True, + scale_factor: float = 2.0, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert ( + self.head_dim * num_heads == self.embed_dim + ), f'embed_dim must be divisible by num_heads ' \ + f'(got `embed_dim`: {self.embed_dim} and `num_heads`: {num_heads}).' + # 1. difference + self.scaling = float(self.head_dim * scale_factor)**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.attn_dropout = nn.Dropout(p=dropout) + self.c_attn = nn.Parameter( + torch.ones((self.num_heads, )), + requires_grad=True) if scale_heads else None + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + r""" + Reshape tensors for multi-head attention. + """ + return tensor.view(bsz, seq_len, self.num_heads, + self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + attn_bias: Optional[torch.Tensor] = None, + ): + r""" + Args: + hidden_states (`torch.FloatTensor` of shape `(bsz, tgt_len, embed_dim)`)`: input states. + key_value_states (`torch.FloatTensor` of shape (bsz, tgt_len, embed_dim), *optional*): key value states. + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): + cached past key value states for fast inference. + attention_mask (`torch.FloatTensor` of shape `(bsz, 1, tgt_len, seq_len)`, *optional*): attention mask. + output_attentions (`bool`, *optional*): whether to output attention weights of all layers. + attn_bias (`torch.FloatTensor` of shape `(bsz, 1, tgt_len, src_len)`, *optional*): + the attention bias for positional information. + + Returns: + attn_output (`torch.FloatTensor` of shape `(bsz, tgt_len, embed_dim)`): attention outputs. + attn_weights_reshaped (`torch.FloatTensor`, *optional*): attention weights of all layers. + past_key_value (`torch.FloatTensor`, *optional*): cached key value states for fast inference. + """ + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + bsz, tgt_len, embed_dim = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, + bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError(f'Attention weights should be of size ' + f'{(bsz * self.num_heads, tgt_len, src_len)}, ' + f'but is {attn_weights.size()}') + + # Add attention bias for positional information + if attn_bias is not None: + attn_weights += attn_bias + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f'Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}' + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, + src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, + src_len) + + attn_weights = F.softmax(attn_weights, dim=-1) + + if output_attentions: + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, + tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, + tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = self.attn_dropout(attn_weights) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, + self.head_dim): + raise ValueError( + f'`attn_output` should be of size ' + f'{(bsz, self.num_heads, tgt_len, self.head_dim)}, ' + f'but is {attn_output.size()}') + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, + self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, embed_dim) + + if self.c_attn is not None: + attn_output = attn_output.view(bsz, tgt_len, self.num_heads, + self.head_dim) + attn_output = torch.einsum('bthd,h->bthd', attn_output, + self.c_attn) + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +class OFAEncoderLayer(nn.Module): + r""" + OFA encoder layer implementation. + + Args: + config: configuration for OFA. + drop_path_rate: the ratio for drop path. + """ + + def __init__(self, config: OFAConfig, drop_path_rate=0.0): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = OFAAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + dropout=config.attention_dropout, + scale_factor=config.attn_scale_factor, + ) + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.self_attn_mid_layer_norm = LayerNorm( + self.embed_dim) if config.normformer else None + self.dropout = nn.Dropout(config.dropout) + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = nn.Dropout(config.activation_dropout) + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.ffn_layer_norm = LayerNorm( + config.encoder_ffn_dim) if config.normformer else None + self.final_layer_norm = LayerNorm(self.embed_dim) + self.normalize_before = config.encoder_normalize_before + self.drop_path = DropPath( + drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + + self.use_gamma_feature = config.use_gamma_feature + if self.use_gamma_feature: + gamma = getattr(config, 'gamma', 1.) + + # `OFA.from_pretrain()` method will replace the `gamma` to `weight` + # in the model key. Here, change the parameters like `xxx_gamma_xxx` + # to `xxx_weight_xxx` to adapt this transformation. + self.weight_self_attn = nn.Parameter( + torch.ones(self.embed_dim) * gamma, requires_grad=True) + self.weight_ffn = nn.Parameter( + torch.ones(self.embed_dim) * gamma, requires_grad=True) + + def residual_connection(self, x, residual): + r""" + Residual connection with drop path. + """ + return residual + self.drop_path(x) + + def forward(self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + output_attentions: bool = False, + attn_bias: Optional[torch.Tensor] = None): + r""" + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape *(bsz, src_len, embed_dim)* + attention_mask (`torch.FloatTensor`): attention mask of size + *(bsz, 1, src_len, src_len)* where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + whether to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + attn_bias (`torch.FloatTensor`): bias for positional information. + + Returns: + outputs (`tuple(torch.FloatTensor)`): + output hidden states of size (bsz, src_len, embed_dim), optionally with attention weights. + """ + + residual = hidden_states + if self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, attn_weights, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + output_attentions=output_attentions, + attn_bias=attn_bias, + ) + if self.self_attn_mid_layer_norm: + hidden_states = self.self_attn_mid_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + if self.use_gamma_feature: + hidden_states = self.weight_self_attn * hidden_states + hidden_states = self.residual_connection(hidden_states, residual) + if not self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + residual = hidden_states + + if self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout(hidden_states) + if self.ffn_layer_norm: + hidden_states = self.ffn_layer_norm(hidden_states) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states) + if self.use_gamma_feature: + hidden_states = self.weight_ffn * hidden_states + hidden_states = self.residual_connection(hidden_states, residual) + if not self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() + or torch.isnan(hidden_states).any()): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states, ) + + if output_attentions: + outputs += (attn_weights, ) + + return outputs + + +class OFADecoderLayer(nn.Module): + r""" + OFA decoder layer implementation. + + Args: + config: configuration for OFA. + drop_path_rate: the ratio for drop path. + """ + + def __init__(self, config: OFAConfig, drop_path_rate=0.0): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = OFAAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + scale_factor=config.attn_scale_factor, + ) + self.dropout = nn.Dropout(p=config.dropout) + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = nn.Dropout(p=config.activation_dropout) + + self.self_attn_layer_norm = LayerNorm(self.embed_dim) + self.self_attn_mid_layer_norm = LayerNorm( + self.embed_dim) if config.normformer else None + self.cross_attn = OFAAttention( + self.embed_dim, + config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + scale_factor=config.attn_scale_factor, + ) + self.cross_attn_layer_norm = LayerNorm(self.embed_dim) + self.cross_attn_mid_layer_norm = LayerNorm( + self.embed_dim) if config.normformer else None + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.ffn_layer_norm = LayerNorm( + config.decoder_ffn_dim) if config.normformer else None + self.final_layer_norm = LayerNorm(self.embed_dim) + self.normalize_before = config.decoder_normalize_before + self.drop_path = DropPath( + drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + + self.use_gamma_feature = config.use_gamma_feature + if self.use_gamma_feature: + gamma = getattr(config, 'gamma', 1.) + + # `OFA.from_pretrain()` method will replace the `gamma` to `weight` + # in the model key. Here, change the parameters like `xxx_gamma_xxx` + # to `xxx_weight_xxx` to adapt this transformation. + self.weight_self_attn = nn.Parameter( + torch.ones(self.embed_dim) * gamma, requires_grad=True) + self.weight_cross_attn = nn.Parameter( + torch.ones(self.embed_dim) * gamma, requires_grad=True) + self.weight_ffn = nn.Parameter( + torch.ones(self.embed_dim) * gamma, requires_grad=True) + + def residual_connection(self, x, residual): + r""" + Residual connection with drop path. + """ + return residual + self.drop_path(x) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + self_attn_bias: Optional[torch.Tensor] = None, + cross_attn_bias: Optional[torch.Tensor] = None, + ): + r""" + Args: + hidden_states (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): input to the layer. + attention_mask (`torch.FloatTensor` of shape `(bsz, 1, tgt_len, src_len)`): + attention mask where padding elements are indicated by very large negative values. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch, seq_len, embed_dim)`): + cross attention input to the layer. + encoder_attention_mask (`torch.FloatTensor` of shape `(bsz, 1, tgt_len, src_len)`): + encoder attention mask where padding elements are indicated by very large negative values. + past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states + output_attentions (`bool`, *optional*): whether to return the attentions tensors of all attention layers. + use_cache (`bool`, *optional*): whether to use cache + self_attn_bias (`torch.FloatTensor`): self attention bias for positional information. + cross_attn_bias (`torch.FloatTensor`): cross attention bias for positional information. + """ + + # Self attention with intermediate layernorm + residual = hidden_states + if self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + # add present self-attn cache to position 1,2 of present_key_value tuple + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=self_attn_past_key_value, + attention_mask=attention_mask, + output_attentions=output_attentions, + attn_bias=self_attn_bias, + ) + if self.self_attn_mid_layer_norm: + hidden_states = self.self_attn_mid_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + if self.use_gamma_feature: + hidden_states = self.weight_self_attn * hidden_states + hidden_states = self.residual_connection(hidden_states, residual) + if not self.normalize_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Cross attention with intermediate layernorm + cross_attn_present_key_value = None + cross_attn_weights = None + if encoder_hidden_states is not None: + residual = hidden_states + if self.normalize_before: + hidden_states = self.cross_attn_layer_norm(hidden_states) + # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attn( + hidden_states=hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + past_key_value=cross_attn_past_key_value, + output_attentions=output_attentions, + attn_bias=cross_attn_bias, + ) + if self.cross_attn_mid_layer_norm: + hidden_states = self.cross_attn_mid_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + if self.use_gamma_feature: + hidden_states = self.weight_cross_attn * hidden_states + hidden_states = self.residual_connection(hidden_states, residual) + if not self.normalize_before: + hidden_states = self.cross_attn_layer_norm(hidden_states) + + # add cross-attn to positions 3,4 of present_key_value tuple + present_key_value = present_key_value + cross_attn_present_key_value + + # FFN with intermediate layernorm + residual = hidden_states + if self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout(hidden_states) + if self.ffn_layer_norm: + hidden_states = self.ffn_layer_norm(hidden_states) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states) + if self.use_gamma_feature: + hidden_states = self.weight_ffn * hidden_states + hidden_states = self.residual_connection(hidden_states, residual) + if not self.normalize_before: + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states, ) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + if use_cache: + outputs += (present_key_value, ) + + return outputs + + +class OFAPreTrainedModel(PreTrainedModel): + r""" + Base class OFA + """ + + config_class = OFAConfig + base_model_prefix = 'model' + supports_gradient_checkpointing = True + + def _init_weights(self, module): + r""" + Weight initialization which follows BERT. + """ + std = self.config.init_std + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + r""" + Turn on the switch of gradient checkpointing. + """ + if isinstance(module, (OFADecoder, OFAEncoder)): + module.gradient_checkpointing = value + + +@dataclass +class OFAEncoderOutput(ModelOutput): + r""" + Base class for OFA's outputs. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): + Sequence of hidden-states at the output of the last layer of the model. + + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed + or when `config.output_hidden_states=True`): + + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of + shape `(bsz, seq_len, hidden)`. + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed + or when `config.output_attentions=True`): + + Tuple of `torch.FloatTensor` (one for each layer) of shape `(bsz, num_heads, seq_len, seq_len)`. + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + + position_embedding (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): + postional embeddings of the inputs. + """ + + last_hidden_state: torch.FloatTensor = None + padding_mask: torch.Tensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + position_embedding: Optional[torch.FloatTensor] = None + + +OFA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`~OFAConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +OFA_GENERATION_EXAMPLE = r""" + Image captioning example: + + ```python + >>> from PIL import Image + >>> from torchvision import transforms + >>> from transformers import OFATokenizer, OFAForConditionalGeneration + + >>> mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5] + >>> resolution = 256 + >>> patch_resize_transform = transforms.Compose([ + lambda image: image.convert("RGB"), + transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + + >>> model = OFAForConditionalGeneration.from_pretrained(ckpt_dir) + >>> tokenizer = OFATokenizer.from_pretrained(ckpt_dir) + + >>> txt = " what is the description of the image?" + >>> inputs = tokenizer([txt], max_length=1024, return_tensors="pt")["input_ids"] + >>> img = Image.open(path_to_image) + >>> patch_img = patch_resize_transform(img).unsqueeze(0) + + >>> gen = model.generate(inputs, patch_img=patch_img, num_beams=4) + >>> print(tokenizer.decode(gen, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` +""" + +OFA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): + indices of input sequence tokens in the vocabular, and padding will be ignored by default; + + indices can be obtained using [`~OFATokenizer`]. + + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the resized image, which are transformed by the default operations. + patch_images_2 (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the second (if it exists) image. + patch_masks (`torch.BoolTensor`): the patches to be masked. + token_embeddings (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): token embeddings. + sample_patch_num (`int`): the number of patches to sample. + decoder_input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the sequence in the vocabulary. + code_masks (`torch.Tensor` of shape `(bsz, seq_len)`): masks only for code generation. + attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): attention mask for decoding. + encoder_outputs (`OFAEncoderOutput`): + encoder outputs with hidden states, positional embeddings, and padding masks. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(bsz, num_heads, tgt_len, head_size)`) and 2 additional tensors of + shape `(bsz, num_heads, src_len, head_size)`. + use_cache (`bool`): whether to use cache for faster inference. + output_attentions (`bool`): whether to output attention weights. + output_hidden_states (`bool`): whether to output hidden states. + return_dict (`bool`): unused. Keep it for generation only. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. +""" + + +class OFAEncoder(OFAPreTrainedModel): + r""" + OFA encoder consisting of layers of [`OFAEncoderLayer`]. + + Args: + config: OFAConfig + embed_tokens (`nn.Embedding`, *optional*): output embedding + """ + + def __init__(self, + config: OFAConfig, + embed_tokens: Optional[nn.Embedding] = None): + super().__init__(config) + + self.dropout = nn.Dropout(config.dropout) + self.encoder_layerdrop = config.encoder_layerdrop + + embed_dim = config.d_model + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_position_embeddings + self.embed_scale = math.sqrt( + embed_dim) if config.scale_embedding else 1.0 + self.num_attention_heads = config.encoder_attention_heads + + if getattr(config, 'layernorm_embedding', False): + self.layernorm_embedding = LayerNorm(embed_dim) + else: + self.layernorm_embedding = None + + if embed_tokens is not None: + self.embed_tokens = embed_tokens + else: + self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, + self.padding_idx) + + if config.add_type_embedding: + if config.use_image_feature: + self.type_embedding = Embedding(2, embed_dim, padding_idx=None) + else: + self.type_embedding = Embedding(1, embed_dim, padding_idx=None) + else: + self.type_embedding = None + + if config.use_image_feature: + if config.use_ofasys: + vit_backbone = { + 'vit_base': vit_base, + 'vit_large': vit_large, + 'vit_large_336': vit_large_336, + 'vit_huge': vit_huge, + }[config.vit_type] + self.embed_images = vit_backbone(config.vit_drop_path_rate) + + self.image_proj = Linear(self.embed_images.width, embed_dim) + + else: + if config.resnet_type == 'resnet18': + self.embed_images = ResNet( + [2, 2, 2], drop_path_rate=config.resnet_drop_path_rate) + elif config.resnet_type == 'resnet34': + self.embed_images = ResNet( + [3, 4, 6], drop_path_rate=config.resnet_drop_path_rate) + elif config.resnet_type == 'resnet50': + self.embed_images = ResNet( + [3, 4, 6], drop_path_rate=config.resnet_drop_path_rate) + elif config.resnet_type == 'resnet101': + self.embed_images = ResNet( + [3, 4, 23], + drop_path_rate=config.resnet_drop_path_rate) + elif config.resnet_type == 'resnet152': + self.embed_images = ResNet( + [3, 8, 36], + drop_path_rate=config.resnet_drop_path_rate) + else: + raise NotImplementedError + + self.image_proj = Linear(1024, embed_dim) + + if not config.use_ofasys and config.resnet_model_path: + print('load resnet {}'.format(config.resnet_model_path)) + resnet_state_dict = torch.load(config.resnet_model_path) + self.embed_images.load_state_dict(resnet_state_dict) + if config.patch_layernorm_embedding: + self.patch_layernorm_embedding = LayerNorm(embed_dim) + else: + self.patch_layernorm_embedding = None + + self.embed_positions = Embedding(self.max_source_positions + 2, + embed_dim) + + if config.use_image_feature: + self.embed_image_positions = Embedding( + config.image_bucket_size**2 + 1, embed_dim) + if not config.use_ofasys: + self.pos_ln = LayerNorm(embed_dim) + + if config.use_image_feature: + self.image_pos_ln = LayerNorm(embed_dim) + self.pos_scaling = float(embed_dim / self.num_attention_heads + * config.attn_scale_factor)**-0.5 + + if not (config.use_ofasys and config.entangle_position_embedding): + self.pos_q_linear = nn.Linear(embed_dim, embed_dim) + self.pos_k_linear = nn.Linear(embed_dim, embed_dim) + + if self.encoder_layerdrop > 0.0: + self.layers = LayerDropModuleList(p=self.encoder_layerdrop) + else: + self.layers = nn.ModuleList([]) + + dpr = [ + x.item() for x in torch.linspace(0, config.encoder_drop_path_rate, + config.encoder_layers) + ] + self.layers.extend([ + OFAEncoderLayer(config, drop_path_rate=dpr[i]) + for i in range(config.encoder_layers) + ]) + self.num_layers = len(self.layers) + + if config.encoder_normalize_before: + self.layer_norm = LayerNorm(embed_dim) + else: + self.layer_norm = None + + self.token_bucket_size = config.token_bucket_size + token_num_rel_dis = 2 * config.token_bucket_size - 1 + token_rp_bucket = make_token_bucket_position(config.token_bucket_size) + self.share_attn_bias = config.share_attn_bias + num_rel_pos_tables = 1 if config.share_attn_bias else config.encoder_layers + self.token_rel_pos_table_list = nn.ModuleList([ + Embedding( + token_num_rel_dis, self.num_attention_heads, zero_init=True) + for _ in range(num_rel_pos_tables) + ]) + + if config.use_image_feature: + self.image_bucket_size = config.image_bucket_size + image_num_rel_dis = (2 * config.image_bucket_size + - 1) * (2 * config.image_bucket_size - 1) + 3 + image_rp_bucket = make_image_bucket_position( + config.image_bucket_size, image_num_rel_dis) + self.image_rel_pos_table_list = nn.ModuleList([ + Embedding( + image_num_rel_dis, + self.num_attention_heads, + zero_init=True) for _ in range(num_rel_pos_tables) + ]) + + self.register_buffer('image_rp_bucket', image_rp_bucket) + + if config.layernorm_embedding: + self.layernorm_embedding = LayerNorm(embed_dim) + else: + self.layernorm_embedding = None + + self.register_buffer('token_rp_bucket', token_rp_bucket) + self.entangle_position_embedding = config.entangle_position_embedding + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + self.use_ofasys = config.use_ofasys + + def get_input_embeddings(self): + r""" + Get the embedding weight. + """ + return self.embed_tokens + + def set_input_embeddings(self, value): + r""" + Set the weight of embedding with the given tensor. + """ + self.embed_tokens = value + + def get_rel_pos_bias(self, x, idx): + r""" + Get the relative positional bias of the text, for attention. + """ + + seq_len = x.size(1) + rp_bucket = self.token_rp_bucket[:seq_len, :seq_len] + values = F.embedding(rp_bucket, + self.token_rel_pos_table_list[idx].weight) + values = values.unsqueeze(0).expand(x.size(0), -1, -1, -1) + values = values.permute([0, 3, 1, 2]) + return values.contiguous() + + def get_image_rel_pos_bias(self, image_position_ids, idx): + r""" + Get the relative positional bias of the image, for attention. + """ + + bsz, seq_len = image_position_ids.shape + rp_bucket_size = self.image_rp_bucket.size(1) + + rp_bucket = self.image_rp_bucket.unsqueeze(0).expand( + bsz, rp_bucket_size, rp_bucket_size).gather( + 1, image_position_ids[:, :, None].expand( + bsz, seq_len, rp_bucket_size)).gather( + 2, image_position_ids[:, None, :].expand( + bsz, seq_len, seq_len)) + values = F.embedding(rp_bucket, + self.image_rel_pos_table_list[idx].weight) + values = values.permute(0, 3, 1, 2) + return values + + def get_patch_images_info(self, patch_images, sample_patch_num, device): + r""" + Get the basic information of the resized image. + + Args: + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): the resized image. + sample_patch_num (`int`): + the number of patches to sample. If it is equal to -1, no sampling will be performed. + device: GPU device. + + Returns: + image_embed (`torch.FloatTensor` of shape `(bsz, h * w, hidden)`): the output of the visual encoder. + image_num_patches (`int`, equal to `h * w`): the number of patches. + image_padding_mask (`torch.BooleanTensor` of shape `(bsz, h*w)`): image padding mask. + image_position_ids (`torch.LongTensor` of shape `(bsz, h*w)`): image position ids. + image_pos_embed (`torch.FloatTensor` of shape (bsz, h*w, hidden)): the positional embedding. + """ + + image_embed = self.embed_images(patch_images) + h, w = image_embed.shape[-2:] + image_num_patches = h * w + image_padding_mask = patch_images.new_zeros( + (patch_images.size(0), image_num_patches)).bool() + image_position_idx = torch.arange(w).unsqueeze(0).expand(h, w)\ + + torch.arange(h).unsqueeze(1) * self.image_bucket_size + 1 + image_position_idx = image_position_idx.view(-1).to(device) + image_position_ids = image_position_idx[None, :].expand( + patch_images.size(0), image_num_patches) + + image_embed = image_embed.flatten(2).transpose(1, 2) + if sample_patch_num is not None: + patch_orders = [ + random.sample(range(image_num_patches), k=sample_patch_num) + for _ in range(patch_images.size(0)) + ] + patch_orders = torch.LongTensor(patch_orders).to(device) + image_embed = image_embed.gather( + 1, + patch_orders.unsqueeze(2).expand(-1, -1, image_embed.size(2))) + image_num_patches = sample_patch_num + image_padding_mask = image_padding_mask.gather(1, patch_orders) + image_position_ids = image_position_ids.gather(1, patch_orders) + orig_num_patches = (self.config.orig_patch_image_size // 16)**2 + orig_hw = self.config.orig_patch_image_size // 16 + if self.config.interpolate_position and image_num_patches > orig_num_patches: + old_image_position_ids = torch.arange(orig_hw).unsqueeze(0).expand(orig_hw, orig_hw) + \ + torch.arange(orig_hw).unsqueeze(1) * \ + self.config.image_bucket_size + 1 # noqa + old_image_position_ids = old_image_position_ids.to(device) + old_image_pos_embed = self.embed_image_positions( + old_image_position_ids) + old_image_pos_embed = old_image_pos_embed.reshape( + 1, orig_hw, orig_hw, -1).permute(0, 3, 1, 2) + image_pos_embed = F.interpolate( + old_image_pos_embed, size=(h, w), mode='bilinear') + image_pos_embed = image_pos_embed.permute(0, 2, 3, 1).reshape( + 1, image_num_patches, -1) + image_pos_embed = image_pos_embed.expand( + patch_images.size(0), -1, -1) + else: + image_pos_embed = self.embed_image_positions(image_position_ids) + + return image_embed, image_num_patches, image_padding_mask, image_position_ids, image_pos_embed + + def forward_embedding(self, + input_ids, + image_embed: Optional[torch.Tensor] = None, + image_embed_2: Optional[torch.Tensor] = None, + token_embedding: Optional[torch.Tensor] = None, + pos_embed: Optional[torch.Tensor] = None, + image_pos_embed: Optional[torch.Tensor] = None, + image_pos_embed_2: Optional[torch.Tensor] = None): + r""" + Generate embeddings of both the image and the text. + Actually since OFA unifies both unimodal and multimodal data, + image inputs are optional. + + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the tokens in the vocabulary. + image_embed (`torch.FloatTensor` of shape `(bsz, h*w, embed_dim)`, *optional*): image embeddings. + image_embed_2 (`torch.FloatTensor` of shape `(bsz, h*w, embed_dim)`, *optional*): + image embeddings of the second image (if it exists). + token_embedding (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`, *optional*): + input token embeddings to replace the embeddings of input ids. + image_pos_embed (`torch.FloatTensor` of shape `(bsz, h*w, embed_dim)`, *optional*): + positional embeddings of the image. + image_pos_embed_2 (`torch.FloatTensor` of shape `(bsz, h*w, embed_dim)`, *optional*): + positional embeddings of the second image. + + Returns: + x (`torch.FloatTensor` of shape `(bsz, h*w+seq_len, embed_dim)`): embeddings of the input. + embed (`torch.FloatTensor` of shape `(bsz, h*w+seq_len, embed_dim)`): + embeddings without adding positional and type embeddings. + """ + + # embed tokens and positions + if token_embedding is None: + token_embedding = self.embed_tokens(input_ids) + x = embed = self.embed_scale * token_embedding + if self.entangle_position_embedding and pos_embed is not None: + x += pos_embed + if self.type_embedding is not None: + x += self.type_embedding(input_ids.new_zeros(x.size()[:2])) + if self.layernorm_embedding is not None: + x = self.layernorm_embedding(x) + x = self.dropout(x) + + # embed raw images + if image_embed is not None: + image_embed = self.image_proj(image_embed) + image_x = image_embed = self.embed_scale * image_embed + if self.entangle_position_embedding and image_pos_embed is not None: + image_x += image_pos_embed + if self.type_embedding is not None: + image_x += self.type_embedding( + input_ids.new_ones(image_x.size()[:2])) + if self.patch_layernorm_embedding is not None: + image_x = self.patch_layernorm_embedding(image_x) + image_x = self.dropout(image_x) + x = torch.cat([image_x, x], dim=1) + embed = torch.cat([image_embed, embed], dim=1) + + if image_embed_2 is not None: + assert self.type_embedding is not None + image_embed_2 = self.image_proj(image_embed_2) + image_x_2 = image_embed_2 = self.embed_scale * image_embed_2 + if self.entangle_position_embedding and image_pos_embed_2 is not None: + image_x_2 += image_pos_embed_2 + if self.type_embedding is not None: + image_x_2 += self.type_embedding( + input_ids.new_full(image_x_2.size()[:2], fill_value=2)) + if self.patch_layernorm_embedding is not None: + image_x_2 = self.patch_layernorm_embedding(image_x_2) + image_x_2 = self.dropout(image_x_2) + if self.quant_noise is not None: + image_x_2 = self.quant_noise(image_x_2) + x = torch.cat([image_x_2, x], dim=1) + embed = torch.cat([image_embed_2, embed], dim=1) + + return x, embed + + def reorder_encoder_out(self, encoder_out, new_order): + """ + Reorder encoder output according to *new_order*. + + Args: + encoder_out: output from the ``forward()`` method + new_order (LongTensor): desired order + + Returns: + *encoder_out* rearranged according to *new_order* + """ + # if encoder_out["last_hidden_state"] is None: + if 'last_hidden_state' not in encoder_out: + new_encoder_out = None + else: + new_encoder_out = encoder_out['last_hidden_state'].index_select( + 0, new_order) + # if encoder_out["padding_mask"] is None: + if 'padding_mask' not in encoder_out: + new_encoder_padding_mask = None + else: + new_encoder_padding_mask = encoder_out[ + 'padding_mask'].index_select(0, new_order) + + # if encoder_out["position_embedding"] is None: + if 'position_embedding' not in encoder_out: + new_position_embeddings = None + else: + new_position_embeddings = encoder_out[ + 'position_embedding'].index_select(0, new_order) + + if 'hidden_states' not in encoder_out: + new_encoer_states = None + else: + encoder_states = encoder_out['hidden_states'] + new_encoer_states = () + if len(encoder_states) > 0: + for idx, state in enumerate(encoder_states): + new_encoer_states += (state.index_select(0, new_order), ) + + if 'attentions' not in encoder_out: + attentions = None + else: + attentions = encoder_out['attentions'] + + return OFAEncoderOutput( + last_hidden_state=new_encoder_out, # B x T x C + padding_mask=new_encoder_padding_mask, # B x T + hidden_states=new_encoer_states, # List[T x B x C] + attentions=attentions, + position_embedding=new_position_embeddings # B x T x C + ) + + def forward( + self, + input_ids=None, + patch_images: Optional[torch.Tensor] = None, + patch_images_2: Optional[torch.Tensor] = None, + patch_masks: Optional[torch.Tensor] = None, + output_attentions: bool = False, + output_hidden_states: bool = False, + token_embeddings: Optional[torch.Tensor] = None, + sample_patch_num: Optional[int] = None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): + indices of input sequence tokens in the vocabular, and padding will be ignored by default; + + indices can be obtained using [`~OFATokenizer`]. + + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the resized image, which are transformed by the default operations. + patch_images_2 (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the second (if it exists) image. + patch_masks (`torch.BoolTensor`): the patches to be masked. + output_attentions (`bool`): whether to return all attention weights, + output_hidden_states (`bool`): whether to return all hidden states. + token_embeddings (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): token embeddings. + sample_patch_num (`int`): the number of patches to sample. + + Returns: + [`OFAEncoderOutput`]: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the states of the last layer. + padding_mask (`torch.BoolTensor` of shape `(bsz, seq_len)`): + the padding mask of the source context. + hidden_states (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the states of all layers including the embeddings. + attentions (`torch.FloatTensor` of shape `(bsz, num_heads, seq_len, seq_len)`): + the attention weights of all layers. + position_embedding (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + positional embeddings of the input image and tokens. + """ + image_embed = None + image_embed_2 = None + image_pos_embed = None + image_pos_embed_2 = None + if patch_images is not None: + image_embed, image_num_patches, image_padding_mask, image_position_ids, image_pos_embed = \ + self.get_patch_images_info(patch_images, sample_patch_num, input_ids.device) + image_padding_mask[~patch_masks] = True + if patch_images_2 is not None: + image_embed_2, image_num_patches_2, image_padding_mask_2, image_position_ids_2, image_pos_embed_2 = \ + self.get_patch_images_info(patch_images_2, sample_patch_num, input_ids.device) + image_padding_mask_2[~patch_masks] = True + + encoder_padding_mask = input_ids.eq(self.padding_idx) + if patch_images is not None: + encoder_padding_mask = torch.cat( + [image_padding_mask, encoder_padding_mask], dim=1) + if patch_images_2 is not None: + encoder_padding_mask = torch.cat( + [image_padding_mask_2, encoder_padding_mask], dim=1) + has_pads = encoder_padding_mask.any() + + pos_embed = self.embed_positions(new_arange(input_ids)) + x, encoder_embedding = self.forward_embedding( + input_ids, image_embed, image_embed_2, token_embeddings, pos_embed, + image_pos_embed, image_pos_embed_2) + + # account for padding while computing the representation + if has_pads: + x = x * (1 - encoder_padding_mask.unsqueeze(-1).type_as(x)) + + if self.use_ofasys: + if patch_images is not None: + pos_embed = torch.cat([image_pos_embed, pos_embed], dim=1) + if patch_images_2 is not None: + pos_embed = torch.cat([image_pos_embed_2, pos_embed], dim=1) + else: + pos_embed = self.pos_ln(pos_embed) + if patch_images is not None: + image_pos_embed = self.image_pos_ln(image_pos_embed) + pos_embed = torch.cat([image_pos_embed, pos_embed], dim=1) + if patch_images_2 is not None: + image_pos_embed_2 = self.image_pos_ln(image_pos_embed_2) + pos_embed = torch.cat([image_pos_embed_2, pos_embed], dim=1) + + def build_abs_pos_bias(pos_embed): + batch_size, seq_length = pos_embed.size(0), pos_embed.size(1) + if not (self.use_ofasys and self.entangle_position_embedding): + pos_q = self.pos_q_linear(pos_embed).view( + batch_size, seq_length, self.num_attention_heads, + -1).transpose(1, 2) * self.pos_scaling + pos_k = self.pos_k_linear(pos_embed).view( + batch_size, seq_length, self.num_attention_heads, + -1).transpose(1, 2) + abs_pos_bias = torch.matmul(pos_q, pos_k.transpose(2, 3)) + else: + abs_pos_bias = torch.zeros( + batch_size, + self.num_attention_heads, + seq_length, + seq_length, + dtype=pos_embed.dtype, + device=pos_embed.device) + return abs_pos_bias + + abs_pos_bias = build_abs_pos_bias(pos_embed) + + # expand attention_mask + if has_pads: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(encoder_padding_mask, dtype=x.dtype) + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # encoder layers + for idx, layer in enumerate(self.layers): + if output_hidden_states: + encoder_states += (x, ) + self_attn_bias = abs_pos_bias.clone() + + real_idx = 0 if self.share_attn_bias else idx + + self_attn_bias[:, :, -input_ids.size(1):, + -input_ids.size(1):] += self.get_rel_pos_bias( + input_ids, real_idx) + if patch_images_2 is not None: + self_attn_bias[:, :, :image_num_patches_2, :image_num_patches_2] += \ + self.get_image_rel_pos_bias(image_position_ids_2, real_idx) + self_attn_bias[:, :, + image_num_patches_2:image_num_patches_2 + image_num_patches, # noqa + image_num_patches_2:image_num_patches_2 + image_num_patches] += \ + self.get_image_rel_pos_bias(image_position_ids, real_idx) # noqa + elif patch_images is not None: + self_attn_bias[:, :, :x.size(1) - input_ids.size(1), :x.size(1) - input_ids.size(1)] += \ + self.get_image_rel_pos_bias(image_position_ids, real_idx) + self_attn_bias = self_attn_bias.reshape(-1, x.size(1), x.size(1)) + + hidden_outputs = layer( + x, + attention_mask if has_pads else None, + attn_bias=self_attn_bias, + output_attentions=output_attentions) + x = hidden_outputs[0] + + if output_attentions: + attention = hidden_outputs[1] + all_attentions = all_attentions + (attention, ) + + if output_hidden_states: + encoder_states += (x, ) + + if self.layer_norm is not None: + x = self.layer_norm(x) + + return OFAEncoderOutput( + last_hidden_state=x, + padding_mask=encoder_padding_mask, + hidden_states=encoder_states, + attentions=all_attentions, + position_embedding=pos_embed) + + +class OFADecoder(OFAPreTrainedModel): + r""" + OFA decoder consisting of layers of [`OFADecoderLayer`] + + Args: + config: OFAConfig + embed_tokens (`nn.Embedding`, *optional*): output embedding + """ + + def __init__(self, + config: OFAConfig, + embed_tokens: Optional[nn.Embedding] = None, + output_projection=None): + super().__init__(config) + self.dropout = nn.Dropout(config.dropout) + self.decoder_layerdrop = config.decoder_layerdrop + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_position_embeddings + self.embed_scale = math.sqrt( + config.d_model) if config.scale_embedding else 1.0 + + self._future_mask = torch.empty(0) + self.share_input_output_embed = config.share_decoder_input_output_embed + self.num_attention_heads = config.decoder_attention_heads + self.use_ofasys = config.use_ofasys + self.disable_entangle = config.disable_entangle + + if embed_tokens is not None: + self.embed_tokens = embed_tokens + else: + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, + self.padding_idx) + + self.embed_dim = config.d_model + self.output_embed_dim = config.d_model + + self.layers = nn.ModuleList( + [OFADecoderLayer(config) for _ in range(config.decoder_layers)]) + if config.layernorm_embedding: + self.layernorm_embedding = LayerNorm(self.embed_dim) + else: + self.layernorm_embedding = None + + if config.use_ofasys: + if config.add_type_embedding: + self.type_embedding = Embedding( + 1, self.embed_dim, padding_idx=None) + else: + self.type_embedding = None + + self.window_size = config.code_image_size // 8 + + self.embed_positions = Embedding(self.max_target_positions + 2, + self.embed_dim) + + if not config.use_ofasys: + self.embed_image_positions = Embedding( + config.image_bucket_size**2 + 1, self.embed_dim) + if not config.use_ofasys: + self.pos_ln = LayerNorm(self.embed_dim) + self.image_pos_ln = LayerNorm(self.embed_dim) + self.pos_scaling = float(self.embed_dim / self.num_attention_heads + * config.attn_scale_factor)**-0.5 + + if not (config.use_ofasys and config.entangle_position_embedding): + self.self_pos_q_linear = nn.Linear(self.embed_dim, self.embed_dim) + self.self_pos_k_linear = nn.Linear(self.embed_dim, self.embed_dim) + + self.cross_pos_q_linear = nn.Linear(self.embed_dim, self.embed_dim) + self.cross_pos_k_linear = nn.Linear(self.embed_dim, self.embed_dim) + + if config.code_layernorm_embedding: + self.code_layernorm_embedding = LayerNorm(self.embed_dim) + else: + self.code_layernorm_embedding = None + + if self.decoder_layerdrop > 0.0: + self.layers = LayerDropModuleList(p=self.decoder_layerdrop) + else: + self.layers = nn.ModuleList([]) + + dpr = [ + x.item() for x in torch.linspace(0, config.decoder_drop_path_rate, + config.decoder_layers) + ] + self.layers.extend([ + OFADecoderLayer(config, drop_path_rate=dpr[i]) + for i in range(config.decoder_layers) + ]) + self.num_layers = len(self.layers) + + if config.decoder_normalize_before: + self.layer_norm = LayerNorm(self.embed_dim) + else: + self.layer_norm = None + + self.adaptive_softmax = None + self.output_projection = output_projection + if self.output_projection is None: + self.build_output_projection(config) + + self.token_bucket_size = config.token_bucket_size + token_num_rel_dis = 2 * config.token_bucket_size - 1 + token_rp_bucket = make_token_bucket_position(config.token_bucket_size) + + self.share_attn_bias = config.share_attn_bias + num_rel_pos_tables = 1 if config.share_attn_bias else config.decoder_layers + self.token_rel_pos_table_list = nn.ModuleList([ + Embedding( + token_num_rel_dis, self.num_attention_heads, zero_init=True) + for _ in range(num_rel_pos_tables) + ]) + + if config.use_image_feature: + if not config.use_ofasys: + self.image_bucket_size = config.image_bucket_size + image_num_rel_dis = (2 * config.image_bucket_size - 1) * ( + 2 * config.image_bucket_size - 1) + 3 + image_rp_bucket = make_image_bucket_position( + config.image_bucket_size, image_num_rel_dis) + image_position_idx = torch.arange(self.window_size).unsqueeze(0).expand(self.window_size, self.window_size) + \ + torch.arange(self.window_size).unsqueeze(1) * config.image_bucket_size + 1 # noqa + image_position_idx = torch.cat( + [torch.tensor([0]), + image_position_idx.view(-1)]) + image_position_idx = torch.cat( + [image_position_idx, + torch.tensor([1024] * 768)]) + self.register_buffer('image_position_idx', image_position_idx) + + self.image_rel_pos_table_list = nn.ModuleList([ + Embedding( + image_num_rel_dis, + self.num_attention_heads, + zero_init=True) for _ in range(num_rel_pos_tables) + ]) + self.register_buffer('image_rp_bucket', image_rp_bucket) + + self.register_buffer('token_rp_bucket', token_rp_bucket) + self.entangle_position_embedding = config.entangle_position_embedding + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def build_output_projection(self, config): + if self.share_input_output_embed: + self.output_projection = nn.Linear( + self.embed_tokens.weight.shape[1], + self.embed_tokens.weight.shape[0], + bias=False, + ) + self.output_projection.weight = self.embed_tokens.weight + else: + self.output_projection = nn.Linear( + self.output_embed_dim, config.vocab_size, bias=False) + nn.init.normal_( + self.output_projection.weight, + mean=0, + std=self.output_embed_dim**-0.5) + + def get_rel_pos_bias(self, x, idx): + r""" + Get the relative positional bias of the text, for attention. + """ + + seq_len = x.size(1) + rp_bucket = self.token_rp_bucket[:seq_len, :seq_len] + values = F.embedding(rp_bucket, + self.token_rel_pos_table_list[idx].weight) + values = values.permute([2, 0, 1]) + return values.contiguous() + + def get_image_rel_pos_bias(self, x, idx): + r""" + Get the relative positional bias of the image, for attention. + """ + + seq_len = x.size(1) + image_position_idx = self.image_position_idx[:seq_len] + rp_bucket = self.image_rp_bucket[ + image_position_idx][:, image_position_idx] + values = F.embedding(rp_bucket, + self.image_rel_pos_table_list[idx].weight) + values = values.permute(2, 0, 1) + return values + + def get_pos_info(self, tgt_pos_embed, src_pos_embed=None, use_image=False): + r""" + Get the positional information. + + Args: + tgt_pos_embed (`torch.FloatTensor` of shape `(bsz, tgt_len, embed_dim)`): + the target-side positional embeddings. + src_pos_embed (`torch.FloatTensor` of shape `(bsz, src_len, embed_dim)`, *optional*): + the source-side positional embeddings. + use_image (`bool`): whether to use image. + + Returns: + abs_pos_bias (`torch.FloatTensor` of shape `(bsz, src_len, tgt_len, src_len)`): + absolute positional bias for attention. + """ + + batch_size = tgt_pos_embed.size(0) + tgt_len = tgt_pos_embed.size(1) + if not self.use_ofasys: + tgt_pos_embed = self.image_pos_ln( + tgt_pos_embed) if use_image else self.pos_ln(tgt_pos_embed) + + if src_pos_embed is not None: + src_len = src_pos_embed.size(1) + if not (self.entangle_position_embedding and self.use_ofasys): + pos_q = self.cross_pos_q_linear(tgt_pos_embed).view( + batch_size, tgt_len, self.num_attention_heads, + -1).transpose(1, 2) * self.pos_scaling + pos_k = self.cross_pos_k_linear(src_pos_embed).view( + batch_size, src_len, self.num_attention_heads, + -1).transpose(1, 2) + abs_pos_bias = torch.matmul(pos_q, pos_k.transpose(2, 3)) + else: + abs_pos_bias = torch.zeros( + batch_size, + self.num_attention_heads, + tgt_len, + src_len, + dtype=tgt_pos_embed.dtype, + device=tgt_pos_embed.device) + else: + # batch_size, seq_length = tgt_pos_embed.size(0), tgt_pos_embed.size(1) + if not (self.entangle_position_embedding and self.use_ofasys): + pos_q = self.self_pos_q_linear(tgt_pos_embed).view( + batch_size, tgt_len, self.num_attention_heads, + -1).transpose(1, 2) * self.pos_scaling + pos_k = self.self_pos_k_linear(tgt_pos_embed).view( + batch_size, tgt_len, self.num_attention_heads, + -1).transpose(1, 2) + abs_pos_bias = torch.matmul(pos_q, pos_k.transpose(2, 3)) + else: + abs_pos_bias = torch.zeros( + batch_size, + self.num_attention_heads, + tgt_len, + tgt_len, + dtype=tgt_pos_embed.dtype, + device=tgt_pos_embed.device) + + return abs_pos_bias + + def get_input_embeddings(self): + r""" + Get the input embeddings + """ + return self.embed_tokens + + def set_input_embeddings(self, value): + r""" + Set the weights of the embeddings with the given tensor. + """ + self.embed_tokens = value + + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, + dtype, past_key_values_length): + r""" + Create causal mask for unidirectional decoding. + [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + """ + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + dtype, + past_key_values_length=past_key_values_length).to(self.device) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask( + attention_mask, dtype, tgt_len=input_shape[-1]) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else + expanded_attn_mask + combined_attention_mask) + + return combined_attention_mask + + def max_positions(self): + """Maximum output length supported by the decoder.""" + if self.embed_positions is None: + return self.max_target_positions + return self.max_target_positions + + def get_normalized_probs( + self, + net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]], + log_probs: bool, + sample: Optional[Dict[str, Tensor]] = None, + ): + """Get normalized probabilities (or log probs) from a net's output.""" + return self.get_normalized_probs_scriptable(net_output, log_probs, + sample) + + def get_normalized_probs_scriptable( + self, + net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]], + log_probs: bool, + sample: Optional[Dict[str, Tensor]] = None, + ): + """Get normalized probabilities (or log probs) from a net's output.""" + + if hasattr(self, + 'adaptive_softmax') and self.adaptive_softmax is not None: + if sample is not None: + assert 'target' in sample + target = sample['target'] + else: + target = None + out = self.adaptive_softmax.get_log_prob( + net_output[0], target=target) + return out.exp_() if not log_probs else out + + logits = net_output[0] + if log_probs: + return utils.log_softmax(logits, dim=-1) + else: + return utils.softmax(logits, dim=-1) + + def reorder_incremental_state_scripting( + self, + past_key_values: Optional[torch.Tensor], + new_order: Tensor, + ): + """Main entry point for reordering the incremental state. + + Due to limitations in TorchScript, we call this function in + :class:`fairseq.sequence_generator.SequenceGenerator` instead of + calling :func:`reorder_incremental_state` directly. + """ + input_buffer = past_key_values + new_past_key_values = [] + if input_buffer is not None: + for input_buffer_k in input_buffer: + new_input_buffer_k = [] + for input in input_buffer_k: + if input is None: + input = None + else: + input = input.index_select(0, new_order) + new_input_buffer_k.append(input) + new_past_key_values.append(new_input_buffer_k) + return new_past_key_values + + def forward( + self, + input_ids: torch.Tensor = None, + attention_mask: torch.Tensor = None, + encoder_hidden_states: torch.Tensor = None, + encoder_attention_mask: torch.Tensor = None, + code_masks: Optional[torch.Tensor] = None, + src_pos_embed: torch.Tensor = None, + past_key_values: Optional[torch.Tensor] = None, + use_cache: bool = False, + output_attentions: bool = False, + output_hidden_states: bool = False, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the sequence in the vocabulary. + attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): mask to avoid attention on padding tokens. + encoder_hidden_states (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): the last hidden state of the encoder. + encoder_attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): the padding mask of the source side. + code_masks (`torch.Tensor` of shape `(bsz, seq_len)`): masks only for code generation. + src_pos_embed (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): the positional embeddings of the source side. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(bsz, num_heads, tgt_len, head_size)`) and 2 additional tensors of + shape `(bsz, num_heads, src_len, head_size)`. + use_cache (`bool`): whether to use cache for faster inference. + output_attentions (`bool`): whether to output attention weights. + output_hidden_states (`bool`): whether to output hidden states. + + Returns: + BaseModelOutputWithPastAndCrossAttentions or a plain tuple: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): the last hidden states. + past_key_values (`tuple(tuple(torch.FloatTensor)): past keys and values for faster inference. + hidden_states (`tuple(torch.FloatTensor)`): hidden states of all layers. + attentions (`tuple(torch.FloatTensor)): self attention weights of all layers. + cross_attentions (`tuple(torch.FloatTensor)): cross attention weights of all layers. + """ # noqa + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if past_key_values is not None and len(past_key_values) > 0: + size = past_key_values[0][0].size() + bsz, tgt_len = size[0], size[-2] + 1 + token_position_idx = torch.arange( + tgt_len, + device=input_ids.device).expand([bsz, tgt_len]).contiguous() + else: + bsz, tgt_len = input_ids.shape + token_position_idx = new_arange(input_ids) + tgt_pos_embed = self.embed_positions(token_position_idx) + if code_masks is not None and torch.any(code_masks): + image_position_idx = self.image_position_idx[:input_ids.size( + 1)].unsqueeze(0).expand(bsz, tgt_len) + tgt_pos_embed[code_masks] = self.embed_image_positions( + image_position_idx)[code_masks] + + # self attn position bias + self_abs_pos_bias = self.get_pos_info(tgt_pos_embed, use_image=False) + if code_masks is not None and torch.any(code_masks): + self_image_abs_pos_bias = self.get_pos_info( + tgt_pos_embed, use_image=True) + self_abs_pos_bias[code_masks] = self_image_abs_pos_bias[code_masks] + # cross attn position bias + cross_abs_pos_bias = self.get_pos_info( + tgt_pos_embed, src_pos_embed=src_pos_embed) + if code_masks is not None and torch.any(code_masks): + cross_image_abs_pos_bias = self.get_pos_info( + tgt_pos_embed, src_pos_embed=src_pos_embed, use_image=True) + cross_abs_pos_bias[code_masks] = cross_image_abs_pos_bias[ + code_masks] + cross_abs_pos_bias = cross_abs_pos_bias.reshape( + -1, + *cross_abs_pos_bias.size()[-2:]) + + all_prev_output_tokens = input_ids.clone() + if past_key_values is not None and len(past_key_values) > 0: + input_ids = input_ids[:, -1:] + cross_abs_pos_bias = cross_abs_pos_bias[:, -1:, :] + tgt_pos_embed = tgt_pos_embed[:, -1:, :] + + # embed tokens and positions + x = self.embed_scale * self.embed_tokens(input_ids) + + if self.entangle_position_embedding and not self.disable_entangle: + x += tgt_pos_embed + + if self.layernorm_embedding is not None: + if code_masks is None or not code_masks.any( + ) or not self.code_layernorm_embedding: + x = self.layernorm_embedding(x) + elif code_masks is not None and code_masks.all(): + x = self.code_layernorm_embedding(x) + else: + x[~code_masks] = self.layernorm_embedding(x[~code_masks]) + x[code_masks] = self.code_layernorm_embedding(x[code_masks]) + + hidden_states = self.dropout(x) + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None and len( + past_key_values) > 0 else 0 + + shape, dtype = input_ids.shape, hidden_states.dtype + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, shape, dtype, past_key_values_length) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if ( + output_attentions and encoder_hidden_states is not None) else None + next_decoder_cache = () if use_cache else None + + # decoder layers + for idx, layer in enumerate(self.layers): + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states, ) + + past_key_value = past_key_values[ + idx] if past_key_values is not None and len( + past_key_values) > 0 else None + + self_attn_bias = self_abs_pos_bias.clone() + real_idx = 0 if self.share_attn_bias else idx + if code_masks is None or not code_masks.any(): + self_attn_bias += self.get_rel_pos_bias( + all_prev_output_tokens, real_idx).unsqueeze(0) + elif code_masks is not None and code_masks.all(): + self_attn_bias += self.get_image_rel_pos_bias( + all_prev_output_tokens, real_idx).unsqueeze(0) + else: + self_attn_bias[~code_masks] += self.get_rel_pos_bias( + all_prev_output_tokens, real_idx).unsqueeze(0) + self_attn_bias[code_masks] += self.get_image_rel_pos_bias( + all_prev_output_tokens, real_idx).unsqueeze(0) + self_attn_bias = self_attn_bias.reshape( + -1, + *self_attn_bias.size()[-2:]) + if past_key_value is not None and len(past_key_values) > 0: + self_attn_bias = self_attn_bias[:, -1:, :] + + layer_outputs = layer( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + self_attn_bias=self_attn_bias, + cross_attn_bias=cross_abs_pos_bias, + ) + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += ( + layer_outputs[3 if output_attentions else 1], ) + + if output_attentions: + all_self_attns += (layer_outputs[1], ) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2], ) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states, ) + + next_cache = next_decoder_cache if use_cache else None + + if self.layer_norm is not None: + hidden_states = self.layer_norm(hidden_states) + + if self.output_projection is not None: + hidden_states = self.output_projection(hidden_states) + + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, # (bz, + past_key_values=next_cache, # (bz, n_heads, seq_len, head_dim) + hidden_states=all_hidden_states, + attentions=all_self_attns, # (bz, n_heads, tgt_len, src_len) + cross_attentions= # noqa + all_cross_attentions # (bz, n_heads, tgt_len, src_len) # noqa + ) + + +@add_start_docstrings( + 'The bare OFA Model outputting raw hidden-states without any specific head on top.', + OFA_START_DOCSTRING, +) +class OFAModel(OFAPreTrainedModel): + r""" + The OFA model built with an encoder and a decoder only, without any classification head. + + Args: + config (OFAConfig): OFA configuration. + """ + + def __init__(self, config: OFAConfig, **kwargs): + super().__init__(config) + self.disable_entangle = getattr(kwargs, 'disable_entangle', False) + + self.padding_idx, vocab_size = config.pad_token_id, config.vocab_size + shared = nn.Embedding(vocab_size, config.d_model, self.padding_idx) + + self.encoder = OFAEncoder(config, shared) + self.decoder = OFADecoder(config, shared) + self.use_ofasys = config.use_ofasys + + # exclude mlp head as default + if not getattr(config, 'exclude_mlp', True): + self.mlp_head = Linear(config.d_model, config.mlp_dim) + # None temperature_init_value as default + if config.temperature_init_value: + self.temp = nn.Parameter(config.temperature_init_value + * torch.ones([])) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + r""" + Retrieve input embeddings. + """ + return self.encoder.get_input_embeddings() + + def set_input_embeddings(self, value): + r""" + Set values for input embeddings + """ + shared = value + self.encoder.embed_tokens = shared + self.decoder.embed_tokens = shared + + def get_encoder(self): + r""" + Retrieve the encoder + """ + return self.encoder + + def get_decoder(self): + r""" + Retrieve the decoder + """ + return self.decoder + + @add_start_docstrings_to_model_forward(OFA_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=Seq2SeqModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + # an adaptor for fairseq generator + def max_decoder_positions(self): + """Maximum length supported by the decoder.""" + return self.decoder.max_positions() + + def get_normalized_probs( + self, + net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]], + log_probs: bool, + sample: Optional[Dict[str, Tensor]] = None, + ): + """Get normalized probabilities (or log probs) from a net's output.""" + return self.get_normalized_probs_scriptable(net_output, log_probs, + sample) + + # TorchScript doesn't support super() method so that the scriptable Subclass + # can't access the base class model in Torchscript. + # Current workaround is to add a helper function with different name and + # call the helper function from scriptable Subclass. + + def get_normalized_probs_scriptable( + self, + net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]], + log_probs: bool, + sample: Optional[Dict[str, Tensor]] = None, + ): + """Scriptable helper function for get_normalized_probs in ~BaseFairseqModel""" + if hasattr(self, 'decoder'): + return self.decoder.get_normalized_probs(net_output, log_probs, + sample) + elif torch.is_tensor(net_output): + # syntactic sugar for simple models which don't have a decoder + # (e.g., the classification tutorial) + logits = net_output.float() + if log_probs: + return F.log_softmax(logits, dim=-1) + else: + return F.softmax(logits, dim=-1) + raise NotImplementedError + + def forward(self, + input_ids=None, + patch_images=None, + patch_images_2=None, + patch_masks=None, + token_embeddings=None, + sample_patch_num=None, + decoder_input_ids=None, + code_masks=None, + attention_mask=None, + encoder_outputs=None, + past_key_values=None, + use_cache=False, + output_attentions=False, + output_hidden_states=False, + return_dict=False): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): + indices of input sequence tokens in the vocabular, and padding will be ignored by default; + + indices can be obtained using [`~OFATokenizer`]. + + patch_images (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the resized image, which are transformed by the default operations. + patch_images_2 (`torch.FloatTensor` of shape `(bsz, 3, height, width)`): + the second (if it exists) image. + patch_masks (`torch.BoolTensor`): the patches to be masked. + token_embeddings (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): token embeddings. + sample_patch_num (`int`): the number of patches to sample. + decoder_input_ids (`torch.LongTensor` of shape `(bsz, seq_len)`): indices of the sequence in the vocabulary. + code_masks (`torch.Tensor` of shape `(bsz, seq_len)`): masks only for code generation. + attention_mask (`torch.Tensor` of shape `(bsz, seq_len)`): attention mask for decoding. + encoder_outputs (`OFAEncoderOutput`): + encoder outputs with hidden states, positional embeddings, and padding masks. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(bsz, num_heads, tgt_len, head_size)`) and 2 additional tensors of + shape `(bsz, num_heads, src_len, head_size)`. + use_cache (`bool`): whether to use cache for faster inference. + output_attentions (`bool`): whether to output attention weights. + output_hidden_states (`bool`): whether to output hidden states. + return_dict (`bool`): unused. Keep it for generation only. + + Returns: + Seq2SeqModelOutput: + last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, hidden)`): the last decoder hidden states. + past_key_values (`tuple(tuple(torch.FloatTensor)): past keys and values for faster inference. + decoder_hidden_states (`tuple(torch.FloatTensor)`): the decoder hidden states of all layers. + decoder_attentions (`tuple(torch.FloatTensor)): the decoder self attention weights of all layers. + cross_attentions (`tuple(torch.FloatTensor)): cross attention weights of all layers. + encoder_last_hidden_state (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the encoder last hidden state. + encoder_hidden_states (`torch.FloatTensor` of shape `(bsz, seq_len, embed_dim)`): + the encoder states of all layers including the embeddings. + encoder_attentions (`torch.FloatTensor` of shape `(bsz, num_heads, seq_len, seq_len)`): + the encoder attention weights of all layers. + """ # noqa + + output_attentions = output_attentions if output_attentions else self.config.output_attentions + output_hidden_states = ( + output_hidden_states + if output_hidden_states else self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + patch_images=patch_images, + patch_images_2=patch_images_2, + patch_masks=patch_masks, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + token_embeddings=token_embeddings, + sample_patch_num=sample_patch_num, + ) + + if decoder_input_ids.eq(self.config.pad_token_id).any(): + attention_mask = decoder_input_ids.eq(self.padding_idx) + + encoder_hidden_states = encoder_outputs.last_hidden_state + encoder_attention_mask = _expand_mask(encoder_outputs.padding_mask, + encoder_hidden_states.dtype, + decoder_input_ids.shape[-1]) + src_pos_embed = encoder_outputs.position_embedding + + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + code_masks=code_masks, + src_pos_embed=src_pos_embed, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + + return Seq2SeqLMOutput( + logits=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + def prepare_inputs_for_generation(self, + decoder_input_ids=None, + past=None, + attention_mask=None, + code_masks=None, + use_cache=False, + encoder_outputs=None, + **kwargs): + # if attention_mask is None: + attention_mask = decoder_input_ids.new_zeros(decoder_input_ids.shape) + + # cut decoder_input_ids if past is used + if past is not None: + decoder_input_ids = decoder_input_ids[:, -1:] + + return { + 'input_ids': None, + 'patch_images': None, + 'patch_images_2': None, + 'patch_masks': None, + 'token_embeddings': None, + 'sample_patch_num': None, + 'attention_mask': attention_mask, + 'encoder_outputs': encoder_outputs, + 'past_key_values': past, + 'decoder_input_ids': decoder_input_ids, + 'code_masks': code_masks, + 'use_cache': use_cache, + } + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return shift_tokens_right(labels, self.config.pad_token_id, + self.config.decoder_start_token_id) + + def _prepare_encoder_decoder_kwargs_for_generation( + self, + inputs_tensor: torch.Tensor, + model_kwargs, + model_input_name: Optional[str] = None): + # 1. get encoder + encoder = self.get_encoder() + + # 2. prepare encoder args and encoder kwargs from model kwargs + irrelevant_prefix = [ + 'decoder_', 'cross_attn', 'use_cache', 'attention_mask' + ] + encoder_kwargs = { + argument: value + for argument, value in model_kwargs.items() + if not any(argument.startswith(p) for p in irrelevant_prefix) + } + + if encoder_kwargs.get('patch_masks') is None: + encoder_kwargs['patch_masks'] = torch.tensor([True]) + + # 3. make sure that encoder returns `ModelOutput` + model_input_name = model_input_name if model_input_name is not None else self.main_input_name + encoder_kwargs[model_input_name] = inputs_tensor + model_kwargs['encoder_outputs']: ModelOutput = encoder( + **encoder_kwargs) + model_kwargs['attention_mask'] = None + + return model_kwargs + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple( + past_state.index_select(0, beam_idx) + for past_state in layer_past), ) + return reordered_past + + @staticmethod + def _expand_inputs_for_generation( + input_ids: torch.LongTensor, + expand_size: int = 1, + is_encoder_decoder: bool = False, + attention_mask: Optional[torch.LongTensor] = None, + encoder_outputs: Optional[ModelOutput] = None, + **model_kwargs, + ): + expanded_return_idx = ( + torch.arange(input_ids.shape[0]).view(-1, 1).repeat( + 1, expand_size).view(-1).to(input_ids.device)) + input_ids = input_ids.index_select(0, expanded_return_idx) + + if 'token_type_ids' in model_kwargs: + token_type_ids = model_kwargs['token_type_ids'] + model_kwargs['token_type_ids'] = token_type_ids.index_select( + 0, expanded_return_idx) + + if attention_mask is not None: + model_kwargs['attention_mask'] = attention_mask.index_select( + 0, expanded_return_idx) + + if is_encoder_decoder: + if encoder_outputs is None: + raise ValueError( + 'If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.' + ) + encoder_outputs[ + 'last_hidden_state'] = encoder_outputs.last_hidden_state.index_select( + 0, + expanded_return_idx.to( + encoder_outputs.last_hidden_state.device)) + encoder_outputs[ + 'position_embedding'] = encoder_outputs.position_embedding.index_select( + 0, + expanded_return_idx.to( + encoder_outputs.position_embedding.device)) + encoder_outputs[ + 'padding_mask'] = encoder_outputs.padding_mask.index_select( + 0, + expanded_return_idx.to( + encoder_outputs.padding_mask.device)) + model_kwargs['encoder_outputs'] = encoder_outputs + return input_ids, model_kwargs diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/resnet.py b/runtime/Lib/site_packages/models/multi_modal/ofa/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c863aad0bd182f53e3fe3040040870eee6346668 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/resnet.py @@ -0,0 +1,348 @@ +# Copyright 2022 OFA-Sys Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a.sh different form of dropout in a.sh separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a.sh layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d( + in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError( + 'Dilation > 1 not supported in BasicBlock') + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + assert False + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None, + drop_path_rate=0.0): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.drop_path = DropPath( + drop_path_rate) if drop_path_rate > 0.0 else nn.Identity() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out = identity + self.drop_path(out) + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + r""" + Deep residual network, copy from https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py. + + You can see more details from https://arxiv.org/abs/1512.03385 + + step 1. Get image embedding with `7` as the patch image size, `2` as stride. + step 2. Do layer normalization, relu activation and max pooling. + step 3. Go through three times residual branch. + """ + + def __init__(self, + layers, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None, + norm_layer=None, + drop_path_rate=0.0): + r""" + Args: + layers (`Tuple[int]`): There are three layers in resnet, so the length + of layers should greater then three. And each element in `layers` is + the number of `Bottleneck` in relative residual branch. + zero_init_residual (`bool`, **optional**, default to `False`): + Whether or not to zero-initialize the last BN in each residual branch. + groups (`int`, **optional**, default to `1`): + The number of groups. So far, only the value of `1` is supported. + width_per_group (`int`, **optional**, default to `64`): + The width in each group. So far, only the value of `64` is supported. + replace_stride_with_dilation (`Tuple[bool]`, **optional**, default to `None`): + Whether or not to replace stride with dilation in each residual branch. + norm_layer (`torch.nn.Module`, **optional**, default to `None`): + The normalization module. If `None`, will use `torch.nn.BatchNorm2d`. + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError('replace_stride_with_dilation should be None ' + 'or a 3-element tuple, got {}'.format( + replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d( + 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer( + Bottleneck, 64, layers[0], drop_path_rate=drop_path_rate) + self.layer2 = self._make_layer( + Bottleneck, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0], + drop_path_rate=drop_path_rate) + self.layer3 = self._make_layer( + Bottleneck, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1], + drop_path_rate=drop_path_rate) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, + (nn.SyncBatchNorm, nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, + block, + planes, + blocks, + stride=1, + dilate=False, + drop_path_rate=0.0): + r""" + Making a single residual branch. + + step 1. If dilate==`True`, switch the value of dilate and stride. + step 2. If the input dimension doesn't equal to th output output dimension + in `block`, initialize a down sample module. + step 3. Build a sequential of `blocks` number of `block`. + + Args: + block (`torch.nn.Module`): The basic block in residual branch. + planes (`int`): The output dimension of each basic block. + blocks (`int`): The number of `block` in residual branch. + stride (`int`, **optional**, default to `1`): + The stride using in conv. + dilate (`bool`, **optional**, default to `False`): + Whether or not to replace dilate with stride. + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + + Returns: + A sequential of basic layer with type `torch.nn.Sequential[block]` + """ + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, blocks)] + for i in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer, + drop_path_rate=dpr[i])) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + return x + + def forward(self, x): + return self._forward_impl(x) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa.py b/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa.py new file mode 100644 index 0000000000000000000000000000000000000000..ea79a32726fd560154497cf01cce0f7a4efc8cc6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa.py @@ -0,0 +1,372 @@ +# Copyright 2022 OFA-Sys Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for OFA.""" +import collections +import os +from typing import List, Optional, Tuple + +from transformers import PreTrainedTokenizer +from transformers.models.bart.tokenization_bart import BartTokenizer +from transformers.models.bert.tokenization_bert import (BasicTokenizer, + WordpieceTokenizer) +from transformers.utils import logging + +from modelscope.utils.constant import ModelFile + +logger = logging.get_logger() + +VOCAB_FILES_NAMES = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': { + 'ofa-base': 'https://huggingface.co/ofa-base/resolve/main/vocab.json', + }, + 'merges_file': { + 'ofa-base': 'https://huggingface.co/ofa-base/resolve/main/merges.txt', + }, + # OFA models are implemented to be compatible with both huggingface + # and modelscope frameworks. For all OFA models available on huggingface, + # please refer to https://huggingface.co/models?filter=ofa +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'ofa-base': 1024, +} + +VOCAB_FILES_NAMES_ZH = {'vocab_file': ModelFile.VOCAB_FILE} + +PRETRAINED_VOCAB_FILES_MAP_ZH = { + 'vocab_file': { + 'bert-base-chinese': + 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt', + } + # OFA models are implemented to be compatible with both huggingface + # and modelscope frameworks. For all OFA models available on huggingface, + # please refer to https://huggingface.co/models?filter=ofa +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES_ZH = { + 'ofa-base': 1024, +} + +PRETRAINED_INIT_CONFIGURATION_ZH = { + 'bert-base-chinese': { + 'do_lower_case': True + }, +} + + +class OFATokenizer(BartTokenizer): + """ + Construct a OFA tokenizer. + + [`~OFATokenizer`] is identical to [`BartTokenizer`] and runs end-to-end tokenization: punctuation splitting and + wordpiece. + + Refer to superclass [`BartTokenizer`] for usage examples and documentation concerning parameters. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, 'r', encoding='utf-8') as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip('\n') + vocab[token] = index + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class OFATokenizerZH(PreTrainedTokenizer): + r""" + Construct a OFA tokenizer. Based on WordPiece. + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + File containing the vocabulary. + do_lower_case (`bool`, *optional*, defaults to `True`): + Whether or not to lowercase the input when tokenizing. + do_basic_tokenize (`bool`, *optional*, defaults to `True`): + Whether or not to do basic tokenization before WordPiece. + never_split (`Iterable`, *optional*): + Collection of tokens which will never be split during tokenization. Only has an effect when + `do_basic_tokenize=True` + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + + + + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + + + + eos_token (`str`, *optional*, defaults to `""`): + The end of sequence token. + + + + When building a sequence using special tokens, this is not the token that is used for the end of sequence. + The token used is the `sep_token`. + + + + sep_token (`str`, *optional*, defaults to `""`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + cls_token (`str`, *optional*, defaults to `""`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (`str`, *optional*, defaults to `""`): + The token used for padding, for example when batching sequences of different lengths. + mask_token (`str`, *optional*, defaults to `""`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + tokenize_chinese_chars (`bool`, *optional*, defaults to `True`): + Whether or not to tokenize Chinese characters. + + This should likely be deactivated for Japanese (see this + [issue](https://github.com/huggingface/transformers/issues/328)). + strip_accents (`bool`, *optional*): + Whether or not to strip all accents. If this option is not specified, then it will be determined by the + value for `lowercase` (as in the original BERT). + """ + + vocab_files_names = VOCAB_FILES_NAMES_ZH + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP_ZH + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION_ZH + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES_ZH + + def __init__(self, + vocab_file, + do_lower_case=True, + do_basic_tokenize=True, + never_split=None, + bos_token='', + eos_token='', + sep_token='', + cls_token='', + unk_token='', + pad_token='', + mask_token='', + tokenize_chinese_chars=True, + strip_accents=None, + **kwargs): + if not os.path.isfile(vocab_file): + raise ValueError( + f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained " + 'model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' + ) + self.vocab = load_vocab(vocab_file) + super().__init__( + do_lower_case=do_lower_case, + do_basic_tokenize=do_basic_tokenize, + never_split=never_split, + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + sep_token=sep_token, + cls_token=cls_token, + pad_token=pad_token, + mask_token=mask_token, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + **kwargs, + ) + + self.ids_to_tokens = collections.OrderedDict([ + (ids, tok) for tok, ids in self.vocab.items() + ]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + ) + self.wordpiece_tokenizer = WordpieceTokenizer( + vocab=self.vocab, unk_token=self.unk_token) + + @property + def do_lower_case(self): + return self.basic_tokenizer.do_lower_case + + @property + def vocab_size(self): + return len(self.vocab) + + def get_vocab(self): + return dict(self.vocab, **self.added_tokens_encoder) + + def _tokenize(self, text): + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize( + text, never_split=self.all_special_tokens): + + # If the token is part of the never_split set + if token in self.basic_tokenizer.never_split: + split_tokens.append(token) + else: + split_tokens += self.wordpiece_tokenizer.tokenize(token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.ids_to_tokens.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + out_string = ' '.join(tokens).replace(' ##', '').strip() + return out_string + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ( + [0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence + pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + + sep) * [1] + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + index = 0 + if os.path.isdir(save_directory): + vocab_file = os.path.join( + save_directory, + (filename_prefix + '-' if filename_prefix else '') + + VOCAB_FILES_NAMES['vocab_file']) + else: + vocab_file = (filename_prefix + + '-' if filename_prefix else '') + save_directory + with open(vocab_file, 'w', encoding='utf-8') as writer: + for token, token_index in sorted( + self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.' + ' Please check that the vocabulary is not corrupted!') + index = token_index + writer.write(token + '\n') + index += 1 + return (vocab_file, ) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa_fast.py b/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..50ad481e635d185f07a69f6ee4c61427f94d9faf --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/tokenization_ofa_fast.py @@ -0,0 +1,215 @@ +# Copyright 2022 OFA-Sys Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for OFA.""" +from typing import List, Optional, Tuple + +import json +from tokenizers import normalizers +from transformers import PreTrainedTokenizerFast +from transformers.models.bart.tokenization_bart_fast import BartTokenizerFast +from transformers.utils import logging + +from modelscope.utils.constant import ModelFile +from .tokenization_ofa import OFATokenizer, OFATokenizerZH + +logger = logging.get_logger() + +VOCAB_FILES_NAMES = { + 'vocab_file': 'vocab.json', + 'merges_file': 'merges.txt', + 'tokenizer_file': 'tokenizer.json' +} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': { + 'ofa-base': 'https://huggingface.co/ofa-base/resolve/main/vocab.json', + }, + 'merges_file': { + 'ofa-base': 'https://huggingface.co/ofa-base/resolve/main/merges.txt', + }, + 'tokenizer_file': { + 'ofa-base': + 'https://huggingface.co/ofa-base/resolve/main/tokenizer.json', + }, + # OFA models are implemented to be compatible with both huggingface + # and modelscope frameworks. For all OFA models available on huggingface, + # please refer to https://huggingface.co/models?filter=ofa +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'ofa-base': 1024, +} + +VOCAB_FILES_NAMES_ZH = {'vocab_file': ModelFile.VOCAB_FILE} + +PRETRAINED_VOCAB_FILES_MAP_ZH = { + 'vocab_file': { + 'bert-base-chinese': + 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt', + } + # OFA models are implemeted to be compatible with both huggingface + # and modelscope frameworks. For all OFA models available on huggingface, + # please refer to https://huggingface.co/models?filter=ofa +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES_ZH = { + 'ofa-base': 1024, +} + +PRETRAINED_INIT_CONFIGURATION_ZH = { + 'bert-base-chinese': { + 'do_lower_case': True + }, +} + + +class OFATokenizerFast(BartTokenizerFast): + r""" + Construct a "fast" OFA tokenizer (backed by HuggingFace's *tokenizers* library). + + [`~OFATokenizerFast`] is identical to [`BartTokenizerFast`] and runs end-to-end tokenization: punctuation splitting + and wordpiece. + + Refer to superclass [`BartTokenizerFast`] for usage examples and documentation concerning parameters. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + slow_tokenizer_class = OFATokenizer + + +class OFATokenizerZHFast(PreTrainedTokenizerFast): + r""" + Construct a "fast" OFA tokenizer (backed by HuggingFace's *tokenizers* library). + + [`~OFATokenizerFast`] is identical to [`BartTokenizerFast`] and runs end-to-end tokenization: punctuation splitting + and wordpiece. + + Refer to superclass [`BartTokenizerFast`] for usage examples and documentation concerning parameters. + """ + vocab_files_names = VOCAB_FILES_NAMES_ZH + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP_ZH + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION_ZH + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES_ZH + slow_tokenizer_class = OFATokenizerZH + + def __init__(self, + vocab_file=None, + tokenizer_file=None, + do_lower_case=True, + bos_token='', + eos_token='', + sep_token='', + cls_token='', + unk_token='', + pad_token='', + mask_token='', + tokenize_chinese_chars=True, + strip_accents=None, + **kwargs): + super().__init__( + vocab_file, + tokenizer_file=tokenizer_file, + do_lower_case=do_lower_case, + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + sep_token=sep_token, + cls_token=cls_token, + pad_token=pad_token, + mask_token=mask_token, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + **kwargs, + ) + + normalizer_state = json.loads( + self.backend_tokenizer.normalizer.__getstate__()) + if (normalizer_state.get('lowercase', do_lower_case) != do_lower_case + or normalizer_state.get('strip_accents', strip_accents) + != strip_accents or normalizer_state.get( + 'handle_chinese_chars', + tokenize_chinese_chars) != tokenize_chinese_chars): + normalizer_class = getattr(normalizers, + normalizer_state.pop('type')) + normalizer_state['lowercase'] = do_lower_case + normalizer_state['strip_accents'] = strip_accents + normalizer_state['handle_chinese_chars'] = tokenize_chinese_chars + self.backend_tokenizer.normalizer = normalizer_class( + **normalizer_state) + + self.do_lower_case = do_lower_case + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + + if token_ids_1: + output += token_ids_1 + [self.sep_token_id] + + return output + + def create_token_type_ids_from_sequences( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence + pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + + sep) * [1] + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + files = self._tokenizer.model.save( + save_directory, name=filename_prefix) + return tuple(files) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/utils/__init__.py b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/utils/constant.py b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..f455e41a7f85aa08219bd45659a0f7dea3758b33 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/constant.py @@ -0,0 +1,17 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import Tasks + +OFA_TASK_KEY_MAPPING = { + Tasks.ocr_recognition: OutputKeys.TEXT, + Tasks.image_captioning: OutputKeys.CAPTION, + Tasks.text_summarization: OutputKeys.TEXT, + Tasks.visual_question_answering: OutputKeys.TEXT, + Tasks.visual_grounding: OutputKeys.BOXES, + Tasks.text_classification: OutputKeys.LABELS, + Tasks.image_classification: OutputKeys.LABELS, + Tasks.visual_entailment: OutputKeys.LABELS, + Tasks.auto_speech_recognition: OutputKeys.TEXT, + Tasks.sudoku: OutputKeys.TEXT, + Tasks.text2sql: OutputKeys.TEXT, +} diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/utils/utils.py b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b874aaa12293ff8b3ec6caff8ff7c659ef5d3c1e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/utils/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Optional + +import torch +import torch.nn as nn + + +def expand_mask(mask: torch.Tensor, + dtype: torch.dtype, + tgt_len: Optional[int] = None): + r""" + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, + src_len).to(dtype) + return expanded_mask.masked_fill(expanded_mask.bool(), + torch.finfo(dtype).min) + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + r""" + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + Args: + x (`nn.Modules`): input nn layers. + drop_prob (`float`): drop path ratio. + training (`bool`): whether is training or inference. + """ + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (1, x.shape[1], 1) + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + r""" + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + See more details about drop path from https://arxiv.org/pdf/1605.07648v4.pdf. + + Args: + drop_prob: drop path ratio. + """ + + def __init__(self, drop_prob=None): + super().__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa/vit.py b/runtime/Lib/site_packages/models/multi_modal/ofa/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..36d1707eae42f59ecc573a2c2d1d86acb678d7ac --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa/vit.py @@ -0,0 +1,251 @@ +# Copyright (c) 2021 OpenAI +# +# This source code is licensed under the MIT license which can be found at +# https://github.com/openai/CLIP/blob/main/LICENSE +from collections import OrderedDict + +import torch +import torch.nn.functional as F +from fairseq.modules import LayerNorm +from torch import nn + +from .utils.utils import DropPath + +__all__ = [ + 'vit_base', + 'vit_large', + 'vit_large_336', + 'vit_huge', +] + + +class QuickGELU(nn.Module): + r""" + An activation function module. + """ + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + r""" + A residual attention block module. + + step 1. Calculate the self attention in input with layer normalization. + step 2. Add input to the result of self attention's result as I. + step 3. Calculate the mlp of input I with layer normalization. + step 4. Add I to the result of mlp. + """ + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None, + drop_path_rate=0.0): + r""" + Args: + d_model (`int`): The embedding dimensions. + n_head (`int`): The number of heads in self attention block. + attn_mask (`Tensor`, **optional**, default to None): + Attention mask using in self attention. + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([ + ('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model)), + ])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + self.drop_path = DropPath(drop_path_rate) + + def attention(self, x: torch.Tensor): + r""" + A wrapper of self attention . + """ + self.attn_mask = ( + self.attn_mask.to(dtype=x.dtype, device=x.device) + if self.attn_mask is not None else None) + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.drop_path(self.attention(self.ln_1(x))) + x = x + self.drop_path(self.mlp(self.ln_2(x))) + return x + + +class Transformer(nn.Module): + r""" + A transformer module using in `VisionTransformer`. + + Execute a sequential of `ResidualAttentionBlock`. + """ + + def __init__( + self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None, + drop_path_rate: float = 0.0, + ): + r""" + Args: + width (`int`): The width of input image. + layers (`int`): The number of `ResidualAttentionBlock` layers. + heads (int): The number of self attention heads. + attn_mask (`Tensor`, **optional**, default to None): + Attention mask using in self attention. + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask, drop_path_rate) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + r""" + Vision transformer module. + + step 1. Using conv2d to get the image embedding. + step 2. If the resolution of input image doesn't equal to the initialized one + do `bilinear` interpolate to get new patch position embedding. + step 3. Add position embedding to image embedding to generate final image representation. + step 4. Do `Transformer` to the image representation. + """ + + def __init__( + self, + input_resolution: int, + patch_size: int, + width: int, + layers: int, + heads: int, + drop_path_rate: float = 0.0, + ): + r""" + Args: + input_resolution (`int`): The resolution of input image. + patch_size (`int`): The resolution of each patch image. + width (`int`): The dimension of each patch image. + layers (`int`): The number of `ResidualAttentionBlock` in `Transformer`. + heads (`int`): The number of heads in self attention block. + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + super().__init__() + self.input_resolution = input_resolution + self.patch_size = patch_size + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False, + ) + + scale = width**-0.5 + self.width = width + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + self.transformer = Transformer( + width, layers, heads, drop_path_rate=drop_path_rate) + + def forward(self, x: torch.Tensor): + resolution = x.shape[-2] + height, width = x.shape[-2] // self.patch_size, x.shape[ + -1] // self.patch_size + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + if resolution != self.input_resolution: + old_pe = self.positional_embedding[1:] + patch_num = self.input_resolution // self.patch_size + old_pe = old_pe.reshape(1, patch_num, patch_num, + -1).permute(0, 3, 1, 2) + new_pe = F.interpolate( + old_pe, size=(height, width), mode='bilinear') + new_pe = new_pe.permute(0, 2, 3, 1).reshape(height * width, -1) + x = x + new_pe.to(x.dtype) + else: + x = x + self.positional_embedding[1:].to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + bz, seq, hidden = x.shape + x = x.transpose(1, 2).reshape(bz, hidden, height, width) + + return x + + +def vit_base(drop_path_rate: float = 0.0): + r""" + An instance of base vision transformer model. + + Args: + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + return VisionTransformer(224, 16, 768, 9, 12, drop_path_rate) + + +def vit_large(drop_path_rate: float = 0.0): + r""" + An instance of large vision transformer model. + + Args: + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + return VisionTransformer(224, 14, 1024, 18, 16, drop_path_rate) + + +def vit_large_336(drop_path_rate: float = 0.0): + r""" + An instance of large vision transformer model with 336 as input image width . + + Args: + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + return VisionTransformer(336, 14, 1024, 18, 16, drop_path_rate) + + +def vit_huge(drop_path_rate: float = 0.0): + r""" + An instance of huge vision transformer model. + + Args: + drop_path_rate (`float`, **optional**, default to 0.0): + Drop path rate. See more details about drop path from + https://arxiv.org/pdf/1605.07648v4.pdf. + """ + return VisionTransformer(224, 14, 1280, 24, 16, drop_path_rate) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa_for_all_tasks.py b/runtime/Lib/site_packages/models/multi_modal/ofa_for_all_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..6bdd5154daa19afa44214f2e9500ad698983bd22 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa_for_all_tasks.py @@ -0,0 +1,521 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os +import re +import string +from functools import partial +from os import path as osp +from typing import Any, Callable, Dict, List, Optional, Union + +import json +import torch.cuda +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.preprocessors.ofa.utils.collate import collate_tokens +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile +from modelscope.utils.trie import Trie +from .ofa import MMSpeechModel, OFAModel, OFATokenizer, OFATokenizerZH +from .ofa.generate import sequence_generator as sg +from .ofa.generate.utils import move_to_device +from .ofa.utils.constant import OFA_TASK_KEY_MAPPING, Tasks +from .ofa.utils.utils import expand_mask + +__all__ = ['OfaForAllTasks'] + + +@MODELS.register_module(Tasks.image_captioning, module_name=Models.ofa) +@MODELS.register_module(Tasks.ocr_recognition, module_name=Models.ofa) +@MODELS.register_module(Tasks.visual_grounding, module_name=Models.ofa) +@MODELS.register_module( + Tasks.visual_question_answering, module_name=Models.ofa) +@MODELS.register_module(Tasks.visual_entailment, module_name=Models.ofa) +@MODELS.register_module(Tasks.image_classification, module_name=Models.ofa) +@MODELS.register_module(Tasks.text_summarization, module_name=Models.ofa) +@MODELS.register_module(Tasks.text_classification, module_name=Models.ofa) +@MODELS.register_module(Tasks.auto_speech_recognition, module_name=Models.ofa) +@MODELS.register_module(Tasks.sudoku, module_name=Models.ofa) +@MODELS.register_module(Tasks.text2sql, module_name=Models.ofa) +class OfaForAllTasks(TorchModel): + r""" + All ofa tasks using uniform ofa model structure. So far, we support three types of tasks: + 1. text generation tasks: ocr_recognition, image_captioning and text_summarization + 2. visual grounding tasks: visual grounding + 3. classification tasks: text classification and image classification. + + Attributes: + cfg: Task configs exclude model configs, such as generator's config. + model: OFA uniform model using in this task. + language: The language using in the model. So far, we support three types of language, `en` for English, + `zh` and `cn` for Chinese, default to `en`. + tokenizer: OFA tokenizer for tokenizing the input for OFA model. + batch_size: Batch size. + patch_image_size: The image size of input image, default to 480. + val_batch_size: The validation batch size. + transtab: A translation table of punctuation. + gen_type: Generation type, so far, we support two types of gen_type, `generation` for generation tasks, + `traverse` for classification tasks, default to `generation`. + bos_item: The id of beginning of a sequence. + pad_item: The id of padding of a sequence. + eos_item: The id of ending of a sequence. + index2ans: A mapping from index to label using in classification tasks. + ans2label_dict: A mapping from label to index using in classification tasks. + constraint_trie: A trie tree building from label using in classification tasks. + val_ans_l: A validation set of label using in classification tasks. + val_masks_l: A validation set of mask using in classification tasks. + generator: A sequence generator with OFA model to generate image code. + task_inference_mapping: A mapping from task name to execution function in task inference. + pattern: Regex pattern which find the blanks after/before the words except ` a-zA-Z0-9.,:!?` + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + if os.path.exists(model_dir): + model_dir = os.path.abspath(model_dir) + super().__init__(model_dir=model_dir, *args, **kwargs) + self.cfg = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + multimodal_type = self.cfg.model.get('multimodal_type', 'default') + if multimodal_type in ['default', 'text2sql']: + model = OFAModel.from_pretrained(model_dir) + elif multimodal_type == 'mmspeech': + model = MMSpeechModel.from_pretrained(model_dir) + else: + raise NotImplementedError + self.model = model.module if hasattr(model, 'module') else model + self.language = self.cfg.model.get('language', 'en') + if self.language == 'en': + self.tokenizer = OFATokenizer.from_pretrained(model_dir) + elif self.language in ['zh', 'cn']: + self.tokenizer = OFATokenizerZH.from_pretrained(model_dir) + else: + raise NotImplementedError + # there is some diff between here and our ofa code, + # there will be no need to use param: use_bpe + + if not model.use_ofasys: + if multimodal_type == 'default': + self.tokenizer.add_tokens( + [''.format(i) for i in range(8192)]) + self.tokenizer.add_tokens( + [''.format(i) for i in range(1000)]) + self.cfg.update({'num_bins': 1000, 'num_codes': 8192}) + elif multimodal_type == 'mmspeech': + self.tokenizer.add_tokens('') + self.tokenizer.add_tokens( + [''.format(i) for i in range(30000)]) + self.cfg.update({'num_bins': 0, 'num_codes': 30000}) + elif multimodal_type == 'text2sql': + self.tokenizer.add_tokens( + [''.format(i) for i in range(8192)]) + self.tokenizer.add_tokens( + [''.format(i) for i in range(1000)]) + self.cfg.update({'num_bins': 1000, 'num_codes': 8192}) + self.tokenizer.add_tokens(['>=', '<=']) + + self.batch_size = self.cfg.model.get('batch_size', 1) + self.patch_image_size = self.cfg.model.get('patch_image_size', 480) + self.max_image_size = self.cfg.model.get('max_image_size', 512) + self.val_batch_size = self.cfg.model.get('valid_batch_size', + self.batch_size) + self.transtab = str.maketrans( + {key: None + for key in string.punctuation}) + self.gen_type = self.cfg.model.get('gen_type', 'generation') + assert self.gen_type in ['generation', 'traverse'], \ + 'model.gen_type must be in ["generation", "traverse"]' + self.bos_item = torch.LongTensor([self.tokenizer.bos_token_id]) + self.pad_item = torch.LongTensor([self.tokenizer.pad_token_id]) + self.eos_item = torch.LongTensor([self.tokenizer.eos_token_id]) + self.index2ans = {} + self.ans2label_dict = {} + self.load_ans2label() + # Initialize generator + sg_args = { + 'tokenizer': self.tokenizer, + 'beam_size': 5, + 'max_len_b': 16, + 'min_len': 1, + 'no_repeat_ngram_size': 3, + 'constraint_range': None + } + if hasattr(self.cfg.model, 'beam_search'): + sg_args.update(self.cfg.model.beam_search) + self.num_return_sequences = self.cfg.model.get('num_return_sequences', + 1) + if len(self.ans2label_dict) > 0: + self.constraint_trie = Trie(self.tokenizer.eos_token_id) + self.val_ans_l = [] + self.val_masks_l = [] + self.build_trie() + sg_args['constraint_trie'] = self.constraint_trie + else: + self.constraint_trie = None + self.generator = sg.SequenceGenerator(**sg_args) + inference_d = { + 'generation': self._text_gen_inference, + 'traverse': self._traverse_inference, + } + self.task_inference_mapping = { + Tasks.ocr_recognition: self._text_gen_inference, + Tasks.image_captioning: self._text_gen_inference, + Tasks.text_summarization: self._text_gen_inference, + Tasks.visual_grounding: self._visual_grounding_inference, + Tasks.visual_entailment: inference_d[self.gen_type], + Tasks.visual_question_answering: inference_d[self.gen_type], + Tasks.text_classification: inference_d[self.gen_type], + Tasks.image_classification: inference_d[self.gen_type], + Tasks.auto_speech_recognition: self._text_gen_inference, + Tasks.sudoku: self._text_gen_inference, + Tasks.text2sql: self._text_gen_inference, + } + pattern_str = '((?<=[^ a-zA-Z0-9.,:!?]) +| +(?=[^ a-zA-Z0-9.,:!?]))' + self.pattern = re.compile(pattern_str) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + r""" + The entry function of task execution. So far, we support two types of execution pipeline: + 1. training, return the model's forward results. + 2. inference, return the result of `self.inference(input)` + + Args: + input (`Dict[Str, Any]`): + The input of the tasks, the actual value depending on the specific tasks. + Returns: + `Dict[Str, Any]` + + """ + input = move_to_device(input, self.model.device) + if self.model.training: + return self.model(**input['net_input']) + else: + return self.inference(input) + + def inference(self, input: Dict[str, Any]) -> Dict[str, Any]: + assert self.generator.beam_size >= self.num_return_sequences, \ + 'beam search can only return beam size sentences' + if self.ans2label_dict and self.gen_type == 'generation': + assert self.generator.beam_size <= len(self.ans2label_dict), \ + 'beam search will not work properly.' + r""" + Task inference function + + Args: + input (`Dict[Str, Any]`): + The input of the tasks, the actual value depending on the specific tasks. + Returns: + `Dict[Str, Any]` + + """ + ret = self.task_inference_mapping[self.cfg.task](input) + if 'samples' in input: + ret['samples'] = input['samples'] + return ret + + def postprocess(self, input: Dict[str, Any], **kwargs) -> Dict[str, Any]: + r""" + Do post processing after task's forward function is executed. So far, we have three strategies while do post + processing. + + 1. If the task is image captioning and using English language, some special words will be removed, such as + `!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~` + 2. If the task is not visual grounding, but a generation task using Chinese language, we will remove the + blank after/before the words except ` a-zA-Z0-9.,:!?` + 3. Other cases will return the input as result. + + Args: + input (`Dict[Str, Any]`): + The result of task's forward function. The key is one of the keys of OFA_TASK_KEY_MAPPING for + distinguishing different ofa tasks, while the value is the result of different tasks. + + Returns: + `Dict[Str, Any]` + """ + if not self.model.training and self.cfg.task == Tasks.image_captioning: + caption = input[OutputKeys.CAPTION] + result_l = list() + for cap in caption: + if self.language == 'en': + result_l.append( + [c.translate(self.transtab).strip() for c in cap]) + else: + result_l.append(cap) + input[OutputKeys.CAPTION] = result_l + if self.gen_type == 'generation' and self.language in [ + 'zh', 'cn' + ] and self.cfg.task != Tasks.visual_grounding: + ret_l = list() + for text in input[OFA_TASK_KEY_MAPPING[self.cfg.task]]: + ret_l.append([self.detokenizer(t) for t in text]) + input[OFA_TASK_KEY_MAPPING[self.cfg.task]] = ret_l + for key in [ + OutputKeys.CAPTION, OutputKeys.TEXT, OutputKeys.BOXES, + OutputKeys.LABELS, OutputKeys.SCORES + ]: + if key not in input: + input[key] = None + else: + if (len(input[key]) == 1 and isinstance(input[key], list)) \ + and self.cfg.task != Tasks.visual_grounding: + input[key] = input[key][0] + return input + + def _text_gen_inference(self, input): + r""" + The inference function fo text generation tasks. + 1. Using OFA sequence generator which match the api of other fairseq generators to generate the token indices. + 2. Decode the token indices to actual language tokens and skip the special tokens. + 3. For the usage of classification scenario, add default score with `len(result)`. + + Args: + input (`Dict[Str, Any]`): + The input of the tasks, the actual value depending on the specific tasks. + Returns: + `Dict[Str, Any]` + """ + gen_outputs = self.generator.generate([self.model], + input, + prefix_tokens=input.get( + 'prefix_tokens', None)) + results = list() + for idx, gen_out in enumerate(gen_outputs): + gen_token_l = [] + for beam_gen_out in gen_out[:self.num_return_sequences]: + decode_tokens = beam_gen_out['tokens'] + if 'prefix_tokens' in input: + prefix_len = input['prefix_tokens'][idx].ne( + self.pad_item.to(self.model.device)).sum() + decode_tokens = decode_tokens[prefix_len:] + gen_token_l.append(decode_tokens) + result = self.tokenizer.batch_decode( + gen_token_l, skip_special_tokens=True) + result = [item.strip() for item in result] + result.extend([''] * (self.num_return_sequences - len(result))) + results.append(result) + # text generation tasks have no score + ret = {OFA_TASK_KEY_MAPPING[self.cfg.task]: results} + if self.ans2label_dict: + ret[OutputKeys.SCORES] = [[1.0]] * len(results) + return ret + + def _visual_grounding_inference(self, input): + r""" + The inference function for visual grounding tasks. + 1. Using OFA sequence generator which match the api of other fairseq generators to generate the token indices. + 2. Decode the token indices into region boxes. + 3. Add default score with `batch_size` + + Args: + input (`Dict[Str, Any]`): + The input of the tasks, the actual value depending on the specific tasks. + Returns: + `Dict[Str, Any]` + """ + gen_output = self.generator.generate([self.model], input) + tokens = [gen_output[i][0]['tokens'] for i in range(len(gen_output))] + region_coord_l = list() + for i in range(len(tokens)): + region_coord_l.append(tokens[i][:-1] + - len(self.tokenizer.get_vocab().items()) + + self.cfg.num_bins) + region_tensor = torch.stack(region_coord_l, dim=0) + region_tensor = region_tensor / (self.cfg.num_bins + - 1) * self.max_image_size + region_tensor[:, ::2] /= input['w_resize_ratios'] + region_tensor[:, 1::2] /= input['h_resize_ratios'] + return { + OutputKeys.BOXES: + move_to_device(region_tensor, torch.device('cpu')).tolist(), + OutputKeys.SCORES: [1.0] * region_tensor.shape[0] + } + + def _traverse_inference(self, input): + r""" + The inference function fo classification tasks. + + Args: + input (`Dict[Str, Any]`): + The input of the tasks, the actual value depending on the specific tasks. + Returns: + `Dict[Str, Any]` + """ + encoder_input = dict() + for key in input['net_input'].keys(): + encoder_input[key] = input['net_input'][key] + encoder_out = self.model.encoder(**encoder_input) + valid_result = [] + for val_ans, val_masks in zip(self.val_ans_l, self.val_masks_l): + valid_size = len(val_ans) + valid_tgt_items = [ + torch.cat([ + torch.tensor(decoder_prompt[1:]).to('cpu'), valid_answer, + self.eos_item + ]) for decoder_prompt in input['decoder_prompts'] + for valid_answer in val_ans + ] + valid_prev_items = [ + torch.cat( + [torch.tensor(decoder_prompt).to('cpu'), valid_answer]) + for decoder_prompt in input['decoder_prompts'] + for valid_answer in val_ans + ] + valid_constraint_mask_items = [ + torch.cat([ + torch.zeros( + len(decoder_prompt) - 1, + valid_constraint_mask.size(1)).bool(), + valid_constraint_mask], dim=0) # yapf: disable + for decoder_prompt in input['decoder_prompts'] # yapf: disable + for valid_constraint_mask in val_masks] # yapf: disable + valid_tgt = collate_tokens( + valid_tgt_items, + pad_idx=self.tokenizer.pad_token_id).to(self.model.device) + valid_prev_output = collate_tokens( + valid_prev_items, + pad_idx=self.tokenizer.pad_token_id).to(self.model.device) + val_masks = collate_tokens( + valid_constraint_mask_items, + pad_idx=self.tokenizer.pad_token_id).to(self.model.device) + new_encoder_out = { + 'last_hidden_state': + encoder_out['last_hidden_state'].repeat_interleave( + valid_size, dim=0), + 'padding_mask': + encoder_out['padding_mask'].repeat_interleave( + valid_size, dim=0), + 'position_embedding': + encoder_out['position_embedding'].repeat_interleave( + valid_size, dim=0) + } + encoder_attention_mask = expand_mask( + new_encoder_out['padding_mask'], + new_encoder_out['last_hidden_state'].dtype, + valid_prev_output.shape[-1]) + + decoder_out = self.model.decoder( + valid_prev_output, + encoder_hidden_states=new_encoder_out['last_hidden_state'], + encoder_attention_mask=encoder_attention_mask, + src_pos_embed=new_encoder_out['position_embedding']) + + decoder_out[0].masked_fill_(~val_masks, -math.inf) + lprobs = self.model.get_normalized_probs( + decoder_out, log_probs=True) + scores = lprobs.gather( + dim=-1, index=valid_tgt.unsqueeze(-1)).squeeze(-1) + scores = scores.masked_fill( + valid_tgt.eq(self.tokenizer.pad_token_id), 0) + scores = scores.masked_fill((~val_masks).all(2), 0) + scores = scores.sum(1) + scores = scores.view(-1, valid_size) + valid_result.append(scores) + valid_result = torch.cat(valid_result, dim=-1) + predicts = valid_result.argmax(1).tolist() + probs = F.softmax(valid_result, dim=-1) + hyps = [self.index2ans[predict_index] for predict_index in predicts] + scores = [ + float(prob[idx].cpu().detach().numpy()) + for prob, idx in zip(probs, predicts) + ] + return {OutputKeys.LABELS: hyps, OutputKeys.SCORES: scores} + + def build_trie(self): + r""" + Building a trie tree for classification label and mask. + """ + answer_item_list = [] + + for i, answer in enumerate(self.ans2label_dict.keys()): + answer_item = self.tokenizer( + ' ' + answer, return_tensors='pt', + add_special_tokens=False).input_ids.squeeze(0) + answer_item_list.append(answer_item) + self.index2ans[i] = answer + self.constraint_trie.insert([self.tokenizer.bos_token_id] + + answer_item.tolist() + + [self.tokenizer.eos_token_id]) + + constraint_mask_list = [] + for answer_item in answer_item_list: + constraint_mask = torch.zeros( + (len(answer_item) + 1, + len(self.tokenizer.get_vocab()))).bool() + for i in range(len(answer_item) + 1): + constraint_prefix_token = [self.tokenizer.bos_token_id + ] + answer_item[:i].tolist() + constraint_nodes = self.constraint_trie.get_next_layer( + constraint_prefix_token) + constraint_mask[i][constraint_nodes] = True + constraint_mask_list.append(constraint_mask) + + for i in range(0, len(answer_item_list), self.val_batch_size): + self.val_ans_l += [answer_item_list[i:i + self.val_batch_size]] + self.val_masks_l += [ + constraint_mask_list[i:i + self.val_batch_size] + ] + + def load_ans2label(self): + r""" + Load answer to label dict from file, using in building trie function. + """ + if self.cfg.model.get('answer2label', None): + ans2label_file = osp.join(self.model_dir, + self.cfg.model.answer2label) + with open(ans2label_file, 'r', encoding='utf-8') as reader: + self.ans2label_dict = json.load(reader) + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = None, + config: Optional[dict] = None, + **kwargs): + r""" + Save the task model, its configuration and other related files to a directory, so that it can be re-loaded + + Args: + target_folder (Union[str, os.PathLike]): + Directory to which to save. Will be created if it doesn't exist. + + save_checkpoint_names (Union[str, List[str]]): + The checkpoint names to be saved in the target_folder + + save_function (Callable, optional): + The function to use to save the state dictionary. + + config (Optional[dict], optional): + The config for the configuration.json, might not be identical with model.config + + """ + super(OfaForAllTasks, self). \ + save_pretrained(target_folder=target_folder, + save_checkpoint_names=save_checkpoint_names, + save_function=partial(save_function, with_meta=False), + config=config, + **kwargs) + + def detokenizer(self, text): + r""" + Remove the blank after/before the words except ` a-zA-Z0-9.,:!?` + """ + return self.pattern.sub('', text) diff --git a/runtime/Lib/site_packages/models/multi_modal/ofa_for_text_to_image_synthesis_model.py b/runtime/Lib/site_packages/models/multi_modal/ofa_for_text_to_image_synthesis_model.py new file mode 100644 index 0000000000000000000000000000000000000000..76ab117082131405ff1604f41142054a3cec87e3 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/ofa_for_text_to_image_synthesis_model.py @@ -0,0 +1,332 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from os import path as osp +from typing import Any, Dict + +import json +import numpy as np +import torch +import torch.cuda +from PIL import Image +from pkg_resources import packaging +from taming.models.vqgan import GumbelVQ, VQModel +from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, + ToTensor) + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.mmr.models.module_clip import CLIP +from modelscope.models.multi_modal.mmr.models.tokenization_clip import \ + SimpleTokenizer as ClipTokenizer +from modelscope.models.multi_modal.ofa import OFAModel, OFATokenizer +from modelscope.models.multi_modal.ofa.generate import sequence_generator as sg +from modelscope.models.multi_modal.ofa.generate.search import Sampling +from modelscope.models.multi_modal.ofa.generate.utils import move_to_device +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +try: + from torchvision.transforms import InterpolationMode + + BICUBIC = InterpolationMode.BICUBIC +except ImportError: + BICUBIC = Image.BICUBIC + +__all__ = ['OfaForTextToImageSynthesis'] + + +def custom_to_pil(x): + r""" + Change the custom array to PIL image. + + Args: + x (`object`) + Object with array interface. + Returns: + A pillow image object. + """ + x = x.detach().cpu() + x = torch.clamp(x, -1., 1.) + x = (x + 1.) / 2. + x = x.permute(1, 2, 0).numpy() + x = (255 * x).astype(np.uint8) + x = Image.fromarray(x) + if not x.mode == 'RGB': + x = x.convert('RGB') + return x + + +def load_vqgan(config, ckpt_path=None, is_gumbel=False): + r""" + Load checkpoint for vqgan model. + + Args: + config (`Dict[Str, Any]`): + Model configs for vgqan model initialization. + ckpt_path (`str` or `os.PathLike`, **optional**, default to `None`): + Checkpoint path. IF not None, it will load model parameters from the checkpoint path. + is_gumbel (`bool`, **optional**, default to `False`): + Whether or not to use gumbel vqgan. + + Returns: + A vqgan model with evaluation state. + """ + if is_gumbel: + model = GumbelVQ(**config['model']['params']) + else: + model = VQModel(**config['model']['params']) + if ckpt_path is not None: + sd = torch.load(ckpt_path, map_location='cpu')['state_dict'] + missing, unexpected = model.load_state_dict(sd, strict=False) + return model.eval() + + +def build_clip_model(model_path): + r""" + Build clip model, the model structure can be found in `modelscope.models.multi_modal.mmr.models.module_clip.CLIP` + + Args: + model_path (`str` or `os.PathLike`): + Model path in which store the clip model's parameters. + + Returns: + A clip model with evaluation state. + """ + state_dict = torch.load(model_path, map_location='cpu').state_dict() + vit = 'visual.proj' in state_dict + if vit: + vision_width = state_dict['visual.conv1.weight'].shape[0] + vision_layers = len([ + k for k in state_dict.keys() + if k.startswith('visual.') and k.endswith('.attn.in_proj_weight') + ]) + vision_patch_size = state_dict['visual.conv1.weight'].shape[-1] + grid_size = round( + (state_dict['visual.positional_embedding'].shape[0] - 1)**0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [ + len( + set( + k.split('.')[2] for k in state_dict + if k.startswith(f'visual.layer{b}'))) + for b in [1, 2, 3, 4] + ] + vision_layers = tuple(counts) + vision_width = state_dict['visual.layer1.0.conv1.weight'].shape[0] + output_width = round( + (state_dict['visual.attnpool.positional_embedding'].shape[0] + - 1)**0.5) + vision_patch_size = None + assert output_width**2 + 1 == state_dict[ + 'visual.attnpool.positional_embedding'].shape[0] + image_resolution = output_width * 32 + + embed_dim = state_dict['text_projection'].shape[1] + context_length = state_dict['positional_embedding'].shape[0] + vocab_size = state_dict['token_embedding.weight'].shape[0] + transformer_width = state_dict['ln_final.weight'].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split('.')[2] for k in state_dict + if k.startswith('transformer.resblocks'))) + + model = CLIP(embed_dim, image_resolution, vision_layers, vision_width, + vision_patch_size, context_length, vocab_size, + transformer_width, transformer_heads, transformer_layers) + + for key in ['input_resolution', 'context_length', 'vocab_size']: + if key in state_dict: + del state_dict[key] + + model.load_state_dict(state_dict) + return model.eval() + + +def _convert_image_to_rgb(image): + r""" + Convert the mode of the image to `RGB`. + """ + return image.convert('RGB') + + +def build_clip_transform(n_px): + r""" + Build image transformation. All images sent to clip model will be transformed in this transformation. + + Args: + n_px(`int` or `sequence`): + Desired output size of resize and crop transformation. + Returns: + A compose of transformations. + """ + return Compose([ + Resize(n_px, interpolation=BICUBIC), + CenterCrop(n_px), + _convert_image_to_rgb, + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)), + ]) + + +@MODELS.register_module(Tasks.text_to_image_synthesis, module_name=Models.ofa) +class OfaForTextToImageSynthesis(Model): + r""" + OFA task for text to image synthesis. + + Attributes: + model: OFA uniform model using in this task. + cfg: Task configs exclude model configs, such as generator's config. + tokenizer: OFA tokenizer for tokenizing the input for OFA model. + vqgan_model: A vqgan model for image decoding. + clip_tokenizer: CLIP tokenizer for tokenizing the input for CLIP model. + clip_model: A CLIP model for ranking the generating image with original input text to select the best one. + generator: A sequence generator with OFA model to generate image code. + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + # Initialize ofa + model = OFAModel.from_pretrained(model_dir) + self.cfg = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + self.model = model.module if hasattr(model, 'module') else model + self.tokenizer = OFATokenizer.from_pretrained(model_dir) + self.tokenizer.add_tokens([''.format(i) for i in range(8192)]) + self.tokenizer.add_tokens([''.format(i) for i in range(1000)]) + self._device = torch.device('cuda') if torch.cuda.is_available() \ + else torch.device('cpu') + self.model.to(self._device) + + # Initialize vqgan + vqgan_config = json.load( + open(os.path.join(model_dir, 'vqgan_config.json'))) + self.vqgan_model = load_vqgan( + vqgan_config, + ckpt_path=os.path.join(model_dir, 'vqgan_model.ckpt'), + is_gumbel=True).to(self._device) + + # Initialize OpenAI clip + + self.clip_tokenizer = ClipTokenizer(model_dir) + self.clip_model = build_clip_model( + os.path.join(model_dir, 'ViT-B-16.pt')) + self.clip_preprocess = build_clip_transform( + self.clip_model.visual.input_resolution) + + self.clip_model.to(self._device) + self.clip_model.eval() + + # Initialize generator + sampling = Sampling(self.tokenizer, sampling_topp=0.9) + sg_args = { + 'tokenizer': self.tokenizer, + 'beam_size': 2, + 'max_len_b': 1024, + 'min_len': 1024, + 'search_strategy': sampling, + 'gen_code': True, + 'constraint_range': '50265,58457' + } + if hasattr(self.cfg.model, 'beam_search'): + sg_args.update(self.cfg.model.beam_search) + self.generator = sg.SequenceGenerator(**sg_args) + + def clip_tokenize(self, texts, context_length=77, truncate=False): + + if isinstance(texts, str): + texts = [texts] + + sot_token = self.clip_tokenizer.encoder['<|startoftext|>'] + eot_token = self.clip_tokenizer.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + self.clip_tokenizer.encode(text) + + [eot_token] for text in texts] + if packaging.version.parse( + torch.__version__) < packaging.version.parse('1.8.0'): + result = torch.zeros( + len(all_tokens), context_length, dtype=torch.long) + else: + result = torch.zeros( + len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError( + f'Input {texts[i]} is too long for context length {context_length}' + ) + result[i, :len(tokens)] = torch.tensor(tokens) + + return result + + def forward(self, input: Dict[str, Any]): + r""" + The entry function of text to image synthesis task. + 1. Using OFA model to generate an image code candidate set. + 2. Using vqgan model to decode the generated images to a pillow image set. + 3. Using CLIP model to rank the candidate set, choosing the best generated image. + + Args: + input (`Dict[Str, Any]`): + The input of the task + Returns: + A generated pillow image. + """ + + text = input['samples'][0]['text'] + input = move_to_device(input, self._device) + clip_text_input = self.clip_tokenize([text]).to(self._device) + + gen_output = self.generator.generate([self.model], input) + gen_tokens = torch.stack( + [item['tokens'][:-1] for item in gen_output[0]], dim=0) + codes = gen_tokens.view(-1, 32, 32) - 50265 + + quant_b = self.vqgan_model.quantize.get_codebook_entry( + codes.view(-1), + list(codes.size()) + [self.vqgan_model.quantize.embedding_dim]) + imgs = self.vqgan_model.decode(quant_b) + + sample_num = imgs.size()[0] + pil_imgs = [custom_to_pil(imgs[i]) for i in range(sample_num)] + + clip_image_input = torch.stack( + [self.clip_preprocess(img) for img in pil_imgs], + dim=0).to(self._device) + + with torch.no_grad(): + hyp_image_features = self.clip_model.encode_image(clip_image_input) + hyp_image_features /= hyp_image_features.norm(dim=-1, keepdim=True) + text_features = self.clip_model.encode_text(clip_text_input) + text_features /= text_features.norm(dim=-1, keepdim=True) + ti_similarity = hyp_image_features @ text_features.T + + sorted_score, ti_indices = torch.sort( + ti_similarity.view(-1), descending=True) + + pil_imgs_orderby_ti = [pil_imgs[index] for index in ti_indices] + return pil_imgs_orderby_ti[0] diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/__init__.py b/runtime/Lib/site_packages/models/multi_modal/prost/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..db249cad0707568b5fcc2f285304e0f5a3b87d11 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .models import ProSTForTVRetrieval diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/dataloaders/__init__.py b/runtime/Lib/site_packages/models/multi_modal/prost/dataloaders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/dataloaders/rawvideo_util.py b/runtime/Lib/site_packages/models/multi_modal/prost/dataloaders/rawvideo_util.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ac3f94719f96e26aa2db4a2228bf6c5aa47282 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/dataloaders/rawvideo_util.py @@ -0,0 +1,117 @@ +# The implementation is adopted from Huaishao Luo, +# made pubicly available under the MIT License at https://github.com/ArrowLuo/CLIP4Clip + +import cv2 +import numpy as np +import torch as th +from PIL import Image +from torchvision.transforms import (CenterCrop, Compose, InterpolationMode, + Normalize, Resize, ToTensor) + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class RawVideoExtractorCV2(): + + def __init__( + self, + centercrop=False, + size=224, + frame_rate=-1, + ): + self.centercrop = centercrop + self.size = size + self.framerate = frame_rate + self.transform = self._transform(self.size) + + def _transform(self, n_px): + return Compose([ + Resize(n_px, interpolation=InterpolationMode.BICUBIC), + CenterCrop(n_px), + lambda image: image.convert('RGB'), + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)), + ]) + + def video_to_tensor(self, + video_file, + preprocess, + sample_fp=0, + start_time=None, + end_time=None): + if start_time is not None or end_time is not None: + assert isinstance(start_time, int) and isinstance(end_time, int) \ + and start_time > -1 and end_time > start_time + assert sample_fp > -1 + + # Samples a frame sample_fp X frames. + cap = cv2.VideoCapture(video_file) + frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = int(cap.get(cv2.CAP_PROP_FPS)) + + if fps == 0: + logger.info(f'{video_file} with fps 0!!!') + total_duration = (frameCount + fps - 1) // fps + start_sec, end_sec = 0, total_duration + + if start_time is not None: + start_sec, end_sec = start_time, end_time if end_time <= total_duration else total_duration + cap.set(cv2.CAP_PROP_POS_FRAMES, int(start_time * fps)) + + interval = 1 + if sample_fp > 0: + interval = fps // sample_fp + else: + sample_fp = fps + if interval == 0: + interval = 1 + + inds = [ind for ind in np.arange(0, fps, interval)] + assert len(inds) >= sample_fp + inds = inds[:sample_fp] + + ret = True + images = [] + + for sec in np.arange(start_sec, end_sec + 1): + if not ret: + break + sec_base = int(sec * fps) + for ind in inds: + cap.set(cv2.CAP_PROP_POS_FRAMES, sec_base + ind) + ret, frame = cap.read() + if not ret: + break + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + images.append( + preprocess(Image.fromarray(frame_rgb).convert('RGB'))) + + cap.release() + + if len(images) > 0: + video_data = th.tensor(np.stack(images)) + else: + video_data = th.zeros(1) + return {'video': video_data} + + def get_video_data(self, video_path, start_time=None, end_time=None): + image_input = self.video_to_tensor( + video_path, + self.transform, + sample_fp=self.framerate, + start_time=start_time, + end_time=end_time) + return image_input + + def process_raw_data(self, raw_video_data): + tensor_size = raw_video_data.size() + tensor = raw_video_data.view(-1, 1, tensor_size[-3], tensor_size[-2], + tensor_size[-1]) + return tensor + + +# An ordinary video frame extractor based CV2 +RawVideoExtractor = RawVideoExtractorCV2 diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/__init__.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a35083a4fb07e8c1440edef396c36651a06ab6df --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .prost_model import ProSTForTVRetrieval diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/modeling.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..37c875a6c7a916af30c353d9bd2f0ae3dd2c5bc8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/modeling.py @@ -0,0 +1,704 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os +import platform +from collections import OrderedDict +from types import SimpleNamespace + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + +from modelscope.models.multi_modal.prost.models.module_clip import ( + _PT_NAME, CLIP, QuickGELU, convert_weights) +from modelscope.models.multi_modal.prost.models.module_cross import ( + CrossConfig, CrossModel) +from modelscope.models.multi_modal.prost.models.module_cross import \ + Transformer as TransformerClip +from modelscope.models.multi_modal.prost.models.until_module import ( + AllGather, CrossEn, Event_decoder, Frame_decoder, LayerNorm, + PreTrainedModel, make_patch_shift) +from modelscope.utils.logger import get_logger + +allgather = AllGather.apply + +logger = get_logger() +__all__ = ['CLIP4Clip'] + + +class MyObject: + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class CLIP4ClipPreTrainedModel(PreTrainedModel, nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, cross_config, *inputs, **kwargs): + super(CLIP4ClipPreTrainedModel, self).__init__(cross_config) + self.cross_config = cross_config + self.clip = None + self.cross = None + + @classmethod + def from_pretrained(cls, + cross_config, + state_dict=None, + cache_dir=None, + type_vocab_size=2, + *inputs, + **kwargs): + + task_config = None + if 'task_config' in kwargs.keys(): + task_config = kwargs['task_config'] + if not hasattr(task_config, 'local_rank'): + task_config['local_rank'] = 0 + elif task_config['local_rank'] == -1: + task_config['local_rank'] = 0 + + if state_dict is None: + state_dict = {} + # pretrained_clip_name = task_config['pretrained_clip_name'] + clip_state_dict = CLIP.get_config(model_dir=task_config['model_dir']) + for key, val in clip_state_dict.items(): + new_key = 'clip.' + key + if new_key not in state_dict: + state_dict[new_key] = val.clone() + + # cross_config, _ = CrossConfig.get_config( + # cross_model_name, + # cache_dir, + # type_vocab_size, + # state_dict=None, + # task_config=task_config) + cross_config = CrossConfig.from_dict(cross_config) + cross_config.type_vocab_size = type_vocab_size + task_config = MyObject(**kwargs['task_config']) + model = cls(cross_config, clip_state_dict, *inputs, task_config) + + # ===> Initialization trick [HARD CODE] + if model.linear_patch == '3d': + contain_conv2 = False + for key in state_dict.keys(): + if key.find('visual.conv2.weight') > -1: + contain_conv2 = True + break + if contain_conv2 is False and hasattr(model.clip.visual, 'conv2'): + cp_weight = state_dict['clip.visual.conv1.weight'].clone() + kernel_size = model.clip.visual.conv2.weight.size(2) + conv2_size = model.clip.visual.conv2.weight.size() + conv2_size = list(conv2_size) + + left_conv2_size = conv2_size.copy() + right_conv2_size = conv2_size.copy() + left_conv2_size[2] = (kernel_size - 1) // 2 + right_conv2_size[2] = kernel_size - 1 - left_conv2_size[2] + + left_zeros, right_zeros = None, None + if left_conv2_size[2] > 0: + left_zeros = torch.zeros( + *tuple(left_conv2_size), + dtype=cp_weight.dtype, + device=cp_weight.device) + if right_conv2_size[2] > 0: + right_zeros = torch.zeros( + *tuple(right_conv2_size), + dtype=cp_weight.dtype, + device=cp_weight.device) + + cat_list = [] + if left_zeros is not None: + cat_list.append(left_zeros) + cat_list.append(cp_weight.unsqueeze(2)) + if right_zeros is not None: + cat_list.append(right_zeros) + cp_weight = torch.cat(cat_list, dim=2) + + state_dict['clip.visual.conv2.weight'] = cp_weight + + # if model.sim_header == 'tightTransf': + # contain_cross = False + # for key in state_dict.keys(): + # if key.find('cross.transformer') > -1: + # contain_cross = True + # break + # if contain_cross is False: + # for key, val in clip_state_dict.items(): + # if key == 'positional_embedding': + # state_dict[ + # 'cross.embeddings.position_embeddings.weight'] = val.clone( + # ) + # continue + # if key.find('transformer.resblocks') == 0: + # num_layer = int(key.split('.')[2]) + + # # cut from beginning + # if num_layer < task_config.cross_num_hidden_layers: + # state_dict['cross.' + key] = val.clone() + # continue + + if model.sim_header == 'seqLSTM' or model.sim_header == 'seqTransf': + # This step is to detect whether in train mode or test mode + contain_frame_position = False + for key in state_dict.keys(): + if key.find('frame_position_embeddings') > -1: + contain_frame_position = True + break + + # train mode + if contain_frame_position is False: + for key, val in clip_state_dict.items(): + if key == 'positional_embedding': + state_dict[ + 'frame_position_embeddings.weight'] = val.clone() + # state_dict["text_prompt_encoder.pos_embedding"] = val[0:3].clone() + continue + if model.sim_header == 'seqTransf' and key.find( + 'transformer.resblocks') == 0: + num_layer = int(key.split('.')[2]) + # cut from beginning + if num_layer < task_config.cross_num_hidden_layers: + state_dict[key.replace( + 'transformer.', + 'transformerClip.')] = val.clone() + continue + + else: + for key, val in state_dict.items(): + # test mode + if key.find('clip.visual.transformer.resblocks') == 0: + num_layer = int(key.split('.')[4]) + # shift layers 10-11 + if num_layer >= 10 and num_layer < 12: + state_dict[key.replace('attn.net.', + 'attn.')] = val.clone() + # <=== End of initialization trick + + if state_dict is not None: + model = cls.init_preweight( + model, state_dict, task_config=task_config) + make_patch_shift(model, video_frame=task_config.max_frames, n_div=14) + return model + + +def show_log(task_config, info): + if task_config is None or task_config.local_rank == 0: + logger.warning(info) + + +def update_attr(target_name, + target_config, + target_attr_name, + source_config, + source_attr_name, + default_value=None): + if hasattr(source_config, source_attr_name): + if default_value is None or getattr(source_config, + source_attr_name) != default_value: + setattr(target_config, target_attr_name, + getattr(source_config, source_attr_name)) + # show_log( + # source_config, "Set {}.{}: {}.".format( + # target_name, target_attr_name, + # getattr(target_config, target_attr_name))) + return target_config + + +def check_attr(target_name, task_config): + return hasattr(task_config, + target_name) and task_config.__dict__[target_name] + + +class CLIP4Clip(CLIP4ClipPreTrainedModel): + + def __init__(self, cross_config, clip_state_dict, task_config): + super(CLIP4Clip, self).__init__(cross_config) + self.task_config = task_config + self.ignore_video_index = -1 + + assert self.task_config.max_words + self.task_config.max_frames <= cross_config.max_position_embeddings + + self._stage_one = True + self._stage_two = False + + # show_log(task_config, "Stage-One:{}, Stage-Two:{}".format(self._stage_one, self._stage_two)) + + self.loose_type = False + if self._stage_one and check_attr('loose_type', self.task_config): + self.loose_type = True + # show_log(task_config, "Test retrieval by loose type.") + + # CLIP Encoders: From OpenAI: CLIP [https://github.com/openai/CLIP] ===> + vit = 'visual.proj' in clip_state_dict + assert vit + if vit: + vision_width = clip_state_dict['visual.conv1.weight'].shape[0] + vision_layers = len([ + k for k in clip_state_dict.keys() if k.startswith('visual.') + and k.endswith('.attn.in_proj_weight') + ]) + vision_patch_size = clip_state_dict['visual.conv1.weight'].shape[ + -1] + grid_size = round( + (clip_state_dict['visual.positional_embedding'].shape[0] + - 1)**0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [ + len( + set( + k.split('.')[2] for k in clip_state_dict + if k.startswith(f'visual.layer{b}'))) + for b in [1, 2, 3, 4] + ] + vision_layers = tuple(counts) + vision_width = clip_state_dict[ + 'visual.layer1.0.conv1.weight'].shape[0] + output_width = round( + (clip_state_dict['visual.attnpool.positional_embedding']. + shape[0] - 1)**0.5) + vision_patch_size = None + assert output_width**2 + 1 == clip_state_dict[ + 'visual.attnpool.positional_embedding'].shape[0] + image_resolution = output_width * 32 + + embed_dim = clip_state_dict['text_projection'].shape[1] + context_length = clip_state_dict['positional_embedding'].shape[0] + vocab_size = clip_state_dict['token_embedding.weight'].shape[0] + transformer_width = clip_state_dict['ln_final.weight'].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split('.')[2] for k in clip_state_dict + if k.startswith('transformer.resblocks'))) + + # show_log(task_config, "\t embed_dim: {}".format(embed_dim)) + # show_log(task_config, "\t image_resolution: {}".format(image_resolution)) + # show_log(task_config, "\t vision_layers: {}".format(vision_layers)) + # show_log(task_config, "\t vision_width: {}".format(vision_width)) + # show_log(task_config, "\t vision_patch_size: {}".format(vision_patch_size)) + # show_log(task_config, "\t context_length: {}".format(context_length)) + # show_log(task_config, "\t vocab_size: {}".format(vocab_size)) + # show_log(task_config, "\t transformer_width: {}".format(transformer_width)) + # show_log(task_config, "\t transformer_heads: {}".format(transformer_heads)) + # show_log(task_config, "\t transformer_layers: {}".format(transformer_layers)) + + self.linear_patch = '2d' + if hasattr(task_config, 'linear_patch'): + self.linear_patch = task_config.linear_patch + # show_log(task_config, "\t\t linear_patch: {}".format(self.linear_patch)) + + # use .float() to avoid overflow/underflow from fp16 weight. https://github.com/openai/CLIP/issues/40 + cut_top_layer = 0 + + self.clip = CLIP( + embed_dim, + image_resolution, + vision_layers - cut_top_layer, + vision_width, + vision_patch_size, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers - cut_top_layer, + linear_patch=self.linear_patch).float() + + for key in ['input_resolution', 'context_length', 'vocab_size']: + if key in clip_state_dict: + del clip_state_dict[key] + + # convert_weights(self.clip) + # <=== End of CLIP Encoders + + self.sim_header = 'seqTransf' + if hasattr(task_config, 'sim_header'): + self.sim_header = task_config.sim_header + # show_log(task_config, "\t sim_header: {}".format(self.sim_header)) + if self.sim_header == 'tightTransf': + assert self.loose_type is False + + cross_config.max_position_embeddings = context_length + if self.loose_type is False: + # Cross Encoder ===> + cross_config = update_attr('cross_config', cross_config, + 'num_hidden_layers', self.task_config, + 'cross_num_hidden_layers') + self.cross = CrossModel(cross_config) + # <=== End of Cross Encoder + self.similarity_dense = nn.Linear(cross_config.hidden_size, 1) + + if self.sim_header == 'seqLSTM' or self.sim_header == 'seqTransf': + self.frame_position_embeddings = nn.Embedding( + cross_config.max_position_embeddings, cross_config.hidden_size) + # self.frame_position_embeddings = nn.Embedding(600, cross_config.hidden_size) + if self.sim_header == 'seqTransf': + self.transformerClip = TransformerClip( + width=transformer_width, + layers=self.task_config.cross_num_hidden_layers, + heads=transformer_heads, + ) + if self.sim_header == 'seqLSTM': + self.lstm_visual = nn.LSTM( + input_size=cross_config.hidden_size, + hidden_size=cross_config.hidden_size, + batch_first=True, + bidirectional=False, + num_layers=1) + + self.loss_fct = CrossEn() + self.apply(self.init_weights) + + self.set_dim = 512 + self.patch_num = self.task_config.max_patch + if hasattr(self.task_config, 'max_word_pro'): + self.word_pro_num = self.task_config.max_word_pro + else: + self.word_pro_num = self.task_config.max_phrase + + self.frame_num = self.task_config.max_frames + if hasattr(self.task_config, 'max_vfea'): + self.event_num = self.task_config.max_vfea + else: + self.event_num = self.task_config.max_event + + self.patch_prototype_weight = nn.Sequential( + nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True), + nn.Linear(self.set_dim, self.patch_num - 1), nn.ReLU(inplace=True)) + + self.word_prototype_weight = nn.Sequential( + nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True), + nn.Linear(self.set_dim, self.word_pro_num), nn.ReLU(inplace=True)) + + self.frame_decoder = Frame_decoder( + num_attris=self.frame_num, + layers=2, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False) + self.event_decoder = Event_decoder( + num_attris=self.event_num, + layers=2, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False) + # ------------------------------------------------------------------------------------------------------- + + def forward(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask=None): + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + # T x 3 x H x W + video = torch.as_tensor(video).float() + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + video_frame = bs * ts + phr_feat, sen_feat, obj_feat, eve_feat = self.get_sequence_visual_output( + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=True, + video_frame=video_frame) + + if self.training: + sim_matrix1, sim_matrix2, sim_matrix3, sim_matrix4 = self.get_max_similarity_logits( + phr_feat, + sen_feat, + obj_feat, + eve_feat, + attention_mask, + video_mask, + shaped=True) + sim_loss = (self.loss_fct(sim_matrix1) + self.loss_fct(sim_matrix2) + + self.loss_fct(sim_matrix3) + + self.loss_fct(sim_matrix4)) / 4.0 + + loss = sim_loss + + return loss + else: + return None + + def get_max_similarity_logits(self, + word_feat, + text_feat, + patch_feat, + video_feat, + text_mask, + video_mask, + shaped=False): + if shaped is False: + text_mask = text_mask.view(-1, text_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + if self.training and torch.cuda.is_available(): # batch merge here + text_feat = allgather(text_feat, self.task_config) + video_feat = allgather(video_feat, self.task_config) + word_feat = allgather(word_feat, self.task_config) + patch_feat = allgather(patch_feat, self.task_config) + + video_mask = allgather(video_mask, self.task_config) + torch.distributed.barrier() # force sync + + # ESPM + text_feat = F.normalize(text_feat, p=2, dim=1) + video_feat = F.normalize(video_feat, p=2, dim=2) + retrieve_logits = torch.einsum('ad,bkd->abk', [text_feat, video_feat]) + retrieve_logits = retrieve_logits.max(2)[0] + + # OPPM + word_feat = F.normalize(word_feat, p=2, dim=2) + patch_feat = F.normalize(patch_feat, p=2, dim=3) + retrieve_logits_2 = torch.einsum('aid, bfjd->abfij', + [word_feat, patch_feat]) + + retrieve_logits_2 = retrieve_logits_2.max(3)[0] + retrieve_logits_2 = retrieve_logits_2.max(2)[0] + retrieve_logits_2 = retrieve_logits_2.sum(2) / self.patch_num + + if self.training: + logit_scale = self.clip.logit_scale.exp() + retrieve_logits = logit_scale * retrieve_logits + retrieve_logits_2 = logit_scale * retrieve_logits_2 + return retrieve_logits, retrieve_logits.t( + ), retrieve_logits_2, retrieve_logits_2.t() + + def get_sequence_output(self, + input_ids, + token_type_ids, + attention_mask, + shaped=False): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + bs_pair = input_ids.size(0) + sequence_hidden = self.clip.encode_text( + input_ids, return_hidden=True)[1].float() + text_feat = sequence_hidden.view(bs_pair, -1, sequence_hidden.size(-1)) + + word_weights = self.word_prototype_weight(text_feat) + text_word_proto = torch.einsum('bmd,bmn->bnd', text_feat, word_weights) + + cls_text_feat = text_feat.contiguous() + cls_text_feat = cls_text_feat[torch.arange(cls_text_feat.shape[0]), + torch.sum(attention_mask, dim=-1) - 1, :] + + return text_word_proto, cls_text_feat + + def get_visual_output(self, + video, + video_mask, + shaped=False, + video_frame=-1): + if shaped is False: + video_mask = video_mask.view(-1, video_mask.shape[-1]) + video = torch.as_tensor(video).float() + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + # video_frame = bs * ts + + bs_pair = video_mask.size(0) + + cls_video_feat, video_patch_feat = self.clip.encode_image_tokens( + video, return_hidden=True) + cls_video_feat = cls_video_feat.float() + video_patch_feat = video_patch_feat.float() + # frame_num = video_patch_feat.shape[0] + patch_dim = video_patch_feat.shape[2] + + patch_weights = self.patch_prototype_weight(video_patch_feat) + # cls_video_feat + video_patch_proto = torch.einsum('bmd,bmn->bnd', video_patch_feat, + patch_weights) + video_patch_proto = torch.cat( + (cls_video_feat.unsqueeze(1), video_patch_proto), 1) + video_patch_proto = video_patch_proto.reshape( + bs_pair, self.task_config.max_frames, self.patch_num, patch_dim) + + video_frame_proto = video_patch_proto.reshape( + bs_pair, self.patch_num * self.task_config.max_frames, patch_dim) + video_frame_proto = self.frame_decoder(video_frame_proto) + + video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape( + bs_pair, self.task_config.max_frames, patch_dim) + video_frame_proto = self.event_decoder(video_frame_proto) + video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape( + bs_pair, self.task_config.max_frames, patch_dim).mean(1).unsqueeze( + 1).repeat(1, video_frame_proto.shape[1], 1) + return video_patch_proto, video_frame_proto + + def get_sequence_visual_output(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=False, + video_frame=-1): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + video = torch.as_tensor(video).float() + + # import pdb;pdb.set_trace() + # b, pair, + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + video_frame = bs * ts + + word_feat, text_feat = self.get_sequence_output( + input_ids, token_type_ids, attention_mask, shaped=True) + + patch_feat, frame_feat = self.get_visual_output( + video, video_mask, shaped=True, video_frame=video_frame) + + return word_feat, text_feat, patch_feat, frame_feat + + def _get_cross_output(self, sequence_output, visual_output, attention_mask, + video_mask): + + concat_features = torch.cat((sequence_output, visual_output), + dim=1) # concatnate tokens and frames + concat_mask = torch.cat((attention_mask, video_mask), dim=1) + text_type_ = torch.zeros_like(attention_mask) + video_type_ = torch.ones_like(video_mask) + concat_type = torch.cat((text_type_, video_type_), dim=1) + + cross_layers, pooled_output = self.cross( + concat_features, + concat_type, + concat_mask, + output_all_encoded_layers=True) + cross_output = cross_layers[-1] + + return cross_output, pooled_output, concat_mask + + def _mean_pooling_for_similarity_sequence(self, sequence_output, + attention_mask): + attention_mask_un = attention_mask.to(dtype=torch.float).unsqueeze(-1) + attention_mask_un[:, 0, :] = 0. + sequence_output = sequence_output * attention_mask_un + text_out = torch.sum( + sequence_output, dim=1) / torch.sum( + attention_mask_un, dim=1, dtype=torch.float) + return text_out + + def _mean_pooling_for_similarity_visual( + self, + visual_output, + video_mask, + ): + video_mask_un = video_mask.to(dtype=torch.float).unsqueeze(-1) + visual_output = visual_output * video_mask_un + video_mask_un_sum = torch.sum(video_mask_un, dim=1, dtype=torch.float) + video_mask_un_sum[video_mask_un_sum == 0.] = 1. + video_out = torch.sum(visual_output, dim=1) / video_mask_un_sum + return video_out + + def _mean_pooling_for_similarity( + self, + sequence_output, + visual_output, + attention_mask, + video_mask, + ): + text_out = self._mean_pooling_for_similarity_sequence( + sequence_output, attention_mask) + video_out = self._mean_pooling_for_similarity_visual( + visual_output, video_mask) + + return text_out, video_out + + def get_global_similarity(self, sequence_output, visual_output, + attention_mask, video_mask): + visual_output = visual_output / visual_output.norm( + dim=-1, keepdim=True) + visual_output = self._mean_pooling_for_similarity_visual( + visual_output, video_mask) + visual_output = visual_output / visual_output.norm( + dim=-1, keepdim=True) + + sequence_output = sequence_output.squeeze(1) + sequence_output = sequence_output / sequence_output.norm( + dim=-1, keepdim=True) + + logit_scale = self.clip.logit_scale.exp() + # retrieve_logits = logit_scale * torch.matmul(sequence_output, visual_output.t()) + sim_matrix_global = logit_scale * torch.matmul(sequence_output, + visual_output.t()) + return sim_matrix_global + + def _cross_similarity(self, sequence_output, visual_output, attention_mask, + video_mask): + sequence_output, visual_output = sequence_output.contiguous( + ), visual_output.contiguous() + + b_text, s_text, h_text = sequence_output.size() + b_visual, s_visual, h_visual = visual_output.size() + + retrieve_logits_list = [] + + step_size = b_text # set smaller to reduce memory cost + split_size = [step_size] * (b_text // step_size) + release_size = b_text - sum(split_size) + if release_size > 0: + split_size += [release_size] + + # due to clip text branch retrun the last hidden + attention_mask = torch.ones(sequence_output.size(0), 1)\ + .to(device=attention_mask.device, dtype=attention_mask.dtype) + + sequence_output_splits = torch.split( + sequence_output, split_size, dim=0) + attention_mask_splits = torch.split(attention_mask, split_size, dim=0) + for i in range(len(split_size)): + sequence_output_row = sequence_output_splits[i] + attention_mask_row = attention_mask_splits[i] + sequence_output_l = sequence_output_row.unsqueeze(1).repeat( + 1, b_visual, 1, 1) + sequence_output_l = sequence_output_l.view(-1, s_text, h_text) + attention_mask_l = attention_mask_row.unsqueeze(1).repeat( + 1, b_visual, 1) + attention_mask_l = attention_mask_l.view(-1, s_text) + + step_truth = sequence_output_row.size(0) + visual_output_r = visual_output.unsqueeze(0).repeat( + step_truth, 1, 1, 1) + visual_output_r = visual_output_r.view(-1, s_visual, h_visual) + video_mask_r = video_mask.unsqueeze(0).repeat(step_truth, 1, 1) + video_mask_r = video_mask_r.view(-1, s_visual) + + cross_output, pooled_output, concat_mask = \ + self._get_cross_output(sequence_output_l, visual_output_r, attention_mask_l, video_mask_r) + retrieve_logits_row = self.similarity_dense(pooled_output).squeeze( + -1).view(step_truth, b_visual) + + retrieve_logits_list.append(retrieve_logits_row) + + retrieve_logits = torch.cat(retrieve_logits_list, dim=0) + return retrieve_logits diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/module_clip.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/module_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..c5aaa1e52f20b65cd06613f768d85c9e93d41006 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/module_clip.py @@ -0,0 +1,538 @@ +# The implementation is adopated from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import hashlib +import os +import urllib +import warnings +from collections import OrderedDict +from typing import Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import nn +from tqdm import tqdm + +_MODELS = {} +_PT_NAME = {'ViT-B/16': 'ViT-B-16.pt'} + + +def available_models(): + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super(Bottleneck, self).__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super(AttentionPool2d, self).__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super(ModifiedResNet, self).__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int, attn_mask=None): + super(ResidualAttentionBlock, self).__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + attn_mask_ = self.attn_mask + if self.attn_mask is not None and hasattr(self.attn_mask, '__call__'): + attn_mask_ = self.attn_mask(x.size(0)) # LND + + attn_mask_ = attn_mask_.to( + dtype=x.dtype, device=x.device) if attn_mask_ is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, x): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask=None, + use_gc=0): + super(Transformer, self).__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + self.use_gc = use_gc + + def forward(self, x: torch.Tensor): + if self.use_gc > 0: + for blk in self.resblocks: + x = checkpoint.checkpoint(blk, x) + return x + else: + return self.resblocks(x) + + +class VisualTransformer(nn.Module): + + def __init__(self, + input_resolution: int, + patch_size: int, + width: int, + layers: int, + heads: int, + output_dim: int, + linear_patch: str = '2d', + use_gc: int = 0): + super(VisualTransformer, self).__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads, use_gc=use_gc) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + # For 3D + assert linear_patch in ['2d', '3d'] + self.linear_patch = linear_patch + if self.linear_patch == '3d': + self.conv2 = nn.Conv3d( + in_channels=3, + out_channels=width, + kernel_size=(3, patch_size, patch_size), + stride=(1, patch_size, patch_size), + padding=(1, 0, 0), + bias=False) + + def forward(self, x: torch.Tensor, video_frame=-1): + + if self.linear_patch == '3d': + assert video_frame != -1 + x_3d = x.reshape(-1, video_frame, x.shape[-3], x.shape[-2], + x.shape[-1]) + x_3d = x_3d.permute(0, 2, 1, 3, 4) + x_3d = self.conv2(x_3d) # shape = [*, width, frame, grid, grid] + x_3d = x_3d.permute(0, 2, 1, 3, + 4) # shape = [*, frame, width, grid, grid] + x = x_3d.reshape( + -1, x_3d.shape[-3], x_3d.shape[-2], + x_3d.shape[-1]).contiguous() # shape = [*, width, grid, grid] + else: + x = self.conv1(x) # shape = [*, width, grid, grid] + + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + _x = self.class_embedding.to(x.dtype) + torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([_x, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + # vision linear of patch + linear_patch: str = '2d', + use_gc: int = 0): + super(CLIP, self).__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim, + linear_patch=linear_patch, + use_gc=use_gc) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([])) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self, context_length): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.zeros(context_length, context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @staticmethod + def get_config(model_dir): + model_path = '{}/ViT-B-16.pt'.format(model_dir) + try: + # loading JIT archive + model = torch.jit.load(model_path, map_location='cpu').eval() + state_dict = model.state_dict() + except RuntimeError: + state_dict = torch.load(model_path, map_location='cpu') + return state_dict + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image_tokens(self, image, return_hidden=False): + hidden = self.visual(image.type(self.dtype)) + hidden = self.visual.ln_post(hidden) @ self.visual.proj + + x = hidden[:, 0, :] + + if return_hidden: + return x, hidden + + return x + + def encode_text(self, text, return_hidden=False, prompt=None): + x = self.token_embedding(text).type( + self.dtype) # [batch_size, n_ctx, d_model] + if prompt: + x = prompt(x) + + pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype) + x = x + pos_emd + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + hidden = self.ln_final(x).type(self.dtype) @ self.text_projection + + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)] + + if return_hidden: + return x, hidden + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logit_scale * text_features @ image_features.t() + + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(lay): + # l = lay + if isinstance(lay, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear)): + lay.weight.data = lay.weight.data.half() + if lay.bias is not None: + lay.bias.data = lay.bias.data.half() + + if isinstance(lay, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(lay, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ['text_projection', 'proj']: + if hasattr(lay, name): + attr = getattr(lay, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/module_cross.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/module_cross.py new file mode 100644 index 0000000000000000000000000000000000000000..fae8e904b288c114a09cfd06463673b7600eff19 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/module_cross.py @@ -0,0 +1,249 @@ +from __future__ import absolute_import, division, print_function +import copy +import logging +import math +import os +import shutil +import tarfile +import tempfile +from collections import OrderedDict + +import json +import torch +import torch.nn.functional as F +from torch import nn + +from .until_config import PreCrossConfig +from .until_module import ACT2FN, LayerNorm, PreTrainedModel + + +# PRETRAINED_MODEL_ARCHIVE_MAP = {} +# CONFIG_NAME = 'cross_config.json' +# WEIGHTS_NAME = 'cross_pytorch_model.bin' +class CrossConfig(PreCrossConfig): + """Configuration class to store the configuration of a `CrossModel`. + """ + + # pretrained_model_archive_map = PRETRAINED_MODEL_ARCHIVE_MAP + # config_name = CONFIG_NAME + # weights_name = WEIGHTS_NAME + def __init__(self, + vocab_size_or_config_json_file, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02): + """Constructs CrossConfig. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CrossModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `CrossModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + if isinstance(vocab_size_or_config_json_file, str): + with open( + vocab_size_or_config_json_file, 'r', + encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + else: + raise ValueError( + 'First argument must be either a vocabulary size (int)' + 'or the path to a pretrained model config file (str)') + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.n_head = n_head + + def attention(self, x: torch.Tensor, attn_mask: torch.Tensor): + attn_mask_ = attn_mask.repeat_interleave(self.n_head, dim=0) + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, para_tuple: tuple): + # x: torch.Tensor, attn_mask: torch.Tensor + # print(para_tuple) + x, attn_mask = para_tuple + x = x + self.attention(self.ln_1(x), attn_mask) + x = x + self.mlp(self.ln_2(x)) + return (x, attn_mask) + + +class Transformer(nn.Module): + + def __init__(self, width: int, layers: int, heads: int): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential( + *[ResidualAttentionBlock(width, heads) for _ in range(layers)]) + + def forward(self, x: torch.Tensor, attn_mask: torch.Tensor): + return self.resblocks((x, attn_mask))[0] + + +class CrossEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(CrossEmbeddings, self).__init__() + + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + # self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + # self.LayerNorm = LayerNorm(config.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, concat_embeddings, concat_type=None): + + _, seq_length = concat_embeddings.size(0), concat_embeddings.size(1) + # if concat_type is None: + # concat_type = torch.zeros(batch_size, concat_type).to(concat_embeddings.device) + + position_ids = torch.arange( + seq_length, dtype=torch.long, device=concat_embeddings.device) + position_ids = position_ids.unsqueeze(0).expand( + concat_embeddings.size(0), -1) + + # token_type_embeddings = self.token_type_embeddings(concat_type) + position_embeddings = self.position_embeddings(position_ids) + + embeddings = concat_embeddings + position_embeddings # + token_type_embeddings + # embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class CrossPooler(nn.Module): + + def __init__(self, config): + super(CrossPooler, self).__init__() + self.ln_pool = LayerNorm(config.hidden_size) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = QuickGELU() + + def forward(self, hidden_states, hidden_mask): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + hidden_states = self.ln_pool(hidden_states) + pooled_output = hidden_states[:, 0] + pooled_output = self.dense(pooled_output) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class CrossModel(PreTrainedModel): + + def initialize_parameters(self): + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + def __init__(self, config): + super(CrossModel, self).__init__(config) + + self.embeddings = CrossEmbeddings(config) + + transformer_width = config.hidden_size + transformer_layers = config.num_hidden_layers + transformer_heads = config.num_attention_heads + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + ) + self.pooler = CrossPooler(config) + self.apply(self.init_weights) + + def build_attention_mask(self, attention_mask): + extended_attention_mask = attention_mask.unsqueeze(1) + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -1000000.0 + extended_attention_mask = extended_attention_mask.expand( + -1, attention_mask.size(1), -1) + return extended_attention_mask + + def forward(self, + concat_input, + concat_type=None, + attention_mask=None, + output_all_encoded_layers=True): + + if attention_mask is None: + attention_mask = torch.ones( + concat_input.size(0), concat_input.size(1)) + if concat_type is None: + concat_type = torch.zeros_like(attention_mask) + + extended_attention_mask = self.build_attention_mask(attention_mask) + + embedding_output = self.embeddings(concat_input, concat_type) + embedding_output = embedding_output.permute(1, 0, 2) # NLD -> LND + embedding_output = self.transformer(embedding_output, + extended_attention_mask) + embedding_output = embedding_output.permute(1, 0, 2) # LND -> NLD + + pooled_output = self.pooler( + embedding_output, hidden_mask=attention_mask) + + return embedding_output, pooled_output diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/prost_model.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/prost_model.py new file mode 100644 index 0000000000000000000000000000000000000000..022903cb7f19f71851c36037137d044decbab44c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/prost_model.py @@ -0,0 +1,267 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import os +import random +import uuid +from os.path import exists +from tempfile import TemporaryDirectory +from typing import Any, Dict +from urllib.parse import urlparse + +import json +import numpy as np +import torch +from decord import VideoReader, cpu +from PIL import Image + +from modelscope.hub.file_download import http_get_file +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.prost.models.modeling import CLIP4Clip +from modelscope.models.multi_modal.prost.models.tokenization_clip import \ + SimpleTokenizer as ClipTokenizer +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from ..dataloaders.rawvideo_util import RawVideoExtractor + +logger = get_logger() + + +@MODELS.register_module(Tasks.text_video_retrieval, module_name=Models.prost) +class ProSTForTVRetrieval(TorchModel): + + def __init__(self, model_dir, **kwargs): + super().__init__(model_dir=model_dir, **kwargs) + # model config parameters + with open( + f'{model_dir}/{ModelFile.CONFIGURATION}', 'r', + encoding='utf-8') as json_file: + all_model_config = json.load(json_file) + model_config = all_model_config['paras'] + + cross_model_config = all_model_config['crossbase'] + # print(all_model_config) + # print(cross_model_config) + model_config['model_dir'] = model_dir + self.SPECIAL_TOKEN = { + 'CLS_TOKEN': '<|startoftext|>', + 'SEP_TOKEN': '<|endoftext|>', + 'MASK_TOKEN': '[MASK]', + 'UNK_TOKEN': '[UNK]', + 'PAD_TOKEN': '[PAD]' + } + self.max_words = model_config['max_words'] + self.max_frames = model_config['max_frames'] + self.feature_framerate = model_config['feature_framerate'] + self.image_resolution = 224 + if torch.cuda.is_available(): + self.device = model_config['device'] + else: + self.device = 'cpu' + self.init_model = f'{model_dir}/{ModelFile.TORCH_MODEL_BIN_FILE}' + + self.tokenizer = ClipTokenizer(model_dir) + self.rawVideoExtractor = RawVideoExtractor( + frame_rate=self.feature_framerate, size=self.image_resolution) + self.local_transform = self.rawVideoExtractor.transform + self.model = CLIP4Clip.from_pretrained( + cross_config=cross_model_config, task_config=model_config) + if hasattr(self.model, 'module'): + self.model = self.model.module.to(self.device) + else: + self.model = self.model.to(self.device) + if self.init_model: + assert exists(self.init_model) + model_state_dict = torch.load(self.init_model, map_location='cpu') + self.model.load_state_dict(model_state_dict, strict=False) + self.model.to(self.device) + + def _get_text(self, caption, tokenizer, enable_zh=False): + + if type(caption) is str: + _caption_text, s, e = caption, None, None + elif type(caption) is tuple: + if len(caption) == 3: + _caption_text, s, e = caption + elif len(caption) == 4: + _caption_text, s, e, pos = caption + else: + NotImplementedError + + if isinstance(_caption_text, list): + caption_text = random.choice(_caption_text) + else: + caption_text = _caption_text + if enable_zh: + _token = tokenizer.encode(caption_text) + input_ids = _token.ids + input_mask = _token.attention_mask + segment_ids = _token.type_ids + else: + words = tokenizer.tokenize(caption_text) + + words = [self.SPECIAL_TOKEN['CLS_TOKEN']] + words + total_length_with_CLS = self.max_words - 1 + if len(words) > total_length_with_CLS: + words = words[:total_length_with_CLS] + words = words + [self.SPECIAL_TOKEN['SEP_TOKEN']] + + input_ids = tokenizer.convert_tokens_to_ids(words) + input_mask = [1] * len(input_ids) + segment_ids = [0] * len(input_ids) + + while len(input_ids) < self.max_words: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + assert len(input_ids) == self.max_words + assert len(input_mask) == self.max_words + assert len(segment_ids) == self.max_words + + pairs_text = np.array(input_ids) + pairs_mask = np.array(input_mask) + pairs_segment = np.array(segment_ids) + + return pairs_text, pairs_mask, pairs_segment, s, e + + def _get_rawvideo_dec(self, + video_path, + rawVideoExtractor, + local_transform, + s=None, + e=None): + video_mask = np.zeros(self.max_frames, dtype=int) + max_video_length = 0 + + # T x 3 x H x W + video = np.zeros((self.max_frames, 3, rawVideoExtractor.size, + rawVideoExtractor.size), + dtype=float) + + if s is None: + start_time, end_time = None, None + else: + start_time = int(s) + end_time = int(e) + start_time = start_time if start_time >= 0. else 0. + end_time = end_time if end_time >= 0. else 0. + if start_time > end_time: + start_time, end_time = end_time, start_time + elif start_time == end_time: + end_time = end_time + 1 + + url_parsed = urlparse(video_path) + if url_parsed.scheme in ('file', '') and exists( + url_parsed.path): # Possibly a local file + vreader = VideoReader(video_path, ctx=cpu(0)) + else: + try: + with TemporaryDirectory() as temporary_cache_dir: + random_str = uuid.uuid4().hex + http_get_file( + url=video_path, + local_dir=temporary_cache_dir, + file_name=random_str, + cookies=None) + temp_file_path = os.path.join(temporary_cache_dir, + random_str) + vreader = VideoReader(temp_file_path, ctx=cpu(0)) + except Exception as ex: + logger.error('non video input, output is {}!!!'.format(ex)) + return video, video_mask + + fps = vreader.get_avg_fps() + f_start = 0 if start_time is None else int(start_time * fps) + f_end = int( + min(1000000000 if end_time is None else end_time * fps, + len(vreader) - 1)) + num_frames = f_end - f_start + 1 + if num_frames > 0: + # L x T x 3 x H x W + sample_fps = int(self.feature_framerate) + t_stride = int(round(float(fps) / sample_fps)) + + all_pos = list(range(f_start, f_end + 1, t_stride)) + if len(all_pos) > self.max_frames: + sample_pos = [ + all_pos[_] for _ in np.linspace( + 0, len(all_pos) - 1, num=self.max_frames, dtype=int) + ] + else: + sample_pos = all_pos + patch_images = [ + Image.fromarray(f) + for f in vreader.get_batch(sample_pos).asnumpy() + ] + patch_images = torch.stack( + [local_transform(img) for img in patch_images]) + slice_len = patch_images.shape[0] + max_video_length = max_video_length if max_video_length > slice_len else slice_len + if slice_len < 1: + pass + else: + video[:slice_len, ...] = patch_images + else: + logger.error('video path: {} error. video id: {}'.format( + video_path, video_id)) + + video_mask[:max_video_length] = [1] * max_video_length + + return video, video_mask + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + + from modelscope.outputs import OutputKeys + output = {} + + if 'video' in input and input['video'] is not None: + video_path = input['video'] + video, video_mask = self._get_rawvideo_dec(video_path, + self.rawVideoExtractor, + self.local_transform) + video = torch.unsqueeze( + torch.from_numpy(video), dim=0).to(self.device) + video_mask = torch.unsqueeze( + torch.from_numpy(video_mask), dim=0).to(self.device) + + if 'text' in input and input['text'] is not None: + caption = input['text'] + pairs_text, pairs_mask, pairs_segment, s, e = self._get_text( + caption, self.tokenizer, enable_zh=False) + input_ids = torch.unsqueeze( + torch.from_numpy(pairs_text), dim=0).to(self.device) + input_mask = torch.unsqueeze( + torch.from_numpy(pairs_mask), dim=0).to(self.device) + segment_ids = torch.unsqueeze( + torch.from_numpy(pairs_segment), dim=0).to(self.device) + + phr_feat, sen_feat, obj_feat, eve_feat = self.model.get_sequence_visual_output( + input_ids, segment_ids, input_mask, video, video_mask) + + sim_espm, _, sim_oppm, _ = self.model.get_max_similarity_logits( + phr_feat, + sen_feat, + obj_feat, + eve_feat, + input_mask, + video_mask, + shaped=True) + # logger.info('sim: {}'.format(sim_espm)) + # logger.info('sim: {}'.format(sim_oppm)) + sim_tv = sim_espm + 1.5 * sim_oppm + + # logger.info('phrase prototype: {}'.format(phr_feat.shape)) + # logger.info('sentence prototype: {}'.format(sen_feat.shape)) + # logger.info('object prototype: {}'.format(obj_feat.shape)) + # logger.info('event prototype: {}'.format(eve_feat.shape)) + output[OutputKeys.TEXTVIDEO_SIM] = sim_tv.cpu().detach().numpy() + output[OutputKeys.PHRASE_PROTOTYPE] = phr_feat.cpu().detach().numpy() + output[OutputKeys.SENTENCE_PROTOTYPE] = sen_feat.cpu().detach().numpy() + output[OutputKeys.OBJECT_PROTOTYPE] = obj_feat.cpu().detach().numpy() + output[OutputKeys.EVENT_PROTOTYPE] = eve_feat.cpu().detach().numpy() + return output + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/tokenization_clip.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/tokenization_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..97ee7156a75d53d4787b01cc018d23eefc244161 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/tokenization_clip.py @@ -0,0 +1,161 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, model_dir): + bpe_path = '{}/bpe_simple_vocab_16e6.txt.gz'.format(model_dir) + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + self.vocab = self.encoder + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + def tokenize(self, text): + tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + tokens.extend( + bpe_token for bpe_token in self.bpe(token).split(' ')) + return tokens + + def convert_tokens_to_ids(self, tokens): + return [self.encoder[bpe_token] for bpe_token in tokens] diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/until_config.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/until_config.py new file mode 100644 index 0000000000000000000000000000000000000000..dc9753d3e14e592ce918605234714f7817ee4fa8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/until_config.py @@ -0,0 +1,59 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +from __future__ import absolute_import, division, print_function +import copy +import logging +import os +import shutil +import tarfile +import tempfile + +import json +import torch + +# from modelscope.utils.logger import get_logger +# logger = get_logger(__name__) + + +class PreCrossConfig(object): + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = cls(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' diff --git a/runtime/Lib/site_packages/models/multi_modal/prost/models/until_module.py b/runtime/Lib/site_packages/models/multi_modal/prost/models/until_module.py new file mode 100644 index 0000000000000000000000000000000000000000..20afc2c3baf62d5373e8293782c09a7d1d9dc445 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/prost/models/until_module.py @@ -0,0 +1,576 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +import copy +import logging +import math + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.models.multi_modal.prost.models.until_config import \ + PreCrossConfig + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class LayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class CrossEn(nn.Module): + + def __init__(self, config=None): + super(CrossEn, self).__init__() + + def forward(self, sim_matrix): + logpt = F.log_softmax(sim_matrix, dim=-1) + logpt = torch.diag(logpt) + nce_loss = -logpt + sim_loss = nce_loss.mean() + return sim_loss + + +class AllGather(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + return ( + grad_output[ctx.batch_size * ctx.rank:ctx.batch_size + * (ctx.rank + 1)], + None, + ) + + +class AllGather2(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + # https://github.com/PyTorchLightning/lightning-bolts/blob/8d3fbf7782e3d3937ab8a1775a7092d7567f2933/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20 + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + grad_input = grad_output.clone() + torch.distributed.all_reduce( + grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False) + return (grad_input[ctx.rank * ctx.batch_size:(ctx.rank + 1) + * ctx.batch_size], None) + + +class PreTrainedModel(nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedModel, self).__init__() + if not isinstance(config, PreCrossConfig): + raise ValueError( + 'Parameter config in `{}(config)` should be an instance of class `PreCrossConfig`. ' + 'To create a model from a Google pretrained model use ' + '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format( + self.__class__.__name__, self.__class__.__name__)) + self.config = config + + def init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, LayerNorm): + if 'beta' in dir(module) and 'gamma' in dir(module): + module.beta.data.zero_() + module.gamma.data.fill_(1.0) + else: + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def resize_token_embeddings(self, new_num_tokens=None): + raise NotImplementedError + + @classmethod + def init_preweight(cls, model, state_dict, prefix=None, task_config=None): + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + if prefix is not None: + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + old_keys.append(key) + new_keys.append(prefix + key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, + True, missing_keys, unexpected_keys, + error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix='') + + # if prefix is None and (task_config is None or task_config.local_rank == 0): + # logger.info("-" * 20) + # if len(missing_keys) > 0: + # logger.info("Weights of {} not initialized from pretrained model: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(missing_keys))) + # if len(unexpected_keys) > 0: + # logger.info("Weights from pretrained model not used in {}: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(unexpected_keys))) + # if len(error_msgs) > 0: + # logger.error("Weights from pretrained model cause errors in {}: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(error_msgs))) + + return model + + @property + def dtype(self): + """ + :obj:`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + try: + return next(self.parameters()).dtype + except StopIteration: + # For nn.DataParallel compatibility in PyTorch 1.5 + def find_tensor_attributes(module: nn.Module): + tuples = [(k, v) for k, v in module.__dict__.items() + if torch.is_tensor(v)] + return tuples + + gen = self._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + @classmethod + def from_pretrained(cls, config, state_dict=None, *inputs, **kwargs): + """ + Instantiate a PreTrainedModel from a pre-trained model file or a pytorch state dict. + Download and cache the pre-trained model file if needed. + """ + # Instantiate model. + model = cls(config, *inputs, **kwargs) + if state_dict is None: + return model + model = cls.init_preweight(model, state_dict) + + return model + + +class PatchShiftModule(nn.Module): + + def __init__(self, net, video_frame, n_div): + super().__init__() + self.net = net + self.video_frame = video_frame + self.n_div = n_div + + def forward(self, + query, + key, + value, + key_padding_mask=None, + need_weights=True, + attn_mask=None): + # here q == k == v, psm means patch shift output + x = query # shape here is LND, not NLD (50, 384, 768) + x = x.permute(1, 0, 2) # LND -> NLD + patch_len = x.shape[-2] + fold = patch_len // self.n_div + x = x.reshape(-1, self.video_frame, x.shape[-2], + x.shape[-1]) # shape = [bs, frame, grid ** 2, width] + psm = torch.zeros_like(x) # shape = [bs, frame, grid ** 2, width] + psm[:, :, :, :] = x[:, :, :, :] + lshift_indices = torch.arange(start=1, end=patch_len, step=fold) + psm[:, 1:, lshift_indices, :] = x[:, :-1, + lshift_indices, :] # f_t = f_t-1 + rshift_indices = torch.arange(start=1 + 3, end=patch_len, step=fold) + psm[:, :-1, rshift_indices, :] = x[:, 1:, + rshift_indices, :] # f_t = f_t+1 + x = psm.reshape(-1, patch_len, x.shape[-1]) + x = x.permute(1, 0, 2) # NLD -> LND + + return self.net( + x, x, x, need_weights=need_weights, attn_mask=attn_mask) + + +def make_patch_shift(net, video_frame=12, shift_layers=4, n_div=7): + ''' + Args: + net: CLIP + video_frame: need predefine here + shift_layers: layers to be shift + ''' + + def make_trans_patch_shift(stage, shift_layers): + blocks = list(stage.children()) + for i, b in enumerate(blocks): + if i >= 10 and i <= 11: + blocks[i].attn = PatchShiftModule( + b.attn, video_frame=video_frame, n_div=n_div) + return nn.Sequential(*blocks) + + net.clip.visual.transformer.resblocks = make_trans_patch_shift( + net.clip.visual.transformer.resblocks, shift_layers=shift_layers) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class Event_Layer(nn.Module): + + def __init__(self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation='relu', + normalize_before=False, + is_weights=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.self_attn_vis = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.norm4 = nn.LayerNorm(d_model) + self.norm5 = nn.LayerNorm(d_model) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = nn.ReLU(inplace=True) + self.normalize_before = normalize_before + self.is_weights = is_weights + + def forward(self, tgt, memory, pos=None, query_pos=None): + + tgt = self.norm1(tgt) + memory = self.norm2(memory) + tgt = self.self_attn(tgt, tgt, tgt)[0] + tgt = self.norm3(tgt) + + tgt2, atten_weights = self.multihead_attn(tgt, memory, memory) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm4(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm5(tgt) + + return tgt, atten_weights + + +def adaptive_mask(aa, bb, ada_para): + tensor = torch.zeros((aa, bb)) + adaptive_num = int(bb * ada_para) + cc = int(bb / aa) + for i in range(aa): + start_col = i * cc + end_col = start_col + cc + adaptive_num + if end_col > bb - 1: + tmp = end_col - (bb - 1) + start_col = start_col - tmp + if start_col < 0: + start_col = 0 + end_col = bb + tensor[i, start_col:end_col] = 1 + tensor = ~tensor.bool() + return tensor + + +class Frame_Layer(nn.Module): + + def __init__(self, + d_model, + nhead, + dim_feedforward=2048, + para=1.0, + dropout=0.1, + activation='relu', + normalize_before=False, + is_weights=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.self_attn_vis = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.norm4 = nn.LayerNorm(d_model) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = nn.ReLU(inplace=True) + self.normalize_before = normalize_before + self.is_weights = is_weights + self.mask_para = para + + def forward(self, tgt, memory, pos=None, query_pos=None): + tgt = self.norm1(tgt) + memory = self.norm2(memory) + mask_new = adaptive_mask(tgt.shape[0], memory.shape[0], ada_para=0.2) + if torch.cuda.is_available(): + mask_new = mask_new.cuda() + tgt2, atten_weights = self.multihead_attn( + tgt, memory, memory, attn_mask=mask_new) + tgt = tgt + self.dropout1(tgt2) + + tgt = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm4(tgt) + + return tgt, atten_weights + + +class TransDecoder(nn.Module): + + def __init__(self, + decoder_layer, + num_layers, + norm=None, + return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward(self, tgt, memory, pos=None, query_pos=None): + output = tgt + + intermediate = [] + all_weights = [] + + for layer in self.layers: + output, weights = layer( + output, memory, pos=pos, query_pos=query_pos) + if self.return_intermediate: + intermediate.append(self.norm(output)) + all_weights.append(weights) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack(all_weights) + return output.unsqueeze(0) + + +class Event_decoder(nn.Module): + + def __init__(self, + num_attris=3, + layers=1, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False): + super().__init__() + embedding_dim = dim_ftr + + d_model = dim_ftr + dim_feedforward = dim_feedforward + + self.V = nn.Parameter( + torch.Tensor(num_attris, dim_feedforward), requires_grad=True) + nn.init.xavier_uniform_(self.V) + decoder_layer = Event_Layer( + d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward) + self.event_decoder = TransDecoder( + decoder_layer, + layers, + nn.LayerNorm(d_model), + return_intermediate=True) + self.use_pos_enc = pos_emb + + if self.use_pos_enc: + self.position_encoding_pre = positionalencoding2d( + embedding_dim, 14, 14).unsqueeze(0) + + def forward(self, features): + batch_size = features.shape[0] + if self.use_pos_enc: # False + pos_encoding = self.position_encoding_pre( + features, + torch.zeros(features.shape[0], 14, 14, + dtype=torch.bool).cuda()) + features = features + pos_encoding + + enco_others = features.permute(1, 0, 2) + h_attr = self.V + h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1) + h_attr_batch = h_attr_batch.permute(1, 0, 2) + + hs, _ = self.event_decoder(h_attr_batch, enco_others) + hs = hs[-1].permute(1, 0, 2) + return hs + + +class Frame_decoder(nn.Module): + + def __init__(self, + num_attris=3, + layers=1, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False): + super().__init__() + embedding_dim = dim_ftr + d_model = dim_ftr + dim_feedforward = dim_feedforward + + self.V = nn.Parameter( + torch.Tensor(num_attris, dim_feedforward), requires_grad=True) + nn.init.xavier_uniform_(self.V) + decoder_layer = Frame_Layer( + d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward) + self.event_decoder = TransDecoder( + decoder_layer, + layers, + nn.LayerNorm(d_model), + return_intermediate=True) + self.use_pos_enc = pos_emb + + if self.use_pos_enc: + self.position_encoding_pre = positionalencoding2d( + embedding_dim, 14, 14).unsqueeze(0) + + def forward(self, features): + batch_size = features.shape[0] + if self.use_pos_enc: + pos_encoding = self.position_encoding_pre( + features, + torch.zeros(features.shape[0], 14, 14, + dtype=torch.bool).cuda()) + features = features + pos_encoding + + enco_others = features.permute(1, 0, 2) + h_attr = self.V + h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1) + h_attr_batch = h_attr_batch.permute(1, 0, 2) + + hs, _ = self.event_decoder(h_attr_batch, enco_others) + hs = hs[-1].permute(1, 0, 2) + + return hs diff --git a/runtime/Lib/site_packages/models/multi_modal/rleg/__init__.py b/runtime/Lib/site_packages/models/multi_modal/rleg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7fec95c79bda58af7af8e760ec716f12cda4beb3 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/rleg/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .rleg import RLEGForMultiModalEmbedding + +else: + _import_structure = { + 'rleg': ['RLEGForMultiModalEmbedding'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/rleg/model.py b/runtime/Lib/site_packages/models/multi_modal/rleg/model.py new file mode 100644 index 0000000000000000000000000000000000000000..efabafdc91330bf38bf04ad50995da7836094e06 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/rleg/model.py @@ -0,0 +1,139 @@ +# Copyright 2021 The OpenAI Team Authors. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# +# The implementation here is modified based on OpenAI CLIP, +# originally MIT License, Copyright (c) 2021 OpenAI, +# and publicly available at https://github.com/openai/CLIP/. +""" Generative Multimodal Model Architecture.""" + +import os + +import json +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.models.multi_modal.gemm import gemm_base, tokenizer + + +class ImageEncoder(nn.Module): + """Image Feature Encoder + ViT Style Transformer + """ + + def __init__(self, configs): + super().__init__() + (embed_dim, image_resolution, vision_layers, vision_width, + vision_patch_size) = configs[:5] + self.visual = gemm_base.VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_width // 64, + output_dim=embed_dim, + use_gc=False) + + def forward(self, image, return_tokens=False): + features = self.visual(image) + tokens = features[:, 1:, :] + embedding = features[:, 0, :] + return (embedding, tokens) if return_tokens else embedding + + +class TextEncoder(nn.Module): + """Text Feature Encoder + BERT style transformer + """ + + def __init__(self, configs): + super().__init__() + (context_length, vocab_size, model_width, model_heads, + model_layers) = configs[-5:] + # text model + self.transformer = gemm_base.Transformer( + width=model_width, + layers=model_layers, + heads=model_heads, + attn_mask=self.build_attention_mask(context_length), + ) + # others + self.token_embedding = nn.Embedding(vocab_size, model_width) + self.positional_embedding = nn.Parameter( + torch.empty(context_length, model_width)) + self.ln_final = nn.LayerNorm(model_width) + self.text_projection = nn.Parameter( + torch.empty(model_width, configs[0])) + + def build_attention_mask(self, seq_length=None): + mask = torch.ones(seq_length, seq_length) * -1e4 + mask.triu_(1) # zero out the lower diagonal + return mask + + def forward(self, text, return_tokens=False): + x = self.token_embedding(text) # [batch_size, n_ctx, d_model] + x = x + self.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + # take features from the eot embedding (eot_token is the highest number in each sequence) + embedding = x[torch.arange(x.shape[0]), + text.argmax(dim=-1), ...] @ self.text_projection + return (embedding, x) if return_tokens else embedding + + +class RLEGModel(nn.Module): + """ Generative multi-modal model, trained with RLEG method. + It takes image or text or both of them as input, and produce + the corresponding features of inputs. + """ + + def __init__(self, model_dir): + super().__init__() + with open( + '{}/encoder_config.json'.format(model_dir), 'r', + encoding='utf-8') as f: + model_config = json.loads(f.read()) + model_name = list(model_config.keys())[0] + config_args = model_config[model_name] + bpe_path = os.path.join(model_dir, 'bpe_vocab_16e6.txt.gz') + self.tokenizer = tokenizer.SimpleTokenizer(bpe_path) + # build model architecture + self.image_encoder = ImageEncoder(config_args) + self.text_encoder = TextEncoder(config_args) + self.logit_scale = nn.Parameter(torch.ones([])) + + def tokenize(self, text_str): + text_tensor = tokenizer.clip_tokenize(self.tokenizer, [text_str])[0] + return text_tensor + + def encode_text(self, text): + feature = self.text_encoder(text) + feature = F.normalize(feature, p=2, dim=-1) + return feature + + def encode_image(self, image): + feature = self.image_encoder(image) + feature = F.normalize(feature, p=2, dim=-1) + return feature + + def parse_feat(self, feat): + out = feat.cpu().numpy() + return out + + @torch.no_grad() + def forward(self, image=None, text=None): + """ It takes image or text as input, + and extracts the features as output. + """ + img_feature, text_feature = None, None + if image is not None: + img_feature = self.parse_feat(self.encode_image(image)) + if text is not None: + text_feature = self.parse_feat(self.encode_text(text)) + out = { + 'image_feature': img_feature, + 'text_feature': text_feature, + } + return out diff --git a/runtime/Lib/site_packages/models/multi_modal/rleg/rleg.py b/runtime/Lib/site_packages/models/multi_modal/rleg/rleg.py new file mode 100644 index 0000000000000000000000000000000000000000..dd9accd7b8f12917232205b24bf63564cc489b66 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/rleg/rleg.py @@ -0,0 +1,85 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +""" Generative Multimodal Model Wrapper.""" +from typing import Any, Dict + +import torch +from torchvision import transforms as T + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.rleg.model import RLEGModel +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['RLEGForMultiModalEmbedding'] + + +@MODELS.register_module( + Tasks.generative_multi_modal_embedding, module_name=Models.rleg) +class RLEGForMultiModalEmbedding(TorchModel): + """ Generative multi-modal model for multi-modal embedding. + The model is trained by representation learning with embedding generation. + Inputs could be image or text or both of them. + Outputs could be features of input image or text, + """ + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + self.model = RLEGModel(model_dir=model_dir) + pretrained_params = torch.load('{}/{}'.format( + model_dir, ModelFile.TORCH_MODEL_BIN_FILE)) + self.model.load_state_dict(pretrained_params) + self.model.eval() + self.device_id = device_id + if self.device_id >= 0 and torch.cuda.is_available(): + self.model.to('cuda:{}'.format(self.device_id)) + logger.info('Use GPU: {}'.format(self.device_id)) + else: + self.device_id = -1 + logger.info('Use CPU for inference') + self.img_preprocessor = T.Compose([ + T.Resize((224, 224)), + T.ToTensor(), + T.Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)) + ]) + + def parse_image(self, input_img): + if input_img is None: + return None + input_img = LoadImage.convert_to_img(input_img) + img_tensor = self.img_preprocessor(input_img)[None, ...] + if self.device_id >= 0: + img_tensor = img_tensor.to('cuda:{}'.format(self.device_id)) + return img_tensor + + def parse_text(self, text_str): + if text_str is None or len(text_str) == 0: + return None + if isinstance(text_str, str): + text_ids_tensor = self.model.tokenize(text_str) + else: + raise TypeError(f'text should be str, but got {type(text_str)}') + if self.device_id >= 0: + text_ids_tensor = text_ids_tensor.to('cuda:{}'.format( + self.device_id)) + return text_ids_tensor.view(1, -1) + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + image_input = input.get('image', input.get('img', None)) + text_input = input.get('text', input.get('txt', None)) + image = self.parse_image(image_input) + text = self.parse_text(text_input) + out = self.model(image, text) + output = { + OutputKeys.IMG_EMBEDDING: out.get('image_feature', None), + OutputKeys.TEXT_EMBEDDING: out.get('text_feature', None), + OutputKeys.CAPTION: out.get('caption', None) + } + return output diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/__init__.py b/runtime/Lib/site_packages/models/multi_modal/soonet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..38b95d26e993277c31e307cde86932dab4a64095 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .tokenizer import SimpleTokenizer + from .model import SOONet + from .utils import decode_video + from .clip import load_clip +else: + _import_structure = { + 'model': ['SOONet'], + 'tokenizer': ['SimpleTokenizer'], + 'utils': ['decode_video'], + 'clip': ['load_clip'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/blocks.py b/runtime/Lib/site_packages/models/multi_modal/soonet/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..28f4a5530e73539b61f5bf38c5869322af880a83 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/blocks.py @@ -0,0 +1,287 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Q2VRankerStage1(nn.Module): + """ + Used to calculate the qv_ctx_score with query embedding and multi anchor context embeddings as input. + The qv_ctx_score is used to pre-rank and retain top-k related anchors. + """ + + def __init__(self, nscales, hidden_dim): + super().__init__() + self.fc = nn.Linear(hidden_dim, hidden_dim) + self.nscales = nscales + + def forward(self, ctx_feats, qfeat): + qfeat = self.fc(qfeat) + qv_ctx_scores = list() + for i in range(self.nscales): + score = torch.einsum('bld,bd->bl', + F.normalize(ctx_feats[i], p=2, dim=2), + F.normalize(qfeat, p=2, dim=1)) + qv_ctx_scores.append(score) + + return qv_ctx_scores + + +class V2QRankerStage1(nn.Module): + """ + Used to calculate the vq_ctx_score with anchor context embeddings and multi query embeddings as input. + """ + + def __init__(self, nscales, hidden_dim): + super().__init__() + self.fc = nn.Linear(hidden_dim, hidden_dim) + self.nscales = nscales + + def forward(self, ctx_feats, qfeat): + vq_ctx_scores = list() + for i in range(self.nscales): + score = torch.einsum( + 'bld,bd->bl', F.normalize(self.fc(ctx_feats[i]), p=2, dim=2), + F.normalize(qfeat, p=2, dim=1)) + vq_ctx_scores.append(score) + + return vq_ctx_scores + + +class Q2VRankerStage2(nn.Module): + """ + Used to calculate the qv_ctn_score with query embedding and video sequence embedding as input. + The qv_ctn_score is used to re-rank anchors. + """ + + def __init__(self, nscales, hidden_dim, snippet_length=10): + super().__init__() + self.nscales = nscales + self.snippet_length = snippet_length + self.qfc = nn.Linear(hidden_dim, hidden_dim) + self.encoder = V2VAttention() + + def forward(self, vfeats, qfeat, hit_indices, qv_ctx_scores): + qfeat = self.qfc(qfeat) + + qv_ctn_scores = list() + qv_merge_scores = list() + + _, L, D = vfeats.size() + ctn_feats = list() + for i in range(self.nscales): + anchor_length = self.snippet_length * 2**i + assert L // anchor_length == qv_ctx_scores[i].size(1) + qv_ctx_score = torch.index_select(qv_ctx_scores[i], 1, + hit_indices[i]) + + ctn_feat = vfeats.view(L // anchor_length, anchor_length, + D).detach() + ctn_feat = torch.index_select(ctn_feat, 0, hit_indices[i]) + ctn_feat = self.encoder( + ctn_feat, + torch.ones(ctn_feat.size()[:2], device=ctn_feat.device)) + ctn_feats.append(ctn_feat) + + qv_ctn_score = torch.einsum( + 'bkld,bd->bkl', F.normalize(ctn_feat.unsqueeze(0), p=2, dim=3), + F.normalize(qfeat, p=2, dim=1)) + qv_ctn_score, _ = torch.max(qv_ctn_score, dim=2) + qv_ctn_scores.append(qv_ctn_score) + qv_merge_scores.append(qv_ctx_score + qv_ctn_score) + + return qv_merge_scores, qv_ctn_scores, ctn_feats + + +class V2QRankerStage2(nn.Module): + """ + Used to calculate the vq_ctn_score with anchor content embeddings and multi query embeddings as input. + """ + + def __init__(self, nscales, hidden_dim): + super().__init__() + self.fc = nn.Linear(hidden_dim, hidden_dim) + self.nscales = nscales + + def forward(self, ctn_feats, qfeat): + vq_ctn_scores = list() + for i in range(self.nscales): + score = torch.einsum( + 'bkld,bd->bkl', + F.normalize(self.fc(ctn_feats[i]).unsqueeze(0), p=2, dim=3), + F.normalize(qfeat, p=2, dim=1)) + score = torch.mean(score, dim=2) + vq_ctn_scores.append(score) + + return vq_ctn_scores + + +class V2VAttention(nn.Module): + """ + Self-attention encoder for anchor frame sequence to encode intra-anchor knowledge. + """ + + def __init__(self): + super().__init__() + self.posemb = PositionEncoding(max_len=400, dim=512, dropout=0.0) + self.encoder = MultiHeadAttention(dim=512, n_heads=8, dropout=0.1) + self.dropout = nn.Dropout(0.0) + + def forward(self, video_feats, video_masks): + mask = torch.einsum('bm,bn->bmn', video_masks, + video_masks).unsqueeze(1) + residual = video_feats + video_feats = video_feats + self.posemb(video_feats) + out = self.encoder( + query=video_feats, key=video_feats, value=video_feats, mask=mask) + video_feats = self.dropout(residual + + out) * video_masks.unsqueeze(2).float() + return video_feats + + +class BboxRegressor(nn.Module): + """ + Predict the offset of bounding box for each candidate anchor. + """ + + def __init__(self, hidden_dim, enable_stage2=False): + super().__init__() + self.fc_ctx = nn.Linear(hidden_dim, hidden_dim) + self.fc_q = nn.Linear(hidden_dim, hidden_dim) + + if enable_stage2: + self.fc_ctn = nn.Linear(hidden_dim, hidden_dim) + self.attn = SelfAttention(hidden_dim) + self.predictor = nn.Sequential( + nn.Linear(2 * hidden_dim, hidden_dim), nn.ReLU(), + nn.Linear(hidden_dim, 2)) + else: + self.predictor = nn.Sequential( + nn.Linear(hidden_dim, hidden_dim), nn.ReLU(), + nn.Linear(hidden_dim, 2)) + self.enable_stage2 = enable_stage2 + + def forward(self, ctx_feats, ctn_feats, qfeat): + qfeat = self.fc_q(qfeat) + + ctx_feats = torch.cat(ctx_feats, dim=1) + ctx_fuse_feats = F.relu(self.fc_ctx(ctx_feats)) * F.relu( + qfeat.unsqueeze(1)) + + if self.enable_stage2 and ctn_feats: + ctn_fuse_feats = list() + for i in range(len(ctn_feats)): + out = F.relu(self.fc_ctn(ctn_feats[i]).unsqueeze(0)) * F.relu( + qfeat.unsqueeze(1).unsqueeze(1)) + out = self.attn(out) + ctn_fuse_feats.append(out) + ctn_fuse_feats = torch.cat(ctn_fuse_feats, dim=1) + fuse_feats = torch.cat([ctx_fuse_feats, ctn_fuse_feats], dim=-1) + else: + fuse_feats = ctx_fuse_feats + + out = self.predictor(fuse_feats) + return out + + +class SelfAttention(nn.Module): + """ + Obtain pooled features by self-attentive pooling. + """ + + def __init__(self, hidden_dim): + super().__init__() + self.fc1 = nn.Linear(hidden_dim, hidden_dim // 2) + self.relu = nn.ReLU() + self.fc2 = nn.Linear(hidden_dim // 2, 1) + + def forward(self, x): + att = self.fc2(self.relu(self.fc1(x))).squeeze(3) + att = F.softmax(att, dim=2).unsqueeze(3) + out = torch.sum(x * att, dim=2) + return out + + +class PositionEncoding(nn.Module): + """ + An implementation of trainable positional embedding which is added to + sequence features to inject time/position information. + + Args: + max_len: The max number of trainable positional embeddings. + dim: the dimension of positional embedding. + """ + + def __init__(self, max_len, dim, dropout=0.0): + super(PositionEncoding, self).__init__() + + self.embed = nn.Embedding(max_len, dim) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + batch_size, seq_len = x.shape[:2] + pos_ids = torch.arange(seq_len, dtype=torch.long, device=x.device) + pos_ids = pos_ids.unsqueeze(0).repeat(batch_size, 1) + pos_emb = self.dropout(self.relu(self.embed(pos_ids))) + + return pos_emb + + +class MultiHeadAttention(nn.Module): + """ + An implementation of multi-head attention module, as described in + 'Attention Is All You Need ' + + Args: + dim: the dimension of features of hidden layers. + n_heads: the number of head. + """ + + def __init__(self, dim, n_heads, dropout=0.0): + super(MultiHeadAttention, self).__init__() + + self.dim = dim + self.n_heads = n_heads + self.head_dim = dim // n_heads + + self.to_q = nn.Linear(dim, dim) + self.to_k = nn.Linear(dim, dim) + self.to_v = nn.Linear(dim, dim) + + self.dropout = nn.Dropout(dropout) + self.softmax = nn.Softmax(dim=-1) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.n_heads, self.head_dim) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) # (N, nh, L, dh) + + def forward(self, query, key, value, mask): + q = self.to_q(query) + k = self.to_k(key) + v = self.to_v(value) + + q_trans = self.transpose_for_scores(q) + k_trans = self.transpose_for_scores(k) + v_trans = self.transpose_for_scores(v) + + att = torch.matmul(q_trans, k_trans.transpose(-1, + -2)) # (N, nh, Lq, L) + att = att / math.sqrt(self.head_dim) + att = mask_logits(att, mask) + att = self.softmax(att) + att = self.dropout(att) + + ctx_v = torch.matmul(att, v_trans) # (N, nh, Lq, dh) + ctx_v = ctx_v.permute(0, 2, 1, 3).contiguous() # (N, Lq, nh, dh) + shape = ctx_v.size()[:-2] + (self.dim, ) + ctx_v = ctx_v.view(*shape) # (N, Lq, D) + return ctx_v + + +def mask_logits(inputs, mask, mask_value=-1e30): + mask = mask.type(torch.float32) + return inputs + (1.0 - mask) * mask_value diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/clip.py b/runtime/Lib/site_packages/models/multi_modal/soonet/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..c43820e518aee7f90b5e4847fb0618d877d69ceb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/clip.py @@ -0,0 +1,342 @@ +# The implementation is adopted from CLIP, made publicly available +# under MIT License at https://github.com/openai/CLIP + +import warnings +from collections import OrderedDict +from typing import Tuple, Union + +import numpy as np +import torch +from torch import nn + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int): + super().__init__() + + self.context_length = context_length + + vision_heads = vision_width // 64 + self.visual = VisionTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask()) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image): + return self.visual(image.type(self.dtype)) + + def encode_text(self, text): + x = self.token_embedding(text).type( + self.dtype) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding.type(self.dtype) + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=1, keepdim=True) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + + def __init__(self, input_resolution: int, patch_size: int, width: int, + layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + class_token = self.class_embedding.to(x.dtype) + torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([class_token, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +def build_model(state_dict: dict): + vision_width = state_dict['visual.conv1.weight'].shape[0] + vision_layers = len([ + k for k in state_dict.keys() + if k.startswith('visual.') and k.endswith('.attn.in_proj_weight') + ]) + vision_patch_size = state_dict['visual.conv1.weight'].shape[-1] + grid_size = round( + (state_dict['visual.positional_embedding'].shape[0] - 1)**0.5) + image_resolution = vision_patch_size * grid_size + + embed_dim = state_dict['text_projection'].shape[1] + context_length = state_dict['positional_embedding'].shape[0] + vocab_size = state_dict['token_embedding.weight'].shape[0] + transformer_width = state_dict['ln_final.weight'].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split('.')[2] for k in state_dict + if k.startswith('transformer.resblocks'))) + + model = CLIP(embed_dim, image_resolution, vision_layers, vision_width, + vision_patch_size, context_length, vocab_size, + transformer_width, transformer_heads, transformer_layers) + + for key in ['input_resolution', 'context_length', 'vocab_size']: + if key in state_dict: + del state_dict[key] + + model.load_state_dict(state_dict) + return model.eval() + + +def load_clip(name: str, + device: Union[str, torch.device] = 'cuda' + if torch.cuda.is_available() else 'cpu', + jit=True): + jit = False + model_path = name + try: + model = torch.jit.load( + model_path, map_location=device if jit else 'cpu').eval() + state_dict = None + except RuntimeError: + if jit: + warnings.warn( + f'File {model_path} is not a JIT archive. Loading as a state dict instead' + ) + jit = False + state_dict = torch.load(model_path, map_location='cpu') + + if not jit: + model = build_model(state_dict or model.state_dict()).to(device) + if str(device) == 'cpu': + model.float() + return model + + device_holder = torch.jit.trace( + lambda: torch.ones([]).to(torch.device(device)), example_inputs=[]) + device_node = [ + n for n in device_holder.graph.findAllNodes('prim::Constant') + if 'Device' in repr(n) + ][-1] + + def patch_device(module): + graphs = [module.graph] if hasattr(module, 'graph') else [] + if hasattr(module, 'forward1'): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes('prim::Constant'): + if 'value' in node.attributeNames() and str( + node['value']).startswith('cuda'): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_image) + patch_device(model.encode_text) + + if str(device) == 'cpu': + float_holder = torch.jit.trace( + lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode('aten::to').inputs())[1] + float_node = float_input.node() + + def patch_float(module): + graphs = [module.graph] if hasattr(module, 'graph') else [] + if hasattr(module, 'forward1'): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes('aten::to'): + inputs = list(node.inputs()) + for i in [1, 2]: + if inputs[i].node()['value'] == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_image) + patch_float(model.encode_text) + + model.float() + + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/model.py b/runtime/Lib/site_packages/models/multi_modal/soonet/model.py new file mode 100644 index 0000000000000000000000000000000000000000..32912bfcab74143f837fb5a0277cacf991376757 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/model.py @@ -0,0 +1,156 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os + +import torch +import torch.nn as nn + +from modelscope.metainfo import Models +from modelscope.models.base.base_torch_model import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .blocks import (BboxRegressor, Q2VRankerStage1, Q2VRankerStage2, + V2QRankerStage1, V2QRankerStage2) +from .swin_transformer import SwinTransformerV2_1D + + +@MODELS.register_module( + Tasks.video_temporal_grounding, module_name=Models.soonet) +class SOONet(TorchModel): + """ + The implementation of 'Scanning Only Once: An End-to-end Framework for Fast Temporal Grounding + in Long Videos'. The model is dynamically initialized with the following parts: + - q2v_stage1: calculate qv_ctx_score. + - v2q_stage1: calculate vq_ctx_score. + - q2v_stage2: calculate qv_ctn_score. + - v2q_stage2: calculate vq_ctn_score. + - regressor: predict the offset of bounding box for each candidate anchor. + """ + + def __init__(self, model_dir: str, *args, **kwargs): + """ + Initialize SOONet Model + + Args: + model_dir: model id or path + """ + super().__init__() + config_path = os.path.join(model_dir, ModelFile.CONFIGURATION) + self.config = Config.from_file(config_path).hyperparams + nscales = self.config.nscales + hidden_dim = self.config.hidden_dim + snippet_length = self.config.snippet_length + self.enable_stage2 = self.config.enable_stage2 + self.stage2_topk = self.config.stage2_topk + self.nscales = nscales + + self.video_encoder = SwinTransformerV2_1D( + patch_size=snippet_length, + in_chans=hidden_dim, + embed_dim=hidden_dim, + depths=[2] * nscales, + num_heads=[8] * nscales, + window_size=[64] * nscales, + mlp_ratio=2., + qkv_bias=True, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.1, + norm_layer=nn.LayerNorm, + patch_norm=True, + use_checkpoint=False, + pretrained_window_sizes=[0] * nscales) + + self.q2v_stage1 = Q2VRankerStage1(nscales, hidden_dim) + self.v2q_stage1 = V2QRankerStage1(nscales, hidden_dim) + if self.enable_stage2: + self.q2v_stage2 = Q2VRankerStage2(nscales, hidden_dim, + snippet_length) + self.v2q_stage2 = V2QRankerStage2(nscales, hidden_dim) + self.regressor = BboxRegressor(hidden_dim, self.enable_stage2) + + # Load trained weights + model_path = os.path.join(model_dir, + 'SOONet_MAD_VIT-B-32_4Scale_10C.pth') + state_dict = torch.load(model_path, map_location='cpu')['model'] + self.load_state_dict(state_dict, strict=True) + + def forward(self, **kwargs): + if self.training: + return self.forward_train(**kwargs) + else: + return self.forward_test(**kwargs) + + def forward_train(self, **kwargs): + raise NotImplementedError + + def forward_test(self, + query_feats=None, + video_feats=None, + start_ts=None, + end_ts=None, + scale_boundaries=None, + **kwargs): + """ + Obtain matching scores and bbox bias of the top-k candidate anchors, with + pre-extracted query features and video features as input. + + Args: + query_feats: the pre-extracted text features. + video_feats: the pre-extracted video features. + start_ts: the start timestamps of pre-defined multi-scale anchors. + end_ts: the end timestamps of pre-defined multi-scale anchors. + scale_boundaries: the begin and end anchor index for each scale in start_ts and end_ts. + + Returns: + [final_scores, bbox_bias, starts, ends] + """ + sent_feat = query_feats + ctx_feats = self.video_encoder(video_feats.permute(0, 2, 1)) + qv_ctx_scores = self.q2v_stage1(ctx_feats, sent_feat) + if self.enable_stage2: + hit_indices = list() + starts = list() + ends = list() + filtered_ctx_feats = list() + for i in range(self.nscales): + _, indices = torch.sort( + qv_ctx_scores[i], dim=1, descending=True) + indices, _ = torch.sort( + torch.LongTensor( + list( + set(indices[:, :self.stage2_topk].flatten().cpu(). + numpy().tolist())))) + indices = indices.to(video_feats.device) + hit_indices.append(indices) + + filtered_ctx_feats.append( + torch.index_select(ctx_feats[i], 1, indices)) + + scale_first = scale_boundaries[i] + scale_last = scale_boundaries[i + 1] + + filtered_start = torch.index_select( + start_ts[scale_first:scale_last], 0, indices) + filtered_end = torch.index_select( + end_ts[scale_first:scale_last], 0, indices) + starts.append(filtered_start) + ends.append(filtered_end) + + starts = torch.cat(starts, dim=0) + ends = torch.cat(ends, dim=0) + + qv_merge_scores, qv_ctn_scores, ctn_feats = self.q2v_stage2( + video_feats, sent_feat, hit_indices, qv_ctx_scores) + ctx_feats = filtered_ctx_feats + else: + ctn_feats = None + qv_merge_scores = qv_ctx_scores + starts = start_ts + ends = end_ts + + bbox_bias = self.regressor(ctx_feats, ctn_feats, sent_feat) + final_scores = torch.sigmoid(torch.cat(qv_merge_scores, dim=1)) + + return final_scores, bbox_bias, starts, ends diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/swin_transformer.py b/runtime/Lib/site_packages/models/multi_modal/soonet/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..459561c02557a991c309da3c7b8291d068b2d392 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/swin_transformer.py @@ -0,0 +1,623 @@ +# The implementation is adopted from Swin-Transformer-1D, made publicly available +# at https://github.com/meraks/Swin-Transformer-1D + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch.nn.init import trunc_normal_ + + +def drop_path(x, + drop_prob: float = 0., + training: bool = False, + scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + def extra_repr(self): + return f'drop_prob={round(self.drop_prob,3):0.3f}' + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, L, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, C) + """ + B, L, C = x.shape + x = x.view(B, L // window_size, window_size, C) + windows = x.permute(0, 1, 2, 3).contiguous().view(-1, window_size, C) + return windows + + +def window_reverse(windows, window_size, L): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + L (int): sequence length + Returns: + x: (B, L, C) + """ + B = int(windows.shape[0] / (L / window_size)) + x = windows.view(B, L // window_size, window_size, -1) + x = x.permute(0, 1, 2, 3).contiguous().view(B, L, -1) + return x + + +class WindowAttention_1D(nn.Module): + r""" Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (int): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + pretrained_window_size (int): The height and width of the window in pre-training. + """ + + def __init__(self, + dim, + window_size, + num_heads, + qkv_bias=True, + attn_drop=0., + proj_drop=0., + pretrained_window_size=0): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wl + self.pretrained_window_size = pretrained_window_size + self.num_heads = num_heads + + self.logit_scale = nn.Parameter( + torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True) + + # mlp to generate continuous relative position bias + self.cpb_mlp = nn.Sequential( + nn.Linear(1, 512, bias=True), nn.ReLU(inplace=True), + nn.Linear(512, num_heads, bias=False)) + + # get relative_coords_table + relative_coords_l = torch.arange( + -(self.window_size - 1), self.window_size, dtype=torch.float32) + relative_coords_table = torch.stack( + torch.meshgrid([relative_coords_l], indexing='ij')).permute( + 1, 0).contiguous().unsqueeze(0) # 1, 2*Wl-1, 1 + if pretrained_window_size > 0: + relative_coords_table[:, :, :] /= (pretrained_window_size - 1) + else: + relative_coords_table[:, :, :] /= (self.window_size - 1) + relative_coords_table *= 8 # normalize to -8, 8 + relative_coords_table = torch.sign(relative_coords_table) * torch.log2( + torch.abs(relative_coords_table) + 1.0) / np.log2(8) + + self.register_buffer('relative_coords_table', relative_coords_table) + + # get pair-wise relative position index for each token inside the window + coords_l = torch.arange(self.window_size) + coords = torch.stack(torch.meshgrid([coords_l], + indexing='ij')) # 1, Wl + coords_flatten = torch.flatten(coords, 1) # 1, Wl + relative_coords = coords_flatten[:, :, + None] - coords_flatten[:, + None, :] # 1, Wl, Wl + relative_coords = relative_coords.permute(1, 2, + 0).contiguous() # Wl, Wl, 1 + relative_coords[:, :, + 0] += self.window_size - 1 # shift to start from 0 + relative_position_index = relative_coords.sum(-1) # Wl, Wl + self.register_buffer('relative_position_index', + relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=False) + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(dim)) + self.v_bias = nn.Parameter(torch.zeros(dim)) + else: + self.q_bias = None + self.v_bias = None + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wl, Wl) or None + """ + B_, N, C = x.shape + qkv_bias = None + if self.q_bias is not None: + qkv_bias = torch.cat( + (self.q_bias, + torch.zeros_like(self.v_bias, + requires_grad=False), self.v_bias)) + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[ + 2] # make torchscript happy (cannot use tensor as tuple) + + # cosine attention + attn = ( + F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1)) + logit_scale = torch.clamp( + self.logit_scale, + max=torch.log(torch.tensor(1. / 0.01, device=attn.device))).exp() + attn = attn * logit_scale + + relative_position_bias_table = self.cpb_mlp( + self.relative_coords_table).view(-1, self.num_heads) + relative_position_bias = relative_position_bias_table[ + self.relative_position_index.view(-1)].view( + self.window_size, self.window_size, -1) # Wl,l,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1).contiguous() # nH, Wl, Wl + relative_position_bias = 16 * torch.sigmoid(relative_position_bias) + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, + N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +def compute_mask(L, window_size, shift_size): + Lp = int(np.ceil(L / window_size)) * window_size + img_mask = torch.zeros((1, Lp, 1)) # 1 Lp 1 + pad_size = int(Lp - L) + if (pad_size == 0) or (pad_size + shift_size == window_size): + segs = (slice(-window_size), slice(-window_size, -shift_size), + slice(-shift_size, None)) + elif pad_size + shift_size > window_size: + seg1 = int(window_size * 2 - L + shift_size) + segs = (slice(-seg1), slice(-seg1, -window_size), + slice(-window_size, -shift_size), slice(-shift_size, None)) + elif pad_size + shift_size < window_size: + seg1 = int(window_size * 2 - L + shift_size) + segs = (slice(-window_size), slice(-window_size, -seg1), + slice(-seg1, -shift_size), slice(-shift_size, None)) + cnt = 0 + for d in segs: + img_mask[:, d, :] = cnt + cnt += 1 + mask_windows = window_partition(img_mask, window_size) # nW, ws, 1 + mask_windows = mask_windows.squeeze(-1) # nW, ws + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, + float(-100.0)).masked_fill( + attn_mask == 0, float(0.0)) + return attn_mask + + +class SwinTransformerBlock_1D(nn.Module): + r""" Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + pretrained_window_size (int): Window size in pre-training. + """ + + def __init__(self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4., + qkv_bias=True, + drop=0., + attn_drop=0., + drop_path=0., + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + pretrained_window_size=0): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + + assert 0 <= self.shift_size < self.window_size, 'shift_size must in 0-window_size' + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention_1D( + dim, + window_size=self.window_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + attn_drop=attn_drop, + proj_drop=drop, + pretrained_window_size=pretrained_window_size) + + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop) + + def forward(self, x): + B, L, C = x.shape + + attn_mask = compute_mask(L, self.window_size, + self.shift_size).to(x.device) + + shortcut = x + # x = x.view(B, L, C) + + # padding x + pad_r = (self.window_size - L % self.window_size) % self.window_size + x = F.pad(x, (0, 0, 0, pad_r)) + _, Lp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size), dims=(1)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition(shifted_x, + self.window_size) # nW*B, window_size, C + x_windows = x_windows.view(-1, self.window_size, + C) # nW*B, window_siz, C + + # W-MSA/SW-MSA + attn_windows = self.attn( + x_windows, mask=attn_mask) # nW*B, window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, + Lp) # B L' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size), dims=(1)) + else: + x = shifted_x + x = x.view(B, Lp, C) + # reverse padding x + x = x[:, :L, :].contiguous() + x = shortcut + self.drop_path(self.norm1(x)) + + # FFN + x = x + self.drop_path(self.norm2(self.mlp(x))) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + # self.reduction = nn.Linear(2 * dim, dim, bias=False) + # self.norm = norm_layer(2 * dim) + + def forward(self, x): + """ Forward function. + Args: + x: Input feature, tensor size (B, L, C). + """ + B, L, C = x.shape + x = F.pad(x, (0, 0, 0, L % 2)) + + x0 = x[:, 0::2, :] # B L/2 C + x1 = x[:, 1::2, :] # B L/2 C + + x = torch.maximum(x0, x1) + + return x + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of input channels. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + pretrained_window_size (int): Local window size in pre-training. + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size, + mlp_ratio=4., + qkv_bias=True, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + pretrained_window_size=0): + + super().__init__() + self.dim = dim + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock_1D( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] + if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + pretrained_window_size=pretrained_window_size) + for i in range(depth) + ]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x): + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + + proposal = x + if self.downsample is not None: + x = self.downsample(x) + return x, proposal + + def _init_respostnorm(self): + for blk in self.blocks: + nn.init.constant_(blk.norm1.bias, 0) + nn.init.constant_(blk.norm1.weight, 0) + nn.init.constant_(blk.norm2.bias, 0) + nn.init.constant_(blk.norm2.weight, 0) + + +class PatchEmbed1D(nn.Module): + """ Video to Patch Embedding. + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input video channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, + patch_size=4, + in_chans=32, + embed_dim=128, + norm_layer=None): + super().__init__() + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv1d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, L = x.size() + pad_r = (self.patch_size - L % self.patch_size) % self.patch_size + x = F.pad(x, (0, pad_r)) + x = self.proj(x) # B C Wl + if self.norm is not None: + # Wl = x.size(2) + x = x.transpose(1, 2) + x = self.norm(x) + # x = x.transpose(1, 2).view(-1, self.embed_dim, Wl) + + return x + + +class SwinTransformerV2_1D(nn.Module): + + def __init__(self, + patch_size=4, + in_chans=32, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=[7, 7, 7, 7], + mlp_ratio=4., + qkv_bias=True, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.1, + norm_layer=nn.LayerNorm, + patch_norm=True, + use_checkpoint=False, + pretrained_window_sizes=[0, 0, 0, 0], + **kwargs): + super().__init__() + + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.patch_norm = patch_norm + self.num_features = int(embed_dim * 2**(self.num_layers - 1)) + self.mlp_ratio = mlp_ratio + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed1D( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + self.pos_drop = nn.Dropout(p=drop_rate) + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=embed_dim, + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size[i_layer], + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if + (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + pretrained_window_size=pretrained_window_sizes[i_layer]) + self.layers.append(layer) + + self.apply(self._init_weights) + for bly in self.layers: + bly._init_respostnorm() + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'absolute_pos_embed'} + + @torch.jit.ignore + def no_weight_decay_keywords(self): + return {'cpb_mlp', 'logit_scale', 'relative_position_bias_table'} + + def forward_features(self, x): + x = self.patch_embed(x) + x = self.pos_drop(x) + + proposals = list() + for layer in self.layers: + x, proposal = layer(x) + proposals.append(proposal) + + return proposals + + def forward(self, x): + return self.forward_features(x) diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/tokenizer.py b/runtime/Lib/site_packages/models/multi_modal/soonet/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4f40a7146e5e01f39234872bdee93eb499aabd --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/tokenizer.py @@ -0,0 +1,152 @@ +# The implementation is adopted from CLIP, made publicly available +# under MIT License at https://github.com/openai/CLIP + +import gzip +import html +from functools import lru_cache + +import ftfy +import regex as re +import torch + + +@lru_cache() +def bytes_to_unicode(): + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except ValueError: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + def tokenize(self, texts, context_length=77): + if isinstance(texts, str): + texts = [texts] + + sot_token = self.encoder['<|startoftext|>'] + eot_token = self.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + self.encode(text) + [eot_token] + for text in texts] + result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + tokens = tokens[:context_length] + tokens[-1] = eot_token + + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/runtime/Lib/site_packages/models/multi_modal/soonet/utils.py b/runtime/Lib/site_packages/models/multi_modal/soonet/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8fe3960e8b02d9beb8be8398120a2d4131291a76 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/soonet/utils.py @@ -0,0 +1,58 @@ +# Copyright 2022-2023 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import copy + +import decord +import numpy as np +from decord import VideoReader, cpu +from decord._ffi.base import DECORDError +from tqdm import tqdm + + +def decode_video(video_path, target_fps=5): + """ + Decode video from 'video_path' and return the sampled frames based on target_fps. + The default value of target_fps is 5. + + Args: + video_path: the absolute path of video. + target_fps: the number of sampled video frames per second. + + Returns: + [imgs, duration] + """ + decord.bridge.set_bridge('torch') + vr = VideoReader(video_path, ctx=cpu(0)) + cur_fps = vr.get_avg_fps() + if cur_fps > target_fps: + interval = float(cur_fps) / float(target_fps) + start = float(interval) / 2. + else: + interval = 1.0 + start = 0.0 + + vid_length = len(vr) + duration = vid_length / cur_fps + sampled_idxs = np.clip( + np.round(np.arange(start, float(vid_length), step=interval)), 0, + vid_length - 1).astype(np.int32) + + imgs = list() + for i in tqdm(sampled_idxs): + bias = 0 + # avoid broken frames + while bias <= 10: + try: + img = vr[i - bias] + break + except DECORDError: + bias += 1 + if bias > 10: + img = copy.deepcopy(imgs[-1]) + imgs.append(img) + else: + img = img / 255. + img = img.permute(2, 0, 1) + imgs.append(img) + + return imgs, duration diff --git a/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/__init__.py b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f0b184b9863c60c6f35d36c3ac5019d607da20e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .stable_diffusion import StableDiffusion +from .stable_diffusion_xl import StableDiffusionXL diff --git a/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion.py b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..06f87287cf3f4aa383dfe0b358d911993fa0c76a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion.py @@ -0,0 +1,168 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +from functools import partial +from typing import Callable, List, Optional, Union + +import torch +import torch.nn.functional as F +from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel +from packaging import version +from transformers import CLIPTextModel, CLIPTokenizer + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.checkpoint import save_checkpoint, save_configuration +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.text_to_image_synthesis, module_name=Models.stable_diffusion) +class StableDiffusion(TorchModel): + """ The implementation of stable diffusion model based on TorchModel. + + This model is constructed with the implementation of stable diffusion model. If you want to + finetune lightweight parameters on your own dataset, you can define you own tuner module + and load in this cls. + """ + + def __init__(self, model_dir, *args, **kwargs): + """ Initialize a vision stable diffusion model. + + Args: + model_dir: model id or path + """ + super().__init__(model_dir, *args, **kwargs) + revision = kwargs.pop('revision', None) + xformers_enable = kwargs.pop('xformers_enable', False) + self.lora_tune = kwargs.pop('lora_tune', False) + self.dreambooth_tune = kwargs.pop('dreambooth_tune', False) + + self.weight_dtype = kwargs.pop('torch_type', torch.float32) + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + # Load scheduler, tokenizer and models + self.noise_scheduler = DDPMScheduler.from_pretrained( + model_dir, subfolder='scheduler') + self.tokenizer = CLIPTokenizer.from_pretrained( + model_dir, subfolder='tokenizer', revision=revision) + self.text_encoder = CLIPTextModel.from_pretrained( + model_dir, subfolder='text_encoder', revision=revision) + self.vae = AutoencoderKL.from_pretrained( + model_dir, subfolder='vae', revision=revision) + self.unet = UNet2DConditionModel.from_pretrained( + model_dir, subfolder='unet', revision=revision) + self.safety_checker = None + + # Freeze gradient calculation and move to device + if self.vae is not None: + self.vae.requires_grad_(False) + self.vae = self.vae.to(self.device, dtype=self.weight_dtype) + if self.text_encoder is not None: + self.text_encoder.requires_grad_(False) + self.text_encoder = self.text_encoder.to( + self.device, dtype=self.weight_dtype) + if self.unet is not None: + if self.lora_tune: + self.unet.requires_grad_(False) + self.unet = self.unet.to(self.device, dtype=self.weight_dtype) + + # xformers accelerate memory efficient attention + if xformers_enable: + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse('0.0.16'): + logger.warn( + 'xFormers 0.0.16 cannot be used for training in some GPUs. ' + 'If you observe problems during training, please update xFormers to at least 0.0.17.' + ) + self.unet.enable_xformers_memory_efficient_attention() + + def tokenize_caption(self, captions): + """ Convert caption text to token data. + + Args: + captions: a batch of texts. + Returns: token's data as tensor. + """ + inputs = self.tokenizer( + captions, + max_length=self.tokenizer.model_max_length, + padding='max_length', + truncation=True, + return_tensors='pt') + return inputs.input_ids + + def forward(self, text='', target=None): + self.unet.train() + self.unet = self.unet.to(self.device) + + # Convert to latent space + with torch.no_grad(): + latents = self.vae.encode( + target.to(dtype=self.weight_dtype)).latent_dist.sample() + latents = latents * self.vae.config.scaling_factor + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.num_train_timesteps, (bsz, ), + device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = self.noise_scheduler.add_noise(latents, noise, + timesteps) + + input_ids = self.tokenize_caption(text).to(self.device) + + # Get the text embedding for conditioning + with torch.no_grad(): + encoder_hidden_states = self.text_encoder(input_ids)[0] + + # Get the target for loss depending on the prediction type + if self.noise_scheduler.config.prediction_type == 'epsilon': + target = noise + elif self.noise_scheduler.config.prediction_type == 'v_prediction': + target = self.noise_scheduler.get_velocity(latents, noise, + timesteps) + else: + raise ValueError( + f'Unknown prediction type {self.noise_scheduler.config.prediction_type}' + ) + + # Predict the noise residual and compute loss + model_pred = self.unet(noisy_latents, timesteps, + encoder_hidden_states).sample + + if model_pred.shape[1] == 6: + model_pred, _ = torch.chunk(model_pred, 2, dim=1) + + loss = F.mse_loss(model_pred.float(), target.float(), reduction='mean') + + output = {OutputKeys.LOSS: loss} + return output + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = partial( + save_checkpoint, with_meta=False), + config: Optional[dict] = None, + save_config_function: Callable = save_configuration, + **kwargs): + # Skip copying the original weights for lora and dreambooth method + if self.lora_tune or self.dreambooth_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion' + pass + else: + super().save_pretrained(target_folder, save_checkpoint_names, + save_function, config, + save_config_function, **kwargs) diff --git a/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion_xl.py b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion_xl.py new file mode 100644 index 0000000000000000000000000000000000000000..e0fa5070a8cbfae8c569666c2ec7cda72851b55e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/stable_diffusion/stable_diffusion_xl.py @@ -0,0 +1,254 @@ +# Copyright 2023-2024 The Alibaba Fundamental Vision Team Authors. All rights reserved. +import os +import random +from functools import partial +from typing import Callable, List, Optional, Union + +import torch +import torch.nn.functional as F +from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel +from packaging import version +from torchvision import transforms +from torchvision.transforms.functional import crop +from transformers import (AutoTokenizer, CLIPTextModel, + CLIPTextModelWithProjection) + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.checkpoint import save_checkpoint, save_configuration +from modelscope.utils.constant import Tasks + + +@MODELS.register_module( + Tasks.text_to_image_synthesis, module_name=Models.stable_diffusion_xl) +class StableDiffusionXL(TorchModel): + """ The implementation of stable diffusion xl model based on TorchModel. + + This model is constructed with the implementation of stable diffusion xl model. If you want to + finetune lightweight parameters on your own dataset, you can define you own tuner module + and load in this cls. + """ + + def __init__(self, model_dir, *args, **kwargs): + """ Initialize a vision stable diffusion xl model. + + Args: + model_dir: model id or path + """ + super().__init__(model_dir, *args, **kwargs) + revision = kwargs.pop('revision', None) + xformers_enable = kwargs.pop('xformers_enable', False) + self.lora_tune = kwargs.pop('lora_tune', False) + self.resolution = kwargs.pop('resolution', 1024) + self.random_flip = kwargs.pop('random_flip', True) + + self.weight_dtype = torch.float32 + self.device = torch.device( + 'cuda' if torch.cuda.is_available() else 'cpu') + + # Load scheduler, tokenizer and models + self.noise_scheduler = DDPMScheduler.from_pretrained( + model_dir, subfolder='scheduler') + self.tokenizer_one = AutoTokenizer.from_pretrained( + model_dir, + subfolder='tokenizer', + revision=revision, + use_fast=False) + self.tokenizer_two = AutoTokenizer.from_pretrained( + model_dir, + subfolder='tokenizer_2', + revision=revision, + use_fast=False) + self.text_encoder_one = CLIPTextModel.from_pretrained( + model_dir, subfolder='text_encoder', revision=revision) + self.text_encoder_two = CLIPTextModelWithProjection.from_pretrained( + model_dir, subfolder='text_encoder_2', revision=revision) + self.vae = AutoencoderKL.from_pretrained( + model_dir, subfolder='vae', revision=revision) + self.unet = UNet2DConditionModel.from_pretrained( + model_dir, subfolder='unet', revision=revision) + self.safety_checker = None + + # Freeze gradient calculation and move to device + if self.vae is not None: + self.vae.requires_grad_(False) + self.vae = self.vae.to(self.device) + if self.text_encoder_one is not None: + self.text_encoder_one.requires_grad_(False) + self.text_encoder_one = self.text_encoder_one.to(self.device) + if self.text_encoder_two is not None: + self.text_encoder_two.requires_grad_(False) + self.text_encoder_two = self.text_encoder_two.to(self.device) + if self.unet is not None: + if self.lora_tune: + self.unet.requires_grad_(False) + self.unet = self.unet.to(self.device) + + # xformers accelerate memory efficient attention + if xformers_enable: + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse('0.0.16'): + logger.warn( + 'xFormers 0.0.16 cannot be used for training in some GPUs. ' + 'If you observe problems during training, please update xFormers to at least 0.0.17.' + ) + self.unet.enable_xformers_memory_efficient_attention() + + def tokenize_caption(self, tokenizer, captions): + """ Convert caption text to token data. + + Args: + tokenizer: the tokenizer one or two. + captions: a batch of texts. + Returns: token's data as tensor. + """ + inputs = tokenizer( + captions, + max_length=tokenizer.model_max_length, + padding='max_length', + truncation=True, + return_tensors='pt') + return inputs.input_ids + + def compute_time_ids(self, original_size, crops_coords_top_left): + target_size = (self.resolution, self.resolution) + add_time_ids = list(original_size + crops_coords_top_left + + target_size) + add_time_ids = torch.tensor([add_time_ids]) + add_time_ids = add_time_ids.to(self.device, dtype=self.weight_dtype) + return add_time_ids + + def encode_prompt(self, + text_encoders, + tokenizers, + prompt, + text_input_ids_list=None): + prompt_embeds_list = [] + + for i, text_encoder in enumerate(text_encoders): + if tokenizers is not None: + tokenizer = tokenizers[i] + text_input_ids = tokenize_prompt(tokenizer, prompt) + else: + assert text_input_ids_list is not None + text_input_ids = text_input_ids_list[i] + + prompt_embeds = text_encoder( + text_input_ids.to(text_encoder.device), + output_hidden_states=True, + ) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + prompt_embeds = prompt_embeds.hidden_states[-2] + bs_embed, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.view(bs_embed, seq_len, -1) + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + pooled_prompt_embeds = pooled_prompt_embeds.view(bs_embed, -1) + return prompt_embeds, pooled_prompt_embeds + + def preprocessing_data(self, text, target): + train_crop = transforms.RandomCrop(self.resolution) + train_resize = transforms.Resize( + self.resolution, + interpolation=transforms.InterpolationMode.BILINEAR) + train_flip = transforms.RandomHorizontalFlip(p=1.0) + image = target + original_size = (image.size()[-1], image.size()[-2]) + image = train_resize(image) + y1, x1, h, w = train_crop.get_params( + image, (self.resolution, self.resolution)) + image = crop(image, y1, x1, h, w) + if self.random_flip and random.random() < 0.5: + # flip + x1 = image.size()[-2] - x1 + image = train_flip(image) + crop_top_left = (y1, x1) + input_ids_one = self.tokenize_caption(self.tokenizer_one, text) + input_ids_two = self.tokenize_caption(self.tokenizer_two, text) + + return original_size, crop_top_left, image, input_ids_one, input_ids_two + + def forward(self, text='', target=None): + self.unet.train() + self.unet = self.unet.to(self.device) + + # processing data + original_size, crop_top_left, image, input_ids_one, input_ids_two = self.preprocessing_data( + text, target) + # Convert to latent space + with torch.no_grad(): + latents = self.vae.encode( + target.to(dtype=self.weight_dtype)).latent_dist.sample() + latents = latents * self.vae.config.scaling_factor + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.num_train_timesteps, (bsz, ), + device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = self.noise_scheduler.add_noise(latents, noise, + timesteps) + + add_time_ids = self.compute_time_ids(original_size, crop_top_left) + + # Predict the noise residual + unet_added_conditions = {'time_ids': add_time_ids} + prompt_embeds, pooled_prompt_embeds = self.encode_prompt( + text_encoders=[self.text_encoder_one, self.text_encoder_two], + tokenizers=None, + prompt=None, + text_input_ids_list=[input_ids_one, input_ids_two]) + unet_added_conditions.update({'text_embeds': pooled_prompt_embeds}) + # Predict the noise residual and compute loss + model_pred = self.unet( + noisy_latents, + timesteps, + prompt_embeds, + added_cond_kwargs=unet_added_conditions).sample + + # Get the target for loss depending on the prediction type + if self.noise_scheduler.config.prediction_type == 'epsilon': + target = noise + elif self.noise_scheduler.config.prediction_type == 'v_prediction': + target = self.noise_scheduler.get_velocity(model_input, noise, + timesteps) + else: + raise ValueError( + f'Unknown prediction type {self.noise_scheduler.config.prediction_type}' + ) + + loss = F.mse_loss(model_pred.float(), target.float(), reduction='mean') + + output = {OutputKeys.LOSS: loss} + return output + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = partial( + save_checkpoint, with_meta=False), + config: Optional[dict] = None, + save_config_function: Callable = save_configuration, + **kwargs): + # Skip copying the original weights for lora and dreambooth method + if self.lora_tune: + config['pipeline']['type'] = 'diffusers-stable-diffusion-xl' + pass + else: + super().save_pretrained(target_folder, save_checkpoint_names, + save_function, config, + save_config_function, **kwargs) diff --git a/runtime/Lib/site_packages/models/multi_modal/team/__init__.py b/runtime/Lib/site_packages/models/multi_modal/team/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..58bbdca55851e584b8c105facd0c55cdb02a6d95 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/team/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from .team_model import TEAMForMultiModalSimilarity diff --git a/runtime/Lib/site_packages/models/multi_modal/team/team_model.py b/runtime/Lib/site_packages/models/multi_modal/team/team_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8c0e288a89c6415ffc7282b41915f1d481ae08dc --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/team/team_model.py @@ -0,0 +1,127 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +from typing import Any, Dict + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image +from tokenizers import BertWordPieceTokenizer +from torchvision.transforms import Compose, Normalize, Resize, ToTensor + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .utils import TEAM, BertWrapper, CLIPVisionWrapper, CrossLayer + +logger = get_logger() + +__all__ = ['TEAMForMultiModalSimilarity'] + + +@MODELS.register_module(Tasks.multi_modal_similarity, module_name=Models.team) +class TEAMForMultiModalSimilarity(TorchModel): + + def __init__(self, model_dir, device_id=0, *args, **kwargs): + super().__init__( + model_dir=model_dir, device_id=device_id, *args, **kwargs) + + text_model = BertWrapper( + config_json='{}/text_config.json'.format(model_dir), + feat_dim=768, + token_dim=1024) + text_model.bert.cls = None + image_model = CLIPVisionWrapper() + + self.model = TEAM( + text_model, + image_model, + pretrained='{}/{}'.format(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE)) + self.model.eval() + + self.device_id = device_id + if self.device_id >= 0 and torch.cuda.is_available(): + self.model.to('cuda:{}'.format(self.device_id)) + logger.info('Use GPU: {}'.format(self.device_id)) + else: + self.device_id = -1 + logger.info('Use CPU for inference') + + self.text_tokenizer = BertWordPieceTokenizer( + '{}/{}'.format(model_dir, ModelFile.VOCAB_FILE), lowercase=False) + self.text_tokenizer.enable_truncation(max_length=30) + + norm_op = Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)) + self.img_preprocessor = Compose([ + Resize((224, 224), interpolation=Image.BICUBIC), + ToTensor(), norm_op + ]) + + def tokenize_text(self, text_str): + tokens = self.text_tokenizer.encode(text_str) + max_tokens = 30 + text_ids_tensor = torch.zeros((1, max_tokens)).long() + text_mask_tensor = torch.zeros((1, max_tokens)) + text_ids, text_mask = tokens.ids, tokens.attention_mask + text_ids_tensor[0, 0:len(text_ids)] = torch.tensor(text_ids) + text_mask_tensor[0, 0:len(text_mask)] = torch.tensor(text_mask) + return text_ids_tensor, text_mask_tensor + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + with torch.no_grad(): + if 'img' in input and input['img'] is not None: + input_img = input['img'] + input_img = LoadImage.convert_to_img(input_img) + img_tensor = self.img_preprocessor(input_img)[None, ...] + + if self.device_id >= 0: + img_tensor = img_tensor.to('cuda:{}'.format( + self.device_id)) + _, _, image_feature, image_tensors = self.model.get_feature( + None, None, img_tensor) + image_feature = image_feature.cpu().numpy() + else: + image_feature, image_tensors = None, None + + if 'text' in input and input['text'] is not None: + text_str = input['text'] + if isinstance(text_str, str): + text_ids_tensor, text_mask_tensor = self.tokenize_text( + text_str) + else: + raise TypeError( + f'text should be str, but got {type(text_str)}') + + if self.device_id >= 0: + text_ids_tensor = text_ids_tensor.to('cuda:{}'.format( + self.device_id)) + text_mask_tensor = text_mask_tensor.to('cuda:{}'.format( + self.device_id)) + text_feature, text_tensors, _, _ = self.model.get_feature( + text_ids_tensor, text_mask_tensor, None) + text_feature = text_feature.cpu().numpy() + else: + text_tensors, text_mask_tensor = None, None + + if text_tensors is not None and text_mask_tensor is not None and image_tensors is not None: + score = self.model.get_cross_score(text_tensors, + text_mask_tensor, + image_tensors)[0].item() + else: + score = None + output = { + OutputKeys.IMG_EMBEDDING: image_feature, + OutputKeys.TEXT_EMBEDDING: text_feature, + OutputKeys.SCORES: score + } + return output + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs diff --git a/runtime/Lib/site_packages/models/multi_modal/team/utils.py b/runtime/Lib/site_packages/models/multi_modal/team/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6ecc217237e9dc0daff3aca78b0b888bc2a21c2b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/team/utils.py @@ -0,0 +1,334 @@ +# Copyright 2021 The OpenAI Team Authors. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. +# +# The implementation here is modified based on OpenAI CLIP, +# originally MIT License, Copyright (c) 2021 OpenAI, +# and publicly available at https://github.com/openai/CLIP/. + +from collections import OrderedDict +from typing import Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import nn +from transformers import BertConfig, BertForMaskedLM + +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask: torch.Tensor = None, + use_gc=False): + super().__init__() + self.use_gc = use_gc + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + def forward(self, x: torch.Tensor): + if self.use_gc: + for each_block in self.resblocks: + x = checkpoint.checkpoint(each_block, x) + return x + else: + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + + def __init__(self, + input_resolution: int, + patch_size: int, + width: int, + layers: int, + heads: int, + output_dim: int, + use_gc=False): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads, use_gc=use_gc) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + class_embedding = self.class_embedding.to(x.dtype) + \ + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([class_embedding, x], + dim=1) # shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +class CLIPVisionWrapper(nn.Module): + + def __init__(self, ): + super().__init__() + self.vision_transformer = VisionTransformer( + input_resolution=224, + patch_size=14, + width=1024, + layers=24, + heads=16, + output_dim=768) + + def forward(self, x): + x = self.vision_transformer.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + class_embedding = self.vision_transformer.class_embedding.to(x.dtype) + \ + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([class_embedding, x], + dim=1) # shape = [*, grid ** 2 + 1, width] + x = x + self.vision_transformer.positional_embedding.to(x.dtype) + x = self.vision_transformer.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.vision_transformer.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x_tensor = x.clone() + x = self.vision_transformer.ln_post(x[:, 0, :]) + + if self.vision_transformer.proj is not None: + x = x @ self.vision_transformer.proj + + return x, x_tensor + + +class BertWrapper(nn.Module): + + def __init__(self, config_json, feat_dim, token_dim): + super(BertWrapper, self).__init__() + bert_config = BertConfig.from_json_file(config_json) + self.bert = BertForMaskedLM(bert_config).bert + + self.projector = nn.Linear(768, feat_dim, bias=False) + self.projector_token_embeds = nn.Linear(768, token_dim) + + def forward(self, input_ids, attention_mask): + trans_features = { + 'input_ids': input_ids, + 'attention_mask': attention_mask + } + output_states = self.bert(**trans_features, return_dict=False) + output_tokens = output_states[0] + + cls_tokens = output_tokens[:, 0, :] # CLS token is first token + + return self.projector(cls_tokens), self.projector_token_embeds( + output_tokens) + + +class Mlp(nn.Module): + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CrossLayer(nn.Module): + + def __init__(self, feat_dim, mlp_ratio): + super(CrossLayer, self).__init__() + self.norm1 = nn.LayerNorm(feat_dim) + self.norm2 = nn.LayerNorm(feat_dim) + self.norm3 = nn.LayerNorm(feat_dim) + + self.self_attn = nn.MultiheadAttention( + embed_dim=feat_dim, num_heads=16) + self.cross_attn = nn.MultiheadAttention( + embed_dim=feat_dim, num_heads=16) + self.ffn = Mlp( + in_features=feat_dim, + hidden_features=feat_dim * mlp_ratio, + drop=0.1) + + self.dropout1 = nn.Dropout(0.1) + self.dropout2 = nn.Dropout(0.1) + self.dropout3 = nn.Dropout(0.1) + + def forward(self, text_tensors, text_masks, image_tensors, + retrieved_tensors): + retrieved_tensors_res = self.norm1(retrieved_tensors) + retrieved_tensors_res = self.self_attn( + (text_tensors + retrieved_tensors_res).permute(1, 0, 2), + (text_tensors + retrieved_tensors_res).permute(1, 0, 2), + retrieved_tensors_res.permute(1, 0, 2), + key_padding_mask=(text_masks == 0), + )[0].permute(1, 0, 2) + retrieved_tensors = retrieved_tensors + self.dropout1( + retrieved_tensors_res) + + retrieved_tensors_res = self.norm2(retrieved_tensors) + retrieved_tensors_res = self.cross_attn( + (text_tensors + retrieved_tensors_res).permute(1, 0, 2), + image_tensors.permute(1, 0, 2), + image_tensors.permute(1, 0, 2))[0].permute(1, 0, 2) + retrieved_tensors = retrieved_tensors + self.dropout2( + retrieved_tensors_res) + + retrieved_tensors_res = self.norm3(retrieved_tensors) + retrieved_tensors = retrieved_tensors + self.dropout3( + self.ffn(retrieved_tensors_res)) + + return retrieved_tensors + + +class TEAM(nn.Module): + + def __init__(self, text_model, image_model, pretrained): + super(TEAM, self).__init__() + self.text_model = text_model + self.image_model = image_model + + self.cross_model = nn.ModuleList( + [CrossLayer(feat_dim=1024, mlp_ratio=2)]) + + self.image_tensor_fc = nn.Linear(1024, 768) + self.text_tensor_fc = nn.Linear(1024, 768) + + params = torch.load(pretrained, 'cpu') + compatible_position_ids(params, + 'text_model.bert.embeddings.position_ids') + self.load_state_dict(params, strict=True) + + def get_feature(self, text_data=None, text_mask=None, img_tensor=None): + if text_data is not None: + text_feature, text_tensors = self.text_model(text_data, text_mask) + text_feature = F.normalize(text_feature, p=2.0, dim=1) + else: + text_feature, text_tensors = None, None + + if img_tensor is not None: + image_feature, image_tensors = self.image_model(img_tensor) + image_feature = F.normalize(image_feature, p=2.0, dim=1) + else: + image_feature, image_tensors = None, None + + return text_feature, text_tensors, image_feature, image_tensors + + def get_cross_score(self, text_tensors, text_mask, image_tensors): + retrieved_tensors = torch.zeros_like(text_tensors) + pair_score_list = [] + text_tensors_proj = self.text_tensor_fc(text_tensors) + text_mask_float = text_mask.type(text_tensors_proj.dtype) + for each_cross_model in self.cross_model: + retrieved_tensors = each_cross_model(text_tensors, text_mask, + image_tensors, + retrieved_tensors) + retrieved_tensors_proj = self.image_tensor_fc(retrieved_tensors) + + pair_score = torch.sum( + F.normalize(retrieved_tensors_proj, p=2.0, dim=2) + * F.normalize(text_tensors_proj, p=2.0, dim=2), + dim=2) + pair_score_reduced = torch.sum( + pair_score * text_mask_float, dim=1) / torch.clamp( + torch.sum(text_mask_float, dim=1), min=1.0) + pair_score_list.append(pair_score_reduced) + return pair_score_list diff --git a/runtime/Lib/site_packages/models/multi_modal/video_synthesis/__init__.py b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7db72e7cd976d82babc96624abc6d9b30684c072 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .text_to_video_synthesis_model import TextToVideoSynthesis + +else: + _import_structure = { + 'text_to_video_synthesis_model': ['TextToVideoSynthesis'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/video_synthesis/autoencoder.py b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7885f2626d412edaaf5c3d3e55a09562f671f78d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/autoencoder.py @@ -0,0 +1,569 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly avaialbe at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['AutoencoderKL'] + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm( + num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +class DiagonalGaussianDistribution(object): + + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean).to(device=self.parameters.device) + + def sample(self): + x = self.mean + self.std * torch.randn( + self.mean.shape).to(device=self.parameters.device) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3]) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var - 1.0 - self.logvar + other.logvar, + dim=[1, 2, 3]) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + + torch.pow(sample - self.mean, 2) / self.var, + dim=dims) + + def mode(self): + return self.mean + + +class ResnetBlock(nn.Module): + + def __init__(self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm( + v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class Upsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x = torch.nn.functional.interpolate( + x, scale_factor=2.0, mode='nearest') + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class Encoder(nn.Module): + + def __init__(self, + *, + ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + **ignore_kwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + **ignorekwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2**(self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class AutoencoderKL(nn.Module): + + def __init__(self, + ddconfig, + embed_dim, + ckpt_path=None, + image_key='image', + colorize_nlabels=None, + monitor=None, + ema_decay=None, + learn_logvar=False): + super().__init__() + self.learn_logvar = learn_logvar + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d(2 * ddconfig['z_channels'], + 2 * embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + + self.use_ema = ema_decay is not None + + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path) + + def init_from_ckpt(self, path): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + + import collections + sd_new = collections.OrderedDict() + + for k in keys: + if k.find('first_stage_model') >= 0: + k_new = k.split('first_stage_model.')[-1] + sd_new[k_new] = sd[k] + + self.load_state_dict(sd_new, strict=True) + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + return x + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, log_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + if log_ema or self.use_ema: + with self.ema_scope(): + xrec_ema, posterior_ema = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec_ema.shape[1] > 3 + xrec_ema = self.to_rgb(xrec_ema) + log['samples_ema'] = self.decode( + torch.randn_like(posterior_ema.sample())) + log['reconstructions_ema'] = xrec_ema + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class IdentityFirstStage(torch.nn.Module): + + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/video_synthesis/diffusion.py b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..138fddae307107718e56ec3252baa6547b869e67 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/diffusion.py @@ -0,0 +1,228 @@ +# Part of the implementation is borrowed and modified from latent-diffusion, +# publicly avaialbe at https://github.com/CompVis/latent-diffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import torch + +__all__ = ['GaussianDiffusion', 'beta_schedule'] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + tensor = tensor.to(x.device) + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t].view(shape).to(x) + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + if schedule == 'linear_sd': + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +class GaussianDiffusion(object): + r""" Diffusion Model for DDIM. + "Denoising diffusion implicit models." by Song, Jiaming, Chenlin Meng, and Stefano Ermon. + See https://arxiv.org/abs/2010.02502 + """ + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + epsilon=1e-12, + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1', + 'charbonnier' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.epsilon = epsilon + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + dim = y_out.size(1) if self.var_type.startswith( + 'fixed') else y_out.size(1) // 2 + a = u_out[:, :dim] + b = guide_scale * (y_out[:, :dim] - u_out[:, :dim]) + c = y_out[:, dim:] + out = torch.cat([a + b, c], dim=1) + + # compute variance + if self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + r"""Sample from p(x_{t-1} | x_t) using DDIM. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) * eps + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / _i( + self.sqrt_recipm1_alphas_cumprod, t, xt) + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + a = (1 - alphas_prev) / (1 - alphas) + b = (1 - alphas / alphas_prev) + sigmas = eta * torch.sqrt(a * b) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0ec66069f320ab6a52c8d790b0f38ccc570534b8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py @@ -0,0 +1,245 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os +from os import path as osp +from typing import Any, Dict + +import open_clip +import torch +import torch.cuda.amp as amp +from einops import rearrange + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.video_synthesis.autoencoder import \ + AutoencoderKL +from modelscope.models.multi_modal.video_synthesis.diffusion import ( + GaussianDiffusion, beta_schedule) +from modelscope.models.multi_modal.video_synthesis.unet_sd import UNetSD +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['TextToVideoSynthesis'] + + +@MODELS.register_module( + Tasks.text_to_video_synthesis, module_name=Models.video_synthesis) +class TextToVideoSynthesis(Model): + r""" + task for text to video synthesis. + + Attributes: + sd_model: denosing model using in this task. + diffusion: diffusion model for DDIM. + autoencoder: decode the latent representation into visual space with VQGAN. + clip_encoder: encode the text into text embedding. + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + self.device = torch.device('cuda') if torch.cuda.is_available() \ + else torch.device('cpu') + self.config = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + cfg = self.config.model.model_cfg + cfg['temporal_attention'] = True if cfg[ + 'temporal_attention'] == 'True' else False + + # Initialize unet + self.sd_model = UNetSD( + in_dim=cfg['unet_in_dim'], + dim=cfg['unet_dim'], + y_dim=cfg['unet_y_dim'], + context_dim=cfg['unet_context_dim'], + out_dim=cfg['unet_out_dim'], + dim_mult=cfg['unet_dim_mult'], + num_heads=cfg['unet_num_heads'], + head_dim=cfg['unet_head_dim'], + num_res_blocks=cfg['unet_res_blocks'], + attn_scales=cfg['unet_attn_scales'], + dropout=cfg['unet_dropout'], + temporal_attention=cfg['temporal_attention']) + self.sd_model.load_state_dict( + torch.load( + osp.join(model_dir, self.config.model.model_args.ckpt_unet)), + strict=True) + self.sd_model.eval() + self.sd_model.to(self.device) + + # Initialize diffusion + betas = beta_schedule( + 'linear_sd', + cfg['num_timesteps'], + init_beta=0.00085, + last_beta=0.0120) + self.diffusion = GaussianDiffusion( + betas=betas, + mean_type=cfg['mean_type'], + var_type=cfg['var_type'], + loss_type=cfg['loss_type'], + rescale_timesteps=False) + + # Initialize autoencoder + ddconfig = { + 'double_z': True, + 'z_channels': 4, + 'resolution': 256, + 'in_channels': 3, + 'out_ch': 3, + 'ch': 128, + 'ch_mult': [1, 2, 4, 4], + 'num_res_blocks': 2, + 'attn_resolutions': [], + 'dropout': 0.0 + } + self.autoencoder = AutoencoderKL( + ddconfig, 4, + osp.join(model_dir, self.config.model.model_args.ckpt_autoencoder)) + if self.config.model.model_args.tiny_gpu == 1: + self.autoencoder.to('cpu') + else: + self.autoencoder.to(self.device) + self.autoencoder.eval() + + # Initialize Open clip + self.clip_encoder = FrozenOpenCLIPEmbedder( + version=osp.join(model_dir, + self.config.model.model_args.ckpt_clip), + layer='penultimate') + if self.config.model.model_args.tiny_gpu == 1: + self.clip_encoder.to('cpu') + else: + self.clip_encoder.to(self.device) + + def forward(self, input: Dict[str, Any]): + r""" + The entry function of text to image synthesis task. + 1. Using diffusion model to generate the video's latent representation. + 2. Using vqgan model (autoencoder) to decode the video's latent representation to visual space. + + Args: + input (`Dict[Str, Any]`): + The input of the task + Returns: + A generated video (as pytorch tensor). + """ + y = input['text_emb'] + zero_y = input['text_emb_zero'] + context = torch.cat([zero_y, y], dim=0).to(self.device) + # synthesis + with torch.no_grad(): + num_sample = 1 # here let b = 1 + max_frames = self.config.model.model_args.max_frames + latent_h, latent_w = input['out_height'] // 8, input[ + 'out_width'] // 8 + with amp.autocast(enabled=True): + x0 = self.diffusion.ddim_sample_loop( + noise=torch.randn(num_sample, 4, max_frames, latent_h, + latent_w).to( + self.device), # shape: b c f h w + model=self.sd_model, + model_kwargs=[{ + 'y': + context[1].unsqueeze(0).repeat(num_sample, 1, 1) + }, { + 'y': + context[0].unsqueeze(0).repeat(num_sample, 1, 1) + }], + guide_scale=9.0, + ddim_timesteps=50, + eta=0.0) + + scale_factor = 0.18215 + video_data = 1. / scale_factor * x0 + bs_vd = video_data.shape[0] + video_data = rearrange(video_data, 'b c f h w -> (b f) c h w') + self.autoencoder.to(self.device) + video_data = self.autoencoder.decode(video_data) + if self.config.model.model_args.tiny_gpu == 1: + self.autoencoder.to('cpu') + video_data = rearrange( + video_data, '(b f) c h w -> b c f h w', b=bs_vd) + return video_data.type(torch.float32).cpu() + + +class FrozenOpenCLIPEmbedder(torch.nn.Module): + """ + Uses the OpenCLIP transformer encoder for text + """ + LAYERS = ['last', 'penultimate'] + + def __init__(self, + arch='ViT-H-14', + version='open_clip_pytorch_model.bin', + device='cuda', + max_length=77, + freeze=True, + layer='last'): + super().__init__() + assert layer in self.LAYERS + model, _, _ = open_clip.create_model_and_transforms( + arch, device=torch.device('cpu'), pretrained=version) + del model.visual + self.model = model + + self.device = device + self.max_length = max_length + if freeze: + self.freeze() + self.layer = layer + if self.layer == 'last': + self.layer_idx = 0 + elif self.layer == 'penultimate': + self.layer_idx = 1 + else: + raise NotImplementedError() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + tokens = open_clip.tokenize(text) + z = self.encode_with_transformer(tokens.to(self.device)) + return z + + def encode_with_transformer(self, text): + x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model] + x = x + self.model.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.model.ln_final(x) + return x + + def text_transformer_forward(self, x: torch.Tensor, attn_mask=None): + for i, r in enumerate(self.model.transformer.resblocks): + if i == len(self.model.transformer.resblocks) - self.layer_idx: + break + x = r(x, attn_mask=attn_mask) + return x + + def encode(self, text): + return self(text) diff --git a/runtime/Lib/site_packages/models/multi_modal/video_synthesis/unet_sd.py b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/unet_sd.py new file mode 100644 index 0000000000000000000000000000000000000000..f3c764eb28aa383a7dd3191aba8e5c996a3c6af8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_synthesis/unet_sd.py @@ -0,0 +1,1098 @@ +# Part of the implementation is borrowed and modified from stable-diffusion, +# publicly avaialbe at https://github.com/Stability-AI/stablediffusion. +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat + +__all__ = ['UNetSD'] + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if callable(d) else d + + +class UNetSD(nn.Module): + + def __init__(self, + in_dim=7, + dim=512, + y_dim=512, + context_dim=512, + out_dim=6, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + use_scale_shift_norm=True, + dropout=0.1, + temporal_attn_times=2, + temporal_attention=True, + use_checkpoint=False, + use_image_dataset=False, + use_fps_condition=False, + use_sim_mask=False): + embed_dim = dim * 4 + num_heads = num_heads if num_heads else dim // 32 + super(UNetSD, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + self.num_heads = num_heads + # parameters for spatial/temporal attention + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.use_scale_shift_norm = use_scale_shift_norm + self.temporal_attn_times = temporal_attn_times + self.temporal_attention = temporal_attention + self.use_checkpoint = use_checkpoint + self.use_image_dataset = use_image_dataset + self.use_fps_condition = use_fps_condition + self.use_sim_mask = use_sim_mask + use_linear_in_temporal = False + transformer_depth = 1 + disabled_sa = False + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embed = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + + if self.use_fps_condition: + self.fps_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + nn.init.zeros_(self.fps_embedding[-1].weight) + nn.init.zeros_(self.fps_embedding[-1].bias) + + # encoder + self.input_blocks = nn.ModuleList() + init_block = nn.ModuleList([nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + + if temporal_attention: + init_block.append( + TemporalTransformer( + dim, + num_heads, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + + self.input_blocks.append(init_block) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList([ + ResBlock( + in_dim, + embed_dim, + dropout, + out_channels=out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + + in_dim = out_dim + self.input_blocks.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + downsample = Downsample( + out_dim, True, dims=2, out_channels=out_dim) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.input_blocks.append(downsample) + + # middle + self.middle_block = nn.ModuleList([ + ResBlock( + out_dim, + embed_dim, + dropout, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ), + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True) + ]) + + if self.temporal_attention: + self.middle_block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset, + )) + + self.middle_block.append( + ResBlock( + out_dim, + embed_dim, + dropout, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + )) + + # decoder + self.output_blocks = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + # residual (+attention) blocks + block = nn.ModuleList([ + ResBlock( + in_dim + shortcut_dims.pop(), + embed_dim, + dropout, + out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=1024, + disable_self_attn=False, + use_linear=True)) + + if self.temporal_attention: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + upsample = Upsample( + out_dim, True, dims=2.0, out_channels=out_dim) + scale *= 2.0 + block.append(upsample) + self.output_blocks.append(block) + + # head + self.out = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.out[-1].weight) + + def forward( + self, + x, + t, + y, + fps=None, + video_mask=None, + focus_present_mask=None, + prob_focus_present=0., + mask_last_frame_num=0 # mask last frame num + ): + """ + prob_focus_present: probability at which a given batch sample will focus on the present + (0. is all off, 1. is completely arrested attention across time) + """ + batch, device = x.shape[0], x.device + self.batch = batch + + # image and video joint training, if mask_last_frame_num is set, prob_focus_present will be ignored + if mask_last_frame_num > 0: + focus_present_mask = None + video_mask[-mask_last_frame_num:] = False + else: + focus_present_mask = default( + focus_present_mask, lambda: prob_mask_like( + (batch, ), prob_focus_present, device=device)) + + time_rel_pos_bias = None + # embeddings + if self.use_fps_condition and fps is not None: + e = self.time_embed(sinusoidal_embedding( + t, self.dim)) + self.fps_embedding( + sinusoidal_embedding(fps, self.dim)) + else: + e = self.time_embed(sinusoidal_embedding(t, self.dim)) + context = y + + # repeat f times for spatial e and context + f = x.shape[2] + e = e.repeat_interleave(repeats=f, dim=0) + context = context.repeat_interleave(repeats=f, dim=0) + + # always in shape (b f) c h w, except for temporal layer + x = rearrange(x, 'b c f h w -> (b f) c h w') + # encoder + xs = [] + for block in self.input_blocks: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + xs.append(x) + + # middle + for block in self.middle_block: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + + # decoder + for block in self.output_blocks: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single( + block, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=xs[-1] if len(xs) > 0 else None) + + # head + x = self.out(x) + # reshape back to (b c f h w) + x = rearrange(x, '(b f) c h w -> b c f h w', b=batch) + return x + + def _forward_single(self, + module, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=None): + if isinstance(module, ResidualBlock): + x = x.contiguous() + x = module(x, e, reference) + elif isinstance(module, ResBlock): + x = x.contiguous() + x = module(x, e, self.batch) + elif isinstance(module, SpatialTransformer): + x = module(x, context) + elif isinstance(module, TemporalTransformer): + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, context) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, CrossAttention): + x = module(x, context) + elif isinstance(module, BasicTransformerBlock): + x = module(x, context) + elif isinstance(module, FeedForward): + x = module(x, context) + elif isinstance(module, Upsample): + x = module(x) + elif isinstance(module, Downsample): + x = module(x) + elif isinstance(module, Resample): + x = module(x, reference) + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context, + time_rel_pos_bias, focus_present_mask, + video_mask, reference) + else: + x = module(x) + return x + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +class CrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + del q, k + + if exists(mask): + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + sim = sim.softmax(dim=-1) + + out = torch.einsum('b i j, b j d -> b i d', sim, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + return self.to_out(out) + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data in spatial axis. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + NEW: use_linear for more efficiency instead of the 1x1 convs + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True): + super().__init__() + if exists(context_dim) and not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + disable_self_attn=disable_self_attn, + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if not isinstance(context, list): + context = [context] + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + if not self.use_linear: + x = self.proj_in(x) + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + if self.use_linear: + x = self.proj_in(x) + for i, block in enumerate(self.transformer_blocks): + x = block(x, context=context[i]) + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = self.proj_out(x) + return x + x_in + + +class TemporalTransformer(nn.Module): + """ + Transformer block for image-like data in temporal axis. + First, reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True, + only_self_att=True, + multiply_zero=False): + super().__init__() + self.multiply_zero = multiply_zero + self.only_self_att = only_self_att + if self.only_self_att: + context_dim = None + if not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv1d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv1d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if self.only_self_att: + context = None + if not isinstance(context, list): + context = [context] + b, c, f, h, w = x.shape + x_in = x + x = self.norm(x) + + if not self.use_linear: + x = rearrange(x, 'b c f h w -> (b h w) c f').contiguous() + x = self.proj_in(x) + if self.use_linear: + x = rearrange( + x, '(b f) c h w -> b (h w) f c', f=self.frames).contiguous() + x = self.proj_in(x) + + if self.only_self_att: + x = rearrange(x, 'bhw c f -> bhw f c').contiguous() + for i, block in enumerate(self.transformer_blocks): + x = block(x) + x = rearrange(x, '(b hw) f c -> b hw f c', b=b).contiguous() + else: + x = rearrange(x, '(b hw) c f -> b hw f c', b=b).contiguous() + for i, block in enumerate(self.transformer_blocks): + context[i] = rearrange( + context[i], '(b f) l con -> b f l con', + f=self.frames).contiguous() + # calculate each batch one by one (since number in shape could not greater then 65,535 for some package) + for j in range(b): + context_i_j = repeat( + context[i][j], + 'f l con -> (f r) l con', + r=(h * w) // self.frames, + f=self.frames).contiguous() + x[j] = block(x[j], context=context_i_j) + + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) f c -> b f c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = rearrange(x, 'b hw f c -> (b hw) c f').contiguous() + x = self.proj_out(x) + x = rearrange( + x, '(b h w) c f -> b c f h w', b=b, h=h, w=w).contiguous() + + if self.multiply_zero: + x = 0.0 * x + x_in + else: + x = x + x_in + return x + + +class BasicTransformerBlock(nn.Module): + + def __init__(self, + dim, + n_heads, + d_head, + dropout=0., + context_dim=None, + gated_ff=True, + checkpoint=True, + disable_self_attn=False): + super().__init__() + attn_cls = CrossAttention + self.disable_self_attn = disable_self_attn + self.attn1 = attn_cls( + query_dim=dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + context_dim=context_dim if self.disable_self_attn else + None) # is a self-attention if not self.disable_self_attn + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = attn_cls( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout) # is self-attn if context is none + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward(self, x, context=None): + x = self.attn1( + self.norm1(x), + context=context if self.disable_self_attn else None) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +# feedforward +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = nn.Conv2d( + self.channels, self.out_channels, 3, padding=padding) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + if self.use_conv: + x = self.conv(x) + return x + + +class ResBlock(nn.Module): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + :param use_temporal_conv: if True, use the temporal convolution. + :param use_image_dataset: if True, the temporal parameters will not be optimized. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + up=False, + down=False, + use_temporal_conv=True, + use_image_dataset=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_scale_shift_norm = use_scale_shift_norm + self.use_temporal_conv = use_temporal_conv + + self.in_layers = nn.Sequential( + nn.GroupNorm(32, channels), + nn.SiLU(), + nn.Conv2d(channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + nn.Linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + nn.GroupNorm(32, self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + nn.Conv2d(self.out_channels, self.out_channels, 3, padding=1)), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = nn.Conv2d(channels, self.out_channels, 1) + + if self.use_temporal_conv: + self.temopral_conv = TemporalConvBlock_v2( + self.out_channels, + self.out_channels, + dropout=0.1, + use_image_dataset=use_image_dataset) + + def forward(self, x, emb, batch_size): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return self._forward(x, emb, batch_size) + + def _forward(self, x, emb, batch_size): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = torch.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + h = self.skip_connection(x) + h + + if self.use_temporal_conv: + h = rearrange(h, '(b f) c h w -> b c f h w', b=batch_size) + h = self.temopral_conv(h) + h = rearrange(h, 'b c f h w -> (b f) c h w') + return h + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if self.use_conv: + self.op = nn.Conv2d( + self.channels, + self.out_channels, + 3, + stride=stride, + padding=padding) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, mode): + assert mode in ['none', 'upsample', 'downsample'] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.mode = mode + + def forward(self, x, reference=None): + if self.mode == 'upsample': + assert reference is not None + x = F.interpolate(x, size=reference.shape[-2:], mode='nearest') + elif self.mode == 'downsample': + x = F.adaptive_avg_pool2d( + x, output_size=tuple(u // 2 for u in x.shape[-2:])) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + mode='none', + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.mode = mode + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, mode) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e, reference=None): + identity = self.resample(x, reference) + x = self.layer1[-1](self.resample(self.layer1[:-1](x), reference)) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + # output + x = self.proj(x) + return x + identity + + +class TemporalConvBlock_v2(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock_v2, self).__init__() + if out_dim is None: + out_dim = in_dim # int(1.5*in_dim) + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv3 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv4 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv4[-1].weight) + nn.init.zeros_(self.conv4[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + + if self.use_image_dataset: + x = identity + 0.0 * x + else: + x = identity + x + return x + + +def prob_mask_like(shape, prob, device): + if prob == 1: + return torch.ones(shape, device=device, dtype=torch.bool) + elif prob == 0: + return torch.zeros(shape, device=device, dtype=torch.bool) + else: + mask = torch.zeros(shape, device=device).float().uniform_(0, 1) < prob + # aviod mask all, which will cause find_unused_parameters error + if mask.all(): + mask[0] = False + return mask + + +def conv_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f'unsupported dimensions: {dims}') + + +def avg_pool_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D average pooling module. + """ + if dims == 1: + return nn.AvgPool1d(*args, **kwargs) + elif dims == 2: + return nn.AvgPool2d(*args, **kwargs) + elif dims == 3: + return nn.AvgPool3d(*args, **kwargs) + raise ValueError(f'unsupported dimensions: {dims}') diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/__init__.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e10a8eb0b02997c176c896696f0fba33c679d72 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .video_to_video_model import VideoToVideo + +else: + _import_structure = { + 'video_to_video_model': ['VideoToVideo'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/__init__.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6c88231819a0af0cf15df8bf4b6483dcc37379eb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .autoencoder import * +from .embedder import * +from .unet_v2v import * diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/autoencoder.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..714a89536f42f843ca890faa3b32c00697ea770b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/autoencoder.py @@ -0,0 +1,590 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import collections + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm( + num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +@torch.no_grad() +def get_first_stage_encoding(encoder_posterior): + scale_factor = 0.18215 + if isinstance(encoder_posterior, DiagonalGaussianDistribution): + z = encoder_posterior.sample() + elif isinstance(encoder_posterior, torch.Tensor): + z = encoder_posterior + else: + raise NotImplementedError( + f"encoder_posterior of type '{type(encoder_posterior)}' not yet implemented" + ) + return scale_factor * z + + +class DiagonalGaussianDistribution(object): + + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean).to(device=self.parameters.device) + + def sample(self): + x = self.mean + self.std * torch.randn( + self.mean.shape).to(device=self.parameters.device) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3]) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var - 1.0 - self.logvar + other.logvar, + dim=[1, 2, 3]) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + + torch.pow(sample - self.mean, 2) / self.var, + dim=dims) + + def mode(self): + return self.mean + + +class ResnetBlock(nn.Module): + + def __init__(self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) + k = k.reshape(b, c, h * w) + w_ = torch.bmm(q, k) + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) + h_ = torch.bmm(v, w_) + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class Upsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x = torch.nn.functional.interpolate( + x, scale_factor=2.0, mode='nearest') + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class Encoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + use_linear_attn=False, + attn_type='vanilla', + **ignore_kwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type='vanilla', + **ignorekwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2**(self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + logger.info('Working with z of shape {} = {} dimensions.'.format( + self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class AutoencoderKL(nn.Module): + + def __init__(self, + ddconfig, + embed_dim, + pretrained=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + ema_decay=None, + learn_logvar=False, + **kwargs): + super().__init__() + self.learn_logvar = learn_logvar + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d(2 * ddconfig['z_channels'], + 2 * embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + + self.use_ema = ema_decay is not None + + if pretrained is not None: + self.init_from_ckpt(pretrained, ignore_keys=ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + sd_new = collections.OrderedDict() + for k in keys: + if k.find('first_stage_model') >= 0: + k_new = k.split('first_stage_model.')[-1] + sd_new[k_new] = sd[k] + self.load_state_dict(sd_new, strict=True) + logger.info(f'Restored from {path}') + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + return x + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, log_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + if log_ema or self.use_ema: + with self.ema_scope(): + xrec_ema, posterior_ema = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec_ema.shape[1] > 3 + xrec_ema = self.to_rgb(xrec_ema) + log['samples_ema'] = self.decode( + torch.randn_like(posterior_ema.sample())) + log['reconstructions_ema'] = xrec_ema + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class IdentityFirstStage(torch.nn.Module): + + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/embedder.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8889a680cbe524ffdd8e70b68b0caa2c72e335 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/embedder.py @@ -0,0 +1,76 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import numpy as np +import open_clip +import torch +import torch.nn as nn +import torchvision.transforms as T + + +class FrozenOpenCLIPEmbedder(nn.Module): + """ + Uses the OpenCLIP transformer encoder for text + """ + LAYERS = ['last', 'penultimate'] + + def __init__(self, + pretrained, + arch='ViT-H-14', + device='cuda', + max_length=77, + freeze=True, + layer='penultimate'): + super().__init__() + assert layer in self.LAYERS + model, _, preprocess = open_clip.create_model_and_transforms( + arch, device=torch.device('cpu'), pretrained=pretrained) + + del model.visual + self.model = model + self.device = device + self.max_length = max_length + + if freeze: + self.freeze() + self.layer = layer + if self.layer == 'last': + self.layer_idx = 0 + elif self.layer == 'penultimate': + self.layer_idx = 1 + else: + raise NotImplementedError() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + tokens = open_clip.tokenize(text) + z = self.encode_with_transformer(tokens.to(self.device)) + return z + + def encode_with_transformer(self, text): + x = self.model.token_embedding(text) + x = x + self.model.positional_embedding + x = x.permute(1, 0, 2) + x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) + x = x.permute(1, 0, 2) + x = self.model.ln_final(x) + return x + + def text_transformer_forward(self, x: torch.Tensor, attn_mask=None): + for i, r in enumerate(self.model.transformer.resblocks): + if i == len(self.model.transformer.resblocks) - self.layer_idx: + break + if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting( + ): + x = checkpoint(r, x, attn_mask) + else: + x = r(x, attn_mask=attn_mask) + return x + + def encode(self, text): + return self(text) diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/unet_v2v.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/unet_v2v.py new file mode 100644 index 0000000000000000000000000000000000000000..43f19f90858958b9acce0e4afa11bce319751e4b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/modules/unet_v2v.py @@ -0,0 +1,1542 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import xformers +import xformers.ops +from einops import rearrange +from fairscale.nn.checkpoint import checkpoint_wrapper +from rotary_embedding_torch import RotaryEmbedding + +USE_TEMPORAL_TRANSFORMER = True + + +class DropPath(nn.Module): + r"""DropPath but without rescaling and supports optional all-zero and/or all-keep. + """ + + def __init__(self, p): + super(DropPath, self).__init__() + self.p = p + + def forward(self, *args, zero=None, keep=None): + if not self.training: + return args[0] if len(args) == 1 else args + + # params + x = args[0] + b = x.size(0) + n = (torch.rand(b) < self.p).sum() + + # non-zero and non-keep mask + mask = x.new_ones(b, dtype=torch.bool) + if keep is not None: + mask[keep] = False + if zero is not None: + mask[zero] = False + + # drop-path index + index = torch.where(mask)[0] + index = index[torch.randperm(len(index))[:n]] + if zero is not None: + index = torch.cat([index, torch.where(zero)[0]], dim=0) + + # drop-path multiplier + multiplier = x.new_ones(b) + multiplier[index] = 0.0 + output = tuple(u * self.broadcast(multiplier, u) for u in args) + return output[0] if len(args) == 1 else output + + def broadcast(self, src, dst): + assert src.size(0) == dst.size(0) + shape = (dst.size(0), ) + (1, ) * (dst.ndim - 1) + return src.view(shape) + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if callable(d) else d + + +def prob_mask_like(shape, prob, device): + if prob == 1: + return torch.ones(shape, device=device, dtype=torch.bool) + elif prob == 0: + return torch.zeros(shape, device=device, dtype=torch.bool) + else: + mask = torch.zeros(shape, device=device).float().uniform_(0, 1) < prob + # aviod mask all, which will cause find_unused_parameters error + if mask.all(): + mask[0] = False + return mask + + +class MemoryEfficientCrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + max_bs=16384, + dropout=0.0): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.max_bs = max_bs + self.heads = heads + self.dim_head = dim_head + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + self.attention_op: Optional[Any] = None + + def forward(self, x, context=None, mask=None): + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + b, _, _ = q.shape + q, k, v = map( + lambda t: t.unsqueeze(3).reshape(b, t.shape[ + 1], self.heads, self.dim_head).permute(0, 2, 1, 3).reshape( + b * self.heads, t.shape[1], self.dim_head).contiguous(), + (q, k, v), + ) + + # actually compute the attention, what we cannot get enough of. + if q.shape[0] > self.max_bs: + q_list = torch.chunk(q, q.shape[0] // self.max_bs, dim=0) + k_list = torch.chunk(k, k.shape[0] // self.max_bs, dim=0) + v_list = torch.chunk(v, v.shape[0] // self.max_bs, dim=0) + out_list = [] + for q_1, k_1, v_1 in zip(q_list, k_list, v_list): + out = xformers.ops.memory_efficient_attention( + q_1, k_1, v_1, attn_bias=None, op=self.attention_op) + out_list.append(out) + out = torch.cat(out_list, dim=0) + else: + out = xformers.ops.memory_efficient_attention( + q, k, v, attn_bias=None, op=self.attention_op) + + if exists(mask): + raise NotImplementedError + out = ( + out.unsqueeze(0).reshape( + b, self.heads, out.shape[1], + self.dim_head).permute(0, 2, 1, + 3).reshape(b, out.shape[1], + self.heads * self.dim_head)) + return self.to_out(out) + + +class RelativePositionBias(nn.Module): + + def __init__(self, heads=8, num_buckets=32, max_distance=128): + super().__init__() + self.num_buckets = num_buckets + self.max_distance = max_distance + self.relative_attention_bias = nn.Embedding(num_buckets, heads) + + @staticmethod + def _relative_position_bucket(relative_position, + num_buckets=32, + max_distance=128): + ret = 0 + n = -relative_position + + num_buckets //= 2 + ret += (n < 0).long() * num_buckets + n = torch.abs(n) + + max_exact = num_buckets // 2 + is_small = n < max_exact + + val_if_large = max_exact + ( + torch.log(n.float() / max_exact) + / math.log(max_distance / max_exact) * # noqa + (num_buckets - max_exact)).long() + val_if_large = torch.min( + val_if_large, torch.full_like(val_if_large, num_buckets - 1)) + + ret += torch.where(is_small, n, val_if_large) + return ret + + def forward(self, n, device): + q_pos = torch.arange(n, dtype=torch.long, device=device) + k_pos = torch.arange(n, dtype=torch.long, device=device) + rel_pos = rearrange(k_pos, 'j -> 1 j') - rearrange(q_pos, 'i -> i 1') + rp_bucket = self._relative_position_bucket( + rel_pos, + num_buckets=self.num_buckets, + max_distance=self.max_distance) + values = self.relative_attention_bias(rp_bucket) + return rearrange(values, 'i j h -> h i j') + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + NEW: use_linear for more efficiency instead of the 1x1 convs + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True): + super().__init__() + if exists(context_dim) and not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + disable_self_attn=disable_self_attn, + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if not isinstance(context, list): + context = [context] + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + if not self.use_linear: + x = self.proj_in(x) + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + if self.use_linear: + x = self.proj_in(x) + for i, block in enumerate(self.transformer_blocks): + x = block(x, context=context[i]) + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = self.proj_out(x) + return x + x_in + + +_ATTN_PRECISION = os.environ.get('ATTN_PRECISION', 'fp32') + + +class CrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + + # force cast to fp32 to avoid overflowing + if _ATTN_PRECISION == 'fp32': + with torch.autocast(enabled=False, device_type='cuda'): + q, k = q.float(), k.float() + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + else: + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + + del q, k + + if exists(mask): + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + sim = sim.softmax(dim=-1) + + out = torch.einsum('b i j, b j d -> b i d', sim, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + return self.to_out(out) + + +class BasicTransformerBlock(nn.Module): + + def __init__(self, + dim, + n_heads, + d_head, + dropout=0., + context_dim=None, + gated_ff=True, + checkpoint=True, + disable_self_attn=False): + super().__init__() + attn_cls = MemoryEfficientCrossAttention + self.disable_self_attn = disable_self_attn + self.attn1 = attn_cls( + query_dim=dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + context_dim=context_dim if self.disable_self_attn else None) + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = attn_cls( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout) + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward_(self, x, context=None): + return checkpoint(self._forward, (x, context), self.parameters(), + self.checkpoint) + + def forward(self, x, context=None): + x = self.attn1( + self.norm1(x), + context=context if self.disable_self_attn else None) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +# feedforward +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = nn.Conv2d( + self.channels, self.out_channels, 3, padding=padding) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + x = x[..., 1:-1, :] + if self.use_conv: + x = self.conv(x) + return x + + +class ResBlock(nn.Module): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + up=False, + down=False, + use_temporal_conv=True, + use_image_dataset=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_scale_shift_norm = use_scale_shift_norm + self.use_temporal_conv = use_temporal_conv + + self.in_layers = nn.Sequential( + nn.GroupNorm(32, channels), + nn.SiLU(), + nn.Conv2d(channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + nn.Linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + nn.GroupNorm(32, self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + nn.Conv2d(self.out_channels, self.out_channels, 3, padding=1)), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = nn.Conv2d(channels, self.out_channels, 1) + + if self.use_temporal_conv: + self.temopral_conv = TemporalConvBlock_v2( + self.out_channels, + self.out_channels, + dropout=0.1, + use_image_dataset=use_image_dataset) + + def forward(self, x, emb, batch_size): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return self._forward(x, emb, batch_size) + + def _forward(self, x, emb, batch_size): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + h = self.skip_connection(x) + h + + if self.use_temporal_conv: + h = rearrange(h, '(b f) c h w -> b c f h w', b=batch_size) + h = self.temopral_conv(h) + h = rearrange(h, 'b c f h w -> (b f) c h w') + return h + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=(2, 1)): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = nn.Conv2d( + self.channels, + self.out_channels, + 3, + stride=stride, + padding=padding) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, mode): + assert mode in ['none', 'upsample', 'downsample'] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.mode = mode + + def forward(self, x, reference=None): + if self.mode == 'upsample': + assert reference is not None + x = F.interpolate(x, size=reference.shape[-2:], mode='nearest') + elif self.mode == 'downsample': + x = F.adaptive_avg_pool2d( + x, output_size=tuple(u // 2 for u in x.shape[-2:])) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + mode='none', + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.mode = mode + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, mode) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e, reference=None): + identity = self.resample(x, reference) + x = self.layer1[-1](self.resample(self.layer1[:-1](x), reference)) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class TemporalAttentionBlock(nn.Module): + + def __init__(self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False): + super().__init__() + # consider num_heads first, as pos_bias needs fixed num_heads + dim_head = dim // heads + assert heads * dim_head == dim + self.use_image_dataset = use_image_dataset + self.use_sim_mask = use_sim_mask + + self.scale = dim_head**-0.5 + self.heads = heads + hidden_dim = dim_head * heads + + self.norm = nn.GroupNorm(32, dim) + self.rotary_emb = rotary_emb + self.to_qkv = nn.Linear(dim, hidden_dim * 3) + self.to_out = nn.Linear(hidden_dim, dim) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + + identity = x + n, height, device = x.shape[2], x.shape[-2], x.device + + x = self.norm(x) + x = rearrange(x, 'b c f h w -> b (h w) f c') + + qkv = self.to_qkv(x).chunk(3, dim=-1) + + if exists(focus_present_mask) and focus_present_mask.all(): + # if all batch samples are focusing on present + # it would be equivalent to passing that token's values (v=qkv[-1]) through to the output + values = qkv[-1] + out = self.to_out(values) + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + return out + identity + + # split out heads + q = rearrange(qkv[0], '... n (h d) -> ... h n d', h=self.heads) + k = rearrange(qkv[1], '... n (h d) -> ... h n d', h=self.heads) + v = rearrange(qkv[2], '... n (h d) -> ... h n d', h=self.heads) + + # scale + + q = q * self.scale + + # rotate positions into queries and keys for time attention + if exists(self.rotary_emb): + q = self.rotary_emb.rotate_queries_or_keys(q) + k = self.rotary_emb.rotate_queries_or_keys(k) + + # similarity + # shape [b (hw) h n n], n=f + sim = torch.einsum('... h i d, ... h j d -> ... h i j', q, k) + + # relative positional bias + + if exists(pos_bias): + sim = sim + pos_bias + + if (focus_present_mask is None and video_mask is not None): + # video_mask: [B, n] + mask = video_mask[:, None, :] * video_mask[:, :, None] + mask = mask.unsqueeze(1).unsqueeze(1) + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + elif exists(focus_present_mask) and not (~focus_present_mask).all(): + attend_all_mask = torch.ones((n, n), + device=device, + dtype=torch.bool) + attend_self_mask = torch.eye(n, device=device, dtype=torch.bool) + + mask = torch.where( + rearrange(focus_present_mask, 'b -> b 1 1 1 1'), + rearrange(attend_self_mask, 'i j -> 1 1 1 i j'), + rearrange(attend_all_mask, 'i j -> 1 1 1 i j'), + ) + + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + + if self.use_sim_mask: + sim_mask = torch.tril( + torch.ones((n, n), device=device, dtype=torch.bool), + diagonal=0) + sim = sim.masked_fill(~sim_mask, -torch.finfo(sim.dtype).max) + + # numerical stability + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + # aggregate values + + out = torch.einsum('... h i j, ... h j d -> ... h i d', attn, v) + out = rearrange(out, '... h n d -> ... n (h d)') + out = self.to_out(out) + + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + if self.use_image_dataset: + out = identity + 0 * out + else: + out = identity + out + return out + + +class TemporalTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True, + only_self_att=True, + multiply_zero=False): + super().__init__() + self.multiply_zero = multiply_zero + self.only_self_att = only_self_att + self.use_adaptor = False + if self.only_self_att: + context_dim = None + if not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv1d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + if self.use_adaptor: + self.adaptor_in = nn.Linear(frames, frames) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv1d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + if self.use_adaptor: + self.adaptor_out = nn.Linear(frames, frames) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if self.only_self_att: + context = None + if not isinstance(context, list): + context = [context] + b, c, f, h, w = x.shape + x_in = x + x = self.norm(x) + + if not self.use_linear: + x = rearrange(x, 'b c f h w -> (b h w) c f').contiguous() + x = self.proj_in(x) + # [16384, 16, 320] + if self.use_linear: + x = rearrange( + x, '(b f) c h w -> b (h w) f c', f=self.frames).contiguous() + x = self.proj_in(x) + + if self.only_self_att: + x = rearrange(x, 'bhw c f -> bhw f c').contiguous() + for i, block in enumerate(self.transformer_blocks): + x = block(x) + x = rearrange(x, '(b hw) f c -> b hw f c', b=b).contiguous() + else: + x = rearrange(x, '(b hw) c f -> b hw f c', b=b).contiguous() + for i, block in enumerate(self.transformer_blocks): + context[i] = rearrange( + context[i], '(b f) l con -> b f l con', + f=self.frames).contiguous() + # calculate each batch one by one (since number in shape could not greater then 65,535 for some package) + for j in range(b): + context_i_j = repeat( + context[i][j], + 'f l con -> (f r) l con', + r=(h * w) // self.frames, + f=self.frames).contiguous() + x[j] = block(x[j], context=context_i_j) + + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) f c -> b f c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = rearrange(x, 'b hw f c -> (b hw) c f').contiguous() + x = self.proj_out(x) + x = rearrange( + x, '(b h w) c f -> b c f h w', b=b, h=h, w=w).contiguous() + + if self.multiply_zero: + x = 0.0 * x + x_in + else: + x = x + x_in + return x + + +class TemporalAttentionMultiBlock(nn.Module): + + def __init__( + self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False, + temporal_attn_times=1, + ): + super().__init__() + self.att_layers = nn.ModuleList([ + TemporalAttentionBlock(dim, heads, dim_head, rotary_emb, + use_image_dataset, use_sim_mask) + for _ in range(temporal_attn_times) + ]) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + for layer in self.att_layers: + x = layer(x, pos_bias, focus_present_mask, video_mask) + return x + + +class InitTemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(InitTemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv[-1].weight) + nn.init.zeros_(self.conv[-1].bias) + + def forward(self, x): + identity = x + x = self.conv(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv2[-1].weight) + nn.init.zeros_(self.conv2[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock_v2(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock_v2, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv3 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv4 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv4[-1].weight) + nn.init.zeros_(self.conv4[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + + if self.use_image_dataset: + x = identity + 0.0 * x + else: + x = identity + x + return x + + +class Vid2VidSDUNet(nn.Module): + + def __init__(self, + in_dim=4, + dim=320, + y_dim=1024, + context_dim=1024, + out_dim=4, + dim_mult=[1, 2, 4, 4], + num_heads=8, + head_dim=64, + num_res_blocks=2, + attn_scales=[1 / 1, 1 / 2, 1 / 4], + use_scale_shift_norm=True, + dropout=0.1, + temporal_attn_times=1, + temporal_attention=True, + use_checkpoint=True, + use_image_dataset=False, + use_fps_condition=False, + use_sim_mask=False, + training=False, + inpainting=True): + embed_dim = dim * 4 + num_heads = num_heads if num_heads else dim // 32 + super(Vid2VidSDUNet, self).__init__() + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + # for temporal attention + self.num_heads = num_heads + # for spatial attention + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.use_scale_shift_norm = use_scale_shift_norm + self.temporal_attn_times = temporal_attn_times + self.temporal_attention = temporal_attention + self.use_checkpoint = use_checkpoint + self.use_image_dataset = use_image_dataset + self.use_fps_condition = use_fps_condition + self.use_sim_mask = use_sim_mask + self.training = training + self.inpainting = inpainting + + use_linear_in_temporal = False + transformer_depth = 1 + disabled_sa = False + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + self.time_embed = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + + if temporal_attention and not USE_TEMPORAL_TRANSFORMER: + self.rotary_emb = RotaryEmbedding(min(32, head_dim)) + self.time_rel_pos_bias = RelativePositionBias( + heads=num_heads, max_distance=32) + + if self.use_fps_condition: + self.fps_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + nn.init.zeros_(self.fps_embedding[-1].weight) + nn.init.zeros_(self.fps_embedding[-1].bias) + + # encoder + self.input_blocks = nn.ModuleList() + init_block = nn.ModuleList([nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + # need an initial temporal attention? + if temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + init_block.append( + TemporalTransformer( + dim, + num_heads, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + init_block.append( + TemporalAttentionMultiBlock( + dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + temporal_attn_times=temporal_attn_times, + use_image_dataset=use_image_dataset)) + self.input_blocks.append(init_block) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + block = nn.ModuleList([ + ResBlock( + in_dim, + embed_dim, + dropout, + out_channels=out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + self.input_blocks.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + downsample = Downsample( + out_dim, True, dims=2, out_channels=out_dim) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.input_blocks.append(downsample) + + self.middle_block = nn.ModuleList([ + ResBlock( + out_dim, + embed_dim, + dropout, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ), + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True) + ]) + + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + self.middle_block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset, + )) + else: + self.middle_block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + + self.middle_block.append( + ResBlock(out_dim, embed_dim, dropout, use_scale_shift_norm=False)) + + # decoder + self.output_blocks = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + block = nn.ModuleList([ + ResBlock( + in_dim + shortcut_dims.pop(), + embed_dim, + dropout, + out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=1024, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + upsample = Upsample( + out_dim, True, dims=2.0, out_channels=out_dim) + scale *= 2.0 + block.append(upsample) + self.output_blocks.append(block) + + # head + self.out = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.out[-1].weight) + + def forward(self, + x, + t, + y, + x_lr=None, + fps=None, + video_mask=None, + focus_present_mask=None, + prob_focus_present=0., + mask_last_frame_num=0): + + batch, x_c, x_f, x_h, x_w = x.shape + device = x.device + self.batch = batch + + # image and video joint training, if mask_last_frame_num is set, prob_focus_present will be ignored + if mask_last_frame_num > 0: + focus_present_mask = None + video_mask[-mask_last_frame_num:] = False + else: + focus_present_mask = default( + focus_present_mask, lambda: prob_mask_like( + (batch, ), prob_focus_present, device=device)) + + if self.temporal_attention and not USE_TEMPORAL_TRANSFORMER: + time_rel_pos_bias = self.time_rel_pos_bias( + x.shape[2], device=x.device) + else: + time_rel_pos_bias = None + + # embeddings + e = self.time_embed(sinusoidal_embedding(t, self.dim)) + context = y + + # repeat f times for spatial e and context + e = e.repeat_interleave(repeats=x_f, dim=0) + context = context.repeat_interleave(repeats=x_f, dim=0) + + # always in shape (b f) c h w, except for temporal layer + x = rearrange(x, 'b c f h w -> (b f) c h w') + # encoder + xs = [] + for block in self.input_blocks: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + xs.append(x) + + # middle + for block in self.middle_block: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + + # decoder + for block in self.output_blocks: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single( + block, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=xs[-1] if len(xs) > 0 else None) + + # head + x = self.out(x) + + # reshape back to (b c f h w) + x = rearrange(x, '(b f) c h w -> b c f h w', b=batch) + return x + + def _forward_single(self, + module, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=None): + if isinstance(module, ResidualBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, reference) + elif isinstance(module, ResBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, self.batch) + elif isinstance(module, SpatialTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, TemporalTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, context) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, CrossAttention): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, MemoryEfficientCrossAttention): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, BasicTransformerBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, FeedForward): + x = module(x, context) + elif isinstance(module, Upsample): + x = module(x) + elif isinstance(module, Downsample): + x = module(x) + elif isinstance(module, Resample): + x = module(x, reference) + elif isinstance(module, TemporalAttentionBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalAttentionMultiBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, InitTemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context, + time_rel_pos_bias, focus_present_mask, + video_mask, reference) + else: + x = module(x) + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/__init__.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..92654e519e5750b9df8b976efbc8ecec635495e3 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/config.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..1c9586eb72fdb97eb19937343129427a67c7c090 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/config.py @@ -0,0 +1,171 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import logging +import os +import os.path as osp +from datetime import datetime + +import torch +from easydict import EasyDict + +cfg = EasyDict(__name__='Config: VideoLDM Decoder') + +# ---------------------------work dir-------------------------- +cfg.work_dir = 'workspace/' + +# ---------------------------Global Variable----------------------------------- +cfg.resolution = [448, 256] +cfg.max_frames = 32 +# ----------------------------------------------------------------------------- + +# ---------------------------Dataset Parameter--------------------------------- +cfg.mean = [0.5, 0.5, 0.5] +cfg.std = [0.5, 0.5, 0.5] +cfg.max_words = 1000 + +# PlaceHolder +cfg.vit_out_dim = 1024 +cfg.vit_resolution = [224, 224] +cfg.depth_clamp = 10.0 +cfg.misc_size = 384 +cfg.depth_std = 20.0 + +cfg.frame_lens = 32 +cfg.sample_fps = 8 + +cfg.batch_sizes = 1 +# ----------------------------------------------------------------------------- + +# ---------------------------Mode Parameters----------------------------------- +# Diffusion +cfg.schedule = 'cosine' +cfg.num_timesteps = 1000 +cfg.mean_type = 'v' +cfg.var_type = 'fixed_small' +cfg.loss_type = 'mse' +cfg.ddim_timesteps = 50 +cfg.ddim_eta = 0.0 +cfg.clamp = 1.0 +cfg.share_noise = False +cfg.use_div_loss = False +cfg.noise_strength = 0.1 + +# classifier-free guidance +cfg.p_zero = 0.1 +cfg.guide_scale = 3.0 + +# clip vision encoder +cfg.vit_mean = [0.48145466, 0.4578275, 0.40821073] +cfg.vit_std = [0.26862954, 0.26130258, 0.27577711] + +# Model +cfg.scale_factor = 0.18215 +cfg.use_fp16 = True +cfg.temporal_attention = True +cfg.decoder_bs = 8 + +cfg.UNet = { + 'type': 'Vid2VidSDUNet', + 'in_dim': 4, + 'dim': 320, + 'y_dim': cfg.vit_out_dim, + 'context_dim': 1024, + 'out_dim': 8 if cfg.var_type.startswith('learned') else 4, + 'dim_mult': [1, 2, 4, 4], + 'num_heads': 8, + 'head_dim': 64, + 'num_res_blocks': 2, + 'attn_scales': [1 / 1, 1 / 2, 1 / 4], + 'dropout': 0.1, + 'temporal_attention': cfg.temporal_attention, + 'temporal_attn_times': 1, + 'use_checkpoint': False, + 'use_fps_condition': False, + 'use_sim_mask': False, + 'num_tokens': 4, + 'default_fps': 8, + 'input_dim': 1024 +} + +cfg.guidances = [] + +# auotoencoder from stabel diffusion +cfg.auto_encoder = { + 'type': 'AutoencoderKL', + 'ddconfig': { + 'double_z': True, + 'z_channels': 4, + 'resolution': 256, + 'in_channels': 3, + 'out_ch': 3, + 'ch': 128, + 'ch_mult': [1, 2, 4, 4], + 'num_res_blocks': 2, + 'attn_resolutions': [], + 'dropout': 0.0 + }, + 'embed_dim': 4, + 'pretrained': 'models/v2-1_512-ema-pruned.ckpt' +} +# clip embedder +cfg.embedder = { + 'type': 'FrozenOpenCLIPEmbedder', + 'layer': 'penultimate', + 'vit_resolution': [224, 224], + 'pretrained': 'open_clip_pytorch_model.bin' +} +# ----------------------------------------------------------------------------- + +# ---------------------------Training Settings--------------------------------- +# training and optimizer +cfg.ema_decay = 0.9999 +cfg.num_steps = 600000 +cfg.lr = 5e-5 +cfg.weight_decay = 0.0 +cfg.betas = (0.9, 0.999) +cfg.eps = 1.0e-8 +cfg.chunk_size = 16 +cfg.alpha = 0.7 +cfg.save_ckp_interval = 1000 +# ----------------------------------------------------------------------------- + +# ----------------------------Pretrain Settings--------------------------------- +# Default: load 2d pretrain +cfg.fix_weight = False +cfg.load_match = False +cfg.pretrained_checkpoint = 'v2-1_512-ema-pruned.ckpt' +cfg.pretrained_image_keys = 'stable_diffusion_image_key_temporal_attention_x1.json' +cfg.resume_checkpoint = 'img2video_ldm_0779000.pth' +# ----------------------------------------------------------------------------- + +# -----------------------------Visual------------------------------------------- +# Visual videos +cfg.viz_interval = 1000 +cfg.visual_train = { + 'type': 'VisualVideoTextDuringTrain', +} +cfg.visual_inference = { + 'type': 'VisualGeneratedVideos', +} +cfg.inference_list_path = '' + +# logging +cfg.log_interval = 100 + +# Default log_dir +cfg.log_dir = 'workspace/output_data' +# ----------------------------------------------------------------------------- + +# ---------------------------Others-------------------------------------------- +# seed +cfg.seed = 8888 +cfg.negative_prompt = 'worst quality, normal quality, low quality, low res, blurry, text, \ +watermark, logo, banner, extra digits, cropped, jpeg artifacts, signature, username, error, \ +sketch ,duplicate, ugly, monochrome, horror, geometry, mutation, disgusting' + +cfg.positive_prompt = ', cinematic, High Contrast, highly detailed, unreal engine, \ +taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, \ +32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, \ +hyper sharpness, perfect without deformations, Unreal Engine 5, 4k render' + +# ----------------------------------------------------------------------------- diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/diffusion_sdedit.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/diffusion_sdedit.py new file mode 100644 index 0000000000000000000000000000000000000000..be5b5f578f2e7dea9b2cc99b955f9f8ae252fa10 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/diffusion_sdedit.py @@ -0,0 +1,247 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import random + +import torch + +from .schedules_sdedit import karras_schedule +from .solvers_sdedit import sample_dpmpp_2m_sde, sample_heun + +__all__ = ['GaussianDiffusion_SDEdit'] + + +def _i(tensor, t, x): + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + return tensor[t.to(tensor.device)].view(shape).to(x.device) + + +class GaussianDiffusion_SDEdit(object): + + def __init__(self, sigmas, prediction_type='eps'): + assert prediction_type in {'x0', 'eps', 'v'} + self.sigmas = sigmas + self.alphas = torch.sqrt(1 - sigmas**2) + self.num_timesteps = len(sigmas) + self.prediction_type = prediction_type + + def diffuse(self, x0, t, noise=None): + noise = torch.randn_like(x0) if noise is None else noise + xt = _i(self.alphas, t, x0) * x0 + _i(self.sigmas, t, x0) * noise + return xt + + def denoise(self, + xt, + t, + s, + model, + model_kwargs={}, + guide_scale=None, + guide_rescale=None, + clamp=None, + percentile=None): + s = t - 1 if s is None else s + + # hyperparams + sigmas = _i(self.sigmas, t, xt) + alphas = _i(self.alphas, t, xt) + alphas_s = _i(self.alphas, s.clamp(0), xt) + alphas_s[s < 0] = 1. + sigmas_s = torch.sqrt(1 - alphas_s**2) + + # precompute variables + betas = 1 - (alphas / alphas_s)**2 + coef1 = betas * alphas_s / sigmas**2 + coef2 = (alphas * sigmas_s**2) / (alphas_s * sigmas**2) + var = betas * (sigmas_s / sigmas)**2 + log_var = torch.log(var).clamp_(-20, 20) + + # prediction + if guide_scale is None: + assert isinstance(model_kwargs, dict) + out = model(xt, t=t, **model_kwargs) + else: + # classifier-free guidance + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, t=t, **model_kwargs[0]) + if guide_scale == 1.: + out = y_out + else: + u_out = model(xt, t=t, **model_kwargs[1]) + out = u_out + guide_scale * (y_out - u_out) + + if guide_rescale is not None: + assert guide_rescale >= 0 and guide_rescale <= 1 + ratio = ( + y_out.flatten(1).std(dim=1) / # noqa + (out.flatten(1).std(dim=1) + 1e-12) + ).view((-1, ) + (1, ) * (y_out.ndim - 1)) + out *= guide_rescale * ratio + (1 - guide_rescale) * 1.0 + + # compute x0 + if self.prediction_type == 'x0': + x0 = out + elif self.prediction_type == 'eps': + x0 = (xt - sigmas * out) / alphas + elif self.prediction_type == 'v': + x0 = alphas * xt - sigmas * out + else: + raise NotImplementedError( + f'prediction_type {self.prediction_type} not implemented') + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 + s = torch.quantile(x0.flatten(1).abs(), percentile, dim=1) + s = s.clamp_(1.0).view((-1, ) + (1, ) * (xt.ndim - 1)) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + + # recompute eps using the restricted x0 + eps = (xt - alphas * x0) / sigmas + + # compute mu (mean of posterior distribution) using the restricted x0 + mu = coef1 * x0 + coef2 * xt + return mu, var, log_var, x0, eps + + @torch.no_grad() + def sample(self, + noise, + model, + model_kwargs={}, + condition_fn=None, + guide_scale=None, + guide_rescale=None, + clamp=None, + percentile=None, + solver='euler_a', + steps=20, + t_max=None, + t_min=None, + discretization=None, + discard_penultimate_step=None, + return_intermediate=None, + show_progress=False, + seed=-1, + **kwargs): + # sanity check + assert isinstance(steps, (int, torch.LongTensor)) + assert t_max is None or (t_max > 0 and t_max <= self.num_timesteps - 1) + assert t_min is None or (t_min >= 0 and t_min < self.num_timesteps - 1) + assert discretization in (None, 'leading', 'linspace', 'trailing') + assert discard_penultimate_step in (None, True, False) + assert return_intermediate in (None, 'x0', 'xt') + + # function of diffusion solver + solver_fn = { + 'heun': sample_heun, + 'dpmpp_2m_sde': sample_dpmpp_2m_sde + }[solver] + + # options + schedule = 'karras' if 'karras' in solver else None + discretization = discretization or 'linspace' + seed = seed if seed >= 0 else random.randint(0, 2**31) + if isinstance(steps, torch.LongTensor): + discard_penultimate_step = False + if discard_penultimate_step is None: + discard_penultimate_step = True if solver in ( + 'dpm2', 'dpm2_ancestral', 'dpmpp_2m_sde', 'dpm2_karras', + 'dpm2_ancestral_karras', 'dpmpp_2m_sde_karras') else False + + # function for denoising xt to get x0 + intermediates = [] + + def model_fn(xt, sigma): + # denoising + t = self._sigma_to_t(sigma).repeat(len(xt)).round().long() + x0 = self.denoise(xt, t, None, model, model_kwargs, guide_scale, + guide_rescale, clamp, percentile)[-2] + + # collect intermediate outputs + if return_intermediate == 'xt': + intermediates.append(xt) + elif return_intermediate == 'x0': + intermediates.append(x0) + return x0 + + # get timesteps + if isinstance(steps, int): + steps += 1 if discard_penultimate_step else 0 + t_max = self.num_timesteps - 1 if t_max is None else t_max + t_min = 0 if t_min is None else t_min + + # discretize timesteps + if discretization == 'leading': + steps = torch.arange(t_min, t_max + 1, + (t_max - t_min + 1) / steps).flip(0) + elif discretization == 'linspace': + steps = torch.linspace(t_max, t_min, steps) + elif discretization == 'trailing': + steps = torch.arange(t_max, t_min - 1, + -((t_max - t_min + 1) / steps)) + else: + raise NotImplementedError( + f'{discretization} discretization not implemented') + steps = steps.clamp_(t_min, t_max) + steps = torch.as_tensor( + steps, dtype=torch.float32, device=noise.device) + + # get sigmas + sigmas = self._t_to_sigma(steps) + sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) + if schedule == 'karras': + if sigmas[0] == float('inf'): + sigmas = karras_schedule( + n=len(steps) - 1, + sigma_min=sigmas[sigmas > 0].min().item(), + sigma_max=sigmas[sigmas < float('inf')].max().item(), + rho=7.).to(sigmas) + sigmas = torch.cat([ + sigmas.new_tensor([float('inf')]), sigmas, + sigmas.new_zeros([1]) + ]) + else: + sigmas = karras_schedule( + n=len(steps), + sigma_min=sigmas[sigmas > 0].min().item(), + sigma_max=sigmas.max().item(), + rho=7.).to(sigmas) + sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) + if discard_penultimate_step: + sigmas = torch.cat([sigmas[:-2], sigmas[-1:]]) + + # sampling + x0 = solver_fn( + noise, model_fn, sigmas, show_progress=show_progress, **kwargs) + return (x0, intermediates) if return_intermediate is not None else x0 + + def _sigma_to_t(self, sigma): + if sigma == float('inf'): + t = torch.full_like(sigma, len(self.sigmas) - 1) + else: + log_sigmas = torch.sqrt(self.sigmas**2 / # noqa + (1 - self.sigmas**2)).log().to(sigma) + log_sigma = sigma.log() + dists = log_sigma - log_sigmas[:, None] + low_idx = dists.ge(0).cumsum(dim=0).argmax(dim=0).clamp( + max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + low, high = log_sigmas[low_idx], log_sigmas[high_idx] + w = (low - log_sigma) / (low - high) + w = w.clamp(0, 1) + t = (1 - w) * low_idx + w * high_idx + t = t.view(sigma.shape) + if t.ndim == 0: + t = t.unsqueeze(0) + return t + + def _t_to_sigma(self, t): + t = t.float() + low_idx, high_idx, w = t.floor().long(), t.ceil().long(), t.frac() + log_sigmas = torch.sqrt(self.sigmas**2 / # noqa + (1 - self.sigmas**2)).log().to(t) + log_sigma = (1 - w) * log_sigmas[low_idx] + w * log_sigmas[high_idx] + log_sigma[torch.isnan(log_sigma) + | torch.isinf(log_sigma)] = float('inf') + return log_sigma.exp() diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/schedules_sdedit.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/schedules_sdedit.py new file mode 100644 index 0000000000000000000000000000000000000000..06fd4e8a66b5d2582932ef59e701a238511fd6d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/schedules_sdedit.py @@ -0,0 +1,85 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch + + +def betas_to_sigmas(betas): + return torch.sqrt(1 - torch.cumprod(1 - betas, dim=0)) + + +def sigmas_to_betas(sigmas): + square_alphas = 1 - sigmas**2 + betas = 1 - torch.cat( + [square_alphas[:1], square_alphas[1:] / square_alphas[:-1]]) + return betas + + +def logsnrs_to_sigmas(logsnrs): + return torch.sqrt(torch.sigmoid(-logsnrs)) + + +def sigmas_to_logsnrs(sigmas): + square_sigmas = sigmas**2 + return torch.log(square_sigmas / (1 - square_sigmas)) + + +def _logsnr_cosine(n, logsnr_min=-15, logsnr_max=15): + t_min = math.atan(math.exp(-0.5 * logsnr_min)) + t_max = math.atan(math.exp(-0.5 * logsnr_max)) + t = torch.linspace(1, 0, n) + logsnrs = -2 * torch.log(torch.tan(t_min + t * (t_max - t_min))) + return logsnrs + + +def _logsnr_cosine_shifted(n, logsnr_min=-15, logsnr_max=15, scale=2): + logsnrs = _logsnr_cosine(n, logsnr_min, logsnr_max) + logsnrs += 2 * math.log(1 / scale) + return logsnrs + + +def _logsnr_cosine_interp(n, + logsnr_min=-15, + logsnr_max=15, + scale_min=2, + scale_max=4): + t = torch.linspace(1, 0, n) + logsnrs_min = _logsnr_cosine_shifted(n, logsnr_min, logsnr_max, scale_min) + logsnrs_max = _logsnr_cosine_shifted(n, logsnr_min, logsnr_max, scale_max) + logsnrs = t * logsnrs_min + (1 - t) * logsnrs_max + return logsnrs + + +def karras_schedule(n, sigma_min=0.002, sigma_max=80.0, rho=7.0): + ramp = torch.linspace(1, 0, n) + min_inv_rho = sigma_min**(1 / rho) + max_inv_rho = sigma_max**(1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho))**rho + sigmas = torch.sqrt(sigmas**2 / (1 + sigmas**2)) + return sigmas + + +def logsnr_cosine_interp_schedule(n, + logsnr_min=-15, + logsnr_max=15, + scale_min=2, + scale_max=4): + return logsnrs_to_sigmas( + _logsnr_cosine_interp(n, logsnr_min, logsnr_max, scale_min, scale_max)) + + +def noise_schedule(schedule='logsnr_cosine_interp', + n=1000, + zero_terminal_snr=False, + **kwargs): + # compute sigmas + sigmas = { + 'logsnr_cosine_interp': logsnr_cosine_interp_schedule + }[schedule](n, **kwargs) + + # post-processing + if zero_terminal_snr and sigmas.max() != 1.0: + scale = (1.0 - sigmas.min()) / (sigmas.max() - sigmas.min()) + sigmas = sigmas.min() + scale * (sigmas - sigmas.min()) + return sigmas diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/seed.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/seed.py new file mode 100644 index 0000000000000000000000000000000000000000..df3c9c50e72e69ca6c93cb74bff272d041fdd78f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/seed.py @@ -0,0 +1,14 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import random + +import numpy as np +import torch + + +def setup_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/solvers_sdedit.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/solvers_sdedit.py new file mode 100644 index 0000000000000000000000000000000000000000..8d00a39fe994b759fafabbeec4361140fd782926 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/solvers_sdedit.py @@ -0,0 +1,194 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torchsde +from tqdm.auto import trange + + +def get_ancestral_step(sigma_from, sigma_to, eta=1.): + """ + Calculates the noise level (sigma_down) to step down to and the amount + of noise to add (sigma_up) when doing an ancestral sampling step. + """ + if not eta: + return sigma_to, 0. + sigma_up = min( + sigma_to, + eta * ( + sigma_to**2 * # noqa + (sigma_from**2 - sigma_to**2) / sigma_from**2)**0.5) + sigma_down = (sigma_to**2 - sigma_up**2)**0.5 + return sigma_down, sigma_up + + +def get_scalings(sigma): + c_out = -sigma + c_in = 1 / (sigma**2 + 1.**2)**0.5 + return c_out, c_in + + +@torch.no_grad() +def sample_heun(noise, + model, + sigmas, + s_churn=0., + s_tmin=0., + s_tmax=float('inf'), + s_noise=1., + show_progress=True): + """ + Implements Algorithm 2 (Heun steps) from Karras et al. (2022). + """ + x = noise * sigmas[0] + for i in trange(len(sigmas) - 1, disable=not show_progress): + gamma = 0. + if s_tmin <= sigmas[i] <= s_tmax and sigmas[i] < float('inf'): + gamma = min(s_churn / (len(sigmas) - 1), 2**0.5 - 1) + eps = torch.randn_like(x) * s_noise + sigma_hat = sigmas[i] * (gamma + 1) + if gamma > 0: + x = x + eps * (sigma_hat**2 - sigmas[i]**2)**0.5 + if sigmas[i] == float('inf'): + # Euler method + denoised = model(noise, sigma_hat) + x = denoised + sigmas[i + 1] * (gamma + 1) * noise + else: + _, c_in = get_scalings(sigma_hat) + denoised = model(x * c_in, sigma_hat) + d = (x - denoised) / sigma_hat + dt = sigmas[i + 1] - sigma_hat + if sigmas[i + 1] == 0: + # Euler method + x = x + d * dt + else: + # Heun's method + x_2 = x + d * dt + _, c_in = get_scalings(sigmas[i + 1]) + denoised_2 = model(x_2 * c_in, sigmas[i + 1]) + d_2 = (x_2 - denoised_2) / sigmas[i + 1] + d_prime = (d + d_2) / 2 + x = x + d_prime * dt + return x + + +class BatchedBrownianTree: + """ + A wrapper around torchsde.BrownianTree that enables batches of entropy. + """ + + def __init__(self, x, t0, t1, seed=None, **kwargs): + t0, t1, self.sign = self.sort(t0, t1) + w0 = kwargs.get('w0', torch.zeros_like(x)) + if seed is None: + seed = torch.randint(0, 2**63 - 1, []).item() + self.batched = True + try: + assert len(seed) == x.shape[0] + w0 = w0[0] + except TypeError: + seed = [seed] + self.batched = False + self.trees = [ + torchsde.BrownianTree(t0, w0, t1, entropy=s, **kwargs) + for s in seed + ] + + @staticmethod + def sort(a, b): + return (a, b, 1) if a < b else (b, a, -1) + + def __call__(self, t0, t1): + t0, t1, sign = self.sort(t0, t1) + w = torch.stack([tree(t0, t1) for tree in self.trees]) * ( + self.sign * sign) + return w if self.batched else w[0] + + +class BrownianTreeNoiseSampler: + """ + A noise sampler backed by a torchsde.BrownianTree. + + Args: + x (Tensor): The tensor whose shape, device and dtype to use to generate + random samples. + sigma_min (float): The low end of the valid interval. + sigma_max (float): The high end of the valid interval. + seed (int or List[int]): The random seed. If a list of seeds is + supplied instead of a single integer, then the noise sampler will + use one BrownianTree per batch item, each with its own seed. + transform (callable): A function that maps sigma to the sampler's + internal timestep. + """ + + def __init__(self, + x, + sigma_min, + sigma_max, + seed=None, + transform=lambda x: x): + self.transform = transform + t0 = self.transform(torch.as_tensor(sigma_min)) + t1 = self.transform(torch.as_tensor(sigma_max)) + self.tree = BatchedBrownianTree(x, t0, t1, seed) + + def __call__(self, sigma, sigma_next): + t0 = self.transform(torch.as_tensor(sigma)) + t1 = self.transform(torch.as_tensor(sigma_next)) + return self.tree(t0, t1) / (t1 - t0).abs().sqrt() + + +@torch.no_grad() +def sample_dpmpp_2m_sde(noise, + model, + sigmas, + eta=1., + s_noise=1., + solver_type='midpoint', + show_progress=True): + """ + DPM-Solver++ (2M) SDE. + """ + assert solver_type in {'heun', 'midpoint'} + + x = noise * sigmas[0] + sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas[ + sigmas < float('inf')].max() + noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max) + old_denoised = None + h_last = None + + for i in trange(len(sigmas) - 1, disable=not show_progress): + if sigmas[i] == float('inf'): + # Euler method + denoised = model(noise, sigmas[i]) + x = denoised + sigmas[i + 1] * noise + else: + _, c_in = get_scalings(sigmas[i]) + denoised = model(x * c_in, sigmas[i]) + if sigmas[i + 1] == 0: + # Denoising step + x = denoised + else: + # DPM-Solver++(2M) SDE + t, s = -sigmas[i].log(), -sigmas[i + 1].log() + h = s - t + eta_h = eta * h + + x = sigmas[i + 1] / sigmas[i] * (-eta_h).exp() * x + \ + (-h - eta_h).expm1().neg() * denoised + + if old_denoised is not None: + r = h_last / h + if solver_type == 'heun': + x = x + ((-h - eta_h).expm1().neg() / (-h - eta_h) + 1) * \ + (1 / r) * (denoised - old_denoised) + elif solver_type == 'midpoint': + x = x + 0.5 * (-h - eta_h).expm1().neg() * \ + (1 / r) * (denoised - old_denoised) + + x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[ + i + 1] * (-2 * eta_h).expm1().neg().sqrt() * s_noise + + old_denoised = denoised + h_last = h + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/transforms.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..3663620f0a646d7574d4d1852a0acce86eace2a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/utils/transforms.py @@ -0,0 +1,404 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import random + +import numpy as np +import torch +import torchvision.transforms.functional as F +from PIL import Image, ImageFilter + +__all__ = [ + 'Compose', 'Resize', 'Rescale', 'CenterCrop', 'CenterCropV2', + 'CenterCropWide', 'RandomCrop', 'RandomCropV2', 'RandomHFlip', + 'GaussianBlur', 'ColorJitter', 'RandomGray', 'ToTensor', 'Normalize', + 'ResizeRandomCrop', 'ExtractResizeRandomCrop', 'ExtractResizeAssignCrop' +] + + +class Compose(object): + + def __init__(self, transforms): + self.transforms = transforms + + def __getitem__(self, index): + if isinstance(index, slice): + return Compose(self.transforms[index]) + else: + return self.transforms[index] + + def __len__(self): + return len(self.transforms) + + def __call__(self, rgb): + for t in self.transforms: + rgb = t(rgb) + return rgb + + +class Resize(object): + + def __init__(self, size=256): + if isinstance(size, int): + size = (size, size) + self.size = size + + def __call__(self, rgb): + if isinstance(rgb, list): + rgb = [u.resize(self.size, Image.BILINEAR) for u in rgb] + else: + rgb = rgb.resize(self.size, Image.BILINEAR) + return rgb + + +class Rescale(object): + + def __init__(self, size=256, interpolation=Image.BILINEAR): + self.size = size + self.interpolation = interpolation + + def __call__(self, rgb): + w, h = rgb[0].size + scale = self.size / min(w, h) + out_w, out_h = int(round(w * scale)), int(round(h * scale)) + rgb = [u.resize((out_w, out_h), self.interpolation) for u in rgb] + return rgb + + +class CenterCrop(object): + + def __init__(self, size=224): + self.size = size + + def __call__(self, rgb): + w, h = rgb[0].size + assert min(w, h) >= self.size + x1 = (w - self.size) // 2 + y1 = (h - self.size) // 2 + rgb = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in rgb] + return rgb + + +class ResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + return rgb + + +class ExtractResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + wh = [x1, y1, x1 + out_w, y1 + out_h] + return rgb, wh + + +class ExtractResizeAssignCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb, wh): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + + rgb = [u.crop(wh) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class CenterCropV2(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + # fast resize + while min(img[0].size) >= 2 * self.size: + img = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in img + ] + scale = self.size / min(img[0].size) + img = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in img + ] + + # center crop + x1 = (img[0].width - self.size) // 2 + y1 = (img[0].height - self.size) // 2 + img = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in img] + return img + + +class CenterCropWide(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + if isinstance(img, list): + scale = min(img[0].size[0] / self.size[0], + img[0].size[1] / self.size[1]) + img = [ + u.resize((round(u.width // scale), round(u.height // scale)), + resample=Image.BOX) for u in img + ] + + # center crop + x1 = (img[0].width - self.size[0]) // 2 + y1 = (img[0].height - self.size[1]) // 2 + img = [ + u.crop((x1, y1, x1 + self.size[0], y1 + self.size[1])) + for u in img + ] + return img + else: + scale = min(img.size[0] / self.size[0], img.size[1] / self.size[1]) + img = img.resize( + (round(img.width // scale), round(img.height // scale)), + resample=Image.BOX) + x1 = (img.width - self.size[0]) // 2 + y1 = (img.height - self.size[1]) // 2 + img = img.crop((x1, y1, x1 + self.size[0], y1 + self.size[1])) + return img + + +class RandomCrop(object): + + def __init__(self, size=224, min_area=0.4): + self.size = size + self.min_area = min_area + + def __call__(self, rgb): + + # consistent crop between rgb and m + w, h = rgb[0].size + area = w * h + out_w, out_h = float('inf'), float('inf') + while out_w > w or out_h > h: + target_area = random.uniform(self.min_area, 1.0) * area + aspect_ratio = random.uniform(3. / 4., 4. / 3.) + out_w = int(round(math.sqrt(target_area * aspect_ratio))) + out_h = int(round(math.sqrt(target_area / aspect_ratio))) + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class RandomCropV2(object): + + def __init__(self, size=224, min_area=0.4, ratio=(3. / 4., 4. / 3.)): + if isinstance(size, (tuple, list)): + self.size = size + else: + self.size = (size, size) + self.min_area = min_area + self.ratio = ratio + + def _get_params(self, img): + width, height = img.size + area = height * width + + for _ in range(10): + target_area = random.uniform(self.min_area, 1.0) * area + log_ratio = (math.log(self.ratio[0]), math.log(self.ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if 0 < w <= width and 0 < h <= height: + i = random.randint(0, height - h) + j = random.randint(0, width - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = float(width) / float(height) + if (in_ratio < min(self.ratio)): + w = width + h = int(round(w / min(self.ratio))) + elif (in_ratio > max(self.ratio)): + h = height + w = int(round(h * max(self.ratio))) + else: + w = width + h = height + i = (height - h) // 2 + j = (width - w) // 2 + return i, j, h, w + + def __call__(self, rgb): + i, j, h, w = self._get_params(rgb[0]) + rgb = [F.resized_crop(u, i, j, h, w, self.size) for u in rgb] + return rgb + + +class RandomHFlip(object): + + def __init__(self, p=0.5): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.transpose(Image.FLIP_LEFT_RIGHT) for u in rgb] + return rgb + + +class GaussianBlur(object): + + def __init__(self, sigmas=[0.1, 2.0], p=0.5): + self.sigmas = sigmas + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + sigma = random.uniform(*self.sigmas) + rgb = [ + u.filter(ImageFilter.GaussianBlur(radius=sigma)) for u in rgb + ] + return rgb + + +class ColorJitter(object): + + def __init__(self, + brightness=0.4, + contrast=0.4, + saturation=0.4, + hue=0.1, + p=0.5): + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + brightness, contrast, saturation, hue = self._random_params() + transforms = [ + lambda f: F.adjust_brightness(f, brightness), + lambda f: F.adjust_contrast(f, contrast), + lambda f: F.adjust_saturation(f, saturation), + lambda f: F.adjust_hue(f, hue) + ] + random.shuffle(transforms) + for t in transforms: + rgb = [t(u) for u in rgb] + + return rgb + + def _random_params(self): + brightness = random.uniform( + max(0, 1 - self.brightness), 1 + self.brightness) + contrast = random.uniform(max(0, 1 - self.contrast), 1 + self.contrast) + saturation = random.uniform( + max(0, 1 - self.saturation), 1 + self.saturation) + hue = random.uniform(-self.hue, self.hue) + return brightness, contrast, saturation, hue + + +class RandomGray(object): + + def __init__(self, p=0.2): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.convert('L').convert('RGB') for u in rgb] + return rgb + + +class ToTensor(object): + + def __call__(self, rgb): + if isinstance(rgb, list): + rgb = torch.stack([F.to_tensor(u) for u in rgb], dim=0) + else: + rgb = F.to_tensor(rgb) + + return rgb + + +class Normalize(object): + + def __init__(self, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]): + self.mean = mean + self.std = std + + def __call__(self, rgb): + rgb = rgb.clone() + rgb.clamp_(0, 1) + if not isinstance(self.mean, torch.Tensor): + self.mean = rgb.new_tensor(self.mean).view(-1) + if not isinstance(self.std, torch.Tensor): + self.std = rgb.new_tensor(self.std).view(-1) + if rgb.dim() == 4: + rgb.sub_(self.mean.view(1, -1, 1, + 1)).div_(self.std.view(1, -1, 1, 1)) + elif rgb.dim() == 3: + rgb.sub_(self.mean.view(-1, 1, 1)).div_(self.std.view(-1, 1, 1)) + return rgb diff --git a/runtime/Lib/site_packages/models/multi_modal/video_to_video/video_to_video_model.py b/runtime/Lib/site_packages/models/multi_modal/video_to_video/video_to_video_model.py new file mode 100644 index 0000000000000000000000000000000000000000..1a13a3fc664110f095fb449e8c66564019e7f132 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/video_to_video/video_to_video_model.py @@ -0,0 +1,230 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +import os.path as osp +import random +from copy import copy +from typing import Any, Dict + +import torch +import torch.cuda.amp as amp +import torch.nn.functional as F + +import modelscope.models.multi_modal.video_to_video.utils.transforms as data +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.video_to_video.modules import * +from modelscope.models.multi_modal.video_to_video.modules import ( + AutoencoderKL, FrozenOpenCLIPEmbedder, Vid2VidSDUNet, + get_first_stage_encoding) +from modelscope.models.multi_modal.video_to_video.utils.config import cfg +from modelscope.models.multi_modal.video_to_video.utils.diffusion_sdedit import \ + GaussianDiffusion_SDEdit +from modelscope.models.multi_modal.video_to_video.utils.schedules_sdedit import \ + noise_schedule +from modelscope.models.multi_modal.video_to_video.utils.seed import setup_seed +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.device import create_device +from modelscope.utils.logger import get_logger + +__all__ = ['VideoToVideo'] + +logger = get_logger() + + +@MODELS.register_module( + Tasks.video_to_video, module_name=Models.video_to_video_model) +class VideoToVideo(TorchModel): + r""" + Video2Video aims to solve the task of generating super-resolution videos based on input + video and text, which is a video generation basic model developed by Alibaba Cloud. + + Paper link: https://arxiv.org/abs/2306.02018 + + Attributes: + diffusion: diffusion model for DDIM. + autoencoder: decode the latent representation of input video into visual space. + clip_encoder: encode the text into text embedding. + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + + self.config = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + + cfg.solver_mode = self.config.model.model_args.solver_mode + + # assign default value + cfg.batch_size = self.config.model.model_cfg.batch_size + cfg.target_fps = self.config.model.model_cfg.target_fps + cfg.max_frames = self.config.model.model_cfg.max_frames + cfg.latent_hei = self.config.model.model_cfg.latent_hei + cfg.latent_wid = self.config.model.model_cfg.latent_wid + cfg.model_path = osp.join(model_dir, + self.config.model.model_args.ckpt_unet) + + required_device = kwargs.pop('device', 'gpu') + self.device = create_device(required_device) + + if 'seed' in self.config.model.model_args.keys(): + cfg.seed = self.config.model.model_args.seed + else: + cfg.seed = random.randint(0, 99999) + setup_seed(cfg.seed) + + # transform + vid_trans = data.Compose( + [data.ToTensor(), + data.Normalize(mean=cfg.mean, std=cfg.std)]) + self.vid_trans = vid_trans + + cfg.embedder.pretrained = osp.join( + model_dir, self.config.model.model_args.ckpt_clip) + clip_encoder = FrozenOpenCLIPEmbedder( + pretrained=cfg.embedder.pretrained, device=self.device) + clip_encoder.model.to(self.device) + self.clip_encoder = clip_encoder + logger.info(f'Build encoder with {cfg.embedder.type}') + + # [unet] + generator = Vid2VidSDUNet() + generator = generator.to(self.device) + generator.eval() + load_dict = torch.load(cfg.model_path, map_location='cpu') + ret = generator.load_state_dict(load_dict['state_dict'], strict=True) + self.generator = generator.half() + logger.info('Load model {} path {}, with local status {}'.format( + cfg.UNet.type, cfg.model_path, ret)) + + # [diffusion] + sigmas = noise_schedule( + schedule='logsnr_cosine_interp', + n=1000, + zero_terminal_snr=True, + scale_min=2.0, + scale_max=4.0) + diffusion = GaussianDiffusion_SDEdit( + sigmas=sigmas, prediction_type='v') + self.diffusion = diffusion + logger.info('Build diffusion with type of GaussianDiffusion_SDEdit') + + # [auotoencoder] + cfg.auto_encoder.pretrained = osp.join( + model_dir, self.config.model.model_args.ckpt_autoencoder) + autoencoder = AutoencoderKL(**cfg.auto_encoder) + autoencoder.eval() + for param in autoencoder.parameters(): + param.requires_grad = False + autoencoder.to(self.device) + self.autoencoder = autoencoder + torch.cuda.empty_cache() + + negative_prompt = cfg.negative_prompt + negative_y = clip_encoder(negative_prompt).detach() + self.negative_y = negative_y + + positive_prompt = cfg.positive_prompt + self.positive_prompt = positive_prompt + + self.cfg = cfg + + def forward(self, input: Dict[str, Any]): + r""" + The entry function of video to video task. + 1. Using CLIP to encode text into embeddings. + 2. Using diffusion model to generate the video's latent representation. + 3. Using autoencoder to decode the video's latent representation to visual space. + + Args: + input (`Dict[Str, Any]`): + The input of the task + Returns: + A generated video (as pytorch tensor). + """ + + video_data = input['video_data'] + y = input['y'] + cfg = self.cfg + + video_data = F.interpolate( + video_data, size=(720, 1280), mode='bilinear') + video_data = video_data.unsqueeze(0) + video_data = video_data.to(self.device) + + batch_size, frames_num, _, _, _ = video_data.shape + video_data = rearrange(video_data, 'b f c h w -> (b f) c h w') + + video_data_list = torch.chunk( + video_data, video_data.shape[0] // 1, dim=0) + with torch.no_grad(): + decode_data = [] + for vd_data in video_data_list: + encoder_posterior = self.autoencoder.encode(vd_data) + tmp = get_first_stage_encoding(encoder_posterior).detach() + decode_data.append(tmp) + video_data_feature = torch.cat(decode_data, dim=0) + video_data_feature = rearrange( + video_data_feature, '(b f) c h w -> b c f h w', b=batch_size) + torch.cuda.empty_cache() + + with amp.autocast(enabled=True): + total_noise_levels = 600 + t = torch.randint( + total_noise_levels - 1, + total_noise_levels, (1, ), + dtype=torch.long).to(self.device) + + noise = torch.randn_like(video_data_feature) + noised_lr = self.diffusion.diffuse(video_data_feature, t, noise) + model_kwargs = [{'y': y}, {'y': self.negative_y}] + + gen_vid = self.diffusion.sample( + noise=noised_lr, + model=self.generator, + model_kwargs=model_kwargs, + guide_scale=7.5, + guide_rescale=0.2, + solver='dpmpp_2m_sde' if cfg.solver_mode == 'fast' else 'heun', + steps=30 if cfg.solver_mode == 'fast' else 50, + t_max=total_noise_levels - 1, + t_min=0, + discretization='trailing') + + torch.cuda.empty_cache() + scale_factor = 0.18215 + vid_tensor_feature = 1. / scale_factor * gen_vid + + vid_tensor_feature = rearrange(vid_tensor_feature, + 'b c f h w -> (b f) c h w') + vid_tensor_feature_list = torch.chunk( + vid_tensor_feature, vid_tensor_feature.shape[0] // 2, dim=0) + decode_data = [] + for vd_data in vid_tensor_feature_list: + tmp = self.autoencoder.decode(vd_data) + decode_data.append(tmp) + vid_tensor_gen = torch.cat(decode_data, dim=0) + + gen_video = rearrange( + vid_tensor_gen, '(b f) c h w -> b c f h w', b=cfg.batch_size) + + return gen_video.type(torch.float32).cpu() diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ae9102c0b1dcc79bfb51e23cd2d13a92a0f8710f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .videocomposer_model import VideoComposer + +else: + _import_structure = { + 'videocomposer_model': ['VideoComposer'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b937315b6e719ae8289fee2908aa486222eb76c5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/canny/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/canny/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bf653d47f8d2921f2bf01737e222d7b27e28b592 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/canny/__init__.py @@ -0,0 +1,44 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import cv2 +import numpy as np +import torch +from tools.annotator.util import HWC3 + + +class CannyDetector: + + def __call__(self, + img, + low_threshold=None, + high_threshold=None, + random_threshold=True): + + # Convert to numpy + if isinstance(img, torch.Tensor): # (h, w, c) + img = img.cpu().numpy() + img_np = cv2.convertScaleAbs((img * 255.)) + elif isinstance(img, np.ndarray): # (h, w, c) + img_np = img # we assume values are in the range from 0 to 255. + else: + assert False + + # Select the threshold + if (low_threshold is None) and (high_threshold is None): + median_intensity = np.median(img_np) + if random_threshold is False: + low_threshold = int(max(0, (1 - 0.33) * median_intensity)) + high_threshold = int(min(255, (1 + 0.33) * median_intensity)) + else: + random_canny = np.random.uniform(0.1, 0.4) + # Might try other values + low_threshold = int( + max(0, (1 - random_canny) * median_intensity)) + high_threshold = 2 * low_threshold + + # Detect canny edge + canny_edge = cv2.Canny(img_np, low_threshold, high_threshold) + + canny_condition = torch.from_numpy( + canny_edge.copy()).unsqueeze(dim=-1).float().cuda() / 255.0 + return canny_condition diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..62f092a1a8c87f3f53fb61427a680460aa234d9c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .palette import * diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/palette.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/palette.py new file mode 100644 index 0000000000000000000000000000000000000000..d28ba85b70bc8ce3146033e5338210d0ba66d236 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/histogram/palette.py @@ -0,0 +1,155 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +r"""Modified from ``https://github.com/sergeyk/rayleigh''. +""" +import os +import os.path as osp + +import numpy as np +from skimage.color import hsv2rgb, lab2rgb, rgb2lab +from skimage.io import imsave +from sklearn.metrics import euclidean_distances + +__all__ = ['Palette'] + + +def rgb2hex(rgb): + return '#%02x%02x%02x' % tuple([int(round(255.0 * u)) for u in rgb]) + + +def hex2rgb(hex): + rgb = hex.strip('#') + fn = lambda u: round(int(u, 16) / 255.0, 5) # noqa + return fn(rgb[:2]), fn(rgb[2:4]), fn(rgb[4:6]) + + +class Palette(object): + r"""Create a color palette (codebook) in the form of a 2D grid of colors. + Further, the rightmost column has num_hues gradations from black to white. + + Parameters: + num_hues: number of colors with full lightness and saturation, in the middle. + num_sat: number of rows above middle row that show the same hues with decreasing saturation. + """ + + def __init__(self, num_hues=11, num_sat=5, num_light=4): + n = num_sat + 2 * num_light + + # hues + if num_hues == 8: + hues = np.tile( + np.array([0., 0.10, 0.15, 0.28, 0.51, 0.58, 0.77, 0.85]), + (n, 1)) + elif num_hues == 9: + hues = np.tile( + np.array([0., 0.10, 0.15, 0.28, 0.49, 0.54, 0.60, 0.7, 0.87]), + (n, 1)) + elif num_hues == 10: + hues = np.tile( + np.array( + [0., 0.10, 0.15, 0.28, 0.49, 0.54, 0.60, 0.66, 0.76, + 0.87]), (n, 1)) + elif num_hues == 11: + hues = np.tile( + np.array([ + 0.0, 0.0833, 0.166, 0.25, 0.333, 0.5, 0.56333, 0.666, 0.73, + 0.803, 0.916 + ]), (n, 1)) + else: + hues = np.tile(np.linspace(0, 1, num_hues + 1)[:-1], (n, 1)) + + # saturations + sats = np.hstack(( + np.linspace(0, 1, num_sat + 2)[1:-1], + 1, + [1] * num_light, + [0.4] * # noqa + (num_light - 1))) + sats = np.tile(np.atleast_2d(sats).T, (1, num_hues)) + + # lights + lights = np.hstack( + ([1] * num_sat, 1, np.linspace(1, 0.2, num_light + 2)[1:-1], + np.linspace(1, 0.2, num_light + 2)[1:-2])) + lights = np.tile(np.atleast_2d(lights).T, (1, num_hues)) + + # colors + rgb = hsv2rgb(np.dstack([hues, sats, lights])) + gray = np.tile( + np.linspace(1, 0, n)[:, np.newaxis, np.newaxis], (1, 1, 3)) + self.thumbnail = np.hstack([rgb, gray]) + + # flatten + rgb = rgb.T.reshape(3, -1).T + gray = gray.T.reshape(3, -1).T + self.rgb = np.vstack((rgb, gray)) + self.lab = rgb2lab(self.rgb[np.newaxis, :, :]).squeeze() + self.hex = [rgb2hex(u) for u in self.rgb] + self.lab_dists = euclidean_distances(self.lab, squared=True) + + def histogram(self, rgb_img, sigma=20): + # compute histogram + lab = rgb2lab(rgb_img).reshape((-1, 3)) + min_ind = np.argmin( + euclidean_distances(lab, self.lab, squared=True), axis=1) + hist = 1.0 * np.bincount( + min_ind, minlength=self.lab.shape[0]) / lab.shape[0] + + # smooth histogram + if sigma > 0: + weight = np.exp(-self.lab_dists / (2.0 * sigma**2)) + weight = weight / weight.sum(1)[:, np.newaxis] + hist = (weight * hist).sum(1) + hist[hist < 1e-5] = 0 + return hist + + def get_palette_image(self, hist, percentile=90, width=200, height=50): + # curate histogram + ind = np.argsort(-hist) + ind = ind[hist[ind] > np.percentile(hist, percentile)] + hist = hist[ind] / hist[ind].sum() + + # draw palette + nums = np.array(hist * width, dtype=int) + array = np.vstack([ + np.tile(np.array(u), (v, 1)) for u, v in zip(self.rgb[ind], nums) + ]) + array = np.tile(array[np.newaxis, :, :], (height, 1, 1)) + if array.shape[1] < width: + array = np.concatenate( + [array, np.zeros((height, width - array.shape[1], 3))], axis=1) + return array + + def quantize_image(self, rgb_img): + lab = rgb2lab(rgb_img).reshape((-1, 3)) + min_ind = np.argmin( + euclidean_distances(lab, self.lab, squared=True), axis=1) + quantized_lab = self.lab[min_ind] + img = lab2rgb(quantized_lab.reshape(rgb_img.shape)) + return img + + def export(self, dirname): + if not osp.exists(dirname): + os.makedirs(dirname) + + # save thumbnail + imsave(osp.join(dirname, 'palette.png'), self.thumbnail) + + # save html + with open(osp.join(dirname, 'palette.html'), 'w') as f: + html = ''' + + ''' + for row in self.thumbnail: + for col in row: + html += '\n'.format( + rgb2hex(col)) + html += '
\n' + f.write(html) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..51e28cc7d988138d54bb450594988669acbc590d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .pidinet import * +from .sketch_simplification import * diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/pidinet.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/pidinet.py new file mode 100644 index 0000000000000000000000000000000000000000..86c17d5daf37e423a01b395aad5084044a520f4a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/pidinet.py @@ -0,0 +1,940 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +r"""Modified from ``https://github.com/zhuoinoulu/pidinet''. + Image augmentation: T.Compose([ + T.ToTensor(), + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]). +""" +import math +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.multi_modal.videocomposer.utils.utils import \ + DOWNLOAD_TO_CACHE + +__all__ = [ + 'PiDiNet', 'pidinet_bsd_tiny', 'pidinet_bsd_small', 'pidinet_bsd', + 'pidinet_nyud', 'pidinet_multicue' +] + +CONFIGS = { + 'baseline': { + 'layer0': 'cv', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'c-v15': { + 'layer0': 'cd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'a-v15': { + 'layer0': 'ad', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'r-v15': { + 'layer0': 'rd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'cvvv4': { + 'layer0': 'cd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'avvv4': { + 'layer0': 'ad', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'ad', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'ad', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'ad', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'rvvv4': { + 'layer0': 'rd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'rd', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'rd', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'rd', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'cccv4': { + 'layer0': 'cd', + 'layer1': 'cd', + 'layer2': 'cd', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'cd', + 'layer6': 'cd', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'cd', + 'layer10': 'cd', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'cd', + 'layer14': 'cd', + 'layer15': 'cv', + }, + 'aaav4': { + 'layer0': 'ad', + 'layer1': 'ad', + 'layer2': 'ad', + 'layer3': 'cv', + 'layer4': 'ad', + 'layer5': 'ad', + 'layer6': 'ad', + 'layer7': 'cv', + 'layer8': 'ad', + 'layer9': 'ad', + 'layer10': 'ad', + 'layer11': 'cv', + 'layer12': 'ad', + 'layer13': 'ad', + 'layer14': 'ad', + 'layer15': 'cv', + }, + 'rrrv4': { + 'layer0': 'rd', + 'layer1': 'rd', + 'layer2': 'rd', + 'layer3': 'cv', + 'layer4': 'rd', + 'layer5': 'rd', + 'layer6': 'rd', + 'layer7': 'cv', + 'layer8': 'rd', + 'layer9': 'rd', + 'layer10': 'rd', + 'layer11': 'cv', + 'layer12': 'rd', + 'layer13': 'rd', + 'layer14': 'rd', + 'layer15': 'cv', + }, + 'c16': { + 'layer0': 'cd', + 'layer1': 'cd', + 'layer2': 'cd', + 'layer3': 'cd', + 'layer4': 'cd', + 'layer5': 'cd', + 'layer6': 'cd', + 'layer7': 'cd', + 'layer8': 'cd', + 'layer9': 'cd', + 'layer10': 'cd', + 'layer11': 'cd', + 'layer12': 'cd', + 'layer13': 'cd', + 'layer14': 'cd', + 'layer15': 'cd', + }, + 'a16': { + 'layer0': 'ad', + 'layer1': 'ad', + 'layer2': 'ad', + 'layer3': 'ad', + 'layer4': 'ad', + 'layer5': 'ad', + 'layer6': 'ad', + 'layer7': 'ad', + 'layer8': 'ad', + 'layer9': 'ad', + 'layer10': 'ad', + 'layer11': 'ad', + 'layer12': 'ad', + 'layer13': 'ad', + 'layer14': 'ad', + 'layer15': 'ad', + }, + 'r16': { + 'layer0': 'rd', + 'layer1': 'rd', + 'layer2': 'rd', + 'layer3': 'rd', + 'layer4': 'rd', + 'layer5': 'rd', + 'layer6': 'rd', + 'layer7': 'rd', + 'layer8': 'rd', + 'layer9': 'rd', + 'layer10': 'rd', + 'layer11': 'rd', + 'layer12': 'rd', + 'layer13': 'rd', + 'layer14': 'rd', + 'layer15': 'rd', + }, + 'carv4': { + 'layer0': 'cd', + 'layer1': 'ad', + 'layer2': 'rd', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'ad', + 'layer6': 'rd', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'ad', + 'layer10': 'rd', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'ad', + 'layer14': 'rd', + 'layer15': 'cv' + } +} + + +def create_conv_func(op_type): + assert op_type in ['cv', 'cd', 'ad', + 'rd'], 'unknown op type: %s' % str(op_type) + if op_type == 'cv': + return F.conv2d + if op_type == 'cd': + + def func(x, + weights, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + assert dilation in [1, + 2], 'dilation for cd_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size( + 3) == 3, 'kernel size for cd_conv should be 3x3' + assert padding == dilation, 'padding for cd_conv set wrong' + + weights_c = weights.sum(dim=[2, 3], keepdim=True) + yc = F.conv2d( + x, weights_c, stride=stride, padding=0, groups=groups) + y = F.conv2d( + x, + weights, + bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups) + return y - yc + + return func + elif op_type == 'ad': + + def func(x, + weights, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + assert dilation in [1, + 2], 'dilation for ad_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size( + 3) == 3, 'kernel size for ad_conv should be 3x3' + assert padding == dilation, 'padding for ad_conv set wrong' + + shape = weights.shape + weights = weights.view(shape[0], shape[1], -1) + weights_conv = (weights + - weights[:, :, [3, 0, 1, 6, 4, 2, 7, 8, 5]]).view( + shape) # clock-wise + y = F.conv2d( + x, + weights_conv, + bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups) + return y + + return func + elif op_type == 'rd': + + def func(x, + weights, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1): + assert dilation in [1, + 2], 'dilation for rd_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size( + 3) == 3, 'kernel size for rd_conv should be 3x3' + padding = 2 * dilation + + shape = weights.shape + if weights.is_cuda: + buffer = torch.cuda.FloatTensor(shape[0], shape[1], + 5 * 5).fill_(0) + else: + buffer = torch.zeros(shape[0], shape[1], 5 * 5) + weights = weights.view(shape[0], shape[1], -1) + buffer[:, :, [0, 2, 4, 10, 14, 20, 22, 24]] = weights[:, :, 1:] + buffer[:, :, [6, 7, 8, 11, 13, 16, 17, 18]] = -weights[:, :, 1:] + buffer[:, :, 12] = 0 + buffer = buffer.view(shape[0], shape[1], 5, 5) + y = F.conv2d( + x, + buffer, + bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups) + return y + + return func + else: + print('impossible to be here unless you force that', flush=True) + return None + + +def config_model(model): + model_options = list(CONFIGS.keys()) + assert model in model_options, \ + 'unrecognized model, please choose from %s' % str(model_options) + + pdcs = [] + for i in range(16): + layer_name = 'layer%d' % i + op = CONFIGS[model][layer_name] + pdcs.append(create_conv_func(op)) + return pdcs + + +def config_model_converted(model): + model_options = list(CONFIGS.keys()) + assert model in model_options, \ + 'unrecognized model, please choose from %s' % str(model_options) + + pdcs = [] + for i in range(16): + layer_name = 'layer%d' % i + op = CONFIGS[model][layer_name] + pdcs.append(op) + return pdcs + + +def convert_pdc(op, weight): + if op == 'cv': + return weight + elif op == 'cd': + shape = weight.shape + weight_c = weight.sum(dim=[2, 3]) + weight = weight.view(shape[0], shape[1], -1) + weight[:, :, 4] = weight[:, :, 4] - weight_c + weight = weight.view(shape) + return weight + elif op == 'ad': + shape = weight.shape + weight = weight.view(shape[0], shape[1], -1) + weight_conv = (weight + - weight[:, :, [3, 0, 1, 6, 4, 2, 7, 8, 5]]).view(shape) + return weight_conv + elif op == 'rd': + shape = weight.shape + buffer = torch.zeros(shape[0], shape[1], 5 * 5, device=weight.device) + weight = weight.view(shape[0], shape[1], -1) + buffer[:, :, [0, 2, 4, 10, 14, 20, 22, 24]] = weight[:, :, 1:] + buffer[:, :, [6, 7, 8, 11, 13, 16, 17, 18]] = -weight[:, :, 1:] + buffer = buffer.view(shape[0], shape[1], 5, 5) + return buffer + raise ValueError('wrong op {}'.format(str(op))) + + +def convert_pidinet(state_dict, config): + pdcs = config_model_converted(config) + new_dict = {} + for pname, p in state_dict.items(): + if 'init_block.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[0], p) + elif 'block1_1.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[1], p) + elif 'block1_2.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[2], p) + elif 'block1_3.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[3], p) + elif 'block2_1.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[4], p) + elif 'block2_2.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[5], p) + elif 'block2_3.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[6], p) + elif 'block2_4.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[7], p) + elif 'block3_1.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[8], p) + elif 'block3_2.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[9], p) + elif 'block3_3.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[10], p) + elif 'block3_4.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[11], p) + elif 'block4_1.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[12], p) + elif 'block4_2.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[13], p) + elif 'block4_3.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[14], p) + elif 'block4_4.conv1.weight' in pname: + new_dict[pname] = convert_pdc(pdcs[15], p) + else: + new_dict[pname] = p + return new_dict + + +class Conv2d(nn.Module): + + def __init__(self, + pdc, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=False): + super(Conv2d, self).__init__() + if in_channels % groups != 0: + raise ValueError('in_channels must be divisible by groups') + if out_channels % groups != 0: + raise ValueError('out_channels must be divisible by groups') + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // groups, kernel_size, + kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + self.reset_parameters() + self.pdc = pdc + + def reset_parameters(self): + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + nn.init.uniform_(self.bias, -bound, bound) + + def forward(self, input): + return self.pdc(input, self.weight, self.bias, self.stride, + self.padding, self.dilation, self.groups) + + +class CSAM(nn.Module): + r""" + Compact Spatial Attention Module + """ + + def __init__(self, channels): + super(CSAM, self).__init__() + + mid_channels = 4 + self.relu1 = nn.ReLU() + self.conv1 = nn.Conv2d( + channels, mid_channels, kernel_size=1, padding=0) + self.conv2 = nn.Conv2d( + mid_channels, 1, kernel_size=3, padding=1, bias=False) + self.sigmoid = nn.Sigmoid() + nn.init.constant_(self.conv1.bias, 0) + + def forward(self, x): + y = self.relu1(x) + y = self.conv1(y) + y = self.conv2(y) + y = self.sigmoid(y) + + return x * y + + +class CDCM(nn.Module): + r""" + Compact Dilation Convolution based Module + """ + + def __init__(self, in_channels, out_channels): + super(CDCM, self).__init__() + + self.relu1 = nn.ReLU() + self.conv1 = nn.Conv2d( + in_channels, out_channels, kernel_size=1, padding=0) + self.conv2_1 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + dilation=5, + padding=5, + bias=False) + self.conv2_2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + dilation=7, + padding=7, + bias=False) + self.conv2_3 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + dilation=9, + padding=9, + bias=False) + self.conv2_4 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + dilation=11, + padding=11, + bias=False) + nn.init.constant_(self.conv1.bias, 0) + + def forward(self, x): + x = self.relu1(x) + x = self.conv1(x) + x1 = self.conv2_1(x) + x2 = self.conv2_2(x) + x3 = self.conv2_3(x) + x4 = self.conv2_4(x) + return x1 + x2 + x3 + x4 + + +class MapReduce(nn.Module): + r""" + Reduce feature maps into a single edge map + """ + + def __init__(self, channels): + super(MapReduce, self).__init__() + self.conv = nn.Conv2d(channels, 1, kernel_size=1, padding=0) + nn.init.constant_(self.conv.bias, 0) + + def forward(self, x): + return self.conv(x) + + +class PDCBlock(nn.Module): + + def __init__(self, pdc, inplane, ouplane, stride=1): + super(PDCBlock, self).__init__() + self.stride = stride + + self.stride = stride + if self.stride > 1: + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.shortcut = nn.Conv2d( + inplane, ouplane, kernel_size=1, padding=0) + self.conv1 = Conv2d( + pdc, + inplane, + inplane, + kernel_size=3, + padding=1, + groups=inplane, + bias=False) + self.relu2 = nn.ReLU() + self.conv2 = nn.Conv2d( + inplane, ouplane, kernel_size=1, padding=0, bias=False) + + def forward(self, x): + if self.stride > 1: + x = self.pool(x) + y = self.conv1(x) + y = self.relu2(y) + y = self.conv2(y) + if self.stride > 1: + x = self.shortcut(x) + y = y + x + return y + + +class PDCBlock_converted(nn.Module): + r""" + CPDC, APDC can be converted to vanilla 3x3 convolution + RPDC can be converted to vanilla 5x5 convolution + """ + + def __init__(self, pdc, inplane, ouplane, stride=1): + super(PDCBlock_converted, self).__init__() + self.stride = stride + + if self.stride > 1: + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.shortcut = nn.Conv2d( + inplane, ouplane, kernel_size=1, padding=0) + if pdc == 'rd': + self.conv1 = nn.Conv2d( + inplane, + inplane, + kernel_size=5, + padding=2, + groups=inplane, + bias=False) + else: + self.conv1 = nn.Conv2d( + inplane, + inplane, + kernel_size=3, + padding=1, + groups=inplane, + bias=False) + self.relu2 = nn.ReLU() + self.conv2 = nn.Conv2d( + inplane, ouplane, kernel_size=1, padding=0, bias=False) + + def forward(self, x): + if self.stride > 1: + x = self.pool(x) + y = self.conv1(x) + y = self.relu2(y) + y = self.conv2(y) + if self.stride > 1: + x = self.shortcut(x) + y = y + x + return y + + +class PiDiNet(nn.Module): + + def __init__(self, inplane, pdcs, dil=None, sa=False, convert=False): + super(PiDiNet, self).__init__() + self.sa = sa + if dil is not None: + assert isinstance(dil, int), 'dil should be an int' + self.dil = dil + + self.fuseplanes = [] + + self.inplane = inplane + if convert: + if pdcs[0] == 'rd': + init_kernel_size = 5 + init_padding = 2 + else: + init_kernel_size = 3 + init_padding = 1 + self.init_block = nn.Conv2d( + 3, + self.inplane, + kernel_size=init_kernel_size, + padding=init_padding, + bias=False) + block_class = PDCBlock_converted + else: + self.init_block = Conv2d( + pdcs[0], 3, self.inplane, kernel_size=3, padding=1) + block_class = PDCBlock + + self.block1_1 = block_class(pdcs[1], self.inplane, self.inplane) + self.block1_2 = block_class(pdcs[2], self.inplane, self.inplane) + self.block1_3 = block_class(pdcs[3], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # C + + inplane = self.inplane + self.inplane = self.inplane * 2 + self.block2_1 = block_class(pdcs[4], inplane, self.inplane, stride=2) + self.block2_2 = block_class(pdcs[5], self.inplane, self.inplane) + self.block2_3 = block_class(pdcs[6], self.inplane, self.inplane) + self.block2_4 = block_class(pdcs[7], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 2C + + inplane = self.inplane + self.inplane = self.inplane * 2 + self.block3_1 = block_class(pdcs[8], inplane, self.inplane, stride=2) + self.block3_2 = block_class(pdcs[9], self.inplane, self.inplane) + self.block3_3 = block_class(pdcs[10], self.inplane, self.inplane) + self.block3_4 = block_class(pdcs[11], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 4C + + self.block4_1 = block_class( + pdcs[12], self.inplane, self.inplane, stride=2) + self.block4_2 = block_class(pdcs[13], self.inplane, self.inplane) + self.block4_3 = block_class(pdcs[14], self.inplane, self.inplane) + self.block4_4 = block_class(pdcs[15], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 4C + + self.conv_reduces = nn.ModuleList() + if self.sa and self.dil is not None: + self.attentions = nn.ModuleList() + self.dilations = nn.ModuleList() + for i in range(4): + self.dilations.append(CDCM(self.fuseplanes[i], self.dil)) + self.attentions.append(CSAM(self.dil)) + self.conv_reduces.append(MapReduce(self.dil)) + elif self.sa: + self.attentions = nn.ModuleList() + for i in range(4): + self.attentions.append(CSAM(self.fuseplanes[i])) + self.conv_reduces.append(MapReduce(self.fuseplanes[i])) + elif self.dil is not None: + self.dilations = nn.ModuleList() + for i in range(4): + self.dilations.append(CDCM(self.fuseplanes[i], self.dil)) + self.conv_reduces.append(MapReduce(self.dil)) + else: + for i in range(4): + self.conv_reduces.append(MapReduce(self.fuseplanes[i])) + + self.classifier = nn.Conv2d(4, 1, kernel_size=1) + nn.init.constant_(self.classifier.weight, 0.25) + nn.init.constant_(self.classifier.bias, 0) + + def get_weights(self): + conv_weights = [] + bn_weights = [] + relu_weights = [] + for pname, p in self.named_parameters(): + if 'bn' in pname: + bn_weights.append(p) + elif 'relu' in pname: + relu_weights.append(p) + else: + conv_weights.append(p) + + return conv_weights, bn_weights, relu_weights + + def forward(self, x): + H, W = x.size()[2:] + + x = self.init_block(x) + + x1 = self.block1_1(x) + x1 = self.block1_2(x1) + x1 = self.block1_3(x1) + + x2 = self.block2_1(x1) + x2 = self.block2_2(x2) + x2 = self.block2_3(x2) + x2 = self.block2_4(x2) + + x3 = self.block3_1(x2) + x3 = self.block3_2(x3) + x3 = self.block3_3(x3) + x3 = self.block3_4(x3) + + x4 = self.block4_1(x3) + x4 = self.block4_2(x4) + x4 = self.block4_3(x4) + x4 = self.block4_4(x4) + + x_fuses = [] + if self.sa and self.dil is not None: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.attentions[i](self.dilations[i](xi))) + elif self.sa: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.attentions[i](xi)) + elif self.dil is not None: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.dilations[i](xi)) + else: + x_fuses = [x1, x2, x3, x4] + + e1 = self.conv_reduces[0](x_fuses[0]) + e1 = F.interpolate(e1, (H, W), mode='bilinear', align_corners=False) + + e2 = self.conv_reduces[1](x_fuses[1]) + e2 = F.interpolate(e2, (H, W), mode='bilinear', align_corners=False) + + e3 = self.conv_reduces[2](x_fuses[2]) + e3 = F.interpolate(e3, (H, W), mode='bilinear', align_corners=False) + + e4 = self.conv_reduces[3](x_fuses[3]) + e4 = F.interpolate(e4, (H, W), mode='bilinear', align_corners=False) + + outputs = [e1, e2, e3, e4] + output = self.classifier(torch.cat(outputs, dim=1)) + + outputs.append(output) + outputs = [torch.sigmoid(r) for r in outputs] + return outputs[-1] + + +def pidinet_bsd_tiny(pretrained=False, vanilla_cnn=True): + pdcs = config_model_converted('carv4') if vanilla_cnn else config_model( + 'carv4') + model = PiDiNet(20, pdcs, dil=8, sa=True, convert=vanilla_cnn) + if pretrained: + state = torch.load( + DOWNLOAD_TO_CACHE('models/pidinet/table5_pidinet-tiny.pth'), + map_location='cpu')['state_dict'] + if vanilla_cnn: + state = convert_pidinet(state, 'carv4') + state = { + k[len('module.'):] if k.startswith('module.') else k: v + for k, v in state.items() + } + model.load_state_dict(state) + return model + + +def pidinet_bsd_small(pretrained=False, vanilla_cnn=True): + pdcs = config_model_converted('carv4') if vanilla_cnn else config_model( + 'carv4') + model = PiDiNet(30, pdcs, dil=12, sa=True, convert=vanilla_cnn) + if pretrained: + state = torch.load( + DOWNLOAD_TO_CACHE('models/pidinet/table5_pidinet-small.pth'), + map_location='cpu')['state_dict'] + if vanilla_cnn: + state = convert_pidinet(state, 'carv4') + state = { + k[len('module.'):] if k.startswith('module.') else k: v + for k, v in state.items() + } + model.load_state_dict(state) + return model + + +def pidinet_bsd(model_dir, pretrained=False, vanilla_cnn=True): + pdcs = config_model_converted('carv4') if vanilla_cnn else config_model( + 'carv4') + model = PiDiNet(60, pdcs, dil=24, sa=True, convert=vanilla_cnn) + if pretrained: + state = torch.load( + os.path.join(model_dir, 'table5_pidinet.pth'), + map_location='cpu')['state_dict'] + if vanilla_cnn: + state = convert_pidinet(state, 'carv4') + state = { + k[len('module.'):] if k.startswith('module.') else k: v + for k, v in state.items() + } + model.load_state_dict(state) + return model + + +def pidinet_nyud(pretrained=False, vanilla_cnn=True): + pdcs = config_model_converted('carv4') if vanilla_cnn else config_model( + 'carv4') + model = PiDiNet(60, pdcs, dil=24, sa=True, convert=vanilla_cnn) + if pretrained: + state = torch.load( + DOWNLOAD_TO_CACHE('models/pidinet/table6_pidinet.pth'), + map_location='cpu')['state_dict'] + if vanilla_cnn: + state = convert_pidinet(state, 'carv4') + state = { + k[len('module.'):] if k.startswith('module.') else k: v + for k, v in state.items() + } + model.load_state_dict(state) + return model + + +def pidinet_multicue(pretrained=False, vanilla_cnn=True): + pdcs = config_model_converted('carv4') if vanilla_cnn else config_model( + 'carv4') + model = PiDiNet(60, pdcs, dil=24, sa=True, convert=vanilla_cnn) + if pretrained: + state = torch.load( + DOWNLOAD_TO_CACHE('models/pidinet/table7_pidinet.pth'), + map_location='cpu')['state_dict'] + if vanilla_cnn: + state = convert_pidinet(state, 'carv4') + state = { + k[len('module.'):] if k.startswith('module.') else k: v + for k, v in state.items() + } + model.load_state_dict(state) + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/sketch_simplification.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/sketch_simplification.py new file mode 100644 index 0000000000000000000000000000000000000000..2555d593082d443a84dae0c4fa0b90d85cbe1d79 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/sketch/sketch_simplification.py @@ -0,0 +1,110 @@ +r"""PyTorch re-implementation adapted from the Lua code in ``https://github.com/bobbens/sketch_simplification''. +""" +import math +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.multi_modal.videocomposer.utils.utils import \ + DOWNLOAD_TO_CACHE + +__all__ = [ + 'SketchSimplification', 'sketch_simplification_gan', + 'sketch_simplification_mse', 'sketch_to_pencil_v1', 'sketch_to_pencil_v2' +] + + +class SketchSimplification(nn.Module): + r"""NOTE: + 1. Input image should has only one gray channel. + 2. Input image size should be divisible by 8. + 3. Sketch in the input/output image is in dark color while background in light color. + """ + + def __init__(self, mean, std): + assert isinstance(mean, float) and isinstance(std, float) + super(SketchSimplification, self).__init__() + self.mean = mean + self.std = std + + # layers + self.layers = nn.Sequential( + nn.Conv2d(1, 48, 5, 2, 2), nn.ReLU(inplace=True), + nn.Conv2d(48, 128, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(128, 128, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(128, 128, 3, 2, 1), nn.ReLU(inplace=True), + nn.Conv2d(128, 256, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(256, 256, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(256, 256, 3, 2, 1), nn.ReLU(inplace=True), + nn.Conv2d(256, 512, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(512, 1024, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(1024, 1024, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(1024, 1024, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(1024, 1024, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(1024, 512, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(512, 256, 3, 1, 1), nn.ReLU(inplace=True), + nn.ConvTranspose2d(256, 256, 4, 2, 1), nn.ReLU(inplace=True), + nn.Conv2d(256, 256, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(256, 128, 3, 1, 1), nn.ReLU(inplace=True), + nn.ConvTranspose2d(128, 128, 4, 2, 1), nn.ReLU(inplace=True), + nn.Conv2d(128, 128, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(128, 48, 3, 1, 1), nn.ReLU(inplace=True), + nn.ConvTranspose2d(48, 48, 4, 2, 1), nn.ReLU(inplace=True), + nn.Conv2d(48, 24, 3, 1, 1), nn.ReLU(inplace=True), + nn.Conv2d(24, 1, 3, 1, 1), nn.Sigmoid()) + + def forward(self, x): + r"""x: [B, 1, H, W] within range [0, 1]. Sketch pixels in dark color. + """ + x = (x - self.mean) / self.std + return self.layers(x) + + +def sketch_simplification_gan(model_dir, pretrained=False): + model = SketchSimplification( + mean=0.9664114577640158, std=0.0858381272736797) + if pretrained: + model.load_state_dict( + torch.load( + os.path.join(model_dir, 'sketch_simplification_gan.pth'), + map_location='cpu')) + return model + + +def sketch_simplification_mse(pretrained=False): + model = SketchSimplification( + mean=0.9664423107454593, std=0.08583666033640507) + if pretrained: + model.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE( + 'models/sketch_simplification/sketch_simplification_mse.pth' + ), + map_location='cpu')) + return model + + +def sketch_to_pencil_v1(pretrained=False): + model = SketchSimplification( + mean=0.9817833515894078, std=0.0925009022585048) + if pretrained: + model.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE( + 'models/sketch_simplification/sketch_to_pencil_v1.pth'), + map_location='cpu')) + return model + + +def sketch_to_pencil_v2(pretrained=False): + model = SketchSimplification( + mean=0.9851298627337799, std=0.07418377454883571) + if pretrained: + model.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE( + 'models/sketch_simplification/sketch_to_pencil_v2.pth'), + map_location='cpu')) + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/util.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/util.py new file mode 100644 index 0000000000000000000000000000000000000000..b02755c9080a1c0836d9c8ac746381c9c342c96e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/annotator/util.py @@ -0,0 +1,42 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import cv2 +import numpy as np + +annotator_ckpts_path = os.path.join(os.path.dirname(__file__), 'ckpts') + + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + + +def resize_image(input_image, resolution): + H, W, C = input_image.shape + H = float(H) + W = float(W) + k = float(resolution) / min(H, W) + H *= k + W *= k + H = int(np.round(H / 64.0)) * 64 + W = int(np.round(W / 64.0)) * 64 + img = cv2.resize( + input_image, (W, H), + interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA) + return img diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/autoencoder.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/autoencoder.py new file mode 100644 index 0000000000000000000000000000000000000000..99b65f879779d46092525e29af091d2318860859 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/autoencoder.py @@ -0,0 +1,650 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['AutoencoderKL'] + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm( + num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +class DiagonalGaussianDistribution(object): + + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean).to(device=self.parameters.device) + + def sample(self): + x = self.mean + self.std * torch.randn( + self.mean.shape).to(device=self.parameters.device) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3]) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var - 1.0 - self.logvar + other.logvar, + dim=[1, 2, 3]) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + + torch.pow(sample - self.mean, 2) / self.var, + dim=dims) + + def mode(self): + return self.mean + + +class Downsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResnetBlock(nn.Module): + + def __init__(self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm( + v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class AttnBlock(nn.Module): # noqa + + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c)**(-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm( + v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class Upsample(nn.Module): + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + x = torch.nn.functional.interpolate( + x, scale_factor=2.0, mode='nearest') + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): # noqa + + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode='constant', value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class Encoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + use_linear_attn=False, + attn_type='vanilla', + **ignore_kwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1, ) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + + def __init__(self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type='vanilla', + **ignorekwargs): + super().__init__() + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute in_ch_mult, block_in and curr_res at lowest res + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2**(self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + print('Working with z of shape {} = {} dimensions.'.format( + self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout)) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class AutoencoderKL(nn.Module): + + def __init__(self, + ddconfig, + embed_dim, + ckpt_path=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + ema_decay=None, + learn_logvar=False): + super().__init__() + self.learn_logvar = learn_logvar + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d(2 * ddconfig['z_channels'], + 2 * embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, + ddconfig['z_channels'], 1) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer('colorize', + torch.randn(3, colorize_nlabels, 1, 1)) + if monitor is not None: + self.monitor = monitor + self.use_ema = ema_decay is not None + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + for key in keys: + print(key, sd[key].shape) + import collections + sd_new = collections.OrderedDict() + for k in keys: + if k.find('first_stage_model') >= 0: + k_new = k.split('first_stage_model.')[-1] + sd_new[k_new] = sd[k] + self.load_state_dict(sd_new, strict=True) + print(f'Restored from {path}') + + def init_from_ckpt2(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + + first_stage_model + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + self.load_state_dict(sd, strict=False) + print(f'Restored from {path}') + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = x.permute(0, 3, 1, + 2).to(memory_format=torch.contiguous_format).float() + return x + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, log_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + if log_ema or self.use_ema: + with self.ema_scope(): + xrec_ema, posterior_ema = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec_ema.shape[1] > 3 + xrec_ema = self.to_rgb(xrec_ema) + log['samples_ema'] = self.decode( + torch.randn_like(posterior_ema.sample())) + log['reconstructions_ema'] = xrec_ema + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer('colorize', + torch.randn(3, x.shape[1], 1, 1).to(x)) + x = F.conv2d(x, weight=self.colorize) + x = 2. * (x - x.min()) / (x.max() - x.min()) - 1. + return x + + +class IdentityFirstStage(torch.nn.Module): + + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/clip.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..f55a520692cd288769840d58586bbd13b8b192a6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/clip.py @@ -0,0 +1,143 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import open_clip +import torch +import torch.nn as nn +import torchvision.transforms as T + + +class FrozenOpenCLIPEmbedder(nn.Module): + """ + Uses the OpenCLIP transformer encoder for text + """ + LAYERS = ['last', 'penultimate'] + + def __init__(self, + arch='ViT-H-14', + pretrained='laion2b_s32b_b79k', + device='cuda', + max_length=77, + freeze=True, + layer='last'): + super().__init__() + assert layer in self.LAYERS + model, _, _ = open_clip.create_model_and_transforms( + arch, device=torch.device('cpu'), pretrained=pretrained) + del model.visual + self.model = model + + self.device = device + self.max_length = max_length + if freeze: + self.freeze() + self.layer = layer + if self.layer == 'last': + self.layer_idx = 0 + elif self.layer == 'penultimate': + self.layer_idx = 1 + else: + raise NotImplementedError() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text): + tokens = open_clip.tokenize(text) + z = self.encode_with_transformer(tokens.to(self.device)) + return z + + def encode_with_transformer(self, text): + x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model] + x = x + self.model.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.model.ln_final(x) + return x + + def text_transformer_forward(self, x: torch.Tensor, attn_mask=None): + for i, r in enumerate(self.model.transformer.resblocks): + if i == len(self.model.transformer.resblocks) - self.layer_idx: + break + if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting( + ): + x = checkpoint(r, x, attn_mask) + else: + x = r(x, attn_mask=attn_mask) + return x + + def encode(self, text): + return self(text) + + +class FrozenOpenCLIPVisualEmbedder(nn.Module): + """ + Uses the OpenCLIP transformer encoder for text + """ + LAYERS = ['last', 'penultimate'] + + def __init__(self, + arch='ViT-H-14', + pretrained='laion2b_s32b_b79k', + device='cuda', + max_length=77, + freeze=True, + layer='last', + input_shape=(224, 224, 3)): + super().__init__() + assert layer in self.LAYERS + model, _, preprocess = open_clip.create_model_and_transforms( + arch, device=torch.device('cpu'), pretrained=pretrained) + del model.transformer + self.model = model + data_white = np.ones(input_shape, dtype=np.uint8) * 255 + self.black_image = preprocess(T.ToPILImage()(data_white)).unsqueeze(0) + self.preprocess = preprocess + + self.device = device + self.max_length = max_length # 77 + if freeze: + self.freeze() + self.layer = layer # 'penultimate' + if self.layer == 'last': + self.layer_idx = 0 + elif self.layer == 'penultimate': + self.layer_idx = 1 + else: + raise NotImplementedError() + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, image): + # tokens = open_clip.tokenize(text) + z = self.model.encode_image(image.to(self.device)) + return z + + def encode_with_transformer(self, text): + x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model] + x = x + self.model.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.model.ln_final(x) + return x + + def text_transformer_forward(self, x: torch.Tensor, attn_mask=None): + for i, r in enumerate(self.model.transformer.resblocks): + if i == len(self.model.transformer.resblocks) - self.layer_idx: + break + if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting( + ): + x = checkpoint(r, x, attn_mask) + else: + x = r(x, attn_mask=attn_mask) + return x + + def encode(self, text): + return self(text) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/config.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/config.py new file mode 100644 index 0000000000000000000000000000000000000000..499d84847bce06a52510642cb5c437f9c3b79969 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/config.py @@ -0,0 +1,156 @@ +import logging +import os +import os.path as osp +from datetime import datetime + +import torch +from easydict import EasyDict + +cfg = EasyDict(__name__='Config: VideoComposer') + +pmi_world_size = int(os.getenv('WORLD_SIZE', 1)) +gpus_per_machine = torch.cuda.device_count() +world_size = pmi_world_size * gpus_per_machine + +cfg.video_compositions = [ + 'text', 'mask', 'depthmap', 'sketch', 'motion', 'image', 'local_image', + 'single_sketch' +] + +# dataset +cfg.root_dir = 'webvid10m/' + +cfg.alpha = 0.7 + +cfg.misc_size = 384 +cfg.depth_std = 20.0 +cfg.depth_clamp = 10.0 +cfg.hist_sigma = 10.0 + +cfg.use_image_dataset = False +cfg.alpha_img = 0.7 + +cfg.resolution = 256 +cfg.mean = [0.5, 0.5, 0.5] +cfg.std = [0.5, 0.5, 0.5] + +# sketch +cfg.sketch_mean = [0.485, 0.456, 0.406] +cfg.sketch_std = [0.229, 0.224, 0.225] + +# dataloader +cfg.max_words = 1000 + +cfg.frame_lens = [ + 16, + 16, + 16, + 16, +] +cfg.feature_framerates = [ + 4, +] +cfg.feature_framerate = 4 +cfg.batch_sizes = { + str(1): 1, + str(4): 1, + str(8): 1, + str(16): 1, +} + +cfg.chunk_size = 64 +cfg.num_workers = 8 +cfg.prefetch_factor = 2 +cfg.seed = 8888 + +# diffusion +cfg.num_timesteps = 1000 +cfg.mean_type = 'eps' +cfg.var_type = 'fixed_small' +cfg.loss_type = 'mse' +cfg.ddim_timesteps = 50 +cfg.ddim_eta = 0.0 +cfg.clamp = 1.0 +cfg.share_noise = False +cfg.use_div_loss = False + +# classifier-free guidance +cfg.p_zero = 0.9 +cfg.guide_scale = 6.0 + +# stabel diffusion +cfg.sd_checkpoint = 'v2-1_512-ema-pruned.ckpt' + +# clip vision encoder +cfg.vit_image_size = 336 +cfg.vit_patch_size = 14 +cfg.vit_dim = 1024 +cfg.vit_out_dim = 768 +cfg.vit_heads = 16 +cfg.vit_layers = 24 +cfg.vit_mean = [0.48145466, 0.4578275, 0.40821073] +cfg.vit_std = [0.26862954, 0.26130258, 0.27577711] +cfg.clip_checkpoint = 'open_clip_pytorch_model.bin' +cfg.mvs_visual = False + +# unet +cfg.unet_in_dim = 4 +cfg.unet_concat_dim = 8 +cfg.unet_y_dim = cfg.vit_out_dim +cfg.unet_context_dim = 1024 +cfg.unet_out_dim = 8 if cfg.var_type.startswith('learned') else 4 +cfg.unet_dim = 320 +cfg.unet_dim_mult = [1, 2, 4, 4] +cfg.unet_res_blocks = 2 +cfg.unet_num_heads = 8 +cfg.unet_head_dim = 64 +cfg.unet_attn_scales = [1 / 1, 1 / 2, 1 / 4] +cfg.unet_dropout = 0.1 +cfg.misc_dropout = 0.5 +cfg.p_all_zero = 0.1 +cfg.p_all_keep = 0.1 +cfg.temporal_conv = False +cfg.temporal_attn_times = 1 +cfg.temporal_attention = True + +cfg.use_fps_condition = False +cfg.use_sim_mask = False + +# Default: load 2d pretrain +cfg.pretrained = False +cfg.fix_weight = False + +# Default resume +cfg.resume = True +cfg.resume_step = 148000 +cfg.resume_check_dir = '.' +cfg.resume_checkpoint = os.path.join( + cfg.resume_check_dir, + f'step_{cfg.resume_step}/non_ema_{cfg.resume_step}.pth') +cfg.resume_optimizer = False +if cfg.resume_optimizer: + cfg.resume_optimizer = os.path.join( + cfg.resume_check_dir, f'optimizer_step_{cfg.resume_step}.pt') + +# acceleration +cfg.use_ema = True +# for debug, no ema +if world_size < 2: + cfg.use_ema = False +cfg.load_from = None + +cfg.use_checkpoint = True +cfg.use_sharded_ddp = False +cfg.use_fsdp = False +cfg.use_fp16 = True + +# training +cfg.ema_decay = 0.9999 +cfg.viz_interval = 1000 +cfg.save_ckp_interval = 1000 + +# logging +cfg.log_interval = 100 +composition_strings = '_'.join(cfg.video_compositions) +# Default log_dir +cfg.log_dir = 'outputs/' diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba8d1233ab16a719114036b8ec45875a91e6299b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .samplers import * +from .tokenizers import * +from .transforms import * diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/samplers.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..4cadef95929f2d16b563a548e9109ce3e3e0a862 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/samplers.py @@ -0,0 +1,158 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os.path as osp + +import json +import numpy as np +from torch.utils.data.sampler import Sampler + +from modelscope.models.multi_modal.videocomposer.ops.distributed import ( + get_rank, get_world_size, shared_random_seed) +from modelscope.models.multi_modal.videocomposer.ops.utils import (ceil_divide, + read) + +__all__ = ['BatchSampler', 'GroupSampler', 'ImgGroupSampler'] + + +class BatchSampler(Sampler): + r"""An infinite batch sampler. + """ + + def __init__(self, + dataset_size, + batch_size, + num_replicas=None, + rank=None, + shuffle=False, + seed=None): + self.dataset_size = dataset_size + self.batch_size = batch_size + self.num_replicas = num_replicas or get_world_size() + self.rank = rank or get_rank() + self.shuffle = shuffle + self.seed = seed or shared_random_seed() + self.rng = np.random.default_rng(self.seed + self.rank) + self.batches_per_rank = ceil_divide( + dataset_size, self.num_replicas * self.batch_size) + self.samples_per_rank = self.batches_per_rank * self.batch_size + + # rank indices + indices = self.rng.permutation( + self.samples_per_rank) if shuffle else np.arange( + self.samples_per_rank) + indices = indices * self.num_replicas + self.rank + indices = indices[indices < dataset_size] + self.indices = indices + + def __iter__(self): + start = 0 + while True: + batch = [ + self.indices[i % len(self.indices)] + for i in range(start, start + self.batch_size) + ] + if self.shuffle and (start + self.batch_size) > len(self.indices): + self.rng.shuffle(self.indices) + start = (start + self.batch_size) % len(self.indices) + yield batch + + +class GroupSampler(Sampler): + + def __init__(self, + group_file, + batch_size, + alpha=0.7, + update_interval=5000, + seed=8888): + self.group_file = group_file + self.group_folder = osp.join(osp.dirname(group_file), 'groups') + self.batch_size = batch_size + self.alpha = alpha + self.update_interval = update_interval + self.seed = seed + self.rng = np.random.default_rng(seed) + + def __iter__(self): + while True: + # keep groups up-to-date + self.update_groups() + + # collect items + items = self.sample() + while len(items) < self.batch_size: + items += self.sample() + + # sample a batch + batch = self.rng.choice( + items, + self.batch_size, + replace=False if len(items) >= self.batch_size else True) + yield [u.strip().split(',') for u in batch] + + def update_groups(self): + if not hasattr(self, '_step'): + self._step = 0 + if self._step % self.update_interval == 0: + self.groups = json.loads(read(self.group_file)) + self._step += 1 + + def sample(self): + scales = np.array( + [float(next(iter(u)).split(':')[-1]) for u in self.groups]) + p = scales**self.alpha / (scales**self.alpha).sum() + group = self.rng.choice(self.groups, p=p) + list_file = osp.join(self.group_folder, + self.rng.choice(next(iter(group.values())))) + return read(list_file).strip().split('\n') + + +class ImgGroupSampler(Sampler): + + def __init__(self, + group_file, + batch_size, + alpha=0.7, + update_interval=5000, + seed=8888): + self.group_file = group_file + self.group_folder = osp.join(osp.dirname(group_file), 'groups') + self.batch_size = batch_size + self.alpha = alpha + self.update_interval = update_interval + self.seed = seed + self.rng = np.random.default_rng(seed) + + def __iter__(self): + while True: + # keep groups up-to-date + self.update_groups() + + # collect items + items = self.sample() + while len(items) < self.batch_size: + items += self.sample() + + # sample a batch + batch = self.rng.choice( + items, + self.batch_size, + replace=False if len(items) >= self.batch_size else True) + yield [u.strip().split(',', 1) for u in batch] + + def update_groups(self): + if not hasattr(self, '_step'): + self._step = 0 + if self._step % self.update_interval == 0: + self.groups = json.loads(read(self.group_file)) + + self._step += 1 + + def sample(self): + scales = np.array( + [float(next(iter(u)).split(':')[-1]) for u in self.groups]) + p = scales**self.alpha / (scales**self.alpha).sum() + group = self.rng.choice(self.groups, p=p) + list_file = osp.join(self.group_folder, + self.rng.choice(next(iter(group.values())))) + return read(list_file).strip().split('\n') diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/tokenizers.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/tokenizers.py new file mode 100644 index 0000000000000000000000000000000000000000..e3ab506b33552bc326ce3d00e7d7d4d9f0a1e632 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/tokenizers.py @@ -0,0 +1,184 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re +import torch +from tokenizers import BertWordPieceTokenizer, CharBPETokenizer + +__all__ = ['CLIPTokenizer'] + + +@lru_cache() +def default_bpe(): + root = os.path.realpath(__file__) + root = '/'.join(root.split('/')[:-1]) + return os.path.join(root, 'bpe_simple_vocab_16e6.txt.gz') + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception as e: + new_word.extend(word[i:]) + print(e) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + +class CLIPTokenizer(object): + + def __init__(self, length=77): + self.length = length + + # init tokenizer + self.tokenizer = SimpleTokenizer(bpe_path=default_bpe()) + self.sos_token = self.tokenizer.encoder['<|startoftext|>'] + self.eos_token = self.tokenizer.encoder['<|endoftext|>'] + self.vocab_size = len(self.tokenizer.encoder) + + def __call__(self, sequence): + if isinstance(sequence, str): + return torch.LongTensor(self._tokenizer(sequence)) + elif isinstance(sequence, list): + return torch.LongTensor([self._tokenizer(u) for u in sequence]) + else: + raise TypeError( + f'Expected the "sequence" to be a string or a list, but got {type(sequence)}' + ) + + def _tokenizer(self, text): + tokens = self.tokenizer.encode(text)[:self.length - 2] + tokens = [self.sos_token] + tokens + [self.eos_token] + tokens = tokens + [0] * (self.length - len(tokens)) + return tokens diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/transforms.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..3573fe86e54ee384b20850ce286cd3623ba58b9a --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/data/transforms.py @@ -0,0 +1,400 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import random + +import numpy as np +import torch +import torchvision.transforms.functional as F +import torchvision.transforms.functional as TF +from PIL import Image, ImageFilter +from torchvision.transforms.functional import InterpolationMode + +__all__ = [ + 'Compose', 'Resize', 'Rescale', 'CenterCrop', 'CenterCropV2', 'RandomCrop', + 'RandomCropV2', 'RandomHFlip', 'GaussianBlur', 'ColorJitter', 'RandomGray', + 'ToTensor', 'Normalize', 'ResizeRandomCrop', 'ExtractResizeRandomCrop', + 'ExtractResizeAssignCrop' +] + + +def random_resize(img, size): + img = [ + TF.resize( + u, + size, + interpolation=random.choice([ + InterpolationMode.BILINEAR, InterpolationMode.BICUBIC, + InterpolationMode.LANCZOS + ])) for u in img + ] + return img + + +class CenterCropV3(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + # fast resize + while min(img.size) >= 2 * self.size: + img = img.resize((img.width // 2, img.height // 2), + resample=Image.BOX) + scale = self.size / min(img.size) + img = img.resize((round(scale * img.width), round(scale * img.height)), + resample=Image.BICUBIC) + + # center crop + x1 = (img.width - self.size) // 2 + y1 = (img.height - self.size) // 2 + img = img.crop((x1, y1, x1 + self.size, y1 + self.size)) + return img + + +class Compose(object): + + def __init__(self, transforms): + self.transforms = transforms + + def __getitem__(self, index): + if isinstance(index, slice): + return Compose(self.transforms[index]) + else: + return self.transforms[index] + + def __len__(self): + return len(self.transforms) + + def __call__(self, rgb): + for t in self.transforms: + rgb = t(rgb) + return rgb + + +class Resize(object): + + def __init__(self, size=256): + if isinstance(size, int): + size = (size, size) + self.size = size + + def __call__(self, rgb): + + rgb = [u.resize(self.size, Image.BILINEAR) for u in rgb] + return rgb + + +class Rescale(object): + + def __init__(self, size=256, interpolation=Image.BILINEAR): + self.size = size + self.interpolation = interpolation + + def __call__(self, rgb): + w, h = rgb[0].size + scale = self.size / min(w, h) + out_w, out_h = int(round(w * scale)), int(round(h * scale)) + rgb = [u.resize((out_w, out_h), self.interpolation) for u in rgb] + return rgb + + +class CenterCrop(object): + + def __init__(self, size=224): + self.size = size + + def __call__(self, rgb): + w, h = rgb[0].size + assert min(w, h) >= self.size + x1 = (w - self.size) // 2 + y1 = (h - self.size) // 2 + rgb = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in rgb] + return rgb + + +class ResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + # self.min_area = min_area + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size # (518, 292) + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + + return rgb + + +class ExtractResizeRandomCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + out_w = self.size + out_h = self.size + w, h = rgb[0].size # (518, 292) + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + wh = [x1, y1, x1 + out_w, y1 + out_h] + + return rgb, wh + + +class ExtractResizeAssignCrop(object): + + def __init__(self, size=256, size_short=292): + self.size = size + self.size_short = size_short + + def __call__(self, rgb, wh): + + # consistent crop between rgb and m + while min(rgb[0].size) >= 2 * self.size_short: + rgb = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in rgb + ] + scale = self.size_short / min(rgb[0].size) + rgb = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in rgb + ] + + rgb = [u.crop(wh) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class CenterCropV2(object): + + def __init__(self, size): + self.size = size + + def __call__(self, img): + # fast resize + while min(img[0].size) >= 2 * self.size: + img = [ + u.resize((u.width // 2, u.height // 2), resample=Image.BOX) + for u in img + ] + scale = self.size / min(img[0].size) + img = [ + u.resize((round(scale * u.width), round(scale * u.height)), + resample=Image.BICUBIC) for u in img + ] + + # center crop + x1 = (img[0].width - self.size) // 2 + y1 = (img[0].height - self.size) // 2 + img = [u.crop((x1, y1, x1 + self.size, y1 + self.size)) for u in img] + return img + + +class RandomCrop(object): + + def __init__(self, size=224, min_area=0.4): + self.size = size + self.min_area = min_area + + def __call__(self, rgb): + + # consistent crop between rgb and m + w, h = rgb[0].size + area = w * h + out_w, out_h = float('inf'), float('inf') + while out_w > w or out_h > h: + target_area = random.uniform(self.min_area, 1.0) * area + aspect_ratio = random.uniform(3. / 4., 4. / 3.) + out_w = int(round(math.sqrt(target_area * aspect_ratio))) + out_h = int(round(math.sqrt(target_area / aspect_ratio))) + x1 = random.randint(0, w - out_w) + y1 = random.randint(0, h - out_h) + + rgb = [u.crop((x1, y1, x1 + out_w, y1 + out_h)) for u in rgb] + rgb = [u.resize((self.size, self.size), Image.BILINEAR) for u in rgb] + + return rgb + + +class RandomCropV2(object): + + def __init__(self, size=224, min_area=0.4, ratio=(3. / 4., 4. / 3.)): + if isinstance(size, (tuple, list)): + self.size = size + else: + self.size = (size, size) + self.min_area = min_area + self.ratio = ratio + + def _get_params(self, img): + width, height = img.size + area = height * width + + for _ in range(10): + target_area = random.uniform(self.min_area, 1.0) * area + log_ratio = (math.log(self.ratio[0]), math.log(self.ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if 0 < w <= width and 0 < h <= height: + i = random.randint(0, height - h) + j = random.randint(0, width - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = float(width) / float(height) + if (in_ratio < min(self.ratio)): + w = width + h = int(round(w / min(self.ratio))) + elif (in_ratio > max(self.ratio)): + h = height + w = int(round(h * max(self.ratio))) + else: # whole image + w = width + h = height + i = (height - h) // 2 + j = (width - w) // 2 + return i, j, h, w + + def __call__(self, rgb): + i, j, h, w = self._get_params(rgb[0]) + rgb = [F.resized_crop(u, i, j, h, w, self.size) for u in rgb] + return rgb + + +class RandomHFlip(object): + + def __init__(self, p=0.5): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.transpose(Image.FLIP_LEFT_RIGHT) for u in rgb] + return rgb + + +class GaussianBlur(object): + + def __init__(self, sigmas=[0.1, 2.0], p=0.5): + self.sigmas = sigmas + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + sigma = random.uniform(*self.sigmas) + rgb = [ + u.filter(ImageFilter.GaussianBlur(radius=sigma)) for u in rgb + ] + return rgb + + +class ColorJitter(object): + + def __init__(self, + brightness=0.4, + contrast=0.4, + saturation=0.4, + hue=0.1, + p=0.5): + self.brightness = brightness + self.contrast = contrast + self.saturation = saturation + self.hue = hue + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + brightness, contrast, saturation, hue = self._random_params() + transforms = [ + lambda f: F.adjust_brightness(f, brightness), + lambda f: F.adjust_contrast(f, contrast), + lambda f: F.adjust_saturation(f, saturation), + lambda f: F.adjust_hue(f, hue) + ] + random.shuffle(transforms) + for t in transforms: + rgb = [t(u) for u in rgb] + + return rgb + + def _random_params(self): + brightness = random.uniform( + max(0, 1 - self.brightness), 1 + self.brightness) + contrast = random.uniform(max(0, 1 - self.contrast), 1 + self.contrast) + saturation = random.uniform( + max(0, 1 - self.saturation), 1 + self.saturation) + hue = random.uniform(-self.hue, self.hue) + return brightness, contrast, saturation, hue + + +class RandomGray(object): + + def __init__(self, p=0.2): + self.p = p + + def __call__(self, rgb): + if random.random() < self.p: + rgb = [u.convert('L').convert('RGB') for u in rgb] + return rgb + + +class ToTensor(object): + + def __call__(self, rgb): + rgb = torch.stack([F.to_tensor(u) for u in rgb], dim=0) + return rgb + + +class Normalize(object): + + def __init__(self, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]): + self.mean = mean + self.std = std + + def __call__(self, rgb): + rgb = rgb.clone() + rgb.clamp_(0, 1) + if not isinstance(self.mean, torch.Tensor): + self.mean = rgb.new_tensor(self.mean).view(-1) + if not isinstance(self.std, torch.Tensor): + self.std = rgb.new_tensor(self.std).view(-1) + rgb.sub_(self.mean.view(1, -1, 1, 1)).div_(self.std.view(1, -1, 1, 1)) + return rgb diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/diffusion.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..8b669ca9516ba2de8a2c58db7de9fb7f404dd94e --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/diffusion.py @@ -0,0 +1,1514 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch + +from .dpm_solver import (DPM_Solver, NoiseScheduleVP, model_wrapper, + model_wrapper_guided_diffusion) +from .ops.losses import discretized_gaussian_log_likelihood, kl_divergence + +__all__ = ['GaussianDiffusion', 'beta_schedule', 'GaussianDiffusion_style'] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + if tensor.device != x.device: + tensor = tensor.to(x.device) + return tensor[t].view(shape).to(x) + + +def beta_schedule(schedule, + num_timesteps=1000, + init_beta=None, + last_beta=None): + ''' + This code defines a function beta_schedule that generates a sequence of beta + values based on the given input parameters. + These beta values can be used in video diffusion processes. The function has the following parameters: + schedule(str): Determines the type of beta schedule to be generated. + It can be 'linear', 'linear_sd', 'quadratic', or 'cosine'. + num_timesteps(int, optional): The number of timesteps for the generated beta schedule. Default is 1000. + init_beta(float, optional): The initial beta value. + If not provided, a default value is used based on the chosen schedule. + last_beta(float, optional): The final beta value. + If not provided, a default value is used based on the chosen schedule. + The function returns a PyTorch tensor containing the generated beta values. + The beta schedule is determined by the schedule parameter: + 1.Linear: Generates a linear sequence of beta values betweeninit_betaandlast_beta. + 2.Linear_sd: Generates a linear sequence of beta values between the square root of + init_beta and the square root oflast_beta, and then squares the result. + 3.Quadratic: Similar to the 'linear_sd' schedule, but with different default values forinit_betaandlast_beta. + 4.Cosine: Generates a sequence of beta values based on a cosine function, + ensuring the values are between 0 and 0.999. + If an unsupported schedule is provided, a ValueError is raised with a message indicating the issue. + ''' + if schedule == 'linear': + scale = 1000.0 / num_timesteps + init_beta = init_beta or scale * 0.0001 + last_beta = last_beta or scale * 0.02 + return torch.linspace( + init_beta, last_beta, num_timesteps, dtype=torch.float64) + elif schedule == 'linear_sd': + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'quadratic': + init_beta = init_beta or 0.0015 + last_beta = last_beta or 0.0195 + return torch.linspace( + init_beta**0.5, last_beta**0.5, num_timesteps, + dtype=torch.float64)**2 + elif schedule == 'cosine': + betas = [] + for step in range(num_timesteps): + t1 = step / num_timesteps + t2 = (step + 1) / num_timesteps + fn = lambda u: math.cos( # noqa + (u + 0.008) / 1.008 * math.pi / 2)**2 # noqa + betas.append(min(1.0 - fn(t2) / fn(t1), 0.999)) + return torch.tensor(betas, dtype=torch.float64) + else: + raise ValueError(f'Unsupported schedule: {schedule}') + + +def load_stable_diffusion_pretrained(state_dict, temporal_attention): + import collections + sd_new = collections.OrderedDict() + keys = list(state_dict.keys()) + + for k in keys: + if k.find('diffusion_model') >= 0: + k_new = k.split('diffusion_model.')[-1] + if k_new in [ + 'input_blocks.3.0.op.weight', 'input_blocks.3.0.op.bias', + 'input_blocks.6.0.op.weight', 'input_blocks.6.0.op.bias', + 'input_blocks.9.0.op.weight', 'input_blocks.9.0.op.bias' + ]: + k_new = k_new.replace('0.op', 'op') + if temporal_attention: + if k_new.find('middle_block.2') >= 0: + k_new = k_new.replace('middle_block.2', 'middle_block.3') + if k_new.find('output_blocks.5.2') >= 0: + k_new = k_new.replace('output_blocks.5.2', + 'output_blocks.5.3') + if k_new.find('output_blocks.8.2') >= 0: + k_new = k_new.replace('output_blocks.8.2', + 'output_blocks.8.3') + sd_new[k_new] = state_dict[k] + + return sd_new + + +class AddGaussianNoise(object): + + def __init__(self, mean=0., std=0.1): + self.std = std + self.mean = mean + + def __call__(self, img): + assert isinstance(img, torch.Tensor) + dtype = img.dtype + if not img.is_floating_point(): + img = img.to(torch.float32) + out = img + self.std * torch.randn_like(img) + self.mean + if out.dtype != dtype: + out = out.to(dtype) + return out + + def __repr__(self): + return self.__class__.__name__ + '(mean={0}, std={1})'.format( + self.mean, self.std) + + +class GaussianDiffusion(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='learned_range', + loss_type='mse', + epsilon=1e-12, + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1', + 'charbonnier' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type # eps + self.var_type = var_type # 'fixed_small' + self.loss_type = loss_type # mse + self.epsilon = epsilon # 1e-12 + self.rescale_timesteps = rescale_timesteps # False + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + noise = torch.randn_like(x0) if noise is None else noise + return _i(self.sqrt_alphas_cumprod, t, x0) * x0 + \ + _i(self.sqrt_one_minus_alphas_cumprod, t, x0) * noise # noqa + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + mask = t.ne(0).float().view( + -1, + *((1, ) * # noqa + (xt.ndim - 1))) + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + mask * torch.exp(0.5 * log_var) * noise + return xt_1, x0 + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, self._scale_timesteps(t), **model_kwargs[0]) + u_out = model(xt, self._scale_timesteps(t), **model_kwargs[1]) + dim = y_out.size(1) if self.var_type.startswith( + 'fixed') else y_out.size(1) // 2 + out = torch.cat( + [ + u_out[:, :dim] + guide_scale * # noqa + (y_out[:, :dim] - u_out[:, :dim]), + y_out[:, dim:] + ], + dim=1) # noqa + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - \ + _i(self.posterior_mean_coef2 / self.posterior_mean_coef1, t, xt) * xt # noqa + elif self.mean_type == 'x0': + x0 = out + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * out # noqa + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1) + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return mu, var, log_var, x0 + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + r"""Sample from p(x_{t-1} | x_t) using DDIM. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + sigmas = eta * torch.sqrt((1 - alphas_prev) / (1 - alphas) * # noqa + (1 - alphas / alphas_prev)) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + mask * sigmas * noise + return xt_1, x0 + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process (TODO: clamp is inaccurate! Consider replacing the stride by explicit prev/next steps) + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + # import ipdb; ipdb.set_trace() + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + stride = self.num_timesteps // ddim_timesteps + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), + (t + stride).clamp(0, self.num_timesteps), xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu, x0 + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b = x0.size(0) + xt = x0 + + # reconstruction steps + steps = torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps) + for step in steps: + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, model, model_kwargs, clamp, + percentile, guide_scale, + ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + stride = self.num_timesteps // plms_timesteps + + # function for compute eps + def compute_eps(xt, t): + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + return eps + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, (t - stride).clamp(0), xt) + direction = torch.sqrt(1 - alphas_prev) * eps + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1, x0 + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, (t - stride).clamp(0)) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for step in steps: + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _, eps = self.plms_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + def loss(self, + x0, + t, + model, + model_kwargs={}, + noise=None, + weight=None, + use_div_loss=False): + noise = torch.randn_like( + x0) if noise is None else noise # [80, 4, 8, 32, 32] + xt = self.q_sample(x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', + 'rescaled_l1']: # self.loss_type: mse + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range' + ]: # self.var_type: 'fixed_small' + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = (out - target).pow(1 if self.loss_type.endswith('l1') else 2 + ).abs().flatten(1).mean(dim=1) + if weight is not None: + loss = loss * weight + + # div loss + if use_div_loss and self.mean_type == 'eps' and x0.shape[2] > 1: + + # derive x0 + x0_ = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * out + + # ncfhw, std on f + div_loss = 0.001 / ( + x0_.std(dim=2).flatten(1).mean(dim=1) + 1e-4) + loss = loss + div_loss + + # total loss + loss = loss + loss_vlb + elif self.loss_type in ['charbonnier']: + out = model(xt, self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([out.detach(), var], dim=1) + loss_vlb, _ = self.variational_lower_bound( + x0, xt, t, model=lambda *args, **kwargs: frozen) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = torch.sqrt((out - target)**2 + self.epsilon) + if weight is not None: + loss = loss * weight + loss = loss.flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None): + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) + kl = kl.flatten(1).mean(dim=1) / math.log(2.0) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) + nll = nll.flatten(1).mean(dim=1) / math.log(2.0) + + # NLL for p(x0 | x1) and KL otherwise + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + r"""Compute the entire variational lower bound, measured in bits-per-dim. + """ + # prepare input and output + b = x0.size(0) + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: # noqa + return t.float() * 1000.0 / self.num_timesteps + return t + + +class GaussianDiffusion_style(object): + + def __init__(self, + betas, + mean_type='eps', + var_type='fixed_small', + loss_type='mse', + rescale_timesteps=False): + # check input + if not isinstance(betas, torch.DoubleTensor): + betas = torch.tensor(betas, dtype=torch.float64) + assert min(betas) > 0 and max(betas) <= 1 + assert mean_type in ['x0', 'x_{t-1}', 'eps'] + assert var_type in [ + 'learned', 'learned_range', 'fixed_large', 'fixed_small' + ] + assert loss_type in [ + 'mse', 'rescaled_mse', 'kl', 'rescaled_kl', 'l1', 'rescaled_l1' + ] + self.betas = betas + self.num_timesteps = len(betas) + self.mean_type = mean_type + self.var_type = var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + # alphas + alphas = 1 - self.betas + self.alphas_cumprod = torch.cumprod(alphas, dim=0) + self.alphas_cumprod_prev = torch.cat( + [alphas.new_ones([1]), self.alphas_cumprod[:-1]]) + self.alphas_cumprod_next = torch.cat( + [self.alphas_cumprod[1:], + alphas.new_zeros([1])]) + + # q(x_t | x_{t-1}) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 + - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = torch.log(1.0 + - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = torch.sqrt(1.0 / self.alphas_cumprod + - 1) + + # q(x_{t-1} | x_t, x_0) + self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / ( + 1.0 - self.alphas_cumprod) + self.posterior_log_variance_clipped = torch.log( + self.posterior_variance.clamp(1e-20)) + self.posterior_mean_coef1 = betas * torch.sqrt( + self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + self.posterior_mean_coef2 = ( + 1.0 - self.alphas_cumprod_prev) * torch.sqrt(alphas) / ( + 1.0 - self.alphas_cumprod) + + def q_sample(self, x0, t, noise=None): + r"""Sample from q(x_t | x_0). + """ + noise = torch.randn_like(x0) if noise is None else noise + xt = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + \ + _i(self.sqrt_one_minus_alphas_cumprod, t, x0) * noise # noqa + return xt.type_as(x0) + + def q_mean_variance(self, x0, t): + r"""Distribution of q(x_t | x_0). + """ + mu = _i(self.sqrt_alphas_cumprod, t, x0) * x0 + var = _i(1.0 - self.alphas_cumprod, t, x0) + log_var = _i(self.log_one_minus_alphas_cumprod, t, x0) + return mu, var, log_var + + def q_posterior_mean_variance(self, x0, xt, t): + r"""Distribution of q(x_{t-1} | x_t, x_0). + """ + mu = _i(self.posterior_mean_coef1, t, xt) * x0 + _i( + self.posterior_mean_coef2, t, xt) * xt + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + return mu, var, log_var + + @torch.no_grad() + def p_sample(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t). + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample (with optional conditional function) + noise = torch.randn_like(xt) + t_mask = t.ne(0).float().view( + -1, + *((1, ) * # noqa + (xt.ndim - 1))) + if condition_fn is not None: + grad = condition_fn(xt, self._scale_timesteps(t), **model_kwargs) + mu = mu.float() + var * grad.float() + xt_1 = mu + t_mask * torch.exp(0.5 * log_var) * noise + return xt_1.type(dtype), x0.type(dtype) + + @torch.no_grad() + def p_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None): + r"""Sample from p(x_{t-1} | x_t) p(x_{t-2} | x_{t-1}) ... p(x_0 | x_1). + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.p_sample(xt, t, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale) + return xt + + def p_mean_variance(self, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""Distribution of p(x_{t-1} | x_t). + """ + # predict distribution + if guide_scale is None: + out = model(xt, t=self._scale_timesteps(t), **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = model(xt, t=self._scale_timesteps(t), **model_kwargs[0]) + if guide_scale != 1.0: + u_out = model( + xt, t=self._scale_timesteps(t), **model_kwargs[1]) + dim = y_out.size(1) if self.var_type.startswith( + 'fixed') else y_out.size(1) // 2 + out = torch.cat( + [ + u_out[:, :dim] + guide_scale * # noqa + (y_out[:, :dim] - u_out[:, :dim]), + y_out[:, dim:] + ], + dim=1) # noqa + else: + out = y_out + + # compute variance + if self.var_type == 'learned': + out, log_var = out.chunk(2, dim=1) + var = torch.exp(log_var) + elif self.var_type == 'learned_range': + out, fraction = out.chunk(2, dim=1) + min_log_var = _i(self.posterior_log_variance_clipped, t, xt) + max_log_var = _i(torch.log(self.betas), t, xt) + fraction = (fraction + 1) / 2.0 + log_var = fraction * max_log_var + (1 - fraction) * min_log_var + var = torch.exp(log_var) + elif self.var_type == 'fixed_large': + var = _i( + torch.cat([self.posterior_variance[1:2], self.betas[1:]]), t, + xt) + log_var = torch.log(var) + elif self.var_type == 'fixed_small': + var = _i(self.posterior_variance, t, xt) + log_var = _i(self.posterior_log_variance_clipped, t, xt) + + # compute mean and x0 + if self.mean_type == 'x_{t-1}': + mu = out # x_{t-1} + x0 = _i(1.0 / self.posterior_mean_coef1, t, xt) * mu - \ + _i(self.posterior_mean_coef2 / self.posterior_mean_coef1, t, xt) * xt # noqa + elif self.mean_type == 'x0': + x0 = out + elif self.mean_type == 'eps': + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * out # noqa + + # restrict the range of x0 + if percentile is not None: + assert percentile > 0 and percentile <= 1 # e.g., 0.995 + s = torch.quantile( + x0.flatten(1).abs(), percentile, + dim=1).clamp_(1.0).view(-1, 1, 1, 1, 1) + # s = torch.quantile(x0.flatten(1).abs(), percentile, dim=1).clamp_(1.0).view(-1, 1, 1, 1) # old + x0 = torch.min(s, torch.max(-s, x0)) / s + elif clamp is not None: + x0 = x0.clamp(-clamp, clamp) + + # recompute mu using the restricted x0 + mu, _, _ = self.q_posterior_mean_variance(x0, xt, t) + return mu, var, log_var, x0 + + @torch.no_grad() + def ddim_sample(self, + xt, + t, + t_prev, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + r"""Sample from p(x_{t-1} | x_t) using DDIM. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + alphas = _i(self.alphas_cumprod, t, xt) + alphas_prev = _i(self.alphas_cumprod, t_prev, xt) + sigmas = eta * torch.sqrt((1 - alphas_prev) / (1 - alphas) * # noqa + (1 - alphas / alphas_prev)) + + # random sample + noise = torch.randn_like(xt) + direction = torch.sqrt(1 - alphas_prev - sigmas**2) * eps + t_mask = t.ne(0).float().view(-1, *((1, ) * (xt.ndim - 1))) + xt_1 = torch.sqrt( + alphas_prev) * x0 + direction + t_mask * sigmas * noise + return xt_1.type(dtype), x0.type(dtype) + + @torch.no_grad() + def ddim_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + ddim_timesteps=20, + eta=0.0): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + for i, step in enumerate(steps): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + t_prev = torch.full((b, ), + steps[i + 1] if i < len(steps) - 1 else 0, + dtype=torch.long, + device=xt.device) + xt, _ = self.ddim_sample(xt, t, t_prev, model, model_kwargs, clamp, + percentile, condition_fn, guide_scale, + ddim_timesteps, eta) + return xt + + @torch.no_grad() + def ddim_reverse_sample(self, + xt, + t, + t_next, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + r"""Sample from p(x_{t+1} | x_t) using DDIM reverse ODE (deterministic). + """ + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, clamp, + percentile, guide_scale) + + # derive variables + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + alphas_next = _i( + torch.cat( + [self.alphas_cumprod, + self.alphas_cumprod.new_zeros([1])]), t_next, xt) + + # reverse sample + mu = torch.sqrt(alphas_next) * x0 + torch.sqrt(1 - alphas_next) * eps + return mu.type(dtype), x0.type(dtype) + + @torch.no_grad() + def ddim_reverse_sample_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + ddim_timesteps=20): + # prepare input + b = x0.size(0) + xt = x0 + + # reconstruction steps + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // ddim_timesteps)).clamp( + 0, self.num_timesteps - 1) + for i, step in enumerate(steps): + t = torch.full((b, ), + steps[i - 1] if i > 0 else 0, + dtype=torch.long, + device=xt.device) + t_next = torch.full((b, ), + step, + dtype=torch.long, + device=xt.device) + xt, _ = self.ddim_reverse_sample(xt, t, t_next, model, + model_kwargs, clamp, percentile, + guide_scale, ddim_timesteps) + return xt + + @torch.no_grad() + def plms_sample(self, + xt, + t, + t_prev, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + r"""Sample from p(x_{t-1} | x_t) using PLMS. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + """ + + # function for compute eps + def compute_eps(xt, t): + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t) + _, _, _, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, guide_scale) + + # condition + if condition_fn is not None: + # x0 -> eps + alpha = _i(self.alphas_cumprod, t, xt) + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + eps = eps - (1 - alpha).sqrt() * condition_fn( + xt, self._scale_timesteps(t), **model_kwargs) + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # derive eps + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + return eps.type(dtype) + + # function for compute x_0 and x_{t-1} + def compute_x0(eps, t): + dtype = eps.dtype + + # eps -> x0 + x0 = _i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) * eps # noqa + + # deterministic sample + alphas_prev = _i(self.alphas_cumprod, t_prev, xt) + direction = torch.sqrt(1 - alphas_prev) * eps + xt_1 = torch.sqrt(alphas_prev) * x0 + direction + return xt_1.type(dtype), x0.type(dtype) + + # PLMS sample + eps = compute_eps(xt, t) + if len(eps_cache) == 0: + # 2nd order pseudo improved Euler + xt_1, x0 = compute_x0(eps, t) + eps_next = compute_eps(xt_1, t_prev) + eps_prime = (eps + eps_next) / 2.0 + elif len(eps_cache) == 1: + # 2nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (3 * eps - eps_cache[-1]) / 2.0 + elif len(eps_cache) == 2: + # 3nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (23 * eps - 16 * eps_cache[-1] + + 5 * eps_cache[-2]) / 12.0 + elif len(eps_cache) >= 3: + # 4nd order pseudo linear multistep (Adams-Bashforth) + eps_prime = (55 * eps - 59 * eps_cache[-1] + 37 * eps_cache[-2] + - 9 * eps_cache[-3]) / 24.0 + xt_1, x0 = compute_x0(eps_prime, t) + return xt_1, x0, eps + + @torch.no_grad() + def plms_sample_loop(self, + noise, + model, + model_kwargs={}, + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + plms_timesteps=20): + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + steps = (1 + torch.arange(0, self.num_timesteps, + self.num_timesteps // plms_timesteps)).clamp( + 0, self.num_timesteps - 1).flip(0) + eps_cache = [] + for i, step in enumerate(steps): + # PLMS sampling step + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + t_prev = torch.full((b, ), + steps[i + 1] if i < len(steps) - 1 else 0, + dtype=torch.long, + device=xt.device) + xt, _, eps = self.plms_sample(xt, t, t_prev, model, model_kwargs, + clamp, percentile, condition_fn, + guide_scale, plms_timesteps, + eps_cache) + + # update eps cache + eps_cache.append(eps) + if len(eps_cache) >= 4: + eps_cache.pop(0) + return xt + + @torch.no_grad() + def dpm_solver_sample_loop(self, + noise, + model, + model_kwargs={}, + order=2, + skip_type='logSNR', + method='multistep', + clamp=None, + percentile=None, + condition_fn=None, + guide_scale=None, + dpm_solver_timesteps=20, + algorithm_type='dpmsolver++', + t_start=None, + t_end=None, + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpmsolver'): + r"""Sample using DPM-Solver-based method. + - condition_fn: for classifier-based guidance (guided-diffusion). + - guide_scale: for classifier-free guidance (glide/dalle-2). + + Please check all the parameters in `dpm_solver.sample` before using. + """ + assert self.mean_type in ('eps', 'x0') + assert percentile in (None, 0.995) + assert clamp is None or percentile is None + noise_schedule = NoiseScheduleVP( + schedule='discrete', betas=self.betas.float()) + model_fn = model_wrapper_guided_diffusion( + model=model, + noise_schedule=noise_schedule, + var_type=self.var_type, + mean_type=self.mean_type, + model_kwargs=model_kwargs, + rescale_timesteps=self.rescale_timesteps, + num_timesteps=self.num_timesteps, + guide_scale=guide_scale, + condition_fn=condition_fn) + dpm_solver = DPM_Solver( + model_fn=model_fn, + noise_schedule=noise_schedule, + algorithm_type=algorithm_type, + percentile=percentile, + clamp=clamp) + xt = dpm_solver.sample( + noise, + steps=dpm_solver_timesteps, + order=order, + skip_type=skip_type, + method=method, + solver_type=solver_type, + t_start=t_start, + t_end=t_end, + lower_order_final=lower_order_final, + denoise_to_zero=denoise_to_zero) + return xt + + @torch.no_grad() + def inpaint_p_sample(self, + xt, + t, + y, + mask, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""DDPM sampling step for inpainting. + """ + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t), conditioned on y and mask + xt = self.q_sample(y, t) * mask + xt * (1 - mask) + mu, var, log_var, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile, + guide_scale) + + # random sample + t_mask = t.ne(0).float().view( + -1, + *((1, ) * # noqa + (xt.ndim - 1))) + xt_1 = mu + t_mask * torch.exp(0.5 * log_var) * torch.randn_like(xt) + return xt_1.type(dtype), x0.type(dtype) + + @torch.no_grad() + def inpaint_p_sample_loop(self, + noise, + y, + mask, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None): + r"""DDPM sampling loop for inpainting. + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.inpaint_p_sample(xt, t, y, mask, model, model_kwargs, + clamp, percentile, guide_scale) + return xt + + @torch.no_grad() + def inpaint_mcg_p_sample(self, + xt, + t, + y, + mask, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + mcg_scale=1.0): + r"""DDPM sampling step for inpainting, with Manifold Constrained Gradient (MCG) correction. + """ + dtype = xt.dtype + + # predict distribution of p(x_{t-1} | x_t), conditioned on y and mask + with torch.enable_grad(): + xt.requires_grad_(True) + mu, var, log_var, x0 = self.p_mean_variance( + xt, t, model, model_kwargs, clamp, percentile, guide_scale) + loss = (y * mask - x0 * mask).square().mean() + grad = torch.autograd.grad(loss, xt)[0] + + # random sample + t_mask = t.ne(0).float().view( + -1, + *((1, ) * # noqa + (xt.ndim - 1))) + xt_1 = mu + t_mask * torch.exp(0.5 * log_var) * torch.randn_like(xt) + xt_1 = xt_1 - mcg_scale * grad + + # merge foreground and background + xt_1 = self.q_sample(y, t) * mask + xt_1 * (1 - mask) + return xt_1.type(dtype), x0.type(dtype) + + @torch.no_grad() + def inpaint_mcg_p_sample_loop(self, + noise, + y, + mask, + model, + model_kwargs={}, + clamp=None, + percentile=None, + guide_scale=None, + mcg_scale=1.0): + r"""DDPM sampling loop for inpainting, with Manifold Constrained Gradient (MCG) correction. + """ + # prepare input + b = noise.size(0) + xt = noise + + # diffusion process + for step in torch.arange(self.num_timesteps).flip(0): + t = torch.full((b, ), step, dtype=torch.long, device=xt.device) + xt, _ = self.inpaint_mcg_p_sample(xt, t, y, mask, model, + model_kwargs, clamp, percentile, + guide_scale, mcg_scale) + return xt + + def loss(self, + x0, + t, + model, + model_kwargs={}, + noise=None, + input_x0=None, + reduction='mean'): + assert reduction in ['mean', 'none'] + noise = torch.randn_like(x0) if noise is None else noise + input_x0 = x0 if input_x0 is None else input_x0 + xt = self.q_sample(input_x0, t, noise=noise) + + # compute loss + if self.loss_type in ['kl', 'rescaled_kl']: + loss, _ = self.variational_lower_bound(x0, xt, t, model, + model_kwargs) + if self.loss_type == 'rescaled_kl': + loss = loss * self.num_timesteps + elif self.loss_type in ['mse', 'rescaled_mse', 'l1', 'rescaled_l1']: + out = model(xt, t=self._scale_timesteps(t), **model_kwargs) + + # VLB for variation + loss_vlb = 0.0 + if self.var_type in ['learned', 'learned_range']: + out, var = out.chunk(2, dim=1) + frozen = torch.cat([ + out.detach(), var + ], dim=1) # learn var without affecting the prediction of mean + loss_vlb, _ = self.variational_lower_bound( + x0, + xt, + t, + model=lambda *args, **kwargs: frozen, + reduction=reduction) + if self.loss_type.startswith('rescaled_'): + loss_vlb = loss_vlb * self.num_timesteps / 1000.0 + + # MSE/L1 for x0/eps + target = { + 'eps': noise, + 'x0': x0, + 'x_{t-1}': self.q_posterior_mean_variance(x0, xt, t)[0] + }[self.mean_type] + loss = ( + out + - target).pow(1 if self.loss_type.endswith('l1') else 2).abs() + if reduction == 'mean': + loss = loss.flatten(1).mean(dim=1) + + # total loss + loss = loss + loss_vlb + return loss + + def variational_lower_bound(self, + x0, + xt, + t, + model, + model_kwargs={}, + clamp=None, + percentile=None, + reduction='mean'): + assert reduction in ['mean', 'none'] + + # compute groundtruth and predicted distributions + mu1, _, log_var1 = self.q_posterior_mean_variance(x0, xt, t) + mu2, _, log_var2, x0 = self.p_mean_variance(xt, t, model, model_kwargs, + clamp, percentile) + + # compute KL loss + kl = kl_divergence(mu1, log_var1, mu2, log_var2) / math.log(2.0) + if reduction == 'mean': + kl = kl.flatten(1).mean(dim=1) + + # compute discretized NLL loss (for p(x0 | x1) only) + nll = -discretized_gaussian_log_likelihood( + x0, mean=mu2, log_scale=0.5 * log_var2) / math.log(2.0) + if reduction == 'mean': + nll = nll.flatten(1).mean(dim=1) + + # NLL for p(x0 | x1) and KL otherwise + t = t.view(-1, *(1, ) * (nll.ndim - 1)) + vlb = torch.where(t == 0, nll, kl) + return vlb, x0 + + @torch.no_grad() + def variational_lower_bound_loop(self, + x0, + model, + model_kwargs={}, + clamp=None, + percentile=None): + r"""Compute the entire variational lower bound, measured in bits-per-dim. + """ + # prepare input and output + b = x0.size(0) + metrics = {'vlb': [], 'mse': [], 'x0_mse': []} + + # loop + for step in torch.arange(self.num_timesteps).flip(0): + # compute VLB + t = torch.full((b, ), step, dtype=torch.long, device=x0.device) + noise = torch.randn_like(x0) + xt = self.q_sample(x0, t, noise) + vlb, pred_x0 = self.variational_lower_bound( + x0, xt, t, model, model_kwargs, clamp, percentile) + + # predict eps from x0 + eps = (_i(self.sqrt_recip_alphas_cumprod, t, xt) * xt - x0) / \ + _i(self.sqrt_recipm1_alphas_cumprod, t, xt) # noqa + + # collect metrics + metrics['vlb'].append(vlb) + metrics['x0_mse'].append( + (pred_x0 - x0).square().flatten(1).mean(dim=1)) + metrics['mse'].append( + (eps - noise).square().flatten(1).mean(dim=1)) + metrics = {k: torch.stack(v, dim=1) for k, v in metrics.items()} + + # compute the prior KL term for VLB, measured in bits-per-dim + mu, _, log_var = self.q_mean_variance(x0, t) + kl_prior = kl_divergence(mu, log_var, torch.zeros_like(mu), + torch.zeros_like(log_var)) + kl_prior = kl_prior.flatten(1).mean(dim=1) / math.log(2.0) + + # update metrics + metrics['prior_bits_per_dim'] = kl_prior + metrics['total_bits_per_dim'] = metrics['vlb'].sum(dim=1) + kl_prior + return metrics + + def _scale_timesteps(self, t): + if self.rescale_timesteps: + return t.float() * 1000.0 / self.num_timesteps + return t diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/dpm_solver.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/dpm_solver.py new file mode 100644 index 0000000000000000000000000000000000000000..a1e15a2f20ebc46a2fdbc7a88fb0b6f99f95ff89 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/dpm_solver.py @@ -0,0 +1,1697 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch +import torch.nn.functional as F + +__all__ = [ + 'NoiseScheduleVP', 'model_wrapper', 'model_wrapper_guided_diffusion', + 'DPM_Solver', 'interpolate_fn' +] + + +def _i(tensor, t, x): + r"""Index tensor using t and format the output according to x. + """ + shape = (x.size(0), ) + (1, ) * (x.ndim - 1) + device = x.device + return tensor[t.to(device)].view(shape).to(device) + + +class NoiseScheduleVP: + + def __init__(self, + schedule='discrete', + betas=None, + alphas_cumprod=None, + continuous_beta_0=0.1, + continuous_beta_1=20., + dtype=torch.float32): + r"""Create a wrapper class for the forward SDE (VP type). + + *** + Update: We support discrete-time diffusion models by implementing a picewise + linear interpolation for log_alpha_t. + We recommend to use schedule='discrete' for the discrete-time diffusion models, + especially for high-resolution images. + *** + + The forward SDE ensures that the condition distribution q_{t|0}(x_t | x_0) = N ( alpha_t * x_0, sigma_t^2 * I ). + We further define lambda_t = log(alpha_t) - log(sigma_t), + which is the half-logSNR (described in the DPM-Solver paper). + Therefore, we implement the functions for computing alpha_t, sigma_t and lambda_t. For t in [0, T], we have: + + log_alpha_t = self.marginal_log_mean_coeff(t) + sigma_t = self.marginal_std(t) + lambda_t = self.marginal_lambda(t) + + Moreover, as lambda(t) is an invertible function, we also support its inverse function: + + t = self.inverse_lambda(lambda_t) + + =============================================================== + + We support both discrete-time DPMs (trained on n = 0, 1, ..., N-1) + and continuous-time DPMs (trained on t in [t_0, T]). + + 1. For discrete-time DPMs: + + For discrete-time DPMs trained on n = 0, 1, ..., N-1, + we convert the discrete steps to continuous time steps by: + t_i = (i + 1) / N + e.g. for N = 1000, we have t_0 = 1e-3 and T = t_{N-1} = 1. + We solve the corresponding diffusion ODE from time T = 1 to time t_0 = 1e-3. + + Args: + betas: A `torch.Tensor`. The beta array for the discrete-time DPM. + (See the original DDPM paper for details) + alphas_cumprod: A `torch.Tensor`. The cumprod alphas for the discrete-time DPM. + (See the original DDPM paper for details) + + Note that we always have alphas_cumprod = cumprod(1 - betas). + Therefore, we only need to set one of `betas` and `alphas_cumprod`. + + **Important**: Please pay special attention for the args for `alphas_cumprod`: + The `alphas_cumprod` is the \hat{alpha_n} arrays in the notations of DDPM. Specifically, + DDPMs assume that + q_{t_n | 0}(x_{t_n} | x_0) = N ( \sqrt{\hat{alpha_n}} * x_0, (1 - \hat{alpha_n}) * I ). + Therefore, the notation \hat{alpha_n} is different from the notation alpha_t in DPM-Solver. + In fact, we have + alpha_{t_n} = \sqrt{\hat{alpha_n}}, + and + log(alpha_{t_n}) = 0.5 * log(\hat{alpha_n}). + + 2. For continuous-time DPMs: + + We support two types of VPSDEs: linear (DDPM) and cosine (improved-DDPM). + The hyperparameters for the noise + schedule are the default settings in DDPM and improved-DDPM: + + Args: + beta_min: A `float` number. The smallest beta for the linear schedule. + beta_max: A `float` number. The largest beta for the linear schedule. + cosine_s: A `float` number. The hyperparameter in the cosine schedule. + cosine_beta_max: A `float` number. The hyperparameter in the cosine schedule. + T: A `float` number. The ending time of the forward process. + + =============================================================== + + Args: + schedule: A `str`. The noise schedule of the forward SDE. + 'discrete' for discrete-time DPMs, + 'linear' or 'cosine' for continuous-time DPMs. + Returns: + A wrapper object of the forward SDE (VP type). + + =============================================================== + + Example: + + # For discrete-time DPMs, given betas (the beta array for n = 0, 1, ..., N - 1): + >>> ns = NoiseScheduleVP('discrete', betas=betas) + + # For discrete-time DPMs, given alphas_cumprod (the \hat{alpha_n} + # array for n = 0, 1, ..., N - 1): + >>> ns = NoiseScheduleVP('discrete', alphas_cumprod=alphas_cumprod) + + # For continuous-time DPMs (VPSDE), linear schedule: + >>> ns = NoiseScheduleVP('linear', continuous_beta_0=0.1, continuous_beta_1=20.) + + """ + + if schedule not in ['discrete', 'linear', 'cosine']: + raise ValueError( + "Unsupported noise schedule {}. The schedule needs to be 'discrete' or 'linear' or 'cosine'" + .format(schedule)) + + self.schedule = schedule + if schedule == 'discrete': + if betas is not None: + log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0) + else: + assert alphas_cumprod is not None + log_alphas = 0.5 * torch.log(alphas_cumprod) + self.total_N = len(log_alphas) + self.T = 1. + self.t_array = torch.linspace(0., 1., + self.total_N + 1)[1:].reshape( + (1, -1)).to(dtype=dtype) + self.log_alpha_array = log_alphas.reshape(( + 1, + -1, + )).to(dtype=dtype) + else: + self.total_N = 1000 + self.beta_0 = continuous_beta_0 + self.beta_1 = continuous_beta_1 + self.cosine_s = 0.008 + self.cosine_beta_max = 999. + self.cosine_t_max = math.atan( + self.cosine_beta_max * (1. + self.cosine_s) / math.pi) * 2. * ( + 1. + self.cosine_s) / math.pi - self.cosine_s + self.cosine_log_alpha_0 = math.log( + math.cos(self.cosine_s / (1. + self.cosine_s) * math.pi / 2.)) + self.schedule = schedule + if schedule == 'cosine': + # For the cosine schedule, + # T = 1 will have numerical issues. So we manually set the ending time T. + # Note that T = 0.9946 may be not the optimal setting. However, we find it works well. + self.T = 0.9946 + else: + self.T = 1. + + def marginal_log_mean_coeff(self, t): + """ + Compute log(alpha_t) of a given continuous-time label t in [0, T]. + """ + if self.schedule == 'discrete': + return interpolate_fn( + t.reshape((-1, 1)), self.t_array.to(t.device), + self.log_alpha_array.to(t.device)).reshape((-1)) + elif self.schedule == 'linear': + return -0.25 * t**2 * (self.beta_1 + - self.beta_0) - 0.5 * t * self.beta_0 + elif self.schedule == 'cosine': + log_alpha_fn = lambda s: torch.log( # noqa + torch.cos((s + self.cosine_s) / # noqa + (1. + self.cosine_s) * math.pi / 2.)) + log_alpha_t = log_alpha_fn(t) - self.cosine_log_alpha_0 + return log_alpha_t + + def marginal_alpha(self, t): + """ + Compute alpha_t of a given continuous-time label t in [0, T]. + """ + return torch.exp(self.marginal_log_mean_coeff(t)) + + def marginal_std(self, t): + """ + Compute sigma_t of a given continuous-time label t in [0, T]. + """ + return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t))) + + def marginal_lambda(self, t): + """ + Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T]. + """ + log_mean_coeff = self.marginal_log_mean_coeff(t) + log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff)) + return log_mean_coeff - log_std + + def inverse_lambda(self, lamb): + """ + Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t. + """ + if self.schedule == 'linear': + tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp( + -2. * lamb, + torch.zeros((1, )).to(lamb)) + Delta = self.beta_0**2 + tmp + return tmp / (torch.sqrt(Delta) + self.beta_0) / ( + self.beta_1 - self.beta_0) + elif self.schedule == 'discrete': + log_alpha = -0.5 * torch.logaddexp( + torch.zeros((1, )).to(lamb.device), -2. * lamb) + t = interpolate_fn( + log_alpha.reshape((-1, 1)), + torch.flip(self.log_alpha_array.to(lamb.device), [1]), + torch.flip(self.t_array.to(lamb.device), [1])) + return t.reshape((-1, )) + else: + log_alpha = -0.5 * torch.logaddexp(-2. * lamb, + torch.zeros((1, )).to(lamb)) + t_fn = lambda log_alpha_t: torch.arccos( # noqa + torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2. * ( + 1. + self.cosine_s) / math.pi - self.cosine_s + t = t_fn(log_alpha) + return t + + +def model_wrapper(model, + noise_schedule, + model_type='noise', + model_kwargs={}, + guidance_type='uncond', + condition=None, + unconditional_condition=None, + guidance_scale=1., + classifier_fn=None, + classifier_kwargs={}): + r"""Create a wrapper function for the noise prediction model. + + DPM-Solver needs to solve the continuous-time diffusion ODEs. + For DPMs trained on discrete-time labels, we need to + firstly wrap the model function to a noise prediction model + that accepts the continuous time as the input. + + We support four types of the diffusion model by setting `model_type`: + + 1. "noise": noise prediction model. (Trained by predicting noise). + + 2. "x_start": data prediction model. (Trained by predicting the data x_0 at time 0). + + 3. "v": velocity prediction model. (Trained by predicting the velocity). + The "v" prediction is derivation detailed in Appendix D of [1], + and is used in Imagen-Video [2]. + + [1] Salimans, Tim, and Jonathan Ho. + "Progressive distillation for fast sampling of diffusion models." + arXiv preprint arXiv:2202.00512 (2022). + [2] Ho, Jonathan, et al. "Imagen Video: High Definition + Video Generation with Diffusion Models." + arXiv preprint arXiv:2210.02303 (2022). + + 4. "score": marginal score function. (Trained by denoising score matching). + Note that the score function and the noise prediction model follows a simple relationship: + ``` + noise(x_t, t) = -sigma_t * score(x_t, t) + ``` + + We support three types of guided sampling by DPMs by setting `guidance_type`: + 1. "uncond": unconditional sampling by DPMs. + The input `model` has the following format: + `` + model(x, t_input, **model_kwargs) -> noise | x_start | v | score + `` + + 2. "classifier": classifier guidance sampling [3] by DPMs and another classifier. + The input `model` has the following format: + `` + model(x, t_input, **model_kwargs) -> noise | x_start | v | score + `` + + The input `classifier_fn` has the following format: + `` + classifier_fn(x, t_input, cond, **classifier_kwargs) -> logits(x, t_input, cond) + `` + + [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis," + in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794. + + 3. "classifier-free": classifier-free guidance sampling by conditional DPMs. + The input `model` has the following format: + `` + model(x, t_input, cond, **model_kwargs) -> noise | x_start | v | score + `` + And if cond == `unconditional_condition`, the model output is the unconditional DPM output. + + [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance." + arXiv preprint arXiv:2207.12598 (2022). + + + The `t_input` is the time label of the model, which may be discrete-time labels (i.e. 0 to 999) + or continuous-time labels (i.e. epsilon to T). + + We wrap the model function to accept only `x` and `t_continuous` as inputs, + and outputs the predicted noise: + `` + def model_fn(x, t_continuous) -> noise: + t_input = get_model_input_time(t_continuous) + return noise_pred(model, x, t_input, **model_kwargs) + `` + where `t_continuous` is the continuous time labels (i.e. epsilon to T). + And we use `model_fn` for DPM-Solver. + + =============================================================== + + Args: + model: A diffusion model with the corresponding format described above. + noise_schedule: A noise schedule object, such as NoiseScheduleVP. + model_type: A `str`. The parameterization type of the diffusion model. + "noise" or "x_start" or "v" or "score". + model_kwargs: A `dict`. A dict for the other inputs of the model function. + guidance_type: A `str`. The type of the guidance for sampling. + "uncond" or "classifier" or "classifier-free". + condition: A pytorch tensor. The condition for the guided sampling. + Only used for "classifier" or "classifier-free" guidance type. + unconditional_condition: A pytorch tensor. The condition for the unconditional sampling. + Only used for "classifier-free" guidance type. + guidance_scale: A `float`. The scale for the guided sampling. + classifier_fn: A classifier function. Only used for the classifier guidance. + classifier_kwargs: A `dict`. A dict for the other inputs of the classifier function. + Returns: + A noise prediction model that accepts the noised data and the continuous time as the inputs. + """ + + def get_model_input_time(t_continuous): + """ + Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time. + For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N]. + For continuous-time DPMs, we just use `t_continuous`. + """ + if noise_schedule.schedule == 'discrete': + return (t_continuous - 1. / noise_schedule.total_N) * 1000. + else: + return t_continuous + + def noise_pred_fn(x, t_continuous, cond=None): + t_input = get_model_input_time(t_continuous) + if cond is None: + output = model(x, t_input, **model_kwargs) + else: + output = model(x, t_input, cond, **model_kwargs) + if model_type == 'noise': + return output + elif model_type == 'x_start': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + return (x - alpha_t * output) / sigma_t + elif model_type == 'v': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + return alpha_t * output + sigma_t * x + elif model_type == 'score': + sigma_t = noise_schedule.marginal_std(t_continuous) + return -sigma_t * output + + def cond_grad_fn(x, t_input): + """ + Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t). + """ + with torch.enable_grad(): + x_in = x.detach().requires_grad_(True) + log_prob = classifier_fn(x_in, t_input, condition, + **classifier_kwargs) + return torch.autograd.grad(log_prob.sum(), x_in)[0] + + def model_fn(x, t_continuous): + """ + The noise predicition model function that is used for DPM-Solver. + """ + if guidance_type == 'uncond': + return noise_pred_fn(x, t_continuous) + elif guidance_type == 'classifier': + assert classifier_fn is not None + t_input = get_model_input_time(t_continuous) + cond_grad = cond_grad_fn(x, t_input) + sigma_t = noise_schedule.marginal_std(t_continuous) + noise = noise_pred_fn(x, t_continuous) + return noise - guidance_scale * sigma_t * cond_grad + elif guidance_type == 'classifier-free': + if guidance_scale == 1. or unconditional_condition is None: + return noise_pred_fn(x, t_continuous, cond=condition) + else: + x_in = torch.cat([x] * 2) + t_in = torch.cat([t_continuous] * 2) + c_in = torch.cat([unconditional_condition, condition]) + noise_uncond, noise = noise_pred_fn( + x_in, t_in, cond=c_in).chunk(2) + return noise_uncond + guidance_scale * (noise - noise_uncond) + + assert model_type in ['noise', 'x_start', 'v'] + assert guidance_type in ['uncond', 'classifier', 'classifier-free'] + return model_fn + + +def model_wrapper_guided_diffusion(model, + noise_schedule, + var_type, + mean_type, + model_kwargs={}, + rescale_timesteps=False, + num_timesteps=1000, + guide_scale=None, + condition_fn=None): + """Create a wrapper function for the noise prediction model guided diffusion. + + DPM-Solver needs to solve the continuous-time diffusion ODEs. + For DPMs trained on discrete-time labels, we need to + firstly wrap the model function to a noise prediction model that accepts the continuous time as the input. + + We support three types of the diffusion model by setting `mean_type`: + + 1. "x_{t-1}": previous data prediction model. + (Trained by predicting the data x_{t-1} at time t-1). + + 2. "x0": data prediction model. (Trained by predicting the data x0 at time 0). + + 3. "eps": noise prediction model. (Trained by predicting the noise). + + We support three types of guided sampling by DPMs: + 1. unconditional sampling by DPMs. + The input `model` has the following format: + `` + model(xt, t_input, **model_kwargs) -> x_{t-1} | x0 | eps + `` + + 2. classifier guidance sampling [3] by DPMs and another classifier. + The input `model` has the following format: + `` + model(xt, t_input, **model_kwargs) -> x_{t-1} | x0 | eps + `` + + The input `condition_fn` has the following format: + `` + condition_fn(xt, t_input, **model_kwargs) -> logits(xt, t_input) + `` + + [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis," + in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794. + + 3. classifier-free guidance sampling by conditional DPMs. + The input `model` has the following format: + `` + model(xt, t_input, **model_kwargs) -> x_{t-1} | x0 | eps + `` + + [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance." + arXiv preprint arXiv:2207.12598 (2022). + + + The `t_input` is the time label of the model, which may be discrete-time labels + or continuous-time labels (i.e. epsilon to T). + + We wrap the model function to accept only `x` and `t_continuous` as inputs, + and outputs the predicted noise: + `` + def model_fn(x, t_continuous) -> noise: + t_input = get_model_input_time(t_continuous) + return noise_pred_fn(model, x, t_input, **model_kwargs) + `` + where `t_continuous` is the continuous time labels (i.e. epsilon to T). + And we use `model_fn` for DPM-Solver. + + =============================================================== + + Args: + model: A diffusion model with the corresponding format described above. + noise_schedule: A noise schedule object, such as NoiseScheduleVP. + var_type: A `str`. The variance type of the diffusion model. + "learned" or "learned_range" or "fixed_large" or "fixed_small". + mean_type: A `str`. The prediction type of the diffusion model. + "x_{t-1}" or "x0" or "eps". + model_kwargs: A `dict`. A dict for the other inputs of the model function. + clamp: A `float`. The range to clamp the data. + rescale_timesteps: A `bool`. Whether to rescale the timesteps. + num_timesteps: An `int`. The number of the total diffusion steps. + guide_scale: A `float`. The strength of the classifier-free guidance. + condition_fn: A function. The function of the classifier guidance. + Returns: + A noise prediction model that accepts the noised data and the continuous time as the inputs. + """ + + def _scale_timesteps(t): + if rescale_timesteps: + return t.float() * 1000.0 / num_timesteps + return t + + def get_model_input_time(t_continuous): + """ + Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time. + For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N]. + For continuous-time DPMs, we just use `t_continuous`. + """ + if noise_schedule.schedule == 'discrete': + return (t_continuous - 1. / noise_schedule.total_N) * 1000. + else: + return t_continuous + + def noise_pred_fn(xt, t_continuous, **model_kwargs): + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((xt.shape[0])) + t_input = get_model_input_time(_scale_timesteps(t_continuous)) + # predict distribution + out = model(xt, t_input, **model_kwargs) + + if var_type == 'learned': + out, _ = out.chunk(2, dim=1) + elif var_type == 'learned_range': + out, _ = out.chunk(2, dim=1) + + if mean_type == 'eps': + eps = out + elif mean_type == 'x_{t-1}': + raise NotImplementedError + assert noise_schedule.schedule == 'discrete' + mu = out + posterior_mean_coef1 = None + posterior_mean_coef2 = None + x0 = expand_dims( + 1. / posterior_mean_coef1, dims) * mu - expand_dims( + posterior_mean_coef2 / posterior_mean_coef1, dims) * xt + eps = (xt - expand_dims(alpha_t, dims) * x0) / expand_dims( + sigma_t, dims) + elif mean_type == 'x0': + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + dims = out.dim() + eps = (xt - expand_dims(alpha_t, dims) * out) / expand_dims( + sigma_t, dims) + + if condition_fn is not None: + alpha_t, sigma_t = noise_schedule.marginal_alpha( + t_continuous), noise_schedule.marginal_std(t_continuous) + eps = eps - (1 - alpha_t).sqrt() * condition_fn( + xt, t_input, **model_kwargs) + + return eps + + def model_fn(x, t_continuous): + """ + The noise predicition model function that is used for DPM-Solver. + """ + if t_continuous.reshape((-1, )).shape[0] == 1: + t_continuous = t_continuous.expand((x.shape[0])) + + if guide_scale is None: + eps = noise_pred_fn(x, t_continuous, **model_kwargs) + else: + # classifier-free guidance + # (model_kwargs[0]: conditional kwargs; model_kwargs[1]: non-conditional kwargs) + assert isinstance(model_kwargs, list) and len(model_kwargs) == 2 + y_out = noise_pred_fn(x, t_continuous, **model_kwargs[0]) + u_out = noise_pred_fn(x, t_continuous, **model_kwargs[1]) + dim = y_out.size(1) if var_type.startswith( + 'fixed') else y_out.size(1) // 2 + eps = u_out[:, :dim] + guide_scale * ( + y_out[:, :dim] - u_out[:, :dim]) + return eps + + return model_fn + + +class DPM_Solver: + + def __init__(self, + model_fn, + noise_schedule, + algorithm_type='dpmsolver++', + percentile=None, + thresholding_max_val=1., + clamp=None): + r"""Construct a DPM-Solver. + + We support both the noise prediction model ("predicting epsilon") + and the data prediction model ("predicting x0"). + If `predict_x0` is False, we use the solver for the noise prediction model (DPM-Solver). + If `predict_x0` is True, we use the solver for the data prediction model (DPM-Solver++). + In such case, we further support the "dynamic thresholding" + in [1] when `thresholding` is True. + The "dynamic thresholding" can greatly improve the sample quality for pixel-space DPMs + with large guidance scales. + + Args: + model_fn: A noise prediction model function which accepts the continuous-time input (t in [epsilon, T]): + `` + def model_fn(x, t_continuous): + return noise + `` + noise_schedule: A noise schedule object, such as NoiseScheduleVP. + predict_x0: A `bool`. If true, use the data prediction model; else, use the noise prediction model. + thresholding: A `bool`. Valid when `predict_x0` is True. Whether to use the "dynamic thresholding" in [1]. + max_val: A `float`. Valid when both `predict_x0` and + `thresholding` are True. The max value for thresholding. + + [1] Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, + Emily Denton, Seyed Kamyar Seyed Ghasemipour, + Burcu Karagol Ayan, S Sara Mahdavi, Rapha Gontijo Lopes, et al. + Photorealistic text-to-image diffusion models with deep language understanding. + arXiv preprint arXiv:2205.11487, 2022b. + """ + self.model = model_fn + self.noise_schedule = noise_schedule + self.algorithm_type = algorithm_type + self.percentile = percentile + self.thresholding_max_val = thresholding_max_val + self.clamp = clamp + + def noise_prediction_fn(self, x, t): + """ + Return the noise prediction model. + """ + return self.model(x, t) + + def data_prediction_fn(self, x, t): + """ + Return the data prediction model (with thresholding). + """ + noise = self.noise_prediction_fn(x, t) + dims = x.dim() + alpha_t, sigma_t = self.noise_schedule.marginal_alpha( + t), self.noise_schedule.marginal_std(t) + x0 = (x - expand_dims(sigma_t, dims) * noise) / expand_dims( + alpha_t, dims) + if self.percentile is not None: + s = torch.quantile( + torch.abs(x0).reshape((x0.shape[0], -1)), + self.percentile, + dim=1) + s = expand_dims( + torch.maximum( + s, + self.thresholding_max_val + * torch.ones_like(s).to(s.device)), dims) + x0 = torch.clamp(x0, -s, s) / s + elif self.clamp is not None: + x0 = x0.clamp(-clamp, clamp) + return x0 + + def model_fn(self, x, t): + """ + Convert the model to the noise prediction model or the data prediction model. + """ + if self.algorithm_type == 'dpmsolver++': + return self.data_prediction_fn(x, t) + else: + return self.noise_prediction_fn(x, t) + + def get_time_steps(self, skip_type, t_T, t_0, N, device): + """Compute the intermediate time steps for sampling. + + Args: + skip_type: A `str`. The type for the spacing of the time steps. We support three types: + - 'logSNR': uniform logSNR for the time steps. + - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.) + - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.) + t_T: A `float`. The starting time of the sampling (default is T). + t_0: A `float`. The ending time of the sampling (default is epsilon). + N: A `int`. The total number of the spacing of the time steps. + device: A torch device. + Returns: + A pytorch tensor of the time steps, with the shape (N + 1,). + """ + if skip_type == 'logSNR': + lambda_T = self.noise_schedule.marginal_lambda( + torch.tensor(t_T).to(device)) + lambda_0 = self.noise_schedule.marginal_lambda( + torch.tensor(t_0).to(device)) + logSNR_steps = torch.linspace(lambda_T.cpu().item(), + lambda_0.cpu().item(), + N + 1).to(device) + return self.noise_schedule.inverse_lambda(logSNR_steps) + elif skip_type == 'time_uniform': + return torch.linspace(t_T, t_0, N + 1).to(device) + elif skip_type == 'time_quadratic': + t_order = 2 + t = torch.linspace(t_T**(1. / t_order), t_0**(1. / t_order), + N + 1).pow(t_order).to(device) + return t + else: + raise ValueError( + "Unsupported skip_type {}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'" + .format(skip_type)) + + def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, + skip_type, t_T, t_0, + device): + """ + Get the order of each step for sampling by the singlestep DPM-Solver. + + We combine both DPM-Solver-1,2,3 to use all the function evaluations, which is named as "DPM-Solver-fast". + Given a fixed number of function evaluations by `steps`, the sampling procedure by DPM-Solver-fast is: + - If order == 1: + We take `steps` of DPM-Solver-1 (i.e. DDIM). + - If order == 2: + - Denote K = (steps // 2). We take K or (K + 1) intermediate time steps for sampling. + - If steps % 2 == 0, we use K steps of DPM-Solver-2. + - If steps % 2 == 1, we use K steps of DPM-Solver-2 and 1 step of DPM-Solver-1. + - If order == 3: + - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling. + - If steps % 3 == 0, we use (K - 2) steps of DPM-Solver-3, + and 1 step of DPM-Solver-2 and 1 step of DPM-Solver-1. + - If steps % 3 == 1, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-1. + - If steps % 3 == 2, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-2. + + ============================================ + Args: + order: A `int`. The max order for the solver (2 or 3). + steps: A `int`. The total number of function evaluations (NFE). + skip_type: A `str`. The type for the spacing of the time steps. We support three types: + - 'logSNR': uniform logSNR for the time steps. + - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.) + - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.) + t_T: A `float`. The starting time of the sampling (default is T). + t_0: A `float`. The ending time of the sampling (default is epsilon). + device: A torch device. + Returns: + orders: A list of the solver order of each step. + """ + if order == 3: + K = steps // 3 + 1 + if steps % 3 == 0: + orders = [ + 3, + ] * (K - 2) + [2, 1] + elif steps % 3 == 1: + orders = [ + 3, + ] * (K - 1) + [1] + else: + orders = [ + 3, + ] * (K - 1) + [2] + elif order == 2: + if steps % 2 == 0: + K = steps // 2 + orders = [ + 2, + ] * K + else: + K = steps // 2 + 1 + orders = [ + 2, + ] * (K - 1) + [1] + elif order == 1: + K = 1 + orders = [ + 1, + ] * steps + else: + raise ValueError("'order' must be '1' or '2' or '3'.") + if skip_type == 'logSNR': + # To reproduce the results in DPM-Solver paper + timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, + device) + else: + timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, + device)[torch.cumsum( + torch.tensor([ + 0, + ] + orders), + 0).to(device)] + return timesteps_outer, orders + + def denoise_to_zero_fn(self, x, s): + """ + Denoise at the final step, which is equivalent to solve + the ODE from lambda_s to infty by first-order discretization. + """ + return self.data_prediction_fn(x, s) + + def dpm_solver_first_update(self, + x, + s, + t, + model_s=None, + return_intermediate=False): + """ + DPM-Solver-1 (equivalent to DDIM) from time `s` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + s: A pytorch tensor. The starting time, with the shape (1,). + t: A pytorch tensor. The ending time, with the shape (1,). + model_s: A pytorch tensor. The model function evaluated at time `s`. + If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it. + return_intermediate: A `bool`. If true, also return the model value at time `s`. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + ns = self.noise_schedule + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + log_alpha_s, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_t = ns.marginal_std(s), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + if self.algorithm_type == 'dpmsolver++': + phi_1 = torch.expm1(-h) + if model_s is None: + model_s = self.model_fn(x, s) + x_t = (sigma_t / sigma_s * x - alpha_t * phi_1 * model_s) + if return_intermediate: + return x_t, {'model_s': model_s} + else: + return x_t + else: + phi_1 = torch.expm1(h) + if model_s is None: + model_s = self.model_fn(x, s) + x_t = ( + torch.exp(log_alpha_t - log_alpha_s) * x # noqa + - (sigma_t * phi_1) * model_s) + if return_intermediate: + return x_t, {'model_s': model_s} + else: + return x_t + + def singlestep_dpm_solver_second_update(self, + x, + s, + t, + r1=0.5, + model_s=None, + return_intermediate=False, + solver_type='dpmsolver'): + """ + Singlestep solver DPM-Solver-2 from time `s` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + s: A pytorch tensor. The starting time, with the shape (1,). + t: A pytorch tensor. The ending time, with the shape (1,). + r1: A `float`. The hyperparameter of the second-order solver. + model_s: A pytorch tensor. The model function evaluated at time `s`. + If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it. + return_intermediate: A `bool`. + If true, also return the model value at time `s` and `s1` (the intermediate time). + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + if solver_type not in ['dpmsolver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpmsolver' or 'taylor', got {}". + format(solver_type)) + if r1 is None: + r1 = 0.5 + ns = self.noise_schedule + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + lambda_s1 = lambda_s + r1 * h + s1 = ns.inverse_lambda(lambda_s1) + log_alpha_s, log_alpha_s1, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_s1, sigma_t = ns.marginal_std(s), ns.marginal_std( + s1), ns.marginal_std(t) + alpha_s1, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_t) + + if self.algorithm_type == 'dpmsolver++': + phi_11 = torch.expm1(-r1 * h) + phi_1 = torch.expm1(-h) + + if model_s is None: + model_s = self.model_fn(x, s) + x_s1 = ((sigma_s1 / sigma_s) * x - (alpha_s1 * phi_11) * model_s) + model_s1 = self.model_fn(x_s1, s1) + if solver_type == 'dpmsolver': + x_t = ((sigma_t / sigma_s) * x - # noqa + (alpha_t * phi_1) * model_s # noqa + - (0.5 / r1) * (alpha_t * phi_1) * (model_s1 - model_s)) + elif solver_type == 'taylor': + x_t = ((sigma_t / sigma_s) * x - # noqa + (alpha_t * phi_1) * model_s # noqa + + (1. / r1) * ( + alpha_t * # noqa + (phi_1 / h + 1.)) * (model_s1 - model_s)) + else: + phi_11 = torch.expm1(r1 * h) + phi_1 = torch.expm1(h) + + if model_s is None: + model_s = self.model_fn(x, s) + x_s1 = ( + torch.exp(log_alpha_s1 - log_alpha_s) * x # noqa + - (sigma_s1 * phi_11) * model_s) + model_s1 = self.model_fn(x_s1, s1) + if solver_type == 'dpmsolver': + x_t = ( + torch.exp(log_alpha_t - log_alpha_s) * x # noqa + - (sigma_t * phi_1) * model_s - (0.5 / r1) # noqa + * (sigma_t * phi_1) * (model_s1 - model_s)) + elif solver_type == 'taylor': + x_t = ( + torch.exp(log_alpha_t - log_alpha_s) * x # noqa + - (sigma_t * phi_1) * model_s - (1. / r1) # noqa + * (sigma_t * (phi_1 / h - 1.)) * (model_s1 - model_s)) + if return_intermediate: + return x_t, {'model_s': model_s, 'model_s1': model_s1} + else: + return x_t + + def singlestep_dpm_solver_third_update(self, + x, + s, + t, + r1=1. / 3., + r2=2. / 3., + model_s=None, + model_s1=None, + return_intermediate=False, + solver_type='dpmsolver'): + """ + Singlestep solver DPM-Solver-3 from time `s` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + s: A pytorch tensor. The starting time, with the shape (1,). + t: A pytorch tensor. The ending time, with the shape (1,). + r1: A `float`. The hyperparameter of the third-order solver. + r2: A `float`. The hyperparameter of the third-order solver. + model_s: A pytorch tensor. The model function evaluated at time `s`. + If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it. + model_s1: A pytorch tensor. The model function evaluated at time `s1` (the intermediate time given by `r1`). + If `model_s1` is None, we evaluate the model at `s1`; otherwise we directly use it. + return_intermediate: A `bool`. If true, + also return the model value at time `s`, `s1` and `s2` (the intermediate times). + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + if solver_type not in ['dpmsolver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpmsolver' or 'taylor', got {}". + format(solver_type)) + if r1 is None: + r1 = 1. / 3. + if r2 is None: + r2 = 2. / 3. + ns = self.noise_schedule + lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t) + h = lambda_t - lambda_s + lambda_s1 = lambda_s + r1 * h + lambda_s2 = lambda_s + r2 * h + s1 = ns.inverse_lambda(lambda_s1) + s2 = ns.inverse_lambda(lambda_s2) + log_alpha_s, log_alpha_s1, log_alpha_s2, log_alpha_t = ns.marginal_log_mean_coeff( + s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff( + s2), ns.marginal_log_mean_coeff(t) + sigma_s, sigma_s1, sigma_s2, sigma_t = ns.marginal_std( + s), ns.marginal_std(s1), ns.marginal_std(s2), ns.marginal_std(t) + alpha_s1, alpha_s2, alpha_t = torch.exp(log_alpha_s1), torch.exp( + log_alpha_s2), torch.exp(log_alpha_t) + + if self.algorithm_type == 'dpmsolver++': + phi_11 = torch.expm1(-r1 * h) + phi_12 = torch.expm1(-r2 * h) + phi_1 = torch.expm1(-h) + phi_22 = torch.expm1(-r2 * h) / (r2 * h) + 1. + phi_2 = phi_1 / h + 1. + phi_3 = phi_2 / h - 0.5 + + if model_s is None: + model_s = self.model_fn(x, s) + if model_s1 is None: + x_s1 = ((sigma_s1 / sigma_s) * x # noqa + - (alpha_s1 * phi_11) * model_s) + model_s1 = self.model_fn(x_s1, s1) + x_s2 = ((sigma_s2 / sigma_s) * x - # noqa + (alpha_s2 * phi_12) * model_s + r2 / r1 # noqa + * (alpha_s2 * phi_22) * (model_s1 - model_s)) + model_s2 = self.model_fn(x_s2, s2) + if solver_type == 'dpmsolver': + x_t = ((sigma_t / sigma_s) * x - # noqa + (alpha_t * phi_1) * model_s # noqa + + (1. / r2) * (alpha_t * phi_2) * (model_s2 - model_s)) + elif solver_type == 'taylor': + D1_0 = (1. / r1) * (model_s1 - model_s) + D1_1 = (1. / r2) * (model_s2 - model_s) + D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1) + D2 = 2. * (D1_1 - D1_0) / (r2 - r1) + x_t = ((sigma_t / sigma_s) * x - # noqa + (alpha_t * phi_1) * model_s # noqa + + (alpha_t * phi_2) * D1 - (alpha_t * phi_3) * D2) + else: + phi_11 = torch.expm1(r1 * h) + phi_12 = torch.expm1(r2 * h) + phi_1 = torch.expm1(h) + phi_22 = torch.expm1(r2 * h) / (r2 * h) - 1. + phi_2 = phi_1 / h - 1. + phi_3 = phi_2 / h - 0.5 + + if model_s is None: + model_s = self.model_fn(x, s) + if model_s1 is None: + x_s1 = ((torch.exp(log_alpha_s1 - log_alpha_s)) * x # noqa + - (sigma_s1 * phi_11) * model_s) + model_s1 = self.model_fn(x_s1, s1) + x_s2 = ((torch.exp(log_alpha_s2 - log_alpha_s)) * x # noqa + - (sigma_s2 * phi_12) * model_s - r2 / r1 # noqa + * (sigma_s2 * phi_22) * (model_s1 - model_s)) + model_s2 = self.model_fn(x_s2, s2) + if solver_type == 'dpmsolver': + x_t = ((torch.exp(log_alpha_t - log_alpha_s)) * x # noqa + - (sigma_t * phi_1) * model_s - (1. / r2) # noqa + * (sigma_t * phi_2) * (model_s2 - model_s)) + elif solver_type == 'taylor': + D1_0 = (1. / r1) * (model_s1 - model_s) + D1_1 = (1. / r2) * (model_s2 - model_s) + D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1) + D2 = 2. * (D1_1 - D1_0) / (r2 - r1) + x_t = ((torch.exp(log_alpha_t - log_alpha_s)) * x # noqa + - (sigma_t * phi_1) * model_s - # noqa + (sigma_t * phi_2) * D1 # noqa + - (sigma_t * phi_3) * D2) + + if return_intermediate: + return x_t, { + 'model_s': model_s, + 'model_s1': model_s1, + 'model_s2': model_s2 + } + else: + return x_t + + def multistep_dpm_solver_second_update(self, + x, + model_prev_list, + t_prev_list, + t, + solver_type='dpmsolver'): + """ + Multistep solver DPM-Solver-2 from time `t_prev_list[-1]` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + model_prev_list: A list of pytorch tensor. The previous computed model values. + t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,) + t: A pytorch tensor. The ending time, with the shape (1,). + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + if solver_type not in ['dpmsolver', 'taylor']: + raise ValueError( + "'solver_type' must be either 'dpmsolver' or 'taylor', got {}". + format(solver_type)) + ns = self.noise_schedule + model_prev_1, model_prev_0 = model_prev_list[-2], model_prev_list[-1] + t_prev_1, t_prev_0 = t_prev_list[-2], t_prev_list[-1] + lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda( + t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t) + log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff( + t_prev_0), ns.marginal_log_mean_coeff(t) + sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + h_0 = lambda_prev_0 - lambda_prev_1 + h = lambda_t - lambda_prev_0 + r0 = h_0 / h + D1_0 = (1. / r0) * (model_prev_0 - model_prev_1) + if self.algorithm_type == 'dpmsolver++': + phi_1 = torch.expm1(-h) + if solver_type == 'dpmsolver': + x_t = ((sigma_t / sigma_prev_0) * x # noqa + - (alpha_t * phi_1) * model_prev_0 - 0.5 # noqa + * (alpha_t * phi_1) * D1_0) + elif solver_type == 'taylor': + x_t = ((sigma_t / sigma_prev_0) * x - # noqa + (alpha_t * phi_1) * model_prev_0 + # noqa + (alpha_t * (phi_1 / h + 1.)) * D1_0) + else: + phi_1 = torch.expm1(h) + if solver_type == 'dpmsolver': + x_t = ((torch.exp(log_alpha_t - log_alpha_prev_0)) * x # noqa + - (sigma_t * phi_1) * model_prev_0 - 0.5 * # noqa + (sigma_t * phi_1) * D1_0) + elif solver_type == 'taylor': + x_t = ((torch.exp(log_alpha_t - log_alpha_prev_0)) * x # noqa + - (sigma_t * phi_1) * model_prev_0 # noqa + - (sigma_t * (phi_1 / h - 1.)) * D1_0) + return x_t + + def multistep_dpm_solver_third_update(self, + x, + model_prev_list, + t_prev_list, + t, + solver_type='dpmsolver'): + """ + Multistep solver DPM-Solver-3 from time `t_prev_list[-1]` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + model_prev_list: A list of pytorch tensor. The previous computed model values. + t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,) + t: A pytorch tensor. The ending time, with the shape (1,). + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + ns = self.noise_schedule + model_prev_2, model_prev_1, model_prev_0 = model_prev_list + t_prev_2, t_prev_1, t_prev_0 = t_prev_list + lambda_prev_2, lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda( + t_prev_2), ns.marginal_lambda(t_prev_1), ns.marginal_lambda( + t_prev_0), ns.marginal_lambda(t) + log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff( + t_prev_0), ns.marginal_log_mean_coeff(t) + sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t) + alpha_t = torch.exp(log_alpha_t) + + h_1 = lambda_prev_1 - lambda_prev_2 + h_0 = lambda_prev_0 - lambda_prev_1 + h = lambda_t - lambda_prev_0 + r0, r1 = h_0 / h, h_1 / h + D1_0 = (1. / r0) * (model_prev_0 - model_prev_1) + D1_1 = (1. / r1) * (model_prev_1 - model_prev_2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1. / (r0 + r1)) * (D1_0 - D1_1) + if self.algorithm_type == 'dpmsolver++': + phi_1 = torch.expm1(-h) + phi_2 = phi_1 / h + 1. + phi_3 = phi_2 / h - 0.5 + x_t = ((sigma_t / sigma_prev_0) * x - # noqa + (alpha_t * phi_1) * model_prev_0 + # noqa + (alpha_t * phi_2) * D1 # noqa + - (alpha_t * phi_3) * D2) + else: + phi_1 = torch.expm1(h) + phi_2 = phi_1 / h - 1. + phi_3 = phi_2 / h - 0.5 + x_t = ((torch.exp(log_alpha_t - log_alpha_prev_0)) * x # noqa + - (sigma_t * phi_1) * model_prev_0 - # noqa + (sigma_t * phi_2) * D1 # noqa + - (sigma_t * phi_3) * D2) + return x_t + + def singlestep_dpm_solver_update(self, + x, + s, + t, + order, + return_intermediate=False, + solver_type='dpmsolver', + r1=None, + r2=None): + """ + Singlestep DPM-Solver with the order `order` from time `s` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + s: A pytorch tensor. The starting time, with the shape (1,). + t: A pytorch tensor. The ending time, with the shape (1,). + order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3. + return_intermediate: A `bool`. If true, also return the model value at time `s`, + `s1` and `s2` (the intermediate times). + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + r1: A `float`. The hyperparameter of the second-order or third-order solver. + r2: A `float`. The hyperparameter of the third-order solver. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + if order == 1: + return self.dpm_solver_first_update( + x, s, t, return_intermediate=return_intermediate) + elif order == 2: + return self.singlestep_dpm_solver_second_update( + x, + s, + t, + return_intermediate=return_intermediate, + solver_type=solver_type, + r1=r1) + elif order == 3: + return self.singlestep_dpm_solver_third_update( + x, + s, + t, + return_intermediate=return_intermediate, + solver_type=solver_type, + r1=r1, + r2=r2) + else: + raise ValueError( + 'Solver order must be 1 or 2 or 3, got {}'.format(order)) + + def multistep_dpm_solver_update(self, + x, + model_prev_list, + t_prev_list, + t, + order, + solver_type='dpmsolver'): + """ + Multistep DPM-Solver with the order `order` from time `t_prev_list[-1]` to time `t`. + + Args: + x: A pytorch tensor. The initial value at time `s`. + model_prev_list: A list of pytorch tensor. The previous computed model values. + t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,) + t: A pytorch tensor. The ending time, with the shape (1,). + order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3. + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_t: A pytorch tensor. The approximated solution at time `t`. + """ + if order == 1: + return self.dpm_solver_first_update( + x, t_prev_list[-1], t, model_s=model_prev_list[-1]) + elif order == 2: + return self.multistep_dpm_solver_second_update( + x, model_prev_list, t_prev_list, t, solver_type=solver_type) + elif order == 3: + return self.multistep_dpm_solver_third_update( + x, model_prev_list, t_prev_list, t, solver_type=solver_type) + else: + raise ValueError( + 'Solver order must be 1 or 2 or 3, got {}'.format(order)) + + def dpm_solver_adaptive(self, + x, + order, + t_T, + t_0, + h_init=0.05, + atol=0.0078, + rtol=0.05, + theta=0.9, + t_err=1e-5, + solver_type='dpmsolver'): + """ + The adaptive step size solver based on singlestep DPM-Solver. + + Args: + x: A pytorch tensor. The initial value at time `t_T`. + order: A `int`. The (higher) order of the solver. We only support order == 2 or 3. + t_T: A `float`. The starting time of the sampling (default is T). + t_0: A `float`. The ending time of the sampling (default is epsilon). + h_init: A `float`. The initial step size (for logSNR). + atol: A `float`. The absolute tolerance of the solver. + For image data, the default setting is 0.0078, followed [1]. + rtol: A `float`. The relative tolerance of the solver. + The default setting is 0.05. + theta: A `float`. The safety hyperparameter for adapting the step size. + The default setting is 0.9, followed [1]. + t_err: A `float`. The tolerance for the time. + We solve the diffusion ODE until the absolute error between the + current time and `t_0` is less than `t_err`. The default setting is 1e-5. + solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers. + The type slightly impacts the performance. We recommend to use 'dpmsolver' type. + Returns: + x_0: A pytorch tensor. The approximated solution at time `t_0`. + + [1] A. Jolicoeur-Martineau, K. Li, R. Piché-Taillefer, T. Kachman, and I. Mitliagkas, + "Gotta go fast when generating data with score-based models," + arXiv preprint arXiv:2105.14080, 2021. + """ + ns = self.noise_schedule + s = t_T * torch.ones((1, )).to(x) + lambda_s = ns.marginal_lambda(s) + lambda_0 = ns.marginal_lambda(t_0 * torch.ones_like(s).to(x)) + h = h_init * torch.ones_like(s).to(x) + x_prev = x + nfe = 0 + if order == 2: + r1 = 0.5 + lower_update = lambda x, s, t: self.dpm_solver_first_update( # noqa + x, s, t, return_intermediate=True) + higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update( # noqa + x, s, t, r1=r1, solver_type=solver_type, **kwargs) + elif order == 3: + r1, r2 = 1. / 3., 2. / 3. + lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update( # noqa + x, + s, + t, + r1=r1, + return_intermediate=True, + solver_type=solver_type) + higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update( # noqa + x, + s, + t, + r1=r1, + r2=r2, + solver_type=solver_type, + **kwargs) + else: + raise ValueError( + 'For adaptive step size solver, order must be 2 or 3, got {}'. + format(order)) + while torch.abs((s - t_0)).mean() > t_err: + t = ns.inverse_lambda(lambda_s + h) + x_lower, lower_noise_kwargs = lower_update(x, s, t) + x_higher = higher_update(x, s, t, **lower_noise_kwargs) + delta = torch.max( + torch.ones_like(x).to(x) * atol, + rtol * torch.max(torch.abs(x_lower), torch.abs(x_prev))) + norm_fn = lambda v: torch.sqrt( # noqa + torch.square(v.reshape( + (v.shape[0], -1))).mean(dim=-1, keepdim=True)) + E = norm_fn((x_higher - x_lower) / delta).max() + if torch.all(E <= 1.): + x = x_higher + s = t + x_prev = x_lower + lambda_s = ns.marginal_lambda(s) + h = torch.min( + theta * h * torch.float_power(E, -1. / order).float(), + lambda_0 - lambda_s) + nfe += order + print('adaptive solver nfe', nfe) + return x + + def add_noise(self, x, t, noise=None): + """ + Compute the noised input xt = alpha_t * x + sigma_t * noise. + + Args: + x: A `torch.Tensor` with shape `(batch_size, *shape)`. + t: A `torch.Tensor` with shape `(t_size,)`. + Returns: + xt with shape `(t_size, batch_size, *shape)`. + """ + alpha_t, sigma_t = self.noise_schedule.marginal_alpha( + t), self.noise_schedule.marginal_std(t) + if noise is None: + noise = torch.randn((t.shape[0], *x.shape), device=x.device) + x = x.reshape((-1, *x.shape)) + xt = expand_dims(alpha_t, x.dim()) * x + expand_dims(sigma_t, + x.dim()) * noise + if t.shape[0] == 1: + return xt.squeeze(0) + else: + return xt + + def inverse(self, + x, + steps=20, + t_start=None, + t_end=None, + order=2, + skip_type='time_uniform', + method='multistep', + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpmsolver', + atol=0.0078, + rtol=0.05, + return_intermediate=False): + """ + Inverse the sample `x` from time `t_start` to `t_end` by DPM-Solver. + For discrete-time DPMs, we use `t_start=1/N`, where `N` is the total time steps during training. + """ + t_0 = 1. / self.noise_schedule.total_N if t_start is None else t_start + t_T = self.noise_schedule.T if t_end is None else t_end + assert t_0 > 0 and t_T > 0, 'Time range needs to be greater than 0. For discrete-time DPMs, \ + it needs to be in [1 / N, 1], where N is the length of betas array' + + return self.sample( + x, + steps=steps, + t_start=t_0, + t_end=t_T, + order=order, + skip_type=skip_type, + method=method, + lower_order_final=lower_order_final, + denoise_to_zero=denoise_to_zero, + solver_type=solver_type, + atol=atol, + rtol=rtol, + return_intermediate=return_intermediate) + + def sample(self, + x, + steps=20, + t_start=None, + t_end=None, + order=2, + skip_type='time_uniform', + method='multistep', + lower_order_final=True, + denoise_to_zero=False, + solver_type='dpmsolver', + atol=0.0078, + rtol=0.05): + """ + Compute the sample at time `t_end` by DPM-Solver, given the initial `x` at time `t_start`. + + ===================================================== + + We support the following algorithms for both noise prediction model and data prediction model: + - 'singlestep': + Singlestep DPM-Solver (i.e. "DPM-Solver-fast" in the paper), + which combines different orders of singlestep DPM-Solver. + We combine all the singlestep solvers with order <= + `order` to use up all the function evaluations (steps). + The total number of function evaluations (NFE) == `steps`. + Given a fixed NFE == `steps`, the sampling procedure is: + - If `order` == 1: + - Denote K = steps. We use K steps of DPM-Solver-1 (i.e. DDIM). + - If `order` == 2: + - Denote K = (steps // 2) + (steps % 2). We take K intermediate time steps for sampling. + - If steps % 2 == 0, we use K steps of singlestep DPM-Solver-2. + - If steps % 2 == 1, we use (K - 1) steps of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1. + - If `order` == 3: + - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling. + - If steps % 3 == 0, we use (K - 2) steps of singlestep DPM-Solver-3, + and 1 step of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1. + - If steps % 3 == 1, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of DPM-Solver-1. + - If steps % 3 == 2, we use (K - 1) + steps of singlestep DPM-Solver-3 and 1 step of singlestep DPM-Solver-2. + - 'multistep': + Multistep DPM-Solver with the order of `order`. + The total number of function evaluations (NFE) == `steps`. + We initialize the first `order` values by lower order multistep solvers. + Given a fixed NFE == `steps`, the sampling procedure is: + Denote K = steps. + - If `order` == 1: + - We use K steps of DPM-Solver-1 (i.e. DDIM). + - If `order` == 2: + - We firstly use 1 step of DPM-Solver-1, then use (K - 1) step of multistep DPM-Solver-2. + - If `order` == 3: + - We firstly use 1 step of DPM-Solver-1, then 1 step of multistep DPM-Solver-2, + then (K - 2) step of multistep DPM-Solver-3. + - 'singlestep_fixed': + Fixed order singlestep DPM-Solver + (i.e. DPM-Solver-1 or singlestep DPM-Solver-2 or singlestep DPM-Solver-3). + We use singlestep DPM-Solver-`order` for `order`=1 or 2 or 3, + with total [`steps` // `order`] * `order` NFE. + - 'adaptive': + Adaptive step size DPM-Solver (i.e. "DPM-Solver-12" and "DPM-Solver-23" in the paper). + We ignore `steps` and use adaptive step size DPM-Solver with a higher order of `order`. + You can adjust the absolute tolerance `atol` and the relative tolerance + `rtol` to balance the computatation costs + (NFE) and the sample quality. + - If `order` == 2, we use DPM-Solver-12 which combines DPM-Solver-1 + and singlestep DPM-Solver-2. + - If `order` == 3, we use DPM-Solver-23 which combines singlestep + DPM-Solver-2 and singlestep DPM-Solver-3. + + ===================================================== + + Some advices for choosing the algorithm: + - For **unconditional sampling** or **guided sampling with small guidance scale** by DPMs: + Use singlestep DPM-Solver ("DPM-Solver-fast" in the paper) with `order = 3`. + e.g. + >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=False) + >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=3, + skip_type='time_uniform', method='singlestep') + - For **guided sampling with large guidance scale** by DPMs: + Use multistep DPM-Solver with `predict_x0 = True` and `order = 2`. + e.g. + >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=True) + >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=2, + skip_type='time_uniform', method='multistep') + + We support three types of `skip_type`: + - 'logSNR': uniform logSNR for the time steps. **Recommended for low-resolutional images** + - 'time_uniform': uniform time for the time steps. **Recommended for high-resolutional images**. + - 'time_quadratic': quadratic time for the time steps. + + ===================================================== + Args: + x: A pytorch tensor. The initial value at time `t_start` + e.g. if `t_start` == T, then `x` is a sample from the standard normal distribution. + steps: A `int`. The total number of function evaluations (NFE). + t_start: A `float`. The starting time of the sampling. + If `T` is None, we use self.noise_schedule.T (default is 1.0). + t_end: A `float`. The ending time of the sampling. + If `t_end` is None, we use 1. / self.noise_schedule.total_N. + e.g. if total_N == 1000, we have `t_end` == 1e-3. + For discrete-time DPMs: + - We recommend `t_end` == 1. / self.noise_schedule.total_N. + For continuous-time DPMs: + - We recommend `t_end` == 1e-3 when `steps` <= 15; and `t_end` == 1e-4 when `steps` > 15. + order: A `int`. The order of DPM-Solver. + skip_type: A `str`. The type for the spacing of the time steps. + 'time_uniform' or 'logSNR' or 'time_quadratic'. + method: A `str`. The method for sampling. + 'singlestep' or 'multistep' or 'singlestep_fixed' or 'adaptive'. + denoise_to_zero: A `bool`. Whether to denoise to time 0 at the final step. + Default is `False`. If `denoise_to_zero` is `True`, the total NFE is (`steps` + 1). + + This trick is firstly proposed by DDPM (https://arxiv.org/abs/2006.11239) and + score_sde (https://arxiv.org/abs/2011.13456). Such trick can improve the FID + for diffusion models sampling by diffusion SDEs for low-resolutional images + (such as CIFAR-10). However, we observed that such trick does not matter for + high-resolutional images. As it needs an additional NFE, we do not recommend + it for high-resolutional images. + lower_order_final: A `bool`. Whether to use lower order solvers at the final steps. + Only valid for `method=multistep` and `steps < 15`. We empirically find that + this trick is a key to stabilizing the sampling by DPM-Solver with very few steps + (especially for steps <= 10). So we recommend to set it to be `True`. + solver_type: A `str`. The taylor expansion type for the solver. + `dpmsolver` or `taylor`. We recommend `dpmsolver`. + atol: A `float`. The absolute tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'. + rtol: A `float`. The relative tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'. + Returns: + x_end: A pytorch tensor. The approximated solution at time `t_end`. + + """ + t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end + t_T = self.noise_schedule.T if t_start is None else t_start + assert t_0 > 0 and t_T > 0, 'Time range needs to be greater than 0. For discrete-time DPMs, \ + it needs to be in [1 / N, 1], where N is the length of betas array' + + device = x.device + with torch.no_grad(): + if method == 'adaptive': + x = self.dpm_solver_adaptive( + x, + order=order, + t_T=t_T, + t_0=t_0, + atol=atol, + rtol=rtol, + solver_type=solver_type) + elif method == 'multistep': + assert steps >= order + timesteps = self.get_time_steps( + skip_type=skip_type, + t_T=t_T, + t_0=t_0, + N=steps, + device=device) + assert timesteps.shape[0] - 1 == steps + # Init the initial values. + step = 0 + t = timesteps[step] + t_prev_list = [t] + model_prev_list = [self.model_fn(x, t)] + # Init the first `order` values by lower order multistep DPM-Solver. + for step in range(1, order): + t = timesteps[step] + x = self.multistep_dpm_solver_update( + x, + model_prev_list, + t_prev_list, + t, + step, + solver_type=solver_type) + t_prev_list.append(t) + model_prev_list.append(self.model_fn(x, t)) + # Compute the remaining values by `order`-th order multistep DPM-Solver. + for step in range(order, steps + 1): + t = timesteps[step] + # We only use lower order for steps < 10 + if lower_order_final and steps < 10: + step_order = min(order, steps + 1 - step) + else: + step_order = order + x = self.multistep_dpm_solver_update( + x, + model_prev_list, + t_prev_list, + t, + step_order, + solver_type=solver_type) + for i in range(order - 1): + t_prev_list[i] = t_prev_list[i + 1] + model_prev_list[i] = model_prev_list[i + 1] + t_prev_list[-1] = t + # We do not need to evaluate the final model value. + if step < steps: + model_prev_list[-1] = self.model_fn(x, t) + elif method in ['singlestep', 'singlestep_fixed']: + if method == 'singlestep': + timesteps_outer, orders = self.get_orders_and_timesteps_for_singlestep_solver( + steps=steps, + order=order, + skip_type=skip_type, + t_T=t_T, + t_0=t_0, + device=device) + elif method == 'singlestep_fixed': + K = steps // order + orders = [ + order, + ] * K + timesteps_outer = self.get_time_steps( + skip_type=skip_type, + t_T=t_T, + t_0=t_0, + N=K, + device=device) + for step, order in enumerate(orders): + s, t = timesteps_outer[step], timesteps_outer[step + 1] + timesteps_inner = self.get_time_steps( + skip_type=skip_type, + t_T=s.item(), + t_0=t.item(), + N=order, + device=device) + lambda_inner = self.noise_schedule.marginal_lambda( + timesteps_inner) + h = lambda_inner[-1] - lambda_inner[0] + r1 = None if order <= 1 else (lambda_inner[1] + - lambda_inner[0]) / h + r2 = None if order <= 2 else (lambda_inner[2] + - lambda_inner[0]) / h + x = self.singlestep_dpm_solver_update( + x, s, t, order, solver_type=solver_type, r1=r1, r2=r2) + + else: + raise ValueError('Got wrong method {}'.format(method)) + if denoise_to_zero: + t = torch.ones((1, )).to(device) * t_0 + x = self.denoise_to_zero_fn(x, t) + return x + + +def interpolate_fn(x, xp, yp): + """ + A piecewise linear function y = f(x), using xp and yp as keypoints. + We implement f(x) in a differentiable way (i.e. applicable for autograd). + The function f(x) is well-defined for all x-axis. (For x beyond the bounds of xp, + we use the outmost points of xp to define the linear function.) + + Args: + x: PyTorch tensor with shape [N, C], where N is the batch size, + C is the number of channels (we use C = 1 for DPM-Solver). + xp: PyTorch tensor with shape [C, K], where K is the number of keypoints. + yp: PyTorch tensor with shape [C, K]. + Returns: + The function values f(x), with shape [N, C]. + """ + N, K = x.shape[0], xp.shape[1] + all_x = torch.cat( + [x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2) + sorted_all_x, x_indices = torch.sort(all_x, dim=2) + x_idx = torch.argmin(x_indices, dim=2) + cand_start_idx = x_idx - 1 + start_idx = torch.where( + torch.eq(x_idx, 0), + torch.tensor(1, device=x.device), + torch.where( + torch.eq(x_idx, K), + torch.tensor(K - 2, device=x.device), + cand_start_idx, + ), + ) + end_idx = torch.where( + torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1) + start_x = torch.gather( + sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2) + end_x = torch.gather( + sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2) + start_idx2 = torch.where( + torch.eq(x_idx, 0), + torch.tensor(0, device=x.device), + torch.where( + torch.eq(x_idx, K), + torch.tensor(K - 2, device=x.device), + cand_start_idx, + ), + ) + y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1) + start_y = torch.gather( + y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2) + end_y = torch.gather( + y_positions_expanded, dim=2, + index=(start_idx2 + 1).unsqueeze(2)).squeeze(2) + cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x) + return cand + + +def expand_dims(v, dims): + """ + Expand the tensor `v` to the dim `dims`. + + Args: + `v`: a PyTorch tensor with shape [N]. + `dim`: a `int`. + Returns: + a PyTorch tensor with shape [N, 1, 1, ..., 1] and the total dimension is `dims`. + """ + return v[(..., ) + (None, ) * (dims - 1)] diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/mha_flash.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/mha_flash.py new file mode 100644 index 0000000000000000000000000000000000000000..009efd442a76a06779418fb68c4baa8b0a51a84f --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/mha_flash.py @@ -0,0 +1,120 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os +import random +import time + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from flash_attn.flash_attention import FlashAttention + + +class FlashAttentionBlock(nn.Module): + + def __init__(self, + dim, + context_dim=None, + num_heads=None, + head_dim=None, + batch_size=4): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(FlashAttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + if self.head_dim <= 128 and (self.head_dim % 8) == 0: + self.flash_attn = FlashAttention( + softmax_scale=None, attention_dropout=0.0) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def _init_weight(self, module): + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=0.15) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Conv2d): + module.weight.data.normal_(mean=0.0, std=0.15) + if module.bias is not None: + module.bias.data.zero_() + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + cq = torch.zeros([b, n, d, 4], dtype=q.dtype, device=q.device) + q = torch.cat([q, cq], dim=-1) + + qkv = torch.cat([q, k, v], dim=1) + origin_dtype = qkv.dtype + qkv = qkv.permute(0, 3, 1, 2).reshape(b, -1, 3, n, + d).half().contiguous() + out, _ = self.flash_attn(qkv) + out.to(origin_dtype) + + if context is not None: + out = out[:, :-4, :, :] + out = out.permute(0, 2, 3, 1).reshape(b, c, h, w) + + # output + x = self.proj(out) + return x + identity + + +if __name__ == '__main__': + batch_size = 8 + flash_net = FlashAttentionBlock( + dim=1280, + context_dim=512, + num_heads=None, + head_dim=64, + batch_size=batch_size).cuda() + + x = torch.randn([batch_size, 1280, 32, 32], dtype=torch.float32).cuda() + context = torch.randn([batch_size, 4, 512], dtype=torch.float32).cuda() + # context = None + flash_net.eval() + + with amp.autocast(enabled=True): + # warm up + for i in range(5): + y = flash_net(x, context) + torch.cuda.synchronize() + s1 = time.time() + for i in range(10): + y = flash_net(x, context) + torch.cuda.synchronize() + s2 = time.time() + + print(f'Average cost time {(s2-s1)*1000/10} ms') diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..13e46148e6024413ea385591194aaa10a81b1ad9 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/__init__.py @@ -0,0 +1,2 @@ +from .clip import * +from .midas import * diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/clip.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce2d8187897c022b004e50dabd5a6fb65fcd8ef --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/clip.py @@ -0,0 +1,460 @@ +import math +import os.path as osp + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import modelscope.models.multi_modal.videocomposer.ops as ops + +__all__ = [ + 'CLIP', 'clip_vit_b_32', 'clip_vit_b_16', 'clip_vit_l_14', + 'clip_vit_l_14_336px', 'clip_vit_h_16' +] + + +def DOWNLOAD_TO_CACHE(oss_key, + file_or_dirname=None, + cache_dir=osp.join( + '/'.join(osp.abspath(__file__).split('/')[:-2]), + 'model_weights')): + r"""Download OSS [file or folder] to the cache folder. + Only the 0th process on each node will run the downloading. + Barrier all processes until the downloading is completed. + """ + # source and target paths + base_path = osp.join(cache_dir, file_or_dirname or osp.basename(oss_key)) + + return base_path + + +def to_fp16(m): + if isinstance(m, (nn.Linear, nn.Conv2d)): + m.weight.data = m.weight.data.half() + if m.bias is not None: + m.bias.data = m.bias.data.half() + elif hasattr(m, 'head'): + p = getattr(m, 'head') + p.data = p.data.half() + + +class QuickGELU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class LayerNorm(nn.LayerNorm): + r"""Subclass of nn.LayerNorm to handle fp16. + """ + + def forward(self, x): + return super(LayerNorm, self).forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.attn_dropout = nn.Dropout(attn_dropout) + self.proj = nn.Linear(dim, dim) + self.proj_dropout = nn.Dropout(proj_dropout) + + def forward(self, x, mask=None): + r"""x: [B, L, C]. + mask: [*, L, L]. + """ + b, l, _, n = *x.size(), self.num_heads + + # compute query, key, and value + q, k, v = self.to_qkv(x.transpose(0, 1)).chunk(3, dim=-1) + q = q.reshape(l, b * n, -1).transpose(0, 1) + k = k.reshape(l, b * n, -1).transpose(0, 1) + v = v.reshape(l, b * n, -1).transpose(0, 1) + + # compute attention + attn = self.scale * torch.bmm(q, k.transpose(1, 2)) + if mask is not None: + attn = attn.masked_fill(mask[:, :l, :l] == 0, float('-inf')) + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + attn = self.attn_dropout(attn) + + # gather context + x = torch.bmm(attn, v) + x = x.view(b, n, l, -1).transpose(1, 2).reshape(b, l, -1) + + # output + x = self.proj(x) + x = self.proj_dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads, attn_dropout=0.0, proj_dropout=0.0): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads, attn_dropout, proj_dropout) + self.norm2 = LayerNorm(dim) + self.mlp = nn.Sequential( + nn.Linear(dim, dim * 4), QuickGELU(), nn.Linear(dim * 4, dim), + nn.Dropout(proj_dropout)) + + def forward(self, x, mask=None): + x = x + self.attn(self.norm1(x), mask) + x = x + self.mlp(self.norm2(x)) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=224, + patch_size=16, + dim=768, + out_dim=512, + num_heads=12, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + assert image_size % patch_size == 0 + super(VisionTransformer, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + gain = 1.0 / math.sqrt(dim) + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.pos_embedding = nn.Parameter( + gain * torch.randn(1, self.num_patches + 1, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.pre_norm = LayerNorm(dim) + self.transformer = nn.Sequential(*[ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.post_norm = LayerNorm(dim) + + # head + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + def forward(self, x): + b, dtype = x.size(0), self.head.dtype + x = x.type(dtype) + + # patch-embedding + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) # [b, n, c] + x = torch.cat([self.cls_embedding.repeat(b, 1, 1).type(dtype), x], + dim=1) + x = self.dropout(x + self.pos_embedding.type(dtype)) + x = self.pre_norm(x) + + # transformer + x = self.transformer(x) + + # head + x = self.post_norm(x) + x = torch.mm(x[:, 0, :], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class TextTransformer(nn.Module): + + def __init__(self, + vocab_size, + text_len, + dim=512, + out_dim=512, + num_heads=8, + num_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(TextTransformer, self).__init__() + self.vocab_size = vocab_size + self.text_len = text_len + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + + # embeddings + self.token_embedding = nn.Embedding(vocab_size, dim) + self.pos_embedding = nn.Parameter(0.01 * torch.randn(1, text_len, dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.transformer = nn.ModuleList([ + AttentionBlock(dim, num_heads, attn_dropout, proj_dropout) + for _ in range(num_layers) + ]) + self.norm = LayerNorm(dim) + + # head + gain = 1.0 / math.sqrt(dim) + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + + # causal attention mask + self.register_buffer('attn_mask', + torch.tril(torch.ones(1, text_len, text_len))) + + def forward(self, x): + eot, dtype = x.argmax(dim=-1), self.head.dtype + + # embeddings + x = self.dropout( + self.token_embedding(x).type(dtype) + + self.pos_embedding.type(dtype)) + + # transformer + for block in self.transformer: + x = block(x, self.attn_mask) + + # head + x = self.norm(x) + x = torch.mm(x[torch.arange(x.size(0)), eot], self.head) + return x + + def fp16(self): + return self.apply(to_fp16) + + +class CLIP(nn.Module): + + def __init__(self, + embed_dim=512, + image_size=224, + patch_size=16, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0): + super(CLIP, self).__init__() + self.embed_dim = embed_dim + self.image_size = image_size + self.patch_size = patch_size + self.vision_dim = vision_dim + self.vision_heads = vision_heads + self.vision_layers = vision_layers + self.vocab_size = vocab_size + self.text_len = text_len + self.text_dim = text_dim + self.text_heads = text_heads + self.text_layers = text_layers + + # models + self.visual = VisionTransformer( + image_size=image_size, + patch_size=patch_size, + dim=vision_dim, + out_dim=embed_dim, + num_heads=vision_heads, + num_layers=vision_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.textual = TextTransformer( + vocab_size=vocab_size, + text_len=text_len, + dim=text_dim, + out_dim=embed_dim, + num_heads=text_heads, + num_layers=text_layers, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout) + self.log_scale = nn.Parameter(math.log(1 / 0.07) * torch.ones([])) + + def forward(self, imgs, txt_tokens): + r"""imgs: [B, C, H, W] of torch.float32. + txt_tokens: [B, T] of torch.long. + """ + xi = self.visual(imgs) + xt = self.textual(txt_tokens) + + # normalize features + xi = F.normalize(xi, p=2, dim=1) + xt = F.normalize(xt, p=2, dim=1) + + # gather features from all ranks + full_xi = ops.diff_all_gather(xi) + full_xt = ops.diff_all_gather(xt) + + # logits + scale = self.log_scale.exp() + logits_i2t = scale * torch.mm(xi, full_xt.t()) + logits_t2i = scale * torch.mm(xt, full_xi.t()) + + # labels + labels = torch.arange( + len(xi) * ops.get_rank(), + len(xi) * (ops.get_rank() + 1), + dtype=torch.long, + device=xi.device) + return logits_i2t, logits_t2i, labels + + def init_weights(self): + # embeddings + nn.init.normal_(self.textual.token_embedding.weight, std=0.02) + nn.init.normal_(self.visual.patch_embedding.weight, tsd=0.1) + + # attentions + for modality in ['visual', 'textual']: + dim = self.vision_dim if modality == 'visual' else 'textual' + transformer = getattr(self, modality).transformer + proj_gain = (1.0 / math.sqrt(dim)) * ( + 1.0 / math.sqrt(2 * transformer.num_layers)) + attn_gain = 1.0 / math.sqrt(dim) + mlp_gain = 1.0 / math.sqrt(2.0 * dim) + for block in transformer.layers: + nn.init.normal_(block.attn.to_qkv.weight, std=attn_gain) + nn.init.normal_(block.attn.proj.weight, std=proj_gain) + nn.init.normal_(block.mlp[0].weight, std=mlp_gain) + nn.init.normal_(block.mlp[2].weight, std=proj_gain) + + def param_groups(self): + groups = [{ + 'params': [ + p for n, p in self.named_parameters() + if 'norm' in n or n.endswith('bias') + ], + 'weight_decay': + 0.0 + }, { + 'params': [ + p for n, p in self.named_parameters() + if not ('norm' in n or n.endswith('bias')) + ] + }] + return groups + + def fp16(self): + return self.apply(to_fp16) + + +def _clip(name, pretrained=False, **kwargs): + model = CLIP(**kwargs) + if pretrained: + model.load_state_dict( + torch.load( + DOWNLOAD_TO_CACHE(f'models/clip/{name}.pth'), + map_location='cpu')) + return model + + +def clip_vit_b_32(pretrained=False, **kwargs): + cfg = dict( + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12) + cfg.update(**kwargs) + return _clip('openai-clip-vit-base-32', pretrained, **cfg) + + +def clip_vit_b_16(pretrained=False, **kwargs): + cfg = dict( + embed_dim=512, + image_size=224, + patch_size=32, + vision_dim=768, + vision_heads=12, + vision_layers=12, + vocab_size=49408, + text_len=77, + text_dim=512, + text_heads=8, + text_layers=12) + cfg.update(**kwargs) + return _clip('openai-clip-vit-base-16', pretrained, **cfg) + + +def clip_vit_l_14(pretrained=False, **kwargs): + cfg = dict( + embed_dim=768, + image_size=224, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + vocab_size=49408, + text_len=77, + text_dim=768, + text_heads=12, + text_layers=12) + cfg.update(**kwargs) + return _clip('openai-clip-vit-large-14', pretrained, **cfg) + + +def clip_vit_l_14_336px(pretrained=False, **kwargs): + cfg = dict( + embed_dim=768, + image_size=336, + patch_size=14, + vision_dim=1024, + vision_heads=16, + vision_layers=24, + vocab_size=49408, + text_len=77, + text_dim=768, + text_heads=12, + text_layers=12) + cfg.update(**kwargs) + return _clip('openai-clip-vit-large-14-336px', pretrained, **cfg) + + +def clip_vit_h_16(pretrained=False, **kwargs): + assert not pretrained, 'pretrained model for openai-clip-vit-huge-16 is not available!' + cfg = dict( + embed_dim=1024, + image_size=256, + patch_size=16, + vision_dim=1280, + vision_heads=16, + vision_layers=32, + vocab_size=49408, + text_len=77, + text_dim=1024, + text_heads=16, + text_layers=24) + cfg.update(**kwargs) + return _clip('openai-clip-vit-huge-16', pretrained, **cfg) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/midas.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/midas.py new file mode 100644 index 0000000000000000000000000000000000000000..4992e721e7eb15a1f4b6ecd8c20d9b869041679c --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/models/midas.py @@ -0,0 +1,320 @@ +r"""A much cleaner re-implementation of ``https://github.com/isl-org/MiDaS''. + Image augmentation: T.Compose([ + Resize( + keep_aspect_ratio=True, + ensure_multiple_of=32, + interpolation=cv2.INTER_CUBIC), + T.ToTensor(), + T.Normalize( + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5])]). + Fast inference: + model = model.to(memory_format=torch.channels_last).half() + input = input.to(memory_format=torch.channels_last).half() + output = model(input) +""" +import math +import os +import os.path as osp + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['MiDaS', 'midas_v3'] + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads): + assert dim % num_heads == 0 + super(SelfAttention, self).__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = 1.0 / math.sqrt(self.head_dim) + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.proj = nn.Linear(dim, dim) + + def forward(self, x): + b, l, c, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + q, k, v = self.to_qkv(x).view(b, l, n * 3, d).chunk(3, dim=2) + + # compute attention + attn = self.scale * torch.einsum('binc,bjnc->bnij', q, k) + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + + # gather context + x = torch.einsum('bnij,bjnc->binc', attn, v) + x = x.reshape(b, l, c) + + # output + x = self.proj(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads): + super(AttentionBlock, self).__init__() + self.dim = dim + self.num_heads = num_heads + + # layers + self.norm1 = nn.LayerNorm(dim) + self.attn = SelfAttention(dim, num_heads) + self.norm2 = nn.LayerNorm(dim) + self.mlp = nn.Sequential( + nn.Linear(dim, dim * 4), nn.GELU(), nn.Linear(dim * 4, dim)) + + def forward(self, x): + x = x + self.attn(self.norm1(x)) + x = x + self.mlp(self.norm2(x)) + return x + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=384, + patch_size=16, + dim=1024, + out_dim=1000, + num_heads=16, + num_layers=24): + assert image_size % patch_size == 0 + super(VisionTransformer, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size) + self.cls_embedding = nn.Parameter(torch.zeros(1, 1, dim)) + self.pos_embedding = nn.Parameter( + torch.empty(1, self.num_patches + 1, dim).normal_(std=0.02)) + + # blocks + self.blocks = nn.Sequential( + *[AttentionBlock(dim, num_heads) for _ in range(num_layers)]) + self.norm = nn.LayerNorm(dim) + + # head + self.head = nn.Linear(dim, out_dim) + + def forward(self, x): + b = x.size(0) + + # embeddings + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) + x = torch.cat([self.cls_embedding.repeat(b, 1, 1), x], dim=1) + x = x + self.pos_embedding + + # blocks + x = self.blocks(x) + x = self.norm(x) + + # head + x = self.head(x) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, dim): + super(ResidualBlock, self).__init__() + self.dim = dim + + # layers + self.residual = nn.Sequential( + nn.ReLU(inplace=False), # NOTE: avoid modifying the input + nn.Conv2d(dim, dim, 3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(dim, dim, 3, padding=1)) + + def forward(self, x): + return x + self.residual(x) + + +class FusionBlock(nn.Module): + + def __init__(self, dim): + super(FusionBlock, self).__init__() + self.dim = dim + + # layers + self.layer1 = ResidualBlock(dim) + self.layer2 = ResidualBlock(dim) + self.conv_out = nn.Conv2d(dim, dim, 1) + + def forward(self, *xs): + assert len(xs) in (1, 2), 'invalid number of inputs' + if len(xs) == 1: + x = self.layer2(xs[0]) + else: + x = self.layer2(xs[0] + self.layer1(xs[1])) + x = F.interpolate( + x, scale_factor=2, mode='bilinear', align_corners=True) + x = self.conv_out(x) + return x + + +class MiDaS(nn.Module): + r"""MiDaS v3.0 DPT-Large from ``https://github.com/isl-org/MiDaS''. + Monocular depth estimation using dense prediction transformers. + """ + + def __init__(self, + image_size=384, + patch_size=16, + dim=1024, + neck_dims=[256, 512, 1024, 1024], + fusion_dim=256, + num_heads=16, + num_layers=24): + assert image_size % patch_size == 0 + assert num_layers % 4 == 0 + super(MiDaS, self).__init__() + self.image_size = image_size + self.patch_size = patch_size + self.dim = dim + self.neck_dims = neck_dims + self.fusion_dim = fusion_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.num_patches = (image_size // patch_size)**2 + + # embeddings + self.patch_embedding = nn.Conv2d( + 3, dim, kernel_size=patch_size, stride=patch_size) + self.cls_embedding = nn.Parameter(torch.zeros(1, 1, dim)) + self.pos_embedding = nn.Parameter( + torch.empty(1, self.num_patches + 1, dim).normal_(std=0.02)) + + # blocks + stride = num_layers // 4 + self.blocks = nn.Sequential( + *[AttentionBlock(dim, num_heads) for _ in range(num_layers)]) + self.slices = [slice(i * stride, (i + 1) * stride) for i in range(4)] + + # stage1 (4x) + self.fc1 = nn.Sequential(nn.Linear(dim * 2, dim), nn.GELU()) + self.conv1 = nn.Sequential( + nn.Conv2d(dim, neck_dims[0], 1), + nn.ConvTranspose2d(neck_dims[0], neck_dims[0], 4, stride=4), + nn.Conv2d(neck_dims[0], fusion_dim, 3, padding=1, bias=False)) + self.fusion1 = FusionBlock(fusion_dim) + + # stage2 (8x) + self.fc2 = nn.Sequential(nn.Linear(dim * 2, dim), nn.GELU()) + self.conv2 = nn.Sequential( + nn.Conv2d(dim, neck_dims[1], 1), + nn.ConvTranspose2d(neck_dims[1], neck_dims[1], 2, stride=2), + nn.Conv2d(neck_dims[1], fusion_dim, 3, padding=1, bias=False)) + self.fusion2 = FusionBlock(fusion_dim) + + # stage3 (16x) + self.fc3 = nn.Sequential(nn.Linear(dim * 2, dim), nn.GELU()) + self.conv3 = nn.Sequential( + nn.Conv2d(dim, neck_dims[2], 1), + nn.Conv2d(neck_dims[2], fusion_dim, 3, padding=1, bias=False)) + self.fusion3 = FusionBlock(fusion_dim) + + # stage4 (32x) + self.fc4 = nn.Sequential(nn.Linear(dim * 2, dim), nn.GELU()) + self.conv4 = nn.Sequential( + nn.Conv2d(dim, neck_dims[3], 1), + nn.Conv2d(neck_dims[3], neck_dims[3], 3, stride=2, padding=1), + nn.Conv2d(neck_dims[3], fusion_dim, 3, padding=1, bias=False)) + self.fusion4 = FusionBlock(fusion_dim) + + # head + self.head = nn.Sequential( + nn.Conv2d(fusion_dim, fusion_dim // 2, 3, padding=1), + nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True), + nn.Conv2d(fusion_dim // 2, 32, 3, padding=1), + nn.ReLU(inplace=True), nn.ConvTranspose2d(32, 1, 1), + nn.ReLU(inplace=True)) + + def forward(self, x): + b, _, h, w, p = *x.size(), self.patch_size + assert h % p == 0 and w % p == 0, f'Image size ({w}, {h}) is not divisible by patch size ({p}, {p})' + hp, wp, grid = h // p, w // p, self.image_size // p + + # embeddings + pos_embedding = torch.cat([ + self.pos_embedding[:, :1], + F.interpolate( + self.pos_embedding[:, 1:].reshape(1, grid, grid, -1).permute( + 0, 3, 1, 2), + size=(hp, wp), + mode='bilinear', + align_corners=False).permute(0, 2, 3, 1).reshape( + 1, hp * wp, -1) + ], + dim=1) # noqa + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) + x = torch.cat([self.cls_embedding.repeat(b, 1, 1), x], dim=1) + x = x + pos_embedding + + # stage1 + x = self.blocks[self.slices[0]](x) + x1 = torch.cat([x[:, 1:], x[:, :1].expand_as(x[:, 1:])], dim=-1) + x1 = self.fc1(x1).permute(0, 2, 1).unflatten(2, (hp, wp)) + x1 = self.conv1(x1) + + # stage2 + x = self.blocks[self.slices[1]](x) + x2 = torch.cat([x[:, 1:], x[:, :1].expand_as(x[:, 1:])], dim=-1) + x2 = self.fc2(x2).permute(0, 2, 1).unflatten(2, (hp, wp)) + x2 = self.conv2(x2) + + # stage3 + x = self.blocks[self.slices[2]](x) + x3 = torch.cat([x[:, 1:], x[:, :1].expand_as(x[:, 1:])], dim=-1) + x3 = self.fc3(x3).permute(0, 2, 1).unflatten(2, (hp, wp)) + x3 = self.conv3(x3) + + # stage4 + x = self.blocks[self.slices[3]](x) + x4 = torch.cat([x[:, 1:], x[:, :1].expand_as(x[:, 1:])], dim=-1) + x4 = self.fc4(x4).permute(0, 2, 1).unflatten(2, (hp, wp)) + x4 = self.conv4(x4) + + # fusion + x4 = self.fusion4(x4) + x3 = self.fusion3(x4, x3) + x2 = self.fusion2(x3, x2) + x1 = self.fusion1(x2, x1) + + # head + x = self.head(x1) + return x + + +def midas_v3(model_dir, pretrained=False, **kwargs): + cfg = dict( + image_size=384, + patch_size=16, + dim=1024, + neck_dims=[256, 512, 1024, 1024], + fusion_dim=256, + num_heads=16, + num_layers=24) + cfg.update(**kwargs) + model = MiDaS(**cfg) + if pretrained: + model.load_state_dict( + torch.load( + os.path.join(model_dir, 'midas_v3_dpt_large.pth'), + map_location='cpu')) + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..48b87bda62d1ee0d5f3b2e8245b99c8b6398300b --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .degration import * +from .distributed import * +from .losses import * +from .random_mask import * +from .utils import * diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/degration.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/degration.py new file mode 100644 index 0000000000000000000000000000000000000000..de97be656373a581c5e88607b29db4fb0210c0ed --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/degration.py @@ -0,0 +1,998 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os +import random +from datetime import datetime + +import numpy as np +import scipy +import scipy.stats as stats +import torch +from scipy import ndimage +from scipy.interpolate import interp2d +from scipy.linalg import orth +from torchvision.utils import make_grid + +os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' + +__all__ = ['degradation_bsrgan_light', 'degradation_bsrgan'] + + +# get uint8 image of size HxWxn_channles (RGB) +def imread_uint(path, n_channels=3): + # input: path + # output: HxWx3(RGB or GGG), or HxWx1 (G) + if n_channels == 1: + img = cv2.imread(path, 0) + img = np.expand_dims(img, axis=2) + elif n_channels == 3: + img = cv2.imread(path, cv2.IMREAD_UNCHANGED) + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) + else: + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + return img + + +def uint2single(img): + return np.float32(img / 255.) + + +def single2uint(img): + return np.uint8((img.clip(0, 1) * 255.).round()) + + +def uint162single(img): + return np.float32(img / 65535.) + + +def single2uint16(img): + return np.uint16((img.clip(0, 1) * 65535.).round()) + + +def rgb2ycbcr(img, only_y=True): + '''same as matlab rgb2ycbcr + only_y: only return Y channel + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + if only_y: + rlt = np.dot(img, [65.481, 128.553, 24.966]) / 255.0 + 16.0 + else: + rlt = np.matmul(img, + [[65.481, -37.797, 112.0], [128.553, -74.203, -93.786], + [24.966, 112.0, -18.214]]) / 255.0 + [16, 128, 128] + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def ycbcr2rgb(img): + '''same as matlab ycbcr2rgb + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + rlt = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0, -0.00153632, 0.00791071], + [0.00625893, -0.00318811, 0]]) * 255.0 + [ + -222.921, 135.576, -276.836 + ] # noqa + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def bgr2ycbcr(img, only_y=True): + '''bgr version of rgb2ycbcr + only_y: only return Y channel + Input: + uint8, [0, 255] + float, [0, 1] + ''' + in_img_type = img.dtype + img.astype(np.float32) + if in_img_type != np.uint8: + img *= 255. + # convert + if only_y: + rlt = np.dot(img, [24.966, 128.553, 65.481]) / 255.0 + 16.0 + else: + rlt = np.matmul(img, + [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786], + [65.481, -37.797, 112.0]]) / 255.0 + [16, 128, 128] + if in_img_type == np.uint8: + rlt = rlt.round() + else: + rlt /= 255. + return rlt.astype(in_img_type) + + +def channel_convert(in_c, tar_type, img_list): + # conversion among BGR, gray and y + if in_c == 3 and tar_type == 'gray': + gray_list = [cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) for img in img_list] + return [np.expand_dims(img, axis=2) for img in gray_list] + elif in_c == 3 and tar_type == 'y': + y_list = [bgr2ycbcr(img, only_y=True) for img in img_list] + return [np.expand_dims(img, axis=2) for img in y_list] + elif in_c == 1 and tar_type == 'RGB': + return [cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) for img in img_list] + else: + return img_list + + +# PSNR +def calculate_psnr(img1, img2, border=0): + # img1 and img2 have range [0, 255] + if not img1.shape == img2.shape: + raise ValueError('Input images must have the same dimensions.') + h, w = img1.shape[:2] + img1 = img1[border:h - border, border:w - border] + img2 = img2[border:h - border, border:w - border] + + img1 = img1.astype(np.float64) + img2 = img2.astype(np.float64) + mse = np.mean((img1 - img2)**2) + if mse == 0: + return float('inf') + return 20 * math.log10(255.0 / math.sqrt(mse)) + + +# SSIM +def calculate_ssim(img1, img2, border=0): + '''calculate SSIM + the same outputs as MATLAB's + img1, img2: [0, 255] + ''' + if not img1.shape == img2.shape: + raise ValueError('Input images must have the same dimensions.') + h, w = img1.shape[:2] + img1 = img1[border:h - border, border:w - border] + img2 = img2[border:h - border, border:w - border] + + if img1.ndim == 2: + return ssim(img1, img2) + elif img1.ndim == 3: + if img1.shape[2] == 3: + ssims = [] + for i in range(3): + ssims.append(ssim(img1[:, :, i], img2[:, :, i])) + return np.array(ssims).mean() + elif img1.shape[2] == 1: + return ssim(np.squeeze(img1), np.squeeze(img2)) + else: + raise ValueError('Wrong input image dimensions.') + + +def ssim(img1, img2): + C1 = (0.01 * 255)**2 + C2 = (0.03 * 255)**2 + + img1 = img1.astype(np.float64) + img2 = img2.astype(np.float64) + kernel = cv2.getGaussianKernel(11, 1.5) + window = np.outer(kernel, kernel.transpose()) + + mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5] + mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5] + mu1_sq = mu1**2 + mu2_sq = mu2**2 + mu1_mu2 = mu1 * mu2 + sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq + sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq + sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2 + + ssim_map = ((2 * mu1_mu2 + C1) # noqa + * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) # noqa + * # noqa + (sigma1_sq + sigma2_sq + C2)) # noqa + return ssim_map.mean() + + +# matlab 'imresize' function, now only support 'bicubic' +def cubic(x): + absx = torch.abs(x) + absx2 = absx**2 + absx3 = absx**3 + return (1.5 * absx3 - 2.5 * absx2 + 1) * ((absx <= 1).type_as(absx)) + \ + (-0.5 * absx3 + 2.5 * absx2 - 4*absx + 2) * (((absx > 1) * (absx <= 2)).type_as(absx)) # noqa + + +def calculate_weights_indices(in_length, out_length, scale, kernel, + kernel_width, antialiasing): + if (scale < 1) and (antialiasing): + # Use a modified kernel to simultaneously interpolate and antialias- larger kernel width + kernel_width = kernel_width / scale + + # Output-space coordinates + x = torch.linspace(1, out_length, out_length) + + # Input-space coordinates. Calculate the inverse mapping such that 0.5 + # in output space maps to 0.5 in input space, and 0.5+scale in output + # space maps to 1.5 in input space. + u = x / scale + 0.5 * (1 - 1 / scale) + + # What is the left-most pixel that can be involved in the computation? + left = torch.floor(u - kernel_width / 2) + + # What is the maximum number of pixels that can be involved in the + # computation? Note: it's OK to use an extra pixel here; if the + # corresponding weights are all zero, it will be eliminated at the end + # of this function. + P = math.ceil(kernel_width) + 2 + + # The indices of the input pixels involved in computing the k-th output + # pixel are in row k of the indices matrix. + indices = left.view(out_length, 1).expand(out_length, P) + torch.linspace( + 0, P - 1, P).view(1, P).expand(out_length, P) + + # The weights used to compute the k-th output pixel are in row k of the + # weights matrix. + distance_to_center = u.view(out_length, 1).expand(out_length, P) - indices + # apply cubic kernel + if (scale < 1) and (antialiasing): + weights = scale * cubic(distance_to_center * scale) + else: + weights = cubic(distance_to_center) + # Normalize the weights matrix so that each row sums to 1. + weights_sum = torch.sum(weights, 1).view(out_length, 1) + weights = weights / weights_sum.expand(out_length, P) + + # If a column in weights is all zero, get rid of it. only consider the first and last column. + weights_zero_tmp = torch.sum((weights == 0), 0) + if not math.isclose(weights_zero_tmp[0], 0, rel_tol=1e-6): + indices = indices.narrow(1, 1, P - 2) + weights = weights.narrow(1, 1, P - 2) + if not math.isclose(weights_zero_tmp[-1], 0, rel_tol=1e-6): + indices = indices.narrow(1, 0, P - 2) + weights = weights.narrow(1, 0, P - 2) + weights = weights.contiguous() + indices = indices.contiguous() + sym_len_s = -indices.min() + 1 + sym_len_e = indices.max() - in_length + indices = indices + sym_len_s - 1 + return weights, indices, int(sym_len_s), int(sym_len_e) + + +# imresize for tensor image [0, 1] +def imresize(img, scale, antialiasing=True): + # Now the scale should be the same for H and W + # input: img: pytorch tensor, CHW or HW [0,1] + # output: CHW or HW [0,1] w/o round + need_squeeze = True if img.dim() == 2 else False + if need_squeeze: + img.unsqueeze_(0) + in_C, in_H, in_W = img.size() + out_C, out_H, out_W = in_C, math.ceil(in_H * scale), math.ceil(in_W + * scale) + kernel_width = 4 + kernel = 'cubic' + + # Return the desired dimension order for performing the resize. The + # strategy is to perform the resize first along the dimension with the + # smallest scale factor. + # Now we do not support this. + + # get weights and indices + weights_H, indices_H, sym_len_Hs, sym_len_He = calculate_weights_indices( + in_H, out_H, scale, kernel, kernel_width, antialiasing) + weights_W, indices_W, sym_len_Ws, sym_len_We = calculate_weights_indices( + in_W, out_W, scale, kernel, kernel_width, antialiasing) + # process H dimension + # symmetric copying + img_aug = torch.FloatTensor(in_C, in_H + sym_len_Hs + sym_len_He, in_W) + img_aug.narrow(1, sym_len_Hs, in_H).copy_(img) + + sym_patch = img[:, :sym_len_Hs, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + img_aug.narrow(1, 0, sym_len_Hs).copy_(sym_patch_inv) + + sym_patch = img[:, -sym_len_He:, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + img_aug.narrow(1, sym_len_Hs + in_H, sym_len_He).copy_(sym_patch_inv) + + out_1 = torch.FloatTensor(in_C, out_H, in_W) + kernel_width = weights_H.size(1) + for i in range(out_H): + idx = int(indices_H[i][0]) + for j in range(out_C): + out_1[j, i, :] = img_aug[j, idx:idx + kernel_width, :].transpose( + 0, 1).mv(weights_H[i]) + + # process W dimension + # symmetric copying + out_1_aug = torch.FloatTensor(in_C, out_H, in_W + sym_len_Ws + sym_len_We) + out_1_aug.narrow(2, sym_len_Ws, in_W).copy_(out_1) + + sym_patch = out_1[:, :, :sym_len_Ws] + inv_idx = torch.arange(sym_patch.size(2) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(2, inv_idx) + out_1_aug.narrow(2, 0, sym_len_Ws).copy_(sym_patch_inv) + + sym_patch = out_1[:, :, -sym_len_We:] + inv_idx = torch.arange(sym_patch.size(2) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(2, inv_idx) + out_1_aug.narrow(2, sym_len_Ws + in_W, sym_len_We).copy_(sym_patch_inv) + + out_2 = torch.FloatTensor(in_C, out_H, out_W) + kernel_width = weights_W.size(1) + for i in range(out_W): + idx = int(indices_W[i][0]) + for j in range(out_C): + out_2[j, :, i] = out_1_aug[j, :, + idx:idx + kernel_width].mv(weights_W[i]) + if need_squeeze: + out_2.squeeze_() + return out_2 + + +# imresize for numpy image [0, 1] +def imresize_np(img, scale, antialiasing=True): + # Now the scale should be the same for H and W + # input: img: Numpy, HWC or HW [0,1] + # output: HWC or HW [0,1] w/o round + img = torch.from_numpy(img) + need_squeeze = True if img.dim() == 2 else False + if need_squeeze: + img.unsqueeze_(2) + + in_H, in_W, in_C = img.size() + out_C, out_H, out_W = in_C, math.ceil(in_H * scale), math.ceil(in_W + * scale) + kernel_width = 4 + kernel = 'cubic' + + # Return the desired dimension order for performing the resize. The + # strategy is to perform the resize first along the dimension with the + # smallest scale factor. + # Now we do not support this. + + # get weights and indices + weights_H, indices_H, sym_len_Hs, sym_len_He = calculate_weights_indices( + in_H, out_H, scale, kernel, kernel_width, antialiasing) + weights_W, indices_W, sym_len_Ws, sym_len_We = calculate_weights_indices( + in_W, out_W, scale, kernel, kernel_width, antialiasing) + # process H dimension + # symmetric copying + img_aug = torch.FloatTensor(in_H + sym_len_Hs + sym_len_He, in_W, in_C) + img_aug.narrow(0, sym_len_Hs, in_H).copy_(img) + + sym_patch = img[:sym_len_Hs, :, :] + inv_idx = torch.arange(sym_patch.size(0) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(0, inv_idx) + img_aug.narrow(0, 0, sym_len_Hs).copy_(sym_patch_inv) + + sym_patch = img[-sym_len_He:, :, :] + inv_idx = torch.arange(sym_patch.size(0) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(0, inv_idx) + img_aug.narrow(0, sym_len_Hs + in_H, sym_len_He).copy_(sym_patch_inv) + + out_1 = torch.FloatTensor(out_H, in_W, in_C) + kernel_width = weights_H.size(1) + for i in range(out_H): + idx = int(indices_H[i][0]) + for j in range(out_C): + out_1[i, :, j] = img_aug[idx:idx + kernel_width, :, + j].transpose(0, 1).mv(weights_H[i]) + + # process W dimension + # symmetric copying + out_1_aug = torch.FloatTensor(out_H, in_W + sym_len_Ws + sym_len_We, in_C) + out_1_aug.narrow(1, sym_len_Ws, in_W).copy_(out_1) + + sym_patch = out_1[:, :sym_len_Ws, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + out_1_aug.narrow(1, 0, sym_len_Ws).copy_(sym_patch_inv) + + sym_patch = out_1[:, -sym_len_We:, :] + inv_idx = torch.arange(sym_patch.size(1) - 1, -1, -1).long() + sym_patch_inv = sym_patch.index_select(1, inv_idx) + out_1_aug.narrow(1, sym_len_Ws + in_W, sym_len_We).copy_(sym_patch_inv) + + out_2 = torch.FloatTensor(out_H, out_W, in_C) + kernel_width = weights_W.size(1) + for i in range(out_W): + idx = int(indices_W[i][0]) + for j in range(out_C): + out_2[:, i, j] = out_1_aug[:, idx:idx + kernel_width, + j].mv(weights_W[i]) + if need_squeeze: + out_2.squeeze_() + + return out_2.numpy() + + +def modcrop_np(img, sf): + ''' + Args: + img: numpy image, WxH or WxHxC + sf: scale factor + Return: + cropped image + ''' + w, h = img.shape[:2] + im = np.copy(img) + return im[:w - w % sf, :h - h % sf, ...] + + +def analytic_kernel(k): + """Calculate the X4 kernel from the X2 kernel (for proof see appendix in paper)""" + k_size = k.shape[0] + # Calculate the big kernels size + big_k = np.zeros((3 * k_size - 2, 3 * k_size - 2)) + # Loop over the small kernel to fill the big one + for r in range(k_size): + for c in range(k_size): + big_k[2 * r:2 * r + k_size, 2 * c:2 * c + k_size] += k[r, c] * k + # Crop the edges of the big kernel to ignore very small values and increase run time of SR + crop = k_size // 2 + cropped_big_k = big_k[crop:-crop, crop:-crop] + # Normalize to 1 + return cropped_big_k / cropped_big_k.sum() + + +def anisotropic_Gaussian(ksize=15, theta=np.pi, l1=6, l2=6): + """ generate an anisotropic Gaussian kernel + Args: + ksize : e.g., 15, kernel size + theta : [0, pi], rotation angle range + l1 : [0.1,50], scaling of eigenvalues + l2 : [0.1,l1], scaling of eigenvalues + If l1 = l2, will get an isotropic Gaussian kernel. + Returns: + k : kernel + """ + + v = np.dot( + np.array([[np.cos(theta), -np.sin(theta)], + [np.sin(theta), np.cos(theta)]]), np.array([1., 0.])) + V = np.array([[v[0], v[1]], [v[1], -v[0]]]) + D = np.array([[l1, 0], [0, l2]]) + Sigma = np.dot(np.dot(V, D), np.linalg.inv(V)) + k = gm_blur_kernel(mean=[0, 0], cov=Sigma, size=ksize) + return k + + +def gm_blur_kernel(mean, cov, size=15): + center = size / 2.0 + 0.5 + k = np.zeros([size, size]) + for y in range(size): + for x in range(size): + cy = y - center + 1 + cx = x - center + 1 + k[y, x] = stats.multivariate_normal.pdf([cx, cy], + mean=mean, + cov=cov) + + k = k / np.sum(k) + return k + + +def shift_pixel(x, sf, upper_left=True): + """shift pixel for super-resolution with different scale factors + Args: + x: WxHxC or WxH + sf: scale factor + upper_left: shift direction + """ + h, w = x.shape[:2] + shift = (sf - 1) * 0.5 + xv, yv = np.arange(0, w, 1.0), np.arange(0, h, 1.0) + if upper_left: + x1 = xv + shift + y1 = yv + shift + else: + x1 = xv - shift + y1 = yv - shift + + x1 = np.clip(x1, 0, w - 1) + y1 = np.clip(y1, 0, h - 1) + + if x.ndim == 2: + x = interp2d(xv, yv, x)(x1, y1) + if x.ndim == 3: + for i in range(x.shape[-1]): + x[:, :, i] = interp2d(xv, yv, x[:, :, i])(x1, y1) + + return x + + +def blur(x, k): + ''' + x: image, NxcxHxW + k: kernel, Nx1xhxw + ''' + n, c = x.shape[:2] + p1, p2 = (k.shape[-2] - 1) // 2, (k.shape[-1] - 1) // 2 + x = torch.nn.functional.pad(x, pad=(p1, p2, p1, p2), mode='replicate') + k = k.repeat(1, c, 1, 1) + k = k.view(-1, 1, k.shape[2], k.shape[3]) + x = x.view(1, -1, x.shape[2], x.shape[3]) + x = torch.nn.functional.conv2d( + x, k, bias=None, stride=1, padding=0, groups=n * c) + x = x.view(n, c, x.shape[2], x.shape[3]) + + return x + + +def gen_kernel( + k_size=np.array([15, 15]), + scale_factor=np.array([4, 4]), + min_var=0.6, + max_var=10., + noise_level=0): + """" + # modified version of https://github.com/assafshocher/BlindSR_dataset_generator + # Kai Zhang + # min_var = 0.175 * sf # variance of the gaussian kernel will be sampled between min_var and max_var + # max_var = 2.5 * sf + """ + # Set random eigen-vals (lambdas) and angle (theta) for COV matrix + lambda_1 = min_var + np.random.rand() * (max_var - min_var) + lambda_2 = min_var + np.random.rand() * (max_var - min_var) + theta = np.random.rand() * np.pi # random theta + noise = -noise_level + np.random.rand(*k_size) * noise_level * 2 + + # Set COV matrix using Lambdas and Theta + LAMBDA = np.diag([lambda_1, lambda_2]) + Q = np.array([[np.cos(theta), -np.sin(theta)], + [np.sin(theta), np.cos(theta)]]) + SIGMA = Q @ LAMBDA @ Q.T + INV_SIGMA = np.linalg.inv(SIGMA)[None, None, :, :] + + # Set expectation position (shifting kernel for aligned image) + MU = k_size // 2 - 0.5 * (scale_factor - 1) + MU = MU[None, None, :, None] + + # Create meshgrid for Gaussian + [X, Y] = np.meshgrid(range(k_size[0]), range(k_size[1])) + Z = np.stack([X, Y], 2)[:, :, :, None] + + # Calcualte Gaussian for every pixel of the kernel + ZZ = Z - MU + ZZ_t = ZZ.transpose(0, 1, 3, 2) + raw_kernel = np.exp(-0.5 * np.squeeze(ZZ_t @ INV_SIGMA @ ZZ)) * (1 + noise) + + # shift the kernel so it will be centered + # raw_kernel_centered = kernel_shift(raw_kernel, scale_factor) + + # Normalize the kernel and return + # kernel = raw_kernel_centered / np.sum(raw_kernel_centered) + kernel = raw_kernel / np.sum(raw_kernel) + return kernel + + +def fspecial_gaussian(hsize, sigma): + hsize = [hsize, hsize] + siz = [(hsize[0] - 1.0) / 2.0, (hsize[1] - 1.0) / 2.0] + std = sigma + [x, y] = np.meshgrid( + np.arange(-siz[1], siz[1] + 1), np.arange(-siz[0], siz[0] + 1)) + arg = -(x * x + y * y) / (2 * std * std) + h = np.exp(arg) + h[h < scipy.finfo(float).eps * h.max()] = 0 + sumh = h.sum() + if sumh != 0: + h = h / sumh + return h + + +def fspecial_laplacian(alpha): + alpha = max([0, min([alpha, 1])]) + h1 = alpha / (alpha + 1) + h2 = (1 - alpha) / (alpha + 1) + h = [[h1, h2, h1], [h2, -4 / (alpha + 1), h2], [h1, h2, h1]] + h = np.array(h) + return h + + +def fspecial(filter_type, *args, **kwargs): + if filter_type == 'gaussian': + return fspecial_gaussian(*args, **kwargs) + if filter_type == 'laplacian': + return fspecial_laplacian(*args, **kwargs) + + +def bicubic_degradation(x, sf=3): + ''' + Args: + x: HxWxC image, [0, 1] + sf: down-scale factor + Return: + bicubicly downsampled LR image + ''' + x = imresize_np(x, scale=1 / sf) + return x + + +def srmd_degradation(x, k, sf=3): + ''' blur + bicubic downsampling + Args: + x: HxWxC image, [0, 1] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + Reference: + @inproceedings{zhang2018learning, + title={Learning a single convolutional super-resolution network for multiple degradations}, + author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, + booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, + pages={3262--3271}, + year={2018} + } + ''' + x = ndimage.filters.convolve(x, np.expand_dims(k, axis=2), mode='wrap') + x = bicubic_degradation(x, sf=sf) + return x + + +def dpsr_degradation(x, k, sf=3): + ''' bicubic downsampling + blur + Args: + x: HxWxC image, [0, 1] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + Reference: + @inproceedings{zhang2019deep, + title={Deep Plug-and-Play Super-Resolution for Arbitrary Blur Kernels}, + author={Zhang, Kai and Zuo, Wangmeng and Zhang, Lei}, + booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, + pages={1671--1681}, + year={2019} + } + ''' + x = bicubic_degradation(x, sf=sf) + x = ndimage.filters.convolve(x, np.expand_dims(k, axis=2), mode='wrap') + return x + + +def classical_degradation(x, k, sf=3): + ''' blur + downsampling + Args: + x: HxWxC image, [0, 1]/[0, 255] + k: hxw, double + sf: down-scale factor + Return: + downsampled LR image + ''' + x = ndimage.filters.convolve(x, np.expand_dims(k, axis=2), mode='wrap') + st = 0 + return x[st::sf, st::sf, ...] + + +def add_sharpening(img, weight=0.5, radius=50, threshold=10): + """USM sharpening. borrowed from real-ESRGAN + Input image: I; Blurry image: B. + 1. K = I + weight * (I - B) + 2. Mask = 1 if abs(I - B) > threshold, else: 0 + 3. Blur mask: + 4. Out = Mask * K + (1 - Mask) * I + Args: + img (Numpy array): Input image, HWC, BGR; float32, [0, 1]. + weight (float): Sharp weight. Default: 1. + radius (float): Kernel size of Gaussian blur. Default: 50. + threshold (int): + """ + if radius % 2 == 0: + radius += 1 + blur = cv2.GaussianBlur(img, (radius, radius), 0) + residual = img - blur + mask = np.abs(residual) * 255 > threshold + mask = mask.astype('float32') + soft_mask = cv2.GaussianBlur(mask, (radius, radius), 0) + + K = img + weight * residual + K = np.clip(K, 0, 1) + return soft_mask * K + (1 - soft_mask) * img + + +def add_blur_1(img, sf=4): + wd2 = 4.0 + sf + wd = 2.0 + 0.2 * sf + + wd2 = wd2 / 4 + wd = wd / 4 + + if random.random() < 0.5: + l1 = wd2 * random.random() + l2 = wd2 * random.random() + k = anisotropic_Gaussian( + ksize=random.randint(2, 11) + 3, + theta=random.random() * np.pi, + l1=l1, + l2=l2) + else: + k = fspecial('gaussian', + random.randint(2, 4) + 3, wd * random.random()) + img = ndimage.filters.convolve( + img, np.expand_dims(k, axis=2), mode='mirror') + + return img + + +def add_resize(img, sf=4): + rnum = np.random.rand() + if rnum > 0.8: + sf1 = random.uniform(1, 2) + elif rnum < 0.7: + sf1 = random.uniform(0.5 / sf, 1) + else: + sf1 = 1.0 + img = cv2.resize( + img, (int(sf1 * img.shape[1]), int(sf1 * img.shape[0])), + interpolation=random.choice([1, 2, 3])) + img = np.clip(img, 0.0, 1.0) + + return img + + +def add_Gaussian_noise(img, noise_level1=2, noise_level2=25): + noise_level = random.randint(noise_level1, noise_level2) + rnum = np.random.rand() + if rnum > 0.6: + img = img + np.random.normal(0, noise_level / 255.0, img.shape).astype( + np.float32) + elif rnum < 0.4: + img = img + np.random.normal(0, noise_level / 255.0, + (*img.shape[:2], 1)).astype(np.float32) + else: + L = noise_level2 / 255. + D = np.diag(np.random.rand(3)) + U = orth(np.random.rand(3, 3)) + conv = np.dot(np.dot(np.transpose(U), D), U) + img = img + np.random.multivariate_normal([0, 0, 0], np.abs( + L**2 * conv), img.shape[:2]).astype(np.float32) + img = np.clip(img, 0.0, 1.0) + return img + + +def add_speckle_noise(img, noise_level1=2, noise_level2=25): + noise_level = random.randint(noise_level1, noise_level2) + img = np.clip(img, 0.0, 1.0) + rnum = random.random() + if rnum > 0.6: + img += img * np.random.normal(0, noise_level / 255.0, + img.shape).astype(np.float32) + elif rnum < 0.4: + img += img * np.random.normal(0, noise_level / 255.0, + (*img.shape[:2], 1)).astype(np.float32) + else: + L = noise_level2 / 255. + D = np.diag(np.random.rand(3)) + U = orth(np.random.rand(3, 3)) + conv = np.dot(np.dot(np.transpose(U), D), U) + img += img * np.random.multivariate_normal( + [0, 0, 0], np.abs(L**2 * conv), img.shape[:2]).astype(np.float32) + img = np.clip(img, 0.0, 1.0) + return img + + +def add_Poisson_noise(img): + img = np.clip((img * 255.0).round(), 0, 255) / 255. + vals = 10**(2 * random.random() + 2.0) + if random.random() < 0.5: + img = np.random.poisson(img * vals).astype(np.float32) / vals + else: + img_gray = np.dot(img[..., :3], [0.299, 0.587, 0.114]) + img_gray = np.clip((img_gray * 255.0).round(), 0, 255) / 255. + noise_gray = np.random.poisson(img_gray * vals).astype( + np.float32) / vals - img_gray + img += noise_gray[:, :, np.newaxis] + img = np.clip(img, 0.0, 1.0) + return img + + +def add_JPEG_noise(img): + quality_factor = random.randint(80, 95) + img = cv2.cvtColor(single2uint(img), cv2.COLOR_RGB2BGR) + result, encimg = cv2.imencode( + '.jpg', img, [int(cv2.IMWRITE_JPEG_QUALITY), quality_factor]) + img = cv2.imdecode(encimg, 1) + img = cv2.cvtColor(uint2single(img), cv2.COLOR_BGR2RGB) + return img + + +def random_crop(lq, hq, sf=4, lq_patchsize=64): + h, w = lq.shape[:2] + rnd_h = random.randint(0, h - lq_patchsize) + rnd_w = random.randint(0, w - lq_patchsize) + lq = lq[rnd_h:rnd_h + lq_patchsize, rnd_w:rnd_w + lq_patchsize, :] + + rnd_h_H, rnd_w_H = int(rnd_h * sf), int(rnd_w * sf) + hq = hq[rnd_h_H:rnd_h_H + lq_patchsize * sf, + rnd_w_H:rnd_w_H + lq_patchsize * sf, :] + return lq, hq + + +def degradation_bsrgan_light(image, sf=4, isp_model=None): + """ + This is the variant of the degradation model of BSRGAN from the paper + "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" + sf: scale factor + isp_model: camera ISP model + Returns + img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] + hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] + """ + image = uint2single(image) + _, jpeg_prob, scale2_prob = 0.25, 0.9, 0.25 + + h1, w1 = image.shape[:2] + image = image.copy()[:w1 - w1 % sf, :h1 - h1 % sf, ...] + h, w = image.shape[:2] + + if sf == 4 and random.random() < scale2_prob: + if np.random.rand() < 0.5: + image = cv2.resize( + image, + (int(1 / 2 * image.shape[1]), int(1 / 2 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + image = imresize_np(image, 1 / 2, True) + image = np.clip(image, 0.0, 1.0) + sf = 2 + + shuffle_order = random.sample(range(7), 7) + idx1, idx2 = shuffle_order.index(2), shuffle_order.index(3) + if idx1 > idx2: + shuffle_order[idx1], shuffle_order[idx2] = shuffle_order[ + idx2], shuffle_order[idx1] + + for i in shuffle_order: + + if i == 0: + image = add_blur_1(image, sf=sf) + elif i == 2: + a, b = image.shape[1], image.shape[0] + # downsample2 + if random.random() < 0.8: + sf1 = random.uniform(1, 2 * sf) + image = cv2.resize( + image, (int(1 / sf1 * image.shape[1]), + int(1 / sf1 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + k = fspecial('gaussian', 25, random.uniform(0.1, 0.6 * sf)) + k_shifted = shift_pixel(k, sf) + k_shifted = k_shifted / k_shifted.sum() + image = ndimage.filters.convolve( + image, np.expand_dims(k_shifted, axis=2), mode='mirror') + image = image[0::sf, 0::sf, ...] + image = np.clip(image, 0.0, 1.0) + elif i == 3: + # downsample3 + image = cv2.resize( + image, (int(1 / sf * a), int(1 / sf * b)), + interpolation=random.choice([1, 2, 3])) + image = np.clip(image, 0.0, 1.0) + elif i == 4: + # add Gaussian noise + image = add_Gaussian_noise(image, noise_level1=1, noise_level2=2) + elif i == 5: + # add JPEG noise + if random.random() < jpeg_prob: + image = add_JPEG_noise(image) + + # add final JPEG compression noise + image = add_JPEG_noise(image) + image = single2uint(image) + return image + + +def add_blur_2(img, sf=4): + wd2 = 4.0 + sf + wd = 2.0 + 0.2 * sf + if random.random() < 0.5: + l1 = wd2 * random.random() + l2 = wd2 * random.random() + k = anisotropic_Gaussian( + ksize=2 * random.randint(2, 11) + 3, + theta=random.random() * np.pi, + l1=l1, + l2=l2) + else: + k = fspecial('gaussian', 2 * random.randint(2, 11) + 3, + wd * random.random()) + img = ndimage.filters.convolve( + img, np.expand_dims(k, axis=2), mode='mirror') + return img + + +def degradation_bsrgan(image, sf=4, isp_model=None): + """ + This is the variant of the degradation model of BSRGAN from the paper + "Designing a Practical Degradation Model for Deep Blind Image Super-Resolution" + sf: scale factor + isp_model: camera ISP model + Returns + img: low-quality patch, size: lq_patchsizeXlq_patchsizeXC, range: [0, 1] + hq: corresponding high-quality patch, size: (lq_patchsizexsf)X(lq_patchsizexsf)XC, range: [0, 1] + """ + image = uint2single(image) + _, jpeg_prob, scale2_prob = 0.25, 0.9, 0.25 + + h1, w1 = image.shape[:2] + image = image.copy()[:w1 - w1 % sf, :h1 - h1 % sf, ...] + h, w = image.shape[:2] + + if sf == 4 and random.random() < scale2_prob: + if np.random.rand() < 0.5: + image = cv2.resize( + image, + (int(1 / 2 * image.shape[1]), int(1 / 2 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + image = imresize_np(image, 1 / 2, True) + image = np.clip(image, 0.0, 1.0) + sf = 2 + + shuffle_order = random.sample(range(7), 7) + idx1, idx2 = shuffle_order.index(2), shuffle_order.index(3) + if idx1 > idx2: + shuffle_order[idx1], shuffle_order[idx2] = shuffle_order[ + idx2], shuffle_order[idx1] + + for i in shuffle_order: + + if i == 0: + image = add_blur_2(image, sf=sf) + elif i == 1: + image = add_blur_2(image, sf=sf) + elif i == 2: + a, b = image.shape[1], image.shape[0] + # downsample2 + if random.random() < 0.75: + sf1 = random.uniform(1, 2 * sf) + image = cv2.resize( + image, (int(1 / sf1 * image.shape[1]), + int(1 / sf1 * image.shape[0])), + interpolation=random.choice([1, 2, 3])) + else: + k = fspecial('gaussian', 25, random.uniform(0.1, 0.6 * sf)) + k_shifted = shift_pixel(k, sf) + k_shifted = k_shifted / k_shifted.sum() + image = ndimage.filters.convolve( + image, np.expand_dims(k_shifted, axis=2), mode='mirror') + image = image[0::sf, 0::sf, ...] + image = np.clip(image, 0.0, 1.0) + elif i == 3: + # downsample3 + image = cv2.resize( + image, (int(1 / sf * a), int(1 / sf * b)), + interpolation=random.choice([1, 2, 3])) + image = np.clip(image, 0.0, 1.0) + elif i == 4: + # add Gaussian noise + image = add_Gaussian_noise(image, noise_level1=2, noise_level2=25) + elif i == 5: + # add JPEG noise + if random.random() < jpeg_prob: + image = add_JPEG_noise(image) + + # add final JPEG compression noise + image = add_JPEG_noise(image) + image = single2uint(image) + return image diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/distributed.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..201e156f4ab47499a55b2e5ac743b56650abeecb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/distributed.py @@ -0,0 +1,460 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import functools +import pickle +from collections import OrderedDict + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.autograd import Function + +__all__ = [ + 'is_dist_initialized', 'get_world_size', 'get_rank', 'new_group', + 'destroy_process_group', 'barrier', 'broadcast', 'all_reduce', 'reduce', + 'gather', 'all_gather', 'reduce_dict', 'get_global_gloo_group', + 'generalized_all_gather', 'generalized_gather', 'scatter', + 'reduce_scatter', 'send', 'recv', 'isend', 'irecv', 'shared_random_seed', + 'diff_all_gather', 'diff_all_reduce', 'diff_scatter', 'diff_copy', + 'spherical_kmeans', 'sinkhorn' +] + + +def is_dist_initialized(): + return dist.is_available() and dist.is_initialized() + + +def get_world_size(group=None): + return dist.get_world_size(group) if is_dist_initialized() else 1 + + +def get_rank(group=None): + return dist.get_rank(group) if is_dist_initialized() else 0 + + +def new_group(ranks=None, **kwargs): + if is_dist_initialized(): + return dist.new_group(ranks, **kwargs) + return None + + +def destroy_process_group(): + if is_dist_initialized(): + dist.destroy_process_group() + + +def barrier(group=None, **kwargs): + if get_world_size(group) > 1: + dist.barrier(group, **kwargs) + + +def broadcast(tensor, src, group=None, **kwargs): + if get_world_size(group) > 1: + return dist.broadcast(tensor, src, group, **kwargs) + + +def all_reduce(tensor, op=dist.ReduceOp.SUM, group=None, **kwargs): + if get_world_size(group) > 1: + return dist.all_reduce(tensor, op, group, **kwargs) + + +def reduce(tensor, dst, op=dist.ReduceOp.SUM, group=None, **kwargs): + if get_world_size(group) > 1: + return dist.reduce(tensor, dst, op, group, **kwargs) + + +def gather(tensor, dst=0, group=None, **kwargs): + rank = get_rank() + world_size = get_world_size(group) + if world_size == 1: + return [tensor] + tensor_list = [torch.empty_like(tensor) + for _ in range(world_size)] if rank == dst else None + dist.gather(tensor, tensor_list, dst, group, **kwargs) + return tensor_list + + +def all_gather(tensor, uniform_size=True, group=None, **kwargs): + world_size = get_world_size(group) + if world_size == 1: + return [tensor] + assert tensor.is_contiguous( + ), 'ops.all_gather requires the tensor to be contiguous()' + + if uniform_size: + tensor_list = [torch.empty_like(tensor) for _ in range(world_size)] + dist.all_gather(tensor_list, tensor, group, **kwargs) + return tensor_list + else: + # collect tensor shapes across GPUs + shape = tuple(tensor.shape) + shape_list = generalized_all_gather(shape, group) + + # flatten the tensor + tensor = tensor.reshape(-1) + size = int(np.prod(shape)) + size_list = [int(np.prod(u)) for u in shape_list] + max_size = max(size_list) + + # pad to maximum size + if size != max_size: + padding = tensor.new_zeros(max_size - size) + tensor = torch.cat([tensor, padding], dim=0) + + # all_gather + tensor_list = [torch.empty_like(tensor) for _ in range(world_size)] + dist.all_gather(tensor_list, tensor, group, **kwargs) + + # reshape tensors + tensor_list = [ + t[:n].view(s) + for t, n, s in zip(tensor_list, size_list, shape_list) + ] + return tensor_list + + +@torch.no_grad() +def reduce_dict(input_dict, group=None, reduction='mean', **kwargs): + assert reduction in ['mean', 'sum'] + world_size = get_world_size(group) + if world_size == 1: + return input_dict + + # ensure that the orders of keys are consistent across processes + if isinstance(input_dict, OrderedDict): + keys = list(input_dict.keys) + else: + keys = sorted(input_dict.keys()) + vals = [input_dict[key] for key in keys] + vals = torch.stack(vals, dim=0) + dist.reduce(vals, dst=0, group=group, **kwargs) + if dist.get_rank(group) == 0 and reduction == 'mean': + vals /= world_size + dist.broadcast(vals, src=0, group=group, **kwargs) + reduced_dict = type(input_dict)([(key, val) + for key, val in zip(keys, vals)]) + return reduced_dict + + +@functools.lru_cache() +def get_global_gloo_group(): + backend = dist.get_backend() + assert backend in ['gloo', 'nccl'] + if backend == 'nccl': + return dist.new_group(backend='gloo') + else: + return dist.group.WORLD + + +def _serialize_to_tensor(data, group): + backend = dist.get_backend(group) + assert backend in ['gloo', 'nccl'] + device = torch.device('cpu' if backend == 'gloo' else 'cuda') + + buffer = pickle.dumps(data) + if len(buffer) > 1024**3: + logger = logging.getLogger(__name__) + logger.warning( + 'Rank {} trying to all-gather {:.2f} GB of data on device' + '{}'.format(get_rank(), + len(buffer) / (1024**3), device)) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to(device=device) + return tensor + + +def _pad_to_largest_tensor(tensor, group): + world_size = dist.get_world_size(group=group) + assert world_size >= 1, \ + 'gather/all_gather must be called from ranks within' \ + 'the give group!' + local_size = torch.tensor([tensor.numel()], + dtype=torch.int64, + device=tensor.device) + size_list = [ + torch.zeros([1], dtype=torch.int64, device=tensor.device) + for _ in range(world_size) + ] + + # gather tensors and compute the maximum size + dist.all_gather(size_list, local_size, group=group) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # pad tensors to the same size + if local_size != max_size: + padding = torch.zeros((max_size - local_size, ), + dtype=torch.uint8, + device=tensor.device) + tensor = torch.cat((tensor, padding), dim=0) + return size_list, tensor + + +def generalized_all_gather(data, group=None): + if get_world_size(group) == 1: + return [data] + if group is None: + group = get_global_gloo_group() + + tensor = _serialize_to_tensor(data, group) + size_list, tensor = _pad_to_largest_tensor(tensor, group) + max_size = max(size_list) + + # receiving tensors from all ranks + tensor_list = [ + torch.empty((max_size, ), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.all_gather(tensor_list, tensor, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + return data_list + + +def generalized_gather(data, dst=0, group=None): + world_size = get_world_size(group) + if world_size == 1: + return [data] + if group is None: + group = get_global_gloo_group() + rank = dist.get_rank() + + tensor = _serialize_to_tensor(data, group) + size_list, tensor = _pad_to_largest_tensor(tensor, group) + + # receiving tensors from all ranks to dst + if rank == dst: + max_size = max(size_list) + tensor_list = [ + torch.empty((max_size, ), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.gather(tensor, tensor_list, dst=dst, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + return data_list + else: + dist.gather(tensor, [], dst=dst, group=group) + return [] + + +def scatter(data, scatter_list=None, src=0, group=None, **kwargs): + r"""NOTE: only supports CPU tensor communication. + """ + if get_world_size(group) > 1: + return dist.scatter(data, scatter_list, src, group, **kwargs) + + +def reduce_scatter(output, + input_list, + op=dist.ReduceOp.SUM, + group=None, + **kwargs): + if get_world_size(group) > 1: + return dist.reduce_scatter(output, input_list, op, group, **kwargs) + + +def send(tensor, dst, group=None, **kwargs): + if get_world_size(group) > 1: + assert tensor.is_contiguous( + ), 'ops.send requires the tensor to be contiguous()' + return dist.send(tensor, dst, group, **kwargs) + + +def recv(tensor, src=None, group=None, **kwargs): + if get_world_size(group) > 1: + assert tensor.is_contiguous( + ), 'ops.recv requires the tensor to be contiguous()' + return dist.recv(tensor, src, group, **kwargs) + + +def isend(tensor, dst, group=None, **kwargs): + if get_world_size(group) > 1: + assert tensor.is_contiguous( + ), 'ops.isend requires the tensor to be contiguous()' + return dist.isend(tensor, dst, group, **kwargs) + + +def irecv(tensor, src=None, group=None, **kwargs): + if get_world_size(group) > 1: + assert tensor.is_contiguous( + ), 'ops.irecv requires the tensor to be contiguous()' + return dist.irecv(tensor, src, group, **kwargs) + + +def shared_random_seed(group=None): + seed = np.random.randint(2**31) + all_seeds = generalized_all_gather(seed, group) + return all_seeds[0] + + +def _all_gather(x): + if not (dist.is_available() + and dist.is_initialized()) or dist.get_world_size() == 1: + return x + rank = dist.get_rank() + world_size = dist.get_world_size() + tensors = [torch.empty_like(x) for _ in range(world_size)] + tensors[rank] = x + dist.all_gather(tensors, x) + return torch.cat(tensors, dim=0).contiguous() + + +def _all_reduce(x): + if not (dist.is_available() + and dist.is_initialized()) or dist.get_world_size() == 1: + return x + dist.all_reduce(x) + return x + + +def _split(x): + if not (dist.is_available() + and dist.is_initialized()) or dist.get_world_size() == 1: + return x + rank = dist.get_rank() + world_size = dist.get_world_size() + return x.chunk(world_size, dim=0)[rank].contiguous() + + +class DiffAllGather(Function): + r"""Differentiable all-gather. + """ + + @staticmethod + def symbolic(graph, input): + return _all_gather(input) + + @staticmethod + def forward(ctx, input): + return _all_gather(input) + + @staticmethod + def backward(ctx, grad_output): + return _split(grad_output) + + +class DiffAllReduce(Function): + r"""Differentiable all-reducd. + """ + + @staticmethod + def symbolic(graph, input): + return _all_reduce(input) + + @staticmethod + def forward(ctx, input): + return _all_reduce(input) + + @staticmethod + def backward(ctx, grad_output): + return grad_output + + +class DiffScatter(Function): + r"""Differentiable scatter. + """ + + @staticmethod + def symbolic(ctx, input): + return _split(input) + + @staticmethod + def backward(ctx, grad_output): + return _all_gather(grad_output) + + +class DiffCopy(Function): + r"""Differentiable copy that reduces all gradients during backward. + """ + + @staticmethod + def symbolic(graph, input): + return input + + @staticmethod + def forward(ctx, input): + return input + + @staticmethod + def backward(ctx, grad_output): + return _all_reduce(grad_output) + + +diff_all_gather = DiffAllGather.apply +diff_all_reduce = DiffAllReduce.apply +diff_scatter = DiffScatter.apply +diff_copy = DiffCopy.apply + + +@torch.no_grad() +def spherical_kmeans(feats, num_clusters, num_iters=10): + k, n, c = num_clusters, *feats.size() + ones = feats.new_ones(n, dtype=torch.long) + + # distributed settings + world_size = get_world_size() + + # init clusters + rand_inds = torch.randperm(n)[:int(np.ceil(k / world_size))] + clusters = torch.cat(all_gather(feats[rand_inds]), dim=0)[:k] + + # variables + new_clusters = feats.new_zeros(k, c) + counts = feats.new_zeros(k, dtype=torch.long) + + # iterative Expectation-Maximization + for step in range(num_iters + 1): + # Expectation step + simmat = torch.mm(feats, clusters.t()) + scores, assigns = simmat.max(dim=1) + if step == num_iters: + break + + # Maximization step + new_clusters.zero_().scatter_add_(0, + assigns.unsqueeze(1).repeat(1, c), + feats) + all_reduce(new_clusters) + + counts.zero_() + counts.index_add_(0, assigns, ones) + all_reduce(counts) + + mask = (counts > 0) + clusters[mask] = new_clusters[mask] / counts[mask].view(-1, 1) + clusters = F.normalize(clusters, p=2, dim=1) + return clusters, assigns, scores + + +@torch.no_grad() +def sinkhorn(Q, eps=0.5, num_iters=3): + # normalize Q + Q = torch.exp(Q / eps).t() + sum_Q = Q.sum() + all_reduce(sum_Q) + Q /= sum_Q + + # variables + n, m = Q.size() + u = Q.new_zeros(n) + r = Q.new_ones(n) / n + c = Q.new_ones(m) / (m * get_world_size()) + + # iterative update + cur_sum = Q.sum(dim=1) + all_reduce(cur_sum) + for i in range(num_iters): + u = cur_sum + Q *= (r / u).unsqueeze(1) + Q *= (c / Q.sum(dim=0)).unsqueeze(0) + cur_sum = Q.sum(dim=1) + all_reduce(cur_sum) + return (Q / Q.sum(dim=0, keepdim=True)).t().float() diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/losses.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..fbcb0b600625158a56dfb47aa9006bc94a4e0409 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/losses.py @@ -0,0 +1,37 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch + +__all__ = ['kl_divergence', 'discretized_gaussian_log_likelihood'] + + +def kl_divergence(mu1, logvar1, mu2, logvar2): + return 0.5 * ( + -1.0 + logvar2 - logvar1 + torch.exp(logvar1 - logvar2) + # noqa + ((mu1 - mu2)**2) * torch.exp(-logvar2)) + + +def standard_normal_cdf(x): + r"""A fast approximation of the cumulative distribution function of the standard normal. + """ + return 0.5 * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x0, mean, log_scale): + assert x0.shape == mean.shape == log_scale.shape + cx = x0 - mean + inv_stdv = torch.exp(-log_scale) + cdf_plus = standard_normal_cdf(inv_stdv * (cx + 1.0 / 255.0)) + cdf_min = standard_normal_cdf(inv_stdv * (cx - 1.0 / 255.0)) + log_cdf_plus = torch.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = torch.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = torch.where( + x0 < -0.999, log_cdf_plus, + torch.where(x0 > 0.999, log_one_minus_cdf_min, + torch.log(cdf_delta.clamp(min=1e-12)))) + assert log_probs.shape == x0.shape + return log_probs diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/random_mask.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/random_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..f23219b53fb3294a706b2af9849a60b5fbe21a19 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/random_mask.py @@ -0,0 +1,81 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import cv2 +import numpy as np + +__all__ = ['make_irregular_mask', 'make_rectangle_mask', 'make_uncrop'] + + +def make_irregular_mask(w, + h, + max_angle=4, + max_length=200, + max_width=100, + min_strokes=1, + max_strokes=5, + mode='line'): + # initialize mask + assert mode in ['line', 'circle', 'square'] + mask = np.zeros((h, w), np.float32) + + # draw strokes + num_strokes = np.random.randint(min_strokes, max_strokes + 1) + for i in range(num_strokes): + x1 = np.random.randint(w) + y1 = np.random.randint(h) + for j in range(1 + np.random.randint(5)): + angle = 0.01 + np.random.randint(max_angle) + if i % 2 == 0: + angle = 2 * 3.1415926 - angle + length = 10 + np.random.randint(max_length) + radius = 5 + np.random.randint(max_width) + x2 = np.clip((x1 + length * np.sin(angle)).astype(np.int32), 0, w) + y2 = np.clip((y1 + length * np.cos(angle)).astype(np.int32), 0, h) + if mode == 'line': + cv2.line(mask, (x1, y1), (x2, y2), 1.0, radius) + elif mode == 'circle': + cv2.circle( + mask, (x1, y1), radius=radius, color=1.0, thickness=-1) + elif mode == 'square': + radius = radius // 2 + mask[y1 - radius:y1 + radius, x1 - radius:x1 + radius] = 1 + x1, y1 = x2, y2 + return mask + + +def make_rectangle_mask(w, + h, + margin=10, + min_size=30, + max_size=150, + min_strokes=1, + max_strokes=4): + # initialize mask + mask = np.zeros((h, w), np.float32) + + # draw rectangles + num_strokes = np.random.randint(min_strokes, max_strokes + 1) + for i in range(num_strokes): + box_w = np.random.randint(min_size, max_size) + box_h = np.random.randint(min_size, max_size) + x1 = np.random.randint(margin, w - margin - box_w + 1) + y1 = np.random.randint(margin, h - margin - box_h + 1) + mask[y1:y1 + box_h, x1:x1 + box_w] = 1 + return mask + + +def make_uncrop(w, h): + # initialize mask + mask = np.zeros((h, w), np.float32) + + # randomly halve the image + side = np.random.choice([0, 1, 2, 3]) + if side == 0: + mask[:h // 2, :] = 1 + elif side == 1: + mask[h // 2:, :] = 1 + elif side == 2: + mask[:, :w // 2] = 1 + elif side == 3: + mask[:, w // 2:] = 1 + return mask diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/utils.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f47ad8e5207e9d4040adee11fc8efc3c457704f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/ops/utils.py @@ -0,0 +1,1039 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import base64 +import binascii +import copy +import glob +import gzip +import hashlib +import logging +import math +import os +import os.path as osp +import pickle +import sys +import time +import urllib.request +import zipfile +from io import BytesIO +from multiprocessing.pool import ThreadPool as Pool + +import imageio +import json +import numpy as np +import oss2 as oss +import requests +import skvideo.io +import torch +import torch.nn.functional as F +import torchvision.utils as tvutils +from einops import rearrange +from PIL import Image + +from modelscope.models.multi_modal.videocomposer.autoencoder import \ + DiagonalGaussianDistribution +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = [ + 'parse_oss_url', 'parse_bucket', 'read', 'read_image', 'read_gzip', + 'ceil_divide', 'to_device', 'put_object', 'put_torch_object', + 'put_object_from_file', 'get_object', 'get_object_to_file', 'rand_name', + 'save_image', 'save_video', 'save_video_vs_conditions', + 'save_video_multiple_conditions_with_data', + 'save_video_multiple_conditions', 'download_video_to_file', + 'save_video_grid_mp4', 'save_caps', 'ema', 'parallel', 'exists', + 'download', 'unzip', 'load_state_dict', 'inverse_indices', + 'detect_duplicates', 'read_tfs', 'md5', 'rope', 'format_state', + 'breakup_grid', 'huggingface_tokenizer', 'huggingface_model' +] + +TFS_CLIENT = None + + +def rand_name(length=8, suffix=''): + name = binascii.b2a_hex(os.urandom(length)).decode('utf-8') + if suffix: + if not suffix.startswith('.'): + suffix = '.' + suffix + name += suffix + return name + + +def save_with_model_kwargs(model_kwargs, video_data, autoencoder, ori_video, + viz_num, step, caps, palette, cfg, duration): + scale_factor = 0.18215 + video_data = 1. / scale_factor * video_data + + bs_vd = video_data.shape[0] + video_data = rearrange(video_data, 'b c f h w -> (b f) c h w') + chunk_size = min(16, video_data.shape[0]) + video_data_list = torch.chunk( + video_data, video_data.shape[0] // chunk_size, dim=0) + decode_data = [] + for vd_data in video_data_list: + tmp = autoencoder.decode(vd_data) + decode_data.append(tmp) + video_data = torch.cat(decode_data, dim=0) + video_data = rearrange(video_data, '(b f) c h w -> b c f h w', b=bs_vd) + ori_video = ori_video[:viz_num] + + oss_key = os.path.join(cfg.log_dir, 'rank.gif') + text_key = osp.join(cfg.log_dir, 'text_description.txt') + + if not os.path.exists(cfg.log_dir): + os.mkdir(cfg.log_dir) + + # Save videos and text inputs. + try: + del model_kwargs[0][list(model_kwargs[0].keys())[0]] + del model_kwargs[1][list(model_kwargs[1].keys())[0]] + + save_video_multiple_conditions( + oss_key, + video_data, + model_kwargs, + ori_video, + palette, + cfg.mean, + cfg.std, + nrow=1, + save_origin_video=cfg.save_origin_video, + duration=duration) + + texts = '\n'.join(caps[:viz_num]) + open(text_key, 'w').writelines(texts) + except Exception as e: + logger.error(f'Save text or video error. {e}') + + logger.info(f'Save videos to {oss_key}') + + +def prepare_model_kwargs(partial_keys, full_model_kwargs, use_fps_condition): + for partial_key in partial_keys: + assert partial_key in [ + 'y', 'depth', 'canny', 'masked', 'sketch', 'image', 'motion', + 'local_image', 'single_sketch' + ] + + if use_fps_condition is True: + partial_keys.append('fps') + + partial_model_kwargs = [{}, {}] + for partial_key in partial_keys: + partial_model_kwargs[0][partial_key] = full_model_kwargs[0][ + partial_key] + partial_model_kwargs[1][partial_key] = full_model_kwargs[1][ + partial_key] + + return partial_model_kwargs + + +@torch.no_grad() +def get_first_stage_encoding(encoder_posterior): + scale_factor = 0.18215 + if isinstance(encoder_posterior, DiagonalGaussianDistribution): + z = encoder_posterior.sample() + elif isinstance(encoder_posterior, torch.Tensor): + z = encoder_posterior + else: + raise NotImplementedError( + f"encoder_posterior of type '{type(encoder_posterior)}' not yet implemented" + ) + return scale_factor * z + + +def make_masked_images(imgs, masks): + masked_imgs = [] + for i, mask in enumerate(masks): + # concatenation + masked_imgs.append( + torch.cat([imgs[i] * (1 - mask), (1 - mask)], dim=1)) + return torch.stack(masked_imgs, dim=0) + + +def DOWNLOAD_TO_CACHE(oss_key, + file_or_dirname=None, + cache_dir=osp.join( + '/'.join(osp.abspath(__file__).split('/')[:-2]), + 'model_weights')): + r"""Download OSS [file or folder] to the cache folder. + Only the 0th process on each node will run the downloading. + Barrier all processes until the downloading is completed. + """ + # source and target paths + base_path = osp.join(cache_dir, file_or_dirname or osp.basename(oss_key)) + + return base_path + + +def parse_oss_url(path): + if path.startswith('oss://'): + path = path[len('oss://'):] + + # configs + configs = { + 'endpoint': os.getenv('OSS_ENDPOINT', None), + 'accessKeyID': os.getenv('OSS_ACCESS_KEY_ID', None), + 'accessKeySecret': os.getenv('OSS_ACCESS_KEY_SECRET', None), + 'securityToken': os.getenv('OSS_SECURITY_TOKEN', None) + } + bucket, path = path.split('/', maxsplit=1) + if '?' in bucket: + bucket, config = bucket.split('?', maxsplit=1) + for pair in config.split('&'): + k, v = pair.split('=', maxsplit=1) + configs[k] = v + + # session + session = parse_oss_url._sessions.setdefault(f'{bucket}@{os.getpid()}', + oss.Session()) + + # bucket + bucket = oss.Bucket( + auth=oss.Auth(configs['accessKeyID'], configs['accessKeySecret']), + endpoint=configs['endpoint'], + bucket_name=bucket, + session=session) + return bucket, path + + +parse_oss_url._sessions = {} + + +def parse_bucket(url): + return parse_oss_url(osp.join(url, '_placeholder'))[0] + + +def read(filename, mode='r', retry=5): + assert mode in ['r', 'rb'] + exception = None + for _ in range(retry): + try: + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + content = bucket.get_object(path).read() + if mode == 'r': + content = content.decode('utf-8') + elif filename.startswith('http'): + content = requests.get(filename).content + if mode == 'r': + content = content.decode('utf-8') + else: + with open(filename, mode=mode) as f: + content = f.read() + return content + except Exception as e: + exception = e + continue + else: + raise exception + + +def read_image(filename, retry=5): + exception = None + for _ in range(retry): + try: + return Image.open(BytesIO(read(filename, mode='rb', retry=retry))) + except Exception as e: + exception = e + continue + else: + raise exception + + +def download_video_to_file(filename, local_file, retry=5): + exception = None + for _ in range(retry): + try: + bucket, path = parse_oss_url(filename) + bucket.get_object_to_file(path, local_file) + break + except Exception as e: + exception = e + continue + else: + raise exception + + +def read_gzip(filename, retry=5): + exception = None + for _ in range(retry): + try: + remove = False + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + filename = rand_name(suffix=osp.splitext(filename)[1]) + bucket.get_object_to_file(path, filename) + remove = True + with gzip.open(filename) as f: + content = f.read() + if remove: + os.remove(filename) + return content + except Exception as e: + exception = e + continue + else: + raise exception + + +def ceil_divide(a, b): + return int(math.ceil(a / b)) + + +def to_device(batch, device, non_blocking=False): + if isinstance(batch, (list, tuple)): + return type(batch)([to_device(u, device, non_blocking) for u in batch]) + elif isinstance(batch, dict): + return type(batch)([(k, to_device(v, device, non_blocking)) + for k, v in batch.items()]) + elif isinstance(batch, torch.Tensor) and batch.device != device: + batch = batch.to(device, non_blocking=non_blocking) + return batch + + +def put_object(bucket, oss_key, data, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.put_object(oss_key, data) + except Exception as e: + exception = e + continue + else: + logger.info( + f'put_object to {oss_key} failed with error: {exception}', + flush=True) + + +def put_torch_object(bucket, oss_key, data, retry=5): + exception = None + for _ in range(retry): + try: + buffer = BytesIO() + torch.save(data, buffer) + return bucket.put_object(oss_key, buffer.getvalue()) + except Exception as e: + exception = e + continue + else: + logger.info( + f'put_torch_object to {oss_key} failed with error: {exception}', + flush=True) + + +def put_object_from_file(bucket, oss_key, filename, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.put_object_from_file(oss_key, filename) + except Exception as e: + exception = e + continue + else: + logger.error( + f'put_object_from_file to {oss_key} failed with error: {exception}', + flush=True) + + +def get_object(bucket, oss_key, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.get_object(oss_key).read() + except Exception as e: + exception = e + continue + else: + logger.error( + f'get_object from {oss_key} failed with error: {exception}', + flush=True) + + +def get_object_to_file(bucket, oss_key, filename, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.get_object_to_file(oss_key, filename) + except Exception as e: + exception = e + continue + else: + logger.error( + f'get_object_to_file from {oss_key} failed with error: {exception}', + flush=True) + + +@torch.no_grad() +def save_image(bucket, + oss_key, + tensor, + nrow=8, + normalize=True, + range=(-1, 1), + retry=5): + filename = rand_name(suffix='.jpg') + for _ in [None] * retry: + try: + tvutils.save_image( + tensor, filename, nrow=nrow, normalize=normalize, range=range) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + logger.error( + 'save image to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def video_tensor_to_gif(tensor, path, duration=200, loop=0, optimize=True): + tensor = tensor.permute(1, 2, 3, 0) + images = tensor.unbind(dim=0) + images = [(image.numpy() * 255).astype('uint8') for image in images] + imageio.mimwrite(path, images, duration=duration) + return images + + +@torch.no_grad() +def save_video(bucket, + oss_key, + tensor, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=tensor.device).view(1, -1, 1, 1, 1) + tensor = tensor.mul_(std).add_(mean) + tensor.clamp_(0, 1) + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + one_gif = rearrange( + tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + video_tensor_to_gif(one_gif, filename) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_multiple_conditions(oss_key, + video_tensor, + model_kwargs, + source_imgs, + palette, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5, + save_origin_video=True, + bucket=None, + duration=200): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + try: + video_tensor.clamp_(0, 1) + except Exception as e: + logger.error(e) + video_tensor = video_tensor.float().clamp_(0, 1) + video_tensor = video_tensor.cpu() + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + model_kwargs_channel3 = {} + for key, conditions in model_kwargs[0].items(): + if conditions.shape[-1] == 1024: + # Skip for style embeding + continue + if len(conditions.shape) == 3: + conditions_np = conditions.cpu().numpy() + conditions = [] + for i in conditions_np: + vis_i = [] + for j in i: + vis_i.append( + palette.get_palette_image( + j, percentile=90, width=256, height=256)) + conditions.append(np.stack(vis_i)) + conditions = torch.from_numpy(np.stack(conditions)) + conditions = rearrange(conditions, 'b n h w c -> b c n h w') + else: + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + if conditions.size(1) == 2: + conditions = torch.cat([conditions, conditions[:, :1, ]], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 3: + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 4: + color = ((conditions[:, 0:3] + 1.) / 2.) + alpha = conditions[:, 3:4] + conditions = color * alpha + 1.0 * (1.0 - alpha) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + model_kwargs_channel3[key] = conditions.cpu( + ) if conditions.is_cuda else conditions + + filename = oss_key + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + cons_list = [ + rearrange(con, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + for _, con in model_kwargs_channel3.items() + ] + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + + if save_origin_video: + vid_gif = torch.cat( + [ + source_imgs, + ] + cons_list + [ + vid_gif, + ], dim=3) + else: + vid_gif = torch.cat( + cons_list + [ + vid_gif, + ], dim=3) + + video_tensor_to_gif(vid_gif, filename, duration=duration) + exception = None + break + except Exception as e: + exception = e + continue + if exception is not None: + logging.info('save video to {} failed, error: {}'.format( + oss_key, exception)) + + +@torch.no_grad() +def save_video_multiple_conditions_with_data(bucket, + video_save_key, + gt_video_save_key, + vis_oss_key, + video_tensor, + model_kwargs, + source_imgs, + palette, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + video_tensor.clamp_(0, 1) + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + model_kwargs_channel3 = {} + for key, conditions in model_kwargs[0].items(): + if len(conditions.shape) == 3: + conditions_np = conditions.cpu().numpy() + conditions = [] + for i in conditions_np: + vis_i = [] + for j in i: + vis_i.append( + palette.get_palette_image( + j, percentile=90, width=256, height=256)) + conditions.append(np.stack(vis_i)) + conditions = torch.from_numpy(np.stack(conditions)) + conditions = rearrange(conditions, 'b n h w c -> b c n h w') + else: + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + if conditions.size(1) == 2: + conditions = torch.cat([conditions, conditions[:, :1, ]], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 3: + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 4: + color = ((conditions[:, 0:3] + 1.) / 2.) + alpha = conditions[:, 3:4] + conditions = color * alpha + 1.0 * (1.0 - alpha) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + model_kwargs_channel3[key] = conditions.cpu( + ) if conditions.is_cuda else conditions + + copy_video_tensor = video_tensor.clone() + copy_source_imgs = source_imgs.clone() + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + cons_list = [ + rearrange(con, '(i j) c f h w -> c f (i h) (j w)', j=nrow) + for _, con in model_kwargs_channel3.items() + ] + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + vid_gif = torch.cat( + [ + source_imgs, + ] + cons_list + [ + vid_gif, + ], dim=3) + + video_tensor_to_gif(vid_gif, filename) + bucket.put_object_from_file(vis_oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + + filename_pred = rand_name(suffix='.pkl') + for _ in [None] * retry: + try: + copy_video_np = (copy_video_tensor.numpy() * 255).astype('uint8') + pickle.dump(copy_video_np, open(filename_pred, 'wb')) + bucket.put_object_from_file(video_save_key, filename_pred) + break + except Exception as e: + logger.error('error! ', video_save_key, e) + continue + + # remove temporary file + if osp.exists(filename_pred): + os.remove(filename_pred) + + filename_gt = rand_name(suffix='.pkl') + for _ in [None] * retry: + try: + copy_source_np = (copy_source_imgs.numpy() * 255).astype('uint8') + pickle.dump(copy_source_np, open(filename_gt, 'wb')) + bucket.put_object_from_file(gt_video_save_key, filename_gt) + break + except Exception as e: + logger.error('error! ', gt_video_save_key, e) + continue + + # remove temporary file + if osp.exists(filename_gt): + os.remove(filename_gt) + + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format( + vis_oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_vs_conditions(bucket, + oss_key, + video_tensor, + conditions, + source_imgs, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + video_tensor.clamp_(0, 1) + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + con_gif = rearrange( + conditions, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + vid_gif = torch.cat([vid_gif, con_gif, source_imgs], dim=2) + + video_tensor_to_gif(vid_gif, filename) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + logger.error(exception) + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_grid_mp4(bucket, + oss_key, + tensor, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=None, + fps=5, + retry=5): + mean = torch.tensor(mean, device=tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=tensor.device).view(1, -1, 1, 1, 1) + tensor = tensor.mul_(std).add_(mean) + tensor.clamp_(0, 1) + b, c, t, h, w = tensor.shape + tensor = tensor.permute(0, 2, 3, 4, 1) + tensor = (tensor.cpu().numpy() * 255).astype('uint8') + + filename = rand_name(suffix='.mp4') + for _ in [None] * retry: + try: + if nrow is None: + nrow = math.ceil(math.sqrt(b)) + ncol = math.ceil(b / nrow) + padding = 1 + video_grid = np.zeros((t, (padding + h) * nrow + padding, + (padding + w) * ncol + padding, c), + dtype='uint8') + for i in range(b): + r = i // ncol + c_ = i % ncol + + start_r = (padding + h) * r + start_c = (padding + w) * c_ + video_grid[:, start_r:start_r + h, + start_c:start_c + w] = tensor[i] + skvideo.io.vwrite(filename, video_grid, inputdict={'-r': str(fps)}) + + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + logger.error(exception) + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_text(bucket, oss_key, tensor, nrow=8, retry=5): + len = tensor.shape[0] + num_per_row = int(len / nrow) + assert (len == nrow * num_per_row) + texts = '' + for i in range(nrow): + for j in range(num_per_row): + text = dec_bytes2obj(tensor[i * num_per_row + j]) + texts += text + '\n' + texts += '\n' + + for _ in [None] * retry: + try: + bucket.put_object(oss_key, texts) + exception = None + break + except Exception as e: + exception = e + logger.error(exception) + continue + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_caps(bucket, oss_key, caps, retry=5): + texts = '' + for cap in caps: + texts += cap + texts += '\n' + + for _ in [None] * retry: + try: + bucket.put_object(oss_key, texts) + exception = None + break + except Exception as e: + exception = e + logger.error(exception) + continue + if exception is not None: + logger.error( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def ema(net_ema, net, beta, copy_buffer=False): + assert 0.0 <= beta <= 1.0 + for p_ema, p in zip(net_ema.parameters(), net.parameters()): + p_ema.copy_(p.lerp(p_ema, beta)) + if copy_buffer: + for b_ema, b in zip(net_ema.buffers(), net.buffers()): + b_ema.copy_(b) + + +def parallel(func, args_list, num_workers=32, timeout=None): + assert isinstance(args_list, list) + if not isinstance(args_list[0], tuple): + args_list = [(args, ) for args in args_list] + if num_workers == 0: + return [func(*args) for args in args_list] + with Pool(processes=num_workers) as pool: + results = [pool.apply_async(func, args) for args in args_list] + results = [res.get(timeout=timeout) for res in results] + return results + + +def exists(filename): + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + return bucket.object_exists(path) + else: + return osp.exists(filename) + + +def download(url, filename=None, replace=False, quiet=False): + if filename is None: + filename = osp.basename(url) + if not osp.exists(filename) or replace: + try: + if url.startswith('oss://'): + bucket, oss_key = parse_oss_url(url) + bucket.get_object_to_file(oss_key, filename) + else: + urllib.request.urlretrieve(url, filename) + if not quiet: + logger.error(f'Downloaded {url} to {filename}', flush=True) + except Exception as e: + raise ValueError(f'Downloading {filename} failed with error {e}') + return osp.abspath(filename) + + +def unzip(filename, dst_dir=None): + if dst_dir is None: + dst_dir = osp.dirname(filename) + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(dst_dir) + + +def load_state_dict(module, state_dict, drop_prefix=''): + # find incompatible key-vals + src, dst = state_dict, module.state_dict() + if drop_prefix: + src = type(src)([ + (k[len(drop_prefix):] if k.startswith(drop_prefix) else k, v) + for k, v in src.items() + ]) + missing = [k for k in dst if k not in src] + unexpected = [k for k in src if k not in dst] + unmatched = [ + k for k in src.keys() & dst.keys() if src[k].shape != dst[k].shape + ] + + # keep only compatible key-vals + incompatible = set(unexpected + unmatched) + src = type(src)([(k, v) for k, v in src.items() if k not in incompatible]) + module.load_state_dict(src, strict=False) + + # report incompatible key-vals + if len(missing) != 0: + logger.info(' Missing: ' + ', '.join(missing), flush=True) + if len(unexpected) != 0: + logger.info(' Unexpected: ' + ', '.join(unexpected), flush=True) + if len(unmatched) != 0: + logger.info(' Shape unmatched: ' + ', '.join(unmatched), flush=True) + + +def inverse_indices(indices): + r"""Inverse map of indices. + E.g., if A[indices] == B, then B[inv_indices] == A. + """ + inv_indices = torch.empty_like(indices) + inv_indices[indices] = torch.arange(len(indices)).to(indices) + return inv_indices + + +def detect_duplicates(feats, thr=0.9): + assert feats.ndim == 2 + + # compute simmat + feats = F.normalize(feats, p=2, dim=1) + simmat = torch.mm(feats, feats.T) + simmat.triu_(1) + torch.cuda.synchronize() + + # detect duplicates + mask = ~simmat.gt(thr).any(dim=0) + return torch.where(mask)[0] + + +class TFSClient(object): + + def __init__(self, + host='restful-store.vip.tbsite.net:3800', + app_key='5354c9fae75f5'): + self.host = host + self.app_key = app_key + + # candidate servers + self.servers = [ + u for u in read(f'http://{host}/url.list').strip().split('\n')[1:] + if ':' in u + ] + assert len(self.servers) >= 1 + self.__server_id = -1 + + @property + def server(self): + self.__server_id = (self.__server_id + 1) % len(self.servers) + return self.servers[self.__server_id] + + def read(self, tfs): + tfs = osp.basename(tfs) + meta = json.loads( + read( + f'http://{self.server}/v1/{self.app_key}/metadata/{tfs}?force=0' + )) + img = Image.open( + BytesIO( + read( + f'http://{self.server}/v1/{self.app_key}/{tfs}?offset=0&size={meta["SIZE"]}', + 'rb'))) + return img + + +def read_tfs(tfs, retry=5): + exception = None + for _ in range(retry): + try: + global TFS_CLIENT + if TFS_CLIENT is None: + TFS_CLIENT = TFSClient() + return TFS_CLIENT.read(tfs) + except Exception as e: + exception = e + continue + else: + raise exception + + +def md5(filename): + with open(filename, 'rb') as f: + return hashlib.md5(f.read()).hexdigest() + + +def rope(x): + r"""Apply rotary position embedding on x of shape [B, *(spatial dimensions), C]. + """ + # reshape + shape = x.shape + x = x.view(x.size(0), -1, x.size(-1)) + l, c = x.shape[-2:] + assert c % 2 == 0 + half = c // 2 + + # apply rotary position embedding on x + sinusoid = torch.outer( + torch.arange(l).to(x), + torch.pow(10000, -torch.arange(half).to(x).div(half))) + sin, cos = torch.sin(sinusoid), torch.cos(sinusoid) + x1, x2 = x.chunk(2, dim=-1) + x = torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1) + + # reshape back + return x.view(shape) + + +def format_state(state, filename=None): + r"""For comparing/aligning state_dict. + """ + content = '\n'.join([f'{k}\t{tuple(v.shape)}' for k, v in state.items()]) + if filename: + with open(filename, 'w') as f: + f.write(content) + + +def breakup_grid(img, grid_size): + r"""The inverse operator of ``torchvision.utils.make_grid``. + """ + # params + nrow = img.height // grid_size + ncol = img.width // grid_size + wrow = wcol = 2 + + # collect grids + grids = [] + for i in range(nrow): + for j in range(ncol): + x1 = j * grid_size + (j + 1) * wcol + y1 = i * grid_size + (i + 1) * wrow + grids.append(img.crop((x1, y1, x1 + grid_size, y1 + grid_size))) + return grids + + +def huggingface_tokenizer(name='google/mt5-xxl', **kwargs): + from transformers import AutoTokenizer + return AutoTokenizer.from_pretrained( + DOWNLOAD_TO_CACHE(f'huggingface/tokenizers/{name}', name), **kwargs) + + +def huggingface_model(name='google/mt5-xxl', model_type='AutoModel', **kwargs): + import transformers + return getattr(transformers, model_type).from_pretrained( + DOWNLOAD_TO_CACHE(f'huggingface/models/{name}', name), **kwargs) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/unet_sd.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/unet_sd.py new file mode 100644 index 0000000000000000000000000000000000000000..23ee29f6f09028725665243dff4ca21cb47844bb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/unet_sd.py @@ -0,0 +1,2102 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os +from functools import partial + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from fairscale.nn.checkpoint import checkpoint_wrapper +from rotary_embedding_torch import RotaryEmbedding +from torch import einsum + +__all__ = ['UNetSD_temporal'] + +USE_TEMPORAL_TRANSFORMER = True + + +# load all keys started with prefix and replace them with new_prefix +def load_Block(state, prefix, new_prefix=None): + if new_prefix is None: + new_prefix = prefix + + state_dict = {} + + state = {key: value for key, value in state.items() if prefix in key} + + for key, value in state.items(): + new_key = key.replace(prefix, new_prefix) + state_dict[new_key] = value + + return state_dict + + +def load_2d_pretrained_state_dict(state, cfg): + + new_state_dict = {} + + dim = cfg.unet_dim + num_res_blocks = cfg.unet_res_blocks + dim_mult = cfg.unet_dim_mult + attn_scales = cfg.unet_attn_scales + + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + + # embeddings + state_dict = load_Block(state, prefix='time_embedding') + new_state_dict.update(state_dict) + state_dict = load_Block(state, prefix='y_embedding') + new_state_dict.update(state_dict) + state_dict = load_Block(state, prefix='context_embedding') + new_state_dict.update(state_dict) + + encoder_idx = 0 + # init block + state_dict = load_Block( + state, + prefix=f'encoder.{encoder_idx}', + new_prefix=f'encoder.{encoder_idx}.0') + new_state_dict.update(state_dict) + encoder_idx += 1 + + shortcut_dims.append(dim) + for i, (in_dim, out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + idx = 0 + idx_ = 0 + # residual (+attention) blocks + state_dict = load_Block( + state, + prefix=f'encoder.{encoder_idx}.{idx}', + new_prefix=f'encoder.{encoder_idx}.{idx_}') + new_state_dict.update(state_dict) + idx += 1 + idx_ = 2 + + if scale in attn_scales: + # block.append(AttentionBlock(out_dim, context_dim, num_heads, head_dim)) + state_dict = load_Block( + state, + prefix=f'encoder.{encoder_idx}.{idx}', + new_prefix=f'encoder.{encoder_idx}.{idx_}') + new_state_dict.update(state_dict) + in_dim = out_dim + encoder_idx += 1 + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + state_dict = load_Block( + state, + prefix='encoder.{encoder_idx}', + new_prefix='encoder.{encoder_idx}.0') + new_state_dict.update(state_dict) + + shortcut_dims.append(out_dim) + scale /= 2.0 + encoder_idx += 1 + + # middle + middle_idx = 0 + + state_dict = load_Block(state, prefix=f'middle.{middle_idx}') + new_state_dict.update(state_dict) + middle_idx += 2 + + state_dict = load_Block( + state, prefix='middle.1', new_prefix=f'middle.{middle_idx}') + new_state_dict.update(state_dict) + middle_idx += 1 + + for _ in range(cfg.temporal_attn_times): + middle_idx += 1 + + state_dict = load_Block( + state, prefix='middle.2', new_prefix=f'middle.{middle_idx}') + new_state_dict.update(state_dict) + middle_idx += 2 + + decoder_idx = 0 + for i, (in_dim, out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + idx = 0 + idx_ = 0 + # residual (+attention) blocks + state_dict = load_Block( + state, + prefix=f'decoder.{decoder_idx}.{idx}', + new_prefix=f'decoder.{decoder_idx}.{idx_}') + new_state_dict.update(state_dict) + idx += 1 + idx_ += 2 + if scale in attn_scales: + state_dict = load_Block( + state, + prefix=f'decoder.{decoder_idx}.{idx}', + new_prefix=f'decoder.{decoder_idx}.{idx_}') + new_state_dict.update(state_dict) + idx += 1 + idx_ += 1 + for _ in range(cfg.temporal_attn_times): + idx_ += 1 + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + state_dict = load_Block( + state, + prefix=f'decoder.{decoder_idx}.{idx}', + new_prefix=f'decoder.{decoder_idx}.{idx_}') + new_state_dict.update(state_dict) + idx += 1 + idx_ += 2 + + scale *= 2.0 + decoder_idx += 1 + + state_dict = load_Block(state, prefix='head') + new_state_dict.update(state_dict) + + return new_state_dict + + +def sinusoidal_embedding(timesteps, dim): + # check input + half = dim // 2 + timesteps = timesteps.float() + + # compute sinusoidal embedding + sinusoid = torch.outer( + timesteps, torch.pow(10000, + -torch.arange(half).to(timesteps).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + if dim % 2 != 0: + x = torch.cat([x, torch.zeros_like(x[:, :1])], dim=1) + return x + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if callable(d) else d + + +def prob_mask_like(shape, prob, device): + if prob == 1: + return torch.ones(shape, device=device, dtype=torch.bool) + elif prob == 0: + return torch.zeros(shape, device=device, dtype=torch.bool) + else: + mask = torch.zeros(shape, device=device).float().uniform_(0, 1) < prob + # aviod mask all, which will cause find_unused_parameters error + if mask.all(): + mask[0] = False + return mask + + +class RelativePositionBias(nn.Module): + + def __init__(self, heads=8, num_buckets=32, max_distance=128): + super().__init__() + self.num_buckets = num_buckets + self.max_distance = max_distance + self.relative_attention_bias = nn.Embedding(num_buckets, heads) + + @staticmethod + def _relative_position_bucket(relative_position, + num_buckets=32, + max_distance=128): + ret = 0 + n = -relative_position + + num_buckets //= 2 + ret += (n < 0).long() * num_buckets + n = torch.abs(n) + + max_exact = num_buckets // 2 + is_small = n < max_exact + + val_if_large = max_exact + ( + torch.log(n.float() / max_exact) + / math.log(max_distance / max_exact) * # noqa + (num_buckets - max_exact)).long() + val_if_large = torch.min( + val_if_large, torch.full_like(val_if_large, num_buckets - 1)) + + ret += torch.where(is_small, n, val_if_large) + return ret + + def forward(self, n, device): + q_pos = torch.arange(n, dtype=torch.long, device=device) + k_pos = torch.arange(n, dtype=torch.long, device=device) + rel_pos = rearrange(k_pos, 'j -> 1 j') - rearrange(q_pos, 'i -> i 1') + rp_bucket = self._relative_position_bucket( + rel_pos, + num_buckets=self.num_buckets, + max_distance=self.max_distance) + values = self.relative_attention_bias(rp_bucket) + return rearrange(values, 'i j h -> h i j') + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + NEW: use_linear for more efficiency instead of the 1x1 convs + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True): + super().__init__() + if exists(context_dim) and not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + disable_self_attn=disable_self_attn, + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv2d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if not isinstance(context, list): + context = [context] + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + if not self.use_linear: + x = self.proj_in(x) + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + if self.use_linear: + x = self.proj_in(x) + for i, block in enumerate(self.transformer_blocks): + x = block(x, context=context[i]) + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = self.proj_out(x) + return x + x_in + + +_ATTN_PRECISION = os.environ.get('ATTN_PRECISION', 'fp32') + + +class CrossAttention(nn.Module): + + def __init__(self, + query_dim, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), + (q, k, v)) + + # force cast to fp32 to avoid overflowing + if _ATTN_PRECISION == 'fp32': + with torch.autocast(enabled=False, device_type='cuda'): + q, k = q.float(), k.float() + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + else: + sim = torch.einsum('b i d, b j d -> b i j', q, k) * self.scale + + del q, k + + if exists(mask): + mask = rearrange(mask, 'b ... -> b (...)') + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, 'b j -> (b h) () j', h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + sim = sim.softmax(dim=-1) + + out = torch.einsum('b i j, b j d -> b i d', sim, v) + out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + return self.to_out(out) + + +class BasicTransformerBlock(nn.Module): + + def __init__(self, + dim, + n_heads, + d_head, + dropout=0., + context_dim=None, + gated_ff=True, + checkpoint=True, + disable_self_attn=False): + super().__init__() + attn_cls = CrossAttention + self.disable_self_attn = disable_self_attn + self.attn1 = attn_cls( + query_dim=dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + context_dim=context_dim if self.disable_self_attn else None) + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = attn_cls( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout) + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward_(self, x, context=None): + return checkpoint(self._forward, (x, context), self.parameters(), + self.checkpoint) + + def forward(self, x, context=None): + x = self.attn1( + self.norm1(x), + context=context if self.disable_self_attn else None) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +# feedforward +class GEGLU(nn.Module): + + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +class FeedForward(nn.Module): + + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = nn.Sequential(nn.Linear( + dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim) + + self.net = nn.Sequential(project_in, nn.Dropout(dropout), + nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.net(x) + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = nn.Conv2d( + self.channels, self.out_channels, 3, padding=padding) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), + mode='nearest') + else: + x = F.interpolate(x, scale_factor=2, mode='nearest') + if self.use_conv: + x = self.conv(x) + return x + + +class ResBlock(nn.Module): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + up=False, + down=False, + use_temporal_conv=True, + use_image_dataset=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_scale_shift_norm = use_scale_shift_norm + self.use_temporal_conv = use_temporal_conv + + self.in_layers = nn.Sequential( + nn.GroupNorm(32, channels), + nn.SiLU(), + nn.Conv2d(channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + nn.Linear( + emb_channels, + 2 * self.out_channels + if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + nn.GroupNorm(32, self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + nn.Conv2d(self.out_channels, self.out_channels, 3, padding=1)), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1) + else: + self.skip_connection = nn.Conv2d(channels, self.out_channels, 1) + + if self.use_temporal_conv: + self.temopral_conv = TemporalConvBlock_v2( + self.out_channels, + self.out_channels, + dropout=0.1, + use_image_dataset=use_image_dataset) + + def forward(self, x, emb, batch_size): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return self._forward(x, emb, batch_size) + + def _forward(self, x, emb, batch_size): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + h = self.skip_connection(x) + h + + if self.use_temporal_conv: + h = rearrange(h, '(b f) c h w -> b c f h w', b=batch_size) + h = self.temopral_conv(h) + h = rearrange(h, 'b c f h w -> (b f) c h w') + return h + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, + channels, + use_conv, + dims=2, + out_channels=None, + padding=1): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = nn.Conv2d( + self.channels, + self.out_channels, + 3, + stride=stride, + padding=padding) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class Resample(nn.Module): + + def __init__(self, in_dim, out_dim, mode): + assert mode in ['none', 'upsample', 'downsample'] + super(Resample, self).__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.mode = mode + + def forward(self, x, reference=None): + if self.mode == 'upsample': + assert reference is not None + x = F.interpolate(x, size=reference.shape[-2:], mode='nearest') + elif self.mode == 'downsample': + x = F.adaptive_avg_pool2d( + x, output_size=tuple(u // 2 for u in x.shape[-2:])) + return x + + +class ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + embed_dim, + out_dim, + use_scale_shift_norm=True, + mode='none', + dropout=0.0): + super(ResidualBlock, self).__init__() + self.in_dim = in_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.use_scale_shift_norm = use_scale_shift_norm + self.mode = mode + + # layers + self.layer1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv2d(in_dim, out_dim, 3, padding=1)) + self.resample = Resample(in_dim, in_dim, mode) + self.embedding = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, + out_dim * 2 if use_scale_shift_norm else out_dim)) + self.layer2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv2d(out_dim, out_dim, 3, padding=1)) + self.shortcut = nn.Identity() if in_dim == out_dim else nn.Conv2d( + in_dim, out_dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.layer2[-1].weight) + + def forward(self, x, e, reference=None): + identity = self.resample(x, reference) + x = self.layer1[-1](self.resample(self.layer1[:-1](x), reference)) + e = self.embedding(e).unsqueeze(-1).unsqueeze(-1).type(x.dtype) + if self.use_scale_shift_norm: + scale, shift = e.chunk(2, dim=1) + x = self.layer2[0](x) * (1 + scale) + shift + x = self.layer2[1:](x) + else: + x = x + e + x = self.layer2(x) + x = x + self.shortcut(identity) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, context_dim=None, num_heads=None, head_dim=None): + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + super(AttentionBlock, self).__init__() + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + + # layers + self.norm = nn.GroupNorm(32, dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + if context_dim is not None: + self.context_kv = nn.Linear(context_dim, dim * 2) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x, context=None): + r"""x: [B, C, H, W]. + context: [B, L, C] or None. + """ + identity = x + b, c, h, w, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + x = self.norm(x) + q, k, v = self.to_qkv(x).view(b, n * 3, d, h * w).chunk(3, dim=1) + if context is not None: + ck, cv = self.context_kv(context).reshape(b, -1, n * 2, + d).permute(0, 2, 3, + 1).chunk( + 2, dim=1) + k = torch.cat([ck, k], dim=-1) + v = torch.cat([cv, v], dim=-1) + + # compute attention + attn = torch.matmul(q.transpose(-1, -2) * self.scale, k * self.scale) + attn = F.softmax(attn, dim=-1) + + # gather context + x = torch.matmul(v, attn.transpose(-1, -2)) + x = x.reshape(b, c, h, w) + + # output + x = self.proj(x) + return x + identity + + +class TemporalAttentionBlock(nn.Module): + + def __init__(self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False): + super().__init__() + # consider num_heads first, as pos_bias needs fixed num_heads + dim_head = dim // heads + assert heads * dim_head == dim + self.use_image_dataset = use_image_dataset + self.use_sim_mask = use_sim_mask + + self.scale = dim_head**-0.5 + self.heads = heads + hidden_dim = dim_head * heads + + self.norm = nn.GroupNorm(32, dim) + self.rotary_emb = rotary_emb + self.to_qkv = nn.Linear(dim, hidden_dim * 3) + self.to_out = nn.Linear(hidden_dim, dim) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + + identity = x + n, height, device = x.shape[2], x.shape[-2], x.device + + x = self.norm(x) + x = rearrange(x, 'b c f h w -> b (h w) f c') + + qkv = self.to_qkv(x).chunk(3, dim=-1) + + if exists(focus_present_mask) and focus_present_mask.all(): + # if all batch samples are focusing on present + # it would be equivalent to passing that token's values (v=qkv[-1]) through to the output + values = qkv[-1] + out = self.to_out(values) + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + return out + identity + + # split out heads + q = rearrange(qkv[0], '... n (h d) -> ... h n d', h=self.heads) + k = rearrange(qkv[1], '... n (h d) -> ... h n d', h=self.heads) + v = rearrange(qkv[2], '... n (h d) -> ... h n d', h=self.heads) + + # scale + q = q * self.scale + + # rotate positions into queries and keys for time attention + if exists(self.rotary_emb): + q = self.rotary_emb.rotate_queries_or_keys(q) + k = self.rotary_emb.rotate_queries_or_keys(k) + + # similarity + sim = torch.einsum('... h i d, ... h j d -> ... h i j', q, k) + + # relative positional bias + + if exists(pos_bias): + sim = sim + pos_bias + + if (focus_present_mask is None and video_mask is not None): + mask = video_mask[:, None, :] * video_mask[:, :, None] + mask = mask.unsqueeze(1).unsqueeze(1) + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + elif exists(focus_present_mask) and not (~focus_present_mask).all(): + attend_all_mask = torch.ones((n, n), + device=device, + dtype=torch.bool) + attend_self_mask = torch.eye(n, device=device, dtype=torch.bool) + + mask = torch.where( + rearrange(focus_present_mask, 'b -> b 1 1 1 1'), + rearrange(attend_self_mask, 'i j -> 1 1 1 i j'), + rearrange(attend_all_mask, 'i j -> 1 1 1 i j'), + ) + + sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max) + + if self.use_sim_mask: + sim_mask = torch.tril( + torch.ones((n, n), device=device, dtype=torch.bool), + diagonal=0) + sim = sim.masked_fill(~sim_mask, -torch.finfo(sim.dtype).max) + + # numerical stability + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + # aggregate values + out = torch.einsum('... h i j, ... h j d -> ... h i d', attn, v) + out = rearrange(out, '... h n d -> ... n (h d)') + out = self.to_out(out) + + out = rearrange(out, 'b (h w) f c -> b c f h w', h=height) + + if self.use_image_dataset: + out = identity + 0 * out + else: + out = identity + out + return out + + +class TemporalTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__(self, + in_channels, + n_heads, + d_head, + depth=1, + dropout=0., + context_dim=None, + disable_self_attn=False, + use_linear=False, + use_checkpoint=True, + only_self_att=True, + multiply_zero=False): + super().__init__() + self.multiply_zero = multiply_zero + self.only_self_att = only_self_att + self.use_adaptor = False + if self.only_self_att: + context_dim = None + if not isinstance(context_dim, list): + context_dim = [context_dim] + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + if not use_linear: + self.proj_in = nn.Conv1d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + else: + self.proj_in = nn.Linear(in_channels, inner_dim) + if self.use_adaptor: + self.adaptor_in = nn.Linear(frames, frames) + + self.transformer_blocks = nn.ModuleList([ + BasicTransformerBlock( + inner_dim, + n_heads, + d_head, + dropout=dropout, + context_dim=context_dim[d], + checkpoint=use_checkpoint) for d in range(depth) + ]) + if not use_linear: + self.proj_out = zero_module( + nn.Conv1d( + inner_dim, in_channels, kernel_size=1, stride=1, + padding=0)) + else: + self.proj_out = zero_module(nn.Linear(in_channels, inner_dim)) + if self.use_adaptor: + self.adaptor_out = nn.Linear(frames, frames) + self.use_linear = use_linear + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + if self.only_self_att: + context = None + if not isinstance(context, list): + context = [context] + b, c, f, h, w = x.shape + x_in = x + x = self.norm(x) + + if not self.use_linear: + x = rearrange(x, 'b c f h w -> (b h w) c f').contiguous() + x = self.proj_in(x) + if self.use_linear: + x = rearrange( + x, '(b f) c h w -> b (h w) f c', f=self.frames).contiguous() + x = self.proj_in(x) + + if self.only_self_att: + x = rearrange(x, 'bhw c f -> bhw f c').contiguous() + for i, block in enumerate(self.transformer_blocks): + x = block(x) + x = rearrange(x, '(b hw) f c -> b hw f c', b=b).contiguous() + else: + x = rearrange(x, '(b hw) c f -> b hw f c', b=b).contiguous() + for i, block in enumerate(self.transformer_blocks): + context[i] = rearrange( + context[i], '(b f) l con -> b f l con', + f=self.frames).contiguous() + # calculate each batch one by one + # (since number in shape could not greater then 65,535 for some package) + for j in range(b): + context_i_j = repeat( + context[i][j], + 'f l con -> (f r) l con', + r=(h * w) // self.frames, + f=self.frames).contiguous() + x[j] = block(x[j], context=context_i_j) + + if self.use_linear: + x = self.proj_out(x) + x = rearrange(x, 'b (h w) f c -> b f c h w', h=h, w=w).contiguous() + if not self.use_linear: + x = rearrange(x, 'b hw f c -> (b hw) c f').contiguous() + x = self.proj_out(x) + x = rearrange( + x, '(b h w) c f -> b c f h w', b=b, h=h, w=w).contiguous() + + if self.multiply_zero: + x = 0.0 * x + x_in + else: + x = x + x_in + return x + + +class TemporalAttentionMultiBlock(nn.Module): + + def __init__( + self, + dim, + heads=4, + dim_head=32, + rotary_emb=None, + use_image_dataset=False, + use_sim_mask=False, + temporal_attn_times=1, + ): + super().__init__() + self.att_layers = nn.ModuleList([ + TemporalAttentionBlock(dim, heads, dim_head, rotary_emb, + use_image_dataset, use_sim_mask) + for _ in range(temporal_attn_times) + ]) + + def forward(self, + x, + pos_bias=None, + focus_present_mask=None, + video_mask=None): + for layer in self.att_layers: + x = layer(x, pos_bias, focus_present_mask, video_mask) + return x + + +class InitTemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(InitTemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv[-1].weight) + nn.init.zeros_(self.conv[-1].bias) + + def forward(self, x): + identity = x + x = self.conv(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv2[-1].weight) + nn.init.zeros_(self.conv2[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + if self.use_image_dataset: + x = identity + 0 * x + else: + x = identity + x + return x + + +class TemporalConvBlock_v2(nn.Module): + + def __init__(self, + in_dim, + out_dim=None, + dropout=0.0, + use_image_dataset=False): + super(TemporalConvBlock_v2, self).__init__() + if out_dim is None: + out_dim = in_dim + self.in_dim = in_dim + self.out_dim = out_dim + self.use_image_dataset = use_image_dataset + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(32, in_dim), nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv2 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv3 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + self.conv4 = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0))) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv4[-1].weight) + nn.init.zeros_(self.conv4[-1].bias) + + def forward(self, x): + identity = x + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + + if self.use_image_dataset: + x = identity + 0.0 * x + else: + x = identity + x + return x + + +class UNetSD_temporal(nn.Module): + + def __init__( + self, + cfg, + in_dim=7, + dim=512, + y_dim=512, + context_dim=512, + hist_dim=156, + concat_dim=8, + out_dim=6, + dim_mult=[1, 2, 3, 4], + num_heads=None, + head_dim=64, + num_res_blocks=3, + attn_scales=[1 / 2, 1 / 4, 1 / 8], + use_scale_shift_norm=True, + dropout=0.1, + temporal_attn_times=1, + temporal_attention=True, + use_checkpoint=False, + use_image_dataset=False, + use_fps_condition=False, + use_sim_mask=False, + misc_dropout=0.5, + training=True, + inpainting=True, + video_compositions=['text', 'mask'], + p_all_zero=0.1, + p_all_keep=0.1, + zero_y=None, + black_image_feature=None, + ): + embed_dim = dim * 4 + num_heads = num_heads if num_heads else dim // 32 + super(UNetSD_temporal, self).__init__() + self.zero_y = zero_y + self.black_image_feature = black_image_feature + self.cfg = cfg + self.in_dim = in_dim + self.dim = dim + self.y_dim = y_dim + self.context_dim = context_dim + self.hist_dim = hist_dim + self.concat_dim = concat_dim + self.embed_dim = embed_dim + self.out_dim = out_dim + self.dim_mult = dim_mult + # for temporal attention + self.num_heads = num_heads + # for spatial attention + self.head_dim = head_dim + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.use_scale_shift_norm = use_scale_shift_norm + self.temporal_attn_times = temporal_attn_times + self.temporal_attention = temporal_attention + self.use_checkpoint = use_checkpoint + self.use_image_dataset = use_image_dataset + self.use_fps_condition = use_fps_condition + self.use_sim_mask = use_sim_mask + self.training = training + self.inpainting = inpainting + self.video_compositions = video_compositions + self.misc_dropout = misc_dropout + self.p_all_zero = p_all_zero + self.p_all_keep = p_all_keep + + use_linear_in_temporal = False + transformer_depth = 1 + disabled_sa = False + # params + enc_dims = [dim * u for u in [1] + dim_mult] + dec_dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + shortcut_dims = [] + scale = 1.0 + if hasattr(cfg, 'adapter_transformer_layers' + ) and cfg.adapter_transformer_layers: + adapter_transformer_layers = cfg.adapter_transformer_layers + else: + adapter_transformer_layers = 1 + + # embeddings + self.time_embed = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + self.pre_image_condition = nn.Sequential( + nn.Linear(1024, 1024), nn.SiLU(), nn.Linear(1024, 1024)) + + # depth embedding + if 'depthmap' in self.video_compositions: + self.depth_embedding = nn.Sequential( + nn.Conv2d(1, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.depth_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + if 'motion' in self.video_compositions: + self.motion_embedding = nn.Sequential( + nn.Conv2d(2, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.motion_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + # canny embedding + if 'canny' in self.video_compositions: + self.canny_embedding = nn.Sequential( + nn.Conv2d(1, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.canny_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + # masked-image embedding + if 'mask' in self.video_compositions: + self.masked_embedding = nn.Sequential( + nn.Conv2d(4, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim + * 4, 3, stride=2, padding=1), nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, + padding=1)) if inpainting else None + self.mask_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + # sketch embedding + if 'sketch' in self.video_compositions: + self.sketch_embedding = nn.Sequential( + nn.Conv2d(1, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.sketch_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + if 'single_sketch' in self.video_compositions: + self.single_sketch_embedding = nn.Sequential( + nn.Conv2d(1, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.single_sketch_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + if 'local_image' in self.video_compositions: + self.local_image_embedding = nn.Sequential( + nn.Conv2d(3, concat_dim * 4, 3, padding=1), nn.SiLU(), + nn.AdaptiveAvgPool2d((128, 128)), + nn.Conv2d( + concat_dim * 4, concat_dim * 4, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(concat_dim * 4, concat_dim, 3, stride=2, padding=1)) + self.local_image_embedding_after = Transformer_v2( + heads=2, + dim=concat_dim, + dim_head_k=concat_dim, + dim_head_v=concat_dim, + dropout_atte=0.05, + mlp_dim=concat_dim, + dropout_ffn=0.05, + depth=adapter_transformer_layers) + + # Condition Dropout + self.misc_dropout = DropPath(misc_dropout) + + if temporal_attention and not USE_TEMPORAL_TRANSFORMER: + self.rotary_emb = RotaryEmbedding(min(32, head_dim)) + self.time_rel_pos_bias = RelativePositionBias( + heads=num_heads, max_distance=32) + + if self.use_fps_condition: + self.fps_embedding = nn.Sequential( + nn.Linear(dim, embed_dim), nn.SiLU(), + nn.Linear(embed_dim, embed_dim)) + nn.init.zeros_(self.fps_embedding[-1].weight) + nn.init.zeros_(self.fps_embedding[-1].bias) + + # encoder + self.input_blocks = nn.ModuleList() + if cfg.resume: + self.pre_image = nn.Sequential() + init_block = nn.ModuleList( + [nn.Conv2d(self.in_dim + concat_dim, dim, 3, padding=1)]) + else: + self.pre_image = nn.Sequential( + nn.Conv2d(self.in_dim + concat_dim, self.in_dim, 1, padding=0)) + init_block = nn.ModuleList( + [nn.Conv2d(self.in_dim, dim, 3, padding=1)]) + + # need an initial temporal attention? + if temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + init_block.append( + TemporalTransformer( + dim, + num_heads, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + init_block.append( + TemporalAttentionMultiBlock( + dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + temporal_attn_times=temporal_attn_times, + use_image_dataset=use_image_dataset)) + + self.input_blocks.append(init_block) + shortcut_dims.append(dim) + for i, (in_dim, + out_dim) in enumerate(zip(enc_dims[:-1], enc_dims[1:])): + for j in range(num_res_blocks): + # residual (+attention) blocks + block = nn.ModuleList([ + ResBlock( + in_dim, + embed_dim, + dropout, + out_channels=out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + # + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + self.input_blocks.append(block) + shortcut_dims.append(out_dim) + + # downsample + if i != len(dim_mult) - 1 and j == num_res_blocks - 1: + downsample = Downsample( + out_dim, True, dims=2, out_channels=out_dim) + shortcut_dims.append(out_dim) + scale /= 2.0 + self.input_blocks.append(downsample) + + # middle + self.middle_block = nn.ModuleList([ + ResBlock( + out_dim, + embed_dim, + dropout, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ), + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=self.context_dim, + disable_self_attn=False, + use_linear=True) + ]) + + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + self.middle_block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset, + )) + else: + self.middle_block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + + self.middle_block.append( + ResBlock(out_dim, embed_dim, dropout, use_scale_shift_norm=False)) + + # decoder + self.output_blocks = nn.ModuleList() + for i, (in_dim, + out_dim) in enumerate(zip(dec_dims[:-1], dec_dims[1:])): + for j in range(num_res_blocks + 1): + block = nn.ModuleList([ + ResBlock( + in_dim + shortcut_dims.pop(), + embed_dim, + dropout, + out_dim, + use_scale_shift_norm=False, + use_image_dataset=use_image_dataset, + ) + ]) + if scale in attn_scales: + block.append( + SpatialTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=1, + context_dim=1024, + disable_self_attn=False, + use_linear=True)) + if self.temporal_attention: + if USE_TEMPORAL_TRANSFORMER: + block.append( + TemporalTransformer( + out_dim, + out_dim // head_dim, + head_dim, + depth=transformer_depth, + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_temporal, + multiply_zero=use_image_dataset)) + else: + block.append( + TemporalAttentionMultiBlock( + out_dim, + num_heads, + head_dim, + rotary_emb=self.rotary_emb, + use_image_dataset=use_image_dataset, + use_sim_mask=use_sim_mask, + temporal_attn_times=temporal_attn_times)) + in_dim = out_dim + + # upsample + if i != len(dim_mult) - 1 and j == num_res_blocks: + upsample = Upsample( + out_dim, True, dims=2.0, out_channels=out_dim) + scale *= 2.0 + block.append(upsample) + self.output_blocks.append(block) + + # head + self.out = nn.Sequential( + nn.GroupNorm(32, out_dim), nn.SiLU(), + nn.Conv2d(out_dim, self.out_dim, 3, padding=1)) + + # zero out the last layer params + nn.init.zeros_(self.out[-1].weight) + + def forward( + self, + x, + t, + y=None, + depth=None, + image=None, + motion=None, + local_image=None, + single_sketch=None, + masked=None, + canny=None, + sketch=None, + histogram=None, + fps=None, + video_mask=None, + focus_present_mask=None, + prob_focus_present=0., + mask_last_frame_num=0 # mask last frame num + ): + + assert self.inpainting or masked is None, 'inpainting is not supported' + + batch, c, f, h, w = x.shape + device = x.device + self.batch = batch + + # image and video joint training, if mask_last_frame_num is set, prob_focus_present will be ignored + if mask_last_frame_num > 0: + focus_present_mask = None + video_mask[-mask_last_frame_num:] = False + else: + focus_present_mask = default( + focus_present_mask, lambda: prob_mask_like( + (batch, ), prob_focus_present, device=device)) + + if self.temporal_attention and not USE_TEMPORAL_TRANSFORMER: + time_rel_pos_bias = self.time_rel_pos_bias( + x.shape[2], device=x.device) + else: + time_rel_pos_bias = None + + # all-zero and all-keep masks + zero = torch.zeros(batch, dtype=torch.bool).to(x.device) + keep = torch.zeros(batch, dtype=torch.bool).to(x.device) + if self.training: + nzero = (torch.rand(batch) < self.p_all_zero).sum() + nkeep = (torch.rand(batch) < self.p_all_keep).sum() + index = torch.randperm(batch) + zero[index[0:nzero]] = True + keep[index[nzero:nzero + nkeep]] = True + assert not (zero & keep).any() + misc_dropout = partial(self.misc_dropout, zero=zero, keep=keep) + + concat = x.new_zeros(batch, self.concat_dim, f, h, w) + if depth is not None: + # DropPath mask + depth = rearrange(depth, 'b c f h w -> (b f) c h w') + depth = self.depth_embedding(depth) + h = depth.shape[2] + depth = self.depth_embedding_after( + rearrange(depth, '(b f) c h w -> (b h w) f c', b=batch)) + + # + depth = rearrange(depth, '(b h w) f c -> b c f h w', b=batch, h=h) + concat = concat + misc_dropout(depth) + + # local_image_embedding + if local_image is not None: + local_image = rearrange(local_image, 'b c f h w -> (b f) c h w') + local_image = self.local_image_embedding(local_image) + + h = local_image.shape[2] + local_image = self.local_image_embedding_after( + rearrange(local_image, '(b f) c h w -> (b h w) f c', b=batch)) + local_image = rearrange( + local_image, '(b h w) f c -> b c f h w', b=batch, h=h) + + concat = concat + misc_dropout(local_image) + + if motion is not None: + motion = rearrange(motion, 'b c f h w -> (b f) c h w') + motion = self.motion_embedding(motion) + + h = motion.shape[2] + motion = self.motion_embedding_after( + rearrange(motion, '(b f) c h w -> (b h w) f c', b=batch)) + motion = rearrange( + motion, '(b h w) f c -> b c f h w', b=batch, h=h) + + if hasattr(self.cfg, 'p_zero_motion_alone' + ) and self.cfg.p_zero_motion_alone and self.training: + motion_d = torch.rand(batch) < self.cfg.p_zero_motion + motion_d = motion_d[:, None, None, None, None] + motion = motion.masked_fill(motion_d.cuda(), 0) + concat = concat + motion + else: + concat = concat + misc_dropout(motion) + + if canny is not None: + # DropPath mask + canny = rearrange(canny, 'b c f h w -> (b f) c h w') + canny = self.canny_embedding(canny) + + h = canny.shape[2] + canny = self.canny_embedding_after( + rearrange(canny, '(b f) c h w -> (b h w) f c', b=batch)) + canny = rearrange(canny, '(b h w) f c -> b c f h w', b=batch, h=h) + + concat = concat + misc_dropout(canny) + + if sketch is not None: + # DropPath mask + sketch = rearrange(sketch, 'b c f h w -> (b f) c h w') + sketch = self.sketch_embedding(sketch) + + h = sketch.shape[2] + sketch = self.sketch_embedding_after( + rearrange(sketch, '(b f) c h w -> (b h w) f c', b=batch)) + sketch = rearrange( + sketch, '(b h w) f c -> b c f h w', b=batch, h=h) + + concat = concat + misc_dropout(sketch) + + if single_sketch is not None: + # DropPath mask + single_sketch = rearrange(single_sketch, + 'b c f h w -> (b f) c h w') + single_sketch = self.single_sketch_embedding(single_sketch) + + h = single_sketch.shape[2] + single_sketch = self.single_sketch_embedding_after( + rearrange( + single_sketch, '(b f) c h w -> (b h w) f c', b=batch)) + single_sketch = rearrange( + single_sketch, '(b h w) f c -> b c f h w', b=batch, h=h) + + concat = concat + misc_dropout(single_sketch) + + if masked is not None: + # DropPath mask + masked = rearrange(masked, 'b c f h w -> (b f) c h w') + masked = self.masked_embedding(masked) + + h = masked.shape[2] + masked = self.mask_embedding_after( + rearrange(masked, '(b f) c h w -> (b h w) f c', b=batch)) + masked = rearrange( + masked, '(b h w) f c -> b c f h w', b=batch, h=h) + + concat = concat + misc_dropout(masked) + + x = torch.cat([x, concat], dim=1) + x = rearrange(x, 'b c f h w -> (b f) c h w') + x = self.pre_image(x) + x = rearrange(x, '(b f) c h w -> b c f h w', b=batch) + + # embeddings + if self.use_fps_condition and fps is not None: + e = self.time_embed(sinusoidal_embedding( + t, self.dim)) + self.fps_embedding( + sinusoidal_embedding(fps, self.dim)) + else: + e = self.time_embed(sinusoidal_embedding(t, self.dim)) + + context = x.new_zeros(batch, 0, self.context_dim) + if y is not None: + y_context = misc_dropout(y) + context = torch.cat([context, y_context], dim=1) + else: + y_context = self.zero_y.repeat(batch, 1, 1) + context = torch.cat([context, y_context], dim=1) + + if image is not None: + image_context = misc_dropout(self.pre_image_condition(image)) + context = torch.cat([context, image_context], dim=1) + + # repeat f times for spatial e and context + e = e.repeat_interleave(repeats=f, dim=0) + context = context.repeat_interleave(repeats=f, dim=0) + + # always in shape (b f) c h w, except for temporal layer + x = rearrange(x, 'b c f h w -> (b f) c h w') + # encoder + xs = [] + for block in self.input_blocks: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + xs.append(x) + + # middle + for block in self.middle_block: + x = self._forward_single(block, x, e, context, time_rel_pos_bias, + focus_present_mask, video_mask) + + # decoder + for block in self.output_blocks: + x = torch.cat([x, xs.pop()], dim=1) + x = self._forward_single( + block, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=xs[-1] if len(xs) > 0 else None) + + # head + x = self.out(x) + + # reshape back to (b c f h w) + x = rearrange(x, '(b f) c h w -> b c f h w', b=batch) + return x + + def _forward_single(self, + module, + x, + e, + context, + time_rel_pos_bias, + focus_present_mask, + video_mask, + reference=None): + if isinstance(module, ResidualBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, reference) + elif isinstance(module, ResBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = x.contiguous() + x = module(x, e, self.batch) + elif isinstance(module, SpatialTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, TemporalTransformer): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, context) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, CrossAttention): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, BasicTransformerBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = module(x, context) + elif isinstance(module, FeedForward): + x = module(x, context) + elif isinstance(module, Upsample): + x = module(x) + elif isinstance(module, Downsample): + x = module(x) + elif isinstance(module, Resample): + x = module(x, reference) + elif isinstance(module, TemporalAttentionBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalAttentionMultiBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x, time_rel_pos_bias, focus_present_mask, video_mask) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, InitTemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, TemporalConvBlock): + module = checkpoint_wrapper( + module) if self.use_checkpoint else module + x = rearrange(x, '(b f) c h w -> b c f h w', b=self.batch) + x = module(x) + x = rearrange(x, 'b c f h w -> (b f) c h w') + elif isinstance(module, nn.ModuleList): + for block in module: + x = self._forward_single(block, x, e, context, + time_rel_pos_bias, focus_present_mask, + video_mask, reference) + else: + x = module(x) + return x + + +class PreNormattention(nn.Module): + + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + x + + +class PreNormattention_qkv(nn.Module): + + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, q, k, v, **kwargs): + return self.fn(self.norm(q), self.norm(k), self.norm(v), **kwargs) + q + + +class Attention(nn.Module): + + def __init__(self, dim, heads=8, dim_head=64, dropout=0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head**-0.5 + + self.attend = nn.Softmax(dim=-1) + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout)) if project_out else nn.Identity() + + def forward(self, x): + _, _, _, h = *x.shape, self.heads + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), qkv) + + dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale + + attn = self.attend(dots) + + out = einsum('b h i j, b h j d -> b h i d', attn, v) + out = rearrange(out, 'b h n d -> b n (h d)') + return self.to_out(out) + + +class Attention_qkv(nn.Module): + + def __init__(self, dim, heads=8, dim_head=64, dropout=0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head**-0.5 + + self.attend = nn.Softmax(dim=-1) + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_k = nn.Linear(dim, inner_dim, bias=False) + self.to_v = nn.Linear(dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout)) if project_out else nn.Identity() + + def forward(self, q, k, v): + _, _, _, h = *q.shape, self.heads + bk = k.shape[0] + q = self.to_q(q) + k = self.to_k(k) + v = self.to_v(v) + q = rearrange(q, 'b n (h d) -> b h n d', h=h) + k = rearrange(k, 'b n (h d) -> b h n d', b=bk, h=h) + v = rearrange(v, 'b n (h d) -> b h n d', b=bk, h=h) + + dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale + + attn = self.attend(dots) + + out = einsum('b h i j, b h j d -> b h i d', attn, v) + out = rearrange(out, 'b h n d -> b n (h d)') + return self.to_out(out) + + +class PostNormattention(nn.Module): + + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, x, **kwargs): + return self.norm(self.fn(x, **kwargs) + x) + + +class Transformer_v2(nn.Module): + + def __init__(self, + heads=8, + dim=2048, + dim_head_k=256, + dim_head_v=256, + dropout_atte=0.05, + mlp_dim=2048, + dropout_ffn=0.05, + depth=1): + super().__init__() + self.layers = nn.ModuleList([]) + self.depth = depth + for _ in range(depth): + self.layers.append( + nn.ModuleList([ + PreNormattention( + dim, + Attention( + dim, + heads=heads, + dim_head=dim_head_k, + dropout=dropout_atte)), + FeedForward(dim, mlp_dim, dropout=dropout_ffn), + ])) + + def forward(self, x): + for attn, ff in self.layers[:1]: + x = attn(x) + x = ff(x) + x + if self.depth > 1: + for attn, ff in self.layers[1:]: + x = attn(x) + x = ff(x) + x + return x + + +class DropPath(nn.Module): + r"""DropPath but without rescaling and supports optional all-zero and/or all-keep. + """ + + def __init__(self, p): + super(DropPath, self).__init__() + self.p = p + + def forward(self, *args, zero=None, keep=None): + if not self.training: + return args[0] if len(args) == 1 else args + + # params + x = args[0] + b = x.size(0) + n = (torch.rand(b) < self.p).sum() + + # non-zero and non-keep mask + mask = x.new_ones(b, dtype=torch.bool) + if keep is not None: + mask[keep] = False + if zero is not None: + mask[zero] = False + + # drop-path index + index = torch.where(mask)[0] + index = index[torch.randperm(len(index))[:n]] + if zero is not None: + index = torch.cat([index, torch.where(zero)[0]], dim=0) + + # drop-path multiplier + multiplier = x.new_ones(b) + multiplier[index] = 0.0 + output = tuple(u * self.broadcast(multiplier, u) for u in args) + return output[0] if len(args) == 1 else output + + def broadcast(self, src, dst): + assert src.size(0) == dst.size(0) + shape = (dst.size(0), ) + (1, ) * (dst.ndim - 1) + return src.view(shape) + + +if __name__ == '__main__': + from config import cfg + # [model] unet + model = UNetSD_temporal( + in_dim=cfg.unet_in_dim, + dim=cfg.unet_dim, + y_dim=cfg.unet_y_dim, + context_dim=cfg.unet_context_dim, + out_dim=cfg.unet_out_dim, + dim_mult=cfg.unet_dim_mult, + num_heads=cfg.unet_num_heads, + head_dim=cfg.unet_head_dim, + num_res_blocks=cfg.unet_res_blocks, + attn_scales=cfg.unet_attn_scales, + dropout=cfg.unet_dropout, + temporal_attn_times=0, + use_checkpoint=cfg.use_checkpoint, + use_image_dataset=True, + use_fps_condition=cfg.use_fps_condition) + + print( + int(sum(p.numel() for k, p in model.named_parameters()) / (1024**2)), + 'M parameters') diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/__init__.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/config.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..059e34630b292fb10b8dfb490a809c15cc43be39 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/config.py @@ -0,0 +1,274 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import argparse +import copy +import os + +import json +import yaml + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def setup_seed(seed): + print('Seed: ', seed) + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +class Config(object): + + def __init__(self, + load=True, + cfg_dict=None, + cfg_level=None, + model_dir=None, + cfg_file_name='exp06_text_depths_vs_style.yaml'): + if model_dir is not None and os.path.isdir(model_dir): + self.model_dir = model_dir + '/configs' + else: + raise Exception(f'model_dir {model_dir} is not exist!') + self.cfg_file_name = cfg_file_name + self._level = 'cfg' + ('.' + + cfg_level if cfg_level is not None else '') + if load: + self.args = self._parse_args() + logger.info('Loading config from {}.'.format(self.args.cfg_file)) + self.need_initialization = True + cfg_base = self._initialize_cfg() + cfg_dict = self._load_yaml(self.args) + cfg_dict = self._merge_cfg_from_base(cfg_base, cfg_dict) + cfg_dict = self._update_from_args(cfg_dict) + self.cfg_dict = cfg_dict + self._update_dict(cfg_dict) + + def _parse_args(self): + parser = argparse.ArgumentParser( + description= + 'Argparser for configuring [code base name to think of] codebase') + parser.add_argument( + '--cfg', + dest='cfg_file', + help='Path to the configuration file', + default=os.path.join(self.model_dir, self.cfg_file_name)) + parser.add_argument( + '--init_method', + help='Initialization method, includes TCP or shared file-system', + default='tcp://localhost:9999', + type=str, + ) + parser.add_argument( + '--seed', + type=int, + default=8888, + help='Need to explore for different videos') + parser.add_argument( + '--debug', + action='store_true', + default=False, + help='Into debug information') + parser.add_argument( + '--input_video', + default='demo_video/video_8800.mp4', + help='input video for full task, or motion vector of input videos', + type=str, + ), + parser.add_argument( + '--image_path', default='', help='Single Image Input', type=str) + parser.add_argument( + '--sketch_path', default='', help='Single Sketch Input', type=str) + parser.add_argument( + '--style_image', help='Single Sketch Input', type=str) + parser.add_argument( + '--input_text_desc', + default= + 'A colorful and beautiful fish swimming in a small glass bowl with \ + multicolored piece of stone, Macro Video', + type=str, + ), + parser.add_argument( + 'opts', + help='other configurations', + default=None, + nargs=argparse.REMAINDER) + return parser.parse_args() + + def _path_join(self, path_list): + path = '' + for p in path_list: + path += p + '/' + return path[:-1] + + def _update_from_args(self, cfg_dict): + args = self.args + for var in vars(args): + cfg_dict[var] = getattr(args, var) + return cfg_dict + + def _initialize_cfg(self): + if self.need_initialization: + self.need_initialization = False + if os.path.exists(os.path.join(self.model_dir, 'base.yaml')): + with open(os.path.join(self.model_dir, 'base.yaml'), 'r') as f: + cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) + else: + with open(os.path.join(self.model_dir, 'base.yaml'), 'r') as f: + cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) + return cfg + + def _load_yaml(self, args, file_name=''): + assert args.cfg_file is not None + if not file_name == '': # reading from base file + with open(file_name, 'r') as f: + cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) + else: + if os.getcwd().split('/')[-1] == args.cfg_file.split('/')[0]: + args.cfg_file = args.cfg_file.replace( + os.getcwd().split('/')[-1], './') + try: + with open(args.cfg_file, 'r') as f: + cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) + file_name = args.cfg_file + except Exception as e: + args.cfg_file = os.path.realpath(__file__).split( + '/')[-3] + '/' + args.cfg_file + with open(args.cfg_file, 'r') as f: + cfg = yaml.load(f.read(), Loader=yaml.SafeLoader) + file_name = args.cfg_file + print(e) + + if '_BASE_RUN' not in cfg.keys() and '_BASE_MODEL' not in cfg.keys( + ) and '_BASE' not in cfg.keys(): + return cfg + + if '_BASE' in cfg.keys(): + if cfg['_BASE'][1] == '.': + prev_count = cfg['_BASE'].count('..') + cfg_base_file = self._path_join( + file_name.split('/')[:(-1 - cfg['_BASE'].count('..'))] + + cfg['_BASE'].split('/')[prev_count:]) + else: + cfg_base_file = cfg['_BASE'].replace( + './', + args.cfg_file.replace(args.cfg_file.split('/')[-1], '')) + cfg_base = self._load_yaml(args, cfg_base_file) + cfg = self._merge_cfg_from_base(cfg_base, cfg) + else: + if '_BASE_RUN' in cfg.keys(): + if cfg['_BASE_RUN'][1] == '.': + prev_count = cfg['_BASE_RUN'].count('..') + cfg_base_file = self._path_join( + file_name.split('/')[:(-1 - prev_count)] + + cfg['_BASE_RUN'].split('/')[prev_count:]) + else: + cfg_base_file = cfg['_BASE_RUN'].replace( + './', + args.cfg_file.replace( + args.cfg_file.split('/')[-1], '')) + cfg_base = self._load_yaml(args, cfg_base_file) + cfg = self._merge_cfg_from_base( + cfg_base, cfg, preserve_base=True) + if '_BASE_MODEL' in cfg.keys(): + if cfg['_BASE_MODEL'][1] == '.': + prev_count = cfg['_BASE_MODEL'].count('..') + cfg_base_file = self._path_join( + file_name.split('/')[:( + -1 - cfg['_BASE_MODEL'].count('..'))] + + cfg['_BASE_MODEL'].split('/')[prev_count:]) + else: + cfg_base_file = cfg['_BASE_MODEL'].replace( + './', + args.cfg_file.replace( + args.cfg_file.split('/')[-1], '')) + cfg_base = self._load_yaml(args, cfg_base_file) + cfg = self._merge_cfg_from_base(cfg_base, cfg) + cfg = self._merge_cfg_from_command(args, cfg) + return cfg + + def _merge_cfg_from_base(self, cfg_base, cfg_new, preserve_base=False): + for k, v in cfg_new.items(): + if k in cfg_base.keys(): + if isinstance(v, dict): + self._merge_cfg_from_base(cfg_base[k], v) + else: + cfg_base[k] = v + else: + if 'BASE' not in k or preserve_base: + cfg_base[k] = v + return cfg_base + + def _merge_cfg_from_command(self, args, cfg): + assert len( + args.opts) % 2 == 0, 'Override list {} has odd length: {}.'.format( + args.opts, len(args.opts)) + keys = args.opts[0::2] + vals = args.opts[1::2] + + # maximum supported depth 3 + for idx, key in enumerate(keys): + key_split = key.split('.') + assert len( + key_split + ) <= 4, 'Key depth error. \nMaximum depth: 3\n Get depth: {}'.format( + len(key_split)) + assert key_split[0] in cfg.keys(), 'Non-existant key: {}.'.format( + key_split[0]) + if len(key_split) == 2: + assert key_split[1] in cfg[ + key_split[0]].keys(), 'Non-existant key: {}.'.format(key) + elif len(key_split) == 3: + assert key_split[1] in cfg[ + key_split[0]].keys(), 'Non-existant key: {}.'.format(key) + assert key_split[2] in cfg[key_split[0]][ + key_split[1]].keys(), 'Non-existant key: {}.'.format(key) + elif len(key_split) == 4: + assert key_split[1] in cfg[ + key_split[0]].keys(), 'Non-existant key: {}.'.format(key) + assert key_split[2] in cfg[key_split[0]][ + key_split[1]].keys(), 'Non-existant key: {}.'.format(key) + assert key_split[3] in cfg[key_split[0]][key_split[1]][ + key_split[2]].keys(), 'Non-existant key: {}.'.format(key) + if len(key_split) == 1: + cfg[key_split[0]] = vals[idx] + elif len(key_split) == 2: + cfg[key_split[0]][key_split[1]] = vals[idx] + elif len(key_split) == 3: + cfg[key_split[0]][key_split[1]][key_split[2]] = vals[idx] + elif len(key_split) == 4: + cfg[key_split[0]][key_split[1]][key_split[2]][ + key_split[3]] = vals[idx] + return cfg + + def _update_dict(self, cfg_dict): + + def recur(key, elem): + if type(elem) is dict: + return key, Config( + load=False, + cfg_dict=elem, + cfg_level=key, + model_dir=self.model_dir, + cfg_file_name=self.cfg_file_name) + else: + if type(elem) is str and elem[1:3] == 'e-': + elem = float(elem) + return key, elem + + dic = dict(recur(k, v) for k, v in cfg_dict.items()) + self.__dict__.update(dic) + + def get_args(self): + return self.args + + def __repr__(self): + return '{}\n'.format(self.dump()) + + def dump(self): + return json.dumps(self.cfg_dict, indent=2) + + def deep_copy(self): + return copy.deepcopy(self) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/distributed.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..b622e700194d106c8838db38d3e126c922e668f5 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/distributed.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +"""Distributed helpers.""" + +import functools +import logging +import pickle + +import torch +import torch.distributed as dist + +_LOCAL_PROCESS_GROUP = None + + +def all_gather(tensors): + """ + All gathers the provided tensors from all processes across machines. + Args: + tensors (list): tensors to perform all gather across all processes in + all machines. + """ + + gather_list = [] + output_tensor = [] + world_size = dist.get_world_size() + for tensor in tensors: + tensor_placeholder = [ + torch.ones_like(tensor) for _ in range(world_size) + ] + dist.all_gather(tensor_placeholder, tensor, async_op=False) + gather_list.append(tensor_placeholder) + for gathered_tensor in gather_list: + output_tensor.append(torch.cat(gathered_tensor, dim=0)) + return output_tensor + + +def all_reduce(tensors, average=True): + """ + All reduce the provided tensors from all processes across machines. + Args: + tensors (list): tensors to perform all reduce across all processes in + all machines. + average (bool): scales the reduced tensor by the number of overall + processes across all machines. + """ + + for tensor in tensors: + dist.all_reduce(tensor, async_op=False) + if average: + world_size = dist.get_world_size() + for tensor in tensors: + tensor.mul_(1.0 / world_size) + return tensors + + +def init_process_group( + local_rank, + local_world_size, + shard_id, + num_shards, + init_method, + dist_backend='nccl', +): + """ + Initializes the default process group. + Args: + local_rank (int): the rank on the current local machine. + local_world_size (int): the world size (number of processes running) on + the current local machine. + shard_id (int): the shard index (machine rank) of the current machine. + num_shards (int): number of shards for distributed training. + init_method (string): supporting three different methods for + initializing process groups: + "file": use shared file system to initialize the groups across + different processes. + "tcp": use tcp address to initialize the groups across different + dist_backend (string): backend to use for distributed training. Options + includes gloo, mpi and nccl, the details can be found here: + https://pytorch.org/docs/stable/distributed.html + """ + # Sets the GPU to use. + torch.cuda.set_device(local_rank) + # Initialize the process group. + proc_rank = local_rank + shard_id * local_world_size + world_size = local_world_size * num_shards + dist.init_process_group( + backend=dist_backend, + init_method=init_method, + world_size=world_size, + rank=proc_rank, + ) + + +def is_master_proc(num_gpus=8): + """ + Determines if the current process is the master process. + """ + if torch.distributed.is_initialized(): + return dist.get_rank() % num_gpus == 0 + else: + return True + + +def get_world_size(): + """ + Get the size of the world. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + """ + Get the rank of the current process. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + dist.barrier() + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + Returns: + (group): pytorch dist group. + """ + if dist.get_backend() == 'nccl': + return dist.new_group(backend='gloo') + else: + return dist.group.WORLD + + +def _serialize_to_tensor(data, group): + """ + Seriialize the tensor to ByteTensor. Note that only `gloo` and `nccl` + backend is supported. + Args: + data (data): data to be serialized. + group (group): pytorch dist group. + Returns: + tensor (ByteTensor): tensor that serialized. + """ + + backend = dist.get_backend(group) + assert backend in ['gloo', 'nccl'] + device = torch.device('cpu' if backend == 'gloo' else 'cuda') + + buffer = pickle.dumps(data) + if len(buffer) > 1024**3: + logger = logging.getLogger(__name__) + logger.warning( + 'Rank {} trying to all-gather {:.2f} GB of data on device {}'. + format(get_rank(), + len(buffer) / (1024**3), device)) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to(device=device) + return tensor + + +def _pad_to_largest_tensor(tensor, group): + """ + Padding all the tensors from different GPUs to the largest ones. + Args: + tensor (tensor): tensor to pad. + group (group): pytorch dist group. + Returns: + list[int]: size of the tensor, on each rank + Tensor: padded tensor that has the max size + """ + world_size = dist.get_world_size(group=group) + assert ( + world_size >= 1 + ), 'comm.gather/all_gather must be called from ranks within the given group!' + local_size = torch.tensor([tensor.numel()], + dtype=torch.int64, + device=tensor.device) + size_list = [ + torch.zeros([1], dtype=torch.int64, device=tensor.device) + for _ in range(world_size) + ] + dist.all_gather(size_list, local_size, group=group) + size_list = [int(size.item()) for size in size_list] + + max_size = max(size_list) + + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + if local_size != max_size: + padding = torch.zeros((max_size - local_size, ), + dtype=torch.uint8, + device=tensor.device) + tensor = torch.cat((tensor, padding), dim=0) + return size_list, tensor + + +def all_gather_unaligned(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return [data] + + tensor = _serialize_to_tensor(data, group) + + size_list, tensor = _pad_to_largest_tensor(tensor, group) + max_size = max(size_list) + + # receiving Tensor from all ranks + tensor_list = [ + torch.empty((max_size, ), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.all_gather(tensor_list, tensor, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def init_distributed_training(cfg): + """ + Initialize variables needed for distributed training. + """ + if cfg.NUM_GPUS <= 1: + return + num_gpus_per_machine = cfg.NUM_GPUS + num_machines = dist.get_world_size() // num_gpus_per_machine + for i in range(num_machines): + ranks_on_i = list( + range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) + pg = dist.new_group(ranks_on_i) + if i == cfg.SHARD_ID: + global _LOCAL_PROCESS_GROUP + _LOCAL_PROCESS_GROUP = pg + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, + i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + assert _LOCAL_PROCESS_GROUP is not None + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/utils.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d01b3dd2be50ea9abffe4e1e7636eb56220a6619 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/utils/utils.py @@ -0,0 +1,955 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import base64 +import binascii +import copy +import glob +import gzip +import hashlib +import logging +import math +import os +import os.path as osp +import pickle +import random +import sys +import time +import urllib.request +import zipfile +from io import BytesIO +from multiprocessing.pool import ThreadPool as Pool + +import imageio +import json +import numpy as np +import oss2 as oss +import requests +import skvideo.io +import torch +import torch.nn.functional as F +import torchvision.utils as tvutils +from einops import rearrange +from PIL import Image + +__all__ = [ + 'parse_oss_url', 'parse_bucket', 'read', 'read_image', 'read_gzip', + 'ceil_divide', 'to_device', 'put_object', 'put_torch_object', + 'put_object_from_file', 'get_object', 'get_object_to_file', 'rand_name', + 'save_image', 'save_video', 'save_video_vs_conditions', + 'save_video_multiple_conditions_with_data', + 'save_video_multiple_conditions', 'download_video_to_file', + 'save_video_grid_mp4', 'save_caps', 'ema', 'parallel', 'exists', + 'download', 'unzip', 'load_state_dict', 'inverse_indices', + 'detect_duplicates', 'read_tfs', 'md5', 'rope', 'format_state', + 'breakup_grid', 'huggingface_tokenizer', 'huggingface_model' +] + +TFS_CLIENT = None + + +def DOWNLOAD_TO_CACHE(oss_key, + file_or_dirname=None, + cache_dir=osp.join( + '/'.join(osp.abspath(__file__).split('/')[:-2]), + 'model_weights')): + r"""Download OSS [file or folder] to the cache folder. + Only the 0th process on each node will run the downloading. + Barrier all processes until the downloading is completed. + """ + # source and target paths + base_path = osp.join(cache_dir, file_or_dirname or osp.basename(oss_key)) + + return base_path + + +def find_free_port(): + """https://stackoverflow.com/questions/1365265/on-localhost-how-do-i-pick-a-free-port-number""" + import socket + from contextlib import closing + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(('', 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return str(s.getsockname()[1]) + + +def setup_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True + + +def parse_oss_url(path): + if path.startswith('oss://'): + path = path[len('oss://'):] + + # configs + configs = { + 'endpoint': os.getenv('OSS_ENDPOINT', None), + 'accessKeyID': os.getenv('OSS_ACCESS_KEY_ID', None), + 'accessKeySecret': os.getenv('OSS_ACCESS_KEY_SECRET', None), + 'securityToken': os.getenv('OSS_SECURITY_TOKEN', None) + } + bucket, path = path.split('/', maxsplit=1) + if '?' in bucket: + bucket, config = bucket.split('?', maxsplit=1) + for pair in config.split('&'): + k, v = pair.split('=', maxsplit=1) + configs[k] = v + + # session + session = parse_oss_url._sessions.setdefault(f'{bucket}@{os.getpid()}', + oss.Session()) + + # bucket + bucket = oss.Bucket( + auth=oss.Auth(configs['accessKeyID'], configs['accessKeySecret']), + endpoint=configs['endpoint'], + bucket_name=bucket, + session=session) + return bucket, path + + +parse_oss_url._sessions = {} + + +def parse_bucket(url): + return parse_oss_url(osp.join(url, '_placeholder'))[0] + + +def read(filename, mode='r', retry=5): + assert mode in ['r', 'rb'] + exception = None + for _ in range(retry): + try: + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + content = bucket.get_object(path).read() + if mode == 'r': + content = content.decode('utf-8') + elif filename.startswith('http'): + content = requests.get(filename).content + if mode == 'r': + content = content.decode('utf-8') + else: + with open(filename, mode=mode) as f: + content = f.read() + return content + except Exception as e: + exception = e + continue + else: + raise exception + + +def read_image(filename, retry=5): + exception = None + for _ in range(retry): + try: + return Image.open(BytesIO(read(filename, mode='rb', retry=retry))) + except Exception as e: + exception = e + continue + else: + raise exception + + +def download_video_to_file(filename, local_file, retry=5): + exception = None + for _ in range(retry): + try: + bucket, path = parse_oss_url(filename) + bucket.get_object_to_file(path, local_file) + break + except Exception as e: + exception = e + continue + else: + raise exception + + +def read_gzip(filename, retry=5): + exception = None + for _ in range(retry): + try: + remove = False + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + filename = rand_name(suffix=osp.splitext(filename)[1]) + bucket.get_object_to_file(path, filename) + remove = True + with gzip.open(filename) as f: + content = f.read() + if remove: + os.remove(filename) + return content + except Exception as e: + exception = e + continue + else: + raise exception + + +def ceil_divide(a, b): + return int(math.ceil(a / b)) + + +def to_device(batch, device, non_blocking=False): + if isinstance(batch, (list, tuple)): + return type(batch)([to_device(u, device, non_blocking) for u in batch]) + elif isinstance(batch, dict): + return type(batch)([(k, to_device(v, device, non_blocking)) + for k, v in batch.items()]) + elif isinstance(batch, torch.Tensor) and batch.device != device: + batch = batch.to(device, non_blocking=non_blocking) + return batch + + +def put_object(bucket, oss_key, data, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.put_object(oss_key, data) + except Exception as e: + exception = e + continue + else: + print( + f'put_object to {oss_key} failed with error: {exception}', + flush=True) + + +def put_torch_object(bucket, oss_key, data, retry=5): + exception = None + for _ in range(retry): + try: + buffer = BytesIO() + torch.save(data, buffer) + return bucket.put_object(oss_key, buffer.getvalue()) + except Exception as e: + exception = e + continue + else: + print( + f'put_torch_object to {oss_key} failed with error: {exception}', + flush=True) + + +def put_object_from_file(bucket, oss_key, filename, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.put_object_from_file(oss_key, filename) + except Exception as e: + exception = e + continue + else: + print( + f'put_object_from_file to {oss_key} failed with error: {exception}', + flush=True) + + +def get_object(bucket, oss_key, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.get_object(oss_key).read() + except Exception as e: + exception = e + continue + else: + print( + f'get_object from {oss_key} failed with error: {exception}', + flush=True) + + +def get_object_to_file(bucket, oss_key, filename, retry=5): + exception = None + for _ in range(retry): + try: + return bucket.get_object_to_file(oss_key, filename) + except Exception as e: + exception = e + continue + else: + print( + f'get_object_to_file from {oss_key} failed with error: {exception}', + flush=True) + + +def rand_name(length=8, suffix=''): + name = binascii.b2a_hex(os.urandom(length)).decode('utf-8') + if suffix: + if not suffix.startswith('.'): + suffix = '.' + suffix + name += suffix + return name + + +@torch.no_grad() +def save_image(bucket, + oss_key, + tensor, + nrow=8, + normalize=True, + range=(-1, 1), + retry=5): + filename = rand_name(suffix='.jpg') + for _ in [None] * retry: + try: + tvutils.save_image( + tensor, filename, nrow=nrow, normalize=normalize, range=range) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + print( + 'save image to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def video_tensor_to_gif(tensor, path, duration=120, loop=0, optimize=True): + tensor = tensor.permute(1, 2, 3, 0) + images = tensor.unbind(dim=0) + images = [(image.numpy() * 255).astype('uint8') for image in images] + imageio.mimwrite(path, images, fps=8) + return images + + +@torch.no_grad() +def save_video(bucket, + oss_key, + tensor, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=tensor.device).view(1, -1, 1, 1, 1) + tensor = tensor.mul_(std).add_(mean) + tensor.clamp_(0, 1) + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + one_gif = rearrange( + tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + video_tensor_to_gif(one_gif, filename) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + print( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_multiple_conditions(oss_key, + video_tensor, + model_kwargs, + source_imgs, + palette, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5, + save_origin_video=True, + bucket=None): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + try: + video_tensor.clamp_(0, 1) + except Exception as e: + video_tensor = video_tensor.float().clamp_(0, 1) + print(e) + video_tensor = video_tensor.cpu() + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + model_kwargs_channel3 = {} + for key, conditions in model_kwargs[0].items(): + if conditions.shape[-1] == 1024: + # Skip for style embeding + continue + if len(conditions.shape) == 3: + conditions_np = conditions.cpu().numpy() + conditions = [] + for i in conditions_np: + vis_i = [] + for j in i: + vis_i.append( + palette.get_palette_image( + j, percentile=90, width=256, height=256)) + conditions.append(np.stack(vis_i)) + conditions = torch.from_numpy(np.stack(conditions)) + conditions = rearrange(conditions, 'b n h w c -> b c n h w') + else: + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + if conditions.size(1) == 2: + conditions = torch.cat([conditions, conditions[:, :1, ]], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 3: + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 4: + color = ((conditions[:, 0:3] + 1.) / 2.) + alpha = conditions[:, 3:4] + conditions = color * alpha + 1.0 * (1.0 - alpha) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + model_kwargs_channel3[key] = conditions.cpu( + ) if conditions.is_cuda else conditions + + filename = oss_key + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + cons_list = [ + rearrange(con, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + for _, con in model_kwargs_channel3.items() + ] + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + + if save_origin_video: + vid_gif = torch.cat( + [ + source_imgs, + ] + cons_list + [ + vid_gif, + ], dim=3) + else: + vid_gif = torch.cat( + cons_list + [ + vid_gif, + ], dim=3) + + video_tensor_to_gif(vid_gif, filename) + exception = None + break + except Exception as e: + exception = e + continue + if exception is not None: + logging.info('save video to {} failed, error: {}'.format( + oss_key, exception)) + + +@torch.no_grad() +def save_video_multiple_conditions_with_data(bucket, + video_save_key, + gt_video_save_key, + vis_oss_key, + video_tensor, + model_kwargs, + source_imgs, + palette, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + video_tensor.clamp_(0, 1) + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + model_kwargs_channel3 = {} + for key, conditions in model_kwargs[0].items(): + if len(conditions.shape) == 3: + conditions_np = conditions.cpu().numpy() + conditions = [] + for i in conditions_np: + vis_i = [] + for j in i: + vis_i.append( + palette.get_palette_image( + j, percentile=90, width=256, height=256)) + conditions.append(np.stack(vis_i)) + conditions = torch.from_numpy(np.stack(conditions)) + conditions = rearrange(conditions, 'b n h w c -> b c n h w') + else: + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + if conditions.size(1) == 2: + conditions = torch.cat([conditions, conditions[:, :1, ]], + dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 3: + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + elif conditions.size(1) == 4: + color = ((conditions[:, 0:3] + 1.) / 2.) + alpha = conditions[:, 3:4] + conditions = color * alpha + 1.0 * (1.0 - alpha) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + model_kwargs_channel3[key] = conditions.cpu( + ) if conditions.is_cuda else conditions + + copy_video_tensor = video_tensor.clone() + copy_source_imgs = source_imgs.clone() + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + cons_list = [ + rearrange(con, '(i j) c f h w -> c f (i h) (j w)', j=nrow) + for _, con in model_kwargs_channel3.items() + ] + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + vid_gif = torch.cat( + [ + source_imgs, + ] + cons_list + [ + vid_gif, + ], dim=3) + + video_tensor_to_gif(vid_gif, filename) + bucket.put_object_from_file(vis_oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + + filename_pred = rand_name(suffix='.pkl') + for _ in [None] * retry: + try: + copy_video_np = (copy_video_tensor.numpy() * 255).astype('uint8') + pickle.dump(copy_video_np, open(filename_pred, 'wb')) + bucket.put_object_from_file(video_save_key, filename_pred) + break + except Exception as e: + print('error! ', video_save_key, e) + continue + + # remove temporary file + if osp.exists(filename_pred): + os.remove(filename_pred) + + filename_gt = rand_name(suffix='.pkl') + for _ in [None] * retry: + try: + copy_source_np = (copy_source_imgs.numpy() * 255).astype('uint8') + pickle.dump(copy_source_np, open(filename_gt, 'wb')) + bucket.put_object_from_file(gt_video_save_key, filename_gt) + break + except Exception as e: + print('error! ', gt_video_save_key, e) + continue + + # remove temporary file + if osp.exists(filename_gt): + os.remove(filename_gt) + + if exception is not None: + print( + 'save video to {} failed, error: {}'.format( + vis_oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_vs_conditions(bucket, + oss_key, + video_tensor, + conditions, + source_imgs, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=8, + retry=5): + mean = torch.tensor(mean, device=video_tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=video_tensor.device).view(1, -1, 1, 1, 1) + video_tensor = video_tensor.mul_(std).add_(mean) + video_tensor.clamp_(0, 1) + + b, c, n, h, w = video_tensor.shape + source_imgs = F.adaptive_avg_pool3d(source_imgs, (n, h, w)) + source_imgs = source_imgs.cpu() + + if conditions.size(1) == 1: + conditions = torch.cat([conditions, conditions, conditions], dim=1) + conditions = F.adaptive_avg_pool3d(conditions, (n, h, w)) + + filename = rand_name(suffix='.gif') + for _ in [None] * retry: + try: + vid_gif = rearrange( + video_tensor, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + con_gif = rearrange( + conditions, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + source_imgs = rearrange( + source_imgs, '(i j) c f h w -> c f (i h) (j w)', i=nrow) + vid_gif = torch.cat([vid_gif, con_gif, source_imgs], dim=2) + + video_tensor_to_gif(vid_gif, filename) + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + print( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_video_grid_mp4(bucket, + oss_key, + tensor, + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5], + nrow=None, + fps=5, + retry=5): + mean = torch.tensor(mean, device=tensor.device).view(1, -1, 1, 1, 1) + std = torch.tensor(std, device=tensor.device).view(1, -1, 1, 1, 1) + tensor = tensor.mul_(std).add_(mean) + tensor.clamp_(0, 1) + b, c, t, h, w = tensor.shape + tensor = tensor.permute(0, 2, 3, 4, 1) + tensor = (tensor.cpu().numpy() * 255).astype('uint8') + + filename = rand_name(suffix='.mp4') + for _ in [None] * retry: + try: + if nrow is None: + nrow = math.ceil(math.sqrt(b)) + ncol = math.ceil(b / nrow) + padding = 1 + video_grid = np.zeros((t, (padding + h) * nrow + padding, + (padding + w) * ncol + padding, c), + dtype='uint8') + for i in range(b): + r = i // ncol + c_ = i % ncol + + start_r = (padding + h) * r + start_c = (padding + w) * c_ + video_grid[:, start_r:start_r + h, + start_c:start_c + w] = tensor[i] + skvideo.io.vwrite(filename, video_grid, inputdict={'-r': str(fps)}) + + bucket.put_object_from_file(oss_key, filename) + exception = None + break + except Exception as e: + exception = e + continue + + # remove temporary file + if osp.exists(filename): + os.remove(filename) + if exception is not None: + print( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_text(bucket, oss_key, tensor, nrow=8, retry=5): + len = tensor.shape[0] + num_per_row = int(len / nrow) + assert (len == nrow * num_per_row) + texts = '' + for i in range(nrow): + for j in range(num_per_row): + text = dec_bytes2obj(tensor[i * num_per_row + j]) + texts += text + '\n' + texts += '\n' + + for _ in [None] * retry: + try: + bucket.put_object(oss_key, texts) + exception = None + break + except Exception as e: + exception = e + continue + if exception is not None: + print( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def save_caps(bucket, oss_key, caps, retry=5): + texts = '' + for cap in caps: + texts += cap + texts += '\n' + + for _ in [None] * retry: + try: + bucket.put_object(oss_key, texts) + exception = None + break + except Exception as e: + exception = e + continue + if exception is not None: + print( + 'save video to {} failed, error: {}'.format(oss_key, exception), + flush=True) + + +@torch.no_grad() +def ema(net_ema, net, beta, copy_buffer=False): + assert 0.0 <= beta <= 1.0 + for p_ema, p in zip(net_ema.parameters(), net.parameters()): + p_ema.copy_(p.lerp(p_ema, beta)) + if copy_buffer: + for b_ema, b in zip(net_ema.buffers(), net.buffers()): + b_ema.copy_(b) + + +def parallel(func, args_list, num_workers=32, timeout=None): + assert isinstance(args_list, list) + if not isinstance(args_list[0], tuple): + args_list = [(args, ) for args in args_list] + if num_workers == 0: + return [func(*args) for args in args_list] + with Pool(processes=num_workers) as pool: + results = [pool.apply_async(func, args) for args in args_list] + results = [res.get(timeout=timeout) for res in results] + return results + + +def exists(filename): + if filename.startswith('oss://'): + bucket, path = parse_oss_url(filename) + return bucket.object_exists(path) + else: + return osp.exists(filename) + + +def download(url, filename=None, replace=False, quiet=False): + if filename is None: + filename = osp.basename(url) + if not osp.exists(filename) or replace: + try: + if url.startswith('oss://'): + bucket, oss_key = parse_oss_url(url) + bucket.get_object_to_file(oss_key, filename) + else: + urllib.request.urlretrieve(url, filename) + if not quiet: + print(f'Downloaded {url} to {filename}', flush=True) + except Exception as e: + raise ValueError(f'Downloading {filename} failed with error {e}') + return osp.abspath(filename) + + +def unzip(filename, dst_dir=None): + if dst_dir is None: + dst_dir = osp.dirname(filename) + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(dst_dir) + + +def load_state_dict(module, state_dict, drop_prefix=''): + # find incompatible key-vals + src, dst = state_dict, module.state_dict() + if drop_prefix: + src = type(src)([ + (k[len(drop_prefix):] if k.startswith(drop_prefix) else k, v) + for k, v in src.items() + ]) + missing = [k for k in dst if k not in src] + unexpected = [k for k in src if k not in dst] + unmatched = [ + k for k in src.keys() & dst.keys() if src[k].shape != dst[k].shape + ] + + # keep only compatible key-vals + incompatible = set(unexpected + unmatched) + src = type(src)([(k, v) for k, v in src.items() if k not in incompatible]) + module.load_state_dict(src, strict=False) + + # report incompatible key-vals + if len(missing) != 0: + print(' Missing: ' + ', '.join(missing), flush=True) + if len(unexpected) != 0: + print(' Unexpected: ' + ', '.join(unexpected), flush=True) + if len(unmatched) != 0: + print(' Shape unmatched: ' + ', '.join(unmatched), flush=True) + + +def inverse_indices(indices): + r"""Inverse map of indices. + E.g., if A[indices] == B, then B[inv_indices] == A. + """ + inv_indices = torch.empty_like(indices) + inv_indices[indices] = torch.arange(len(indices)).to(indices) + return inv_indices + + +def detect_duplicates(feats, thr=0.9): + assert feats.ndim == 2 + + # compute simmat + feats = F.normalize(feats, p=2, dim=1) + simmat = torch.mm(feats, feats.T) + simmat.triu_(1) + torch.cuda.synchronize() + + # detect duplicates + mask = ~simmat.gt(thr).any(dim=0) + return torch.where(mask)[0] + + +class TFSClient(object): + + def __init__(self, + host='restful-store.vip.tbsite.net:3800', + app_key='5354c9fae75f5'): + self.host = host + self.app_key = app_key + + # candidate servers + self.servers = [ + u for u in read(f'http://{host}/url.list').strip().split('\n')[1:] + if ':' in u + ] + assert len(self.servers) >= 1 + self.__server_id = -1 + + @property + def server(self): + self.__server_id = (self.__server_id + 1) % len(self.servers) + return self.servers[self.__server_id] + + def read(self, tfs): + tfs = osp.basename(tfs) + meta = json.loads( + read( + f'http://{self.server}/v1/{self.app_key}/metadata/{tfs}?force=0' + )) + img = Image.open( + BytesIO( + read( + f'http://{self.server}/v1/{self.app_key}/{tfs}?offset=0&size={meta["SIZE"]}', + 'rb'))) + return img + + +def read_tfs(tfs, retry=5): + exception = None + for _ in range(retry): + try: + global TFS_CLIENT + if TFS_CLIENT is None: + TFS_CLIENT = TFSClient() + return TFS_CLIENT.read(tfs) + except Exception as e: + exception = e + continue + else: + raise exception + + +def md5(filename): + with open(filename, 'rb') as f: + return hashlib.md5(f.read()).hexdigest() + + +def rope(x): + r"""Apply rotary position embedding on x of shape [B, *(spatial dimensions), C]. + """ + # reshape + shape = x.shape + x = x.view(x.size(0), -1, x.size(-1)) + l, c = x.shape[-2:] + assert c % 2 == 0 + half = c // 2 + + # apply rotary position embedding on x + sinusoid = torch.outer( + torch.arange(l).to(x), + torch.pow(10000, -torch.arange(half).to(x).div(half))) + sin, cos = torch.sin(sinusoid), torch.cos(sinusoid) + x1, x2 = x.chunk(2, dim=-1) + x = torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1) + + # reshape back + return x.view(shape) + + +def format_state(state, filename=None): + r"""For comparing/aligning state_dict. + """ + content = '\n'.join([f'{k}\t{tuple(v.shape)}' for k, v in state.items()]) + if filename: + with open(filename, 'w') as f: + f.write(content) + + +def breakup_grid(img, grid_size): + r"""The inverse operator of ``torchvision.utils.make_grid``. + """ + # params + nrow = img.height // grid_size + ncol = img.width // grid_size + wrow = wcol = 2 + + # collect grids + grids = [] + for i in range(nrow): + for j in range(ncol): + x1 = j * grid_size + (j + 1) * wcol + y1 = i * grid_size + (i + 1) * wrow + grids.append(img.crop((x1, y1, x1 + grid_size, y1 + grid_size))) + return grids + + +def huggingface_tokenizer(name='google/mt5-xxl', **kwargs): + from transformers import AutoTokenizer + return AutoTokenizer.from_pretrained( + DOWNLOAD_TO_CACHE(f'huggingface/tokenizers/{name}', name), **kwargs) + + +def huggingface_model(name='google/mt5-xxl', model_type='AutoModel', **kwargs): + import transformers + return getattr(transformers, model_type).from_pretrained( + DOWNLOAD_TO_CACHE(f'huggingface/models/{name}', name), **kwargs) diff --git a/runtime/Lib/site_packages/models/multi_modal/videocomposer/videocomposer_model.py b/runtime/Lib/site_packages/models/multi_modal/videocomposer/videocomposer_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c085e16f8fe53d7a268b84498b9b24d3d43aa3eb --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/videocomposer/videocomposer_model.py @@ -0,0 +1,490 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from copy import copy, deepcopy +from os import path as osp +from typing import Any, Dict + +import open_clip +import pynvml +import torch +import torch.cuda.amp as amp +import torch.nn as nn +from einops import rearrange + +import modelscope.models.multi_modal.videocomposer.models as models +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.videocomposer.annotator.sketch import ( + pidinet_bsd, sketch_simplification_gan) +from modelscope.models.multi_modal.videocomposer.autoencoder import \ + AutoencoderKL +from modelscope.models.multi_modal.videocomposer.clip import ( + FrozenOpenCLIPEmbedder, FrozenOpenCLIPVisualEmbedder) +from modelscope.models.multi_modal.videocomposer.diffusion import ( + GaussianDiffusion, beta_schedule) +from modelscope.models.multi_modal.videocomposer.ops.utils import ( + get_first_stage_encoding, make_masked_images, prepare_model_kwargs, + save_with_model_kwargs) +from modelscope.models.multi_modal.videocomposer.unet_sd import UNetSD_temporal +from modelscope.models.multi_modal.videocomposer.utils.config import Config +from modelscope.models.multi_modal.videocomposer.utils.utils import ( + find_free_port, setup_seed, to_device) +from modelscope.outputs import OutputKeys +from modelscope.preprocessors.image import load_image +from modelscope.utils.constant import ModelFile, Tasks +from .config import cfg + +__all__ = ['VideoComposer'] + + +@MODELS.register_module( + Tasks.text_to_video_synthesis, module_name=Models.videocomposer) +class VideoComposer(TorchModel): + r""" + task for video composer. + + Attributes: + sd_model: denosing model using in this task. + diffusion: diffusion model for DDIM. + autoencoder: decode the latent representation into visual space with VQGAN. + clip_encoder: encode the text into text embedding. + """ + + def __init__(self, model_dir, *args, **kwargs): + r""" + Args: + model_dir (`str` or `os.PathLike`) + Can be either: + - A string, the *model id* of a pretrained model hosted inside a model repo on modelscope + or modelscope.cn. Valid model ids can be located at the root-level, like `bert-base-uncased`, + or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In + this case, `from_tf` should be set to `True` and a configuration object should be provided as + `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a + PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g, + `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to + `True`. + """ + super().__init__(model_dir=model_dir, *args, **kwargs) + self.device = torch.device('cuda') if torch.cuda.is_available() \ + else torch.device('cpu') + self.duration = kwargs.pop('duration', 200) + clip_checkpoint = kwargs.pop('clip_checkpoint', + 'open_clip_pytorch_model.bin') + sd_checkpoint = kwargs.pop('sd_checkpoint', 'v2-1_512-ema-pruned.ckpt') + cfg_file_name = kwargs.pop('cfg_file_name', + 'exp06_text_depths_vs_style.yaml') + _cfg = Config( + load=True, + cfg_dict=None, + cfg_level=None, + model_dir=model_dir, + cfg_file_name=cfg_file_name) + cfg.update(_cfg.cfg_dict) + + # rank-wise params + l1 = len(cfg.frame_lens) + l2 = len(cfg.feature_framerates) + cfg.max_frames = cfg.frame_lens[0 % (l1 * l2) // l2] + cfg.batch_size = cfg.batch_sizes[str(cfg.max_frames)] + # Copy update input parameter to current task + self.cfg = cfg + if 'MASTER_ADDR' not in os.environ: + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = find_free_port() + self.cfg.pmi_rank = int(os.getenv('RANK', 0)) + self.cfg.pmi_world_size = int(os.getenv('WORLD_SIZE', 1)) + setup_seed(self.cfg.seed) + self.read_image = kwargs.pop('read_image', False) + self.read_style = kwargs.pop('read_style', True) + self.read_sketch = kwargs.pop('read_sketch', False) + self.save_origin_video = kwargs.pop('save_origin_video', True) + self.video_compositions = kwargs.pop('video_compositions', [ + 'text', 'mask', 'depthmap', 'sketch', 'motion', 'image', + 'local_image', 'single_sketch' + ]) + self.viz_num = self.cfg.batch_size + self.clip_encoder = FrozenOpenCLIPEmbedder( + layer='penultimate', + pretrained=os.path.join(model_dir, clip_checkpoint)) + self.clip_encoder = self.clip_encoder.to(self.device) + self.clip_encoder_visual = FrozenOpenCLIPVisualEmbedder( + layer='penultimate', + pretrained=os.path.join(model_dir, clip_checkpoint)) + self.clip_encoder_visual.model.to(self.device) + ddconfig = { + 'double_z': True, + 'z_channels': 4, + 'resolution': 256, + 'in_channels': 3, + 'out_ch': 3, + 'ch': 128, + 'ch_mult': [1, 2, 4, 4], + 'num_res_blocks': 2, + 'attn_resolutions': [], + 'dropout': 0.0 + } + self.autoencoder = AutoencoderKL( + ddconfig, 4, ckpt_path=os.path.join(model_dir, sd_checkpoint)) + self.zero_y = self.clip_encoder('').detach() + black_image_feature = self.clip_encoder_visual( + self.clip_encoder_visual.black_image).unsqueeze(1) + black_image_feature = torch.zeros_like(black_image_feature) + self.autoencoder.eval() + for param in self.autoencoder.parameters(): + param.requires_grad = False + self.autoencoder.cuda() + self.model = UNetSD_temporal( + cfg=self.cfg, + in_dim=self.cfg.unet_in_dim, + concat_dim=self.cfg.unet_concat_dim, + dim=self.cfg.unet_dim, + y_dim=self.cfg.unet_y_dim, + context_dim=self.cfg.unet_context_dim, + out_dim=self.cfg.unet_out_dim, + dim_mult=self.cfg.unet_dim_mult, + num_heads=self.cfg.unet_num_heads, + head_dim=self.cfg.unet_head_dim, + num_res_blocks=self.cfg.unet_res_blocks, + attn_scales=self.cfg.unet_attn_scales, + dropout=self.cfg.unet_dropout, + temporal_attention=self.cfg.temporal_attention, + temporal_attn_times=self.cfg.temporal_attn_times, + use_checkpoint=self.cfg.use_checkpoint, + use_fps_condition=self.cfg.use_fps_condition, + use_sim_mask=self.cfg.use_sim_mask, + video_compositions=self.cfg.video_compositions, + misc_dropout=self.cfg.misc_dropout, + p_all_zero=self.cfg.p_all_zero, + p_all_keep=self.cfg.p_all_zero, + zero_y=self.zero_y, + black_image_feature=black_image_feature, + ).to(self.device) + + # Load checkpoint + if self.cfg.resume and self.cfg.resume_checkpoint: + if hasattr(self.cfg, 'text_to_video_pretrain' + ) and self.cfg.text_to_video_pretrain: + checkpoint_name = cfg.resume_checkpoint.split('/')[-1] + ss = torch.load( + os.path.join(self.model_dir, cfg.resume_checkpoint)) + ss = { + key: p + for key, p in ss.items() if 'input_blocks.0.0' not in key + } + self.model.load_state_dict(ss, strict=False) + else: + checkpoint_name = cfg.resume_checkpoint.split('/')[-1] + self.model.load_state_dict( + torch.load( + os.path.join(self.model_dir, checkpoint_name), + map_location='cpu'), + strict=False) + + torch.cuda.empty_cache() + else: + raise ValueError( + f'The checkpoint file {self.cfg.resume_checkpoint} is wrong ') + + # diffusion + betas = beta_schedule( + 'linear_sd', + self.cfg.num_timesteps, + init_beta=0.00085, + last_beta=0.0120) + self.diffusion = GaussianDiffusion( + betas=betas, + mean_type=self.cfg.mean_type, + var_type=self.cfg.var_type, + loss_type=self.cfg.loss_type, + rescale_timesteps=False) + + def forward(self, input: Dict[str, Any]): + frame_in = None + if self.read_image: + image_key = input['style_image'] + frame = load_image(image_key) + frame_in = misc_transforms([frame]) + + frame_sketch = None + if self.read_sketch: + sketch_key = self.cfg.sketch_path + frame_sketch = load_image(sketch_key) + frame_sketch = misc_transforms([frame_sketch]) + + frame_style = None + if self.read_style: + frame_style = load_image(input['style_image']) + + # Generators for various conditions + if 'depthmap' in self.video_compositions: + midas = models.midas_v3( + pretrained=True, + model_dir=self.model_dir).eval().requires_grad_(False).to( + memory_format=torch.channels_last).half().to(self.device) + if 'canny' in self.video_compositions: + canny_detector = CannyDetector() + if 'sketch' in self.video_compositions: + pidinet = pidinet_bsd( + self.model_dir, pretrained=True, + vanilla_cnn=True).eval().requires_grad_(False).to(self.device) + cleaner = sketch_simplification_gan( + self.model_dir, + pretrained=True).eval().requires_grad_(False).to(self.device) + pidi_mean = torch.tensor(self.cfg.sketch_mean).view( + 1, -1, 1, 1).to(self.device) + pidi_std = torch.tensor(self.cfg.sketch_std).view(1, -1, 1, 1).to( + self.device) + # Placeholder for color inference + palette = None + + self.model.eval() + caps = input['cap_txt'] + if self.cfg.max_frames == 1 and self.cfg.use_image_dataset: + ref_imgs = input['ref_frame'] + video_data = input['video_data'] + misc_data = input['misc_data'] + mask = input['mask'] + mv_data = input['mv_data'] + fps = torch.tensor( + [self.cfg.feature_framerate] * self.cfg.batch_size, + dtype=torch.long, + device=self.device) + else: + ref_imgs = input['ref_frame'] + video_data = input['video_data'] + misc_data = input['misc_data'] + mask = input['mask'] + mv_data = input['mv_data'] + # add fps test + fps = torch.tensor( + [self.cfg.feature_framerate] * self.cfg.batch_size, + dtype=torch.long, + device=self.device) + + # save for visualization + misc_backups = copy(misc_data) + misc_backups = rearrange(misc_backups, 'b f c h w -> b c f h w') + mv_data_video = [] + if 'motion' in self.cfg.video_compositions: + mv_data_video = rearrange(mv_data, 'b f c h w -> b c f h w') + + # mask images + masked_video = [] + if 'mask' in self.cfg.video_compositions: + masked_video = make_masked_images( + misc_data.sub(0.5).div_(0.5), mask) + masked_video = rearrange(masked_video, 'b f c h w -> b c f h w') + + # Single Image + image_local = [] + if 'local_image' in self.cfg.video_compositions: + frames_num = misc_data.shape[1] + bs_vd_local = misc_data.shape[0] + if self.cfg.read_image: + image_local = frame_in.unsqueeze(0).repeat( + bs_vd_local, frames_num, 1, 1, 1).cuda() + else: + image_local = misc_data[:, :1].clone().repeat( + 1, frames_num, 1, 1, 1) + image_local = rearrange( + image_local, 'b f c h w -> b c f h w', b=bs_vd_local) + + # encode the video_data + bs_vd = video_data.shape[0] + video_data = rearrange(video_data, 'b f c h w -> (b f) c h w') + misc_data = rearrange(misc_data, 'b f c h w -> (b f) c h w') + + video_data_list = torch.chunk( + video_data, video_data.shape[0] // self.cfg.chunk_size, dim=0) + misc_data_list = torch.chunk( + misc_data, misc_data.shape[0] // self.cfg.chunk_size, dim=0) + + with torch.no_grad(): + decode_data = [] + for vd_data in video_data_list: + encoder_posterior = self.autoencoder.encode(vd_data) + tmp = get_first_stage_encoding(encoder_posterior).detach() + decode_data.append(tmp) + video_data = torch.cat(decode_data, dim=0) + video_data = rearrange( + video_data, '(b f) c h w -> b c f h w', b=bs_vd) + + depth_data = [] + if 'depthmap' in self.cfg.video_compositions: + for misc_imgs in misc_data_list: + depth = midas( + misc_imgs.sub(0.5).div_(0.5).to( + memory_format=torch.channels_last).half()) + depth = (depth / self.cfg.depth_std).clamp_( + 0, self.cfg.depth_clamp) + depth_data.append(depth) + depth_data = torch.cat(depth_data, dim=0) + depth_data = rearrange( + depth_data, '(b f) c h w -> b c f h w', b=bs_vd) + + canny_data = [] + if 'canny' in self.cfg.video_compositions: + for misc_imgs in misc_data_list: + misc_imgs = rearrange(misc_imgs.clone(), + 'k c h w -> k h w c') + canny_condition = torch.stack( + [canny_detector(misc_img) for misc_img in misc_imgs]) + canny_condition = rearrange(canny_condition, + 'k h w c-> k c h w') + canny_data.append(canny_condition) + canny_data = torch.cat(canny_data, dim=0) + canny_data = rearrange( + canny_data, '(b f) c h w -> b c f h w', b=bs_vd) + + sketch_data = [] + if 'sketch' in self.cfg.video_compositions: + sketch_list = misc_data_list + if self.cfg.read_sketch: + sketch_repeat = frame_sketch.repeat(frames_num, 1, 1, + 1).cuda() + sketch_list = [sketch_repeat] + + for misc_imgs in sketch_list: + sketch = pidinet(misc_imgs.sub(pidi_mean).div_(pidi_std)) + sketch = 1.0 - cleaner(1.0 - sketch) + sketch_data.append(sketch) + sketch_data = torch.cat(sketch_data, dim=0) + sketch_data = rearrange( + sketch_data, '(b f) c h w -> b c f h w', b=bs_vd) + + single_sketch_data = [] + if 'single_sketch' in self.cfg.video_compositions: + single_sketch_data = sketch_data.clone()[:, :, :1].repeat( + 1, 1, frames_num, 1, 1) + + # preprocess for input text descripts + y = self.clip_encoder(caps).detach() + y0 = y.clone() + + y_visual = [] + if 'image' in self.cfg.video_compositions: + with torch.no_grad(): + if self.cfg.read_style: + y_visual = self.clip_encoder_visual( + self.clip_encoder_visual.preprocess( + frame_style).unsqueeze(0).cuda()).unsqueeze(0) + y_visual0 = y_visual.clone() + else: + ref_imgs = ref_imgs.squeeze(1) + y_visual = self.clip_encoder_visual(ref_imgs).unsqueeze(1) + y_visual0 = y_visual.clone() + + with torch.no_grad(): + # Log memory + pynvml.nvmlInit() + # Sample images (DDIM) + with amp.autocast(enabled=self.cfg.use_fp16): + if self.cfg.share_noise: + b, c, f, h, w = video_data.shape + noise = torch.randn((self.viz_num, c, h, w), + device=self.device) + noise = noise.repeat_interleave(repeats=f, dim=0) + noise = rearrange( + noise, '(b f) c h w->b c f h w', b=self.viz_num) + noise = noise.contiguous() + else: + noise = torch.randn_like(video_data[:self.viz_num]) + + full_model_kwargs = [{ + 'y': + y0[:self.viz_num], + 'local_image': + None + if len(image_local) == 0 else image_local[:self.viz_num], + 'image': + None if len(y_visual) == 0 else y_visual0[:self.viz_num], + 'depth': + None + if len(depth_data) == 0 else depth_data[:self.viz_num], + 'canny': + None + if len(canny_data) == 0 else canny_data[:self.viz_num], + 'sketch': + None + if len(sketch_data) == 0 else sketch_data[:self.viz_num], + 'masked': + None + if len(masked_video) == 0 else masked_video[:self.viz_num], + 'motion': + None if len(mv_data_video) == 0 else + mv_data_video[:self.viz_num], + 'single_sketch': + None if len(single_sketch_data) == 0 else + single_sketch_data[:self.viz_num], + 'fps': + fps[:self.viz_num] + }, { + 'y': + self.zero_y.repeat(self.viz_num, 1, 1) + if not self.cfg.use_fps_condition else + torch.zeros_like(y0)[:self.viz_num], + 'local_image': + None + if len(image_local) == 0 else image_local[:self.viz_num], + 'image': + None if len(y_visual) == 0 else torch.zeros_like( + y_visual0[:self.viz_num]), + 'depth': + None + if len(depth_data) == 0 else depth_data[:self.viz_num], + 'canny': + None + if len(canny_data) == 0 else canny_data[:self.viz_num], + 'sketch': + None + if len(sketch_data) == 0 else sketch_data[:self.viz_num], + 'masked': + None + if len(masked_video) == 0 else masked_video[:self.viz_num], + 'motion': + None if len(mv_data_video) == 0 else + mv_data_video[:self.viz_num], + 'single_sketch': + None if len(single_sketch_data) == 0 else + single_sketch_data[:self.viz_num], + 'fps': + fps[:self.viz_num] + }] + + # Save generated videos + partial_keys = self.cfg.guidances + noise_motion = noise.clone() + model_kwargs = prepare_model_kwargs( + partial_keys=partial_keys, + full_model_kwargs=full_model_kwargs, + use_fps_condition=self.cfg.use_fps_condition) + video_output = self.diffusion.ddim_sample_loop( + noise=noise_motion, + model=self.model.eval(), + model_kwargs=model_kwargs, + guide_scale=9.0, + ddim_timesteps=self.cfg.ddim_timesteps, + eta=0.0) + + save_with_model_kwargs( + model_kwargs=model_kwargs, + video_data=video_output, + autoencoder=self.autoencoder, + ori_video=misc_backups, + viz_num=self.viz_num, + step=0, + caps=caps, + palette=palette, + cfg=self.cfg, + duration=self.duration) + + return { + 'video': video_output.type(torch.float32).cpu(), + 'video_path': self.cfg + } diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/__init__.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a231402a37529d357e7eec452178e1ed8d28ff8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .model import VLDocForDocVLEmbedding diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/conv_fpn_trans.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/conv_fpn_trans.py new file mode 100644 index 0000000000000000000000000000000000000000..65e27ab574cdb89f25998cc9630a19c4fd026868 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/conv_fpn_trans.py @@ -0,0 +1,293 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import random +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath, trunc_normal_ + +from modelscope.models.multi_modal.vldoc.convnext import convnext_tiny +from modelscope.utils.logger import get_logger + +try: + import apex + import apex.normalization + LN = apex.normalization.FusedLayerNorm +except ImportError: + LN = torch.nn.LayerNorm + +logging = get_logger() + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, + d_model: int, + n_head: int, + attn_mask: torch.Tensor = None, + expand_ratio=4.0, + init_values: float = None): + """ + The implementation of the transformer block refers to: + https://github.com/openai/CLIP/blob/b46f5ac7587d2e1862f8b7b1573179d80dcdd620/clip/model.py + """ + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LN(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * expand_ratio)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * expand_ratio, + d_model))])) + self.ln_2 = LN(d_model) + self.attn_mask = attn_mask + if init_values is not None: + self.gamma_1 = nn.Parameter( + init_values * torch.ones((d_model)), requires_grad=True) + self.gamma_2 = nn.Parameter( + init_values * torch.ones((d_model)), requires_grad=True) + else: + self.gamma_1, self.gamma_2 = 1.0, 1.0 + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to( + dtype=x.dtype, + device=x.device) if self.attn_mask is not None else None + return self.attn( + x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.gamma_1 * self.attention(self.ln_1(x)) + x = x + self.gamma_2 * self.mlp(self.ln_2(x)) + return x + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +def drop_grid(grid_map, drop_range=(0.3, 0.8), training=False): + """ + only drop in the training phase. + grid_map: [N, D, T1, ...] + """ + if training: + drop_ratio = random.random() * (drop_range[1] + - drop_range[0]) + drop_range[0] + # [N, T1, ...], True will be dropped + mask = (torch.rand_like(grid_map[:, 0]) < drop_ratio).bool() + grid_map = grid_map.masked_fill(mask.unsqueeze(1), 0.0) + return grid_map + + +class GumbelSample(nn.Module): + + def __init__(self, in_dim, num_keep): + super(GumbelSample, self).__init__() + self.keep_layer = nn.Sequential( + nn.Conv2d( + in_dim, 256, 3, stride=1, padding=3, dilation=3, bias=False), + nn.BatchNorm2d(256), + nn.ReLU(inplace=True), + nn.Conv2d( + 256, 256, 3, stride=1, padding=2, dilation=2, bias=False), + nn.BatchNorm2d(256), + nn.ReLU(inplace=True), + nn.Conv2d(256, 1, 3, stride=1, padding=1, dilation=1), + nn.Sigmoid(), + ) + self.num_keep = num_keep + self.diffusion = nn.Conv2d(in_dim, in_dim, 3, padding=1) + self.dropout = nn.Dropout(0.1) + + def forward(self, x, tau=1): + """ + x: [N, C, H, W] + """ + N = x.size(0) + keep_score = self.keep_layer(x) + keep_score = torch.clamp(keep_score, min=0.0, max=1.0) + keep_score = torch.cat([keep_score, 1 - keep_score], + dim=1) + 1e-5 # [N, 2, H, W] + gumbel_score = F.gumbel_softmax( + keep_score.log(), tau=tau, hard=False, dim=1) + # differentiable hard mode + index = gumbel_score.max(dim=1, keepdim=True)[1] + gumbel_hard = torch.zeros_like( + gumbel_score, + memory_format=torch.legacy_contiguous_format).scatter_( + 1, index, 1.0) + gumbel_hard = gumbel_hard - gumbel_score.detach() + gumbel_score + # + gumbel_score = gumbel_score[:, 0].contiguous().view(N, + -1) # [N, H x W] + gumbel_hard = gumbel_hard[:, 0].contiguous().view(N, -1) + # sort by score + idx_true = torch.topk( + gumbel_score, self.num_keep, dim=1)[1] # [N, num_keep] + topk_mask = torch.zeros_like(gumbel_score).bool().fill_( + False).scatter_(1, idx_true, True) # [N, H x W] + return topk_mask, gumbel_hard, keep_score[:, 0] + + def sample(self, x, topk_mask, gumbel_hard): + N, D, H, W = x.size() + x = x.contiguous().view(N, D, -1) # [N, D, HxW] + x = x * gumbel_hard.unsqueeze(1) + x = x.transpose(1, 2) # [N, HxW, D] + x = x[topk_mask].contiguous().view(N, -1, D) # [N, num_keep, D] + x = drop_grid( + x.transpose(1, 2), drop_range=(0.0, 0.2), + training=self.training).transpose(1, 2) + return x + + def random_sample(self, x): + N, D, H, W = x.size() + x = x.contiguous().view(N, D, -1) # [N, D, HxW] + x = x.transpose(1, 2) # [N, HxW, D] + # generate random mask + idx_true = torch.topk( + torch.rand_like(x[:, :, 0]), self.num_keep, + dim=1)[1] # [N, num_keep] + topk_mask = torch.zeros_like(x[:, :, 0]).bool().fill_(False).scatter_( + 1, idx_true, True) # [N, H x W] + # apply the mask + x = x[topk_mask].contiguous().view(N, -1, D) # [N, num_keep, D] + x = drop_grid( + x.transpose(1, 2), drop_range=(0.0, 0.2), + training=self.training).transpose(1, 2) + return x, topk_mask + + def restore(self, x, topk_mask, src): + """ + x: [N, D, H, W] + topk_mask: [N, HxW] + src: [N, num_keep, D] + """ + N, D, H, W = x.size() + x = drop_grid(x, drop_range=(0.2, 0.8), training=self.training) + x = x.contiguous().view(N, D, -1).transpose(1, 2) # [N, HxW, D] + x = x.masked_scatter(topk_mask.unsqueeze(-1), src) + x = x.transpose(1, 2).contiguous().view(N, D, H, W) + x = self.dropout(self.diffusion(x)) + return x + + +class FPNTrans(nn.Module): + + def __init__(self, + trans_layers=2, + inner_channels=256, + img_size=(896, 896), + inner_vit=False, + out_sampling=False): + super(FPNTrans, self).__init__() + self.cnn = convnext_tiny(pretrained=True, in_22k=True) + self.dims = self.cnn.dims + self.img_size = img_size + # FPN in DB + self.up5 = nn.Upsample(scale_factor=2, mode='nearest') + self.up4 = nn.Upsample(scale_factor=2, mode='nearest') + self.up3 = nn.Upsample(scale_factor=2, mode='nearest') + + self.in5 = nn.Conv2d(self.dims[-1], inner_channels, 1, bias=False) + self.in4 = nn.Conv2d(self.dims[-2], inner_channels, 1, bias=False) + self.in3 = nn.Conv2d(self.dims[-3], inner_channels, 1, bias=False) + self.in2 = nn.Conv2d(self.dims[-4], inner_channels, 1, bias=False) + + self.out5 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=False), + nn.Upsample(scale_factor=8, mode='nearest')) + self.out4 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=False), + nn.Upsample(scale_factor=4, mode='nearest')) + self.out3 = nn.Sequential( + nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=False), + nn.Upsample(scale_factor=2, mode='nearest')) + self.out2 = nn.Conv2d( + inner_channels, inner_channels // 4, 3, padding=1, bias=False) + self.inner_vit = inner_vit + if inner_vit: + # mini vit + self.num_keep1 = (self.img_size[0] // 64)**2 + self.gumble_sample1 = GumbelSample( + inner_channels, num_keep=self.num_keep1) + self.pos_emb1 = nn.Parameter( + torch.randn(inner_channels, self.img_size[0] // 32, + self.img_size[1] // 32)) + trunc_normal_(self.pos_emb1, std=.02) + self.mini_vit = nn.Sequential(*[ + ResidualAttentionBlock( + inner_channels, 4, expand_ratio=2, init_values=0.1) + for _ in range(trans_layers) + ]) + self.dropout_pos = nn.Dropout(0.1) + if out_sampling: + # sample for co-attention + self.num_keep2 = (self.img_size[0] // 64)**2 + self.gumble_sample2 = GumbelSample( + inner_channels, num_keep=self.num_keep2) + self.pos_emb2 = nn.Parameter( + torch.randn(inner_channels, self.img_size[0] // 4, + self.img_size[1] // 4)) + trunc_normal_(self.pos_emb2, std=.02) + self.out_sampling = out_sampling + + self.drop_path = DropPath(0.1) + + def forward(self, x): + ms_features = self.cnn(x) + c2, c3, c4, c5 = ms_features + in5 = self.in5(c5) + in4 = self.in4(c4) + in3 = self.in3(c3) + in2 = self.in2(c2) + N, D5, H5, W5 = in5.size() + if self.inner_vit: + # random sample + keep_score = None + in5_pos = self.dropout_pos(in5 + self.pos_emb1.unsqueeze(0)) + in5_pos_in, topk_mask = self.gumble_sample1.random_sample(in5_pos) + in5_pos_in = in5_pos_in.transpose(0, 1) # [num_keep1, N, D5] + in5_pos_out = self.mini_vit(in5_pos_in).permute( + 1, 2, 0) # [N, D5, num_keep1] + in5 = self.gumble_sample1.restore(in5, topk_mask, + in5_pos_out.transpose(1, 2)) + else: + keep_score = None + # FPN for fused multi-scale visual feature + out4 = self.up5(in5) + self.drop_path(in4) + out3 = self.up4(out4) + self.drop_path(in3) + out2 = self.up3(out3) + self.drop_path(in2) + p5 = self.out5(in5) + p4 = self.out4(out4) + p3 = self.out3(out3) + p2 = self.out2(out2) + feat_ms = torch.cat((p5, p4, p3, p2), 1) + ret_dict = dict( + feat_ms=feat_ms, + keep_score=keep_score, + ) + if self.out_sampling: + # gumbel sampling + topk_mask2, gumbel_hard2, keep_score2 = self.gumble_sample2( + feat_ms) + feat_ms_pos = self.dropout_pos(feat_ms + + self.pos_emb2.unsqueeze(0)) + feat_ms_pos_sampled = self.gumble_sample2.sample( + feat_ms_pos, topk_mask2, + gumbel_hard2).transpose(0, 1) # [num_keep2, N, inner_c] + ret_dict_sup = dict( + feat_ms_pos_sampled=feat_ms_pos_sampled, + sampler=self.gumble_sample2, + keep_score2=keep_score2, + ) + ret_dict.update(ret_dict_sup) + return ret_dict diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/convnext.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/convnext.py new file mode 100644 index 0000000000000000000000000000000000000000..3cf7cd63463a32c32438ae0f366a690888c6cdc8 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/convnext.py @@ -0,0 +1,168 @@ +# The implementation is borrowed and partly modified from ConvNext, +# made publicly available under the MIT License at https://github.com/facebookresearch/ConvNeXt. + +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath, trunc_normal_ +from timm.models.registry import register_model + + +class Block(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + self.dwconv = nn.Conv2d( + dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv + self.norm = LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, + 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter( + layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) + + x = input + self.drop_path(x) + return x + + +class ConvNeXt(nn.Module): + r""" ConvNeXt + A PyTorch impl of : `A ConvNet for the 2020s` - + https://arxiv.org/pdf/2201.03545.pdf + Args: + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3] + dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768] + drop_path_rate (float): Stochastic depth rate. Default: 0. + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1. + """ + + def __init__( + self, + in_chans=3, + depths=[3, 3, 9, 3], + dims=[96, 192, 384, 768], + drop_path_rate=0., + layer_scale_init_value=1e-6, + ): + super().__init__() + + self.downsample_layers = nn.ModuleList( + ) # stem and 3 intermediate downsampling conv layers + stem = nn.Sequential( + nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4), + LayerNorm(dims[0], eps=1e-6, data_format='channels_first')) + self.downsample_layers.append(stem) + for i in range(3): + downsample_layer = nn.Sequential( + LayerNorm(dims[i], eps=1e-6, data_format='channels_first'), + nn.Conv2d(dims[i], dims[i + 1], kernel_size=2, stride=2), + ) + self.downsample_layers.append(downsample_layer) + + self.stages = nn.ModuleList( + ) # 4 feature resolution stages, each consisting of multiple residual blocks + dp_rates = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] + cur = 0 + for i in range(4): + stage = nn.Sequential(*[ + Block( + dim=dims[i], + drop_path=dp_rates[cur + j], + layer_scale_init_value=layer_scale_init_value) + for j in range(depths[i]) + ]) + self.stages.append(stage) + cur += depths[i] + + # self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer + self.dims = dims + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, (nn.Conv2d, nn.Linear)): + trunc_normal_(m.weight, std=.02) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + xs = [] + for i in range(4): + x = self.downsample_layers[i](x) + x = self.stages[i](x) + xs.append(x) + # x = x.permute(0, 2, 3, 1) # [N, H, W, C] + # x = self.norm(x) + # x = x.permute(0, 3, 1, 2) # [N, C, H, W] + return tuple(xs) + + +class LayerNorm(nn.Module): + r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with + shape (batch_size, height, width, channels) while channels_first corresponds to inputs + with shape (batch_size, channels, height, width). + """ + + def __init__(self, + normalized_shape, + eps=1e-6, + data_format='channels_last'): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ['channels_last', 'channels_first']: + raise NotImplementedError + self.normalized_shape = (normalized_shape, ) + + def forward(self, x): + if self.data_format == 'channels_last': + return F.layer_norm(x, self.normalized_shape, self.weight, + self.bias, self.eps) + elif self.data_format == 'channels_first': + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + + +@register_model +def convnext_tiny(pretrained=False, in_22k=False, **kwargs): + model = ConvNeXt(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs) + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/model.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5b21bf102594837b33e355f1fbd55e7a36af7df6 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/model.py @@ -0,0 +1,433 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import copy +import logging +import math +import os +import re +import sys + +import json +import torch +import torch.distributed as dist +import torch.nn as nn +from torchvision.ops import roi_align + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.vldoc.conv_fpn_trans import FPNTrans +from modelscope.models.multi_modal.vldoc.modeling_layout_roberta import ( + LayoutRobertaModel, LayoutRobertaPreTrainedModel) +from modelscope.models.multi_modal.vldoc.transformer_local import ( + TransformerDecoder, TransformerDecoderLayer) +from modelscope.utils.constant import ModeKeys, ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['VLDocForDocVLEmbedding'] + + +class GeoVLDocModelOutputs(object): + + def __init__( + self, + text_features, + text_mm_features, + block_vis_features, + block_vis_mm_features, + image_mm_features, + ): + # [batch size, sequence length, hidden size] + self.text_features = text_features + # [batch size, sequence length, hidden size] + self.text_mm_features = text_mm_features + # [batch size, block num, hidden size] + self.block_vis_features = block_vis_features + # [batch size, block num, hidden size] + self.block_vis_mm_features = block_vis_mm_features + # [batch size, hidden size] + self.image_mm_features = image_mm_features + + +class GeoVLDocModel(LayoutRobertaPreTrainedModel): + + def __init__(self, config, hard_negtive_sampling=False): + super().__init__(config) + self.config = config + self.hard_negtive_sampling = hard_negtive_sampling + + if getattr(self.config, 'architectures', None): + if self.config.architectures[0] == 'LayoutRobertaModel': + self.text_encoder = LayoutRobertaModel(config) + else: + self.text_encoder = LayoutRobertaModel(config) + else: + self.text_encoder = LayoutRobertaModel(config) + self.visual_encoder = FPNTrans( + img_size=self.config.image_size, inner_vit=False) + self.pool = nn.AdaptiveAvgPool2d([1, 1]) + self.vis_linear = nn.Linear(256, self.config.hidden_size) + + cross_modal_text_layer = TransformerDecoderLayer( + self.config.hidden_size, + self.config.num_attention_heads, + self.config.intermediate_size, + self_attn=True) + self.cross_modal_text = TransformerDecoder(cross_modal_text_layer, 1) + + cross_modal_visual_layer = TransformerDecoderLayer( + self.config.hidden_size, + self.config.num_attention_heads, + self.config.intermediate_size, + self_attn=True) + self.cross_modal_visual = TransformerDecoder(cross_modal_visual_layer, + 1) + + self.init_weights() + + def from_pretrained(self, ckpt_path: str): + state_dict = torch.load(ckpt_path, map_location='cpu') + state_dict_new = {} + for k, v in state_dict.items(): + k = k.replace('geo_vl_doc_model.', '') + state_dict_new[k] = v + self.load_state_dict(state_dict_new) + + def forward(self, + input_ids=None, + image=None, + bbox=None, + bbox_4p_normalized=None, + attention_mask=None, + first_token_idxes=None, + first_token_idxes_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + + batch_size, seq_len = input_ids.shape + + return_dict = ( + return_dict + if return_dict is not None else self.config.use_return_dict) + + kwargs['line_bbox'] = bbox + # ################ get text representation ################ + if self.config.architectures[0] == 'LayoutRobertaModel': + outputs = self.text_encoder( + input_ids, + bbox=bbox_4p_normalized, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + **kwargs) + else: + outputs = self.text_encoder( + input_ids, + bbox=bbox_4p_normalized, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + **kwargs) + + # sequence_output: [batch_size, seq_len, hidden_size] + # pooled_output: [batch_size, hidden_size] + sequence_output, pooled_output = outputs[:2] + + # ################ get visual representation ################ + _, num_first = first_token_idxes.shape + B_batch_dim = torch.arange( + 0, batch_size, + device=input_ids.device).reshape(batch_size, + 1).expand(batch_size, num_first) + + feature_bbox = bbox[B_batch_dim, first_token_idxes] + _, block_num, _ = feature_bbox.shape + + visual_out = self.visual_encoder(image) + batch_idxs = torch.arange( + 0, batch_size, device=sequence_output.device).reshape( + batch_size, 1).expand(batch_size, block_num).unsqueeze(-1) + + # [batch_size*block_num, 5] + batch_idx_with_bbox = torch.cat( + (batch_idxs, feature_bbox), + 2).reshape(batch_size * block_num, + 5).to(dtype=visual_out['feat_ms'].dtype) + + if visual_out['feat_ms'].dtype == torch.float16: + # [batch_size*block_num, 256, 1, 1] + blk_vis_features = roi_align( + visual_out['feat_ms'].to(torch.float32), + batch_idx_with_bbox.to(torch.float32), + 1, + spatial_scale=visual_out['feat_ms'].size(-1) / 1000.0) + blk_vis_features = blk_vis_features.to( + dtype=visual_out['feat_ms'].dtype) + else: + blk_vis_features = roi_align( + visual_out['feat_ms'], + batch_idx_with_bbox.to(torch.float32), + 1, + spatial_scale=visual_out['feat_ms'].size(-1) / 1000.0) + + # [batch_size*block_num, 256] + blk_vis_features = blk_vis_features.squeeze(2).squeeze(2).reshape( + batch_size, block_num, 256) + + # visual block features: + # blk_vis_features: [batch_size, block_num, hidden_size] + blk_vis_features = self.vis_linear(blk_vis_features) + blk_vis_features = blk_vis_features * first_token_idxes_mask.unsqueeze( + 2) + # [batch_size, 256] + full_img_features = self.pool( + visual_out['feat_ms']).squeeze(2).squeeze(2) + # [batch_size, hidden_size] + full_img_features = self.vis_linear(full_img_features).unsqueeze(1) + + # ################ multi-modal fusion ################ + + # cross attention inputs + vis_inps = torch.cat((full_img_features, blk_vis_features), 1) + + glb_feat_attn = torch.ones((batch_size, 1)).to(input_ids.device) + + vis_mask = torch.cat((glb_feat_attn, first_token_idxes_mask), 1) + + # When we use transformer in torch.nn, the input size is + # [seq_len, batch_size, hidden_size] + # In attention_mask, 1 denotes masked + new_attention_mask = (1 - attention_mask) > 0 + new_vis_mask = (1 - vis_mask) > 0 + + text_mm_feat = self.cross_modal_text( + tgt=sequence_output.transpose(0, 1), + memory=vis_inps.transpose(0, 1), + tgt_key_padding_mask=new_attention_mask, + memory_key_padding_mask=new_vis_mask) + + vis_mm_feat = self.cross_modal_visual( + tgt=vis_inps.transpose(0, 1), + memory=sequence_output.transpose(0, 1), + tgt_key_padding_mask=new_vis_mask, + memory_key_padding_mask=new_attention_mask, + ) + + # [batch_size, seq_len, hidden_size] + text_mm_feat = text_mm_feat.transpose(0, 1) + # [batch_size, 1+block_num, hidden_size] + vis_mm_feat = vis_mm_feat.transpose(0, 1) + + # image_mm_features = vis_mm_feat[:, 0, :] + block_vis_mm_features = vis_mm_feat[:, 1:] + + return GeoVLDocModelOutputs( + text_features=sequence_output, + text_mm_features=text_mm_feat, + block_vis_features=blk_vis_features, + block_vis_mm_features=block_vis_mm_features, + image_mm_features=vis_mm_feat, + ) + + +@MODELS.register_module(Tasks.document_vl_embedding, module_name=Models.vldoc) +class VLDocForDocVLEmbedding(TorchModel): + """ + Generate multi-modal document embeddings in segment-level and token-level. + + Args: + model_dir: + the path in model hub, e.g., 'damo/multi-modal_convnext-roberta-base_vldoc-embedding' + """ + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir=model_dir, *args, **kwargs) + + # Initialize the model. + from modelscope.models.multi_modal.vldoc.modeling_layout_roberta import LayoutRobertaConfig + model_cfg_path = os.path.join(model_dir, 'config.json') + logger.info('Loading config file from {}'.format(model_cfg_path)) + assert os.path.exists(model_cfg_path) + self.config = LayoutRobertaConfig.from_json_file(model_cfg_path) + self.doc_model = GeoVLDocModel(self.config) + + # restore the pretrained weight + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + assert os.path.exists(model_path) + self.doc_model.from_pretrained(model_path) + logger.info('Loading model from {}'.format(model_path)) + + # Initialize the tokenizer. + from modelscope.models.multi_modal.vldoc.tokenization import VLDocXLMTokenizer + tokenizer_path = os.path.join(model_dir, ModelFile.TOKENIZER_FOLDER) + self.tokenizer = VLDocXLMTokenizer.from_pretrained(tokenizer_path) + + # place the model + self.device = 'cuda:{}'.format(int(os.environ.get( + 'LOCAL_RANK', 0))) if torch.cuda.is_available() else 'cpu' + if torch.cuda.is_available(): + self.doc_model.to(self.device) + logger.info('Use GPU {} for finetuning & inference'.format( + int(os.environ.get('LOCAL_RANK', 0)))) + else: + self.doc_model.float() + logger.info('Use CPU for finetuning & inference') + + def forward(self, + input_ids=None, + image=None, + bbox=None, + bbox_4p_normalized=None, + attention_mask=None, + first_token_idxes=None, + first_token_idxes_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + """ + Args: + - input_ids: :math:`(B, T, E)`, the input tokens, where B is the batch size, + T is the max token size, E is the embedding dimension. + - image: :math:`(B, C, H, W)`, normalized images. + - bbox: :math:`(B, T, 4)`, segment boxes denoted by top-left and bottom-right + vertexes whose values are normalized to [0, 1000). + - bbox_4p_normalized: :math:`(B, T, 8)`, word boxes denoted by 4 vertexes, whose + values are normalized to [0, 1). + - attention_mask: :math:`(B, T)`, mask for input tokens, where 0 means masked. + - first_token_idxes: :math:`(B, S)`, indexes of the corresponding first tokens + of all segments, where S is the max segment size. + - first_token_idxes_mask: :math:`(B, S)`, mask for segments, where 0 means masked. + Optional: + - line_rank_id: :math:`(B, T)`, orders of segments. + - line_rank_inner_id: :math:`(B, T)`, BIE-like tags. + + To be more specific, please refer to the class `TextLayoutSerializer` in + `modelscope/models/multi_modal/vldoc/processing.py`. + """ + + vldoc_outputs = self.doc_model( + input_ids=input_ids, + image=image, + bbox=bbox, + bbox_4p_normalized=bbox_4p_normalized, + attention_mask=attention_mask, + first_token_idxes=first_token_idxes, + first_token_idxes_mask=first_token_idxes_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + **kwargs) + + return dict( + img_embedding=vldoc_outputs.image_mm_features, + text_embedding=vldoc_outputs.text_mm_features, + ) + + +def init_pretrained_weight( + model, + pretrained_model_path, + state_dict=None, + cache_dir=None, + init_backbone='roberta', +): + if state_dict is None: + state_dict = torch.load(pretrained_model_path, map_location='cpu') + + old_keys = [] + new_keys = [] + state_dict_keys = list(state_dict.keys()) + + if init_backbone == 'roberta': + for i in range(len(state_dict_keys)): + key = state_dict_keys[i] + new_key = None + + if key.startswith('roberta.'): + new_key = key.replace('roberta.', + 'geo_vl_doc_model.text_encoder.') + key = copy.deepcopy(new_key) + + if new_key: + old_keys.append(state_dict_keys[i]) + new_keys.append(new_key) + + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + missing_keys, unexpected_keys, error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + start_prefix = '' + if not hasattr(model, 'geo_vl_doc_model') and any( + s.startswith('geo_vl_doc_model.') for s in state_dict.keys()): + start_prefix = 'geo_vl_doc_model.' + load(model, prefix=start_prefix) + if len(missing_keys) > 0: + logger.info( + 'Weights of {} not initialized from pretrained model: {}'.format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info('Weights from pretrained model not used in {}: {}'.format( + model.__class__.__name__, unexpected_keys)) + if len(error_msgs) > 0: + raise RuntimeError( + 'Error(s) in loading state_dict for {}:\n\t{}'.format( + model.__class__.__name__, '\n\t'.join(error_msgs))) + + return model diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/modeling_layout_roberta.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/modeling_layout_roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..c47294e3d2bdf453cd8e6ab39d0a56b773fb91a7 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/modeling_layout_roberta.py @@ -0,0 +1,1140 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright 2021-2022 The Alibaba DAMO Duguang Team Authors. All rights reserved. + +import math +import os + +import torch +import torch.utils.checkpoint +from packaging import version +from torch import nn +from transformers.activations import ACT2FN, gelu +from transformers.configuration_utils import PretrainedConfig +from transformers.file_utils import (add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings) +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions, MaskedLMOutput, + MultipleChoiceModelOutput, QuestionAnsweringModelOutput, + SequenceClassifierOutput, TokenClassifierOutput) +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.utils import logging + +logger = logging.get_logger() + + +class LayoutRobertaConfig(PretrainedConfig): + model_type = 'layoutroberta' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + bbox_scale=100.0, + pe_type='crel', + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + **kwargs): + super().__init__( + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + layer_norm_eps=layer_norm_eps, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + **kwargs, + ) + + self.bbox_scale = bbox_scale + self.pe_type = pe_type + + +class PositionalEmbedding1D(nn.Module): + # Reference: + # https://github.com/kimiyoung/transformer-xl/blob/master/pytorch/mem_transformer.py#L15 + + def __init__(self, demb): + super(PositionalEmbedding1D, self).__init__() + + self.demb = demb + + inv_freq = 1 / (10000**(torch.arange(0.0, demb, 2.0) / demb)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, pos_seq, bsz=None): + seq_size = pos_seq.size() + + if len(seq_size) == 2: + b1, b2 = seq_size + sinusoid_inp = pos_seq.view(b1, b2, 1) * self.inv_freq.view( + 1, 1, self.demb // 2) + elif len(seq_size) == 3: + b1, b2, b3 = seq_size + sinusoid_inp = pos_seq.view(b1, b2, b3, 1) * self.inv_freq.view( + 1, 1, 1, self.demb // 2) + else: + raise ValueError(f'Invalid seq_size={len(seq_size)}') + + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + + return pos_emb + + +class PositionalEmbedding2D(nn.Module): + + def __init__(self, demb, dim_bbox=8): + super(PositionalEmbedding2D, self).__init__() + + self.demb = demb + self.dim_bbox = dim_bbox + + self.x_pos_emb = PositionalEmbedding1D(demb // dim_bbox) + self.y_pos_emb = PositionalEmbedding1D(demb // dim_bbox) + + inv_freq = 1 / (10000**(torch.arange(0.0, demb, 2.0) / demb)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, bbox): + # bbox: [seq_length, batch_size, dim_bbox] + stack = [] + for i in range(self.dim_bbox): + if i % 2 == 0: + stack.append(self.x_pos_emb(bbox[..., i])) + else: + stack.append(self.y_pos_emb(bbox[..., i])) + bbox_pos_emb = torch.cat(stack, dim=-1) + return bbox_pos_emb + + +class LayoutRobertaEmbeddings(nn.Module): + """ + Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. + """ + + # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__ + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # layout-related embeddings + self.line_rank_embeddings = nn.Embedding( + config.max_position_embeddings, config.hidden_size) + + self.line_rank_inner_embeddings = nn.Embedding(4, config.hidden_size) + + self.x_position_embeddings = nn.Embedding( + config.max_2d_position_embeddings, config.coordinate_size) + self.y_position_embeddings = nn.Embedding( + config.max_2d_position_embeddings, config.coordinate_size) + self.h_position_embeddings = nn.Embedding( + config.max_2d_position_embeddings, config.shape_size) + self.w_position_embeddings = nn.Embedding( + config.max_2d_position_embeddings, config.shape_size) + + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + + if version.parse(torch.__version__) > version.parse('1.6.0'): + self.register_buffer( + 'token_type_ids', + torch.zeros(self.position_ids.size(), dtype=torch.long), + persistent=False, + ) + + if config.pe_type == 'pdpdq_ws': + dim_bbox_sinusoid_emb = config.hidden_size + dim_bbox_projection = config.hidden_size + elif config.pe_type == 'crel': + dim_bbox_sinusoid_emb = config.hidden_size // 4 + dim_bbox_projection = config.hidden_size // config.num_attention_heads + else: + raise ValueError(f'Unknown config.pe_type={config.pe_type}') + + self.bbox_sinusoid_emb = PositionalEmbedding2D( + dim_bbox_sinusoid_emb, dim_bbox=8) + self.bbox_projection = nn.Linear( + dim_bbox_sinusoid_emb, dim_bbox_projection, bias=False) + + # End copy + self.padding_idx = config.pad_token_id + self.position_embeddings = nn.Embedding( + config.max_position_embeddings, + config.hidden_size, + padding_idx=self.padding_idx) + + def _cal_spatial_position_embeddings(self, bbox): + try: + left_position_embeddings = self.x_position_embeddings(bbox[:, :, + 0]) + upper_position_embeddings = self.y_position_embeddings(bbox[:, :, + 1]) + right_position_embeddings = self.x_position_embeddings(bbox[:, :, + 2]) + lower_position_embeddings = self.y_position_embeddings(bbox[:, :, + 3]) + except IndexError as e: + raise IndexError( + 'The :obj:`bbox`coordinate values should be within 0-1000 range.' + ) from e + + h_position_embeddings = self.h_position_embeddings(bbox[:, :, 3] + - bbox[:, :, 1]) + w_position_embeddings = self.w_position_embeddings(bbox[:, :, 2] + - bbox[:, :, 0]) + + spatial_position_embeddings = torch.cat( + [ + left_position_embeddings, + upper_position_embeddings, + right_position_embeddings, + lower_position_embeddings, + h_position_embeddings, + w_position_embeddings, + ], + dim=-1, + ) + return spatial_position_embeddings + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + **kwargs): + if position_ids is None: + if input_ids is not None: + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = create_position_ids_from_input_ids( + input_ids, self.padding_idx, past_key_values_length) + else: + position_ids = self.create_position_ids_from_inputs_embeds( + inputs_embeds) + + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + # Setting the token_type_ids to the registered buffer in constructor + # where it is all zeros, which usually occurs + # when its auto-generated, registered buffer helps users when + # tracing the model without passing token_type_ids, solves + # issue #5664 + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + + if 'line_bbox' in kwargs: + embeddings += self._cal_spatial_position_embeddings( + kwargs['line_bbox']) + + if 'line_rank_id' in kwargs: + embeddings += self.line_rank_embeddings(kwargs['line_rank_id']) + + if 'line_rank_inner_id' in kwargs: + embeddings += self.line_rank_inner_embeddings( + kwargs['line_rank_inner_id']) + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + def calc_bbox_pos_emb(self, bbox, pe_type): + # bbox_t: [seq_length, batch_size, dim_bbox] + bbox_t = bbox.transpose(0, 1) + + if pe_type == 'pdpdq_ws': + bbox_pos = bbox_t + elif pe_type == 'crel': + # bbox_pos: [seq_length, seq_length, batch_size, dim_bbox] + bbox_pos = bbox_t[None, :, :, :] - bbox_t[:, None, :, :] + else: + raise ValueError(f'Unknown pe_type={pe_type}') + + bbox_pos_emb = self.bbox_sinusoid_emb(bbox_pos) + bbox_pos_emb = self.bbox_projection(bbox_pos_emb) + + return bbox_pos_emb + + def create_position_ids_from_inputs_embeds(self, inputs_embeds): + """ + We are provided embeddings directly. + We cannot infer which are padded so just generate sequential position ids. + + Args: + inputs_embeds: torch.Tensor + + Returns: torch.Tensor + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange( + self.padding_idx + 1, + sequence_length + self.padding_idx + 1, + dtype=torch.long, + device=inputs_embeds.device) + return position_ids.unsqueeze(0).expand(input_shape) + + +# Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->Roberta +class LayoutRobertaSelfAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = position_embedding_type or getattr( + config, 'position_embedding_type', 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + self.pe_type = config.pe_type + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + ( + self.num_attention_heads, + self.attention_head_size, + ) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + bbox_pos_emb=None, + bbox_pos_mask=None, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if (self.position_embedding_type == 'relative_key' + or self.position_embedding_type == 'relative_key_query'): + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = ( + attention_scores + relative_position_scores_query + + relative_position_scores_key) + + # bbox positional encoding + batch_size, n_head, seq_length, d_head = query_layer.shape + if self.pe_type == 'pdpdq_ws': + head_q_pos = self.query(bbox_pos_emb) + head_k_pos = self.key(bbox_pos_emb) + head_q_pos = head_q_pos.view(seq_length, batch_size, n_head, + d_head) + head_k_pos = head_k_pos.view(seq_length, batch_size, n_head, + d_head) + head_q_pos = head_q_pos.permute([1, 2, 0, 3]) + head_k_pos = head_k_pos.permute([1, 2, 0, 3]) + + bbox_pos_scores_1 = torch.einsum( + 'bnid,bnjd->bnij', + (torch.mul(query_layer, head_q_pos), head_k_pos)) + bbox_pos_scores_2 = torch.einsum('bnid,bnjd->bnij', + (head_q_pos, head_k_pos)) + bbox_pos_scores = bbox_pos_scores_1 + bbox_pos_scores_2 + elif self.pe_type == 'crel': + bbox_pos_emb = bbox_pos_emb.view(seq_length, seq_length, + batch_size, d_head) + bbox_pos_emb = bbox_pos_emb.permute([2, 0, 1, 3]) + bbox_pos_scores = torch.einsum('bnid,bijd->bnij', + (query_layer, bbox_pos_emb)) + else: + raise ValueError(f'Unknown self.pe_type={self.pe_type}') + + if bbox_pos_mask is not None: + # bbox_pos_mask is [batch_size, seq_length] + bbox_pos_mask = 1 - bbox_pos_mask + # [batch_size, 1, seq_length] + M1 = bbox_pos_mask.unsqueeze(1) + # [batch_size, seq_length, 1] + MT = M1.permute(0, 2, 1) + # [batch_size, seq_length, seq_length] + bbox_pos_mask_final = torch.matmul( + MT.to(bbox_pos_scores.dtype), M1.to(bbox_pos_scores.dtype)) + else: + bbox_pos_mask_final = None + + if bbox_pos_mask_final is not None: + bbox_pos_scores = torch.mul(bbox_pos_scores, + bbox_pos_mask_final.unsqueeze(1)) + + # [batch_size, d_head, seq_length, seq_length] + attention_scores = attention_scores + bbox_pos_scores + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +# Copied from transformers.models.bert.modeling_bert.BertSelfOutput +class LayoutRobertaSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Roberta +class LayoutRobertaAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + self.self = LayoutRobertaSelfAttention( + config, position_embedding_type=position_embedding_type) + self.output = LayoutRobertaSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + bbox_pos_emb=None, + bbox_pos_mask=None, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + bbox_pos_emb=bbox_pos_emb, + bbox_pos_mask=bbox_pos_mask, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +# Copied from transformers.models.bert.modeling_bert.BertIntermediate +class LayoutRobertaIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertOutput +class LayoutRobertaOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Roberta +class LayoutRobertaLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = LayoutRobertaAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = LayoutRobertaAttention( + config, position_embedding_type='absolute') + self.intermediate = LayoutRobertaIntermediate(config) + self.output = LayoutRobertaOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + bbox_pos_emb=None, + bbox_pos_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + bbox_pos_emb=bbox_pos_emb, + bbox_pos_mask=bbox_pos_mask, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise ValueError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers' + ' by setting `config.add_cross_attention=True`') + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output, + ) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +# Copied from transformers.models.bert.modeling_bert.BertEncoder with Bert->Roberta +class LayoutRobertaEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList([ + LayoutRobertaLayer(config) for _ in range(config.num_hidden_layers) + ]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + bbox_pos_emb=None, + bbox_pos_mask=None, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + bbox_pos_emb, + bbox_pos_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + bbox_pos_emb, + bbox_pos_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +# Copied from transformers.models.bert.modeling_bert.BertPooler +class LayoutRobertaPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class LayoutRobertaPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = LayoutRobertaConfig + base_model_prefix = 'layoutroberta' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, LayoutRobertaEncoder): + module.gradient_checkpointing = value + + def update_keys_to_ignore(self, config, del_keys_to_ignore): + """Remove some keys from ignore list""" + if not config.tie_word_embeddings: + # must make a new list, or the class variable gets modified! + self._keys_to_ignore_on_save = [ + k for k in self._keys_to_ignore_on_save + if k not in del_keys_to_ignore + ] + self._keys_to_ignore_on_load_missing = [ + k for k in self._keys_to_ignore_on_load_missing + if k not in del_keys_to_ignore + ] + + +class LayoutRobertaModel(LayoutRobertaPreTrainedModel): + """ + + BROS + Roberta + + """ + + # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->Roberta + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = LayoutRobertaEmbeddings(config) + self.encoder = LayoutRobertaEncoder(config) + + self.pooler = LayoutRobertaPooler( + config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + # Copied from transformers.models.bert.modeling_bert.BertModel.forward + def forward(self, + input_ids=None, + bbox=None, + bbox_mask=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple + having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """ + output_attentions = ( + output_attentions if output_attentions is not None else + self.config.output_attentions) + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + + if token_type_ids is None: + if hasattr(self.embeddings, 'token_type_ids'): + buffered_token_type_ids = self.embeddings.token_type_ids[:, : + seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + batch_size, seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + **kwargs) + + scaled_bbox = bbox * self.config.bbox_scale + bbox_pos_emb = self.embeddings.calc_bbox_pos_emb( + scaled_bbox, self.config.pe_type) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + bbox_pos_emb=bbox_pos_emb, + bbox_pos_mask=bbox_mask, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +def create_position_ids_from_input_ids(input_ids, + padding_idx, + past_key_values_length=0): + """ + Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols + are ignored. This is modified from fairseq's `utils.make_positions`. + + Args: + x: torch.Tensor x: + + Returns: torch.Tensor + """ + # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA. + mask = input_ids.ne(padding_idx).int() + incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + + past_key_values_length) * mask + return incremental_indices.long() + padding_idx diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/processing.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/processing.py new file mode 100644 index 0000000000000000000000000000000000000000..afef8bdbcecd6f05aa4bd4d407ecfefd0788ce89 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/processing.py @@ -0,0 +1,538 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" +Processor class for GeoLayoutLM. +""" + +from collections import defaultdict +from typing import Dict, Iterable, List, Union + +import cv2 +import numpy as np +import PIL +import torch +from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from torchvision import transforms + +from modelscope.preprocessors.image import LoadImage + + +def custom_tokenize(tokenizer, text): + toks = tokenizer.tokenize('pad ' + text)[1:] + toks2 = toks[1:] if len(toks) > 0 and toks[0] == '▁' else toks + return toks2 + + +class ImageProcessor(object): + r""" + Construct a GeoLayoutLM image processor + Args: + do_preprocess (`bool`): whether to do preprocess to unify the image format, + resize and convert to tensor. + do_rescale: only works when we disable do_preprocess. + """ + + def __init__(self, + do_preprocess: bool = True, + do_resize: bool = False, + image_size: Dict[str, int] = None, + do_rescale: bool = False, + rescale_factor: float = 1. / 255, + do_normalize: bool = True, + image_mean: Union[float, Iterable[float]] = None, + image_std: Union[float, Iterable[float]] = None, + apply_ocr: bool = True, + **kwargs) -> None: + self.do_preprocess = do_preprocess + self.do_resize = do_resize + self.size = image_size if image_size is not None else { + 'height': 768, + 'width': 768 + } + self.do_rescale = do_rescale and (not do_preprocess) + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + image_mean = IMAGENET_DEFAULT_MEAN if image_mean is None else image_mean + image_std = IMAGENET_DEFAULT_STD if image_std is None else image_std + self.image_mean = (image_mean, image_mean, image_mean) if isinstance( + image_mean, float) else image_mean + self.image_std = (image_std, image_std, image_std) if isinstance( + image_std, float) else image_std + self.apply_ocr = apply_ocr + self.kwargs = kwargs + + self.totensor = transforms.ToTensor() + + def preprocess(self, image: Union[np.ndarray, PIL.Image.Image]): + """ unify the image format, resize and convert to tensor. + """ + image = LoadImage.convert_to_ndarray(image)[:, :, ::-1] + size_raw = image.shape[:2] + if self.do_resize: + image = cv2.resize(image, + (self.size['width'], self.size['height'])) + # convert to pytorch tensor + image_pt = self.totensor(image) + return image_pt, size_raw + + def __call__(self, images: Union[list, np.ndarray, PIL.Image.Image, str]): + """ + Args: + images: list of np.ndarrays, PIL images or image tensors. + """ + if not isinstance(images, list): + images = [images] + sizes_raw = [] + if self.do_preprocess: + for i in range(len(images)): + images[i], size_raw = self.preprocess(images[i]) + sizes_raw.append(size_raw) + images_pt = torch.stack(images, dim=0) # [b, c, h, w] + if self.do_rescale: + images_pt = images_pt * self.rescale_factor + if self.do_normalize: + mu = torch.tensor(self.image_mean).view(1, 3, 1, 1) + std = torch.tensor(self.image_std).view(1, 3, 1, 1) + images_pt = (images_pt - mu) / (std + 1e-8) + + # TODO: apply OCR + ocr_infos = None + if self.apply_ocr: + raise NotImplementedError('OCR service is not available yet!') + if len(sizes_raw) == 0: + sizes_raw = None + data = { + 'images': images_pt, + 'ocr_infos': ocr_infos, + 'sizes_raw': sizes_raw + } + return data + + +class OCRUtils(object): + + def __init__(self): + self.version = 'v0' + + def __call__(self, ocr_infos): + """ + sort boxes, filtering or other preprocesses + should return sorted ocr_infos + """ + raise NotImplementedError + + +def bound_box(box, height, width): + # box: [x_tl, y_tl, x_br, y_br] or ... + assert len(box) == 4 or len(box) == 8 + for i in range(len(box)): + if i & 1: + box[i] = max(0, min(box[i], height)) + else: + box[i] = max(0, min(box[i], width)) + return box + + +def bbox2pto4p(box2p): + box4p = [ + box2p[0], box2p[1], box2p[2], box2p[1], box2p[2], box2p[3], box2p[0], + box2p[3] + ] + return box4p + + +def bbox4pto2p(box4p): + box2p = [ + min(box4p[0], box4p[2], box4p[4], box4p[6]), + min(box4p[1], box4p[3], box4p[5], box4p[7]), + max(box4p[0], box4p[2], box4p[4], box4p[6]), + max(box4p[1], box4p[3], box4p[5], box4p[7]), + ] + return box2p + + +def stack_tensor_dict(tensor_dicts: List[Dict[str, torch.Tensor]]): + one_dict = defaultdict(list) + for td in tensor_dicts: + for k, v in td.items(): + one_dict[k].append(v) + res_dict = {} + for k, v in one_dict.items(): + res_dict[k] = torch.stack(v, dim=0) + return res_dict + + +class TextLayoutSerializer(object): + + def __init__(self, + max_seq_length: int, + max_block_num: int, + tokenizer, + width=768, + height=768, + use_roberta_tokenizer: bool = True, + ocr_utils: OCRUtils = None): + self.version = 'v0' + + self.max_seq_length = max_seq_length + self.max_block_num = max_block_num + self.tokenizer = tokenizer + self.width = width + self.height = height + self.use_roberta_tokenizer = use_roberta_tokenizer + self.ocr_utils = ocr_utils + + self.pad_token_id = tokenizer.pad_token_id + self.cls_token_id = tokenizer.bos_token_id + self.sep_token_id = tokenizer.eos_token_id + self.unk_token_id = tokenizer.unk_token_id + self.cls_bbs_word = [0.0] * 8 + self.cls_bbs_line = [0] * 4 + + def label2seq(self, ocr_info: list, label_info: list): + raise NotImplementedError + + def serialize_single( + self, + ocr_info: list = None, + input_ids: list = None, + bbox_line: List[List] = None, + bbox_word: List[List] = None, + width: int = 768, + height: int = 768, + ): + r""" + Either ocr_info or (input_ids, bbox_line, bbox_word) + should be provided. + If (input_ids, bbox_line, bbox_word) is provided, + convinient plug into the serialization (customization) + is offered. The tokens must be organised by blocks and words. + Else, ocr_info must be provided, to be parsed + to sequences directly (the simplest way). + Args: + ocr_info: [ + {"text": "xx", "box": [a,b,c,d], + "words": [{"text": "x", "box": [e,f,g,h]}, ...]}, + ... + ] + bbox_line: the coordinate value should match the original image + (i.e., not be normalized). + """ + if input_ids is not None: + assert len(input_ids) == len(bbox_line) + assert len(input_ids) == len(bbox_word) + input_ids, bbs_word, bbs_line, first_token_idxes, \ + line_rank_ids, line_rank_inner_ids, word_rank_ids = \ + self.halfseq2seq(input_ids, bbox_line, bbox_word, width, height) + else: + assert ocr_info is not None + input_ids, bbs_word, bbs_line, first_token_idxes, \ + line_rank_ids, line_rank_inner_ids, word_rank_ids = \ + self.ocr_info2seq(ocr_info, width, height) + + token_seq = {} + token_seq['input_ids'] = torch.ones( + self.max_seq_length, dtype=torch.int64) * self.pad_token_id + token_seq['attention_mask'] = torch.zeros( + self.max_seq_length, dtype=torch.int64) + token_seq['first_token_idxes'] = torch.zeros( + self.max_block_num, dtype=torch.int64) + token_seq['first_token_idxes_mask'] = torch.zeros( + self.max_block_num, dtype=torch.int64) + token_seq['bbox_4p_normalized'] = torch.zeros( + self.max_seq_length, 8, dtype=torch.float32) + token_seq['bbox'] = torch.zeros( + self.max_seq_length, 4, dtype=torch.float32) + token_seq['line_rank_id'] = torch.zeros( + self.max_seq_length, dtype=torch.int64) # start from 1 + token_seq['line_rank_inner_id'] = torch.ones( + self.max_seq_length, dtype=torch.int64) # 1 2 2 3 + token_seq['word_rank_id'] = torch.zeros( + self.max_seq_length, dtype=torch.int64) # start from 1 + + # expand using cls and sep tokens + sep_bbs_word = [width, height] * 4 + sep_bbs_line = [width, height] * 2 + input_ids = [self.cls_token_id] + input_ids + [self.sep_token_id] + bbs_line = [self.cls_bbs_line] + bbs_line + [sep_bbs_line] + bbs_word = [self.cls_bbs_word] + bbs_word + [sep_bbs_word] + + # assign + len_tokens = len(input_ids) + len_lines = len(first_token_idxes) + token_seq['input_ids'][:len_tokens] = torch.tensor(input_ids) + token_seq['attention_mask'][:len_tokens] = 1 + token_seq['first_token_idxes'][:len_lines] = torch.tensor( + first_token_idxes) + token_seq['first_token_idxes_mask'][:len_lines] = 1 + token_seq['line_rank_id'][1:len_tokens + - 1] = torch.tensor(line_rank_ids) + token_seq['line_rank_inner_id'][1:len_tokens - 1] = torch.tensor( + line_rank_inner_ids) + token_seq['line_rank_inner_id'] = token_seq[ + 'line_rank_inner_id'] * token_seq['attention_mask'] + token_seq['word_rank_id'][1:len_tokens + - 1] = torch.tensor(word_rank_ids) + + token_seq['bbox_4p_normalized'][:len_tokens, :] = torch.tensor( + bbs_word) + # word bbox normalization -> [0, 1] + token_seq['bbox_4p_normalized'][:, [0, 2, 4, 6]] = \ + token_seq['bbox_4p_normalized'][:, [0, 2, 4, 6]] / width + token_seq['bbox_4p_normalized'][:, [1, 3, 5, 7]] = \ + token_seq['bbox_4p_normalized'][:, [1, 3, 5, 7]] / height + + token_seq['bbox'][:len_tokens, :] = torch.tensor(bbs_line) + # line bbox -> [0, 1000) + token_seq['bbox'][:, + [0, 2]] = token_seq['bbox'][:, [0, 2]] / width * 1000 + token_seq['bbox'][:, + [1, 3]] = token_seq['bbox'][:, + [1, 3]] / height * 1000 + token_seq['bbox'] = token_seq['bbox'].long() + + return token_seq + + def ocr_info2seq(self, ocr_info: list, width: int, height: int): + input_ids = [] + bbs_word = [] + bbs_line = [] + first_token_idxes = [] + line_rank_ids = [] + line_rank_inner_ids = [] + word_rank_ids = [] + + early_stop = False + for line_idx, line in enumerate(ocr_info): + if line_idx == self.max_block_num: + early_stop = True + if early_stop: + break + lbox = line['box'] + lbox = bound_box(lbox, height, width) + is_first_word = True + for word_id, word_info in enumerate(line['words']): + wtext = word_info['text'] + wbox = word_info['box'] + wbox = bound_box(wbox, height, width) + wbox4p = bbox2pto4p(wbox) + if self.use_roberta_tokenizer: + wtokens = custom_tokenize(self.tokenizer, wtext) + else: + wtokens = self.tokenizer.tokenize(wtext) + wtoken_ids = self.tokenizer.convert_tokens_to_ids(wtokens) + if len(wtoken_ids) == 0: + wtoken_ids.append(self.unk_token_id) + n_tokens = len(wtoken_ids) + # reserve for cls and sep + if len(input_ids) + n_tokens > self.max_seq_length - 2: + early_stop = True + break # chunking early for long documents + if is_first_word: + first_token_idxes.append(len(input_ids) + 1) + input_ids.extend(wtoken_ids) + bbs_word.extend([wbox4p] * n_tokens) + bbs_line.extend([lbox] * n_tokens) + word_rank_ids.extend([word_id + 1] * n_tokens) + line_rank_ids.extend([line_idx + 1] * n_tokens) + if is_first_word: + if len(line_rank_inner_ids + ) > 0 and line_rank_inner_ids[-1] == 2: + line_rank_inner_ids[-1] = 3 + line_rank_inner_ids.extend([1] + (n_tokens - 1) * [2]) + is_first_word = False + else: + line_rank_inner_ids.extend(n_tokens * [2]) + if len(line_rank_inner_ids) > 0 and line_rank_inner_ids[-1] == 2: + line_rank_inner_ids[-1] = 3 + + return input_ids, bbs_word, bbs_line, first_token_idxes, line_rank_ids, \ + line_rank_inner_ids, word_rank_ids + + def halfseq2seq(self, input_ids: list, bbox_line: List[List], + bbox_word: List[List], width: int, height: int): + """ + for convinient plug into the serialization, given the 3 customized sequences. + They should not contain special tokens like [CLS] or [SEP]. + """ + bbs_word = [] + bbs_line = [] + first_token_idxes = [] + line_rank_ids = [] + line_rank_inner_ids = [] + word_rank_ids = [] + + n_real_tokens = len(input_ids) + lb_prev, wb_prev = None, None + line_id = 0 + word_id = 1 + for i in range(n_real_tokens): + lb_now = bbox_line[i] + wb_now = bbox_word[i] + line_start = lb_prev is None or lb_now != lb_prev + word_start = wb_prev is None or wb_now != wb_prev + lb_prev, wb_prev = lb_now, wb_now + + if len(lb_now) == 8: + lb_now = bbox4pto2p(lb_now) + assert len(lb_now) == 4 + lb_now = bound_box(lb_now, height, width) + if len(wb_now) == 4: + wb_now = bbox2pto4p(wb_now) + assert len(wb_now) == 8 + wb_now = bound_box(wb_now, height, width) + + bbs_word.append(wb_now) + bbs_line.append(lb_now) + + if word_start: + word_id += 1 + if line_start: + line_id += 1 + first_token_idxes.append(i + 1) + if len(line_rank_inner_ids + ) > 0 and line_rank_inner_ids[-1] == 2: + line_rank_inner_ids[-1] = 3 + line_rank_inner_ids.append(1) + word_id = 1 + else: + line_rank_inner_ids.append(2) + line_rank_ids.append(line_id) + word_rank_ids.append(word_id) + + if len(line_rank_inner_ids) > 0 and line_rank_inner_ids[-1] == 2: + line_rank_inner_ids[-1] = 3 + + return input_ids, bbs_word, bbs_line, first_token_idxes, \ + line_rank_ids, line_rank_inner_ids, word_rank_ids + + def __call__( + self, + ocr_infos: List[List] = None, + input_ids: list = None, + bboxes_line: List[List] = None, + bboxes_word: List[List] = None, + sizes_raw: list = None, + **kwargs, + ): + n_samples = len(ocr_infos) if ocr_infos is not None else len(input_ids) + if sizes_raw is None: + sizes_raw = [(self.height, self.width)] * n_samples + seqs = [] + if input_ids is not None: + assert len(input_ids) == len(bboxes_line) + assert len(input_ids) == len(bboxes_word) + for input_id, bbox_line, bbox_word, size_raw in zip( + input_ids, bboxes_line, bboxes_word, sizes_raw): + height, width = size_raw + token_seq = self.serialize_single(None, input_id, bbox_line, + bbox_word, width, height) + seqs.append(token_seq) + else: + assert ocr_infos is not None, 'For serialization, ocr_infos must not be NoneType!' + if self.ocr_utils is not None: + ocr_infos = self.ocr_utils(ocr_infos) + for ocr_info, size_raw in zip(ocr_infos, sizes_raw): + height, width = size_raw + token_seq = self.serialize_single( + ocr_info, width=width, height=height) + seqs.append(token_seq) + pt_seqs = stack_tensor_dict(seqs) + return pt_seqs + + +class Processor(object): + r"""Construct a GeoLayoutLM processor. + + Args: + max_seq_length: max length for token + max_block_num: max number of text lines (blocks or segments) + img_processor: type of ImageProcessor. + tokenizer: to tokenize strings. + use_roberta_tokenizer: Whether the tokenizer is originated from RoBerta tokenizer + (True by default). + ocr_utils: a tool to preprocess ocr_infos. + width: default width. It can be used only when all the images are of the same shape. + height: default height. It can be used only when all the images are of the same shape. + + In `serialize_from_tokens`, the 3 sequences (i.e., `input_ids`, `bboxes_line`, `bboxes_word`) + must not contain special tokens like [CLS] or [SEP]. + The boxes in `bboxes_line` and `bboxes_word` can be presented by either 2 points or 4 points. + The value in boxes should keep original. + Here is an example of the 3 arguments: + ``` + input_ids -> + [[6, 2391, 6, 31833, 6, 10132, 6, 2283, 6, 17730, 6, 2698, 152]] + bboxes_line -> + [[[230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], + [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], + [257, 155, 338, 191], [257, 155, 338, 191], [257, 155, 338, 191], [257, 155, 338, 191], + [257, 155, 338, 191]]] + bboxes_word -> + [[[231, 2, 267, 2, 267, 38, 231, 38], [231, 2, 267, 2, 267, 38, 231, 38], + [264, 7, 298, 7, 298, 36, 264, 36], [264, 7, 298, 7, 298, 36, 264, 36], + [293, 3, 329, 3, 329, 41, 293, 41], [293, 3, 329, 3, 329, 41, 293, 41], + [330, 4, 354, 4, 354, 39, 330, 39], [330, 4, 354, 4, 354, 39, 330, 39], + [258, 156, 289, 156, 289, 193, 258, 193], [258, 156, 289, 156, 289, 193, 258, 193], + [288, 158, 321, 158, 321, 192, 288, 192], [288, 158, 321, 158, 321, 192, 288, 192], + [321, 156, 336, 156, 336, 190, 321, 190]]] + ``` + + """ + + def __init__(self, + max_seq_length, + max_block_num, + img_processor: ImageProcessor, + tokenizer=None, + use_roberta_tokenizer: bool = True, + ocr_utils: OCRUtils = None, + width=768, + height=768, + **kwargs): + self.img_processor = img_processor + self.tokenizer = tokenizer + self.kwargs = kwargs + + self.serializer = TextLayoutSerializer( + max_seq_length, + max_block_num, + tokenizer, + width, + height, + use_roberta_tokenizer=use_roberta_tokenizer, + ocr_utils=ocr_utils) + + def __call__( + self, + images: Union[list, np.ndarray, PIL.Image.Image, str], + ocr_infos: List[List] = None, + token_seqs: dict = None, + sizes_raw: list = None, + ): + img_data = self.img_processor(images) + images = img_data['images'] + ocr_infos = img_data['ocr_infos'] if ocr_infos is None else ocr_infos + sizes_raw = img_data['sizes_raw'] if sizes_raw is None else sizes_raw + if token_seqs is None: + token_seqs = self.serializer(ocr_infos, sizes_raw=sizes_raw) + else: + token_seqs = self.serializer( + None, sizes_raw=sizes_raw, **token_seqs) + assert token_seqs is not None, 'token_seqs must not be NoneType!' + batch = {} + batch['image'] = images + for k, v in token_seqs.items(): + batch[k] = token_seqs[k] + return batch + + def serialize_from_tokens(self, + images, + input_ids, + bboxes_line, + bboxes_word, + sizes_raw=None): + half_batch = {} + half_batch['input_ids'] = input_ids + half_batch['bboxes_line'] = bboxes_line + half_batch['bboxes_word'] = bboxes_word + return self(images, None, half_batch, sizes_raw) diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/tokenization.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..a16c58490e428491b42a4b5e0bd2f932c2f89b45 --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/tokenization.py @@ -0,0 +1,89 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +from transformers import XLMRobertaTokenizer + +SPIECE_UNDERLINE = '▁' + + +class VLDocXLMTokenizer(XLMRobertaTokenizer): + """ + Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on + [SentencePiece](https://github.com/google/sentencepiece). + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + + + + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + + + + eos_token (`str`, *optional*, defaults to `""`): + The end of sequence token. + + + + When building a sequence using special tokens, this is not the token that is used for the end of sequence. + The token used is the `sep_token`. + + + + sep_token (`str`, *optional*, defaults to `"
"`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + cls_token (`str`, *optional*, defaults to `""`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (`str`, *optional*, defaults to `""`): + The token used for padding, for example when batching sequences of different lengths. + mask_token (`str`, *optional*, defaults to `""`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + cls_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`): + The bounding box to use for the special [CLS] token. + sep_token_box (`List[int]`, *optional*, defaults to `[1000, 1000, 1000, 1000]`): + The bounding box to use for the special [SEP] token. + pad_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`): + The bounding box to use for the special [PAD] token. + pad_token_label (`int`, *optional*, defaults to -100): + The label to use for padding tokens. Defaults to -100, which is the `ignore_index` of PyTorch's + CrossEntropyLoss. + only_label_first_subword (`bool`, *optional*, defaults to `True`): + Whether or not to only label the first subword, in case word labels are provided. + additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`): + Additional special tokens used by the tokenizer. + sp_model_kwargs (`dict`, *optional*): + Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for + SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, + to set: + + - `enable_sampling`: Enable subword regularization. + - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. + + - `nbest_size = {0,1}`: No sampling is performed. + - `nbest_size > 1`: samples from the nbest_size results. + - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) + using forward-filtering-and-backward-sampling algorithm. + + - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for + BPE-dropout. + + Attributes: + sp_model (`SentencePieceProcessor`): + The *SentencePiece* processor that is used for every conversion (string, tokens and IDs). + """ + model_input_names = ['input_ids', 'attention_mask'] diff --git a/runtime/Lib/site_packages/models/multi_modal/vldoc/transformer_local.py b/runtime/Lib/site_packages/models/multi_modal/vldoc/transformer_local.py new file mode 100644 index 0000000000000000000000000000000000000000..e819d4a4fa8357e9586c4a6b9f01c7a5356e3d6d --- /dev/null +++ b/runtime/Lib/site_packages/models/multi_modal/vldoc/transformer_local.py @@ -0,0 +1,204 @@ +# The implementation is borrowed and modified from the official PyTorch website and ABINet: +# https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html +# https://github.com/FangShancheng/ABINet/blob/main/modules/transformer.py + +import copy + +import torch.nn as nn +from torch import Tensor +from torch.nn import Dropout, LayerNorm, Linear, Module, ModuleList +from torch.nn import functional as F + + +class TransformerDecoder(Module): + r"""TransformerDecoder is a stack of N decoder layers + + Args: + decoder_layer: an instance of the TransformerDecoderLayer() class (required). + num_layers: the number of sub-decoder-layers in the decoder (required). + norm: the layer normalization component (optional). + + Examples: + >>> decoder_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8) + >>> transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=6) + >>> memory = torch.rand(10, 32, 512) + >>> tgt = torch.rand(20, 32, 512) + >>> out = transformer_decoder(tgt, memory) + """ + __constants__ = ['norm'] + + def __init__(self, decoder_layer, num_layers, norm=None): + super(TransformerDecoder, self).__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward(self, + tgt, + memory, + memory2=None, + tgt_mask=None, + memory_mask=None, + memory_mask2=None, + tgt_key_padding_mask=None, + memory_key_padding_mask=None, + memory_key_padding_mask2=None): + r"""Pass the inputs (and mask) through the decoder layer in turn. + + Args: + tgt: the sequence to the decoder (required). + memory: the sequence from the last layer of the encoder (required). + tgt_mask: the mask for the tgt sequence (optional). + memory_mask: the mask for the memory sequence (optional). + tgt_key_padding_mask: the mask for the tgt keys per batch (optional). + memory_key_padding_mask: the mask for the memory keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + output = tgt + + for mod in self.layers: + output = mod( + output, + memory, + memory2=memory2, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + memory_mask2=memory_mask2, + tgt_key_padding_mask=tgt_key_padding_mask, + memory_key_padding_mask=memory_key_padding_mask, + memory_key_padding_mask2=memory_key_padding_mask2) + + if self.norm is not None: + output = self.norm(output) + + return output + + +class TransformerDecoderLayer(Module): + r"""TransformerDecoderLayer is made up of self-attn, multi-head-attn and feedforward network. + + Args: + d_model: the number of expected features in the input (required). + nhead: the number of heads in the multiheadattention models (required). + dim_feedforward: the dimension of the feedforward network model (default=2048). + dropout: the dropout value (default=0.1). + activation: the activation function of intermediate layer, relu or gelu (default=relu). + """ + + def __init__(self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation='relu', + self_attn=True, + siamese=False, + debug=False): + super(TransformerDecoderLayer, self).__init__() + self.has_self_attn, self.siamese = self_attn, siamese + self.debug = debug + if self.has_self_attn: + self.self_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + self.norm1 = LayerNorm(d_model) + self.dropout1 = Dropout(dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = Linear(d_model, dim_feedforward) + self.dropout = Dropout(dropout) + self.linear2 = Linear(dim_feedforward, d_model) + + self.norm2 = LayerNorm(d_model) + self.norm3 = LayerNorm(d_model) + self.dropout2 = Dropout(dropout) + self.dropout3 = Dropout(dropout) + if self.siamese: + self.multihead_attn2 = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + + self.activation = _get_activation_fn(activation) + + def __setstate__(self, state): + if 'activation' not in state: + state['activation'] = F.relu + super(TransformerDecoderLayer, self).__setstate__(state) + + def forward(self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + tgt_key_padding_mask=None, + memory_key_padding_mask=None, + memory2=None, + memory_mask2=None, + memory_key_padding_mask2=None): + r"""Pass the inputs (and mask) through the decoder layer. + + Args: + tgt: the sequence to the decoder layer (required). + memory: the sequence from the last layer of the encoder (required). + tgt_mask: the mask for the tgt sequence (optional). + memory_mask: the mask for the memory sequence (optional). + tgt_key_padding_mask: the mask for the tgt keys per batch (optional). + memory_key_padding_mask: the mask for the memory keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + if self.has_self_attn: + tgt2, attn = self.self_attn( + tgt, + tgt, + tgt, + attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask, + ) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + if self.debug: + self.attn = attn + tgt2, attn2 = self.multihead_attn( + tgt, + memory, + memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask) + if self.debug: + self.attn2 = attn2 + + if self.siamese: + tgt3, attn3 = self.multihead_attn2( + tgt, + memory2, + memory2, + attn_mask=memory_mask2, + key_padding_mask=memory_key_padding_mask2) + tgt = tgt + self.dropout2(tgt3) + if self.debug: + self.attn3 = attn3 + + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout3(tgt2) + tgt = self.norm3(tgt) + + return tgt + + +def _get_clones(module, N): + return ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation): + if activation == 'relu': + return F.relu + elif activation == 'gelu': + return F.gelu + + raise RuntimeError( + 'activation should be relu/gelu, not {}'.format(activation)) diff --git a/runtime/Lib/site_packages/models/nlp/T5/__init__.py b/runtime/Lib/site_packages/models/nlp/T5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb0921c6ee8bb58f312325957997b448b0cd8df3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/T5/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import T5Model + from .text2text_generation import T5ForConditionalGeneration + +else: + _import_structure = { + 'backbone': ['T5Model'], + 'text2text_generation': ['T5ForConditionalGeneration'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/T5/backbone.py b/runtime/Lib/site_packages/models/nlp/T5/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..800ab6c62c777fcde7546ca1d1240a3ae8e8b23e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/T5/backbone.py @@ -0,0 +1,1536 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch T5 model.""" + +import copy +import math +import os +import warnings +from typing import Optional, Tuple, Union + +import torch +from torch import nn +from torch.utils.checkpoint import checkpoint +from transformers.activations import ACT2FN +from transformers.modeling_outputs import \ + BaseModelOutputWithPastAndCrossAttentions +from transformers.modeling_utils import (PreTrainedModel, + find_pruneable_heads_and_indices, + prune_linear_layer) +from transformers.utils import (DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, + add_start_docstrings_to_model_forward, + is_torch_fx_proxy, replace_return_docstrings) +from transformers.utils.model_parallel_utils import (assert_device_map, + get_device_map) + +from modelscope.metainfo import Models +from modelscope.models.base import Model, Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput, Seq2SeqModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .configuration import T5Config + +logger = get_logger() + + +################################################### +# This is a conversion method from TF 1.0 to PyTorch +# More details: https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28 +#################################################### +def load_tf_weights_in_t5(model, config, tf_checkpoint_path): + """Load tf checkpoints in a pytorch model.""" + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + 'Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see ' + 'https://www.tensorflow.org/install/ for installation instructions.' + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info(f'Converting TensorFlow checkpoint from {tf_path}') + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + tf_weights = {} + for name, shape in init_vars: + logger.info(f'Loading TF weight {name} with shape {shape}') + array = tf.train.load_variable(tf_path, name) + names.append(name) + tf_weights[name] = array + + for txt_name in names: + name = txt_name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in [ + 'adam_v', 'adam_m', 'AdamWeightDecayOptimizer', + 'AdamWeightDecayOptimizer_1', 'global_step' + ] for n in name): + logger.info(f"Skipping {'/'.join(name)}") + tf_weights.pop(txt_name, None) + continue + if '_slot_' in name[-1]: + logger.info(f"Skipping {'/'.join(name)}") + tf_weights.pop(txt_name, None) + continue + pointer = model + array = tf_weights[txt_name] + + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + scope_names = re.split(r'_(\d+)', m_name) + else: + scope_names = [m_name] + if scope_names[0] in ['kernel', 'scale', 'embedding']: + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'self_attention': + pointer = getattr(pointer, 'layer') + pointer = pointer[0] + elif scope_names[0] == 'enc_dec_attention': + pointer = getattr(pointer, 'layer') + pointer = pointer[1] + elif scope_names[0] == 'dense_relu_dense': + pointer = getattr(pointer, 'layer') + pointer = pointer[2] + elif scope_names[0] == 'rms_norm': + if hasattr(pointer, 'layer_norm'): + pointer = getattr(pointer, 'layer_norm') + elif hasattr(pointer, 'final_layer_norm'): + pointer = getattr(pointer, 'final_layer_norm') + elif scope_names[0] == 'scale': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'output_bias' or scope_names[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif scope_names[0] == 'squad': + pointer = getattr(pointer, 'classifier') + elif scope_names[0] == 'decoder' and name[1] == 'logits': + continue + elif scope_names[0] == 'logits': + pointer = getattr(pointer, 'lm_head') + elif scope_names[0] == 'wi' and len( + scope_names) > 1 and scope_names[1].isdigit(): + pointer = getattr(pointer, f'wi_{scope_names[1]}') + continue + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info(f"Skipping {'/'.join(name)}") + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if scope_names[0] not in ['kernel', 'scale', 'embedding']: + pointer = getattr(pointer, 'weight') + if scope_names[0] != 'embedding': + logger.info( + f'Transposing numpy weight of shape {array.shape} for {name}') + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f'Pointer shape {pointer.shape} and array shape {array.shape} mismatched' + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info(f'Initialize PyTorch weight {name}') + pointer.data = torch.from_numpy(array.astype(np.float32)) + tf_weights.pop(txt_name, None) + + logger.info( + f"Weights not copied to PyTorch model: {', '.join(tf_weights.keys())}." + ) + return model + + +class T5LayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-6): + """ + Construct a layernorm module in the T5 style. No bias and no subtraction of mean. + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + + # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean + # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated + # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for + # half-precision inputs is done in fp32 + + variance = hidden_states.to(torch.float32).pow(2).mean( + -1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + + self.variance_epsilon) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +class T5DenseReluDense(nn.Module): + + def __init__(self, config: T5Config): + super().__init__() + self.wi = nn.Linear(config.d_model, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.d_model, bias=False) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states): + hidden_states = self.wi(hidden_states) + hidden_states = nn.functional.relu(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.wo(hidden_states) + return hidden_states + + +class T5DenseGatedGeluDense(nn.Module): + + def __init__(self, config: T5Config): + super().__init__() + self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=False) + self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.d_model, bias=False) + self.dropout = nn.Dropout(config.dropout_rate) + self.gelu_act = ACT2FN['gelu_new'] + + def forward(self, hidden_states): + hidden_gelu = self.gelu_act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + hidden_states = self.wo(hidden_states) + return hidden_states + + +class T5LayerFF(nn.Module): + + def __init__(self, config: T5Config): + super().__init__() + if config.feed_forward_proj == 'relu': + self.DenseReluDense = T5DenseReluDense(config) + elif config.feed_forward_proj == 'gated-gelu': + self.DenseReluDense = T5DenseGatedGeluDense(config) + else: + raise ValueError( + f'{self.config.feed_forward_proj} is not supported. Choose between `relu` and `gated-gelu`' + ) + + self.layer_norm = T5LayerNorm( + config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states): + forwarded_states = self.layer_norm(hidden_states) + forwarded_states = self.DenseReluDense(forwarded_states) + hidden_states = hidden_states + self.dropout(forwarded_states) + return hidden_states + + +class T5Attention(nn.Module): + + def __init__(self, config: T5Config, has_relative_attention_bias=False): + super().__init__() + self.is_decoder = config.is_decoder + self.has_relative_attention_bias = has_relative_attention_bias + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.relative_attention_max_distance = config.relative_attention_max_distance + self.d_model = config.d_model + self.key_value_proj_dim = config.d_kv + self.n_heads = config.num_heads + self.dropout = config.dropout_rate + self.inner_dim = self.n_heads * self.key_value_proj_dim + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.q = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.k = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.v = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.o = nn.Linear(self.inner_dim, self.d_model, bias=False) + + if self.has_relative_attention_bias: + self.relative_attention_bias = nn.Embedding( + self.relative_attention_num_buckets, self.n_heads) + self.pruned_heads = set() + self.gradient_checkpointing = False + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads) + # Prune linear layers + self.q = prune_linear_layer(self.q, index) + self.k = prune_linear_layer(self.k, index) + self.v = prune_linear_layer(self.v, index) + self.o = prune_linear_layer(self.o, index, dim=1) + # Update hyper params + self.n_heads = self.n_heads - len(heads) + self.inner_dim = self.key_value_proj_dim * self.n_heads + self.pruned_heads = self.pruned_heads.union(heads) + + @staticmethod + def _relative_position_bucket(relative_position, + bidirectional=True, + num_buckets=32, + max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. The relative position is defined as + memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for + small absolute relative_position and larger buckets for larger absolute relative_positions. All relative + positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. + This should allow for more graceful generalization to longer sequences than the model has been trained on + + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + + Returns: + a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) + """ + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > 0).to( + torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = -torch.min(relative_position, + torch.zeros_like(relative_position)) + # now relative_position is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + # The other half of the buckets are for logarithmically bigger bins in + # positions up to max_distance + relateive_pos_log = torch.log(relative_position.float() / max_exact) + max_dis_log = math.log(max_distance / max_exact) + origin_relative_position = relateive_pos_log / max_dis_log * ( + num_buckets - max_exact) + relative_postion_if_large = max_exact + origin_relative_position.to( + torch.long) + relative_postion_if_large = torch.min( + relative_postion_if_large, + torch.full_like(relative_postion_if_large, num_buckets - 1)) + + relative_buckets += torch.where(is_small, relative_position, + relative_postion_if_large) + return relative_buckets + + def compute_bias(self, query_length, key_length): + """Compute binned relative position bias""" + context_position = torch.arange( + query_length, + dtype=torch.long, + device=self.relative_attention_bias.weight.device)[:, None] + memory_position = torch.arange( + key_length, + dtype=torch.long, + device=self.relative_attention_bias.weight.device)[None, :] + relative_position = memory_position - context_position # shape (query_length, key_length) + relative_position_bucket = self._relative_position_bucket( + relative_position, # shape (query_length, key_length) + bidirectional=(not self.is_decoder), + num_buckets=self.relative_attention_num_buckets, + max_distance=self.relative_attention_max_distance, + ) + values = self.relative_attention_bias( + relative_position_bucket + ) # shape (query_length, key_length, num_heads) + values = values.permute([2, 0, 1]).unsqueeze( + 0) # shape (1, num_heads, query_length, key_length) + return values + + def forward( + self, + hidden_states, + mask=None, + key_value_states=None, + position_bias=None, + past_key_value=None, + layer_head_mask=None, + query_length=None, + use_cache=False, + output_attentions=False, + ): + """ + Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). + """ + # Input is (batch_size, seq_length, dim) + # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length) + # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head) + batch_size, seq_length = hidden_states.shape[:2] + + real_seq_length = seq_length + + if past_key_value is not None: + assert ( + len(past_key_value) == 2 + ), f'past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states' + real_seq_length += past_key_value[0].shape[ + 2] if query_length is None else query_length + + key_length = real_seq_length if key_value_states is None else key_value_states.shape[ + 1] + + def shape(states): + """projection""" + return states.view(batch_size, -1, self.n_heads, + self.key_value_proj_dim).transpose(1, 2) + + def unshape(states): + """reshape""" + return states.transpose(1, 2).contiguous().view( + batch_size, -1, self.inner_dim) + + def project(hidden_states, proj_layer, key_value_states, + past_key_value): + """projects hidden states correctly to key/query states""" + if key_value_states is None: + # self-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = shape(proj_layer(hidden_states)) + elif past_key_value is None: + # cross-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = shape(proj_layer(key_value_states)) + + if past_key_value is not None: + if key_value_states is None: + # self-attn + # (batch_size, n_heads, key_length, dim_per_head) + hidden_states = torch.cat([past_key_value, hidden_states], + dim=2) + else: + # cross-attn + hidden_states = past_key_value + return hidden_states + + # get query states + query_states = shape(self.q( + hidden_states)) # (batch_size, n_heads, seq_length, dim_per_head) + + # get key/value states + key_states = project( + hidden_states, self.k, key_value_states, + past_key_value[0] if past_key_value is not None else None) + value_states = project( + hidden_states, self.v, key_value_states, + past_key_value[1] if past_key_value is not None else None) + + # compute scores + scores = torch.matmul( + query_states, key_states.transpose(3, 2) + ) # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9 + + if position_bias is None: + if not self.has_relative_attention_bias: + position_bias = torch.zeros( + (1, self.n_heads, real_seq_length, key_length), + device=scores.device, + dtype=scores.dtype) + if self.gradient_checkpointing and self.training: + position_bias.requires_grad = True + else: + position_bias = self.compute_bias(real_seq_length, key_length) + + # if key and values are already calculated + # we want only the last query position bias + if past_key_value is not None: + position_bias = position_bias[:, :, -hidden_states.size(1):, :] + + if mask is not None: + position_bias = position_bias + mask # (batch_size, n_heads, seq_length, key_length) + + scores += position_bias + attn_weights = nn.functional.softmax( + scores.float(), dim=-1).type_as( + scores) # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.dropout( + attn_weights, p=self.dropout, training=self.training + ) # (batch_size, n_heads, seq_length, key_length) + + # Mask heads if we want to + if layer_head_mask is not None: + attn_weights = attn_weights * layer_head_mask + + attn_output = unshape(torch.matmul( + attn_weights, value_states)) # (batch_size, seq_length, dim) + attn_output = self.o(attn_output) + + present_key_value_state = (key_states, + value_states) if (self.is_decoder + and use_cache) else None + outputs = (attn_output, ) + (present_key_value_state, ) + ( + position_bias, ) + + if output_attentions: + outputs = outputs + (attn_weights, ) + return outputs + + +class T5LayerSelfAttention(nn.Module): + + def __init__(self, config, has_relative_attention_bias=False): + super().__init__() + self.SelfAttention = T5Attention( + config, has_relative_attention_bias=has_relative_attention_bias) + self.layer_norm = T5LayerNorm( + config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.SelfAttention( + normed_hidden_states, + mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = hidden_states + self.dropout(attention_output[0]) + outputs = (hidden_states, + ) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5LayerCrossAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.EncDecAttention = T5Attention( + config, has_relative_attention_bias=False) + self.layer_norm = T5LayerNorm( + config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + key_value_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + query_length=None, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention( + normed_hidden_states, + mask=attention_mask, + key_value_states=key_value_states, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + query_length=query_length, + output_attentions=output_attentions, + ) + layer_output = hidden_states + self.dropout(attention_output[0]) + outputs = (layer_output, + ) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5Block(nn.Module): + + def __init__(self, config, has_relative_attention_bias=False): + super().__init__() + self.is_decoder = config.is_decoder + self.layer = nn.ModuleList() + self.layer.append( + T5LayerSelfAttention( + config, + has_relative_attention_bias=has_relative_attention_bias)) + if self.is_decoder: + self.layer.append(T5LayerCrossAttention(config)) + + self.layer.append(T5LayerFF(config)) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + encoder_decoder_position_bias=None, + layer_head_mask=None, + cross_attn_layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + return_dict=True, + ): + + if past_key_value is not None: + if not self.is_decoder: + logger.warning( + '`past_key_values` is passed to the encoder. Please make sure this is intended.' + ) + expected_num_past_key_values = 2 if encoder_hidden_states is None else 4 + + if len(past_key_value) != expected_num_past_key_values: + raise ValueError( + f'There should be {expected_num_past_key_values} past states. ' + f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}" + f'Got {len(past_key_value)} past key / value states') + + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + self_attention_outputs = self.layer[0]( + hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=self_attn_past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states, present_key_value_state = self_attention_outputs[:2] + attention_outputs = self_attention_outputs[ + 2:] # Keep self-attention outputs and relative position weights + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf( + hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value) + + do_cross_attention = self.is_decoder and encoder_hidden_states is not None + if do_cross_attention: + # the actual query length is unknown for cross attention + # if using past key value states. Need to inject it here + if present_key_value_state is not None: + query_length = present_key_value_state[0].shape[2] + else: + query_length = None + + cross_attention_outputs = self.layer[1]( + hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + layer_head_mask=cross_attn_layer_head_mask, + past_key_value=cross_attn_past_key_value, + query_length=query_length, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = cross_attention_outputs[0] + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf( + hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value) + + # Combine self attn and cross attn key value states + if present_key_value_state is not None: + present_key_value_state = present_key_value_state + cross_attention_outputs[ + 1] + + # Keep cross-attention outputs and relative position weights + attention_outputs = attention_outputs + cross_attention_outputs[2:] + + # Apply Feed Forward layer + hidden_states = self.layer[-1](hidden_states) + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf( + hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states, ) + + if use_cache: + outputs = outputs + (present_key_value_state, ) + attention_outputs + else: + outputs = outputs + attention_outputs + + # hidden-states, present_key_value_states, (self-attention position + # bias), (self-attention weights), (cross-attention position bias), + # (cross-attention weights) + return outputs + + +class T5PreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface + for downloading and loading pretrained models. + """ + + config_class = T5Config + load_tf_weights = load_tf_weights_in_t5 + base_model_prefix = 'transformer' + is_parallelizable = True + supports_gradient_checkpointing = True + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + @property + def dummy_inputs(self): + input_ids = torch.tensor(DUMMY_INPUTS) + input_mask = torch.tensor(DUMMY_MASK) + dummy_inputs = { + 'decoder_input_ids': input_ids, + 'input_ids': input_ids, + 'decoder_attention_mask': input_mask, + } + return dummy_inputs + + def _init_weights(self, module): + """Initialize the weights""" + factor = self.config.initializer_factor # Used for testing weights initialization + if isinstance(module, T5LayerNorm): + module.weight.data.fill_(factor * 1.0) + elif isinstance(module, T5Model): + # Mesh TensorFlow embeddings initialization See + # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L1624 + module.shared.weight.data.normal_(mean=0.0, std=factor * 1.0) + elif isinstance(module, T5DenseReluDense): + # Mesh TensorFlow FF initialization See + # https://github.com/tensorflow/mesh/blob/master/mesh_tensorflow/transformer/transformer_layers.py#L56 + # and + # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L89 + module.wi.weight.data.normal_( + mean=0.0, std=factor * ((self.config.d_model)**-0.5)) + if hasattr(module.wi, 'bias') and module.wi.bias is not None: + module.wi.bias.data.zero_() + module.wo.weight.data.normal_( + mean=0.0, std=factor * ((self.config.d_ff)**-0.5)) + if hasattr(module.wo, 'bias') and module.wo.bias is not None: + module.wo.bias.data.zero_() + elif isinstance(module, T5DenseGatedGeluDense): + module.wi_0.weight.data.normal_( + mean=0.0, std=factor * ((self.config.d_model)**-0.5)) + if hasattr(module.wi_0, 'bias') and module.wi_0.bias is not None: + module.wi_0.bias.data.zero_() + module.wi_1.weight.data.normal_( + mean=0.0, std=factor * ((self.config.d_model)**-0.5)) + if hasattr(module.wi_1, 'bias') and module.wi_1.bias is not None: + module.wi_1.bias.data.zero_() + module.wo.weight.data.normal_( + mean=0.0, std=factor * ((self.config.d_ff)**-0.5)) + if hasattr(module.wo, 'bias') and module.wo.bias is not None: + module.wo.bias.data.zero_() + elif isinstance(module, T5Attention): + # Mesh TensorFlow attention initialization to avoid scaling before + # softmax See + # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136 + d_model = self.config.d_model + key_value_proj_dim = self.config.d_kv + n_heads = self.config.num_heads + module.q.weight.data.normal_( + mean=0.0, std=factor * ((d_model * key_value_proj_dim)**-0.5)) + module.k.weight.data.normal_( + mean=0.0, std=factor * (d_model**-0.5)) + module.v.weight.data.normal_( + mean=0.0, std=factor * (d_model**-0.5)) + module.o.weight.data.normal_( + mean=0.0, std=factor * ((n_heads * key_value_proj_dim)**-0.5)) + if module.has_relative_attention_bias: + module.relative_attention_bias.weight.data.normal_( + mean=0.0, std=factor * ((d_model)**-0.5)) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (T5Attention, T5Stack)): + module.gradient_checkpointing = value + + def _shift_right(self, input_ids): + decoder_start_token_id = self.config.decoder_start_token_id + pad_token_id = self.config.pad_token_id + + assert ( + decoder_start_token_id is not None + ), 'self.model.config.decoder_start_token_id has to be defined.' + + # shift inputs to the right + if is_torch_fx_proxy(input_ids): + # Item assignment is not supported natively for proxies. + shifted_input_ids = torch.full(input_ids.shape[:-1] + (1, ), + decoder_start_token_id) + shifted_input_ids = torch.cat( + [shifted_input_ids, input_ids[..., :-1]], dim=-1) + else: + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids[..., 0] = decoder_start_token_id + + assert pad_token_id is not None, 'self.model.config.pad_token_id has to be defined.' + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + assert torch.all(shifted_input_ids >= 0).item( + ), 'Verify that `shifted_input_ids` has only positive values' + + return shifted_input_ids + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the + label information. num_labels: An optional arg to tell the + model how many classes to initialize. + Method will call utils.parse_label_mapping + if num_labels not supplied. If num_labels is + not found, the model will use the default + setting (2 classes). + + Returns: + The loaded model, which is initialized by + transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.get('model_dir', None) + if model_dir is None: + config = T5Config(**kwargs) + model = cls(config) + else: + model_kwargs = {} + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_kwargs) + model.model_dir = model_dir + return model + + +class T5Stack(T5PreTrainedModel): + + def __init__(self, config, embed_tokens=None): + super().__init__(config) + + self.embed_tokens = embed_tokens + self.is_decoder = config.is_decoder + + self.block = nn.ModuleList([ + T5Block(config, has_relative_attention_bias=bool(i == 0)) + for i in range(config.num_layers) + ]) + self.final_layer_norm = T5LayerNorm( + config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + # Initialize weights and apply final processing + self.post_init() + # Model parallel + self.model_parallel = False + self.device_map = None + self.gradient_checkpointing = False + + def parallelize(self, device_map=None): + r""" + This is an experimental feature and is a subject to change at a + moment's notice. + + Uses a device map to distribute attention modules of the model + across several devices. If no device map is given, it will evenly + distribute blocks across all devices. + + Args: + device_map (`Dict[int, list]`, optional, defaults to None): + A dictionary that maps attention modules to devices. Note + that the embedding module and LMHead are always + automatically mapped to the first device (for esoteric + reasons). That means that the first device should have fewer + attention modules mapped to it than other devices. For + reference, the t5 models have the following number of + attention modules: + + - t5-small: 6 + - t5-base: 12 + - t5-large: 24 + - t5-3b: 24 + - t5-11b: 24 + + Example: + + >>> # Here is an example of a device map on a machine with 4 GPUs + >>> # using t5-3b, which has a total of 24 attention modules: + >>> model = T5ForConditionalGeneration.from_pretrained("t5-3b") + >>> device_map = { + >>> 0: [0, 1, 2], 1: [3, 4, 5, 6, 7, 8, 9], 2: [10, 11, 12, 13, 14, + >>> 15, 16], 3: [17, 18, 19, 20, 21, 22, 23], + >>> } + >>> model.parallelize(device_map) + >>> # all of the parallelize methods in this file are the same + + """ + # Check validity of device_map + self.device_map = ( + get_device_map(len(self.block), range(torch.cuda.device_count())) + if device_map is None else device_map) + assert_device_map(self.device_map, len(self.block)) + self.model_parallel = True + self.first_device = 'cpu' if 'cpu' in self.device_map.keys( + ) else 'cuda:' + str(min(self.device_map.keys())) + self.last_device = 'cuda:' + str(max(self.device_map.keys())) + # Load onto devices + for k, v in self.device_map.items(): + for layer in v: + cuda_device = 'cuda:' + str(k) + self.block[layer] = self.block[layer].to(cuda_device) + + # Set embed_tokens to first layer + self.embed_tokens = self.embed_tokens.to(self.first_device) + # Set final layer norm to last device + self.final_layer_norm = self.final_layer_norm.to(self.last_device) + + def deparallelize(self): + r""" + Moves the model to cpu from a model parallel state. + + Example: + + >>> # On a 4 GPU machine with t5-3b: + >>> model = T5ForConditionalGeneration.from_pretrained("t5-3b") + >>> device_map = { + >>> 0: [0, 1, 2], 1: [3, 4, 5, 6, 7, 8, 9], 2: [10, 11, 12, 13, 14, + >>> 15, 16], 3: [17, 18, 19, 20, 21, 22, 23], + >>> } + >>> model.parallelize(device_map) + >>> # Splits the model across several devices model.deparallelize() + >>> # Put the model back on cpu and + >>> # cleans memory by calling torch.cuda.empty_cache() + >>> # all of the deparallelize methods in this file are the same + """ + self.model_parallel = False + self.device_map = None + self.first_device = 'cpu' + self.last_device = 'cpu' + for i in range(len(self.block)): + self.block[i] = self.block[i].to('cpu') + self.embed_tokens = self.embed_tokens.to('cpu') + self.final_layer_norm = self.final_layer_norm.to('cpu') + torch.cuda.empty_cache() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, new_embeddings): + self.embed_tokens = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + inputs_embeds=None, + head_mask=None, + cross_attn_head_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + # Model parallel + if self.model_parallel: + torch.cuda.set_device(self.first_device) + self.embed_tokens = self.embed_tokens.to(self.first_device) + use_cache = use_cache if use_cache is not None else self.config.use_cache + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + err_msg_prefix = 'decoder_' if self.is_decoder else '' + raise ValueError( + f'You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + err_msg_prefix = 'decoder_' if self.is_decoder else '' + raise ValueError( + f'You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds' + ) + + if inputs_embeds is None: + assert self.embed_tokens is not None, 'You have to initialize the model with valid token embeddings' + inputs_embeds = self.embed_tokens(input_ids) + + batch_size, seq_length = input_shape + + # required mask seq length can be calculated via length of past + mask_seq_length = past_key_values[0][0].shape[ + 2] + seq_length if past_key_values is not None else seq_length + + if use_cache is True: + assert self.is_decoder, f'`use_cache` can only be set to `True` if {self} is used as a decoder' + + if attention_mask is None: + attention_mask = torch.ones(batch_size, mask_seq_length).to( + inputs_embeds.device) + if self.is_decoder and encoder_attention_mask is None and encoder_hidden_states is not None: + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = torch.ones( + batch_size, + encoder_seq_length, + device=inputs_embeds.device, + dtype=torch.long) + + # initialize past_key_values with `None` if past does not exist + if past_key_values is None: + past_key_values = [None] * len(self.block) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape, inputs_embeds.device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=inputs_embeds.device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + head_mask = self.get_head_mask(head_mask, self.config.num_layers) + cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, + self.config.num_layers) + present_key_value_states = () if use_cache else None + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + all_cross_attentions = () if (output_attentions + and self.is_decoder) else None + position_bias = None + encoder_decoder_position_bias = None + + hidden_states = self.dropout(inputs_embeds) + + for i, (layer_module, + past_key_value) in enumerate(zip(self.block, past_key_values)): + layer_head_mask = head_mask[i] + cross_attn_layer_head_mask = cross_attn_head_mask[i] + # Model parallel + if self.model_parallel: + torch.cuda.set_device(hidden_states.device) + # Ensure that attention_mask is always on the same device as hidden_states + if attention_mask is not None: + attention_mask = attention_mask.to(hidden_states.device) + if position_bias is not None: + position_bias = position_bias.to(hidden_states.device) + if encoder_hidden_states is not None: + encoder_hidden_states = encoder_hidden_states.to( + hidden_states.device) + if encoder_extended_attention_mask is not None: + encoder_extended_attention_mask = encoder_extended_attention_mask.to( + hidden_states.device) + if encoder_decoder_position_bias is not None: + encoder_decoder_position_bias = encoder_decoder_position_bias.to( + hidden_states.device) + if layer_head_mask is not None: + layer_head_mask = layer_head_mask.to(hidden_states.device) + if cross_attn_layer_head_mask is not None: + cross_attn_layer_head_mask = cross_attn_layer_head_mask.to( + hidden_states.device) + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return tuple( + module(*inputs, use_cache, output_attentions)) + + return custom_forward + + layer_outputs = checkpoint( + create_custom_forward(layer_module), + hidden_states, + extended_attention_mask, + position_bias, + encoder_hidden_states, + encoder_extended_attention_mask, + encoder_decoder_position_bias, + layer_head_mask, + cross_attn_layer_head_mask, + None, # past_key_value is always None with gradient checkpointing + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + layer_head_mask=layer_head_mask, + cross_attn_layer_head_mask=cross_attn_layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + # layer_outputs is a tuple with: hidden-states, key-value-states, + # (self-attention position bias), (self-attention weights), + # (cross-attention position bias), (cross-attention weights) + if use_cache is False: + layer_outputs = layer_outputs[:1] + ( + None, ) + layer_outputs[1:] + + hidden_states, present_key_value_state = layer_outputs[:2] + + # We share the position biases between the layers - the first layer + # store them layer_outputs = hidden-states, key-value-states + # (self-attention position bias), (self-attention weights), + # (cross-attention position bias), (cross-attention weights) + position_bias = layer_outputs[2] + if self.is_decoder and encoder_hidden_states is not None: + encoder_decoder_position_bias = layer_outputs[ + 4 if output_attentions else 3] + # append next layer key value states + if use_cache: + present_key_value_states = present_key_value_states + ( + present_key_value_state, ) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[3], ) + if self.is_decoder: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[5], ) + + # Model Parallel: If it's the last layer for that device, put things on the next device + if self.model_parallel: + for k, v in self.device_map.items(): + if i == v[-1] and 'cuda:' + str(k) != self.last_device: + hidden_states = hidden_states.to('cuda:' + str(k + 1)) + + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + + # Add last layer + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + present_key_value_states, + all_hidden_states, + all_attentions, + all_cross_attentions, + ] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=present_key_value_states, + hidden_states=all_hidden_states, + attentions=all_attentions, + cross_attentions=all_cross_attentions, + ) + + +# Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask +__HEAD_MASK_WARNING_MSG = """ +The input argument `head_mask` was split into two arguments `head_mask` and +`decoder_head_mask`. Currently, `decoder_head_mask` is set to copy `head_mask`, +but this feature is deprecated and will be removed in future versions. If you do +not want to use any `decoder_head_mask` now, please set `decoder_head_mask = +torch.ones(num_layers, num_heads)`. +""" + + +@MODELS.register_module(group_key=Tasks.backbone, module_name=Models.T5) +class T5Model(T5PreTrainedModel): + """The bare T5 Model transformer outputting raw hidden-states without any + specific head on top. + + The T5 model was proposed in [Exploring the Limits of Transfer Learning with + a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by + Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, + Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder + transformer pre-trained in a text-to-text denoising generative setting. + + This model inherits from [`PreTrainedModel`]. Check the superclass + documentation for the generic methods the library implements for all its + model (such as downloading or saving, resizing the input embeddings, pruning + heads etc.) + + This model is also a PyTorch + [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch + documentation for all matter related to general usage and behavior. + + Parameters: + config ([`T5Config`]): Model configuration class with all the parameters + of the model. + Initializing with a config file does not load the weights associated + with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + """ + _keys_to_ignore_on_load_missing = [ + r'encoder\.embed_tokens\.weight', + r'decoder\.embed_tokens\.weight', + ] + _keys_to_ignore_on_load_unexpected = [ + r'decoder\.block\.0\.layer\.1\.EncDecAttention\.relative_attention_bias\.weight', + ] + + def __init__(self, config: T5Config): + super().__init__(config) + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + encoder_config.is_decoder = False + encoder_config.use_cache = False + encoder_config.is_encoder_decoder = False + self.encoder = T5Stack(encoder_config, self.shared) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + decoder_config.is_encoder_decoder = False + decoder_config.num_layers = config.num_decoder_layers + self.decoder = T5Stack(decoder_config, self.shared) + + # Initialize weights and apply final processing + self.post_init() + + # Model parallel + self.model_parallel = False + self.device_map = None + + def parallelize(self, device_map=None): + self.device_map = ( + get_device_map( + len(self.encoder.block), range(torch.cuda.device_count())) + if device_map is None else device_map) + assert_device_map(self.device_map, len(self.encoder.block)) + self.encoder.parallelize(self.device_map) + self.decoder.parallelize(self.device_map) + self.model_parallel = True + + def deparallelize(self): + self.encoder.deparallelize() + self.decoder.deparallelize() + self.encoder = self.encoder.to('cpu') + self.decoder = self.decoder.to('cpu') + self.model_parallel = False + self.device_map = None + torch.cuda.empty_cache() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + self.encoder.set_input_embeddings(new_embeddings) + self.decoder.set_input_embeddings(new_embeddings) + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of + heads to prune in this layer} See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + decoder_head_mask: Optional[torch.FloatTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + decoder_inputs_embeds: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model + with relative position embeddings so you should be able to pad the + inputs on both the right and the left. + + Indices can be obtained using [`T5Tokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a + look a [T5 Training](./t5#training). + attention_mask (`torch.FloatTensor` of shape `(batch_size, + sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, + target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`T5Tokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for + `decoder_input_ids` generation. If `past_key_values` is used, + optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining + take a look at [T5 Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, + target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in + `decoder_input_ids`. Causal mask will also be used by default. + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, + num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the + encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or + `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the + decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or + `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in + the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, + `optional`: *attentions*) `last_hidden_state` of shape `(batch_size, + sequence_length, hidden_size)` is a sequence of hidden states at the + output of the last layer of the encoder. Used in the cross-attention + of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length + `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only + the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead + of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, + sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, + target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to + directly pass an embedded representation. If `past_key_values` is + used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more + control over how to convert `decoder_input_ids` indices into + associated vectors than the model's internal embedding lookup + matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, + `decoder_inputs_embeds` takes the value of `inputs_embeds`. + + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned + and can be used to speed up decoding (see `past_key_values`). + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain + tuple. + Returns: + + Example: + + >>> from transformers import T5Tokenizer, T5Model + + >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> model = T5Model.from_pretrained("t5-small") + + >>> input_ids = tokenizer( + ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" + >>> ).input_ids # Batch size 1 + >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 + + >>> # forward pass + >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) + >>> last_hidden_states = outputs.last_hidden_state + """ + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask + if head_mask is not None and decoder_head_mask is None: + if self.config.num_layers == self.config.num_decoder_layers: + warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning) + decoder_head_mask = head_mask + + # Encode if needed (training, first prediction pass) + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + elif return_dict and not isinstance(encoder_outputs, + AttentionBackboneModelOutput): + encoder_outputs = AttentionBackboneModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] + if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] + if len(encoder_outputs) > 2 else None, + ) + + hidden_states = encoder_outputs[0] + if self.model_parallel: + torch.cuda.set_device(self.decoder.first_device) + # Set device for model parallelism + if self.model_parallel: + torch.cuda.set_device(self.decoder.first_device) + hidden_states = hidden_states.to(self.decoder.first_device) + if decoder_input_ids is not None: + decoder_input_ids = decoder_input_ids.to( + self.decoder.first_device) + if attention_mask is not None: + attention_mask = attention_mask.to(self.decoder.first_device) + if decoder_attention_mask is not None: + decoder_attention_mask = decoder_attention_mask.to( + self.decoder.first_device) + + # Decode + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + inputs_embeds=decoder_inputs_embeds, + past_key_values=past_key_values, + encoder_hidden_states=hidden_states, + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/T5/configuration.py b/runtime/Lib/site_packages/models/nlp/T5/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..d64793ade57a5d2bd48ecb7eb280c467f68987f4 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/T5/configuration.py @@ -0,0 +1,175 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2020, The T5 Authors and HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" T5 model configuration""" +from typing import Mapping + +from transformers.configuration_utils import PretrainedConfig +from transformers.onnx import OnnxSeq2SeqConfigWithPast + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class T5Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`T5Model`] or a [`TFT5Model`]. It is used to + instantiate a T5 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the T5 + [t5-small](https://huggingface.co/t5-small) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Arguments: + vocab_size (`int`, *optional*, defaults to 32128): + Vocabulary size of the T5 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`T5Model`] or [`TFT5Model`]. + d_model (`int`, *optional*, defaults to 512): + Size of the encoder layers and the pooler layer. + d_kv (`int`, *optional*, defaults to 64): + Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model // + num_heads`. + d_ff (`int`, *optional*, defaults to 2048): + Size of the intermediate feed forward layer in each `T5Block`. + num_layers (`int`, *optional*, defaults to 6): + Number of hidden layers in the Transformer encoder. + num_decoder_layers (`int`, *optional*): + Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not set. + num_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer encoder. + relative_attention_num_buckets (`int`, *optional*, defaults to 32): + The number of buckets to use for each attention layer. + relative_attention_max_distance (`int`, *optional*, defaults to 128): + The maximum distance of the longer sequences for the bucket separation. + dropout_rate (`float`, *optional*, defaults to 0.1): + The ratio for all dropout layers. + layer_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + initializer_factor (`float`, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + feed_forward_proj (`string`, *optional*, defaults to `"relu"`): + Type of feed forward layer to be used. Should be one of `"relu"` or `"gated-gelu"`. T5v1.1 uses the + `"gated-gelu"` feed forward projection. Original T5 uses `"relu"`. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + """ + model_type = 't5' + keys_to_ignore_at_inference = ['past_key_values'] + attribute_map = { + 'hidden_size': 'd_model', + 'num_attention_heads': 'num_heads', + 'num_hidden_layers': 'num_layers' + } + + def __init__(self, + vocab_size=32128, + d_model=512, + d_kv=64, + d_ff=2048, + num_layers=6, + num_decoder_layers=None, + num_heads=8, + relative_attention_num_buckets=32, + relative_attention_max_distance=128, + dropout_rate=0.1, + layer_norm_epsilon=1e-6, + initializer_factor=1.0, + feed_forward_proj='relu', + is_encoder_decoder=True, + use_cache=True, + pad_token_id=0, + eos_token_id=1, + **kwargs): + self.vocab_size = vocab_size + self.d_model = d_model + self.d_kv = d_kv + self.d_ff = d_ff + self.num_layers = num_layers + self.num_decoder_layers = (num_decoder_layers if num_decoder_layers + is not None else self.num_layers + ) # default = symmetry + self.num_heads = num_heads + self.relative_attention_num_buckets = relative_attention_num_buckets + self.relative_attention_max_distance = relative_attention_max_distance + self.dropout_rate = dropout_rate + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_factor = initializer_factor + self.feed_forward_proj = feed_forward_proj + self.use_cache = use_cache + + act_info = self.feed_forward_proj.split('-') + self.dense_act_fn = act_info[-1] + self.is_gated_act = act_info[0] == 'gated' + + if len(act_info) > 1 and act_info[0] != 'gated' or len(act_info) > 2: + raise ValueError( + f'`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.' + 'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. ' + "'gated-gelu' or 'relu'") + + # for backwards compatibility + if feed_forward_proj == 'gated-gelu': + self.dense_act_fn = 'gelu_new' + + super().__init__( + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + **kwargs, + ) + + +class T5OnnxConfig(OnnxSeq2SeqConfigWithPast): + + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + common_inputs = { + 'input_ids': { + 0: 'batch', + 1: 'encoder_sequence' + }, + 'attention_mask': { + 0: 'batch', + 1: 'encoder_sequence' + }, + } + if self.use_past: + common_inputs['attention_mask'][ + 1] = 'past_encoder_sequence + sequence' + common_inputs['decoder_input_ids'] = {0: 'batch'} + common_inputs['decoder_attention_mask'] = { + 0: 'batch', + 1: 'past_decoder_sequence + sequence' + } + else: + common_inputs['decoder_input_ids'] = { + 0: 'batch', + 1: 'decoder_sequence' + } + common_inputs['decoder_attention_mask'] = { + 0: 'batch', + 1: 'decoder_sequence' + } + + if self.use_past: + self.fill_with_past_key_values_(common_inputs, direction='inputs') + + return common_inputs + + @property + def default_onnx_opset(self) -> int: + return 13 diff --git a/runtime/Lib/site_packages/models/nlp/T5/text2text_generation.py b/runtime/Lib/site_packages/models/nlp/T5/text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..aa2826a8ba1b3715e39563f33603a217ff8c7115 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/T5/text2text_generation.py @@ -0,0 +1,462 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import warnings +from typing import Optional, Tuple, Union + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.utils.model_parallel_utils import (assert_device_map, + get_device_map) + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import (AttentionBackboneModelOutput, Seq2SeqLMOutput, + TokenGeneratorOutput) +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .backbone import T5PreTrainedModel, T5Stack +from .configuration import T5Config + +logger = get_logger() + +# Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask +__HEAD_MASK_WARNING_MSG = """ +The input argument `head_mask` was split into two arguments `head_mask` and +`decoder_head_mask`. Currently, `decoder_head_mask` is set to copy `head_mask`, +but this feature is deprecated and will be removed in future versions. If you do +not want to use any `decoder_head_mask` now, please set `decoder_head_mask = +torch.ones(num_layers, num_heads)`. +""" + + +@MODELS.register_module( + group_key=Tasks.text2text_generation, + module_name=Models.T5, +) +class T5ForConditionalGeneration(T5PreTrainedModel): + _keys_to_ignore_on_load_missing = [ + r'encoder\.embed_tokens\.weight', + r'decoder\.embed_tokens\.weight', + r'lm_head\.weight', + ] + _keys_to_ignore_on_load_unexpected = [ + r'decoder\.block\.0\.layer\.1\.EncDecAttention\.relative_attention_bias\.weight', + ] + + def __init__(self, config: T5Config, device_map=None, **kwargs): + super().__init__(config) + self.model_dim = config.d_model + + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + encoder_config.is_decoder = False + encoder_config.use_cache = False + encoder_config.is_encoder_decoder = False + self.encoder = T5Stack(encoder_config, self.shared) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + decoder_config.is_encoder_decoder = False + decoder_config.num_layers = config.num_decoder_layers + self.decoder = T5Stack(decoder_config, self.shared) + + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + # Model parallel + self.model_parallel = False + if device_map == 'auto': + self.parallelize() + + def parallelize(self, device_map=None): + self.device_map = ( + get_device_map( + len(self.encoder.block), range(torch.cuda.device_count())) + if device_map is None else device_map) + assert_device_map(self.device_map, len(self.encoder.block)) + self.encoder.parallelize(self.device_map) + self.decoder.parallelize(self.device_map) + self.lm_head = self.lm_head.to(self.decoder.first_device) + self.model_parallel = True + + def deparallelize(self): + self.encoder.deparallelize() + self.decoder.deparallelize() + self.encoder = self.encoder.to('cpu') + self.decoder = self.decoder.to('cpu') + self.lm_head = self.lm_head.to('cpu') + self.model_parallel = False + self.device_map = None + torch.cuda.empty_cache() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + self.encoder.set_input_embeddings(new_embeddings) + self.decoder.set_input_embeddings(new_embeddings) + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def get_output_embeddings(self): + return self.lm_head + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + def forward(self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + decoder_head_mask: Optional[torch.FloatTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.Tensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. T5 is a model + with relative position embeddings so you should be able to pad the + inputs on both the right and the left. + + Indices can be obtained using [`T5Tokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a + look a [T5 Training](./t5#training). + attention_mask (`torch.FloatTensor` of shape `(batch_size,sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`T5Tokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + T5 uses the `pad_token_id` as the starting token for + `decoder_input_ids` generation. If `past_key_values` is used, + optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining + take a look at [T5 Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in + `decoder_input_ids`. Causal mask will also be used by default. + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the + encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or + `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the + decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in + the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, + `optional`: *attentions*) `last_hidden_state` of shape `(batch_size, + sequence_length, hidden_size)` is a sequence of hidden states at the + output of the last layer of the encoder. Used in the cross-attention + of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length + `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only + the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead + of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, + *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to + directly pass an embedded representation. If `past_key_values` is + used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more + control over how to convert `decoder_input_ids` indices into + associated vectors than the model's internal embedding lookup + matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, + `decoder_inputs_embeds` takes the value of `inputs_embeds`. + + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned + and can be used to speed up decoding (see `past_key_values`). + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain + tuple. + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. + Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All + labels set to `-100` are ignored (masked), the loss is only computed + for labels in `[0, ..., config.vocab_size]` + + Returns: + + Examples: + + >>> from transformers import T5Tokenizer, T5ForConditionalGeneration + + >>> tokenizer = T5Tokenizer.from_pretrained("t5-small") + >>> model = T5ForConditionalGeneration.from_pretrained("t5-small") + + >>> # training + >>> input_ids = tokenizer("The walks in park", return_tensors="pt").input_ids + >>> labels = tokenizer(" cute dog the ", return_tensors="pt").input_ids + >>> outputs = model(input_ids=input_ids, labels=labels) + >>> loss = outputs.loss + >>> logits = outputs.logits + + >>> # inference + >>> input_ids = tokenizer( + ... "summarize: studies have shown that owning a dog is good for you", return_tensors="pt" + >>> ).input_ids # Batch size 1 + >>> outputs = model.generate(input_ids) + >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) + >>> # studies have shown that owning a dog is good for you. + """ + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask + if head_mask is not None and decoder_head_mask is None: + if self.config.num_layers == self.config.num_decoder_layers: + warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning) + decoder_head_mask = head_mask + + # Encode if needed (training, first prediction pass) + if encoder_outputs is None: + # Convert encoder inputs in embeddings if needed + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + elif return_dict and not isinstance(encoder_outputs, + AttentionBackboneModelOutput): + encoder_outputs = AttentionBackboneModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] + if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] + if len(encoder_outputs) > 2 else None, + ) + + hidden_states = encoder_outputs[0] + + if self.model_parallel: + torch.cuda.set_device(self.decoder.first_device) + + if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None: + # get decoder inputs from shifting lm labels to the right + decoder_input_ids = self._shift_right(labels) + + # Set device for model parallelism + if self.model_parallel: + torch.cuda.set_device(self.decoder.first_device) + hidden_states = hidden_states.to(self.decoder.first_device) + if decoder_input_ids is not None: + decoder_input_ids = decoder_input_ids.to( + self.decoder.first_device) + if attention_mask is not None: + attention_mask = attention_mask.to(self.decoder.first_device) + if decoder_attention_mask is not None: + decoder_attention_mask = decoder_attention_mask.to( + self.decoder.first_device) + + # Decode + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + inputs_embeds=decoder_inputs_embeds, + past_key_values=past_key_values, + encoder_hidden_states=hidden_states, + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = decoder_outputs[0] + + # Set device for model parallelism + if self.model_parallel: + torch.cuda.set_device(self.encoder.first_device) + self.lm_head = self.lm_head.to(self.encoder.first_device) + sequence_output = sequence_output.to(self.lm_head.weight.device) + + if self.config.tie_word_embeddings: + # Rescale output before projecting on vocab See + # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 + sequence_output = sequence_output * (self.model_dim**-0.5) + + lm_logits = self.lm_head(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-100) + loss = loss_fct( + lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1)) + # TODO(thom): Add z_loss + # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 + + if not return_dict: + output = (lm_logits, ) + decoder_outputs[1:] + encoder_outputs + return ((loss, ) + output) if loss is not None else output + + return Seq2SeqLMOutput( + loss=loss, + logits=lm_logits, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + def prepare_inputs_for_generation(self, + input_ids, + past=None, + attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, + use_cache=None, + encoder_outputs=None, + **kwargs): + + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + + return { + 'decoder_input_ids': input_ids, + 'past_key_values': past, + 'encoder_outputs': encoder_outputs, + 'attention_mask': attention_mask, + 'head_mask': head_mask, + 'decoder_head_mask': decoder_head_mask, + 'cross_attn_head_mask': cross_attn_head_mask, + 'use_cache': use_cache, + } + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return self._shift_right(labels) + + def generate( + self, + *args, + **kwargs, + ): + output = super().generate(*args, **kwargs) + return TokenGeneratorOutput( + sequences=output if isinstance(output, torch.Tensor) else output[0] + ) + + def _reorder_cache(self, past, beam_idx): + # if decoder past is not included in output + # speedy decoding is disabled and no need to reorder + if past is None: + logger.warning( + 'You might want to consider setting `use_cache=True` to speed up decoding' + ) + return past + + reordered_decoder_past = () + for layer_past_states in past: + # get the correct batch idx from layer past batch dim + # batch dim of `past` is at 2nd position + reordered_layer_past_states = () + for layer_past_state in layer_past_states: + # need to set correct `past` for each of the four key / value states + reordered_layer_past_states = reordered_layer_past_states + ( + layer_past_state.index_select( + 0, beam_idx.to(layer_past_state.device)), ) + + assert reordered_layer_past_states[0].shape == layer_past_states[ + 0].shape + assert len(reordered_layer_past_states) == len(layer_past_states) + + reordered_decoder_past = reordered_decoder_past + ( + reordered_layer_past_states, ) + return reordered_decoder_past diff --git a/runtime/Lib/site_packages/models/nlp/__init__.py b/runtime/Lib/site_packages/models/nlp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9878217c19b58ea61ee1a960e1a9ce6047e4ec0e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/__init__.py @@ -0,0 +1,194 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .bart import BartForTextErrorCorrection + from .bert import ( + BertForMaskedLM, + BertForTextRanking, + BertForSentenceEmbedding, + BertForSequenceClassification, + BertForTokenClassification, + BertForDocumentSegmentation, + BertModel, + BertConfig, + SiameseUieModel, + ) + from .bloom import BloomModel, BloomForTextGeneration + from .codegeex import CodeGeeXForCodeTranslation, CodeGeeXForCodeGeneration + from .glm_130b import GLM130bForTextGeneration + from .csanmt import CsanmtForTranslation + from .canmt import CanmtForTranslation + from .polylm import PolyLMForTextGeneration + from .deberta_v2 import DebertaV2ForMaskedLM, DebertaV2Model + from .chatglm import ChatGLMForConditionalGeneration, ChatGLMTokenizer, ChatGLMConfig + from .chatglm2 import ChatGLM2ForConditionalGeneration, ChatGLM2Tokenizer, ChatGLM2Config + from .gpt_neo import GPTNeoModel + from .gpt2 import GPT2Model + from .gpt3 import GPT3ForTextGeneration, DistributedGPT3 + from .gpt_moe import GPTMoEForTextGeneration, DistributedGPTMoE + from .heads import TextClassificationHead + from .hf_transformers import TransformersModel + from .lstm import ( + LSTMModel, + LSTMForTokenClassificationWithCRF, + ) + from .megatron_bert import ( + MegatronBertConfig, + MegatronBertForMaskedLM, + MegatronBertModel, + ) + from .mglm import MGLMForTextSummarization + from .palm_v2 import PalmForTextGeneration + from .plug_mental import (PlugMentalConfig, PlugMentalModel, + PlugMentalForSequenceClassification) + from .ponet import PoNetForMaskedLM, PoNetModel, PoNetConfig + from .space import SpaceForDialogIntent, SpaceForDialogModeling, SpaceForDST + from .space_T_cn import TableQuestionAnswering + from .space_T_en import StarForTextToSql + from .structbert import ( + SbertForFaqQuestionAnswering, + SbertForMaskedLM, + SbertForSequenceClassification, + SbertForTokenClassification, + SbertModel, + ) + from .T5 import T5ForConditionalGeneration + from .task_models import ( + ModelForFeatureExtraction, + ModelForInformationExtraction, + ModelForTextClassification, + SingleBackboneTaskModelBase, + ModelForTextGeneration, + ModelForTextRanking, + ModelForTokenClassification, + ModelForTokenClassificationWithCRF, + ModelForMachineReadingComprehension, + ) + from .unite import UniTEForTranslationEvaluation + from .use import UserSatisfactionEstimation + from .veco import (VecoConfig, VecoForMaskedLM, + VecoForSequenceClassification, + VecoForTokenClassification, VecoModel) + from .dgds import (DocumentGroundedDialogGenerateModel, + DocumentGroundedDialogRetrievalModel, + DocumentGroundedDialogRerankModel) + from .xlm_roberta import XLMRobertaConfig, XLMRobertaModel + from .llama import LlamaForTextGeneration, LlamaConfig, LlamaModel, LlamaTokenizer, LlamaTokenizerFast + from .llama2 import Llama2ForTextGeneration, Llama2Config, Llama2Model, Llama2Tokenizer, Llama2TokenizerFast + from .qwen import QWenForTextGeneration, QWenConfig, QWenModel, QWenTokenizer +else: + _import_structure = { + 'bart': ['BartForTextErrorCorrection'], + 'bert': [ + 'BertForMaskedLM', + 'BertForTextRanking', + 'BertForSentenceEmbedding', + 'BertForSequenceClassification', + 'BertForTokenClassification', + 'BertForDocumentSegmentation', + 'BertModel', + 'BertConfig', + 'SiameseUieModel', + ], + 'bloom': ['BloomModel', 'BloomForTextGeneration'], + 'csanmt': ['CsanmtForTranslation'], + 'canmt': ['CanmtForTranslation'], + 'polylm': ['PolyLMForTextGeneration'], + 'codegeex': + ['CodeGeeXForCodeTranslation', 'CodeGeeXForCodeGeneration'], + 'glm_130b': ['GLM130bForTextGeneration'], + 'deberta_v2': ['DebertaV2ForMaskedLM', 'DebertaV2Model'], + 'chatglm': [ + 'ChatGLMForConditionalGeneration', 'ChatGLMTokenizer', + 'ChatGLMConfig' + ], + 'chatglm2': [ + 'ChatGLM2ForConditionalGeneration', 'ChatGLM2Tokenizer', + 'ChatGLM2Config' + ], + 'heads': ['TextClassificationHead'], + 'hf_transformers': ['TransformersModel'], + 'gpt2': ['GPT2Model'], + 'gpt3': ['GPT3ForTextGeneration', 'DistributedGPT3'], + 'gpt_moe': ['GPTMoEForTextGeneration', 'DistributedGPTMoE'], + 'gpt_neo': ['GPTNeoModel'], + 'structbert': [ + 'SbertForFaqQuestionAnswering', + 'SbertForMaskedLM', + 'SbertForSequenceClassification', + 'SbertForTokenClassification', + 'SbertModel', + ], + 'veco': [ + 'VecoConfig', + 'VecoForMaskedLM', + 'VecoForSequenceClassification', + 'VecoForTokenClassification', + 'VecoModel', + ], + 'lstm': [ + 'LSTM', + 'LSTMForTokenClassificationWithCRF', + ], + 'megatron_bert': [ + 'MegatronBertConfig', + 'MegatronBertForMaskedLM', + 'MegatronBertModel', + ], + 'mglm': ['MGLMForTextSummarization'], + 'palm_v2': ['PalmForTextGeneration'], + 'plug_mental': [ + 'PlugMentalConfig', + 'PlugMentalModel', + 'PlugMentalForSequenceClassification', + ], + 'ponet': ['PoNetForMaskedLM', 'PoNetModel', 'PoNetConfig'], + 'space_T_en': ['StarForTextToSql'], + 'space_T_cn': ['TableQuestionAnswering'], + 'space': + ['SpaceForDialogIntent', 'SpaceForDialogModeling', 'SpaceForDST'], + 'task_models': [ + 'ModelForFeatureExtraction', + 'ModelForInformationExtraction', + 'ModelForTextClassification', + 'SingleBackboneTaskModelBase', + 'ModelForTextGeneration', + 'ModelForTextRanking', + 'ModelForTokenClassification', + 'ModelForTokenClassificationWithCRF', + 'ModelForMachineReadingComprehension', + ], + 'sentence_embedding': ['SentenceEmbedding'], + 'T5': ['T5ForConditionalGeneration'], + 'unite': ['UniTEForTranslationEvaluation'], + 'use': ['UserSatisfactionEstimation'], + 'dgds': [ + 'DocumentGroundedDialogGenerateModel', + 'DocumentGroundedDialogRetrievalModel', + 'DocumentGroundedDialogRerankModel' + ], + 'xlm_roberta': ['XLMRobertaConfig', 'XLMRobertaModel'], + 'llama': [ + 'LlamaForTextGeneration', 'LlamaConfig', 'LlamaModel', + 'LlamaTokenizer', 'LlamaTokenizerFast' + ], + 'llama2': [ + 'Llama2ForTextGeneration', 'Llama2Config', 'Llama2Model', + 'Llama2Tokenizer', 'Llama2TokenizerFast' + ], + 'qwen': + ['QWenForTextGeneration', 'QWenConfig', 'QWenModel', 'QWenTokenizer'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/bart/__init__.py b/runtime/Lib/site_packages/models/nlp/bart/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..31912efce0d5251d56310b91181c3f20ed9dbdc1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bart/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .text_error_correction import BartForTextErrorCorrection diff --git a/runtime/Lib/site_packages/models/nlp/bart/text_error_correction.py b/runtime/Lib/site_packages/models/nlp/bart/text_error_correction.py new file mode 100644 index 0000000000000000000000000000000000000000..97c3a7a9e7b8624d79ce583a801c575039f81f85 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bart/text_error_correction.py @@ -0,0 +1,86 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path as osp +from typing import Any, Dict + +import torch.cuda + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import TextErrorCorrectionOutput +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['BartForTextErrorCorrection'] + + +@MODELS.register_module(Tasks.text_error_correction, module_name=Models.bart) +class BartForTextErrorCorrection(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir=model_dir, *args, **kwargs) + """initialize the text error correction model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + ckpt_name = ModelFile.TORCH_MODEL_FILE + local_model = osp.join(model_dir, ckpt_name) + + bart_vocab_dir = model_dir + # turn on cuda if GPU is available + from fairseq import checkpoint_utils, utils + if torch.cuda.is_available(): + self._device = torch.device('cuda') + else: + self._device = torch.device('cpu') + self.use_fp16 = kwargs[ + 'use_fp16'] if 'use_fp16' in kwargs and torch.cuda.is_available()\ + else False + + overrides = { + 'data': bart_vocab_dir, + 'beam': 2, + } + models, cfg, task = checkpoint_utils.load_model_ensemble_and_task( + utils.split_paths(local_model), arg_overrides=overrides) + # Move models to GPU + for model in models: + model.eval() + model.to(self._device) + if self.use_fp16: + model.half() + model.prepare_for_inference_(cfg) + self.models = models + # Initialize generator + self.generator = task.build_generator(models, 'translation') + + self.task = task + + def forward(self, input: Dict[str, Dict]) -> TextErrorCorrectionOutput: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data which contains following: + - src_tokens: tensor with shape (2478,242,24,4), + - src_lengths: tensor with shape (4) + + + Returns: + Dict[str, Tensor]: results which contains following: + - predictions: tokens need to be decode by tokenizer with shape [1377, 4959, 2785, 6392...] + """ + import fairseq.utils + + batch_size = input['src_tokens'].size(0) + + input = {'net_input': input} + if torch.cuda.is_available(): + input = fairseq.utils.move_to_cuda(input, device=self._device) + + translations = self.task.inference_step(self.generator, self.models, + input) + batch_preds = [] + for i in range(batch_size): + # get 1-best List[Tensor] + batch_preds.append(translations[i][0]['tokens']) + return TextErrorCorrectionOutput(predictions=batch_preds) diff --git a/runtime/Lib/site_packages/models/nlp/bert/__init__.py b/runtime/Lib/site_packages/models/nlp/bert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..00373f1c704cecf2036b5435449dcd5f62843eb0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/__init__.py @@ -0,0 +1,45 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import ( + BertLayer, + BertModel, + BertPreTrainedModel, + ) + from .configuration import BertConfig + from .fill_mask import BertForMaskedLM + from .text_ranking import BertForTextRanking + from .sentence_embedding import BertForSentenceEmbedding + from .text_classification import BertForSequenceClassification + from .token_classification import BertForTokenClassification + from .document_segmentation import BertForDocumentSegmentation + from .siamese_uie import SiameseUieModel + from .word_alignment import MBertForWordAlignment +else: + _import_structure = { + 'backbone': [ + 'BertModel', + 'BertPreTrainedModel', + ], + 'configuration': ['BertConfig'], + 'fill_mask': ['BertForMaskedLM'], + 'text_ranking': ['BertForTextRanking'], + 'sentence_embedding': ['BertForSentenceEmbedding'], + 'text_classification': ['BertForSequenceClassification'], + 'token_classification': ['BertForTokenClassification'], + 'document_segmentation': ['BertForDocumentSegmentation'], + 'siamese_uie': ['SiameseUieModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/bert/backbone.py b/runtime/Lib/site_packages/models/nlp/bert/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..827a00d8ce7f0a83398f56f2536ccf3e6303fd4b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/backbone.py @@ -0,0 +1,933 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model. """ + +import math + +import torch +import torch.utils.checkpoint +from packaging import version +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import BertConfig + +logger = get_logger() + +_CONFIG_FOR_DOC = 'BertConfig' + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model + # variable name and be able to load any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and + # exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + if version.parse(torch.__version__) > version.parse('1.6.0'): + self.register_buffer( + 'token_type_ids', + torch.zeros(self.position_ids.size(), dtype=torch.long), + persistent=False, + ) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + # Setting the token_type_ids to the registered buffer in constructor + # where it is all zeros, which usually occurs when its auto-generated, + # registered buffer helps users when tracing the model without passing + # token_type_ids, solves issue #5664 + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = position_embedding_type or getattr( + config, 'position_embedding_type', 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all + # cross attention key/value_states. Further calls to cross_attention + # layer can then reuse all cross-attention key/value_states (first + # "if" case) if uni-directional self-attention (decoder) save + # Tuple(torch.Tensor, torch.Tensor) of all previous decoder + # key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected + # key/value_states (third "elif" case) if encoder bi-directional + # self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + self.self = BertSelfAttention( + config, position_embedding_type=position_embedding_type) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BertAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = BertAttention( + config, position_embedding_type='absolute') + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise ValueError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated ' + f'with cross-attention layers by setting `config.add_cross_attention=True`' + ) + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [BertLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class BertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface + for downloading and loading pretrained models. + """ + + config_class = BertConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, BertEncoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels not supplied. + If num_labels is not found, the model will use the default setting (2 classes). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + if model_dir is None: + config = BertConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + model.model_dir = model_dir + return model + + +@MODELS.register_module(group_key=Tasks.backbone, module_name=Models.bert) +class BertModel(BertPreTrainedModel): + """The Bert Model transformer outputting raw hidden-states without any + specific head on top. + + This model inherits from [`PreTrainedModel`]. Check the superclass + documentation for the generic methods the library implements for all its + model (such as downloading or saving, resizing the input embeddings, pruning + heads etc.) + + This model is also a PyTorch + [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch + documentation for all matter related to general usage and behavior. + + Parameters: + config ([`BertConfig`]): Model configuration class with all the + parameters of the model. + Initializing with a config file does not load the weights associated + with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a + decoder, in which case a layer of cross-attention is added between the + self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam + Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz + Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the + `is_decoder` argument of the configuration set to `True`. To be used in a + Seq2Seq model, the model needs to initialized with both `is_decoder` + argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` + is then expected as an input to the forward pass. + + + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder(config) + + self.pooler = BertPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + @classmethod + def _instantiate(cls, model_dir=None, add_pooling_layer=True, **config): + config = BertConfig(**config) + model = cls(config, add_pooling_layer) + return model + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs) -> AttentionBackboneModelOutput: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`BertTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the + inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position + embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, + num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask + values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, + *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a + plain tuple. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, + sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the + encoder. Used in the cross-attention if the model is configured as a + decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, + sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of + the encoder input. This mask is used in the cross-attention if the + model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length + `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only + the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead + of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned + and can be used to speed up decoding (see `past_key_values`). + Others (**kwargs) + some additional parameters might passed in from upstream pipeline, + which not influence the results. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + + if token_type_ids is None: + if hasattr(self.embeddings, 'token_type_ids'): + buffered_token_type_ids = self.embeddings.token_type_ids[:, : + seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + batch_size, seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return AttentionBackboneModelOutput( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + def extract_sequence_outputs(self, outputs): + return outputs['last_hidden_state'] + + def extract_pooled_outputs(self, outputs): + return outputs['pooler_output'] diff --git a/runtime/Lib/site_packages/models/nlp/bert/configuration.py b/runtime/Lib/site_packages/models/nlp/bert/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f837d40c7d6bd6aa705874f829d62b98089250 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/configuration.py @@ -0,0 +1,163 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERT model configuration """ +from collections import OrderedDict +from typing import Mapping + +from transformers.configuration_utils import PretrainedConfig +from transformers.onnx import OnnxConfig + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class BertConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a + [`BertModel`] or a [`TFBertModel`]. It is used to instantiate a BERT model + according to the specified arguments, defining the model architecture. + Instantiating a configuration with the defaults will yield a similar + configuration to that of the BERT + [bert-base-uncased](https://huggingface.co/bert-base-uncased) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to + control the model outputs. Read the documentation from [`PretrainedConfig`] + for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different + tokens that can be represented by the `inputs_ids` passed when + calling [`BertModel`] or [`TFBertModel`]. + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the + Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) + layer in the Transformer encoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the + encoder and pooler. If string, `"gelu"`, `"relu"`, `"silu"` and + `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the + embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. + Typically set this to something large just in case (e.g., 512 or + 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling + [`BertModel`] or [`TFBertModel`]. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (`str`, *optional*, defaults to `"absolute"`): + Type of position embedding. Choose one of `"absolute"`, + `"relative_key"`, `"relative_key_query"`. For positional embeddings + use `"absolute"`. For more information on `"relative_key"`, please + refer to [Self-Attention with Relative Position Representations + (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more + information on `"relative_key_query"`, please refer to *Method 4* in + [Improve Transformer Models with Better Relative Position Embeddings + (Huang et al.)](https://arxiv.org/abs/2009.13658). + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values + attentions (not used by all models). Only relevant if + `config.is_decoder=True`. + classifier_dropout (`float`, *optional*): + The dropout ratio for the classification head. + + Examples: + + >>> from transformers import BertModel, BertConfig + + >>> # Initializing a BERT bert-base-uncased style configuration + >>> configuration = BertConfig() + + >>> # Initializing a model from the bert-base-uncased style configuration + >>> model = BertModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + model_type = 'bert' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + + +class BertOnnxConfig(OnnxConfig): + + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + return OrderedDict([ + ('input_ids', { + 0: 'batch', + 1: 'sequence' + }), + ('attention_mask', { + 0: 'batch', + 1: 'sequence' + }), + ('token_type_ids', { + 0: 'batch', + 1: 'sequence' + }), + ]) diff --git a/runtime/Lib/site_packages/models/nlp/bert/document_segmentation.py b/runtime/Lib/site_packages/models/nlp/bert/document_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..0f2f288062e2b03940bd07186807fe773cdcb354 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/document_segmentation.py @@ -0,0 +1,112 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.builder import MODELS +from modelscope.models.nlp.ponet import PoNetConfig +from modelscope.outputs import AttentionTokenClassificationModelOutput +from modelscope.utils.constant import Tasks +from .backbone import BertModel, BertPreTrainedModel +from .configuration import BertConfig + +__all__ = ['BertForDocumentSegmentation'] + + +@MODELS.register_module( + Tasks.document_segmentation, module_name=Models.bert_for_ds) +class BertForDocumentSegmentation(BertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.sentence_pooler_type = None + self.bert = BertModel(config, add_pooling_layer=False) + + classifier_dropout = config.hidden_dropout_prob + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + self.class_weights = None + self.init_weights() + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + sentence_attention_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None): + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + if self.sentence_pooler_type is not None: + raise NotImplementedError + else: + sequence_output = self.dropout(sequence_output) + + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss(weight=self.class_weights) + if sentence_attention_mask is not None: + active_loss = sentence_attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + @classmethod + def _instantiate(cls, model_dir, model_config: Dict[str, Any], **kwargs): + if model_config['type'] == 'bert': + config = BertConfig.from_pretrained(model_dir, num_labels=2) + elif model_config['type'] == 'ponet': + config = PoNetConfig.from_pretrained(model_dir, num_labels=2) + else: + raise ValueError( + f'Expected config type bert and ponet, which is : {model_config["type"]}' + ) + model = super(Model, cls).from_pretrained( + model_dir, from_tf=False, config=config) + model.model_dir = model_dir + model.model_cfg = model_config + return model diff --git a/runtime/Lib/site_packages/models/nlp/bert/fill_mask.py b/runtime/Lib/site_packages/models/nlp/bert/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..5c625ecaf24b6ed2a93b4e5526638be2be607d21 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/fill_mask.py @@ -0,0 +1,16 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from modelscope.metainfo import Heads, Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.fill_mask import ModelForFillMask +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.fill_mask, module_name=Models.bert) +class BertForMaskedLM(ModelForFillMask): + + base_model_type = Models.bert + head_type = Heads.bert_mlm diff --git a/runtime/Lib/site_packages/models/nlp/bert/sentence_embedding.py b/runtime/Lib/site_packages/models/nlp/bert/sentence_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..b7df5ef9b4404bbdfcdd9137e0fa747a2bff348c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/sentence_embedding.py @@ -0,0 +1,156 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.builder import MODELS +from modelscope.outputs import SentencEmbeddingModelOutput +from modelscope.utils.constant import Tasks +from .backbone import BertModel, BertPreTrainedModel + + +class Pooler(nn.Module): + """ + Parameter-free poolers to get the sentence embedding + 'cls': [CLS] representation with BERT/RoBERTa's MLP pooler. + 'cls_before_pooler': [CLS] representation without the original MLP pooler. + 'avg': average of the last layers' hidden states at each token. + 'avg_top2': average of the last two layers. + 'avg_first_last': average of the first and the last layers. + """ + + def __init__(self, pooler_type): + super().__init__() + self.pooler_type = pooler_type + assert self.pooler_type in [ + 'cls', 'avg', 'avg_top2', 'avg_first_last' + ], 'unrecognized pooling type %s' % self.pooler_type + + def forward(self, outputs, attention_mask): + last_hidden = outputs.last_hidden_state + hidden_states = outputs.hidden_states + + if self.pooler_type in ['cls']: + return last_hidden[:, 0] + elif self.pooler_type == 'avg': + return ((last_hidden * attention_mask.unsqueeze(-1)).sum(1) + / attention_mask.sum(-1).unsqueeze(-1)) + elif self.pooler_type == 'avg_first_last': + first_hidden = hidden_states[1] + last_hidden = hidden_states[-1] + pooled_result = ((first_hidden + last_hidden) / 2.0 + * attention_mask.unsqueeze(-1) + ).sum(1) / attention_mask.sum(-1).unsqueeze(-1) + return pooled_result + elif self.pooler_type == 'avg_top2': + second_last_hidden = hidden_states[-2] + last_hidden = hidden_states[-1] + pooled_result = ((last_hidden + second_last_hidden) / 2.0 + * attention_mask.unsqueeze(-1) + ).sum(1) / attention_mask.sum(-1).unsqueeze(-1) + return pooled_result + else: + raise NotImplementedError + + +@MODELS.register_module(Tasks.sentence_embedding, module_name=Models.bert) +class BertForSentenceEmbedding(BertPreTrainedModel): + + def __init__(self, config, **kwargs): + super().__init__(config) + self.config = config + self.pooler_type = kwargs.get('emb_pooler_type', 'cls') + self.pooler = Pooler(self.pooler_type) + self.normalize = kwargs.get('normalize', False) + setattr(self, self.base_model_prefix, + BertModel(config, add_pooling_layer=False)) + + def forward(self, query=None, docs=None, labels=None): + r""" + Args: + query (:obj: `dict`): Dict of pretrained models's input for the query sequence. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + docs (:obj: `dict`): Dict of pretrained models's input for the query sequence. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + Returns: + Returns `modelscope.outputs.SentencEmbeddingModelOutput + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_corom_sentence-embedding_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_corom_sentence-embedding_chinese-base') + >>> print(model(**preprocessor('source_sentence':['This is a test']))) + """ + query_embeddings, doc_embeddings = None, None + if query is not None: + query_embeddings = self.encode(**query) + if docs is not None: + doc_embeddings = self.encode(**docs) + outputs = SentencEmbeddingModelOutput( + query_embeddings=query_embeddings, doc_embeddings=doc_embeddings) + if query_embeddings is None or doc_embeddings is None: + return outputs + if self.base_model.training: + loss_fct = nn.CrossEntropyLoss() + scores = torch.matmul(query_embeddings, doc_embeddings.T) + if labels is None: + labels = torch.arange( + scores.size(0), device=scores.device, dtype=torch.long) + labels = labels * ( + doc_embeddings.size(0) // query_embeddings.size(0)) + loss = loss_fct(scores, labels) + outputs.loss = loss + return outputs + + def encode( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + outputs = self.base_model.forward( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + outputs = self.pooler(outputs, attention_mask) + if self.normalize: + outputs = F.normalize(outputs, p=2, dim=-1) + return outputs + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + model_dir = kwargs.get('model_dir') + model_kwargs = { + 'emb_pooler_type': kwargs.get('emb_pooler_type', 'cls'), + 'normalize': kwargs.get('normalize', False) + } + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_kwargs) + model.model_dir = model_dir + return model diff --git a/runtime/Lib/site_packages/models/nlp/bert/siamese_uie.py b/runtime/Lib/site_packages/models/nlp/bert/siamese_uie.py new file mode 100644 index 0000000000000000000000000000000000000000..33ec925ee02725adaf688db4af6bc86b64a6d878 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/siamese_uie.py @@ -0,0 +1,151 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from copy import deepcopy + +import torch +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import BertEncoder, BertModel, BertPreTrainedModel + +__all__ = ['SiameseUieModel'] + + +@MODELS.register_module(Tasks.siamese_uie, module_name=Models.bert) +class SiameseUieModel(BertPreTrainedModel): + r"""SiameseUIE general information extraction model, + based on the construction idea of prompt (Prompt) + text (Text), + uses pointer network (Pointer Network) to + realize segment extraction (Span Extraction), so as to + realize named entity recognition (NER), relation extraction (RE), + Extraction of various tasks such as event extraction (EE), + attribute sentiment extraction (ABSA), etc. Different from + the existing general information extraction tasks on the market: + """ + + def __init__(self, config, **kwargs): + super().__init__(config) + self.config = config + self.plm = BertModel(self.config, add_pooling_layer=True) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.head_clsf = nn.Linear(config.hidden_size, 1) + self.tail_clsf = nn.Linear(config.hidden_size, 1) + self.set_crossattention_layer() + + def set_crossattention_layer(self, num_hidden_layers=6): + crossattention_config = deepcopy(self.config) + crossattention_config.num_hidden_layers = num_hidden_layers + self.config.num_hidden_layers -= num_hidden_layers + self.crossattention = BertEncoder(crossattention_config) + self.crossattention.layer = self.plm.encoder.layer[self.config. + num_hidden_layers:] + self.plm.encoder.layer = self.plm.encoder.layer[:self.config. + num_hidden_layers] + + def circle_loss(self, y_pred, y_true): + batch_size = y_true.size(0) + y_true = y_true.view(batch_size, -1) + y_pred = y_pred.view(batch_size, -1) + y_pred = (1 - 2 * y_true) * y_pred + y_pred_neg = y_pred - y_true * 1e12 + y_pred_pos = y_pred - (1 - y_true) * 1e12 + zeros = torch.zeros_like(y_pred[:, :1]) + y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1) + y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1) + neg_loss = torch.logsumexp(y_pred_neg, dim=-1) + pos_loss = torch.logsumexp(y_pred_pos, dim=-1) + return (neg_loss + pos_loss).mean() + + def get_cross_attention_output(self, hidden_states, attention_mask, + encoder_hidden_states, + encoder_attention_mask): + cat_hidden_states = torch.cat([hidden_states, encoder_hidden_states], + dim=1) + cat_attention_mask = torch.cat( + [attention_mask, encoder_attention_mask], dim=1) + cat_attention_mask = self.plm.get_extended_attention_mask( + cat_attention_mask, + cat_hidden_states.size()[:2]) + hidden_states = self.crossattention( + hidden_states=cat_hidden_states, attention_mask=cat_attention_mask + )[0][:, :hidden_states.size()[1], :] + return hidden_states + + def get_plm_sequence_output(self, + input_ids, + attention_mask, + position_ids=None, + is_hint=False): + token_type_ids = torch.ones_like(attention_mask) if is_hint else None + sequence_output = self.plm( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids)[0] + return sequence_output + + def forward(self, input_ids, attention_masks, hint_ids, + cross_attention_masks, head_labels, tail_labels): + """train forward + + Args: + input_ids (Tensor): input token ids of text. + attention_masks (Tensor): attention_masks of text. + hint_ids (Tensor): input token ids of prompt. + cross_attention_masks (Tensor): attention_masks of prompt. + head_labels (Tensor): labels of start position. + tail_labels (Tensor): labels of end position. + + Returns: + Dict[str, float]: the loss + Example: + {"loss": 0.5091743} + """ + sequence_output = self.get_plm_sequence_output(input_ids, + attention_masks) + assert hint_ids.size(1) + input_ids.size(1) <= 512 + position_ids = torch.arange(hint_ids.size(1)).expand( + (1, -1)) + input_ids.size(1) + position_ids = position_ids.to(sequence_output.device) + hint_sequence_output = self.get_plm_sequence_output( + hint_ids, cross_attention_masks, position_ids, is_hint=True) + sequence_output = self.get_cross_attention_output( + sequence_output, attention_masks, hint_sequence_output, + cross_attention_masks) + # (b, l, n) + head_logits = self.head_clsf(sequence_output).squeeze(-1) + tail_logits = self.tail_clsf(sequence_output).squeeze(-1) + loss_func = self.circle_loss + head_loss = loss_func(head_logits, head_labels) + tail_loss = loss_func(tail_logits, tail_labels) + return {'loss': head_loss + tail_loss} + + def fast_inference(self, sequence_output, attention_masks, hint_ids, + cross_attention_masks): + """ + + Args: + sequence_output(tensor): 3-dimension tensor (batch size, sequence length, hidden size) + attention_masks(tensor): attention mask, 2-dimension tensor (batch size, sequence length) + hint_ids(tensor): token ids of prompt 2-dimension tensor (batch size, sequence length) + cross_attention_masks(tensor): cross attention mask, 2-dimension tensor (batch size, sequence length) + Default Returns: + head_probs(tensor): 2-dimension tensor(batch size, sequence length) + tail_probs(tensor): 2-dimension tensor(batch size, sequence length) + """ + position_ids = torch.arange(hint_ids.size(1)).expand( + (1, -1)) + sequence_output.size(1) + position_ids = position_ids.to(sequence_output.device) + hint_sequence_output = self.get_plm_sequence_output( + hint_ids, cross_attention_masks, position_ids, is_hint=True) + sequence_output = self.get_cross_attention_output( + sequence_output, attention_masks, hint_sequence_output, + cross_attention_masks) + # (b, l, n) + head_logits = self.head_clsf(sequence_output).squeeze(-1) + tail_logits = self.tail_clsf(sequence_output).squeeze(-1) + head_probs = head_logits + (1 - attention_masks) * -10000 + tail_probs = tail_logits + (1 - attention_masks) * -10000 + return head_probs, tail_probs diff --git a/runtime/Lib/site_packages/models/nlp/bert/text_classification.py b/runtime/Lib/site_packages/models/nlp/bert/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..0a38202f5f263279dc5a91145f4ac7d6a7219801 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/text_classification.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp import ModelForTextClassification +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.text_classification, module_name=Models.bert) +@MODELS.register_module(Tasks.nli, module_name=Models.bert) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.bert) +@MODELS.register_module(Tasks.sentence_similarity, module_name=Models.bert) +@MODELS.register_module( + Tasks.zero_shot_classification, module_name=Models.bert) +class BertForSequenceClassification(ModelForTextClassification): + r"""Bert Model transformer with a sequence classification/regression head on top + (a linear layer on top of the pooled output) e.g. for GLUE tasks. + + This model inherits from :class:`SequenceClassificationModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + """ + base_model_type = 'bert' diff --git a/runtime/Lib/site_packages/models/nlp/bert/text_ranking.py b/runtime/Lib/site_packages/models/nlp/bert/text_ranking.py new file mode 100644 index 0000000000000000000000000000000000000000..c366b95ac4a2a420ee70d1f4dd67f3683a8ecb70 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/text_ranking.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp import ModelForTextRanking +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.text_ranking, module_name=Models.bert) +class BertForTextRanking(ModelForTextRanking): + r"""Bert Model transformer with a sequence classification/regression head on top + (a linear layer on top of the pooled output) e.g. for GLUE tasks. + + This model inherits from :class:`SequenceClassificationModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + """ + base_model_type = 'bert' diff --git a/runtime/Lib/site_packages/models/nlp/bert/token_classification.py b/runtime/Lib/site_packages/models/nlp/bert/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..fb21a6579f9a33cdba1e87c1f0e40ab6320539ba --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/token_classification.py @@ -0,0 +1,45 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from modelscope.metainfo import Heads, Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.token_classification import ( + ModelForTokenClassification, ModelForTokenClassificationWithCRF) +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.token_classification, module_name=Models.bert) +@MODELS.register_module(Tasks.part_of_speech, module_name=Models.bert) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.bert) +class BertForTokenClassification(ModelForTokenClassification): + r"""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks, word-segmentation. + + This model inherits from :class:`TokenClassificationModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + """ + + base_model_type = 'bert' diff --git a/runtime/Lib/site_packages/models/nlp/bert/word_alignment.py b/runtime/Lib/site_packages/models/nlp/bert/word_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..526ec07d7cf42752121d5143e747a95bfc17e884 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bert/word_alignment.py @@ -0,0 +1,173 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import WordAlignmentOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import BertModel, BertPreTrainedModel + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.word_alignment, module_name=Models.bert) +class MBertForWordAlignment(BertPreTrainedModel): + r"""MBert Model for the Word Alignment task. + + Code for EMNLP Findings 2022 paper, "Third-Party Aligner for Neural Word Alignments". + https://arxiv.org/abs/2211.04198 + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config, **kwargs): + super().__init__(config) + + if config.is_decoder: + logger.warning( + 'If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for ' + 'bi-directional self-attention.') + config.num_hidden_layers = kwargs.get('encoder_layers', 8) + + self.bert = BertModel(config, add_pooling_layer=False) + + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + src_input_ids=None, + src_attention_mask=None, + src_b2w_map=None, + tgt_input_ids=None, + tgt_attention_mask=None, + tgt_b2w_map=None, + threshold=0.001, + bpe_level=False, + ): + """ + Args: src_input_ids: + Indices of source input sequence tokens in the vocabulary. + src_attention_mask: + Source mask to avoid performing attention on padding token indices. + src_b2w_map: + Word order numner of subword in source sequence. + tgt_input_ids: + Indices of target input sequence tokens in the vocabulary. + tgt_attention_mask: + Target mask to avoid performing attention on padding token indices. + tgt_b2w_map: + Word order numner of subword in target sequence. + threshold: + The threshold used to extract alignment. + bpe_level: + Return subword-level alignment or not. + Example: + { + 'src_input_ids': LongTensor([[2478,242,24,4]]), + 'src_attention_mask': BoolTensor([[1,1,1,1]]), + 'src_b2w_map': LongTensor([[0,1,2,3]]), + 'tgt_input_ids': LongTensor([[1056,356,934,263,7]]), + 'tgt_attention_mask': BoolTensor([[1,1,1,1,1]]), + 'tgt_b2w_map': longtensor([[0,1,1,2,3]]), + 'threshold': 0.001, + 'bpe_level': False, + } + Returns `modelscope.outputs.WordAlignmentOutput` + """ + with torch.no_grad(): + src_encoder_out = self.bert( + input_ids=src_input_ids, + attention_mask=src_attention_mask.float(), + head_mask=None, + inputs_embeds=None, + output_hidden_states=True, + ) + tgt_encoder_out = self.bert( + input_ids=tgt_input_ids, + attention_mask=tgt_attention_mask.float(), + head_mask=None, + inputs_embeds=None, + output_hidden_states=True, + ) + + atten_mask_src = (1 - ( + (src_input_ids != 101) & (src_input_ids != 102) + & src_attention_mask)[:, None, None, :].float()) * -10000 + atten_mask_tgt = (1 - ( + (tgt_input_ids != 101) & (tgt_input_ids != 102) + & tgt_attention_mask)[:, None, None, :].float()) * -10000 + + src_align_out = src_encoder_out[0] + tgt_align_out = tgt_encoder_out[0] + + bpe_sim = torch.bmm(src_align_out, tgt_align_out.transpose(1, 2)) + + attention_scores_src = bpe_sim.unsqueeze(1) + atten_mask_tgt + attention_scores_tgt = bpe_sim.unsqueeze(1) + atten_mask_src.transpose( + -1, -2) + + attention_probs_src = nn.Softmax(dim=-1)(attention_scores_src) + attention_probs_tgt = nn.Softmax(dim=-2)(attention_scores_tgt) + + align_matrix = (attention_probs_src > threshold) * ( + attention_probs_tgt > threshold) + align_matrix = align_matrix.squeeze(1) + + len_src = (atten_mask_src == 0).sum(dim=-1).unsqueeze(-1) + len_tgt = (atten_mask_tgt == 0).sum(dim=-1).unsqueeze(-1) + + attention_probs_src = nn.Softmax(dim=-1)( + attention_scores_src / torch.sqrt(len_src.float())) + attention_probs_tgt = nn.Softmax(dim=-2)( + attention_scores_tgt / torch.sqrt(len_tgt.float())) + + word_aligns = [] + + for idx, (line_align, b2w_src, b2w_tgt) in enumerate( + zip(align_matrix, src_b2w_map, tgt_b2w_map)): + aligns = dict() + non_specials = torch.where(line_align) + for i, j in zip(*non_specials): + if not bpe_level: + word_pair = (src_b2w_map[idx][i - 1].item(), + tgt_b2w_map[idx][j - 1].item()) + if word_pair not in aligns: + aligns[word_pair] = bpe_sim[idx][i, j].item() + else: + aligns[word_pair] = max(aligns[word_pair], + bpe_sim[idx][i, j].item()) + else: + aligns[(i.item() - 1, + j.item() - 1)] = bpe_sim[idx][i, j].item() + word_aligns.append(aligns) + + return WordAlignmentOutput(predictions=word_aligns) diff --git a/runtime/Lib/site_packages/models/nlp/bloom/__init__.py b/runtime/Lib/site_packages/models/nlp/bloom/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..24d7202def904e266488d138d5b1c3d7a69d33bf --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bloom/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import BloomModel + from .text_generation import BloomForTextGeneration + from .sentence_embedding import BloomForSentenceEmbedding +else: + _import_structure = { + 'backbone': ['BloomModel'], + 'text_generation': ['BloomForTextGeneration'], + 'sentence_embedding': ['BloomForSentenceEmbedding'] + } + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/bloom/backbone.py b/runtime/Lib/site_packages/models/nlp/bloom/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..5b2089eb019fcb7298af2f3147797f66e768bf07 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bloom/backbone.py @@ -0,0 +1,38 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from transformers import BloomConfig +from transformers import BloomModel as BloomModelTransform + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import BACKBONES +from modelscope.utils.constant import Tasks + + +class MsModelMixin: + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + kwargs.pop('device', None) + if model_dir is None: + config = BloomConfig(**kwargs) + model = cls(config) + else: + model = super(MsModelMixin, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model + + +@BACKBONES.register_module(group_key=Tasks.backbone, module_name=Models.bloom) +class BloomModel(MsModelMixin, BloomModelTransform, TorchModel): + + pass diff --git a/runtime/Lib/site_packages/models/nlp/bloom/sentence_embedding.py b/runtime/Lib/site_packages/models/nlp/bloom/sentence_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..ec35db387bc35efd5037e774c2c7a9f1bf0aaffb --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bloom/sentence_embedding.py @@ -0,0 +1,165 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +from transformers import BloomConfig +from transformers import BloomModel as BloomModelTransform + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.outputs import SentencEmbeddingModelOutput +from modelscope.utils.constant import Tasks + + +class DecoderPooler(torch.nn.Module): + """ + Parameter-free poolers to get the sentence embedding + 'last': the last token state. + 'weighted_mean': position weighted average of all token states. + """ + + def __init__(self, pooler_type): + super().__init__() + self.pooler_type = pooler_type + assert self.pooler_type in [ + 'last', 'weighted_mean' + ], 'unrecognized pooling type %s' % self.pooler_type + + def forward(self, outputs, attention_mask): + last_hidden = outputs.last_hidden_state + + if self.pooler_type in ['last']: + n, l, h = last_hidden.shape + + # Get shape [n] indices of the last token (i.e. the last token for each batch item) + # Any sequence where min == 1, we use the entire sequence lenth since argmin = 0 + values, indices = torch.min(attention_mask, 1, keepdim=False) + gather_indices = torch.where(values == 0, indices, + l) - 1 # Shape [n] + + # There are empty sequences, where the index would become -1 which will crash + gather_indices = torch.clamp(gather_indices, min=0) + + # Turn indices from shape [n] --> [n, 1, h] + gather_indices = gather_indices.unsqueeze(1).unsqueeze(1).expand( + n, 1, h) + + # Gather along the 1st dim (l) (n, l, h -> n, h) + pooled_output = torch.gather(last_hidden, 1, + gather_indices).squeeze(dim=1) + + elif self.pooler_type == 'weighted_mean': + input_mask_expanded = attention_mask.unsqueeze(-1).expand( + last_hidden.size()).float() + # last_hidden shape: bs, seq, hidden_dim + weights = ( + torch.arange(start=1, end=last_hidden.shape[1] + + 1).unsqueeze(0).unsqueeze(-1).expand( + last_hidden.size()).float().to( + last_hidden.device)) + assert weights.shape == last_hidden.shape == input_mask_expanded.shape + input_mask_expanded = input_mask_expanded * weights + + sum_embeddings = torch.sum(last_hidden * input_mask_expanded, 1) + sum_mask = input_mask_expanded.sum(1) + sum_mask = torch.clamp(sum_mask, min=1e-9) + pooled_output = sum_embeddings / sum_mask + + else: + raise NotImplementedError + + return pooled_output + + +@MODELS.register_module( + group_key=Tasks.sentence_embedding, module_name=Models.bloom) +class BloomForSentenceEmbedding(BloomModelTransform, TorchModel): + r""" + This model represent a text to a dense vector by the last token state or weighted mean of all token states. + See `Language Models are Universal Embedders + `_ for details. + """ + + def __init__(self, config, **kwargs): + super().__init__(config) + self.config = config + self.pooler_type = kwargs.get('emb_pooler_type', 'weighted_mean') + self.pooler = DecoderPooler(self.pooler_type) + self.normalize = kwargs.get('normalize', False) + setattr(self, self.base_model_prefix, BloomModelTransform(config)) + + def forward(self, query=None, docs=None, labels=None): + r""" + Args: + query (:obj: `dict`): Dict of pretrained models's input for the query sequence. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + docs (:obj: `dict`): Dict of pretrained models's input for the query sequence. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + Returns: + Returns `modelscope.outputs.SentencEmbeddingModelOutput + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_udever_bloom_560m') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_udever_bloom_560m') + >>> inputs = preprocessor({'source_sentence': ['This is a test']}) + >>> outputs = model(**inputs) + >>> print(outputs) + """ + query_embeddings, doc_embeddings = None, None + if query is not None: + query_embeddings = self.encode(**query) + if docs is not None: + doc_embeddings = self.encode(**docs) + outputs = SentencEmbeddingModelOutput( + query_embeddings=query_embeddings, doc_embeddings=doc_embeddings) + if query_embeddings is None or doc_embeddings is None: + return outputs + if self.base_model.training: + loss_fct = torch.nn.CrossEntropyLoss() + scores = torch.matmul(query_embeddings, doc_embeddings.T) + if labels is None: + labels = torch.arange( + scores.size(0), device=scores.device, dtype=torch.long) + labels = labels * ( + doc_embeddings.size(0) // query_embeddings.size(0)) + loss = loss_fct(scores, labels) + outputs.loss = loss + return outputs + + def encode( + self, + input_ids=None, + attention_mask=None, + ): + outputs = self.base_model.forward( + input_ids, attention_mask=attention_mask) + embeddings = self.pooler(outputs, attention_mask) + if self.normalize: + embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=-1) + return embeddings + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + model_dir = kwargs.get('model_dir') + model_kwargs = { + 'emb_pooler_type': kwargs.get('emb_pooler_type', 'weighted_mean'), + 'normalize': kwargs.get('normalize', False) + } + if model_dir is None: + config = BloomConfig(**kwargs) + model = cls(config) + else: + model = super(BloomModelTransform, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_kwargs) + model.model_dir = model_dir + return model diff --git a/runtime/Lib/site_packages/models/nlp/bloom/text_generation.py b/runtime/Lib/site_packages/models/nlp/bloom/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..73031a828c0c7d7e6714e35148b14f0169f2561f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/bloom/text_generation.py @@ -0,0 +1,15 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from transformers import BloomForCausalLM as BloomForCausalLMTransform + +from modelscope.metainfo import Models +from modelscope.models import MODELS +from modelscope.utils.constant import Tasks +from .backbone import MsModelMixin, TorchModel + + +@MODELS.register_module( + group_key=Tasks.text_generation, module_name=Models.bloom) +class BloomForTextGeneration(MsModelMixin, BloomForCausalLMTransform, + TorchModel): + + pass diff --git a/runtime/Lib/site_packages/models/nlp/canmt/__init__.py b/runtime/Lib/site_packages/models/nlp/canmt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..affe5b1f3eeb2de4b2aa98111084cf7d3c68dd77 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/canmt/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from .canmt_translation import CanmtForTranslation diff --git a/runtime/Lib/site_packages/models/nlp/canmt/canmt_model.py b/runtime/Lib/site_packages/models/nlp/canmt/canmt_model.py new file mode 100644 index 0000000000000000000000000000000000000000..e55c6f8906a4ab791b172bc40ffd7c51ff9c9faa --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/canmt/canmt_model.py @@ -0,0 +1,1301 @@ +# Part of the implementation is borrowed and modified from FAIRSEQ, +# publicly available at https://github.com/facebookresearch/fairseq +# Copyright 2022-2023 The Alibaba MT Team Authors. All rights reserved. +import math +from typing import Any, Dict, List, Optional, Tuple + +import numpy +import torch +import torch.nn as nn +from fairseq import utils +from fairseq.distributed import fsdp_wrap +from fairseq.models import (FairseqEncoder, FairseqEncoderDecoderModel, + FairseqIncrementalDecoder, register_model, + register_model_architecture) +from fairseq.modules import (AdaptiveSoftmax, BaseLayer, FairseqDropout, + LayerDropModuleList, LayerNorm, + PositionalEmbedding, + SinusoidalPositionalEmbedding, + TransformerDecoderLayer, TransformerEncoderLayer) +from fairseq.modules.checkpoint_activations import checkpoint_wrapper +from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_ +from torch import Tensor + +DEFAULT_MAX_SOURCE_POSITIONS = 1024 +DEFAULT_MAX_TARGET_POSITIONS = 1024 + +DEFAULT_MIN_PARAMS_TO_WRAP = int(1e8) + + +class CanmtModel(FairseqEncoderDecoderModel): + """ + + Args: + encoder (TransformerEncoder): the encoder + decoder (TransformerDecoder): the decoder + + The CanmtModel provides the following named architectures and + command-line arguments: + + .. argparse:: + :ref: fairseq.models.transformer_parser + :prog: + """ + + def __init__(self, args, encoder, decoder, second_decoder): + super().__init__(encoder, decoder) + self.args = args + self.supports_align_args = True + self.encoder = encoder + self.decoder = decoder + self.second_decoder = second_decoder + + @staticmethod + def add_args(parser): + """Add model-specific arguments to the parser.""" + # fmt: off + parser.add_argument( + '--activation-fn', + choices=utils.get_available_activation_fns(), + help='activation function to use') + parser.add_argument( + '--dropout', type=float, metavar='D', help='dropout probability') + parser.add_argument( + '--attention-dropout', + type=float, + metavar='D', + help='dropout probability for attention weights') + parser.add_argument( + '--activation-dropout', + '--relu-dropout', + type=float, + metavar='D', + help='dropout probability after activation in FFN.') + parser.add_argument( + '--encoder-embed-path', + type=str, + metavar='STR', + help='path to pre-trained encoder embedding') + parser.add_argument( + '--encoder-embed-dim', + type=int, + metavar='N', + help='encoder embedding dimension') + parser.add_argument( + '--encoder-ffn-embed-dim', + type=int, + metavar='N', + help='encoder embedding dimension for FFN') + parser.add_argument( + '--encoder-layers', + type=int, + metavar='N', + help='num encoder layers') + parser.add_argument( + '--encoder-attention-heads', + type=int, + metavar='N', + help='num encoder attention heads') + parser.add_argument( + '--encoder-normalize-before', + action='store_true', + help='apply layernorm before each encoder block') + parser.add_argument( + '--encoder-learned-pos', + action='store_true', + help='use learned positional embeddings in the encoder') + parser.add_argument( + '--decoder-embed-path', + type=str, + metavar='STR', + help='path to pre-trained decoder embedding') + parser.add_argument( + '--decoder-embed-dim', + type=int, + metavar='N', + help='decoder embedding dimension') + parser.add_argument( + '--decoder-ffn-embed-dim', + type=int, + metavar='N', + help='decoder embedding dimension for FFN') + parser.add_argument( + '--decoder-layers', + type=int, + metavar='N', + help='num decoder layers') + parser.add_argument( + '--decoder-attention-heads', + type=int, + metavar='N', + help='num decoder attention heads') + parser.add_argument( + '--decoder-learned-pos', + action='store_true', + help='use learned positional embeddings in the decoder') + parser.add_argument( + '--decoder-normalize-before', + action='store_true', + help='apply layernorm before each decoder block') + parser.add_argument( + '--decoder-output-dim', + type=int, + metavar='N', + help='decoder output dimension (extra linear layer ' + 'if different from decoder embed dim') + parser.add_argument( + '--share-decoder-input-output-embed', + action='store_true', + help='share decoder input and output embeddings') + parser.add_argument( + '--share-all-embeddings', + action='store_true', + help='share encoder, decoder and output embeddings' + ' (requires shared dictionary and embed dim)') + parser.add_argument( + '--no-token-positional-embeddings', + default=False, + action='store_true', + help= + 'if set, disables positional embeddings (outside self attention)') + parser.add_argument( + '--adaptive-softmax-cutoff', + metavar='EXPR', + help='comma separated list of adaptive softmax cutoff points. ' + 'Must be used with adaptive_loss criterion'), + parser.add_argument( + '--adaptive-softmax-dropout', + type=float, + metavar='D', + help='sets adaptive softmax dropout for the tail projections') + parser.add_argument( + '--layernorm-embedding', + action='store_true', + help='add layernorm to embedding') + parser.add_argument( + '--no-scale-embedding', + action='store_true', + help='if True, dont scale embeddings') + parser.add_argument( + '--checkpoint-activations', + action='store_true', + help='checkpoint activations at each layer, which saves GPU ' + 'memory usage at the cost of some additional compute') + parser.add_argument( + '--offload-activations', + action='store_true', + help='checkpoint activations at each layer, then save to gpu.' + 'Sets --checkpoint-activations.') + parser.add_argument( + '--no-cross-attention', + default=False, + action='store_true', + help='do not perform cross-attention') + parser.add_argument( + '--cross-self-attention', + default=False, + action='store_true', + help='perform cross+self-attention') + parser.add_argument( + '--encoder-layerdrop', + type=float, + metavar='D', + default=0, + help='LayerDrop probability for encoder') + parser.add_argument( + '--decoder-layerdrop', + type=float, + metavar='D', + default=0, + help='LayerDrop probability for decoder') + parser.add_argument( + '--encoder-layers-to-keep', + default=None, + help='which layers to *keep* when pruning as a comma-separated list' + ) + parser.add_argument( + '--decoder-layers-to-keep', + default=None, + help='which layers to *keep* when pruning as a comma-separated list' + ) + parser.add_argument( + '--quant-noise-pq', + type=float, + metavar='D', + default=0, + help='iterative PQ quantization noise at training time') + parser.add_argument( + '--quant-noise-pq-block-size', + type=int, + metavar='D', + default=8, + help='block size of quantization noise at training time') + parser.add_argument( + '--quant-noise-scalar', + type=float, + metavar='D', + default=0, + help= + 'scalar quantization noise and scalar quantization at training time' + ) + parser.add_argument( + '--min-params-to-wrap', + type=int, + metavar='D', + default=DEFAULT_MIN_PARAMS_TO_WRAP, + help= + ('minimum number of params for a layer to be wrapped with FSDP() when ' + 'training with --ddp-backend=fully_sharded. Smaller values will ' + 'improve memory efficiency, but may make torch.distributed ' + 'communication less efficient due to smaller input sizes. This option ' + 'is set to 0 (i.e., always wrap) when --checkpoint-activations or ' + '--offload-activations are passed.')) + # fmt: on + + @classmethod + def build_model(cls, args, task): + """Build a new model instance.""" + + # make sure all arguments are present in older models + base_architecture(args) + + if args.encoder_layers_to_keep: + args.encoder_layers = len(args.encoder_layers_to_keep.split(',')) + if args.decoder_layers_to_keep: + args.decoder_layers = len(args.decoder_layers_to_keep.split(',')) + + if getattr(args, 'max_source_positions', None) is None: + args.max_source_positions = DEFAULT_MAX_SOURCE_POSITIONS + if getattr(args, 'max_target_positions', None) is None: + args.max_target_positions = DEFAULT_MAX_TARGET_POSITIONS + + src_dict, tgt_dict = task.vocab_src, task.vocab_tgt + + if args.share_all_embeddings: + if src_dict != tgt_dict: + raise ValueError( + '--share-all-embeddings requires a joined dictionary') + if args.encoder_embed_dim != args.decoder_embed_dim: + raise ValueError( + '--share-all-embeddings requires --encoder-embed-dim to match --decoder-embed-dim' + ) + if args.decoder_embed_path and \ + (args.decoder_embed_path != args.encoder_embed_path): + raise ValueError( + '--share-all-embeddings not compatible with --decoder-embed-path' + ) + encoder_embed_tokens = cls.build_embedding(args, src_dict, + args.encoder_embed_dim, + args.encoder_embed_path) + decoder_embed_tokens = encoder_embed_tokens + args.share_decoder_input_output_embed = True + else: + encoder_embed_tokens = cls.build_embedding(args, src_dict, + args.encoder_embed_dim, + args.encoder_embed_path) + decoder_embed_tokens = cls.build_embedding(args, tgt_dict, + args.decoder_embed_dim, + args.decoder_embed_path) + if getattr(args, 'offload_activations', False): + args.checkpoint_activations = True # offloading implies checkpointing + encoder = cls.build_encoder(args, src_dict, encoder_embed_tokens) + decoder = cls.build_decoder(args, tgt_dict, decoder_embed_tokens) + second_decoder = cls.build_decoder(args, src_dict, + encoder_embed_tokens) + if not args.share_all_embeddings: + min_params_to_wrap = getattr(args, 'min_params_to_wrap', + DEFAULT_MIN_PARAMS_TO_WRAP) + # fsdp_wrap is a no-op when --ddp-backend != fully_sharded + encoder = fsdp_wrap(encoder, min_num_params=min_params_to_wrap) + decoder = fsdp_wrap(decoder, min_num_params=min_params_to_wrap) + return cls(args, encoder, decoder, second_decoder) + + @classmethod + def build_embedding(cls, args, dictionary, embed_dim, path=None): + num_embeddings = len(dictionary) + padding_idx = dictionary.pad() + + emb = Embedding(num_embeddings, embed_dim, padding_idx) + # if provided, load from preloaded dictionaries + if path: + embed_dict = utils.parse_embedding(path) + utils.load_embedding(embed_dict, dictionary, emb) + return emb + + @classmethod + def build_encoder(cls, args, src_dict, embed_tokens): + return TransformerEncoder(args, src_dict, embed_tokens) + + @classmethod + def build_decoder(cls, args, tgt_dict, embed_tokens): + return TransformerDecoder( + args, + tgt_dict, + embed_tokens, + no_encoder_attn=getattr(args, 'no_cross_attention', False), + ) + + def forward( + self, + src_tokens, + src_lengths, + prev_output_tokens, + prev_src_tokens, + return_all_hiddens: bool = True, + features_only: bool = False, + alignment_layer: Optional[int] = None, + alignment_heads: Optional[int] = None, + ): + """ + Run the forward pass for an encoder-decoder model. + + Copied from the base class, but without ``**kwargs``, + which are not supported by TorchScript. + """ + + encoder_out = self.encoder( + src_tokens, + src_lengths=src_lengths, + return_all_hiddens=return_all_hiddens) + decoder_out = self.decoder( + prev_output_tokens, + encoder_out=encoder_out, + features_only=features_only, + alignment_layer=alignment_layer, + alignment_heads=alignment_heads, + src_lengths=src_lengths, + return_all_hiddens=return_all_hiddens, + ) + + decoder_out_re = self.decoder( + prev_output_tokens, + encoder_out=None, + features_only=features_only, + full_context_alignment=True, + alignment_layer=alignment_layer, + alignment_heads=alignment_heads, + src_lengths=src_lengths, + return_all_hiddens=return_all_hiddens, + ) + decoder_out_tensor = decoder_out_re[1]['last_layer'] + decoder_padding = decoder_out_re[1]['self_attn_padding_mask'] + + decoder_kvs = { + 'encoder_out': [decoder_out_tensor], + 'encoder_padding_mask': [decoder_padding] + } + src_out = self.second_decoder( + prev_src_tokens, + encoder_out=decoder_kvs, + features_only=features_only, + alignment_layer=alignment_layer, + alignment_heads=alignment_heads, + src_lengths=None, + return_all_hiddens=return_all_hiddens, + ) + return decoder_out, src_out, decoder_kvs + + @torch.jit.export + def get_normalized_probs( + self, + net_output: Tuple[Tensor, Optional[Dict[str, List[Optional[Tensor]]]]], + log_probs: bool, + sample: Optional[Dict[str, Tensor]] = None, + ): + """Get normalized probabilities (or log probs) from a net's output.""" + return self.get_normalized_probs_scriptable(net_output, log_probs, + sample) + + def forward_decoder( + self, + tokens, + encoder_outs: Dict[str, List[Tensor]], + incremental_states: Dict[str, Dict[str, Optional[Tensor]]], + temperature: float = 1.0, + ): + encoder_out: Optional[Dict[str, List[Tensor]]] = None + encoder_out = encoder_outs + # decode + decoder_out = self.decoder.forward( + tokens, + encoder_out=encoder_out, + incremental_state=incremental_states, + ) + + attn: Optional[Tensor] = None + decoder_len = len(decoder_out) + if decoder_len > 1 and decoder_out[1] is not None: + if isinstance(decoder_out[1], Tensor): + attn = decoder_out[1] + else: + attn_holder = decoder_out[1]['attn'] + if isinstance(attn_holder, Tensor): + attn = attn_holder + elif attn_holder is not None: + attn = attn_holder[0] + if attn is not None: + attn = attn[:, -1, :] + + decoder_out_tuple = ( + decoder_out[0][:, -1:, :].div_(temperature), + None if decoder_len <= 1 else decoder_out[1], + ) + probs = self.get_normalized_probs( + decoder_out_tuple, log_probs=True, sample=None) + probs = probs[:, -1, :] + decoder_out_tensor = decoder_out[1]['last_layer'] + return probs, attn, decoder_out_tensor + + def forward_decoder_src( + self, + tokens, + encoder_outs: Dict[str, List[Tensor]], + incremental_states: Dict[str, Dict[str, Optional[Tensor]]], + temperature: float = 1.0, + ): + encoder_out: Optional[Dict[str, List[Tensor]]] = None + encoder_out = encoder_outs + # decode each model + decoder_out = self.second_decoder.forward( + tokens, encoder_out=encoder_out) + + attn: Optional[Tensor] = None + decoder_len = len(decoder_out) + if decoder_len > 1 and decoder_out[1] is not None: + if isinstance(decoder_out[1], Tensor): + attn = decoder_out[1] + else: + attn_holder = decoder_out[1]['attn'] + if isinstance(attn_holder, Tensor): + attn = attn_holder + elif attn_holder is not None: + attn = attn_holder[0] + if attn is not None: + attn = attn[:, -1, :] + + decoder_out_tuple = ( + decoder_out[0][:, -1:, :].div_(temperature), + None if decoder_len <= 1 else decoder_out[1], + ) + probs = self.get_normalized_probs( + decoder_out_tuple, log_probs=True, sample=None) + probs = probs[:, -1, :] + decoder_out_tensor = decoder_out[1]['last_layer'] + return probs, attn, decoder_out_tensor, decoder_out + + def forward_encoder(self, net_input: Dict[str, Tensor]): + encoder_input = { + k: v + for k, v in net_input.items() if k != 'prev_output_tokens' + and k != 'prev_src_tokens' and k != 'sources' + } + return self.encoder.forward_torchscript(encoder_input) + + def reorder_encoder_out(self, encoder_outs: Optional[Dict[str, + List[Tensor]]], + new_order): + """ + Reorder encoder output according to *new_order*. + + Args: + encoder_out: output from the ``forward()`` method + new_order (LongTensor): desired order + + Returns: + *encoder_out* rearranged according to *new_order* + """ + assert encoder_outs is not None + return self.encoder.reorder_encoder_out(encoder_outs, new_order) + + def reorder_incremental_state( + self, + incremental_states: Dict[str, Dict[str, Optional[Tensor]]], + new_order, + ): + self.decoder.reorder_incremental_state_scripting( + incremental_states, new_order) + + +class TransformerEncoder(FairseqEncoder): + """ + Transformer encoder consisting of *args.encoder_layers* layers. Each layer + is a :class:`TransformerEncoderLayer`. + + Args: + args (argparse.Namespace): parsed command-line arguments + dictionary (~fairseq.data.Dictionary): encoding dictionary + embed_tokens (torch.nn.Embedding): input embedding + """ + + def __init__(self, args, dictionary, embed_tokens): + self.args = args + super().__init__(dictionary) + self.register_buffer('version', torch.Tensor([3])) + + self.dropout_module = FairseqDropout( + args.dropout, module_name=self.__class__.__name__) + self.encoder_layerdrop = args.encoder_layerdrop + + embed_dim = embed_tokens.embedding_dim + self.padding_idx = embed_tokens.padding_idx + self.max_source_positions = args.max_source_positions + self.embed_tokens = embed_tokens + + self.embed_scale = 1.0 if args.no_scale_embedding else math.sqrt( + embed_dim) + + self.embed_positions = ( + PositionalEmbedding( + args.max_source_positions, + embed_dim, + self.padding_idx, + learned=args.encoder_learned_pos, + ) if not args.no_token_positional_embeddings else None) + export = getattr(args, 'export', False) + if getattr(args, 'layernorm_embedding', False): + self.layernorm_embedding = LayerNorm(embed_dim, export=export) + else: + self.layernorm_embedding = None + + if not args.adaptive_input and args.quant_noise_pq > 0: + self.quant_noise = apply_quant_noise_( + nn.Linear(embed_dim, embed_dim, bias=False), + args.quant_noise_pq, + args.quant_noise_pq_block_size, + ) + else: + self.quant_noise = None + + if self.encoder_layerdrop > 0.0: + self.layers = LayerDropModuleList(p=self.encoder_layerdrop) + else: + self.layers = nn.ModuleList([]) + self.layers.extend([ + self.build_encoder_layer(args) for i in range(args.encoder_layers) + ]) + self.num_layers = len(self.layers) + + if args.encoder_normalize_before: + self.layer_norm = LayerNorm(embed_dim, export=export) + else: + self.layer_norm = None + + def build_encoder_layer(self, args): + layer = TransformerEncoderLayer(args) + checkpoint = getattr(args, 'checkpoint_activations', False) + if checkpoint: + offload_to_cpu = getattr(args, 'offload_activations', False) + layer = checkpoint_wrapper(layer, offload_to_cpu=offload_to_cpu) + min_params_to_wrap = ( + getattr(args, 'min_params_to_wrap', DEFAULT_MIN_PARAMS_TO_WRAP) + if not checkpoint else 0) + layer = fsdp_wrap(layer, min_num_params=min_params_to_wrap) + return layer + + def forward_embedding(self, + src_tokens, + token_embedding: Optional[torch.Tensor] = None): + # embed tokens and positions + if token_embedding is None: + token_embedding = self.embed_tokens(src_tokens) + x = embed = self.embed_scale * token_embedding + if self.embed_positions is not None: + x = embed + self.embed_positions(src_tokens) + if self.layernorm_embedding is not None: + x = self.layernorm_embedding(x) + x = self.dropout_module(x) + if self.quant_noise is not None: + x = self.quant_noise(x) + return x, embed + + def forward( + self, + src_tokens, + src_lengths: Optional[torch.Tensor] = None, + return_all_hiddens: bool = False, + token_embeddings: Optional[torch.Tensor] = None, + ): + """ + Args: + src_tokens (LongTensor): tokens in the source language of shape + `(batch, src_len)` + src_lengths (torch.LongTensor): lengths of each source sentence of + shape `(batch)` + return_all_hiddens (bool, optional): also return all of the + intermediate hidden states (default: False). + token_embeddings (torch.Tensor, optional): precomputed embeddings + default `None` will recompute embeddings + + Returns: + dict: + - **encoder_out** (Tensor): the last encoder layer's output of + shape `(src_len, batch, embed_dim)` + - **encoder_padding_mask** (ByteTensor): the positions of + padding elements of shape `(batch, src_len)` + - **encoder_embedding** (Tensor): the (scaled) embedding lookup + of shape `(batch, src_len, embed_dim)` + - **encoder_states** (List[Tensor]): all intermediate + hidden states of shape `(src_len, batch, embed_dim)`. + Only populated if *return_all_hiddens* is True. + """ + return self.forward_scriptable(src_tokens, src_lengths, + return_all_hiddens, token_embeddings) + + def forward_scriptable( + self, + src_tokens, + src_lengths: Optional[torch.Tensor] = None, + return_all_hiddens: bool = False, + token_embeddings: Optional[torch.Tensor] = None, + ): + """ + Args: + src_tokens (LongTensor): tokens in the source language of shape + `(batch, src_len)` + src_lengths (torch.LongTensor): lengths of each source sentence of + shape `(batch)` + return_all_hiddens (bool, optional): also return all of the + intermediate hidden states (default: False). + token_embeddings (torch.Tensor, optional): precomputed embeddings + default `None` will recompute embeddings + + Returns: + dict: + - **encoder_out** (Tensor): the last encoder layer's output of + shape `(src_len, batch, embed_dim)` + - **encoder_padding_mask** (ByteTensor): the positions of + padding elements of shape `(batch, src_len)` + - **encoder_embedding** (Tensor): the (scaled) embedding lookup + of shape `(batch, src_len, embed_dim)` + - **encoder_states** (List[Tensor]): all intermediate + hidden states of shape `(src_len, batch, embed_dim)`. + Only populated if *return_all_hiddens* is True. + """ + # compute padding mask + encoder_padding_mask = src_tokens.eq(self.padding_idx) + has_pads = src_tokens.device.type == 'xla' or encoder_padding_mask.any( + ) + x, encoder_embedding = self.forward_embedding(src_tokens, + token_embeddings) + + # account for padding while computing the representation + if has_pads: + x = x * (1 - encoder_padding_mask.unsqueeze(-1).type_as(x)) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + encoder_states = [] + + if return_all_hiddens: + encoder_states.append(x) + + # encoder layers + for layer in self.layers: + x = layer( + x, + encoder_padding_mask=encoder_padding_mask + if has_pads else None) + if return_all_hiddens: + assert encoder_states is not None + encoder_states.append(x) + + if self.layer_norm is not None: + x = self.layer_norm(x) + + return { + 'encoder_out': [x], # T x B x C + 'encoder_padding_mask': [encoder_padding_mask], # B x T + 'encoder_embedding': [encoder_embedding], # B x T x C + 'encoder_states': encoder_states, # List[T x B x C] + 'src_tokens': [], + 'src_lengths': [], + } + + @torch.jit.export + def reorder_encoder_out(self, encoder_out: Dict[str, List[Tensor]], + new_order): + """ + Reorder encoder output according to *new_order*. + + Args: + encoder_out: output from the ``forward()`` method + new_order (LongTensor): desired order + + Returns: + *encoder_out* rearranged according to *new_order* + """ + if len(encoder_out['encoder_out']) == 0: + new_encoder_out = [] + else: + new_encoder_out = [ + encoder_out['encoder_out'][0].index_select(1, new_order) + ] + if len(encoder_out['encoder_padding_mask']) == 0: + new_encoder_padding_mask = [] + else: + new_encoder_padding_mask = [ + encoder_out['encoder_padding_mask'][0].index_select( + 0, new_order) + ] + if len(encoder_out['encoder_embedding']) == 0: + new_encoder_embedding = [] + else: + new_encoder_embedding = [ + encoder_out['encoder_embedding'][0].index_select(0, new_order) + ] + + if len(encoder_out['src_tokens']) == 0: + src_tokens = [] + else: + src_tokens = [ + (encoder_out['src_tokens'][0]).index_select(0, new_order) + ] + + if len(encoder_out['src_lengths']) == 0: + src_lengths = [] + else: + src_lengths = [ + (encoder_out['src_lengths'][0]).index_select(0, new_order) + ] + + encoder_states = encoder_out['encoder_states'] + if len(encoder_states) > 0: + for idx, state in enumerate(encoder_states): + encoder_states[idx] = state.index_select(1, new_order) + + return { + 'encoder_out': new_encoder_out, # T x B x C + 'encoder_padding_mask': new_encoder_padding_mask, # B x T + 'encoder_embedding': new_encoder_embedding, # B x T x C + 'encoder_states': encoder_states, # List[T x B x C] + 'src_tokens': src_tokens, # B x T + 'src_lengths': src_lengths, # B x 1 + } + + def max_positions(self): + """Maximum input length supported by the encoder.""" + if self.embed_positions is None: + return self.max_source_positions + return min(self.max_source_positions, + self.embed_positions.max_positions) + + def upgrade_state_dict_named(self, state_dict, name): + """Upgrade a (possibly old) state dict for new versions of fairseq.""" + if isinstance(self.embed_positions, SinusoidalPositionalEmbedding): + weights_key = '{}.embed_positions.weights'.format(name) + if weights_key in state_dict: + print('deleting {0}'.format(weights_key)) + del state_dict[weights_key] + state_dict['{}.embed_positions._float_tensor'.format( + name)] = torch.FloatTensor(1) + for i in range(self.num_layers): + # update layer norms + self.layers[i].upgrade_state_dict_named( + state_dict, '{}.layers.{}'.format(name, i)) + + version_key = '{}.version'.format(name) + if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) < 2: + # earlier checkpoints did not normalize after the stack of layers + self.layer_norm = None + self.normalize = False + state_dict[version_key] = torch.Tensor([1]) + return state_dict + + +class TransformerDecoder(FairseqIncrementalDecoder): + """ + Transformer decoder consisting of *args.decoder_layers* layers. Each layer + is a :class:`TransformerDecoderLayer`. + + Args: + args (argparse.Namespace): parsed command-line arguments + dictionary (~fairseq.data.Dictionary): decoding dictionary + embed_tokens (torch.nn.Embedding): output embedding + no_encoder_attn (bool, optional): whether to attend to encoder outputs + (default: False). + """ + + def __init__( + self, + args, + dictionary, + embed_tokens, + no_encoder_attn=False, + output_projection=None, + ): + self.args = args + super().__init__(dictionary) + self.register_buffer('version', torch.Tensor([3])) + self._future_mask = torch.empty(0) + + self.dropout_module = FairseqDropout( + args.dropout, module_name=self.__class__.__name__) + self.decoder_layerdrop = args.decoder_layerdrop + self.share_input_output_embed = args.share_decoder_input_output_embed + + input_embed_dim = embed_tokens.embedding_dim + embed_dim = args.decoder_embed_dim + self.embed_dim = embed_dim + self.output_embed_dim = args.decoder_output_dim + + self.padding_idx = embed_tokens.padding_idx + self.max_target_positions = args.max_target_positions + + self.embed_tokens = embed_tokens + + self.embed_scale = 1.0 if args.no_scale_embedding else math.sqrt( + embed_dim) + + if not args.adaptive_input and args.quant_noise_pq > 0: + self.quant_noise = apply_quant_noise_( + nn.Linear(embed_dim, embed_dim, bias=False), + args.quant_noise_pq, + args.quant_noise_pq_block_size, + ) + else: + self.quant_noise = None + + self.project_in_dim = ( + Linear(input_embed_dim, embed_dim, bias=False) + if embed_dim != input_embed_dim else None) + self.embed_positions = ( + PositionalEmbedding( + self.max_target_positions, + embed_dim, + self.padding_idx, + learned=args.decoder_learned_pos, + ) if not args.no_token_positional_embeddings else None) + export = getattr(args, 'export', False) + if getattr(args, 'layernorm_embedding', False): + self.layernorm_embedding = LayerNorm(embed_dim, export=export) + else: + self.layernorm_embedding = None + + self.cross_self_attention = getattr(args, 'cross_self_attention', + False) + + if self.decoder_layerdrop > 0.0: + self.layers = LayerDropModuleList(p=self.decoder_layerdrop) + else: + self.layers = nn.ModuleList([]) + self.layers.extend([ + self.build_decoder_layer(args, no_encoder_attn) + for _ in range(args.decoder_layers) + ]) + self.num_layers = len(self.layers) + + if args.decoder_normalize_before and not getattr( + args, 'no_decoder_final_norm', False): + self.layer_norm = LayerNorm(embed_dim, export=export) + else: + self.layer_norm = None + + self.project_out_dim = ( + Linear(embed_dim, self.output_embed_dim, bias=False) + if embed_dim != self.output_embed_dim + and not args.tie_adaptive_weights else None) + + self.adaptive_softmax = None + self.output_projection = output_projection + if self.output_projection is None: + self.build_output_projection(args, dictionary, embed_tokens) + + def build_output_projection(self, args, dictionary, embed_tokens): + if args.adaptive_softmax_cutoff is not None: + self.adaptive_softmax = AdaptiveSoftmax( + len(dictionary), + self.output_embed_dim, + utils.eval_str_list(args.adaptive_softmax_cutoff, type=int), + dropout=args.adaptive_softmax_dropout, + adaptive_inputs=embed_tokens + if args.tie_adaptive_weights else None, + factor=args.adaptive_softmax_factor, + tie_proj=args.tie_adaptive_proj, + ) + elif self.share_input_output_embed: + self.output_projection = nn.Linear( + self.embed_tokens.weight.shape[1], + self.embed_tokens.weight.shape[0], + bias=False, + ) + self.output_projection.weight = self.embed_tokens.weight + else: + self.output_projection = nn.Linear( + self.output_embed_dim, len(dictionary), bias=False) + nn.init.normal_( + self.output_projection.weight, + mean=0, + std=self.output_embed_dim**-0.5) + num_base_layers = getattr(args, 'base_layers', 0) + for i in range(num_base_layers): + self.layers.insert( + ((i + 1) * args.decoder_layers) // (num_base_layers + 1), + BaseLayer(args), + ) + + def build_decoder_layer(self, args, no_encoder_attn=False): + layer = TransformerDecoderLayer(args, no_encoder_attn) + checkpoint = getattr(args, 'checkpoint_activations', False) + if checkpoint: + offload_to_cpu = getattr(args, 'offload_activations', False) + layer = checkpoint_wrapper(layer, offload_to_cpu=offload_to_cpu) + min_params_to_wrap = ( + getattr(args, 'min_params_to_wrap', DEFAULT_MIN_PARAMS_TO_WRAP) + if not checkpoint else 0) + layer = fsdp_wrap(layer, min_num_params=min_params_to_wrap) + return layer + + def forward( + self, + prev_output_tokens, + encoder_out: Optional[Dict[str, List[Tensor]]] = None, + incremental_state: Optional[Dict[str, Dict[str, + Optional[Tensor]]]] = None, + features_only: bool = False, + full_context_alignment: bool = False, + alignment_layer: Optional[int] = None, + alignment_heads: Optional[int] = None, + src_lengths: Optional[Any] = None, + return_all_hiddens: bool = False, + ): + """ + Args: + prev_output_tokens (LongTensor): previous decoder outputs of shape + `(batch, tgt_len)`, for teacher forcing + encoder_out (optional): output from the encoder, used for + encoder-side attention, should be of size T x B x C + incremental_state (dict): dictionary used for storing state during + :ref:`Incremental decoding` + features_only (bool, optional): only return features without + applying output layer (default: False). + full_context_alignment (bool, optional): don't apply + auto-regressive mask to self-attention (default: False). + + Returns: + tuple: + - the decoder's output of shape `(batch, tgt_len, vocab)` + - a dictionary with any model-specific outputs + """ + + x, extra = self.extract_features( + prev_output_tokens, + encoder_out=encoder_out, + incremental_state=incremental_state, + full_context_alignment=full_context_alignment, + alignment_layer=alignment_layer, + alignment_heads=alignment_heads, + ) + + if not features_only: + x = self.output_layer(x) + return x, extra + + def extract_features( + self, + prev_output_tokens, + encoder_out: Optional[Dict[str, List[Tensor]]], + incremental_state: Optional[Dict[str, Dict[str, + Optional[Tensor]]]] = None, + full_context_alignment: bool = False, + alignment_layer: Optional[int] = None, + alignment_heads: Optional[int] = None, + ): + return self.extract_features_scriptable( + prev_output_tokens, + encoder_out, + incremental_state, + full_context_alignment, + alignment_layer, + alignment_heads, + ) + + """ + A scriptable subclass of this class has an extract_features method and calls + super().extract_features, but super() is not supported in torchscript. A copy of + this function is made to be used in the subclass instead. + """ + + def extract_features_scriptable( + self, + prev_output_tokens, + encoder_out: Optional[Dict[str, List[Tensor]]], + incremental_state: Optional[Dict[str, Dict[str, + Optional[Tensor]]]] = None, + full_context_alignment: bool = False, + alignment_layer: Optional[int] = None, + alignment_heads: Optional[int] = None, + ): + """ + Similar to *forward* but only return features. + + Includes several features from "Jointly Learning to Align and + Translate with Transformer Models" (Garg et al., EMNLP 2019). + + Args: + full_context_alignment (bool, optional): don't apply + auto-regressive mask to self-attention (default: False). + alignment_layer (int, optional): return mean alignment over + heads at this layer (default: last layer). + alignment_heads (int, optional): only average alignment over + this many heads (default: all heads). + + Returns: + tuple: + - the decoder's features of shape `(batch, tgt_len, embed_dim)` + - a dictionary with any model-specific outputs + """ + bs, slen = prev_output_tokens.size() + if alignment_layer is None: + alignment_layer = self.num_layers - 1 + + enc: Optional[Tensor] = None + padding_mask: Optional[Tensor] = None + if encoder_out is not None and len(encoder_out['encoder_out']) > 0: + enc = encoder_out['encoder_out'][0] + assert (enc.size()[1] == bs + ), f'Expected enc.shape == (t, {bs}, c) got {enc.shape}' + if encoder_out is not None and len( + encoder_out['encoder_padding_mask']) > 0: + padding_mask = encoder_out['encoder_padding_mask'][0] + + # embed positions + positions = None + if self.embed_positions is not None: + positions = self.embed_positions( + prev_output_tokens, incremental_state=incremental_state) + if incremental_state is not None: + prev_output_tokens = prev_output_tokens[:, -1:] + if positions is not None: + positions = positions[:, -1:] + + # embed tokens and positions + x = self.embed_scale * self.embed_tokens(prev_output_tokens) + + if self.quant_noise is not None: + x = self.quant_noise(x) + + if self.project_in_dim is not None: + x = self.project_in_dim(x) + + if positions is not None: + x += positions + + if self.layernorm_embedding is not None: + x = self.layernorm_embedding(x) + + x = self.dropout_module(x) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + self_attn_padding_mask: Optional[Tensor] = None + if self.cross_self_attention or prev_output_tokens.eq( + self.padding_idx).any(): + self_attn_padding_mask = prev_output_tokens.eq(self.padding_idx) + + # decoder layers + attn: Optional[Tensor] = None + inner_states: List[Optional[Tensor]] = [x] + for idx, layer in enumerate(self.layers): + if incremental_state is None and not full_context_alignment: + self_attn_mask = self.buffered_future_mask(x) + else: + self_attn_mask = None + + x, layer_attn, self_attn_hidden = layer( + x, + enc, + padding_mask, + incremental_state, + self_attn_mask=self_attn_mask, + self_attn_padding_mask=self_attn_padding_mask, + need_attn=bool((idx == alignment_layer)), + need_head_weights=bool((idx == alignment_layer)), + ) + inner_states.append(x) + if layer_attn is not None and idx == alignment_layer: + attn = layer_attn.float().to(x) + + if attn is not None: + if alignment_heads is not None: + attn = attn[:alignment_heads] + + attn = attn.mean(dim=0) + + if self.layer_norm is not None: + x = self.layer_norm(x) + last_layer = x + # T x B x C -> B x T x C + x = x.transpose(0, 1) + if self.project_out_dim is not None: + x = self.project_out_dim(x) + return x, { + 'attn': [attn], + 'inner_states': inner_states, + 'last_layer': last_layer, + 'self_attn_padding_mask': self_attn_padding_mask + } + + def output_layer(self, features): + """Project features to the vocabulary size.""" + if self.adaptive_softmax is None: + # project back to size of vocabulary + return self.output_projection(features) + else: + return features + + def max_positions(self): + """Maximum output length supported by the decoder.""" + if self.embed_positions is None: + return self.max_target_positions + return min(self.max_target_positions, + self.embed_positions.max_positions) + + def buffered_future_mask(self, tensor): + dim = tensor.size(0) + if (self._future_mask.size(0) == 0 + or (not self._future_mask.device == tensor.device) + or self._future_mask.size(0) < dim): + self._future_mask = torch.triu( + utils.fill_with_neg_inf(torch.zeros([dim, dim])), 1) + self._future_mask = self._future_mask.to(tensor) + return self._future_mask[:dim, :dim] + + def upgrade_state_dict_named(self, state_dict, name): + """Upgrade a (possibly old) state dict for new versions of fairseq.""" + if isinstance(self.embed_positions, SinusoidalPositionalEmbedding): + weights_key = '{}.embed_positions.weights'.format(name) + if weights_key in state_dict: + del state_dict[weights_key] + state_dict['{}.embed_positions._float_tensor'.format( + name)] = torch.FloatTensor(1) + + if f'{name}.output_projection.weight' not in state_dict: + if self.share_input_output_embed: + embed_out_key = f'{name}.embed_tokens.weight' + else: + embed_out_key = f'{name}.embed_out' + if embed_out_key in state_dict: + state_dict[f'{name}.output_projection.weight'] = state_dict[ + embed_out_key] + if not self.share_input_output_embed: + del state_dict[embed_out_key] + + for i in range(self.num_layers): + # update layer norms + layer_norm_map = { + '0': 'self_attn_layer_norm', + '1': 'encoder_attn_layer_norm', + '2': 'final_layer_norm', + } + for old, new in layer_norm_map.items(): + for m in ('weight', 'bias'): + k = '{}.layers.{}.layer_norms.{}.{}'.format( + name, i, old, m) + if k in state_dict: + state_dict['{}.layers.{}.{}.{}'.format( + name, i, new, m)] = state_dict[k] + del state_dict[k] + + version_key = '{}.version'.format(name) + if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) <= 2: + # earlier checkpoints did not normalize after the stack of layers + self.layer_norm = None + self.normalize = False + state_dict[version_key] = torch.Tensor([1]) + + return state_dict + + +def Embedding(num_embeddings, embedding_dim, padding_idx): + m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx) + nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5) + nn.init.constant_(m.weight[padding_idx], 0) + return m + + +def Linear(in_features, out_features, bias=True): + m = nn.Linear(in_features, out_features, bias) + nn.init.xavier_uniform_(m.weight) + if bias: + nn.init.constant_(m.bias, 0.0) + return m + + +def base_architecture(args): + args.encoder_embed_path = getattr(args, 'encoder_embed_path', None) + args.encoder_embed_dim = getattr(args, 'encoder_embed_dim', 512) + args.encoder_ffn_embed_dim = getattr(args, 'encoder_ffn_embed_dim', 2048) + args.encoder_layers = getattr(args, 'encoder_layers', 6) + args.encoder_attention_heads = getattr(args, 'encoder_attention_heads', 8) + args.encoder_normalize_before = getattr(args, 'encoder_normalize_before', + False) + args.encoder_learned_pos = getattr(args, 'encoder_learned_pos', False) + args.decoder_embed_path = getattr(args, 'decoder_embed_path', None) + args.decoder_embed_dim = getattr(args, 'decoder_embed_dim', + args.encoder_embed_dim) + args.decoder_ffn_embed_dim = getattr(args, 'decoder_ffn_embed_dim', + args.encoder_ffn_embed_dim) + args.decoder_layers = getattr(args, 'decoder_layers', 6) + args.decoder_attention_heads = getattr(args, 'decoder_attention_heads', 8) + args.decoder_normalize_before = getattr(args, 'decoder_normalize_before', + False) + args.decoder_learned_pos = getattr(args, 'decoder_learned_pos', False) + args.attention_dropout = getattr(args, 'attention_dropout', 0.0) + args.activation_dropout = getattr(args, 'activation_dropout', 0.0) + args.activation_fn = getattr(args, 'activation_fn', 'relu') + args.dropout = getattr(args, 'dropout', 0.1) + args.adaptive_softmax_cutoff = getattr(args, 'adaptive_softmax_cutoff', + None) + args.adaptive_softmax_dropout = getattr(args, 'adaptive_softmax_dropout', + 0) + args.share_decoder_input_output_embed = getattr( + args, 'share_decoder_input_output_embed', False) + args.share_all_embeddings = getattr(args, 'share_all_embeddings', False) + args.no_token_positional_embeddings = getattr( + args, 'no_token_positional_embeddings', False) + args.adaptive_input = getattr(args, 'adaptive_input', False) + args.no_cross_attention = getattr(args, 'no_cross_attention', False) + args.cross_self_attention = getattr(args, 'cross_self_attention', False) + + args.decoder_output_dim = getattr(args, 'decoder_output_dim', + args.decoder_embed_dim) + args.decoder_input_dim = getattr(args, 'decoder_input_dim', + args.decoder_embed_dim) + + args.no_scale_embedding = getattr(args, 'no_scale_embedding', False) + args.layernorm_embedding = getattr(args, 'layernorm_embedding', False) + args.tie_adaptive_weights = getattr(args, 'tie_adaptive_weights', False) + args.checkpoint_activations = getattr(args, 'checkpoint_activations', + False) + args.offload_activations = getattr(args, 'offload_activations', False) + if args.offload_activations: + args.checkpoint_activations = True + args.encoder_layers_to_keep = getattr(args, 'encoder_layers_to_keep', None) + args.decoder_layers_to_keep = getattr(args, 'decoder_layers_to_keep', None) + args.encoder_layerdrop = getattr(args, 'encoder_layerdrop', 0) + args.decoder_layerdrop = getattr(args, 'decoder_layerdrop', 0) + args.quant_noise_pq = getattr(args, 'quant_noise_pq', 0) + args.quant_noise_pq_block_size = getattr(args, 'quant_noise_pq_block_size', + 8) + args.quant_noise_scalar = getattr(args, 'quant_noise_scalar', 0) + + +def transformer_deep(args): + args.encoder_embed_dim = getattr(args, 'encoder_embed_dim', 768) + args.encoder_ffn_embed_dim = getattr(args, 'encoder_ffn_embed_dim', 3072) + args.encoder_attention_heads = getattr(args, 'encoder_attention_heads', 12) + args.encoder_layers = getattr(args, 'encoder_layers', 24) + args.encoder_normalize_before = getattr(args, 'encoder_normalize_before', + True) + args.decoder_normalize_before = getattr(args, 'decoder_normalize_before', + True) + args.decoder_layers = getattr(args, 'decoder_layers', 3) + args.decoder_embed_dim = getattr(args, 'decoder_embed_dim', 768) + args.decoder_ffn_embed_dim = getattr(args, 'decoder_ffn_embed_dim', 3072) + args.decoder_attention_heads = getattr(args, 'decoder_attention_heads', 12) + args.attention_dropout = getattr(args, 'attention_dropout', 0.01) + args.activation_dropout = getattr(args, 'activation_dropout', 0.01) + args.dropout = getattr(args, 'dropout', 0.01) + base_architecture(args) diff --git a/runtime/Lib/site_packages/models/nlp/canmt/canmt_translation.py b/runtime/Lib/site_packages/models/nlp/canmt/canmt_translation.py new file mode 100644 index 0000000000000000000000000000000000000000..fe9b933f9dd6303f813806f65e266e0bf729cc54 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/canmt/canmt_translation.py @@ -0,0 +1,78 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math +import os.path as osp +from typing import Any, Dict, List, Optional, Tuple + +import numpy +import torch +import torch.nn as nn +from torch import Tensor + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['CanmtForTranslation'] + + +@MODELS.register_module( + Tasks.competency_aware_translation, module_name=Models.canmt) +class CanmtForTranslation(TorchModel): + + def __init__(self, model_dir, **args): + """ + CanmtForTranslation implements a Competency-Aware Neural Machine Translaton, + which has both translation and self-estimation abilities. + + For more details, please refer to https://aclanthology.org/2022.emnlp-main.330.pdf + """ + super().__init__(model_dir=model_dir, **args) + self.args = args + cfg_file = osp.join(model_dir, ModelFile.CONFIGURATION) + self.cfg = Config.from_file(cfg_file) + from fairseq.data import Dictionary + self.vocab_src = Dictionary.load(osp.join(model_dir, 'dict.src.txt')) + self.vocab_tgt = Dictionary.load(osp.join(model_dir, 'dict.tgt.txt')) + self.model = self.build_model(model_dir) + self.generator = self.build_generator(self.model, self.vocab_tgt, + self.cfg['decode']) + + def build_model(self, model_dir): + from .canmt_model import CanmtModel + state = self.load_checkpoint( + osp.join(model_dir, ModelFile.TORCH_MODEL_FILE), 'cpu') + cfg = state['cfg'] + model = CanmtModel.build_model(cfg['model'], self) + model.load_state_dict(state['model'], model_cfg=cfg['model']) + return model + + def build_generator(cls, model, vocab_tgt, args): + from .sequence_generator import SequenceGenerator + return SequenceGenerator( + model, + vocab_tgt, + beam_size=args['beam'], + len_penalty=args['lenpen']) + + def load_checkpoint(self, path: str, device: torch.device): + state_dict = torch.load(path, map_location=device) + self.load_state_dict(state_dict, strict=False) + return state_dict + + def forward(self, input: Dict[str, Dict]): + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data which contains following: + - src_tokens: tensor with shape (2478,242,24,4), + - src_lengths: tensor with shape (4) + + + Returns: + Dict[str, Tensor]: results which contains following: + - predictions: tokens need to be decode by tokenizer with shape [1377, 4959, 2785, 6392...] + """ + input = {'net_input': input} + return self.generator.generate(input) diff --git a/runtime/Lib/site_packages/models/nlp/canmt/sequence_generator.py b/runtime/Lib/site_packages/models/nlp/canmt/sequence_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..8a5134a4d678176bbfc8a9acdfa36b6b2ac06dd1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/canmt/sequence_generator.py @@ -0,0 +1,850 @@ +# Part of the implementation is borrowed and modified from FAIRSEQ, +# publicly available at https://github.com/facebookresearch/fairseq +# Copyright 2022-2023 The Alibaba MT Team Authors. All rights reserved. +import math +import sys +from typing import Dict, List, Optional + +import numpy +import torch +import torch.nn as nn +from fairseq import search, utils +from fairseq.data import data_utils +from fairseq.models import FairseqIncrementalDecoder +from fairseq.ngram_repeat_block import NGramRepeatBlock +from torch import Tensor + + +def label_smoothed_nll_loss(lprobs, + target, + epsilon, + ignore_index=None, + reduce=True): + if target.dim() == lprobs.dim() - 1: + target = target.unsqueeze(-1) + if target.dtype != torch.int64: + target = target.type(torch.int64) + nll_loss = -lprobs.gather(dim=-1, index=target) + smooth_loss = -lprobs.sum(dim=-1, keepdim=True) + if ignore_index is not None: + pad_mask = target.eq(ignore_index) + nll_loss.masked_fill_(pad_mask, 0.0) + smooth_loss.masked_fill_(pad_mask, 0.0) + else: + nll_loss = nll_loss.squeeze(-1) + smooth_loss = smooth_loss.squeeze(-1) + if reduce: + nll_loss = nll_loss.sum() + smooth_loss = smooth_loss.sum() + eps_i = epsilon / (lprobs.size(-1) - 1) + loss = (1.0 - epsilon - eps_i) * nll_loss + eps_i * smooth_loss + return loss, nll_loss + + +class SequenceGenerator(nn.Module): + + def __init__( + self, + model, + tgt_dict, + beam_size=1, + max_len_a=0, + max_len_b=200, + max_len=200, + min_len=1, + normalize_scores=True, + len_penalty=1.0, + unk_penalty=0.0, + temperature=1.0, + match_source_len=False, + no_repeat_ngram_size=0, + search_strategy=None, + eos=None, + symbols_to_strip_from_output=None, + lm_model=None, + lm_weight=1.0, + recon_force_decoding=True, + trans_force_decoding=False, + ): + """Generates translations of a given source sentence. + + Args: + models (List[~fairseq.models.FairseqModel]): ensemble of models, + currently support fairseq.models.TransformerModel for scripting + beam_size (int, optional): beam width (default: 1) + max_len_a/b (int, optional): generate sequences of maximum length + ax + b, where x is the source length + max_len (int, optional): the maximum length of the generated output + (not including end-of-sentence) + min_len (int, optional): the minimum length of the generated output + (not including end-of-sentence) + normalize_scores (bool, optional): normalize scores by the length + of the output (default: True) + len_penalty (float, optional): length penalty, where <1.0 favors + shorter, >1.0 favors longer sentences (default: 1.0) + unk_penalty (float, optional): unknown word penalty, where <0 + produces more unks, >0 produces fewer (default: 0.0) + temperature (float, optional): temperature, where values + >1.0 produce more uniform samples and values <1.0 produce + sharper samples (default: 1.0) + match_source_len (bool, optional): outputs should match the source + length (default: False) + """ + super().__init__() + self.model = model + self.recon_force_decoding = recon_force_decoding + self.trans_force_decoding = trans_force_decoding + self.tgt_dict = tgt_dict + self.pad = tgt_dict.pad() + self.unk = tgt_dict.unk() + self.eos = tgt_dict.eos() if eos is None else eos + self.symbols_to_strip_from_output = ( + symbols_to_strip_from_output.union({self.eos}) + if symbols_to_strip_from_output is not None else {self.eos}) + self.vocab_size = len(tgt_dict) + self.beam_size = beam_size + # the max beam size is the dictionary size - 1, since we never select pad + self.beam_size = min(beam_size, self.vocab_size - 1) + self.max_len_a = max_len_a + self.max_len_b = max_len_b + self.min_len = min_len + self.max_len = max_len + + self.normalize_scores = normalize_scores + self.len_penalty = len_penalty + self.unk_penalty = unk_penalty + self.temperature = temperature + self.match_source_len = match_source_len + self.eps = 0.1 + + if no_repeat_ngram_size > 0: + self.repeat_ngram_blocker = NGramRepeatBlock(no_repeat_ngram_size) + else: + self.repeat_ngram_blocker = None + + assert temperature > 0, '--temperature must be greater than 0' + + self.search = ( + search.BeamSearch(tgt_dict) + if search_strategy is None else search_strategy) + # We only need to set src_lengths in LengthConstrainedBeamSearch. + # As a module attribute, setting it would break in multithread + # settings when the model is shared. + self.should_set_src_lengths = ( + hasattr(self.search, 'needs_src_lengths') + and self.search.needs_src_lengths) + + self.model.eval() + + self.lm_model = lm_model + self.lm_weight = lm_weight + if self.lm_model is not None: + self.lm_model.eval() + + def cuda(self): + self.model.cuda() + return self + + @torch.no_grad() + def forward( + self, + sample: Dict[str, Dict[str, Tensor]], + prefix_tokens: Optional[Tensor] = None, + bos_token: Optional[int] = None, + ): + """Generate a batch of translations. + + Args: + sample (dict): batch + prefix_tokens (torch.LongTensor, optional): force decoder to begin + with these tokens + bos_token (int, optional): beginning of sentence token + (default: self.eos) + """ + return self._generate(sample, prefix_tokens, bos_token=bos_token) + + # TODO(myleott): unused, deprecate after pytorch-translate migration + def generate_batched_itr(self, + data_itr, + beam_size=None, + cuda=False, + timer=None): + """Iterate over a batched dataset and yield individual translations. + Args: + cuda (bool, optional): use GPU for generation + timer (StopwatchMeter, optional): time generations + """ + for sample in data_itr: + s = utils.move_to_cuda(sample) if cuda else sample + if 'net_input' not in s: + continue + input = s['net_input'] + # model.forward normally channels prev_output_tokens into the decoder + # separately, but SequenceGenerator directly calls model.encoder + encoder_input = { + k: v + for k, v in input.items() if k != 'prev_output_tokens' + } + if timer is not None: + timer.start() + with torch.no_grad(): + hypos = self.generate(encoder_input) + if timer is not None: + timer.stop(sum(len(h[0]['tokens']) for h in hypos)) + for i, id in enumerate(s['id'].data): + # remove padding + src = utils.strip_pad(input['src_tokens'].data[i, :], self.pad) + ref = ( + utils.strip_pad(s['target'].data[i, :], self.pad) + if s['target'] is not None else None) + yield id, src, ref, hypos[i] + + @torch.no_grad() + def generate(self, sample: Dict[str, Dict[str, Tensor]], + **kwargs) -> List[List[Dict[str, Tensor]]]: + """Generate translations. Match the api of other fairseq generators. + + Args: + models (List[~fairseq.models.FairseqModel]): ensemble of models + sample (dict): batch + prefix_tokens (torch.LongTensor, optional): force decoder to begin + with these tokens + constraints (torch.LongTensor, optional): force decoder to include + the list of constraints + bos_token (int, optional): beginning of sentence token + (default: self.eos) + """ + from torch import tensor + finalized = self._generate(sample, **kwargs) + tokens_list = [] + decoder_list = [] + for i in range(len(finalized)): + sent = finalized[i][0] + tokens = sent['tokens'] + tokens_list.append(tokens) + decoder_out = sent['decoder_out'] + decoder_list.append(decoder_out) + # padding tokens + size = max(v.size(0) for v in tokens_list) + batch_size = len(tokens_list) + + for i in range(len(tokens_list)): + tokens_list[i] = tokens_list[i].roll(1, 0) + decoder_list[i] = decoder_list[i].roll(1, 0) + res = tokens_list[0].new(batch_size, size).fill_(self.pad) + + def copy_tensor(src, dst): + assert dst.numel() == src.numel() + dst.copy_(src) + + for i, v in enumerate(tokens_list): + copy_tensor(v, res[i][:len(v)] if True else res[i][:len(v)]) + tgt_tokens = res + decoder_padding_mask = tgt_tokens.eq(self.pad) + + # generate for src reconstruction + decoder_out_re = self.model.decoder( + tgt_tokens, + encoder_out=None, + full_context_alignment=True, + ) + decoder_out_re = decoder_out_re[1]['last_layer'] + decoder_outs = dict() + decoder_outs['encoder_out'] = [decoder_out_re] + decoder_outs['encoder_padding_mask'] = [decoder_padding_mask] + decoder_outs['src_tokens'] = [tgt_tokens] + decoder_outs['encoder_embedding'] = [] + decoder_outs['encoder_states'] = [] + decoder_outs['src_lengths'] = [] + + scores = self._forward_src(decoder_outs, sample) + return finalized, scores + + def _forward( + self, + sample: Dict[str, Dict[str, Tensor]], + ): + net_input = sample['net_input'] + src_tokens = net_input['src_tokens'] + prev_output_tokens = net_input['prev_output_tokens'] + encoder_outs = self.model.forward_encoder(net_input) + final_encoder_out = encoder_outs['encoder_out'][0].transpose(0, 1) + final_encoder_embedding = encoder_outs['encoder_embedding'][0] + self.model.has_incremental = False + lprobs, avg_attn_scores, decoder_outs = self.model.forward_decoder( + prev_output_tokens, encoder_outs, incremental_states=None) + decoder_outs = decoder_outs.transpose(0, 1) + self.model.has_incremental = True + batch_size = src_tokens.size(0) + # list of completed sentences + finalized = torch.jit.annotate( + List[List[Dict[str, Tensor]]], + [ + torch.jit.annotate(List[Dict[str, Tensor]], []) + for i in range(batch_size) + ], + ) # contains lists of dictionaries of infomation about the hypothesis being finalized at each step + for i in range(batch_size): + eos_idx = numpy.where(sample['target'][i].cpu() == self.eos)[0][0] + finalized[i].append({ + 'tokens': + sample['target'][i][:eos_idx + 1], + 'decoder_out': + decoder_outs[i][:eos_idx + 1], + 'final_encoder_embedding': + final_encoder_embedding[i], + 'final_encoder_out': + final_encoder_out[i], + }) + return finalized + + def _forward_src( + self, + encoder_outs, + sample: Dict[str, Dict[str, Tensor]], + ): + net_input = sample['net_input'] + src_tokens = net_input['src_tokens'] + src_lengths = net_input['src_lengths'] + prev_output_tokens = net_input['prev_src_tokens'] + self.model.has_incremental = False + lprobs, avg_attn_scores, _, decoder_outs = self.model.forward_decoder_src( + prev_output_tokens, encoder_outs, incremental_states=None) + logits = decoder_outs[0] + lprobs = utils.log_softmax(logits, dim=-1) + sources = net_input['sources'].view(-1) + lprobs = lprobs.view(-1, lprobs.size(-1)) + + scores, _ = label_smoothed_nll_loss( + lprobs, + sources, + epsilon=self.eps, + ignore_index=self.pad, + reduce=False) + batch_size = src_tokens.shape[0] + scores = scores.reshape(batch_size, -1) + scores = torch.cat((scores.sum(axis=-1, keepdim=True) + / src_lengths.reshape(batch_size, 1), scores), + axis=-1) + return scores + + def _generate( + self, + sample: Dict[str, Dict[str, Tensor]], + prefix_tokens: Optional[Tensor] = None, + constraints: Optional[Tensor] = None, + bos_token: Optional[int] = None, + ): + incremental_states = torch.jit.annotate( + Dict[str, Dict[str, Optional[Tensor]]], + torch.jit.annotate(Dict[str, Dict[str, Optional[Tensor]]], {}), + ) + + net_input = sample['net_input'] + if 'src_tokens' in net_input: + src_tokens = net_input['src_tokens'] + # length of the source text being the character length except EndOfSentence and pad + src_lengths = ((src_tokens.ne(self.eos) + & src_tokens.ne(self.pad)).long().sum(dim=1)) + else: + raise Exception( + 'expected src_tokens or source in net input. input keys: ' + + str(net_input.keys())) + + # bsz: total number of sentences in beam + # Note that src_tokens may have more than 2 dimensions (i.e. audio features) + bsz, src_len = src_tokens.size()[:2] + beam_size = self.beam_size + + max_len: int = -1 + if self.match_source_len: + max_len = src_lengths.max().item() + else: + max_len = min( + int(self.max_len_a * src_len + self.max_len_b), + self.max_len - 1, + ) + assert ( + self.min_len <= max_len + ), 'min_len cannot be larger than max_len, please adjust these!' + # compute the encoder output for each beam + encoder_outs = self.model.forward_encoder(net_input) + + final_encoder_out = encoder_outs['encoder_out'][0].transpose(0, 1) + final_encoder_embedding = encoder_outs['encoder_embedding'][0] + + # placeholder of indices for bsz * beam_size to hold tokens and accumulative scores + new_order = torch.arange(bsz).view(-1, 1).repeat(1, beam_size).view(-1) + new_order = new_order.to(src_tokens.device).long() + encoder_outs = self.model.reorder_encoder_out(encoder_outs, new_order) + + # ensure encoder_outs is a List. + assert encoder_outs is not None + + # initialize buffers + scores = (torch.zeros(bsz * beam_size, + max_len + 1).to(src_tokens).float() + ) # +1 for eos; pad is never chosen for scoring + tokens = (torch.zeros(bsz * beam_size, + max_len + 2).to(src_tokens).long().fill_( + self.pad)) # +2 for eos and pad + tokens[:, 0] = self.eos if bos_token is None else bos_token + attn: Optional[Tensor] = None + + cands_to_ignore = (torch.zeros(bsz, beam_size).to(src_tokens).eq(-1) + ) # forward and backward-compatible False mask + + # list of completed sentences + finalized = torch.jit.annotate( + List[List[Dict[str, Tensor]]], + [ + torch.jit.annotate(List[Dict[str, Tensor]], []) + for i in range(bsz) + ], + ) # contains lists of dictionaries of infomation about the hypothesis being finalized at each step + + # a boolean array indicating if the sentence at the index is finished or not + finished = [False for i in range(bsz)] + num_remaining_sent = bsz # number of sentences remaining + + # number of candidate hypos per step + cand_size = 2 * beam_size # 2 x beam size in case half are EOS + + # offset arrays for converting between different indexing schemes + bbsz_offsets = ((torch.arange(0, bsz) + * beam_size).unsqueeze(1).type_as(tokens).to( + src_tokens.device)) + cand_offsets = torch.arange(0, cand_size).type_as(tokens).to( + src_tokens.device) + + reorder_state: Optional[Tensor] = None + batch_idxs: Optional[Tensor] = None + + original_batch_idxs: Optional[Tensor] = None + if 'id' in sample and isinstance(sample['id'], Tensor): + original_batch_idxs = sample['id'] + else: + original_batch_idxs = torch.arange(0, bsz).type_as(tokens) + + for step in range(max_len + 1): # one extra step for EOS marker + # reorder decoder internal states based on the prev choice of beams + if reorder_state is not None: + if batch_idxs is not None: + # update beam indices to take into account removed sentences + corr = batch_idxs - torch.arange( + batch_idxs.numel()).type_as(batch_idxs) + reorder_state.view(-1, beam_size).add_( + corr.unsqueeze(-1) * beam_size) + original_batch_idxs = original_batch_idxs[batch_idxs] + + self.model.reorder_incremental_state(incremental_states, + reorder_state) + encoder_outs = self.model.reorder_encoder_out( + encoder_outs, reorder_state) + + lprobs, avg_attn_scores, decoder_out_word = self.model.forward_decoder( + tokens[:, :step + 1], + encoder_outs, + incremental_states, + self.temperature, + ) + # (length, batch*beam, hidden_size) - >(batch*beam, length, hidden_size) + decoder_out_word = decoder_out_word.transpose(0, 1) + if step == 0: + decoder_out_tensor = decoder_out_word + else: + decoder_out_tensor = torch.cat( + [decoder_out_tensor, decoder_out_word], dim=1) + + lprobs[lprobs != lprobs] = torch.tensor(-math.inf).to(lprobs) + + lprobs[:, self.pad] = -math.inf # never select pad + lprobs[:, self.unk] -= self.unk_penalty # apply unk penalty + + # handle max length constraint + if step >= max_len: + lprobs[:, :self.eos] = -math.inf + lprobs[:, self.eos + 1:] = -math.inf + + # handle prefix tokens (possibly with different lengths) + if (prefix_tokens is not None and step < prefix_tokens.size(1) + and step < max_len): + + lprobs, tokens, scores = self._prefix_tokens( + step, lprobs, scores, tokens, prefix_tokens, beam_size) + elif step < self.min_len: + # minimum length constraint (does not apply if using prefix_tokens) + lprobs[:, self.eos] = -math.inf + + # Record attention scores, only support avg_attn_scores is a Tensor + if avg_attn_scores is not None: + if attn is None: + attn = torch.empty(bsz * beam_size, + avg_attn_scores.size(1), + max_len + 2).to(scores) + attn[:, :, step + 1].copy_(avg_attn_scores) + + scores = scores.type_as(lprobs) + eos_bbsz_idx = torch.empty(0).to( + tokens + ) # indices of hypothesis ending with eos (finished sentences) + eos_scores = torch.empty(0).to( + scores + ) # scores of hypothesis ending with eos (finished sentences) + + if self.should_set_src_lengths: + self.search.set_src_lengths(src_lengths) + + if self.repeat_ngram_blocker is not None: + lprobs = self.repeat_ngram_blocker(tokens, lprobs, bsz, + beam_size, step) + + # Shape: (batch, cand_size) + cand_scores, cand_indices, cand_beams = self.search.step( + step, + lprobs.view(bsz, -1, self.vocab_size), + scores.view(bsz, beam_size, -1)[:, :, :step], + tokens[:, :step + 1], + original_batch_idxs, + ) + + # cand_bbsz_idx contains beam indices for the top candidate + # hypotheses, with a range of values: [0, bsz*beam_size), + # and dimensions: [bsz, cand_size] + cand_bbsz_idx = cand_beams.add(bbsz_offsets) + + # finalize hypotheses that end in eos + # Shape of eos_mask: (batch size, beam size) + eos_mask = cand_indices.eq(self.eos) & cand_scores.ne(-math.inf) + eos_mask[:, :beam_size][cands_to_ignore] = torch.tensor(0).to( + eos_mask) + + # only consider eos when it's among the top beam_size indices + # Now we know what beam item(s) to finish + # Shape: 1d list of absolute-numbered + eos_bbsz_idx = torch.masked_select( + cand_bbsz_idx[:, :beam_size], mask=eos_mask[:, :beam_size]) + + finalized_sents: List[int] = [] + if eos_bbsz_idx.numel() > 0: + eos_scores = torch.masked_select( + cand_scores[:, :beam_size], mask=eos_mask[:, :beam_size]) + + finalized_sents = self.finalize_hypos( + step, + eos_bbsz_idx, + eos_scores, + tokens, + scores, + finalized, + finished, + beam_size, + attn, + src_lengths, + max_len, + decoder_out_tensor, + ) + num_remaining_sent -= len(finalized_sents) + + assert num_remaining_sent >= 0 + if num_remaining_sent == 0: + break + if self.search.stop_on_max_len and step >= max_len: + break + assert step < max_len, f'{step} < {max_len}' + + # Remove finalized sentences (ones for which {beam_size} + # finished hypotheses have been generated) from the batch. + if len(finalized_sents) > 0: + new_bsz = bsz - len(finalized_sents) + + # construct batch_idxs which holds indices of batches to keep for the next pass + batch_mask = torch.ones( + bsz, dtype=torch.bool, device=cand_indices.device) + batch_mask[finalized_sents] = False + # TODO replace `nonzero(as_tuple=False)` after TorchScript supports it + batch_idxs = torch.arange( + bsz, device=cand_indices.device).masked_select(batch_mask) + + # Choose the subset of the hypothesized constraints that will continue + self.search.prune_sentences(batch_idxs) + + eos_mask = eos_mask[batch_idxs] + cand_beams = cand_beams[batch_idxs] + bbsz_offsets.resize_(new_bsz, 1) + cand_bbsz_idx = cand_beams.add(bbsz_offsets) + cand_scores = cand_scores[batch_idxs] + cand_indices = cand_indices[batch_idxs] + + if prefix_tokens is not None: + prefix_tokens = prefix_tokens[batch_idxs] + src_lengths = src_lengths[batch_idxs] + cands_to_ignore = cands_to_ignore[batch_idxs] + + scores = scores.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, -1) + tokens = tokens.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, -1) + decoder_out_tensor = decoder_out_tensor.contiguous().view( + bsz, -1)[batch_idxs].view(new_bsz * beam_size, + decoder_out_tensor.size(1), -1) + if attn is not None: + attn = attn.view(bsz, -1)[batch_idxs].view( + new_bsz * beam_size, attn.size(1), -1) + bsz = new_bsz + else: + batch_idxs = None + + # Set active_mask so that values > cand_size indicate eos hypos + # and values < cand_size indicate candidate active hypos. + # After, the min values per row are the top candidate active hypos + + eos_mask[:, :beam_size] = ~((~cands_to_ignore) + & (~eos_mask[:, :beam_size])) + active_mask = torch.add( + eos_mask.type_as(cand_offsets) * cand_size, + cand_offsets[:eos_mask.size(1)], + ) + + # get the top beam_size active hypotheses, which are just + # the hypos with the smallest values in active_mask. + # {active_hypos} indicates which {beam_size} hypotheses + # from the list of {2 * beam_size} candidates were + # selected. Shapes: (batch size, beam size) + new_cands_to_ignore, active_hypos = torch.topk( + active_mask, k=beam_size, dim=1, largest=False) + + # update cands_to_ignore to ignore any finalized hypos. + cands_to_ignore = new_cands_to_ignore.ge(cand_size)[:, :beam_size] + # Make sure there is at least one active item for each sentence in the batch. + assert (~cands_to_ignore).any(dim=1).all() + + # update cands_to_ignore to ignore any finalized hypos + + # {active_bbsz_idx} denotes which beam number is continued for each new hypothesis (a beam + # can be selected more than once). + active_bbsz_idx = torch.gather( + cand_bbsz_idx, dim=1, index=active_hypos) + active_scores = torch.gather( + cand_scores, dim=1, index=active_hypos) + + active_bbsz_idx = active_bbsz_idx.view(-1) + active_scores = active_scores.view(-1) + + # copy tokens and scores for active hypotheses + + # Set the tokens for each beam (can select the same row more than once) + tokens[:, :step + 1] = torch.index_select( + tokens[:, :step + 1], dim=0, index=active_bbsz_idx) + # Select the next token for each of them + tokens.view(bsz, beam_size, -1)[:, :, step + 1] = torch.gather( + cand_indices, dim=1, index=active_hypos) + if step > 0: + scores[:, :step] = torch.index_select( + scores[:, :step], dim=0, index=active_bbsz_idx) + scores.view(bsz, beam_size, -1)[:, :, step] = torch.gather( + cand_scores, dim=1, index=active_hypos) + + # Update constraints based on which candidates were selected for the next beam + self.search.update_constraints(active_hypos) + + # copy attention for active hypotheses + if attn is not None: + attn[:, :, :step + 2] = torch.index_select( + attn[:, :, :step + 2], dim=0, index=active_bbsz_idx) + + # reorder incremental state in decoder + reorder_state = active_bbsz_idx + + # sort by score descending + for sent in range(len(finalized)): + scores = torch.tensor( + [float(elem['score'].item()) for elem in finalized[sent]]) + _, sorted_scores_indices = torch.sort(scores, descending=True) + finalized[sent] = [ + finalized[sent][ssi] for ssi in sorted_scores_indices + ] + finalized[sent] = torch.jit.annotate(List[Dict[str, Tensor]], + finalized[sent]) + finalized[sent][0][ + 'final_encoder_embedding'] = final_encoder_embedding[sent] + finalized[sent][0]['final_encoder_out'] = final_encoder_out[sent] + return finalized + + def _prefix_tokens(self, step: int, lprobs, scores, tokens, prefix_tokens, + beam_size: int): + """Handle prefix tokens""" + prefix_toks = prefix_tokens[:, step].unsqueeze(-1).repeat( + 1, beam_size).view(-1) + prefix_lprobs = lprobs.gather(-1, prefix_toks.unsqueeze(-1)) + prefix_mask = prefix_toks.ne(self.pad) + lprobs[prefix_mask] = torch.tensor(-math.inf).to(lprobs) + lprobs[prefix_mask] = lprobs[prefix_mask].scatter( + -1, prefix_toks[prefix_mask].unsqueeze(-1), + prefix_lprobs[prefix_mask]) + # if prefix includes eos, then we should make sure tokens and + # scores are the same across all beams + eos_mask = prefix_toks.eq(self.eos) + if eos_mask.any(): + # validate that the first beam matches the prefix + first_beam = tokens[eos_mask].view(-1, beam_size, + tokens.size(-1))[:, 0, + 1:step + 1] + eos_mask_batch_dim = eos_mask.view(-1, beam_size)[:, 0] + target_prefix = prefix_tokens[eos_mask_batch_dim][:, :step] + assert (first_beam == target_prefix).all() + + # copy tokens, scores and lprobs from the first beam to all beams + tokens = self.replicate_first_beam(tokens, eos_mask_batch_dim, + beam_size) + scores = self.replicate_first_beam(scores, eos_mask_batch_dim, + beam_size) + lprobs = self.replicate_first_beam(lprobs, eos_mask_batch_dim, + beam_size) + return lprobs, tokens, scores + + def replicate_first_beam(self, tensor, mask, beam_size: int): + tensor = tensor.view(-1, beam_size, tensor.size(-1)) + tensor[mask] = tensor[mask][:, :1, :] + return tensor.view(-1, tensor.size(-1)) + + def finalize_hypos( + self, + step: int, + bbsz_idx, + eos_scores, + tokens, + scores, + finalized: List[List[Dict[str, Tensor]]], + finished: List[bool], + beam_size: int, + attn: Optional[Tensor], + src_lengths, + max_len: int, + decoder_out=None, + ): + """Finalize hypothesis, store finalized information in `finalized`, and change `finished` accordingly. + A sentence is finalized when {beam_size} finished items have been collected for it. + + Returns number of sentences (not beam items) being finalized. + These will be removed from the batch and not processed further. + Args: + bbsz_idx (Tensor): + """ + assert bbsz_idx.numel() == eos_scores.numel() + + # clone relevant token and attention tensors. + # tokens is (batch * beam, max_len). So the index_select + # gets the newly EOS rows, then selects cols 1..{step + 2} + if decoder_out is not None: + decoder_out_clone = decoder_out.index_select(0, bbsz_idx) + + tokens_clone = tokens.index_select(0, bbsz_idx)[:, 1:step + 2] + # skip the first index, which is EOS + + tokens_clone[:, step] = self.eos + attn_clone = ( + attn.index_select(0, bbsz_idx)[:, :, 1:step + + 2] if attn is not None else None) + + # compute scores per token position + pos_scores = scores.index_select(0, bbsz_idx)[:, :step + 1] + pos_scores[:, step] = eos_scores + # convert from cumulative to per-position scores + pos_scores[:, 1:] = pos_scores[:, 1:] - pos_scores[:, :-1] + + # normalize sentence-level scores + if self.normalize_scores: + eos_scores /= (step + 1)**self.len_penalty + + # cum_unfin records which sentences in the batch are finished. + # It helps match indexing between (a) the original sentences + # in the batch and (b) the current, possibly-reduced set of + # sentences. + cum_unfin: List[int] = [] + prev = 0 + for f in finished: + if f: + prev += 1 + else: + cum_unfin.append(prev) + + # The keys here are of the form "{sent}_{unfin_idx}", where + # "unfin_idx" is the index in the current (possibly reduced) + # list of sentences, and "sent" is the index in the original, + # unreduced batch + # set() is not supported in script export + sents_seen: Dict[str, Optional[Tensor]] = {} + + # For every finished beam item + for i in range(bbsz_idx.size()[0]): + idx = bbsz_idx[i] + score = eos_scores[i] + # sentence index in the current (possibly reduced) batch + unfin_idx = idx // beam_size + # sentence index in the original (unreduced) batch + sent = unfin_idx + cum_unfin[unfin_idx] + # Cannot create dict for key type '(int, int)' in torchscript. + # The workaround is to cast int to string + seen = str(sent.item()) + '_' + str(unfin_idx.item()) + if seen not in sents_seen: + sents_seen[seen] = None + + if self.match_source_len and step > src_lengths[unfin_idx]: + score = torch.tensor(-math.inf).to(score) + + # An input sentence (among those in a batch) is finished when + # beam_size hypotheses have been collected for it + if len(finalized[sent]) < beam_size: + if attn_clone is not None: + # remove padding tokens from attn scores + hypo_attn = attn_clone[i] + else: + hypo_attn = torch.empty(0) + + finalized[sent].append({ + 'tokens': + tokens_clone[i], + 'score': + score, + 'attention': + hypo_attn, # src_len x tgt_len + 'alignment': + torch.empty(0), + 'positional_scores': + pos_scores[i], + 'decoder_out': + decoder_out_clone[i] if decoder_out is not None else [], + }) + + newly_finished: List[int] = [] + + for seen in sents_seen.keys(): + # check termination conditions for this sentence + sent: int = int(float(seen.split('_')[0])) + unfin_idx: int = int(float(seen.split('_')[1])) + + if not finished[sent] and self.is_finished( + step, unfin_idx, max_len, len(finalized[sent]), beam_size): + finished[sent] = True + newly_finished.append(unfin_idx) + + return newly_finished + + def is_finished( + self, + step: int, + unfin_idx: int, + max_len: int, + finalized_sent_len: int, + beam_size: int, + ): + """ + Check whether decoding for a sentence is finished, which + occurs when the list of finalized sentences has reached the + beam size, or when we reach the maximum length. + """ + assert finalized_sent_len <= beam_size + if finalized_sent_len == beam_size or step == max_len: + return True + return False diff --git a/runtime/Lib/site_packages/models/nlp/chatglm/__init__.py b/runtime/Lib/site_packages/models/nlp/chatglm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a0a073fe3b84017c85b687efb755cdbf08d253c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm/__init__.py @@ -0,0 +1,46 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import ChatGLMConfig + from .tokenization import ChatGLMTokenizer + from .text_generation import ChatGLMForConditionalGeneration + from .quantization import ( + quantize, ) + +else: + _import_structure = { + 'configuration': ['ChatGLMConfig'], + 'text_generation': ['ChatGLMForConditionalGeneration'], + 'quantization': ['quantize'], + 'tokenization': [ + 'ChatGLMTokenizer', + ], + } + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__) diff --git a/runtime/Lib/site_packages/models/nlp/chatglm/configuration.py b/runtime/Lib/site_packages/models/nlp/chatglm/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..5ecf3484b6724325305e46d2e1acbd1414d27bd2 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm/configuration.py @@ -0,0 +1,102 @@ +""" ChatGLM model configuration """ + +from transformers.configuration_utils import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class ChatGLMConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`~ChatGLMModel`]. + It is used to instantiate an ChatGLM model according to the specified arguments, defining the model + architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of + the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used + to control the model outputs. Read the documentation from [`PretrainedConfig`] + for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 150528): + Vocabulary size of the ChatGLM-6B model. + Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`~ChatGLMModel`] or + [`~TFChatGLMModel`]. + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 28): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + inner_hidden_size (`int`, *optional*, defaults to 16384): + Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + max_sequence_length (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. + Typically set this to something large just in case (e.g., 512 or 1024 or 2048). + layernorm_epsilon (`float`, *optional*, defaults to 1e-5): + The epsilon used by the layer normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether the model should return the last key/values attentions (not used by all models). + Example: + + ```python + >>> from modelscope.models.nlp.chatglm.configuration import ChatGLMConfig + >>> from modelscope.models.nlp.chatglm.text_generation import ChatGLMModel + + >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration + >>> configuration = ChatGLMConfig() + + >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration + >>> model = ChatGLMModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` +""" + model_type = 'chatglm' + + def __init__(self, + vocab_size=150528, + hidden_size=4096, + num_layers=28, + num_attention_heads=32, + layernorm_epsilon=1e-5, + use_cache=False, + bos_token_id=150004, + eos_token_id=150005, + mask_token_id=150000, + gmask_token_id=150001, + pad_token_id=0, + max_sequence_length=2048, + inner_hidden_size=16384, + position_encoding_2d=True, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + **kwargs): + self.num_layers = num_layers + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.max_sequence_length = max_sequence_length + self.layernorm_epsilon = layernorm_epsilon + self.inner_hidden_size = inner_hidden_size + self.use_cache = use_cache + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.mask_token_id = mask_token_id + self.gmask_token_id = gmask_token_id + self.position_encoding_2d = position_encoding_2d + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + **kwargs) diff --git a/runtime/Lib/site_packages/models/nlp/chatglm/quantization.py b/runtime/Lib/site_packages/models/nlp/chatglm/quantization.py new file mode 100644 index 0000000000000000000000000000000000000000..4e568c71076d627cb4bececbe13bca31f427d69a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm/quantization.py @@ -0,0 +1,235 @@ +import base64 +import bz2 +import ctypes +from typing import List + +import torch +from torch.nn import Linear +from torch.nn.parameter import Parameter + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + +try: + from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up + + class Kernel: + + def __init__(self, code: bytes, function_names: List[str]): + self.code = code + self._function_names = function_names + self._cmodule = LazyKernelCModule(self.code) + + for name in self._function_names: + setattr(self, name, KernelFunction(self._cmodule, name)) + + quantization_code = '$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ' # noqa + + kernels = Kernel( + bz2.decompress(base64.b64decode(quantization_code)), + [ + 'int4WeightCompression', + 'int4WeightExtractionFloat', + 'int4WeightExtractionHalf', + 'int8WeightExtractionFloat', + 'int8WeightExtractionHalf', + ], + ) +except Exception as exception: + kernels = None + logger.warning('Failed to load cpm_kernels:' + str(exception)) + + +class W8A16Linear(torch.autograd.Function): + + @staticmethod + def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, + scale_w: torch.Tensor, weight_bit_width): + ctx.inp_shape = inp.size() + ctx.weight_bit_width = weight_bit_width + out_features = quant_w.size(0) + inp = inp.contiguous().view(-1, inp.size(-1)) + weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width) + ctx.weight_shape = weight.size() + output = inp.mm(weight.t()) + ctx.save_for_backward(inp, quant_w, scale_w) + return output.view(*(ctx.inp_shape[:-1] + (out_features, ))) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + inp, quant_w, scale_w = ctx.saved_tensors + weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width) + grad_output = grad_output.contiguous().view(-1, weight.size(0)) + grad_input = grad_output.mm(weight) + grad_weight = grad_output.t().mm(inp) + return grad_input.view(ctx.inp_shape), grad_weight.view( + ctx.weight_shape), None, None + + +def compress_int4_weight(weight: torch.Tensor): # (n, m) + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + assert m % 2 == 0 + m = m // 2 + out = torch.empty(n, m, dtype=torch.int8, device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + if kernels is None: + raise RuntimeError( + 'kernels is None, please check whether it is correctly initialized.' + ) + kernels.int4WeightCompression( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m) + ], + ) + return out + + +def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, + source_bit_width: int): + if kernels is None: + raise RuntimeError( + 'kernels is None, please check whether it is correctly initialized.' + ) + if source_bit_width == 8: + func = kernels.int8WeightExtractionHalf + elif source_bit_width == 4: + func = kernels.int4WeightExtractionHalf + else: + assert False, 'Unsupported bit-width' + + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + out = torch.empty( + n, m * (8 // source_bit_width), dtype=torch.half, device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + func( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(scale_list.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m), + ], + ) + return out + + +class QuantizedLinear(Linear): + + def __init__(self, + weight_bit_width: int, + weight_tensor=None, + bias_tensor=None, + empty_init=False, + *args, + **kwargs): + super(QuantizedLinear, self).__init__(*args, **kwargs) + self.weight_bit_width = weight_bit_width + + shape = self.weight.shape + del self.weight + + if weight_tensor is None or empty_init: + self.weight = torch.empty( + shape[0], + shape[1] * weight_bit_width // 8, + dtype=torch.int8, + device=kwargs['device']) + self.weight_scale = torch.empty( + shape[0], dtype=kwargs['dtype'], device=kwargs['device']) + else: + self.weight_scale = (weight_tensor.abs().max(dim=-1).values / ( + (2**(weight_bit_width - 1)) - 1)).half() # noqa + self.weight = torch.round( + weight_tensor / self.weight_scale[:, None]).to(torch.int8) + if weight_bit_width == 4: + self.weight = compress_int4_weight(self.weight) + + self.weight = Parameter( + self.weight.to(kwargs['device']), requires_grad=False) + self.weight_scale = Parameter( + self.weight_scale.to(kwargs['device']), requires_grad=False) + if bias_tensor is not None: + self.bias = Parameter( + bias_tensor.to(kwargs['device']), requires_grad=False) + else: + self.bias = None + + def forward(self, input): + output = W8A16Linear.apply(input, self.weight, self.weight_scale, + self.weight_bit_width) + if self.bias is not None: + output = output + self.bias + return output + + +def quantize(model, weight_bit_width, empty_init=False, **kwargs): + """Replace fp16 linear with quantized linear""" + + for layer in model.layers: + layer.attention.query_key_value = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight_tensor=layer.attention.query_key_value.weight.to( + torch.cuda.current_device()), + bias_tensor=layer.attention.query_key_value.bias, + in_features=layer.attention.query_key_value.in_features, + out_features=layer.attention.query_key_value.out_features, + bias=True, + dtype=torch.half, + device=layer.attention.query_key_value.weight.device, + empty_init=empty_init) + layer.attention.dense = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight_tensor=layer.attention.dense.weight.to( + torch.cuda.current_device()), + bias_tensor=layer.attention.dense.bias, + in_features=layer.attention.dense.in_features, + out_features=layer.attention.dense.out_features, + bias=True, + dtype=torch.half, + device=layer.attention.dense.weight.device, + empty_init=empty_init) + layer.mlp.dense_h_to_4h = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight_tensor=layer.mlp.dense_h_to_4h.weight.to( + torch.cuda.current_device()), + bias_tensor=layer.mlp.dense_h_to_4h.bias, + in_features=layer.mlp.dense_h_to_4h.in_features, + out_features=layer.mlp.dense_h_to_4h.out_features, + bias=True, + dtype=torch.half, + device=layer.mlp.dense_h_to_4h.weight.device, + empty_init=empty_init) + layer.mlp.dense_4h_to_h = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight_tensor=layer.mlp.dense_4h_to_h.weight.to( + torch.cuda.current_device()), + bias_tensor=layer.mlp.dense_4h_to_h.bias, + in_features=layer.mlp.dense_4h_to_h.in_features, + out_features=layer.mlp.dense_4h_to_h.out_features, + bias=True, + dtype=torch.half, + device=layer.mlp.dense_4h_to_h.weight.device, + empty_init=empty_init) + return model diff --git a/runtime/Lib/site_packages/models/nlp/chatglm/text_generation.py b/runtime/Lib/site_packages/models/nlp/chatglm/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..95ea33db1f10ed58eb2cc7e6544b1f435fc24e66 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm/text_generation.py @@ -0,0 +1,1572 @@ +""" PyTorch ChatGLM model. """ + +import copy +import math +import os +import re +import sys +import warnings +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss, LayerNorm +from torch.nn.utils import skip_init +from transformers.generation.logits_process import LogitsProcessor +from transformers.generation.utils import (GenerationConfig, + LogitsProcessorList, ModelOutput, + StoppingCriteriaList) +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithPast) +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import (add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward) + +from modelscope.metainfo import Models +from modelscope.models import MODELS, Model, TorchModel +from modelscope.outputs import OutputKeys +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .configuration import ChatGLMConfig +from .tokenization import ChatGLMTokenizer + +# flags required to enable jit fusion kernels + +if sys.platform != 'darwin': + torch._C._jit_set_profiling_mode(False) + torch._C._jit_set_profiling_executor(False) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + +logger = logging.get_logger() + +_CHECKPOINT_FOR_DOC = 'THUDM/ChatGLM-6B' +_CONFIG_FOR_DOC = 'ChatGLM6BConfig' + +CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [ + 'THUDM/chatglm-6b', + # See all ChatGLM-6B models at https://huggingface.co/models?filter=chatglm +] + + +class InvalidScoreLogitsProcessor(LogitsProcessor): + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + if torch.isnan(scores).any() or torch.isinf(scores).any(): + scores.zero_() + scores[..., 5] = 5e4 + return scores + + +def load_tf_weights_in_chatglm_6b(model, config, tf_checkpoint_path): + """Load tf checkpoints in a pytorch model.""" + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + 'Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see ' + 'https://www.tensorflow.org/install/ for installation instructions.' + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info(f'Converting TensorFlow checkpoint from {tf_path}') + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info(f'Loading TF weight {name} with shape {shape}') + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in [ + 'adam_v', 'adam_m', 'AdamWeightDecayOptimizer', + 'AdamWeightDecayOptimizer_1', 'global_step' + ] for n in name): + logger.info(f"Skipping {'/'.join(name)}") + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + scope_names = re.split(r'_(\d+)', m_name) + else: + scope_names = [m_name] + if scope_names[0] == 'kernel' or scope_names[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'output_bias' or scope_names[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif scope_names[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif scope_names[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info(f"Skipping {'/'.join(name)}") + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f'Pointer shape {pointer.shape} and array shape {array.shape} mismatched' + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info(f'Initialize PyTorch weight {name}') + pointer.data = torch.from_numpy(array) + return model + + +class PrefixEncoder(torch.nn.Module): + """ + The torch.nn model to encode the prefix + Input shape: (batch-size, prefix-length) + Output shape: (batch-size, prefix-length, 2*layers*hidden) + """ + + def __init__(self, config): + super().__init__() + self.prefix_projection = config.prefix_projection + if self.prefix_projection: + # Use a two-layer MLP to encode the prefix + self.embedding = torch.nn.Embedding(config.pre_seq_len, + config.hidden_size) + self.trans = torch.nn.Sequential( + torch.nn.Linear(config.hidden_size, config.hidden_size), + torch.nn.Tanh(), + torch.nn.Linear(config.hidden_size, + config.num_layers * config.hidden_size * 2)) + else: + self.embedding = torch.nn.Embedding( + config.pre_seq_len, config.num_layers * config.hidden_size * 2) + + def forward(self, prefix: torch.Tensor): + if self.prefix_projection: + prefix_tokens = self.embedding(prefix) + past_key_values = self.trans(prefix_tokens) + else: + past_key_values = self.embedding(prefix) + return past_key_values + + +@torch.jit.script +def gelu_impl(x): + """OpenAI's gelu implementation.""" + return 0.5 * x * ( + 1.0 + torch.tanh(0.7978845608028654 * x * # noqa + (1.0 + 0.044715 * x * x))) # noqa + + +def gelu(x): + return gelu_impl(x) + + +class RotaryEmbedding(torch.nn.Module): + + def __init__(self, dim, base=10000, precision=torch.half, learnable=False): + super().__init__() + inv_freq = 1. / (base**(torch.arange(0, dim, 2).float() / dim)) + inv_freq = inv_freq.half() + self.learnable = learnable + if learnable: + self.inv_freq = torch.nn.Parameter(inv_freq) + self.max_seq_len_cached = None + else: + self.register_buffer('inv_freq', inv_freq) + self.max_seq_len_cached = None + self.cos_cached = None + self.sin_cached = None + self.precision = precision + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + pass + + def forward(self, x, seq_dim=1, seq_len=None): + if seq_len is None: + seq_len = x.shape[seq_dim] + if self.max_seq_len_cached is None or ( + seq_len > self.max_seq_len_cached): # noqa + self.max_seq_len_cached = None if self.learnable else seq_len + t = torch.arange( + seq_len, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum('i,j->ij', t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + if self.precision == torch.bfloat16: + emb = emb.float() + + # [sx, 1 (b * np), hn] + cos_cached = emb.cos()[:, None, :] + sin_cached = emb.sin()[:, None, :] + if self.precision == torch.bfloat16: + cos_cached = cos_cached.bfloat16() + sin_cached = sin_cached.bfloat16() + if self.learnable: + return cos_cached, sin_cached + self.cos_cached, self.sin_cached = cos_cached, sin_cached + return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...] + + def _apply(self, fn): + if self.cos_cached is not None: + self.cos_cached = fn(self.cos_cached) + if self.sin_cached is not None: + self.sin_cached = fn(self.sin_cached) + return super()._apply(fn) + + +def rotate_half(x): + x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:] + return torch.cat( + (-x2, x1), + dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions + + +@torch.jit.script +def apply_rotary_pos_emb_index(q, k, cos, sin, position_id): + # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn] + cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \ + F.embedding(position_id, sin.squeeze(1)).unsqueeze(2) + q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + ( + rotate_half(k) * sin) + return q, k + + +def attention_fn( + self, + query_layer, + key_layer, + value_layer, + attention_mask, + hidden_size_per_partition, + layer_id, + layer_past=None, + scaling_attention_score=True, + use_cache=False, +): + if layer_past is not None: + past_key, past_value = layer_past[0], layer_past[1] + key_layer = torch.cat((past_key, key_layer), dim=0) + value_layer = torch.cat((past_value, value_layer), dim=0) + + # seqlen, batch, num_attention_heads, hidden_size_per_attention_head + seq_len, b, nh, hidden_size = key_layer.shape + + if use_cache: + present = (key_layer, value_layer) + else: + present = None + + query_key_layer_scaling_coeff = float(layer_id + 1) + if scaling_attention_score: + query_layer = query_layer / ( + math.sqrt(hidden_size) * query_key_layer_scaling_coeff) + + # =================================== + # Raw attention scores. [b, np, s, s] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view(output_size[2], + output_size[0] * output_size[1], -1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], + -1) + + matmul_result = torch.zeros( + 1, + 1, + 1, + dtype=query_layer.dtype, + device=query_layer.device, + ) + + matmul_result = torch.baddbmm( + matmul_result, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=1.0, + ) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + if self.scale_mask_softmax: + self.scale_mask_softmax.scale = query_key_layer_scaling_coeff + attention_probs = self.scale_mask_softmax(attention_scores, + attention_mask.contiguous()) + else: + if not (attention_mask == 0).all(): + # if auto-regressive, skip + attention_scores.masked_fill_(attention_mask, -10000.0) + dtype = attention_scores.dtype + attention_scores = attention_scores.float() + attention_scores = attention_scores * query_key_layer_scaling_coeff + + attention_probs = F.softmax(attention_scores, dim=-1) + + attention_probs = attention_probs.type(dtype) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + + # change view [sk, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], + output_size[2], -1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + ( + hidden_size_per_partition, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, present, attention_probs) + + return outputs + + +class SelfAttention(torch.nn.Module): + + def __init__(self, + hidden_size, + num_attention_heads, + layer_id, + hidden_size_per_attention_head=None, + bias=True, + params_dtype=torch.float, + position_encoding_2d=True): + super(SelfAttention, self).__init__() + + self.layer_id = layer_id + self.hidden_size = hidden_size + self.hidden_size_per_partition = hidden_size + self.num_attention_heads = num_attention_heads + self.num_attention_heads_per_partition = num_attention_heads + self.position_encoding_2d = position_encoding_2d + self.rotary_emb = RotaryEmbedding( # noqa + self.hidden_size // # noqa + (self.num_attention_heads * 2) if position_encoding_2d else # noqa + self.hidden_size // self.num_attention_heads, # noqa + base=10000, # noqa + precision=torch.half, # noqa + learnable=False, # noqa + ) # noqa + + self.scale_mask_softmax = None + + if hidden_size_per_attention_head is None: + self.hidden_size_per_attention_head = hidden_size // num_attention_heads + else: + self.hidden_size_per_attention_head = hidden_size_per_attention_head + + self.inner_hidden_size = num_attention_heads * self.hidden_size_per_attention_head + + # Strided linear layer. + self.query_key_value = skip_init( + torch.nn.Linear, + hidden_size, + 3 * self.inner_hidden_size, + bias=bias, + dtype=params_dtype, + ) + + self.dense = skip_init( + torch.nn.Linear, + self.inner_hidden_size, + hidden_size, + bias=bias, + dtype=params_dtype, + ) + + @staticmethod + def attention_mask_func(attention_scores, attention_mask): + attention_scores.masked_fill_(attention_mask, -10000.0) + return attention_scores + + def split_tensor_along_last_dim(self, + tensor, + num_partitions, + contiguous_split_chunks=False): + """Split a tensor along its last dimension. + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = tensor.size()[last_dim] // num_partitions + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + def forward( + self, + hidden_states: torch.Tensor, + position_ids, + attention_mask: torch.Tensor, + layer_id, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + """ + hidden_states: [seq_len, batch, hidden_size] + attention_mask: [(1, 1), seq_len, seq_len] + """ + + # [seq_len, batch, 3 * hidden_size] + mixed_raw_layer = self.query_key_value(hidden_states) + + # [seq_len, batch, 3 * hidden_size] --> [seq_len, batch, num_attention_heads, + # 3 * hidden_size_per_attention_head] + new_tensor_shape = mixed_raw_layer.size()[:-1] + ( + self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head, + ) + mixed_raw_layer = mixed_raw_layer.view(*new_tensor_shape) + + # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head] + (query_layer, key_layer, + value_layer) = self.split_tensor_along_last_dim(mixed_raw_layer, 3) + + if self.position_encoding_2d: + q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1)) + k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1)) + cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1) + position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \ + position_ids[:, 1, :].transpose(0, 1).contiguous() + q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids) + q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, + block_position_ids) + query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1)) + key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1)) + else: + position_ids = position_ids.transpose(0, 1) + cos, sin = self.rotary_emb( + value_layer, seq_len=position_ids.max() + 1) + # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head] + query_layer, key_layer = apply_rotary_pos_emb_index( + query_layer, key_layer, cos, sin, position_ids) + + # [seq_len, batch, hidden_size] + context_layer, present, attention_probs = attention_fn( + self=self, + query_layer=query_layer, + key_layer=key_layer, + value_layer=value_layer, + attention_mask=attention_mask, + hidden_size_per_partition=self.hidden_size_per_partition, + layer_id=layer_id, + layer_past=layer_past, + use_cache=use_cache) + + output = self.dense(context_layer) + + outputs = (output, present) + + if output_attentions: + outputs += (attention_probs, ) + + return outputs # output, present, attention_probs + + +class GEGLU(torch.nn.Module): + + def __init__(self): + super().__init__() + self.activation_fn = F.gelu + + def forward(self, x): + # dim=-1 breaks in jit for pt<1.10 + x1, x2 = x.chunk(2, dim=(x.ndim - 1)) + return x1 * self.activation_fn(x2) + + +class GLU(torch.nn.Module): + + def __init__(self, + hidden_size, + inner_hidden_size=None, + layer_id=None, + bias=True, + activation_func=gelu, + params_dtype=torch.float): + super(GLU, self).__init__() + self.layer_id = layer_id + self.activation_func = activation_func + + # Project to 4h. + self.hidden_size = hidden_size + if inner_hidden_size is None: + inner_hidden_size = 4 * hidden_size + self.inner_hidden_size = inner_hidden_size + self.dense_h_to_4h = skip_init( + torch.nn.Linear, + self.hidden_size, + self.inner_hidden_size, + bias=bias, + dtype=params_dtype, + ) + # Project back to h. + self.dense_4h_to_h = skip_init( + torch.nn.Linear, + self.inner_hidden_size, + self.hidden_size, + bias=bias, + dtype=params_dtype, + ) + + def forward(self, hidden_states): + """ + hidden_states: [seq_len, batch, hidden_size] + """ + + # [seq_len, batch, inner_hidden_size] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + + intermediate_parallel = self.activation_func(intermediate_parallel) + + output = self.dense_4h_to_h(intermediate_parallel) + + return output + + +class GLMBlock(torch.nn.Module): + + def __init__(self, + hidden_size, + num_attention_heads, + layernorm_epsilon, + layer_id, + inner_hidden_size=None, + hidden_size_per_attention_head=None, + layernorm=LayerNorm, + use_bias=True, + params_dtype=torch.float, + num_layers=28, + position_encoding_2d=True): + super(GLMBlock, self).__init__() + # Set output layer initialization if not provided. + + self.layer_id = layer_id + + # Layernorm on the input data. + self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon) + + self.position_encoding_2d = position_encoding_2d + + # Self attention. + self.attention = SelfAttention( + hidden_size, + num_attention_heads, + layer_id, + hidden_size_per_attention_head=hidden_size_per_attention_head, + bias=use_bias, + params_dtype=params_dtype, + position_encoding_2d=self.position_encoding_2d) + + # Layernorm on the input data. + self.post_attention_layernorm = layernorm( + hidden_size, eps=layernorm_epsilon) + + self.num_layers = num_layers + + # GLU + self.mlp = GLU( + hidden_size, + inner_hidden_size=inner_hidden_size, + bias=use_bias, + layer_id=layer_id, + params_dtype=params_dtype, + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_ids, + attention_mask: torch.Tensor, + layer_id, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + """ + hidden_states: [seq_len, batch, hidden_size] + attention_mask: [(1, 1), seq_len, seq_len] + """ + + # Layer norm at the begining of the transformer layer. + # [seq_len, batch, hidden_size] + attention_input = self.input_layernorm(hidden_states) + + # Self attention. + attention_outputs = self.attention( + attention_input, + position_ids, + attention_mask=attention_mask, + layer_id=layer_id, + layer_past=layer_past, + use_cache=use_cache, + output_attentions=output_attentions) + + attention_output = attention_outputs[0] + + outputs = attention_outputs[1:] + + # Residual connection. + alpha = (2 * self.num_layers)**0.5 + hidden_states = attention_input * alpha + attention_output + + mlp_input = self.post_attention_layernorm(hidden_states) + + # MLP. + mlp_output = self.mlp(mlp_input) + + # Second residual connection. + output = mlp_input * alpha + mlp_output + + if use_cache: + outputs = (output, ) + outputs + else: + outputs = (output, ) + outputs[1:] + + return outputs # hidden_states, present, attentions + + +class ChatGLMPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and + a simple interface for downloading and loading pretrained models. + """ + + is_parallelizable = False + supports_gradient_checkpointing = True + config_class = ChatGLMConfig + base_model_prefix = 'transformer' + _no_split_modules = ['GLMBlock'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module: nn.Module): + """Initialize the weights.""" + return + + def get_masks(self, input_ids, device): + batch_size, seq_length = input_ids.shape + context_lengths = [ + seq.tolist().index(self.config.bos_token_id) for seq in input_ids + ] + attention_mask = torch.ones((batch_size, seq_length, seq_length), + device=device) + attention_mask.tril_() + for i, context_length in enumerate(context_lengths): + attention_mask[i, :, :context_length] = 1 + attention_mask.unsqueeze_(1) + attention_mask = (attention_mask < 0.5).bool() + + return attention_mask + + def get_position_ids(self, input_ids, mask_positions, device, gmask=False): + batch_size, seq_length = input_ids.shape + context_lengths = [ + seq.tolist().index(self.config.bos_token_id) for seq in input_ids + ] + if self.position_encoding_2d: + position_ids = torch.arange( + seq_length, dtype=torch.long, + device=device).unsqueeze(0).repeat(batch_size, 1) + for i, context_length in enumerate(context_lengths): + position_ids[i, context_length:] = mask_positions[i] + block_position_ids = [ + torch.cat(( + torch.zeros( # noqa + context_length, + dtype=torch.long, + device=device), # noqa + torch.arange( # noqa + seq_length - context_length, # noqa + dtype=torch.long, # noqa + device=device) + 1)) # noqa + for context_length in context_lengths + ] + block_position_ids = torch.stack(block_position_ids, dim=0) + position_ids = torch.stack((position_ids, block_position_ids), + dim=1) + else: + position_ids = torch.arange( + seq_length, dtype=torch.long, + device=device).unsqueeze(0).repeat(batch_size, 1) + if not gmask: + for i, context_length in enumerate(context_lengths): + position_ids[context_length:] = mask_positions[i] + + return position_ids + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, ChatGLMModel): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + kwargs.pop('cfg', None) + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model + + +CHATGLM_6B_START_DOCSTRING = r""" + This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general + usage and behavior. + + Parameters: + config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +CHATGLM_6B_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `({0})`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`ChatGLM6BTokenizer`]. + See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range `[0, config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert *input_ids* indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + 'The bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.', + CHATGLM_6B_START_DOCSTRING, +) +class ChatGLMModel(ChatGLMPreTrainedModel): + """ + + The model can behave as an encoder (with only self-attention) as well + as a decoder, in which case a layer of cross-attention is added between + the self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, + Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the + `is_decoder` argument of the configuration set to `True`. + To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` + argument and `add_cross_attention` set to `True`; an + `encoder_hidden_states` is then expected as an input to the forward pass. + """ + + def __init__(self, config: ChatGLMConfig): + super().__init__(config) + + # recording parameters + self.max_sequence_length = config.max_sequence_length + self.hidden_size = config.hidden_size + self.params_dtype = torch.half + self.num_attention_heads = config.num_attention_heads + self.vocab_size = config.vocab_size + self.num_layers = config.num_layers + self.layernorm_epsilon = config.layernorm_epsilon + self.inner_hidden_size = config.inner_hidden_size + self.hidden_size_per_attention_head = self.hidden_size // self.num_attention_heads + self.position_encoding_2d = config.position_encoding_2d + self.pre_seq_len = config.pre_seq_len + self.prefix_projection = config.prefix_projection + + self.word_embeddings = skip_init( + torch.nn.Embedding, + num_embeddings=self.vocab_size, + embedding_dim=self.hidden_size, + dtype=self.params_dtype) + self.gradient_checkpointing = False + + def get_layer(layer_id): + return GLMBlock( + self.hidden_size, + self.num_attention_heads, + self.layernorm_epsilon, + layer_id, + inner_hidden_size=self.inner_hidden_size, + hidden_size_per_attention_head=self. + hidden_size_per_attention_head, + layernorm=LayerNorm, + use_bias=True, + params_dtype=self.params_dtype, + position_encoding_2d=self.position_encoding_2d, + ) + + self.layers = torch.nn.ModuleList( + [get_layer(layer_id) for layer_id in range(self.num_layers)]) + + # Final layer norm before output. + self.final_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon) + + if self.pre_seq_len is not None: + for param in self.parameters(): + param.requires_grad = False + self.prefix_tokens = torch.arange(self.pre_seq_len).long() + self.prefix_encoder = PrefixEncoder(config) + self.dropout = torch.nn.Dropout(0.1) + + # total_params = sum(p.numel() for p in self.parameters()) + # trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad) + # print("Using p-tuning v2: # trainable_params = {} / {}".format(trainable_params, total_params)) + + def get_input_embeddings(self): + return self.word_embeddings + + def set_input_embeddings(self, new_embeddings: torch.Tensor): + self.word_embeddings = new_embeddings + + def get_prompt(self, batch_size, device, dtype=torch.half): + prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, + -1).to(device) + past_key_values = self.prefix_encoder(prefix_tokens).type(dtype) + past_key_values = past_key_values.view( + batch_size, self.pre_seq_len, self.num_layers * 2, + self.num_attention_heads, + self.hidden_size // self.num_attention_heads) + # seq_len, b, nh, hidden_size + past_key_values = self.dropout(past_key_values) + past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2) + # past_key_values = [(v[0], v[1]) for v in past_key_values] + return past_key_values + + @add_start_docstrings_to_model_forward( + CHATGLM_6B_INPUTS_DOCSTRING.format('batch_size, sequence_length')) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=BaseModelOutputWithPastAndCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], + ...]] = None, + inputs_embeds: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPast]: + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.gradient_checkpointing and self.training: + if use_cache: + # logger.warning_once( + # "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + # ) + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + batch_size, seq_length = input_ids.shape[:2] + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape[:2] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + if past_key_values is None: + if self.pre_seq_len is not None: + past_key_values = self.get_prompt( + batch_size=input_ids.shape[0], + device=input_ids.device, + dtype=inputs_embeds.dtype) + else: + past_key_values = tuple([None] * len(self.layers)) + + if attention_mask is None: + attention_mask = self.get_masks( + input_ids, device=input_ids.device) + + if position_ids is None: + MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id + mask_token = gMASK if gMASK in input_ids else MASK + use_gmask = True if gMASK in input_ids else False + + mask_positions = [ + seq.tolist().index(mask_token) for seq in input_ids + ] + position_ids = self.get_position_ids( + input_ids, + mask_positions=mask_positions, + device=input_ids.device, + gmask=use_gmask) + + if self.pre_seq_len is not None and attention_mask is not None: + prefix_attention_mask = torch.ones( + batch_size, 1, input_ids.size(-1), + self.pre_seq_len).to(attention_mask.device) + prefix_attention_mask = (prefix_attention_mask < 0.5).bool() + attention_mask = torch.cat((prefix_attention_mask, attention_mask), + dim=3) + + # [seq_len, batch, hidden_size] + hidden_states = inputs_embeds.transpose(0, 1) + + presents = () if use_cache else None + all_self_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + + if attention_mask is None: + attention_mask = torch.zeros(1, 1, device=input_ids.device).bool() + + else: + attention_mask = attention_mask.to(input_ids.device) + + for i, layer in enumerate(self.layers): + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + layer_past = past_key_values[i] + + if self.gradient_checkpointing and self.training: + layer_ret = torch.utils.checkpoint.checkpoint( + layer, hidden_states, position_ids, attention_mask, + torch.tensor(i), layer_past, use_cache, output_attentions) + else: + layer_ret = layer( + hidden_states, + position_ids=position_ids, + attention_mask=attention_mask, + layer_id=torch.tensor(i), + layer_past=layer_past, + use_cache=use_cache, + output_attentions=output_attentions) + + hidden_states = layer_ret[0] + + if use_cache: + presents = presents + (layer_ret[1], ) + + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_ret[2 if use_cache else 1], ) + + # Final layer norm. + hidden_states = self.final_layernorm(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, presents, all_hidden_states, all_self_attentions + ] if v is not None) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +@MODELS.register_module(Tasks.chat, module_name=Models.chatglm_6b) +class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel): + + def __init__(self, config: ChatGLMConfig): + super().__init__(config) + + # self.hidden_size = config.hidden_size + # self.params_dtype = torch.half + # self.vocab_size = config.vocab_size + self.max_sequence_length = config.max_sequence_length + + self.position_encoding_2d = config.position_encoding_2d + + self.transformer = ChatGLMModel(config) + + self.lm_head = skip_init( + nn.Linear, + config.hidden_size, + config.vocab_size, + bias=False, + dtype=torch.half) + + self.config = config + + self.quantized = False + + if self.config.quantization_bit: + self.quantize(self.config.quantization_bit, empty_init=True) + + # loading tokenizer + self.tokenizer = ChatGLMTokenizer.from_pretrained(config.name_or_path) + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def _update_model_kwargs_for_generation( + self, + outputs: ModelOutput, + model_kwargs: Dict[str, Any], + is_encoder_decoder: bool = False, + standardize_cache_format: bool = False, + ) -> Dict[str, Any]: + # update past_key_values + model_kwargs['past_key_values'] = self._extract_past_from_model_output( + outputs, standardize_cache_format=standardize_cache_format) + + # update attention mask + if 'attention_mask' in model_kwargs: + attention_mask = model_kwargs['attention_mask'] + if attention_mask is not None and attention_mask.dtype == torch.bool: + attention_mask = torch.cat([ + attention_mask, + attention_mask.new_ones((*attention_mask.shape[:3], 1)) + ], + dim=3) # noqa + new_attention_mask = attention_mask[:, :, -1:].clone() + new_attention_mask[..., -1] = False + model_kwargs['attention_mask'] = torch.cat( + [attention_mask, new_attention_mask], dim=2) + + # update position ids + if 'position_ids' in model_kwargs: + position_ids = model_kwargs['position_ids'] + new_position_id = position_ids[..., -1:].clone() + new_position_id[:, 1, :] += 1 + model_kwargs['position_ids'] = torch.cat( + [position_ids, new_position_id], dim=-1) + + return model_kwargs + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past: Optional[torch.Tensor] = None, + past_key_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + **kwargs) -> dict: + batch_size, seq_length = input_ids.shape + MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id + mask_token = gMASK if gMASK in input_ids else MASK + use_gmask = True if gMASK in input_ids else False + seqs = input_ids.tolist() + mask_positions = [seq.index(mask_token) for seq in seqs] + + # only last token for input_ids if past is not None + if past is not None or past_key_values is not None: + last_token = input_ids[:, -1].unsqueeze(-1) + if attention_mask is not None and attention_mask.dtype == torch.bool: + attention_mask = attention_mask[:, :, -1:] + else: + attention_mask = None + if position_ids is not None: + position_ids = position_ids[..., -1:] + else: + context_lengths = [ + seq.index(self.config.bos_token_id) for seq in seqs + ] + if self.position_encoding_2d: + position_ids = torch.tensor( + [[mask_position, seq_length - context_length] + for mask_position, context_length in zip( + mask_positions, context_lengths)], + dtype=torch.long, + device=input_ids.device).unsqueeze(-1) + else: + position_ids = torch.tensor( + [mask_position for mask_position in mask_positions], + dtype=torch.long, + device=input_ids.device).unsqueeze(-1) + + if past is None: + past = past_key_values + return { + 'input_ids': last_token, + 'past_key_values': past, + 'position_ids': position_ids, + 'attention_mask': attention_mask + } + else: + if attention_mask is not None and attention_mask.dtype != torch.bool: + # logger.warning_once(f"The dtype of attention mask ({attention_mask.dtype}) is not bool") + attention_mask = None + if attention_mask is None: + attention_mask = self.get_masks( + input_ids, device=input_ids.device) + if position_ids is None: + position_ids = self.get_position_ids( + input_ids, + device=input_ids.device, + mask_positions=mask_positions, + gmask=use_gmask) + + return { + 'input_ids': input_ids, + 'past_key_values': past, + 'position_ids': position_ids, + 'attention_mask': attention_mask + } + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.transformer( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states).permute(1, 0, 2).contiguous() + + loss = None + if labels is not None: + lm_logits = lm_logits.to(torch.float32) + + # Shift so that tokens < n predict n + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss(ignore_index=-100) + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + + lm_logits = lm_logits.to(hidden_states.dtype) + loss = loss.to(hidden_states.dtype) + + if not return_dict: + output = (lm_logits, ) + transformer_outputs[1:] + return ((loss, ) + output) if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + @staticmethod + def _reorder_cache( + past: Tuple[Tuple[torch.Tensor, torch.Tensor], + ...], beam_idx: torch.LongTensor + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]: + """ + This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct + beam_idx at every generation step. + + Output shares the same memory storage as `past`. + """ + return tuple(( + layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)), + layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)), + ) for layer_past in past) + + def process_response(self, response): + response = response.strip() + response = response.replace('[[训练时间]]', '2023年') + punkts = [ + [',', ','], + ['!', '!'], + [':', ':'], + [';', ';'], + ['\?', '?'], # noqa + ] + for item in punkts: + response = re.sub(r'([\u4e00-\u9fff])%s' % item[0], + r'\1%s' % item[1], response) + response = re.sub(r'%s([\u4e00-\u9fff])' % item[0], + r'%s\1' % item[1], response) + return response + + @torch.no_grad() + def _chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None, + max_length: int = 2048, + num_beams=1, + do_sample=True, + top_p=0.7, + temperature=0.95, + logits_processor=None, + **kwargs): + if history is None: + history = [] + if logits_processor is None: + logits_processor = LogitsProcessorList() + logits_processor.append(InvalidScoreLogitsProcessor()) + gen_kwargs = { + 'max_length': max_length, + 'num_beams': num_beams, + 'do_sample': do_sample, + 'top_p': top_p, + 'temperature': temperature, + 'logits_processor': logits_processor, + **kwargs + } + if not history: + prompt = query + else: + prompt = '' + for i, (old_query, response) in enumerate(history): + prompt += '[Round {}]\n问:{}\n答:{}\n'.format( + i, old_query, response) + prompt += '[Round {}]\n问:{}\n答:'.format(len(history), query) + inputs = tokenizer([prompt], return_tensors='pt') + inputs = inputs.to(self.device) + outputs = self.generate(**inputs, **gen_kwargs) + outputs = outputs.tolist()[0][len(inputs['input_ids'][0]):] + response = tokenizer.decode(outputs) + response = self.process_response(response) + history = history + [(query, response)] + return response, history + + @torch.no_grad() + def stream_chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None, + max_length: int = 2048, + do_sample=True, + top_p=0.7, + temperature=0.95, + logits_processor=None, + **kwargs): + if history is None: + history = [] + if logits_processor is None: + logits_processor = LogitsProcessorList() + logits_processor.append(InvalidScoreLogitsProcessor()) + gen_kwargs = { + 'max_length': max_length, + 'do_sample': do_sample, + 'top_p': top_p, + 'temperature': temperature, + 'logits_processor': logits_processor, + **kwargs + } + if not history: + prompt = query + else: + prompt = '' + for i, (old_query, response) in enumerate(history): + prompt += '[Round {}]\n问:{}\n答:{}\n'.format( + i, old_query, response) + prompt += '[Round {}]\n问:{}\n答:'.format(len(history), query) + inputs = tokenizer([prompt], return_tensors='pt') + inputs = inputs.to(self.device) + for outputs in self.stream_generate(**inputs, **gen_kwargs): + outputs = outputs.tolist()[0][len(inputs['input_ids'][0]):] + response = tokenizer.decode(outputs) + response = self.process_response(response) + new_history = history + [(query, response)] + yield response, new_history + + @torch.no_grad() + def stream_generate( + self, + input_ids, + generation_config: Optional[GenerationConfig] = None, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], + List[int]]] = None, + **kwargs, + ): + _, input_ids_seq_length = input_ids.shape[0], input_ids.shape[ + -1] # noqa + + if generation_config is None: + generation_config = self.generation_config + generation_config = copy.deepcopy(generation_config) + model_kwargs = generation_config.update(**kwargs) + _, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id + + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + + has_default_max_length = kwargs.get( + 'max_length') is None and generation_config.max_length is not None + if has_default_max_length and generation_config.max_new_tokens is None: + warnings.warn( + f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. " + 'This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we' + ' recommend using `max_new_tokens` to control the maximum length of the generation.', + UserWarning, + ) + elif generation_config.max_new_tokens is not None: + generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length + if not has_default_max_length: + logger.warn( + f'Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(=' + f'{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. ' + 'Please refer to the documentation for more information. ' + '(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)', + UserWarning, + ) + + if input_ids_seq_length >= generation_config.max_length: + input_ids_string = 'decoder_input_ids' if self.config.is_encoder_decoder else 'input_ids' + logger.warning( + f'Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to' + f' {generation_config.max_length}. This can lead to unexpected behavior. You should consider' + ' increasing `max_new_tokens`.') + + # 2. Set generation parameters if not already defined + logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList( + ) + stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList( + ) + + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_seq_length, + encoder_input_ids=input_ids, + prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, + logits_processor=logits_processor, + ) + + stopping_criteria = self._get_stopping_criteria( + generation_config=generation_config, + stopping_criteria=stopping_criteria) + logits_warper = self._get_logits_warper(generation_config) + + unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1) + scores = None + while True: + model_inputs = self.prepare_inputs_for_generation( + input_ids, **model_kwargs) + # forward pass to get next token + outputs = self( + **model_inputs, + return_dict=True, + output_attentions=False, + output_hidden_states=False, + ) + + next_token_logits = outputs.logits[:, -1, :] + + # pre-process distribution + next_token_scores = logits_processor(input_ids, next_token_logits) + next_token_scores = logits_warper(input_ids, next_token_scores) + + # sample + probs = nn.functional.softmax(next_token_scores, dim=-1) + if generation_config.do_sample: + next_tokens = torch.multinomial( + probs, num_samples=1).squeeze(1) + else: + next_tokens = torch.argmax(probs, dim=-1) + + # update generated ids, model inputs, and length for next step + input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + model_kwargs = self._update_model_kwargs_for_generation( + outputs, + model_kwargs, + is_encoder_decoder=self.config.is_encoder_decoder) + unfinished_sequences = unfinished_sequences.mul( + (sum(next_tokens != i for i in eos_token_id)).long()) + + # stop when each sentence is finished, or if we exceed the maximum length + if unfinished_sequences.max() == 0 or stopping_criteria( + input_ids, scores): + break + yield input_ids + + def quantize(self, bits: int, empty_init=False, **kwargs): + if bits == 0: + return + + from .quantization import quantize + + if self.quantized: + logger.info('Already quantized.') + return self + + self.quantized = True + + self.config.quantization_bit = bits + + self.transformer = quantize( + self.transformer, bits, empty_init=empty_init, **kwargs) + return self + + def chat(self, input: Dict) -> Dict: + text = input['text'] + history = input['history'] + # args + if 'max_length' in input: + max_length = input['max_length'] + else: + max_length = 2048 + + if 'temperature' in input: + temperature = input['temperature'] + else: + temperature = 0.95 + + if 'num_beams' in input: + num_beams = input['num_beams'] + else: + num_beams = 1 + + if 'do_sample' in input: + do_sample = input['do_sample'] + else: + do_sample = True + + if type(history) == torch.Tensor: + history = history.tolist() + response, history = self._chat( + self.tokenizer, + text, + history, + max_length=max_length, + temperature=temperature, + num_beams=num_beams, + do_sample=do_sample) + logger.info('Generation finished.') + return {OutputKeys.RESPONSE: response, OutputKeys.HISTORY: history} diff --git a/runtime/Lib/site_packages/models/nlp/chatglm/tokenization.py b/runtime/Lib/site_packages/models/nlp/chatglm/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..6ce1b90d5fe96049f62b156a15a75f0182994734 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm/tokenization.py @@ -0,0 +1,466 @@ +"""Tokenization classes for ChatGLM.""" +import os +from typing import Dict, List, Optional, Union + +import numpy as np +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer +from transformers.tokenization_utils_base import BatchEncoding, EncodedInput +from transformers.utils import PaddingStrategy + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'THUDM/chatglm-6b': 2048, +} + + +class TextTokenizer: + + def __init__(self, model_path): + self.sp = spm.SentencePieceProcessor() + self.sp.Load(model_path) + self.num_tokens = self.sp.vocab_size() + + def encode(self, text): + return self.sp.EncodeAsIds(text) + + def decode(self, ids: List[int]): + return self.sp.DecodeIds(ids) + + def tokenize(self, text): + return self.sp.EncodeAsPieces(text) + + def convert_tokens_to_ids(self, tokens): + return [self.sp.PieceToId(token) for token in tokens] + + def convert_token_to_id(self, token): + return self.sp.PieceToId(token) + + def convert_id_to_token(self, idx): + return self.sp.IdToPiece(idx) + + def __len__(self): + return self.num_tokens + + +class SPTokenizer: + + def __init__( + self, + vocab_file, + num_image_tokens=20000, + max_blank_length=80, + byte_fallback=True, + ): + assert vocab_file is not None + self.vocab_file = vocab_file + self.num_image_tokens = num_image_tokens + self.special_tokens = [ + '[MASK]', '[gMASK]', '[sMASK]', '', '', '', + '', '' + ] + self.max_blank_length = max_blank_length + self.byte_fallback = byte_fallback + self.text_tokenizer = TextTokenizer(vocab_file) + + def _get_text_tokenizer(self): + return self.text_tokenizer + + @staticmethod + def get_blank_token(length: int): + assert length >= 2 + return f'<|blank_{length}|>' + + @staticmethod + def get_tab_token(): + return '<|tab|>' + + @property + def num_text_tokens(self): + return self.text_tokenizer.num_tokens + + @property + def num_tokens(self): + return self.num_image_tokens + self.num_text_tokens + + @staticmethod + def _encode_whitespaces(text: str, max_len: int = 80): + text = text.replace('\t', SPTokenizer.get_tab_token()) + for i in range(max_len, 1, -1): + text = text.replace(' ' * i, SPTokenizer.get_blank_token(i)) + return text + + def _preprocess(self, text: str, linebreak=True, whitespaces=True): + if linebreak: + text = text.replace('\n', '') + if whitespaces: + text = self._encode_whitespaces( + text, max_len=self.max_blank_length) + return text + + def encode(self, + text: str, + linebreak=True, + whitespaces=True, + add_dummy_prefix=True) -> List[int]: + """ + @param text: Text to encode. + @param linebreak: Whether to encode newline (\n) in text. + @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. + @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. + @param add_dummy_prefix: Whether to add dummy blank space in the beginning. + """ + text = self._preprocess(text, linebreak, whitespaces) + if not add_dummy_prefix: + text = '' + text + tmp = self._get_text_tokenizer().encode(text) + tokens = [x + self.num_image_tokens for x in tmp] + return tokens if add_dummy_prefix else tokens[2:] + + def decode(self, text_ids: List[int]) -> str: + ids = [int(_id) - self.num_image_tokens for _id in text_ids] + ids = [_id for _id in ids if _id >= 0] + text = self._get_text_tokenizer().decode(ids) + text = text.replace('', '\n') + text = text.replace(SPTokenizer.get_tab_token(), '\t') + for i in range(2, self.max_blank_length + 1): + text = text.replace(self.get_blank_token(i), ' ' * i) + return text + + def tokenize(self, + text: str, + linebreak=True, + whitespaces=True, + add_dummy_prefix=True) -> List[str]: + """ + @param text: Text to encode. + @param linebreak: Whether to encode newline (\n) in text. + @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. + @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. + @param add_dummy_prefix: Whether to add dummy blank space in the beginning. + """ + text = self._preprocess(text, linebreak, whitespaces) + if not add_dummy_prefix: + text = '' + text + tokens = self._get_text_tokenizer().tokenize(text) + return tokens if add_dummy_prefix else tokens[2:] + + def __getitem__(self, x: Union[int, str]): + if isinstance(x, int): + if x < self.num_image_tokens: + return ''.format(x) + else: + return self.text_tokenizer.convert_id_to_token( + x - self.num_image_tokens) + elif isinstance(x, str): + if x.startswith('') and x[7:-1].isdigit(): + return int(x[7:-1]) + else: + return self.text_tokenizer.convert_token_to_id( + x) + self.num_image_tokens + else: + raise ValueError('The key should be str or int.') + + +class ChatGLMTokenizer(PreTrainedTokenizer): + """ + Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file: Path to the vocabulary file. + do_lower_case: Use lower case letters. + remove_space: Remove spaces. + bos_token: The bos token + eos_token: The Eos Token + end_token: The end token + mask_token: The mask token + gmask_token: The gmask token + padding_side: The padding side + num_image_tokens: The `num_image_tokens` in `SPTokenizer` + """ + + vocab_files_names = {'vocab_file': 'ice_text.model'} + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ['input_ids', 'attention_mask', 'position_ids'] + + def __init__(self, + vocab_file, + do_lower_case=False, + remove_space=False, + bos_token='', + eos_token='', + end_token='', + mask_token='[MASK]', + gmask_token='[gMASK]', + padding_side='left', + num_image_tokens=20000, + **kwargs) -> None: + + self.sp_tokenizer = SPTokenizer( + vocab_file, num_image_tokens=num_image_tokens) + + super().__init__( + do_lower_case=do_lower_case, + remove_space=remove_space, + padding_side=padding_side, + bos_token=bos_token, + eos_token=eos_token, + end_token=end_token, + mask_token=mask_token, + gmask_token=gmask_token, + num_image_tokens=num_image_tokens, + **kwargs) + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.vocab_file = vocab_file + + self.bos_token = bos_token + self.eos_token = eos_token + self.end_token = end_token + self.mask_token = mask_token + self.gmask_token = gmask_token + """ Initialisation """ + + @property + def gmask_token_id(self) -> Optional[int]: + if self.gmask_token is None: + return None + return self.convert_tokens_to_ids(self.gmask_token) + + @property + def end_token_id(self) -> Optional[int]: + """ + `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been + set. + """ + if self.end_token is None: + return None + return self.convert_tokens_to_ids(self.end_token) + + @property + def vocab_size(self): + """ Returns vocab size """ + return self.sp_tokenizer.num_tokens + + def get_vocab(self): + """ Returns vocab as a dict """ + vocab = { + self._convert_id_to_token(i): i + for i in range(self.vocab_size) + } + vocab.update(self.added_tokens_encoder) + return vocab + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, **kwargs): + """ Returns a tokenized string. """ + text = self.preprocess_text(text) + + seq = self.sp_tokenizer.tokenize(text) + + return seq + + def _decode(self, + token_ids: Union[int, List[int]], + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool = True, + **kwargs) -> str: + if isinstance(token_ids, int): + token_ids = [token_ids] + if len(token_ids) == 0: + return '' + if self.pad_token_id in token_ids: # remove pad + token_ids = list(filter((self.pad_token_id).__ne__, token_ids)) + return self.sp_tokenizer.decode(token_ids) + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.sp_tokenizer[token] + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.sp_tokenizer[index] + + def save_vocabulary(self, save_directory, filename_prefix=None): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if os.path.isdir(save_directory): + vocab_file = os.path.join(save_directory, + self.vocab_files_names['vocab_file']) + else: + vocab_file = save_directory + + with open(self.vocab_file, 'rb') as fin: + proto_str = fin.read() + + with open(vocab_file, 'wb') as writer: + writer.write(proto_str) + + return (vocab_file, ) + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + mask_ids = self.sp_tokenizer[self.mask_token] + gmask_ids = self.sp_tokenizer[self.gmask_token] + eos_id = self.sp_tokenizer[self.eos_token] + if mask_ids not in token_ids_0 and gmask_ids not in token_ids_0: + token_ids_0 += [gmask_ids] + + if token_ids_0[-1] != mask_ids and token_ids_0[-1] != gmask_ids: + token_ids_0 += [self.sp_tokenizer[self.end_token]] + + token_ids_0 += [self.sp_tokenizer[self.bos_token]] + + if token_ids_1 is not None: + if not token_ids_1 or token_ids_1[-1] != eos_id: + token_ids_1 += [eos_id] + token_ids_0 += token_ids_1 + + return token_ids_0 + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: + Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_attention_mask: + (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + bos_token_id = self.sp_tokenizer[self.bos_token] + mask_token_id = self.sp_tokenizer[self.mask_token] + gmask_token_id = self.sp_tokenizer[self.gmask_token] + assert self.padding_side == 'left' + + required_input = encoded_inputs[self.model_input_names[0]] + seq_length = len(required_input) + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + # Initialize attention mask if not present. + if max_length is not None: + if 'attention_mask' not in encoded_inputs: + if bos_token_id in required_input: + context_length = required_input.index(bos_token_id) + else: + context_length = seq_length + attention_mask = np.ones((1, seq_length, seq_length)) + attention_mask = np.tril(attention_mask) + attention_mask[:, :, :context_length] = 1 + attention_mask = np.bool_(attention_mask < 0.5) + encoded_inputs['attention_mask'] = attention_mask + + if 'position_ids' not in encoded_inputs: + position_ids = np.arange(seq_length, dtype=np.int64) + mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id + if mask_token in required_input: + mask_position = required_input.index(mask_token) + position_ids[context_length:] = mask_position + block_position_ids = np.concatenate([ + np.zeros(context_length, dtype=np.int64), + np.arange( + 1, seq_length - context_length + 1, dtype=np.int64) + ]) + encoded_inputs['position_ids'] = np.stack( + [position_ids, block_position_ids], axis=0) + + if needs_to_be_padded: + difference = max_length - len(required_input) + + if 'attention_mask' in encoded_inputs: + encoded_inputs['attention_mask'] = np.pad( + encoded_inputs['attention_mask'], + pad_width=[(0, 0), (difference, 0), (difference, 0)], + mode='constant', + constant_values=True) + if 'token_type_ids' in encoded_inputs: + encoded_inputs['token_type_ids'] = [ + self.pad_token_type_id + ] * difference + encoded_inputs['token_type_ids'] + if 'special_tokens_mask' in encoded_inputs: + encoded_inputs['special_tokens_mask'] = [ + 1 + ] * difference + encoded_inputs['special_tokens_mask'] + if 'position_ids' in encoded_inputs: + encoded_inputs['position_ids'] = np.pad( + encoded_inputs['position_ids'], + pad_width=[(0, 0), (difference, 0)]) + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + + return encoded_inputs diff --git a/runtime/Lib/site_packages/models/nlp/chatglm2/__init__.py b/runtime/Lib/site_packages/models/nlp/chatglm2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a2b5bfea23efea93a9dfea8ff2fc5fe2e154357d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm2/__init__.py @@ -0,0 +1,46 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import ChatGLM2Config + from .tokenization import ChatGLM2Tokenizer + from .text_generation import ChatGLM2ForConditionalGeneration + from .quantization import ( + quantize, ) + +else: + _import_structure = { + 'configuration': ['ChatGLM2Config'], + 'text_generation': ['ChatGLM2ForConditionalGeneration'], + 'quantization': ['quantize'], + 'tokenization': [ + 'ChatGLM2Tokenizer', + ], + } + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__) diff --git a/runtime/Lib/site_packages/models/nlp/chatglm2/configuration.py b/runtime/Lib/site_packages/models/nlp/chatglm2/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..ec056ccee90197907783cc1ab31b68caaffde6d6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm2/configuration.py @@ -0,0 +1,66 @@ +""" ChatGLM model configuration """ + +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class ChatGLM2Config(PretrainedConfig): + model_type = 'chatglm' + + def __init__(self, + num_layers=28, + padded_vocab_size=65024, + hidden_size=4096, + ffn_hidden_size=13696, + kv_channels=128, + num_attention_heads=32, + seq_length=2048, + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-5, + rmsnorm=True, + apply_residual_connection_post_layernorm=False, + post_layer_norm=True, + add_bias_linear=False, + add_qkv_bias=False, + bias_dropout_fusion=True, + multi_query_attention=False, + multi_query_group_num=1, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=True, + fp32_residual_connection=False, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + rope_ratio=1.0, + **kwargs): + self.num_layers = num_layers + self.vocab_size = padded_vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.kv_channels = kv_channels + self.num_attention_heads = num_attention_heads + self.seq_length = seq_length + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.layernorm_epsilon = layernorm_epsilon + self.rmsnorm = rmsnorm + self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm + self.post_layer_norm = post_layer_norm + self.add_bias_linear = add_bias_linear + self.add_qkv_bias = add_qkv_bias + self.bias_dropout_fusion = bias_dropout_fusion + self.multi_query_attention = multi_query_attention + self.multi_query_group_num = multi_query_group_num + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.fp32_residual_connection = fp32_residual_connection + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + self.rope_ratio = rope_ratio + super().__init__(**kwargs) diff --git a/runtime/Lib/site_packages/models/nlp/chatglm2/quantization.py b/runtime/Lib/site_packages/models/nlp/chatglm2/quantization.py new file mode 100644 index 0000000000000000000000000000000000000000..a1e8b8f2507762dca73f5d524be17985c247c6bf --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm2/quantization.py @@ -0,0 +1,222 @@ +import base64 +import bz2 +import ctypes +from typing import List + +import torch +from torch.nn.parameter import Parameter + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + +try: + from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up + + class Kernel: + + def __init__(self, code: bytes, function_names: List[str]): + self.code = code + self._function_names = function_names + self._cmodule = LazyKernelCModule(self.code) + + for name in self._function_names: + setattr(self, name, KernelFunction(self._cmodule, name)) + + quantization_code = '$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ' # noqa + + kernels = Kernel( + bz2.decompress(base64.b64decode(quantization_code)), + [ + 'int4WeightCompression', + 'int4WeightExtractionFloat', + 'int4WeightExtractionHalf', + 'int8WeightExtractionFloat', + 'int8WeightExtractionHalf', + ], + ) +except Exception as exception: + kernels = None + logger.warning('Failed to load cpm_kernels:' + str(exception)) + + +class W8A16Linear(torch.autograd.Function): + + @staticmethod + def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, + scale_w: torch.Tensor, weight_bit_width): + ctx.inp_shape = inp.size() + ctx.weight_bit_width = weight_bit_width + out_features = quant_w.size(0) + inp = inp.contiguous().view(-1, inp.size(-1)) + weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width) + ctx.weight_shape = weight.size() + output = inp.mm(weight.t()) + ctx.save_for_backward(inp, quant_w, scale_w) + return output.view(*(ctx.inp_shape[:-1] + (out_features, ))) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + inp, quant_w, scale_w = ctx.saved_tensors + weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width) + grad_output = grad_output.contiguous().view(-1, weight.size(0)) + grad_input = grad_output.mm(weight) + grad_weight = grad_output.t().mm(inp) + return grad_input.view(ctx.inp_shape), grad_weight.view( + ctx.weight_shape), None, None + + +def compress_int4_weight(weight: torch.Tensor): # (n, m) + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + assert m % 2 == 0 + m = m // 2 + out = torch.empty(n, m, dtype=torch.int8, device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + kernels.int4WeightCompression( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m) + ], + ) + return out + + +def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, + source_bit_width: int): + assert scale_list.dtype in [torch.half, torch.bfloat16] + assert weight.dtype in [torch.int8] + if source_bit_width == 8: + return weight.to(scale_list.dtype) * scale_list[:, None] + elif source_bit_width == 4: + func = ( + kernels.int4WeightExtractionHalf if scale_list.dtype == torch.half + else kernels.int4WeightExtractionBFloat16) + else: + assert False, 'Unsupported bit-width' + + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + out = torch.empty( + n, + m * (8 // source_bit_width), + dtype=scale_list.dtype, + device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + func( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(scale_list.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m), + ], + ) + return out + + +class QuantizedLinear(torch.nn.Module): + + def __init__(self, + weight_bit_width: int, + weight, + bias=None, + device='cpu', + dtype=None, + empty_init=False, + *args, + **kwargs): + super().__init__() + self.weight_bit_width = weight_bit_width + + shape = weight.shape + + if weight is None or empty_init: + self.weight = torch.empty( + shape[0], + shape[1] * weight_bit_width // 8, + dtype=torch.int8, + device=device) + self.weight_scale = torch.empty( + shape[0], dtype=dtype, device=device) + else: + self.weight_scale = weight.abs().max(dim=-1).values / ( + (2**(weight_bit_width - 1)) - 1) + self.weight = torch.round(weight / self.weight_scale[:, None]).to( + torch.int8) + if weight_bit_width == 4: + self.weight = compress_int4_weight(self.weight) + + self.weight = Parameter(self.weight.to(device), requires_grad=False) + self.weight_scale = Parameter( + self.weight_scale.to(device), requires_grad=False) + self.bias = Parameter( + bias.to(device), requires_grad=False) if bias is not None else None + + def forward(self, input): + output = W8A16Linear.apply(input, self.weight, self.weight_scale, + self.weight_bit_width) + if self.bias is not None: + output = output + self.bias + return output + + +def quantize(model, weight_bit_width, empty_init=False, device=None): + """Replace fp16 linear with quantized linear""" + for layer in model.layers: + layer.self_attention.query_key_value = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight=layer.self_attention.query_key_value.weight.to( + torch.cuda.current_device()), + bias=layer.self_attention.query_key_value.bias, + dtype=layer.self_attention.query_key_value.weight.dtype, + device=layer.self_attention.query_key_value.weight.device + if device is None else device, + empty_init=empty_init) + layer.self_attention.dense = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight=layer.self_attention.dense.weight.to( + torch.cuda.current_device()), + bias=layer.self_attention.dense.bias, + dtype=layer.self_attention.dense.weight.dtype, + device=layer.self_attention.dense.weight.device + if device is None else device, + empty_init=empty_init) + layer.mlp.dense_h_to_4h = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight=layer.mlp.dense_h_to_4h.weight.to( + torch.cuda.current_device()), + bias=layer.mlp.dense_h_to_4h.bias, + dtype=layer.mlp.dense_h_to_4h.weight.dtype, + device=layer.mlp.dense_h_to_4h.weight.device + if device is None else device, + empty_init=empty_init) + layer.mlp.dense_4h_to_h = QuantizedLinear( + weight_bit_width=weight_bit_width, + weight=layer.mlp.dense_4h_to_h.weight.to( + torch.cuda.current_device()), + bias=layer.mlp.dense_4h_to_h.bias, + dtype=layer.mlp.dense_4h_to_h.weight.dtype, + device=layer.mlp.dense_4h_to_h.weight.device + if device is None else device, + empty_init=empty_init) + + return model diff --git a/runtime/Lib/site_packages/models/nlp/chatglm2/text_generation.py b/runtime/Lib/site_packages/models/nlp/chatglm2/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..21323e64bd8369904ed225ac45731a8e7d29c069 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm2/text_generation.py @@ -0,0 +1,1440 @@ +""" PyTorch ChatGLM model. """ + +import copy +import math +import sys +import warnings +from typing import Any, Callable, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss, LayerNorm +from torch.nn.utils import skip_init +from transformers.generation.logits_process import LogitsProcessor +from transformers.generation.utils import (GenerationConfig, + LogitsProcessorList, ModelOutput, + StoppingCriteriaList) +from transformers.modeling_outputs import (BaseModelOutputWithPast, + CausalLMOutputWithPast) +from transformers.modeling_utils import PreTrainedModel + +from modelscope import Model, TorchModel +from modelscope.metainfo import Models +from modelscope.outputs import OutputKeys +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from ... import MODELS +from .configuration import ChatGLM2Config + +# flags required to enable jit fusion kernels + +if sys.platform != 'darwin': + torch._C._jit_set_profiling_mode(False) + torch._C._jit_set_profiling_executor(False) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + +logger = logging.get_logger() + +_CHECKPOINT_FOR_DOC = 'THUDM/ChatGLM2-6B' +_CONFIG_FOR_DOC = 'ChatGLM6BConfig' + +CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [ + 'THUDM/chatglm2-6b', + # See all ChatGLM models at https://huggingface.co/models?filter=chatglm +] + + +def default_init(cls, *args, **kwargs): + return cls(*args, **kwargs) + + +class InvalidScoreLogitsProcessor(LogitsProcessor): + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + if torch.isnan(scores).any() or torch.isinf(scores).any(): + scores.zero_() + scores[..., 5] = 5e4 + return scores + + +class PrefixEncoder(torch.nn.Module): + """ + The torch.nn model to encode the prefix + Input shape: (batch-size, prefix-length) + Output shape: (batch-size, prefix-length, 2*layers*hidden) + """ + + def __init__(self, config: ChatGLM2Config): + super().__init__() + self.prefix_projection = config.prefix_projection + if self.prefix_projection: + # Use a two-layer MLP to encode the prefix + kv_size = config.num_layers * config.kv_channels * config.multi_query_group_num * 2 + self.embedding = torch.nn.Embedding(config.pre_seq_len, kv_size) + self.trans = torch.nn.Sequential( + torch.nn.Linear(kv_size, config.hidden_size), torch.nn.Tanh(), + torch.nn.Linear(config.hidden_size, kv_size)) + else: + self.embedding = torch.nn.Embedding( + config.pre_seq_len, config.num_layers * config.kv_channels + * config.multi_query_group_num * 2) + + def forward(self, prefix: torch.Tensor): + if self.prefix_projection: + prefix_tokens = self.embedding(prefix) + past_key_values = self.trans(prefix_tokens) + else: + past_key_values = self.embedding(prefix) + return past_key_values + + +def split_tensor_along_last_dim( + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, +) -> List[torch.Tensor]: + """Split a tensor along its last dimension. + + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + + Returns: + A list of Tensors + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = tensor.size()[last_dim] // num_partitions + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + +class RotaryEmbedding(nn.Module): + + def __init__(self, + dim, + rope_ratio=1, + original_impl=False, + device=None, + dtype=None): + super().__init__() + inv_freq = 1.0 / (10000**( + torch.arange(0, dim, 2, device=device).to(dtype=dtype) / dim)) + self.register_buffer('inv_freq', inv_freq) + self.dim = dim + self.original_impl = original_impl + self.rope_ratio = rope_ratio + + def forward_impl(self, + seq_len: int, + n_elem: int, + dtype: torch.dtype, + device: torch.device, + base: int = 10000): + """Enhanced Transformer with Rotary Position Embedding. + + Derived from: https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/ + transformers/rope/__init__.py. MIT License: + https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/license. + """ + # $\Theta = {\theta_i = 10000^{\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$ + theta = 1.0 / ( + base**(torch.arange(0, n_elem, 2, dtype=dtype, device=device) + / n_elem)) + + # Create position indexes `[0, 1, ..., seq_len - 1]` + seq_idx = torch.arange( + seq_len, dtype=dtype, device=device) / self.rope_ratio + + # Calculate the product of position index and $\theta_i$ + idx_theta = torch.outer(seq_idx, theta).float() + + cache = torch.stack( + [torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1) + + # this is to mimic the behaviour of complex32, else we will get different results + if dtype in (torch.float16, torch.bfloat16, torch.int8): + cache = cache.bfloat16( + ) if dtype == torch.bfloat16 else cache.half() + return cache + + def forward(self, max_seq_len, offset=0): + return self.forward_impl( + max_seq_len, + self.dim, + dtype=self.inv_freq.dtype, + device=self.inv_freq.device) + + +@torch.jit.script +def apply_rotary_pos_emb(x: torch.Tensor, + rope_cache: torch.Tensor) -> torch.Tensor: + # x: [sq, b, np, hn] + sq, _, np, _ = x.size(0), x.size(1), x.size(2), x.size(3) + rot_dim = rope_cache.shape[-2] * 2 + x, x_pass = x[..., :rot_dim], x[..., rot_dim:] + # truncate to support variable sizes + rope_cache = rope_cache[:sq] + xshaped = x.reshape(sq, -1, np, rot_dim // 2, 2) + rope_cache = rope_cache.view(sq, -1, 1, xshaped.size(3), 2) + x_out2 = torch.stack( + [ + xshaped[..., 0] * rope_cache[..., 0] + - xshaped[..., 1] * rope_cache[..., 1], + xshaped[..., 1] * rope_cache[..., 0] + + xshaped[..., 0] * rope_cache[..., 1], + ], + -1, + ) + x_out2 = x_out2.flatten(3) + return torch.cat((x_out2, x_pass), dim=-1) + + +class RMSNorm(torch.nn.Module): + + def __init__(self, + normalized_shape, + eps=1e-5, + device=None, + dtype=None, + **kwargs): + super().__init__() + self.weight = torch.nn.Parameter( + torch.empty(normalized_shape, device=device, dtype=dtype)) + self.eps = eps + + def forward(self, hidden_states: torch.Tensor): + input_dtype = hidden_states.dtype + variance = hidden_states.to(torch.float32).pow(2).mean( + -1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.eps) + + return (self.weight * hidden_states).to(input_dtype) + + +class CoreAttention(torch.nn.Module): + + def __init__(self, config: ChatGLM2Config, layer_number): + super(CoreAttention, self).__init__() + + self.apply_query_key_layer_scaling = config.apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = config.attention_softmax_in_fp32 + if self.apply_query_key_layer_scaling: + self.attention_softmax_in_fp32 = True + self.layer_number = max(1, layer_number) + + projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + self.hidden_size_per_partition = projection_size + self.hidden_size_per_attention_head = projection_size // config.num_attention_heads + self.num_attention_heads_per_partition = config.num_attention_heads + + coeff = None + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + if self.apply_query_key_layer_scaling: + coeff = self.layer_number + self.norm_factor *= coeff + self.coeff = coeff + + self.attention_dropout = torch.nn.Dropout(config.attention_dropout) + + def forward(self, query_layer, key_layer, value_layer, attention_mask): + pytorch_major_version = int(torch.__version__.split('.')[0]) + if pytorch_major_version >= 2: + query_layer, key_layer, value_layer = [ + k.permute(1, 2, 0, 3) + for k in [query_layer, key_layer, value_layer] + ] + if attention_mask is None and query_layer.shape[ + 2] == key_layer.shape[2]: + context_layer = torch.nn.functional.scaled_dot_product_attention( + query_layer, key_layer, value_layer, is_causal=True) + else: + if attention_mask is not None: + attention_mask = ~attention_mask + context_layer = torch.nn.functional.scaled_dot_product_attention( + query_layer, key_layer, value_layer, attention_mask) + context_layer = context_layer.permute(2, 0, 1, 3) + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size_per_partition, ) + context_layer = context_layer.reshape(*new_context_layer_shape) + else: + # Raw attention scores + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view(output_size[2], + output_size[0] * output_size[1], -1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], + output_size[0] * output_size[1], -1) + + # preallocting input tensor: [b * np, sq, sk] + matmul_input_buffer = torch.empty( + output_size[0] * output_size[1], + output_size[2], + output_size[3], + dtype=query_layer.dtype, + device=query_layer.device) + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.baddbmm( + matmul_input_buffer, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / self.norm_factor), + ) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + # =========================== + # Attention probs and dropout + # =========================== + + # attention scores and attention mask [b, np, sq, sk] + if self.attention_softmax_in_fp32: + attention_scores = attention_scores.float() + if self.coeff is not None: + attention_scores = attention_scores * self.coeff + if attention_mask is None and attention_scores.shape[ + 2] == attention_scores.shape[3]: + attention_mask = torch.ones( + output_size[0], + 1, + output_size[2], + output_size[3], + device=attention_scores.device, + dtype=torch.bool) + attention_mask.tril_() + attention_mask = ~attention_mask + if attention_mask is not None: + attention_scores = attention_scores.masked_fill( + attention_mask, float('-inf')) + attention_probs = F.softmax(attention_scores, dim=-1) + attention_probs = attention_probs.type_as(value_layer) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.attention_dropout(attention_probs) + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + # change view [sk, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + # change view [b * np, sq, sk] + attention_probs = attention_probs.view( + output_size[0] * output_size[1], output_size[2], -1) + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, + value_layer.transpose(0, 1)) + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size_per_partition, ) + context_layer = context_layer.view(*new_context_layer_shape) + + return context_layer + + +class SelfAttention(torch.nn.Module): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__(self, config: ChatGLM2Config, layer_number, device=None): + super(SelfAttention, self).__init__() + self.layer_number = max(1, layer_number) + + self.projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + self.hidden_size_per_attention_head = self.projection_size // config.num_attention_heads + self.num_attention_heads_per_partition = config.num_attention_heads + + self.multi_query_attention = config.multi_query_attention + self.qkv_hidden_size = 3 * self.projection_size + if self.multi_query_attention: + self.num_multi_query_groups_per_partition = config.multi_query_group_num + self.qkv_hidden_size = ( + self.projection_size + 2 * self.hidden_size_per_attention_head + * config.multi_query_group_num) + self.query_key_value = nn.Linear( + config.hidden_size, + self.qkv_hidden_size, + bias=config.add_bias_linear or config.add_qkv_bias, + device=device, + **_config_to_kwargs(config)) + + self.core_attention = CoreAttention(config, self.layer_number) + + # Output. + self.dense = nn.Linear( + self.projection_size, + config.hidden_size, + bias=config.add_bias_linear, + device=device, + **_config_to_kwargs(config)) + + def _allocate_memory(self, + inference_max_sequence_len, + batch_size, + device=None, + dtype=None): + if self.multi_query_attention: + num_attention_heads = self.num_multi_query_groups_per_partition + else: + num_attention_heads = self.num_attention_heads_per_partition + return torch.empty( + inference_max_sequence_len, + batch_size, + num_attention_heads, + self.hidden_size_per_attention_head, + dtype=dtype, + device=device, + ) + + def forward(self, + hidden_states, + attention_mask, + rotary_pos_emb, + kv_cache=None, + use_cache=True): + # hidden_states: [sq, b, h] + + # ================================================= + # Pre-allocate memory for key-values for inference. + # ================================================= + # ===================== + # Query, Key, and Value + # ===================== + + # Attention heads [sq, b, h] --> [sq, b, (np * 3 * hn)] + mixed_x_layer = self.query_key_value(hidden_states) + + if self.multi_query_attention: + (query_layer, key_layer, value_layer) = mixed_x_layer.split( + [ + self.num_attention_heads_per_partition + * self.hidden_size_per_attention_head, + self.num_multi_query_groups_per_partition + * self.hidden_size_per_attention_head, + self.num_multi_query_groups_per_partition + * self.hidden_size_per_attention_head, + ], + dim=-1, + ) + query_layer = query_layer.view(query_layer.size()[:-1] + ( + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head)) + key_layer = key_layer.view(key_layer.size()[:-1] + ( + self.num_multi_query_groups_per_partition, + self.hidden_size_per_attention_head)) + value_layer = value_layer.view(value_layer.size()[:-1] + ( + self.num_multi_query_groups_per_partition, + self.hidden_size_per_attention_head)) + else: + new_tensor_shape = mixed_x_layer.size()[:-1] + \ + (self.num_attention_heads_per_partition, # noqa + 3 * self.hidden_size_per_attention_head) # noqa + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) + + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] + (query_layer, key_layer, + value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3) + + # apply relative positional encoding (rotary embedding) + if rotary_pos_emb is not None: + query_layer = apply_rotary_pos_emb(query_layer, rotary_pos_emb) + key_layer = apply_rotary_pos_emb(key_layer, rotary_pos_emb) + + # adjust key and value for inference + if kv_cache is not None: + cache_k, cache_v = kv_cache + key_layer = torch.cat((cache_k, key_layer), dim=0) + value_layer = torch.cat((cache_v, value_layer), dim=0) + if use_cache: + kv_cache = (key_layer, value_layer) + else: + kv_cache = None + + if self.multi_query_attention: + key_layer = key_layer.unsqueeze(-2) + key_layer = key_layer.expand( + -1, -1, -1, self.num_attention_heads_per_partition + // self.num_multi_query_groups_per_partition, -1) + key_layer = key_layer.contiguous().view( + key_layer.size()[:2] + (self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head)) + value_layer = value_layer.unsqueeze(-2) + value_layer = value_layer.expand( + -1, -1, -1, self.num_attention_heads_per_partition + // self.num_multi_query_groups_per_partition, -1) + value_layer = value_layer.contiguous().view( + value_layer.size()[:2] + + (self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head)) + + # ================================== + # core attention computation + # ================================== + + context_layer = self.core_attention(query_layer, key_layer, + value_layer, attention_mask) + + # ================= + # Output. [sq, b, h] + # ================= + + output = self.dense(context_layer) + + return output, kv_cache + + +def _config_to_kwargs(args): + common_kwargs = { + 'dtype': args.torch_dtype, + } + return common_kwargs + + +class MLP(torch.nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__(self, config: ChatGLM2Config, device=None): + super(MLP, self).__init__() + + self.add_bias = config.add_bias_linear + + # Project to 4h. If using swiglu double the output width, see https://arxiv.org/pdf/2002.05202.pdf + self.dense_h_to_4h = nn.Linear( + config.hidden_size, + config.ffn_hidden_size * 2, + bias=self.add_bias, + device=device, + **_config_to_kwargs(config)) + + def swiglu(x): + x = torch.chunk(x, 2, dim=-1) + return F.silu(x[0]) * x[1] + + self.activation_func = swiglu + + # Project back to h. + self.dense_4h_to_h = nn.Linear( + config.ffn_hidden_size, + config.hidden_size, + bias=self.add_bias, + device=device, + **_config_to_kwargs(config)) + + def forward(self, hidden_states): + # [s, b, 4hp] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output = self.dense_4h_to_h(intermediate_parallel) + return output + + +class GLMBlock(torch.nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__(self, config: ChatGLM2Config, layer_number, device=None): + super(GLMBlock, self).__init__() + self.layer_number = layer_number + + self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm + + self.fp32_residual_connection = config.fp32_residual_connection + + LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm + # Layernorm on the input data. + self.input_layernorm = LayerNormFunc( + config.hidden_size, + eps=config.layernorm_epsilon, + device=device, + dtype=config.torch_dtype) + + # Self attention. + self.self_attention = SelfAttention( + config, layer_number, device=device) + self.hidden_dropout = config.hidden_dropout + + # Layernorm on the attention output + self.post_attention_layernorm = LayerNormFunc( + config.hidden_size, + eps=config.layernorm_epsilon, + device=device, + dtype=config.torch_dtype) + + # MLP + self.mlp = MLP(config, device=device) + + def forward( + self, + hidden_states, + attention_mask, + rotary_pos_emb, + kv_cache=None, + use_cache=True, + ): + # hidden_states: [s, b, h] + + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output, kv_cache = self.self_attention( + layernorm_output, + attention_mask, + rotary_pos_emb, + kv_cache=kv_cache, + use_cache=use_cache) + + # Residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + layernorm_input = torch.nn.functional.dropout( + attention_output, p=self.hidden_dropout, training=self.training) + layernorm_input = residual + layernorm_input + + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + + # MLP. + mlp_output = self.mlp(layernorm_output) + + # Second residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + output = torch.nn.functional.dropout( + mlp_output, p=self.hidden_dropout, training=self.training) + output = residual + output + + return output, kv_cache + + +class GLMTransformer(torch.nn.Module): + """Transformer class.""" + + def __init__(self, config: ChatGLM2Config, device=None): + super(GLMTransformer, self).__init__() + + self.fp32_residual_connection = config.fp32_residual_connection + self.post_layer_norm = config.post_layer_norm + + # Number of layers. + self.num_layers = config.num_layers + + # Transformer layers. + def build_layer(layer_number): + return GLMBlock(config, layer_number, device=device) + + self.layers = torch.nn.ModuleList( + [build_layer(i + 1) for i in range(self.num_layers)]) + + if self.post_layer_norm: + LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm + # Final layer norm before output. + self.final_layernorm = LayerNormFunc( + config.hidden_size, + eps=config.layernorm_epsilon, + device=device, + dtype=config.torch_dtype) + + self.gradient_checkpointing = False + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def forward( + self, + hidden_states, + attention_mask, + rotary_pos_emb, + kv_caches=None, + use_cache: Optional[bool] = True, + output_hidden_states: Optional[bool] = False, + ): + if not kv_caches: + kv_caches = [None for _ in range(self.num_layers)] + presents = () if use_cache else None + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + all_self_attentions = None + all_hidden_states = () if output_hidden_states else None + for index in range(self.num_layers): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer = self._get_layer(index) + if self.gradient_checkpointing and self.training: + layer_ret = torch.utils.checkpoint.checkpoint( + layer, hidden_states, attention_mask, rotary_pos_emb, + kv_caches[index], use_cache) + else: + layer_ret = layer( + hidden_states, + attention_mask, + rotary_pos_emb, + kv_cache=kv_caches[index], + use_cache=use_cache) + hidden_states, kv_cache = layer_ret + if use_cache: + presents = presents + (kv_cache, ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + # Final layer norm. + if self.post_layer_norm: + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states, presents, all_hidden_states, all_self_attentions + + +class ChatGLMPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and + a simple interface for downloading and loading pretrained models. + """ + + is_parallelizable = False + supports_gradient_checkpointing = True + config_class = ChatGLM2Config + base_model_prefix = 'transformer' + _no_split_modules = ['GLMBlock'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module: nn.Module): + """Initialize the weights.""" + return + + def get_masks(self, input_ids, past_key_values, padding_mask=None): + batch_size, seq_length = input_ids.shape + full_attention_mask = torch.ones( + batch_size, seq_length, seq_length, device=input_ids.device) + full_attention_mask.tril_() + past_length = 0 + if past_key_values: + past_length = past_key_values[0][0].shape[0] + if past_length: + full_attention_mask = torch.cat( + ( + torch.ones( # noqa + batch_size, + seq_length, + past_length, # noqa + device=input_ids.device), + full_attention_mask), # noqa + dim=-1) # noqa + if padding_mask is not None: + full_attention_mask = full_attention_mask * padding_mask.unsqueeze( + 1) + if not past_length and padding_mask is not None: + full_attention_mask -= padding_mask.unsqueeze(-1) - 1 + full_attention_mask = (full_attention_mask < 0.5).bool() + full_attention_mask.unsqueeze_(1) + return full_attention_mask + + def get_position_ids(self, input_ids, device): + batch_size, seq_length = input_ids.shape + position_ids = torch.arange( + seq_length, dtype=torch.long, + device=device).unsqueeze(0).repeat(batch_size, 1) + return position_ids + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, GLMTransformer): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + kwargs.pop('cfg', None) + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model + + +class Embedding(torch.nn.Module): + """Language model embeddings.""" + + def __init__(self, config: ChatGLM2Config, device=None): + super(Embedding, self).__init__() + + self.hidden_size = config.hidden_size + # Word embeddings (parallel). + self.word_embeddings = nn.Embedding( + config.padded_vocab_size, + self.hidden_size, + dtype=config.torch_dtype, + device=device) + self.fp32_residual_connection = config.fp32_residual_connection + + def forward(self, input_ids): + # Embeddings. + words_embeddings = self.word_embeddings(input_ids) + embeddings = words_embeddings + # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. + embeddings = embeddings.transpose(0, 1).contiguous() + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + return embeddings + + +class ChatGLMModel(ChatGLMPreTrainedModel): + + def __init__(self, config: ChatGLM2Config, device=None, empty_init=True): + super().__init__(config) + if empty_init: + init_method = skip_init + else: + init_method = default_init + init_kwargs = {} + if device is not None: + init_kwargs['device'] = device + self.embedding = init_method(Embedding, config, **init_kwargs) + self.num_layers = config.num_layers + self.multi_query_group_num = config.multi_query_group_num + self.kv_channels = config.kv_channels + + # Rotary positional embeddings + self.seq_length = config.seq_length + rotary_dim = ( + config.hidden_size // config.num_attention_heads + if config.kv_channels is None else config.kv_channels) + + self.rotary_pos_emb = RotaryEmbedding( + rotary_dim // 2, + rope_ratio=config.rope_ratio, + original_impl=config.original_rope, + device=device, + dtype=config.torch_dtype) + self.encoder = init_method(GLMTransformer, config, **init_kwargs) + self.output_layer = init_method( + nn.Linear, + config.hidden_size, + config.padded_vocab_size, + bias=False, + dtype=config.torch_dtype, + **init_kwargs) + self.pre_seq_len = config.pre_seq_len + self.prefix_projection = config.prefix_projection + if self.pre_seq_len is not None: + for param in self.parameters(): + param.requires_grad = False + self.prefix_tokens = torch.arange(self.pre_seq_len).long() + self.prefix_encoder = PrefixEncoder(config) + self.dropout = torch.nn.Dropout(0.1) + + def get_input_embeddings(self): + return self.embedding.word_embeddings + + def get_prompt(self, batch_size, device, dtype=torch.half): + prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, + -1).to(device) + past_key_values = self.prefix_encoder(prefix_tokens).type(dtype) + past_key_values = past_key_values.view(batch_size, self.pre_seq_len, + self.num_layers * 2, + self.multi_query_group_num, + self.kv_channels) + # seq_len, b, nh, hidden_size + past_key_values = self.dropout(past_key_values) + past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2) + return past_key_values + + def forward( + self, + input_ids, + position_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.BoolTensor] = None, + full_attention_mask: Optional[torch.BoolTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], + ...]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + batch_size, seq_length = input_ids.shape + + if inputs_embeds is None: + inputs_embeds = self.embedding(input_ids) + + if self.pre_seq_len is not None: + if past_key_values is None: + past_key_values = self.get_prompt( + batch_size=batch_size, + device=input_ids.device, + dtype=inputs_embeds.dtype) + if attention_mask is not None: + attention_mask = torch.cat( + [ + attention_mask.new_ones( # noqa + (batch_size, self.pre_seq_len)), + attention_mask # noqa + ], # noqa + dim=-1) # noqa + + if full_attention_mask is None: + if (attention_mask is not None + and not attention_mask.all()) or (past_key_values + and seq_length != 1): + full_attention_mask = self.get_masks( + input_ids, past_key_values, padding_mask=attention_mask) + + # Rotary positional embeddings + rotary_pos_emb = self.rotary_pos_emb(self.seq_length) + if position_ids is not None: + rotary_pos_emb = rotary_pos_emb[position_ids] + else: + rotary_pos_emb = rotary_pos_emb[None, :seq_length] + rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous() + + # Run encoder. + hidden_states, presents, all_hidden_states, all_self_attentions = self.encoder( + inputs_embeds, + full_attention_mask, + rotary_pos_emb=rotary_pos_emb, + kv_caches=past_key_values, + use_cache=use_cache, + output_hidden_states=output_hidden_states) + + if not return_dict: + return tuple(v for v in [ + hidden_states, presents, all_hidden_states, all_self_attentions + ] if v is not None) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + def quantize(self, weight_bit_width: int): + from .quantization import quantize + quantize(self.encoder, weight_bit_width) + return self + + +@MODELS.register_module(Tasks.chat, module_name=Models.chatglm2_6b) +class ChatGLM2ForConditionalGeneration(ChatGLMPreTrainedModel): + + def __init__(self, config: ChatGLM2Config, empty_init=True, device=None): + super().__init__(config) + + self.max_sequence_length = config.max_length + self.transformer = ChatGLMModel( + config, empty_init=empty_init, device=device) + self.config = config + self.quantized = False + + if self.config.quantization_bit: + self.quantize(self.config.quantization_bit, empty_init=True) + + def _update_model_kwargs_for_generation( + self, + outputs: ModelOutput, + model_kwargs: Dict[str, Any], + is_encoder_decoder: bool = False, + standardize_cache_format: bool = False, + ) -> Dict[str, Any]: + # update past_key_values + model_kwargs['past_key_values'] = self._extract_past_from_model_output( + outputs, standardize_cache_format=standardize_cache_format) + + # update attention mask + if 'attention_mask' in model_kwargs: + attention_mask = model_kwargs['attention_mask'] + model_kwargs['attention_mask'] = torch.cat( + [ # noqa + attention_mask, # noqa + attention_mask.new_ones( + (attention_mask.shape[0], 1)) # noqa + ], # noqa + dim=-1) # noqa + + # update position ids + if 'position_ids' in model_kwargs: + position_ids = model_kwargs['position_ids'] + new_position_id = position_ids[..., -1:].clone() + new_position_id += 1 + model_kwargs['position_ids'] = torch.cat( + [position_ids, new_position_id], dim=-1) + + model_kwargs['is_first_forward'] = False + return model_kwargs + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + is_first_forward: bool = True, + **kwargs) -> dict: + # only last token for input_ids if past is not None + if position_ids is None: + position_ids = self.get_position_ids( + input_ids, device=input_ids.device) + if not is_first_forward: + position_ids = position_ids[..., -1:] + input_ids = input_ids[:, -1:] + return { + 'input_ids': input_ids, + 'past_key_values': past_key_values, + 'position_ids': position_ids, + 'attention_mask': attention_mask, + 'return_last_logit': True + } + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + return_last_logit: Optional[bool] = False, + ): + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.transformer( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = transformer_outputs[0] + if return_last_logit: + hidden_states = hidden_states[-1:] + lm_logits = self.transformer.output_layer(hidden_states) + lm_logits = lm_logits.transpose(0, 1).contiguous() + + loss = None + if labels is not None: + lm_logits = lm_logits.to(torch.float32) + + # Shift so that tokens < n predict n + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss(ignore_index=-100) + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + + lm_logits = lm_logits.to(hidden_states.dtype) + loss = loss.to(hidden_states.dtype) + + if not return_dict: + output = (lm_logits, ) + transformer_outputs[1:] + return ((loss, ) + output) if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + @staticmethod + def _reorder_cache( + past: Tuple[Tuple[torch.Tensor, torch.Tensor], + ...], beam_idx: torch.LongTensor + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]: + """ + This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct + beam_idx at every generation step. + + Output shares the same memory storage as `past`. + """ + return tuple(( + layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)), + layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)), + ) for layer_past in past) + + def process_response(self, response): + response = response.strip() + response = response.replace('[[训练时间]]', '2023年') + return response + + def build_inputs(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None): + prompt = tokenizer.build_prompt(query, history=history) + inputs = tokenizer([prompt], return_tensors='pt') + inputs = inputs.to(self.device) + return inputs + + def build_stream_inputs(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None): + if history: + prompt = '\n\n[Round {}]\n\n问:{}\n\n答:'.format( + len(history) + 1, query) + input_ids = tokenizer.encode(prompt, add_special_tokens=False) + input_ids = input_ids[1:] + inputs = tokenizer.batch_encode_plus([(input_ids, None)], + return_tensors='pt', + add_special_tokens=False) + else: + prompt = '[Round {}]\n\n问:{}\n\n答:'.format(len(history) + 1, query) + inputs = tokenizer([prompt], return_tensors='pt') + inputs = inputs.to(self.device) + return inputs + + @torch.no_grad() + def _chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None, + max_length: int = None, + num_beams=1, + do_sample=True, + top_p=0.8, + temperature=0.8, + logits_processor=None, + **kwargs): + if history is None: + history = [] + if logits_processor is None: + logits_processor = LogitsProcessorList() + logits_processor.append(InvalidScoreLogitsProcessor()) + if max_length is None: + max_length = self.seq_length + gen_kwargs = { + 'max_length': max_length, + 'num_beams': num_beams, + 'do_sample': do_sample, + 'top_p': top_p, + 'temperature': temperature, + 'logits_processor': logits_processor, + **kwargs + } + inputs = self.build_inputs(tokenizer, query, history=history) + outputs = self.generate(**inputs, **gen_kwargs) + outputs = outputs.tolist()[0][len(inputs['input_ids'][0]):] + response = tokenizer.decode(outputs) + response = self.process_response(response) + history = history + [(query, response)] + return response, history + + @torch.no_grad() + def stream_chat(self, + tokenizer, + query: str, + history: List[Tuple[str, str]] = None, + past_key_values=None, + max_length: int = None, + do_sample=True, + top_p=0.8, + temperature=0.8, + logits_processor=None, + return_past_key_values=False, + **kwargs): + if history is None: + history = [] + if logits_processor is None: + logits_processor = LogitsProcessorList() + logits_processor.append(InvalidScoreLogitsProcessor()) + if max_length is None: + max_length = self.seq_length + gen_kwargs = { + 'max_length': max_length, + 'do_sample': do_sample, + 'top_p': top_p, + 'temperature': temperature, + 'logits_processor': logits_processor, + **kwargs + } + if past_key_values is None and not return_past_key_values: + inputs = self.build_inputs(tokenizer, query, history=history) + else: + inputs = self.build_stream_inputs( + tokenizer, query, history=history) + if past_key_values is not None: + past_length = past_key_values[0][0].shape[0] + if self.transformer.pre_seq_len is not None: + past_length -= self.transformer.pre_seq_len + inputs.position_ids += past_length + attention_mask = inputs.attention_mask + attention_mask = torch.cat( + (attention_mask.new_ones(1, past_length), attention_mask), + dim=1) + inputs['attention_mask'] = attention_mask + for outputs in self.stream_generate( + **inputs, + past_key_values=past_key_values, + return_past_key_values=return_past_key_values, + **gen_kwargs): + if return_past_key_values: + outputs, past_key_values = outputs + outputs = outputs.tolist()[0][len(inputs['input_ids'][0]):] + response = tokenizer.decode(outputs) + if response and response[-1] != '�': + response = self.process_response(response) + new_history = history + [(query, response)] + if return_past_key_values: + yield response, new_history, past_key_values + else: + yield response, new_history + + @torch.no_grad() + def stream_generate( + self, + input_ids, + generation_config: Optional[GenerationConfig] = None, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], + List[int]]] = None, + return_past_key_values=False, + **kwargs, + ): + _, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1] + + if generation_config is None: + generation_config = self.generation_config + generation_config = copy.deepcopy(generation_config) + model_kwargs = generation_config.update(**kwargs) + _, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id + + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + + has_default_max_length = kwargs.get( + 'max_length') is None and generation_config.max_length is not None + if has_default_max_length and generation_config.max_new_tokens is None: + warnings.warn( + f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. " + 'This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we' + ' recommend using `max_new_tokens` to control the maximum length of the generation.', + UserWarning, + ) + elif generation_config.max_new_tokens is not None: + generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length + if not has_default_max_length: + logger.warn( + f'Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(=' + f'{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. ' + 'Please refer to the documentation for more information. ' + '(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)', + UserWarning, + ) + + if input_ids_seq_length >= generation_config.max_length: + input_ids_string = 'decoder_input_ids' if self.config.is_encoder_decoder else 'input_ids' + logger.warning( + f'Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to' + f' {generation_config.max_length}. This can lead to unexpected behavior. You should consider' + ' increasing `max_new_tokens`.') + + # 2. Set generation parameters if not already defined + logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList( + ) + stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList( + ) + + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_seq_length, + encoder_input_ids=input_ids, + prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, + logits_processor=logits_processor, + ) + + stopping_criteria = self._get_stopping_criteria( + generation_config=generation_config, + stopping_criteria=stopping_criteria) + logits_warper = self._get_logits_warper(generation_config) + + unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1) + scores = None + while True: + model_inputs = self.prepare_inputs_for_generation( + input_ids, **model_kwargs) + # forward pass to get next token + outputs = self( + **model_inputs, + return_dict=True, + output_attentions=False, + output_hidden_states=False, + ) + + next_token_logits = outputs.logits[:, -1, :] + + # pre-process distribution + next_token_scores = logits_processor(input_ids, next_token_logits) + next_token_scores = logits_warper(input_ids, next_token_scores) + + # sample + probs = nn.functional.softmax(next_token_scores, dim=-1) + if generation_config.do_sample: + next_tokens = torch.multinomial( + probs, num_samples=1).squeeze(1) + else: + next_tokens = torch.argmax(probs, dim=-1) + + # update generated ids, model inputs, and length for next step + input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + model_kwargs = self._update_model_kwargs_for_generation( + outputs, + model_kwargs, + is_encoder_decoder=self.config.is_encoder_decoder) + unfinished_sequences = unfinished_sequences.mul( + (sum(next_tokens != i for i in eos_token_id)).long()) + if return_past_key_values: + yield input_ids, outputs.past_key_values + else: + yield input_ids + # stop when each sentence is finished, or if we exceed the maximum length + if unfinished_sequences.max() == 0 or stopping_criteria( + input_ids, scores): + break + + def quantize(self, bits: int, empty_init=False, device=None, **kwargs): + if bits == 0: + return + + from .quantization import quantize + + if self.quantized: + logger.info('Already quantized.') + return self + + self.quantized = True + + self.config.quantization_bit = bits + + self.transformer.encoder = quantize( + self.transformer.encoder, + bits, + empty_init=empty_init, + device=device, + **kwargs) + return self + + def chat(self, input: Dict, tokenizer) -> Dict: + text = input['text'] + history = input['history'] + # args + if 'max_length' in input: + max_length = input['max_length'] + else: + max_length = 2048 + + if 'temperature' in input: + temperature = input['temperature'] + else: + temperature = 0.95 + + if 'num_beams' in input: + num_beams = input['num_beams'] + else: + num_beams = 1 + + if 'do_sample' in input: + do_sample = input['do_sample'] + else: + do_sample = True + + if type(history) == torch.Tensor: + history = history.tolist() + response, history = self._chat( + tokenizer, + text, + history, + max_length=max_length, + temperature=temperature, + num_beams=num_beams, + do_sample=do_sample) + return {OutputKeys.RESPONSE: response, OutputKeys.HISTORY: history} diff --git a/runtime/Lib/site_packages/models/nlp/chatglm2/tokenization.py b/runtime/Lib/site_packages/models/nlp/chatglm2/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..4523dcdd3d9d425ebc402147816cd77e2c32e2b0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/chatglm2/tokenization.py @@ -0,0 +1,273 @@ +import os +from typing import Dict, List, Optional, Union + +from sentencepiece import SentencePieceProcessor +from transformers import PreTrainedTokenizer +from transformers.tokenization_utils_base import BatchEncoding, EncodedInput +from transformers.utils import PaddingStrategy + + +class SPTokenizer: + + def __init__(self, model_path: str): + # reload tokenizer + assert os.path.isfile(model_path), model_path + self.sp_model = SentencePieceProcessor(model_file=model_path) + + # BOS / EOS token IDs + self.n_words: int = self.sp_model.vocab_size() + self.bos_id: int = self.sp_model.bos_id() + self.eos_id: int = self.sp_model.eos_id() + self.pad_id: int = self.sp_model.unk_id() + assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() + + special_tokens = ['[MASK]', '[gMASK]', '[sMASK]', 'sop', 'eop'] + self.special_tokens = {} + self.index_special_tokens = {} + for token in special_tokens: + self.special_tokens[token] = self.n_words + self.index_special_tokens[self.n_words] = token + self.n_words += 1 + + def tokenize(self, s: str): + return self.sp_model.EncodeAsPieces(s) + + def encode(self, + s: str, + bos: bool = False, + eos: bool = False) -> List[int]: + assert type(s) is str + t = self.sp_model.encode(s) + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + return t + + def decode(self, t: List[int]) -> str: + return self.sp_model.decode(t) + + def decode_tokens(self, tokens: List[str]) -> str: + text = self.sp_model.DecodePieces(tokens) + return text + + def convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + if token in self.special_tokens: + return self.special_tokens[token] + return self.sp_model.PieceToId(token) + + def convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.index_special_tokens or index in [ + self.eos_id, self.bos_id, self.pad_id + ] or index < 0: + return '' + return self.sp_model.IdToPiece(index) + + +class ChatGLM2Tokenizer(PreTrainedTokenizer): + vocab_files_names = {'vocab_file': 'tokenizer.model'} + + model_input_names = ['input_ids', 'attention_mask', 'position_ids'] + + def __init__(self, vocab_file, padding_side='left', **kwargs): + self.name = 'GLMTokenizer' + + self.vocab_file = vocab_file + self.tokenizer = SPTokenizer(vocab_file) + self.special_tokens = { + '': self.tokenizer.bos_id, + '': self.tokenizer.eos_id, + '': self.tokenizer.pad_id + } + super().__init__(padding_side=padding_side, **kwargs) + + def get_command(self, token): + if token in self.special_tokens: + return self.special_tokens[token] + assert token in self.tokenizer.special_tokens, f'{token} is not a special token for {self.name}' + return self.tokenizer.special_tokens[token] + + @property + def pad_token(self) -> str: + return '' + + @property + def pad_token_id(self): + return self.get_command('') + + @property + def eos_token(self) -> str: + return '' + + @property + def eos_token_id(self): + return self.get_command('') + + @property + def vocab_size(self): + return self.tokenizer.n_words + + def get_vocab(self): + """ Returns vocab as a dict """ + vocab = { + self._convert_id_to_token(i): i + for i in range(self.vocab_size) + } + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text, **kwargs): + return self.tokenizer.tokenize(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.tokenizer.convert_token_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.tokenizer.convert_id_to_token(index) + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + return self.tokenizer.decode_tokens(tokens) + + def save_vocabulary(self, save_directory, filename_prefix=None): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if os.path.isdir(save_directory): + vocab_file = os.path.join(save_directory, + self.vocab_files_names['vocab_file']) + else: + vocab_file = save_directory + + with open(self.vocab_file, 'rb') as fin: + proto_str = fin.read() + + with open(vocab_file, 'wb') as writer: + writer.write(proto_str) + + return (vocab_file, ) + + def get_prefix_tokens(self): + prefix_tokens = [self.get_command('[gMASK]'), self.get_command('sop')] + return prefix_tokens + + def build_prompt(self, query, history=None): + if history is None: + history = [] + prompt = '' + for i, (old_query, response) in enumerate(history): + prompt += '[Round {}]\n\n问:{}\n\n答:{}\n\n'.format( + i + 1, old_query, response) + prompt += '[Round {}]\n\n问:{}\n\n答:'.format(len(history) + 1, query) + return prompt + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + prefix_tokens = self.get_prefix_tokens() + token_ids_0 = prefix_tokens + token_ids_0 + if token_ids_1 is not None: + token_ids_0 = token_ids_0 + token_ids_1 + [ + self.get_command('') + ] + return token_ids_0 + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: + Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_attention_mask: + (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + assert self.padding_side == 'left' + + required_input = encoded_inputs[self.model_input_names[0]] + seq_length = len(required_input) + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + # Initialize attention mask if not present. + if 'attention_mask' not in encoded_inputs: + encoded_inputs['attention_mask'] = [1] * seq_length + + if 'position_ids' not in encoded_inputs: + encoded_inputs['position_ids'] = list(range(seq_length)) + + if needs_to_be_padded: + difference = max_length - len(required_input) + + if 'attention_mask' in encoded_inputs: + encoded_inputs['attention_mask'] = [ + 0 + ] * difference + encoded_inputs['attention_mask'] + if 'position_ids' in encoded_inputs: + encoded_inputs['position_ids'] = [ + 0 + ] * difference + encoded_inputs['position_ids'] + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + + return encoded_inputs diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/__init__.py b/runtime/Lib/site_packages/models/nlp/codegeex/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0bcdb4bc5e98fe3216141bc78e6ba2a8101f3c6c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/__init__.py @@ -0,0 +1,24 @@ +# Modified by Zhipu.AI +# Original Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING, Union + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .codegeex_for_code_translation import CodeGeeXForCodeTranslation + from .codegeex_for_code_generation import CodeGeeXForCodeGeneration +else: + _import_structure = { + 'codegeex_for_code_translation': ['CodeGeeXForCodeTranslation'], + 'codegeex_for_code_generation': ['CodeGeeXForCodeGeneration'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/codegeex.py b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex.py new file mode 100644 index 0000000000000000000000000000000000000000..f8d43008f68bc522ef089b38811c09b98eef79ca --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex.py @@ -0,0 +1,1030 @@ +# Copyright (c) 2022 Zhipu.AI +import math + +import torch +import torch.nn.functional as F + + +def fast_gelu(x): + """Mindspore's fast gelu implementation.""" + return x / (1 + torch.exp(-1.702 * torch.abs(x))) * torch.exp( + 0.851 * (x - torch.abs(x))) + + +class MLP(torch.nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. At the end, dropout is also + applied. + """ + + def __init__( + self, + hidden_size, + ): + super(MLP, self).__init__() + self.hidden_size = hidden_size + # Project to 4h. + self.dense_h_to_4h = torch.nn.Linear( + self.hidden_size, + 4 * self.hidden_size, + ) + + self.activation_func = fast_gelu + + # Project back to h. + self.dense_4h_to_h = torch.nn.Linear( + 4 * self.hidden_size, + self.hidden_size, + ) + + def forward(self, hidden_states): + # [s, b, 4hp] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output = self.dense_4h_to_h(intermediate_parallel) + + return output + + +class SelfAttention(torch.nn.Module): + """self-attention layer abstract class. + + Self-attention layer takes input with size [b, s, h] + and returns output of the same size. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + layer_number, + fp16=True, + attention_softmax_in_fp32=True, + ): + super(SelfAttention, self).__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.fp16 = fp16 + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.layer_number = max(1, layer_number) + + assert self.hidden_size % self.num_attention_heads == 0 + self.hidden_size_per_attention_head = int(self.hidden_size + // self.num_attention_heads) + + self.query = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.key = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.value = torch.nn.Linear(self.hidden_size, self.hidden_size) + + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + self.softmax = torch.nn.Softmax(dim=-1) + + self.dense = torch.nn.Linear(self.hidden_size, self.hidden_size) + + def forward( + self, + hidden_states, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + # hidden_states: [sq, b, h] + + # ===================== + # Query, Key, and Value + # ===================== + + query_layer = self.query(hidden_states) + key_layer = self.key(hidden_states) + value_layer = self.value(hidden_states) + + new_query_layer_shape = query_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head + ) # noqa + query_layer = query_layer.view(*new_query_layer_shape) + + new_query_layer_shape = key_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head) + key_layer = key_layer.view(*new_query_layer_shape) + + new_query_layer_shape = value_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head + ) # noqa + value_layer = value_layer.view(*new_query_layer_shape) + + # ================================== + # Adjust key and value for inference + # ================================== + + if layer_past is not None: + past_key, past_value = layer_past + key_layer = torch.cat((past_key.type_as(key_layer), key_layer), + dim=0) + value_layer = torch.cat( + (past_value.type_as(value_layer), value_layer), dim=0) + if get_key_value: + present = (key_layer, value_layer) + + # =================================== + # Raw attention scores. [b, np, sq, sk] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.contiguous().view( + output_size[2], output_size[0] * output_size[1], -1) + key_layer = key_layer.contiguous().view( + output_size[3], output_size[0] * output_size[1], -1) + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.matmul( + query_layer.transpose(0, 1), + key_layer.transpose(0, 1).transpose(1, 2)) / self.norm_factor + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + # ================================================== + # Update attention mask for inference. [b, np, sq, sk] + # ================================================== + + if get_key_value: + with torch.no_grad(): + if layer_past is not None: + attention_mask = attention_mask[ + ..., + attention_scores.size(3) + - 1, :attention_scores.size(3)].unsqueeze(2) + else: + attention_mask = attention_mask[ + ..., :attention_scores.size(3), :attention_scores. + size(3)] + + if context_length is not None: + attention_mask = torch.clone(attention_mask) + attention_mask[:, :, context_length:, :] = True + + # attention scores and attention mask [b, np, sq, sk] + # attention_scores = attention_mask_func(attention_scores, attention_mask) + attention_scores = attention_scores - attention_mask * 10000.0 + if self.attention_softmax_in_fp32: + attention_probs = self.softmax(attention_scores.float()).half() + else: + attention_probs = self.softmax(attention_scores) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sq, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + + # change view [sq, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], + output_size[2], -1) + + context_layer = torch.bmm( + attention_probs, + value_layer.unsqueeze(0).transpose(1, 2).squeeze(0)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + # ================= + # Output. [sq, b, h] + # ================= + + output = self.dense(context_layer) + + if get_key_value: + output = [output, present] + + return output + + +class TopQuerySelfAttention(torch.nn.Module): + """Top query self-attention layer abstract class. + + Self-attention layer takes input with size [b, s, h] + and returns output of the same size. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + layer_number, + fp16=True, + attention_softmax_in_fp32=True, + ): + super(TopQuerySelfAttention, self).__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.fp16 = fp16 + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.layer_number = max(1, layer_number) + + assert self.hidden_size % self.num_attention_heads == 0 + self.hidden_size_per_attention_head = int(self.hidden_size + // self.num_attention_heads) + + self.query = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.key = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.value = torch.nn.Linear(self.hidden_size, self.hidden_size) + + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + self.softmax = torch.nn.Softmax(dim=-1) + + self.dense = torch.nn.Linear(self.hidden_size, self.hidden_size) + + def forward( + self, + hidden_states, + query_hidden_state, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + + # hidden_states: [sq, b, h] + query_layer = self.query(query_hidden_state) + key_layer = self.key(hidden_states) + value_layer = self.value(hidden_states) + + new_query_layer_shape = query_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head + ) # noqa + query_layer = query_layer.view(*new_query_layer_shape) + + new_query_layer_shape = key_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head) + key_layer = key_layer.view(*new_query_layer_shape) + + new_query_layer_shape = value_layer.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head + ) # noqa + value_layer = value_layer.view(*new_query_layer_shape) + + # ================================== + # Adjust key and value for inference + # ================================== + + if layer_past is not None: + past_key, past_value = layer_past + key_layer = torch.cat((past_key.type_as(key_layer), key_layer), + dim=0) + value_layer = torch.cat( + (past_value.type_as(value_layer), value_layer), dim=0) + if get_key_value: + present = (key_layer, value_layer) + + # =================================== + # Raw attention scores. [b, np, sq, sk] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [s, b, np, hn] -> [s, b * np, hn] + query_layer = query_layer.contiguous().view( + output_size[2], output_size[0] * output_size[1], -1) + key_layer = key_layer.contiguous().view( + output_size[3], output_size[0] * output_size[1], -1) + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.matmul( + query_layer.transpose(0, 1), + key_layer.transpose(0, 1).transpose(1, 2)) / self.norm_factor + + # change view to [b, np, s, s] + attention_scores = matmul_result.view(*output_size) + + # ================================================== + # Update attention mask for inference. [b, np, sq, sk] + # ================================================== + + if get_key_value: + with torch.no_grad(): + if layer_past is not None: + attention_mask = attention_mask[ + ..., + attention_scores.size(3) + - 1, :attention_scores.size(3)].unsqueeze(2) + else: + attention_mask = attention_mask[ + ..., :attention_scores.size(3), :attention_scores. + size(3)] + + if context_length is not None: + attention_mask = torch.clone(attention_mask) + attention_mask[:, :, context_length:, :] = True + + # attention scores and attention mask [b, np, sq, sk] + # attention_scores = attention_mask_func(attention_scores, attention_mask) + attention_scores = attention_scores - attention_mask * 10000.0 + if self.attention_softmax_in_fp32: + attention_probs = self.softmax(attention_scores.float()).half() + else: + attention_probs = self.softmax(attention_scores) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sq, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + + # change view [sq, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], + output_size[2], -1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm( + attention_probs, + value_layer.unsqueeze(0).transpose(1, 2).squeeze(0)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size,) # noqa + context_layer = context_layer.view(*new_context_layer_shape) + + # ================= + # Output. [sq, b, h] + # ================= + + output = self.dense(context_layer) + + if get_key_value: + output = [output, present] + + return output + + +class TransformerLayer(torch.nn.Module): + """A single transformer layer. + + Transformore layer takes input with size [b, s, h] and returns an + output of the same size. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + layer_number, + layernorm_epsilon=1e-5, + fp16=True, + attention_softmax_in_fp32=True, + ): + super(TransformerLayer, self).__init__() + self.hidden_size = hidden_size + self.layernorm_epsilon = layernorm_epsilon + self.layer_number = layer_number + + # Layernorm on the input data. + self.input_layernorm = torch.nn.LayerNorm( + hidden_size, eps=self.layernorm_epsilon) + + # Self attention. + self.attention = SelfAttention(hidden_size, num_attention_heads, + layer_number, fp16, + attention_softmax_in_fp32) + + # Layernorm on the input data. + self.post_attention_layernorm = torch.nn.LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon) + self.mlp = MLP(self.hidden_size) + + def forward( + self, + hidden_states, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + # hidden_states: [b, s, h] + # Use FP32 for Layernorm + # layernorm_output = self.input_layernorm(hidden_states.float()).half() + layernorm_output = self.input_layernorm(hidden_states) + + # Self attention. + attention_output = self.attention( + layernorm_output, + attention_mask, + layer_past=layer_past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + + if get_key_value: + attention_output, presents = attention_output + + # Residual connection. + residual = hidden_states + layernorm_input = attention_output + residual + + # Use FP32 for Layernorm + # layernorm_output = self.post_attention_layernorm(layernorm_input.float()).half() + layernorm_output = self.post_attention_layernorm(layernorm_input) + mlp_output = self.mlp(layernorm_output) + output = mlp_output + layernorm_input + + if get_key_value: + output = [output, presents] + + return output + + +class TopQueryLayer(torch.nn.Module): + """A single top query layer. + + Top query layer takes input with size [b, s, h] and returns an + output of the same size. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + layer_number, + layernorm_epsilon=1e-5, + ): + super(TopQueryLayer, self).__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.layernorm_epsilon = layernorm_epsilon + self.layer_number = layer_number + + # Use FP32 for Layernorm + self.input_layernorm = torch.nn.LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon) + + # Self attention. + self.attention = TopQuerySelfAttention(self.hidden_size, + self.num_attention_heads, + self.layer_number) + # Layernorm on the input data. + self.post_attention_layernorm = torch.nn.LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon) + + # MLP + self.mlp = MLP(self.hidden_size) + + def forward( + self, + hidden_states, + query_hidden_state, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + # hidden_states: [b, s, h] + assert query_hidden_state != None # noqa + + # Use FP32 for Layernorm + # layernorm_output = self.input_layernorm(hidden_states.float()).half() + layernorm_output = self.input_layernorm(hidden_states) + + # Self attention. + attention_output = self.attention( + layernorm_output, + query_hidden_state, + attention_mask, + layer_past=layer_past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + + if get_key_value: + attention_output, presents = attention_output + + # Residual connection. + residual = hidden_states + layernorm_input = attention_output + residual + + # Use FP32 for Layernorm + # layernorm_output = self.post_attention_layernorm(layernorm_input.float()).half() + layernorm_output = self.post_attention_layernorm(layernorm_input) + + # MLP. + mlp_output = self.mlp(layernorm_output) + + # Second residual connection. + residual = layernorm_input + output = mlp_output + residual + + if get_key_value: + output = [output, presents] + + return output + + +class Transformer(torch.nn.Module): + """Transformer class.""" + + def __init__( + self, + hidden_size, + num_attention_heads, + num_layers, + layernorm_epsilon=1e-5, + ): + super(Transformer, self).__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.layernorm_epsilon = layernorm_epsilon + # Number of layers: + self.num_layers = num_layers + self.num_unique_layers = None + + ################# + assert self.num_unique_layers is None + ################# + + if self.num_unique_layers is None: + self.num_unique_layers = self.num_layers + assert self.num_layers % self.num_unique_layers == 0, \ + 'number of layers should be divisible by number of unique layers' + + # Transformer layers. + def build_layer(layer_number): + return TransformerLayer(self.hidden_size, self.num_attention_heads, + layer_number) + + self.layers = torch.nn.ModuleList( + [build_layer(i + 1) for i in range(self.num_unique_layers)]) + + self.topQueryLayer = TopQueryLayer(self.hidden_size, + self.num_attention_heads, + self.num_unique_layers) + + self.final_layernorm = torch.nn.LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon) + + def _get_layer_index(self, layer_number): + return layer_number % self.num_unique_layers + + def _get_layer(self, layer_number): + return self.layers[self._get_layer_index(layer_number)] + + def forward( + self, + hidden_states, + query_hidden_state, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + # data format change to avoid explicit tranposes : [b s h] --> [s b h] + hidden_states = hidden_states.transpose(0, 1).contiguous() + query_hidden_state = query_hidden_state.transpose(0, 1).contiguous() + + if get_key_value: + presents = [] + for index in range(self.num_layers): + layer = self._get_layer(index) + past = None + if layer_past is not None: + past = layer_past[index] + hidden_states = layer( + hidden_states, + attention_mask, + layer_past=past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + if get_key_value: + hidden_states, present = hidden_states + presents.append(present) + + # Use FP32 for Layernorm + # hidden_states_ = self.final_layernorm(hidden_states.float()).half() + hidden_states_ = self.final_layernorm(hidden_states) + + ################################# + # top query layer + ################################# + past = None + if layer_past is not None: + past = layer_past[self.num_layers] + hidden_states = self.topQueryLayer( + hidden_states_, + query_hidden_state, + attention_mask, + layer_past=past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + + if get_key_value: + hidden_states, present = hidden_states + presents.append(present) + + # reverting data format change [s b h] --> [b s h] + output = hidden_states.transpose(0, 1).contiguous() + + if get_key_value: + output = [output, presents] + + return output + + def state_dict_for_save_checkpoint(self, + destination=None, + prefix='', + keep_vars=False): + return self.state_dict(destination, prefix, keep_vars) + + +class Embedding(torch.nn.Module): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + """ + + def __init__( + self, + hidden_size, + vocab_size, + max_sequence_length, + ): + super(Embedding, self).__init__() + self.hidden_size = hidden_size + self.vocab_size = vocab_size + self.max_sequence_length = max_sequence_length + + # Word embeddings. + self.word_embeddings = torch.nn.Embedding(self.vocab_size, + self.hidden_size) + self._word_embeddings_key = 'word_embeddings' + + # Position embedding. + self.position_embeddings = torch.nn.Embedding(self.max_sequence_length, + self.hidden_size) + self.position_embeddings = self.position_embeddings.half() + self._position_embeddings_key = 'position_embeddings' + + def forward(self, input_ids, position_ids): + # Embeddings. + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + embeddings = words_embeddings + position_embeddings + + return embeddings + + def state_dict_for_save_checkpoint(self, + destination=None, + prefix='', + keep_vars=False): + """For easy load.""" + + state_dict_ = {} + state_dict_[self._word_embeddings_key] \ + = self.word_embeddings.state_dict(destination, prefix, keep_vars) + state_dict_[self._position_embeddings_key] \ + = self.position_embeddings.state_dict( + destination, prefix, keep_vars) + + return state_dict_ + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + # Word embedding. + if self._word_embeddings_key in state_dict: + state_dict_ = state_dict[self._word_embeddings_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'word_embeddings' in key: + state_dict_[key.split('word_embeddings.')[1]] \ + = state_dict[key] + state_dict_['weight'] = state_dict_['weight'][:self.vocab_size] + self.word_embeddings.load_state_dict(state_dict_, strict=strict) + + # Position embedding. + if self._position_embeddings_key in state_dict: + state_dict_ = state_dict[self._position_embeddings_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'position_embeddings' in key: + state_dict_[key.split('position_embeddings.')[1]] \ + = state_dict[key] + self.position_embeddings.load_state_dict(state_dict_, strict=strict) + + +class QueryEmbedding(torch.nn.Module): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + """ + + def __init__( + self, + hidden_size, + vocab_size, + max_sequence_length, + ): + super(QueryEmbedding, self).__init__() + + self.hidden_size = hidden_size + self.vocab_size = vocab_size + self.max_sequence_length = max_sequence_length + + # Top query position embedding (serial). + self.top_query_embeddings = torch.nn.Embedding( + self.max_sequence_length, self.hidden_size) + self.top_query_embeddings = self.top_query_embeddings.half() + self._top_query_embeddings_key = 'top_query_embeddings' + + def forward(self, position_ids): + # Embeddings. + embeddings = self.top_query_embeddings(position_ids) + + return embeddings + + def state_dict_for_save_checkpoint(self, + destination=None, + prefix='', + keep_vars=False): + """For easy load.""" + + state_dict_ = {} + state_dict_[self._top_query_embeddings_key] \ + = self.top_query_embeddings.state_dict( + destination, prefix, keep_vars) + + return state_dict_ + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + # Position embedding. + if self._top_query_embeddings_key in state_dict: + state_dict_ = state_dict[self._top_query_embeddings_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'top_query_embeddings' in key: + state_dict_[key.split('top_query_embeddings.')[1]] \ + = state_dict[key] + self.top_query_embeddings.load_state_dict(state_dict_, strict=strict) + + +class TransformerLanguageModel(torch.nn.Module): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + attention_mask_func: a function that takes `unmaksed-attention-scores` + with size [b, np, s, s] and an `attention-mask` and will apply + the masking. The function should return a masked score of the + same size [b, np, s, s]. + masked-attention-scores = attention_mask_func( + unmaksed-attention-scores, attention-mask) + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + """ + + def __init__( + self, + hidden_size, + num_layers, + num_attention_heads, + padded_vocab_size, + max_position_embeddings, + ): + super(TransformerLanguageModel, self).__init__() + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.padded_vocab_size = padded_vocab_size + self.max_position_embeddings = max_position_embeddings + + # Embeddings + self.embedding = Embedding(self.hidden_size, self.padded_vocab_size, + self.max_position_embeddings) + self._embedding_key = 'embedding' + + # Query embeddings + self.topQueryEmbedding = QueryEmbedding(self.hidden_size, + self.padded_vocab_size, + self.max_position_embeddings) + self._topQueryEmbedding_key = 'topQueryEmbedding' + + # Transformer + self.transformer = Transformer(self.hidden_size, + self.num_attention_heads, + self.num_layers) + self._transformer_key = 'transformer' + + def forward( + self, + input_ids, + position_ids, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + + # Embeddings. + embedding_output = self.embedding(input_ids, position_ids) + query_position_ids = position_ids + queryEmbedding_out = self.topQueryEmbedding(query_position_ids) + + # Transformer. + transformer_output = self.transformer( + embedding_output, + queryEmbedding_out, + attention_mask, + layer_past=layer_past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + + return transformer_output + + def state_dict_for_save_checkpoint(self, + destination=None, + prefix='', + keep_vars=False): + """For easy load.""" + + state_dict_ = {} + state_dict_[self._embedding_key] \ + = self.embedding.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + state_dict_[self._topQueryEmbedding_key] \ + = self.topQueryEmbedding.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + state_dict_[self._transformer_key] \ + = self.transformer.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + + return state_dict_ + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + # Embedding. + if self._embedding_key in state_dict: + state_dict_ = state_dict[self._embedding_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if '_embeddings' in key: + state_dict_[key] = state_dict[key] + self.embedding.load_state_dict(state_dict_, strict=strict) + + if self._topQueryEmbedding_key in state_dict: + state_dict_ = state_dict[self._topQueryEmbedding_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if '_embeddings' in key: + state_dict_[key] = state_dict[key] + self.topQueryEmbedding.load_state_dict(state_dict_, strict=strict) + + # Transformer. + if self._transformer_key in state_dict: + state_dict_ = state_dict[self._transformer_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'transformer.' in key: + state_dict_[key.split('transformer.')[1]] = state_dict[key] + self.transformer.load_state_dict(state_dict_, strict=strict) + + +class CodeGeeXModel(torch.nn.Module): + """CodeGeeX: A Multilingual Code Generation Model.""" + + def __init__( + self, + hidden_size, + num_layers, + num_attention_heads, + padded_vocab_size, + max_position_embeddings, + ): + super(CodeGeeXModel, self).__init__() + + self.language_model = TransformerLanguageModel( + hidden_size, num_layers, num_attention_heads, padded_vocab_size, + max_position_embeddings) + self._language_model_key = 'language_model' + + def forward( + self, + input_ids, + position_ids, + attention_mask, + layer_past=None, + get_key_value=False, + prompt_length=None, + context_length=None, + ): + # Language model. + lm_output = self.language_model( + input_ids, + position_ids, + attention_mask, + layer_past=layer_past, + get_key_value=get_key_value, + prompt_length=prompt_length, + context_length=context_length) + + if get_key_value: + lm_output, presents = lm_output + + output = F.linear( + lm_output, + self.language_model.embedding.word_embeddings.weight.half()) + + if get_key_value: + output = [output, presents] + + return output + + def state_dict_for_save_checkpoint(self, + destination=None, + prefix='', + keep_vars=False): + + state_dict_ = {} + state_dict_[self._language_model_key] \ + = self.language_model.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + return state_dict_ + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + if self._language_model_key in state_dict: + state_dict = state_dict[self._language_model_key] + self.language_model.load_state_dict(state_dict, strict=strict) diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_generation.py b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..ff191cbaf83e2970e49d37bbf4027609f85ef2a3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_generation.py @@ -0,0 +1,110 @@ +# Copyright (c) 2022 Zhipu.AI +import copy +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .codegeex import CodeGeeXModel +from .inference import get_token_stream +from .tokenizer import CodeGeeXTokenizer + + +def model_provider(): + """Build the model.""" + + hidden_size = 5120 + num_attention_heads = 40 + num_layers = 39 + padded_vocab_size = 52224 + max_position_embeddings = 2048 + + model = CodeGeeXModel(hidden_size, num_layers, num_attention_heads, + padded_vocab_size, max_position_embeddings) + + return model + + +@MODELS.register_module(Tasks.code_generation, module_name=Models.codegeex) +class CodeGeeXForCodeGeneration(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the fast poem model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + logger = get_logger() + # loading tokenizer + logger.info('Loading tokenizer ...') + self.tokenizer = CodeGeeXTokenizer( + tokenizer_path=model_dir + '/tokenizer', mode='codegeex-13b') + # loading model + state_dict_path = model_dir + '/ckpt_ms_213000_fp32_52224.pt' + logger.info('Loading state dict ...') + state_dict = torch.load(state_dict_path, map_location='cpu') + state_dict = state_dict['module'] + + logger.info('Building CodeGeeX model ...') + self.model = model_provider() + self.model.load_state_dict(state_dict) + self.model.eval() + self.model.half() + self.model.cuda() + + def forward(self, input: Dict[str, str]) -> Dict[str, str]: + micro_batch_size = 1 + seq_length = 2048 + out_seq_length = 256 + bad_ids = None + lang = input['language'] + prompt = input['prompt'] + prompt = f'# language: {lang}\n{prompt}' + logger = get_logger() + tokenizer = self.tokenizer + model = self.model + for prompt in [prompt]: + tokens = tokenizer.encode_code(prompt) + n_token_prompt = len(tokens) + token_stream = get_token_stream( + model, + tokenizer, + seq_length, + out_seq_length, + [copy.deepcopy(tokens) for _ in range(micro_batch_size)], + micro_batch_size=micro_batch_size, + bad_ids=bad_ids, + topk=1, + topp=0.9, + temperature=0.9, + greedy=True) + is_finished = [False for _ in range(micro_batch_size)] + for i, generated in enumerate(token_stream): + generated_tokens = generated[0] + for j in range(micro_batch_size): + if is_finished[j]: + continue + if generated_tokens[j].cpu().numpy( + )[-1] == tokenizer.eos_token_id or len( + generated_tokens[j]) >= out_seq_length: + is_finished[j] = True + generated_tokens_ = generated_tokens[j].cpu().numpy( + ).tolist() + generated_code = tokenizer.decode_code( + generated_tokens_[n_token_prompt:]) + generated_code = ''.join(generated_code) + logger.info( + '================================= Generated code:' + ) + logger.info(generated_code) + if all(is_finished): + break + + logger.info('Generation finished.') + return {OutputKeys.TEXT: generated_code} diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_translation.py b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_translation.py new file mode 100644 index 0000000000000000000000000000000000000000..fece907d11220c425306ba5bdbdf4ee7ee399a0d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/codegeex_for_code_translation.py @@ -0,0 +1,109 @@ +# Copyright (c) 2022 Zhipu.AI +import copy +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .codegeex import CodeGeeXModel +from .inference import get_token_stream +from .tokenizer import CodeGeeXTokenizer + + +def model_provider(): + """Build the model.""" + + hidden_size = 5120 + num_attention_heads = 40 + num_layers = 39 + padded_vocab_size = 52224 + max_position_embeddings = 2048 + + model = CodeGeeXModel(hidden_size, num_layers, num_attention_heads, + padded_vocab_size, max_position_embeddings) + + return model + + +@MODELS.register_module(Tasks.code_translation, module_name=Models.codegeex) +class CodeGeeXForCodeTranslation(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the fast poem model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + logger = get_logger() + # loading tokenizer + logger.info('Loading tokenizer ...') + self.tokenizer = CodeGeeXTokenizer( + tokenizer_path=model_dir + '/tokenizer', mode='codegeex-13b') + # loading model + state_dict_path = model_dir + '/ckpt_ms_translation_0817.pt' + logger.info('Loading state dict ...') + state_dict = torch.load(state_dict_path, map_location='cpu') + state_dict = state_dict['module'] + + logger.info('Building CodeGeeX model ...') + self.model = model_provider() + self.model.load_state_dict(state_dict) + self.model.eval() + self.model.half() + self.model.cuda() + + def forward(self, input: Dict[str, str]) -> Dict[str, str]: + micro_batch_size = 1 + seq_length = 2048 + out_seq_length = 256 + bad_ids = None + src_lang = input['source language'] + dst_lang = input['target language'] + prompt = input['prompt'] + prompt = f'code translation\n{src_lang}:\n{prompt}\n{dst_lang}:\n' + logger = get_logger() + tokenizer = self.tokenizer + model = self.model + for prompt in [prompt]: + tokens = tokenizer.encode_code(prompt) + n_token_prompt = len(tokens) + token_stream = get_token_stream( + model, + tokenizer, + seq_length, + out_seq_length, + [copy.deepcopy(tokens) for _ in range(micro_batch_size)], + micro_batch_size=micro_batch_size, + bad_ids=bad_ids, + greedy=True, + ) + is_finished = [False for _ in range(micro_batch_size)] + for i, generated in enumerate(token_stream): + generated_tokens = generated[0] + for j in range(micro_batch_size): + if is_finished[j]: + continue + if generated_tokens[j].cpu().numpy( + )[-1] == tokenizer.eos_token_id or len( + generated_tokens[j]) >= out_seq_length: + is_finished[j] = True + generated_tokens_ = generated_tokens[j].cpu().numpy( + ).tolist() + generated_code = tokenizer.decode_code( + generated_tokens_[n_token_prompt:]) + generated_code = ''.join(generated_code) + logger.info( + '================================= Generated code:' + ) + logger.info(generated_code) + if all(is_finished): + break + + logger.info('Generation finished.') + return {OutputKeys.TEXT: generated_code} diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/inference.py b/runtime/Lib/site_packages/models/nlp/codegeex/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..38f14d6cc386254d7923a90950aec5900ba77efe --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/inference.py @@ -0,0 +1,301 @@ +# Copyright (c) 2022 Zhipu.AI + +from typing import List + +import torch +import torch.nn.functional as F + + +def get_ltor_masks_and_position_ids( + data, + eod_token, + reset_position_ids, + reset_attention_mask, +): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size() + + # Attention mask (lower triangular). + if reset_attention_mask: + att_mask_batch = micro_batch_size + else: + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), + device=data.device)).view(att_mask_batch, 1, seq_length, + seq_length) + + # Position ids. + position_ids = torch.arange( + seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data) + # We need to clone as the ids will be modifed based on batch index. + if reset_position_ids: + position_ids = position_ids.clone() + + if reset_position_ids or reset_attention_mask: + # Loop through the batches: + for b in range(micro_batch_size): + + # Find indecies where EOD token is. + eod_index = position_ids[b, data[b] == eod_token] + # Detach indecies from positions if going to modify positions. + if reset_position_ids: + eod_index = eod_index.clone() + + # Loop through EOD indecies: + prev_index = 0 + for j in range(eod_index.size()[0]): + i = eod_index[j] + # Mask attention loss. + if reset_attention_mask: + attention_mask[b, 0, (i + 1):, :(i + 1)] = 0 + # Reset positions. + if reset_position_ids: + position_ids[b, (i + 1):] -= i + 1 - prev_index + prev_index = i + 1 + + # Convert attention mask to binary: + attention_mask = attention_mask < 0.5 + + return attention_mask, position_ids + + +def get_batch( + context_tokens, + micro_batch_size, + eod_token, + reset_position_ids=False, + reset_attention_mask=False, +): + """Generate batch from context tokens.""" + tokens = context_tokens.view(micro_batch_size, -1).contiguous().cuda() + # Get the attention mask and postition ids. + attention_mask, position_ids = get_ltor_masks_and_position_ids( + tokens, + eod_token, + reset_position_ids, + reset_attention_mask, + ) + + return tokens, attention_mask, position_ids + + +def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')): + """This function has been mostly taken from huggingface conversational + ai code at + https://medium.com/huggingface/how-to-build-a-state-of-the-art- + conversational-ai-with-transfer-learning-2d818ac26313""" + + if top_k > 0: + # Remove all tokens with a probability less than the + # last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p > 0.0: + # Cconvert to 1D + sorted_logits, sorted_indices = torch.sort( + logits, descending=True, dim=-1) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep also the first token + # above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + for i in range(sorted_indices.size(0)): + indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]] + logits[i][indices_to_remove] = filter_value + + return logits + + +def pad_batch(batch, pad_id, seq_length): + context_lengths = [] + for tokens in batch: + context_length = len(tokens) + if context_length < seq_length: + tokens.extend([pad_id] * (seq_length - context_length)) + context_lengths.append(context_length) + return batch, context_lengths + + +def get_token_stream( + model, + tokenizer, + seq_length, + out_seq_length, + context_tokens, + return_scores: bool = False, + prompt_length: int = None, + micro_batch_size: int = None, + bad_ids: List = None, + temperature: float = 1.0, + topp: float = 1.0, + topk: int = 0.0, + greedy: bool = False, +): + context_tokens, context_lengths = pad_batch(context_tokens, + tokenizer.eos_token_id, + seq_length) + + context_tokens_tensor = torch.cuda.LongTensor(context_tokens) + context_length_tensor = torch.cuda.LongTensor(context_lengths) + context_length = context_length_tensor.min().item() + tokens, attention_mask, position_ids = get_batch( + context_tokens_tensor, + micro_batch_size, + tokenizer.eos_token_id, + ) + + batch_token_iterator = sample_sequence_batch( + model, + tokenizer, + context_tokens_tensor, + context_length_tensor, + attention_mask, + position_ids, + seq_length=seq_length, + out_seq_length=out_seq_length, + return_scores=return_scores, + prompt_length=prompt_length, + bad_ids=bad_ids, + temperature=temperature, + topp=topp, + topk=topk, + greedy=greedy, + ) + + for tokens, lengths in batch_token_iterator: + context_length += 1 + if tokens is not None: + yield tokens[:, :context_length], lengths + else: + yield None, None + + +def switch(val1, val2, boolean): + boolean = boolean.type_as(val1) + return (1 - boolean) * val1 + boolean * val2 + + +def sample_sequence_batch( + model, + tokenizer, + context_tokens, + context_lengths, + attention_mask, + position_ids, + seq_length, + out_seq_length, + maxlen=None, + return_scores: bool = False, + prompt_length: int = None, + bad_ids: List = None, + temperature: float = 1.0, + topp: float = 1.0, + topk: int = 0.0, + recompute: bool = False, + greedy: bool = False, +): + model.eval() + with torch.no_grad(): + context_length = context_lengths.min().item() + eos_id = tokenizer.eos_token_id + + counter = 0 + org_context_length = context_length + + layer_past = None + batch_size = context_tokens.size(0) + is_done = torch.zeros([batch_size]).byte().cuda() + tokens = context_tokens + if maxlen is None: + maxlen = seq_length - 1 + if maxlen > (org_context_length + out_seq_length): + maxlen = org_context_length + out_seq_length + + lengths = torch.ones([batch_size]).long().cuda() * maxlen + if return_scores: + scores = torch.zeros([batch_size]).float().cuda() + + while context_length <= (maxlen): + + if recompute: + logits = model( + tokens, + position_ids, + attention_mask, + prompt_length=prompt_length, + context_length=context_length, + ) + logits = logits[:, context_length - 1, :] + else: + if counter == 0: + tokens2use = tokens[:, :context_length] + positions2use = position_ids[:, :context_length] + else: + tokens2use = tokens[:, context_length - 1].view( + batch_size, -1) + positions2use = position_ids[:, context_length - 1].view( + batch_size, -1) + logits, layer_past = model( + tokens2use, + positions2use, + attention_mask, + layer_past=layer_past, + get_key_value=True, + prompt_length=prompt_length, + context_length=context_length, + ) + logits = logits[:, -1].view(batch_size, -1).contiguous() + + if bad_ids is not None: + for bad_id in bad_ids: + logits[:, bad_id] = -10000 + if greedy: + prev = torch.argmax(logits, dim=-1).view(-1) + else: + logits = logits.float() + if return_scores: + orig_log_probs = torch.log_softmax(logits, dim=-1) + logits /= temperature + logits = top_k_logits(logits, top_k=topk, top_p=topp) + log_probs = F.softmax(logits, dim=-1) + prev = torch.multinomial(log_probs, num_samples=1).view(-1) + + started = context_lengths <= context_length + + new_tokens = switch(tokens[:, context_length].view(-1), prev, + started) + + if not greedy and return_scores: + indices = prev.view(-1, 1) + new_scores = orig_log_probs.gather(1, indices).view(-1) + new_scores = new_scores * started + new_scores = new_scores * is_done.bool().logical_not() + scores += new_scores + + tokens[:, context_length] = new_tokens + done_token = (prev == eos_id).byte() & started.byte() + just_finished = (done_token & ~is_done).bool() + lengths[just_finished.view(-1)] = context_length + is_done = is_done | done_token + done = torch.all(is_done) + + if return_scores: + yield tokens, (lengths, scores) + else: + yield tokens, lengths + + context_length += 1 + counter += 1 + if done: + break diff --git a/runtime/Lib/site_packages/models/nlp/codegeex/tokenizer.py b/runtime/Lib/site_packages/models/nlp/codegeex/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..a5da9a3c8303087c24fdc0caf7dca2e96208023b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/codegeex/tokenizer.py @@ -0,0 +1,187 @@ +# Copyright (c) 2022 Zhipu.AI +from typing import List, Union + +import torch +from transformers import AutoTokenizer +from transformers.models.gpt2 import GPT2TokenizerFast + + +def encode_whitespaces(text, start_extra_id: int, max_len: int): + """ Encode whitespaces to extra tokens in GPT-J. + + >>> encode_whitespaces('a\\n b\\n c', 10, 10) + 'a\\n<|extratoken_10|>b\\n<|extratoken_11|>c' + """ + + def push_acc_space(acc_len: int, text: str): + if acc_len == 0: + return text + if acc_len == 1: + return text + ' ' + assert acc_len <= max_len, f'Max whitespace run length {max_len}, but found {acc_len}' + extra_id = start_extra_id - 2 + acc_len + extra_token = f'<|extratoken_{extra_id}|>' + return text + extra_token + + acc_len = 0 + res = '' + for ch in text: + if ch == ' ': + acc_len += 1 + if acc_len == max_len: + res = push_acc_space(acc_len, res) + acc_len = 0 + else: + res = push_acc_space(acc_len, res) + acc_len = 0 + res = res + ch + + res = push_acc_space(acc_len, res) + + return res + + +def decode_whitespaces(text: str, start_extra_id: int, max_len: int): + """ Decode the whitespace-encoded strings produced by encode_whitespace. + + >>> text = 'a\\n b\\n c' + >>> s, l = 10, 10 + >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l) + True + """ + for l in range(2, max_len + 1): # noqa + token_id = start_extra_id - 2 + l + token = f'<|extratoken_{token_id}|>' + text = text.replace(token, ' ' * l) + return text + + +class Code13BDictionary(object): + + def __init__( + self, + dict_file: str, + extra_token_ids: List[str] = None, + pad_to_vocab_size: int = -1, + ): + self._idx = dict() + self._count = dict() + self._num_symbols = 0 + self._symbols = [] + + self._add_symbol('', 0) + self._add_symbol('', 0) + self._add_symbol('', 0) + self._add_symbol('', 0) + self._load_dict(dict_file) + + if extra_token_ids is None: + extra_token_ids = [str(x) for x in range(50257, 50400) + ] # follows GPT-J settings + + for token_id in extra_token_ids: + self._add_symbol(token_id, 0) + + if pad_to_vocab_size > 0: + self._pad_to_vocab_size(pad_to_vocab_size) + + def _pad_to_vocab_size(self, vocab_size: int): + num_pad = vocab_size - len(self) + if num_pad <= 0: + return + for i in range(1, num_pad + 1): + self._add_symbol('vocab_pad_token{}'.format(i), 0) + + def _load_dict(self, dict_file: str): + with open(dict_file, 'r') as f: + for line in f: + line = line.strip() + if line == '' or line.startswith('#'): + continue + sym, count = line.split() + self._add_symbol(sym, int(count)) + + def _add_symbol(self, sym: str, count: int): + self._idx[sym] = self._num_symbols + self._count[sym] = count + self._symbols.append(sym) + self._num_symbols += 1 + + def __len__(self): + return self._num_symbols + + def index(self, sym: str): + return self._idx[sym] + + def string(self, idx: int): + return self._symbols[idx] + + def map_token(self, token: Union[int, str]): + if isinstance(token, int): + token = str(token) + return self.index(token) + + def map_tokens(self, tokens): + return [self.map_token(token) for token in tokens] + + def decode_tokens(self, tokens): + decoded = [ + '50256' if token == 50256 else self.string(token) + for token in tokens + ] + return [int(x) for x in decoded if not x.startswith('vocab_pad_token')] + + +class CodeGeeXTokenizer(object): + + def __init__( + self, + tokenizer: GPT2TokenizerFast = None, + tokenizer_path: str = 'EleutherAI/gpt-j-6B', + start_extra_id: int = 10, + max_len: int = 10, + mode='codegeex-13b', + dict_file: str = None, + ): + self.tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained( + tokenizer_path) + if mode not in ['codegeex-13b', 'codegeex-python-13b']: + raise ValueError( + f"Invalid mode {mode}, choose from ['codegeex-13b', 'codegeex-python-13b']" + ) + self.start_extra_id = start_extra_id + self.max_len = max_len + self.mode = mode + if dict_file is not None: + self.code_dict = Code13BDictionary( + dict_file, pad_to_vocab_size=51200 + ) if self.mode == 'codegeex-python-13b' else None + else: + self.code_dict = None + self.eos_token_id = self.tokenizer.eos_token_id + + def encode_code(self, code: str): + if self.mode == 'codegeex-13b': + code = encode_whitespaces(code, self.start_extra_id, self.max_len) + input_ids = self.tokenizer( + code, is_split_into_words=False).input_ids + + elif self.mode == 'codegeex-python-13b': + code = encode_whitespaces(code, self.start_extra_id, self.max_len) + input_ids = self.code_dict.map_tokens(self.tokenizer.encode(code)) + input_ids = torch.LongTensor(input_ids).reshape(1, -1) + + return input_ids + + def decode_code(self, input_ids): + if self.mode == 'codegeex-13b': + text = self.tokenizer.decode(input_ids, skip_special_tokens=False) + output_code = decode_whitespaces(text, self.start_extra_id, + self.max_len) + elif self.mode == 'codegeex-python-13b': + input_ids = [self.code_dict.decode_tokens(input_ids.tolist()[0])] + text = self.tokenizer.decode(input_ids, skip_special_tokens=False) + output_code = decode_whitespaces(text, self.start_extra_id, + self.max_len) + + return output_code diff --git a/runtime/Lib/site_packages/models/nlp/csanmt/__init__.py b/runtime/Lib/site_packages/models/nlp/csanmt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..85531617faba07a526a692a8457881a510d6071d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/csanmt/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .translation import CsanmtForTranslation diff --git a/runtime/Lib/site_packages/models/nlp/csanmt/translation.py b/runtime/Lib/site_packages/models/nlp/csanmt/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..6ece4255e4c8a7d1f0582d837d82f1d1936d4aeb --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/csanmt/translation.py @@ -0,0 +1,1574 @@ +# Part of the implementation is borrowed and modified from THUMT, +# publicly available at https://github.com/THUNLP-MT/THUMT +# Copyright 2017-2022 The Alibaba MT Team Authors. All rights reserved. +import math +from collections import namedtuple +from typing import Dict + +import tensorflow as tf + +from modelscope.metainfo import Models +from modelscope.models.base import Model, Tensor +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks + +__all__ = ['CsanmtForTranslation'] + + +@MODELS.register_module(Tasks.translation, module_name=Models.translation) +class CsanmtForTranslation(Model): + + def __init__(self, model_dir, *args, **kwargs): + """ + Args: + params (dict): the model configuration. + """ + super().__init__(model_dir, *args, **kwargs) + self.params = kwargs + print(self.params) + + def __call__(self, + input: Dict[str, Tensor], + label: Dict[str, Tensor] = None, + prefix: Dict[str, Tensor] = None, + prefix_hit: Dict[bool, Tensor] = None) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input: the preprocessed input source sequence + label: the ground truth target data for model training + prefix: the preprocessed input target prefix sequence for interactive translation + prefix_hit: the preprocessed target prefix subword vector for interactive translation + + Returns: + output_seqs: output sequence of target ids + """ + if label is None: + with tf.compat.v1.variable_scope('NmtModel'): + output_seqs, output_scores = self.beam_search( + { + 'input_wids': input, + 'prefix_wids': prefix, + 'prefix_hit': prefix_hit + }, self.params) + return { + 'output_seqs': output_seqs, + 'output_scores': output_scores, + } + else: + train_op, loss = self.transformer_model_train_fn(input, label) + return { + 'train_op': train_op, + 'loss': loss, + } + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """ + Run the forward pass for a model. + + Args: + input (Dict[str, Tensor]): the dict of the model inputs for the forward method + + Returns: + Dict[str, Tensor]: output from the model forward pass + """ + ... + + def encoding_graph(self, features, params): + src_vocab_size = params['src_vocab_size'] + hidden_size = params['hidden_size'] + + initializer = tf.compat.v1.random_normal_initializer( + 0.0, hidden_size**-0.5, dtype=tf.float32) + + if params['shared_source_target_embedding']: + with tf.compat.v1.variable_scope( + 'Shared_Embedding', reuse=tf.compat.v1.AUTO_REUSE): + src_embedding = tf.compat.v1.get_variable( + 'Weights', [src_vocab_size, hidden_size], + initializer=initializer) + else: + with tf.compat.v1.variable_scope('Source_Embedding'): + src_embedding = tf.compat.v1.get_variable( + 'Weights', [src_vocab_size, hidden_size], + initializer=initializer) + src_bias = tf.compat.v1.get_variable('encoder_input_bias', + [hidden_size]) + + eos_padding = tf.zeros_like(features, dtype=tf.int64)[:, :1] + src_seq = tf.concat([features, eos_padding], 1) + src_mask = tf.cast(tf.not_equal(src_seq, 0), dtype=tf.float32) + shift_src_mask = src_mask[:, :-1] + shift_src_mask = tf.pad( + tensor=shift_src_mask, + paddings=[[0, 0], [1, 0]], + constant_values=1) + + encoder_input = tf.gather(src_embedding, tf.cast(src_seq, tf.int32)) + encoder_input = encoder_input * (hidden_size**0.5) + if params['position_info_type'] == 'absolute': + encoder_input = add_timing_signal(encoder_input) + encoder_input = tf.multiply(encoder_input, + tf.expand_dims(shift_src_mask, 2)) + + encoder_input = tf.nn.bias_add(encoder_input, src_bias) + encoder_self_attention_bias = attention_bias(shift_src_mask, 'masking') + + if params['residual_dropout'] > 0.0: + encoder_input = tf.nn.dropout( + encoder_input, rate=params['residual_dropout']) + + # encode + encoder_output = transformer_encoder(encoder_input, + encoder_self_attention_bias, + shift_src_mask, params) + return encoder_output, encoder_self_attention_bias + + def semantic_encoding_graph(self, features, params, name=None): + hidden_size = params['hidden_size'] + initializer = tf.compat.v1.random_normal_initializer( + 0.0, hidden_size**-0.5, dtype=tf.float32) + scope = None + if params['shared_source_target_embedding']: + vocab_size = params['src_vocab_size'] + scope = 'Shared_Semantic_Embedding' + elif name == 'source': + vocab_size = params['src_vocab_size'] + scope = 'Source_Semantic_Embedding' + elif name == 'target': + vocab_size = params['trg_vocab_size'] + scope = 'Target_Semantic_Embedding' + else: + raise ValueError('error: no right name specified.') + + with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE): + embedding_mat = tf.compat.v1.get_variable( + 'Weights', [vocab_size, hidden_size], initializer=initializer) + + eos_padding = tf.zeros_like(features, dtype=tf.int64)[:, :1] + input_seq = tf.concat([features, eos_padding], 1) + input_mask = tf.cast(tf.not_equal(input_seq, 0), dtype=tf.float32) + shift_input_mask = input_mask[:, :-1] + shift_input_mask = tf.pad( + tensor=shift_input_mask, + paddings=[[0, 0], [1, 0]], + constant_values=1) + + encoder_input = tf.gather(embedding_mat, tf.cast(input_seq, tf.int32)) + encoder_input = encoder_input * (hidden_size**0.5) + encoder_input = tf.multiply(encoder_input, + tf.expand_dims(shift_input_mask, 2)) + + encoder_self_attention_bias = attention_bias(shift_input_mask, + 'masking') + + if params['residual_dropout'] > 0.0: + encoder_input = tf.nn.dropout( + encoder_input, rate=params['residual_dropout']) + + # encode + encoder_output = transformer_semantic_encoder( + encoder_input, encoder_self_attention_bias, shift_input_mask, + params) + return encoder_output + + def build_contrastive_training_graph(self, features, labels, params): + # representations + source_name = 'source' + target_name = 'target' + if params['shared_source_target_embedding']: + source_name = None + target_name = None + feature_output = self.semantic_encoding_graph( + features, params, name=source_name) + label_output = self.semantic_encoding_graph( + labels, params, name=target_name) + + return feature_output, label_output + + def MGMC_sampling(self, x_embedding, y_embedding, params, epsilon=1e-12): + K = params['num_of_samples'] + eta = params['eta'] + assert K % 2 == 0 + + def get_samples(x_vector, y_vector): + bias_vector = y_vector - x_vector + w_r = tf.math.divide( + tf.abs(bias_vector) - tf.reduce_min( + input_tensor=tf.abs(bias_vector), axis=2, keepdims=True) + + epsilon, + tf.reduce_max( + input_tensor=tf.abs(bias_vector), axis=2, keepdims=True) + - tf.reduce_min( + input_tensor=tf.abs(bias_vector), axis=2, keepdims=True) + + 2 * epsilon) + + R = [] + for i in range(K // 2): + omega = eta * tf.random.normal(tf.shape(input=bias_vector), 0.0, w_r) + \ + (1.0 - eta) * tf.random.normal(tf.shape(input=bias_vector), 0.0, 1.0) + sample = x_vector + omega * bias_vector + R.append(sample) + return R + + ALL_SAMPLES = [] + ALL_SAMPLES = get_samples(x_embedding, y_embedding) + ALL_SAMPLES.extend(get_samples(y_embedding, x_embedding)) + + assert len(ALL_SAMPLES) == K + + return tf.concat(ALL_SAMPLES, axis=0) + + def decoding_graph(self, + encoder_output, + encoder_self_attention_bias, + labels, + params={}, + embedding_augmentation=None): + trg_vocab_size = params['trg_vocab_size'] + hidden_size = params['hidden_size'] + + initializer = tf.compat.v1.random_normal_initializer( + 0.0, hidden_size**-0.5, dtype=tf.float32) + + if params['shared_source_target_embedding']: + with tf.compat.v1.variable_scope( + 'Shared_Embedding', reuse=tf.compat.v1.AUTO_REUSE): + trg_embedding = tf.compat.v1.get_variable( + 'Weights', [trg_vocab_size, hidden_size], + initializer=initializer) + else: + with tf.compat.v1.variable_scope('Target_Embedding'): + trg_embedding = tf.compat.v1.get_variable( + 'Weights', [trg_vocab_size, hidden_size], + initializer=initializer) + + eos_padding = tf.zeros_like(labels, dtype=tf.int64)[:, :1] + trg_seq = tf.concat([labels, eos_padding], 1) + trg_mask = tf.cast(tf.not_equal(trg_seq, 0), dtype=tf.float32) + shift_trg_mask = trg_mask[:, :-1] + shift_trg_mask = tf.pad( + tensor=shift_trg_mask, + paddings=[[0, 0], [1, 0]], + constant_values=1) + + decoder_input = tf.gather(trg_embedding, tf.cast(trg_seq, tf.int32)) + + decoder_input *= hidden_size**0.5 + decoder_self_attention_bias = attention_bias( + tf.shape(input=decoder_input)[1], 'causal') + decoder_input = tf.pad( + tensor=decoder_input, paddings=[[0, 0], [1, 0], [0, 0]])[:, :-1, :] + if params['position_info_type'] == 'absolute': + decoder_input = add_timing_signal(decoder_input) + + decoder_input = tf.nn.dropout( + decoder_input, rate=1 - (1.0 - params['residual_dropout'])) + + # training + decoder_output, attention_weights = transformer_decoder( + decoder_input, + encoder_output, + decoder_self_attention_bias, + encoder_self_attention_bias, + states_key=None, + states_val=None, + embedding_augmentation=embedding_augmentation, + params=params) + + logits = self.prediction(decoder_output, params) + + on_value = params['confidence'] + off_value = (1.0 - params['confidence']) / tf.cast( + trg_vocab_size - 1, dtype=tf.float32) + soft_targets = tf.one_hot( + tf.cast(trg_seq, tf.int32), + depth=trg_vocab_size, + on_value=on_value, + off_value=off_value) + mask = tf.cast(shift_trg_mask, logits.dtype) + xentropy = tf.nn.softmax_cross_entropy_with_logits( + logits=logits, labels=tf.stop_gradient(soft_targets)) * mask + loss = tf.reduce_sum(input_tensor=xentropy) / tf.reduce_sum( + input_tensor=mask) + + return loss + + def build_training_graph(self, + features, + labels, + params, + feature_embedding=None, + label_embedding=None): + # encode + encoder_output, encoder_self_attention_bias = self.encoding_graph( + features, params) + embedding_augmentation = None + if feature_embedding is not None and label_embedding is not None: + embedding_augmentation = self.MGMC_sampling( + feature_embedding, label_embedding, params) + + encoder_output = tf.tile(encoder_output, + [params['num_of_samples'], 1, 1]) + encoder_self_attention_bias = tf.tile( + encoder_self_attention_bias, + [params['num_of_samples'], 1, 1, 1]) + labels = tf.tile(labels, [params['num_of_samples'], 1]) + + # decode + loss = self.decoding_graph( + encoder_output, + encoder_self_attention_bias, + labels, + params, + embedding_augmentation=embedding_augmentation) + + return loss + + def transformer_model_train_fn(self, features, labels): + initializer = get_initializer(self.params) + with tf.compat.v1.variable_scope('NmtModel', initializer=initializer): + num_gpus = self.params['num_gpus'] + gradient_clip_norm = self.params['gradient_clip_norm'] + global_step = tf.compat.v1.train.get_global_step() + print(global_step) + + # learning rate + learning_rate = get_learning_rate_decay( + self.params['learning_rate'], global_step, self.params) + learning_rate = tf.convert_to_tensor( + value=learning_rate, dtype=tf.float32) + + # optimizer + if self.params['optimizer'] == 'sgd': + optimizer = tf.compat.v1.train.GradientDescentOptimizer( + learning_rate) + elif self.params['optimizer'] == 'adam': + optimizer = tf.compat.v1.train.AdamOptimizer( + learning_rate=learning_rate, + beta1=self.params['adam_beta1'], + beta2=self.params['adam_beta2'], + epsilon=self.params['adam_epsilon']) + else: + tf.compat.v1.logging.info('optimizer not supported') + sys.exit() + opt = MultiStepOptimizer(optimizer, self.params['update_cycle']) + + def fill_gpus(inputs, num_gpus): + outputs = inputs + for i in range(num_gpus): + outputs = tf.concat([outputs, inputs], axis=0) + outputs = outputs[:num_gpus, ] + return outputs + + features = tf.cond( + pred=tf.shape(input=features)[0] < num_gpus, + true_fn=lambda: fill_gpus(features, num_gpus), + false_fn=lambda: features) + labels = tf.cond( + pred=tf.shape(input=labels)[0] < num_gpus, + true_fn=lambda: fill_gpus(labels, num_gpus), + false_fn=lambda: labels) + + if num_gpus > 0: + feature_shards = shard_features(features, num_gpus) + label_shards = shard_features(labels, num_gpus) + else: + feature_shards = [features] + label_shards = [labels] + + if num_gpus > 0: + devices = ['gpu:%d' % d for d in range(num_gpus)] + else: + devices = ['cpu:0'] + multi_grads = [] + sharded_losses = [] + + for i, device in enumerate(devices): + with tf.device(device), tf.compat.v1.variable_scope( + tf.compat.v1.get_variable_scope(), + reuse=True if i > 0 else None): + with tf.name_scope('%s_%d' % ('GPU', i)): + feature_output, label_output = self.build_contrastive_training_graph( + feature_shards[i], label_shards[i], self.params) + mle_loss = self.build_training_graph( + feature_shards[i], label_shards[i], self.params, + feature_output, label_output) + sharded_losses.append(mle_loss) + tf.compat.v1.summary.scalar('mle_loss_{}'.format(i), + mle_loss) + + # Optimization + trainable_vars_list = [ + v for v in tf.compat.v1.trainable_variables() + if 'Semantic_Embedding' not in v.name + and 'mini_xlm_encoder' not in v.name + ] + grads_and_vars = opt.compute_gradients( + mle_loss, + var_list=trainable_vars_list, + colocate_gradients_with_ops=True) + multi_grads.append(grads_and_vars) + + total_loss = tf.add_n(sharded_losses) / len(sharded_losses) + + # Average gradients + grads_and_vars = average_gradients(multi_grads) + + if gradient_clip_norm > 0.0: + grads, var_list = list(zip(*grads_and_vars)) + grads, _ = tf.clip_by_global_norm(grads, gradient_clip_norm) + grads_and_vars = zip(grads, var_list) + + train_op = opt.apply_gradients( + grads_and_vars, + global_step=tf.compat.v1.train.get_global_step()) + + return train_op, total_loss + + def prediction(self, decoder_output, params): + hidden_size = params['hidden_size'] + trg_vocab_size = params['trg_vocab_size'] + + if params['shared_embedding_and_softmax_weights']: + embedding_scope = 'Shared_Embedding' if params[ + 'shared_source_target_embedding'] else 'Target_Embedding' + with tf.compat.v1.variable_scope(embedding_scope, reuse=True): + weights = tf.compat.v1.get_variable('Weights') + else: + weights = tf.compat.v1.get_variable('Softmax', + [tgt_vocab_size, hidden_size]) + shape = tf.shape(input=decoder_output)[:-1] + decoder_output = tf.reshape(decoder_output, [-1, hidden_size]) + logits = tf.matmul(decoder_output, weights, transpose_b=True) + logits = tf.reshape(logits, tf.concat([shape, [trg_vocab_size]], 0)) + return logits + + def inference_func(self, + encoder_output, + feature_output, + encoder_self_attention_bias, + trg_seq, + states_key, + states_val, + params={}, + is_prefix=False): + trg_vocab_size = params['trg_vocab_size'] + hidden_size = params['hidden_size'] + + initializer = tf.compat.v1.random_normal_initializer( + 0.0, hidden_size**-0.5, dtype=tf.float32) + + if params['shared_source_target_embedding']: + with tf.compat.v1.variable_scope( + 'Shared_Embedding', reuse=tf.compat.v1.AUTO_REUSE): + trg_embedding = tf.compat.v1.get_variable( + 'Weights', [trg_vocab_size, hidden_size], + initializer=initializer) + else: + with tf.compat.v1.variable_scope('Target_Embedding'): + trg_embedding = tf.compat.v1.get_variable( + 'Weights', [trg_vocab_size, hidden_size], + initializer=initializer) + + decoder_input = tf.gather(trg_embedding, tf.cast(trg_seq, tf.int32)) + decoder_input *= hidden_size**0.5 + decoder_self_attention_bias = attention_bias( + tf.shape(input=decoder_input)[1], 'causal') + decoder_input = tf.pad( + tensor=decoder_input, paddings=[[0, 0], [1, 0], [0, 0]])[:, :-1, :] + if params['position_info_type'] == 'absolute': + decoder_input = add_timing_signal(decoder_input) + if not is_prefix: + decoder_input = decoder_input[:, -1:, :] + decoder_self_attention_bias = decoder_self_attention_bias[:, :, + -1:, :] + decoder_output, attention_weights = transformer_decoder( + decoder_input, + encoder_output, + decoder_self_attention_bias, + encoder_self_attention_bias, + states_key=states_key, + states_val=states_val, + embedding_augmentation=feature_output, + params=params) + if not is_prefix: + decoder_output_last = decoder_output[:, -1, :] + attention_weights_last = attention_weights[:, -1, :] + else: + decoder_output_last = decoder_output + attention_weights_last = attention_weights + + if params['shared_embedding_and_softmax_weights']: + embedding_scope = \ + 'Shared_Embedding' if params['shared_source_target_embedding'] else 'Target_Embedding' + with tf.compat.v1.variable_scope(embedding_scope, reuse=True): + weights = tf.compat.v1.get_variable('Weights') + else: + weights = tf.compat.v1.get_variable('Softmax', + [trg_vocab_size, hidden_size]) + logits = tf.matmul(decoder_output_last, weights, transpose_b=True) + log_prob = tf.nn.log_softmax(logits) + return log_prob, attention_weights_last, states_key, states_val + + def beam_search(self, features, params): + beam_size = params['beam_size'] + trg_vocab_size = params['trg_vocab_size'] + hidden_size = params['hidden_size'] + num_decoder_layers = params['num_decoder_layers'] + lp_rate = params['lp_rate'] + max_decoded_trg_len = params['max_decoded_trg_len'] + src_input = features['input_wids'] + if 'prefix_wids' in features: + prefix = features['prefix_wids'] + prefix_hit = features['prefix_hit'] + else: + prefix = None + prefix_hit = None + batch_size = tf.shape(src_input)[0] + + src_input = tile_to_beam_size(src_input, beam_size) + src_input = merge_first_two_dims(src_input) + if prefix is not None: + prefix = tf.cast(tile_to_beam_size(prefix, beam_size), tf.int32) + prefix_hit = tile_to_beam_size(prefix_hit, beam_size) + + encoder_output, encoder_self_attention_bias = self.encoding_graph( + src_input, params) + source_name = 'source' + if params['shared_source_target_embedding']: + source_name = None + feature_output = self.semantic_encoding_graph( + src_input, params, name=source_name) + + states_key = [ + tf.fill([batch_size, 0, hidden_size], 0.0) + for layer in range(num_decoder_layers) + ] + states_val = [ + tf.fill([batch_size, 0, hidden_size], 0.0) + for layer in range(num_decoder_layers) + ] + for layer in range(num_decoder_layers): + states_key[layer].set_shape( + tf.TensorShape([None, None, hidden_size])) + states_val[layer].set_shape( + tf.TensorShape([None, None, hidden_size])) + states_key = [ + tile_to_beam_size(states_key[layer], beam_size) + for layer in range(num_decoder_layers) + ] + states_val = [ + tile_to_beam_size(states_val[layer], beam_size) + for layer in range(num_decoder_layers) + ] + fixed_length = 1 + if prefix is not None: + init_seqs = tf.concat( + [prefix, tf.fill([batch_size, beam_size, 1], 0)], axis=2) + fixed_length = tf.shape(init_seqs)[-1] + flat_seqs = merge_first_two_dims(init_seqs) + flat_states_key = [ + merge_first_two_dims(states_key[layer]) + for layer in range(num_decoder_layers) + ] + flat_states_val = [ + merge_first_two_dims(states_val[layer]) + for layer in range(num_decoder_layers) + ] + + step_log_probs, step_attn_weights, step_states_key, step_states_val = self.inference_func( + encoder_output, + feature_output, + encoder_self_attention_bias, + flat_seqs, + flat_states_key, + flat_states_val, + params=params, + is_prefix=True) + + states_key = [ + split_first_two_dims(step_states_key[layer], batch_size, + beam_size) + for layer in range(num_decoder_layers) + ] + states_val = [ + split_first_two_dims(step_states_val[layer], batch_size, + beam_size) + for layer in range(num_decoder_layers) + ] + + prefix_hit = merge_first_two_dims(prefix_hit) + log_probs = tf.where( + prefix_hit, step_log_probs[:, -1, :], + tf.ones_like(step_log_probs[:, -1, :]) * tf.float32.min) + + init_seqs = tf.concat([ + flat_seqs[:, :-1], + tf.expand_dims( + tf.cast(tf.argmax(log_probs, -1), tf.int32), -1) + ], -1) + + init_seqs = split_first_two_dims(init_seqs, batch_size, beam_size) + init_seqs = tf.concat( + [init_seqs, tf.fill([batch_size, beam_size, 1], 0)], axis=2) + else: + init_seqs = tf.fill([batch_size, beam_size, 1], 0) + + init_log_probs = \ + tf.constant([[0.] + [tf.float32.min] * (beam_size - 1)]) + init_log_probs = tf.tile(init_log_probs, [batch_size, 1]) + init_scores = tf.zeros_like(init_log_probs) + fin_seqs = init_seqs + fin_scores = tf.fill([batch_size, beam_size], tf.float32.min) + fin_flags = tf.cast(tf.fill([batch_size, beam_size], 0), tf.bool) + + state = BeamSearchState( + inputs=(init_seqs, init_log_probs, init_scores), + state=(states_key, states_val), + finish=(fin_flags, fin_seqs, fin_scores), + ) + + def _beam_search_step(time, state): + seqs, log_probs = state.inputs[:2] + states_key, states_val = state.state + + flat_seqs = merge_first_two_dims(seqs) + flat_states_key = [ + merge_first_two_dims(states_key[layer]) + for layer in range(num_decoder_layers) + ] + flat_states_val = [ + merge_first_two_dims(states_val[layer]) + for layer in range(num_decoder_layers) + ] + + step_log_probs, step_attn_weights, step_states_key, step_states_val = self.inference_func( + encoder_output, + feature_output, + encoder_self_attention_bias, + flat_seqs, + flat_states_key, + flat_states_val, + params=params, + is_prefix=False) + + step_log_probs = split_first_two_dims(step_log_probs, batch_size, + beam_size) + curr_log_probs = tf.expand_dims(log_probs, 2) + step_log_probs + + next_states_key = [ + split_first_two_dims(step_states_key[layer], batch_size, + beam_size) + for layer in range(num_decoder_layers) + ] + next_states_val = [ + split_first_two_dims(step_states_val[layer], batch_size, + beam_size) + for layer in range(num_decoder_layers) + ] + + # Apply length penalty + length_penalty = tf.pow( + (5.0 + tf.cast(time + 1, dtype=tf.float32)) / 6.0, lp_rate) + curr_scores = curr_log_probs / length_penalty + + # Select top-k candidates + # [batch_size, beam_size * vocab_size] + curr_scores = tf.reshape(curr_scores, + [-1, beam_size * trg_vocab_size]) + # [batch_size, 2 * beam_size] + top_scores, top_indices = tf.nn.top_k(curr_scores, k=2 * beam_size) + # Shape: [batch_size, 2 * beam_size] + beam_indices = top_indices // trg_vocab_size + symbol_indices = top_indices % trg_vocab_size + # Expand sequences + # [batch_size, 2 * beam_size, time] + candidate_seqs = gather_2d(seqs, beam_indices) + candidate_seqs = tf.concat( + [candidate_seqs[:, :, :-1], + tf.expand_dims(symbol_indices, 2)], + axis=2) + pad_seqs = tf.fill([batch_size, 2 * beam_size, 1], + tf.constant(0, tf.int32)) + candidate_seqs = tf.concat([candidate_seqs, pad_seqs], axis=2) + + # Expand sequences + # Suppress finished sequences + flags = tf.equal(symbol_indices, 0) + # [batch, 2 * beam_size] + alive_scores = top_scores + tf.cast( + flags, dtype=tf.float32) * tf.float32.min + # [batch, beam_size] + alive_scores, alive_indices = tf.nn.top_k(alive_scores, beam_size) + alive_symbols = gather_2d(symbol_indices, alive_indices) + alive_indices = gather_2d(beam_indices, alive_indices) + alive_seqs = gather_2d(seqs, alive_indices) + alive_seqs = tf.concat( + [alive_seqs[:, :, :-1], + tf.expand_dims(alive_symbols, 2)], + axis=2) + pad_seqs = tf.fill([batch_size, beam_size, 1], + tf.constant(0, tf.int32)) + alive_seqs = tf.concat([alive_seqs, pad_seqs], axis=2) + alive_states_key = [ + gather_2d(next_states_key[layer], alive_indices) + for layer in range(num_decoder_layers) + ] + alive_states_val = [ + gather_2d(next_states_val[layer], alive_indices) + for layer in range(num_decoder_layers) + ] + alive_log_probs = alive_scores * length_penalty + + # Select finished sequences + prev_fin_flags, prev_fin_seqs, prev_fin_scores = state.finish + # [batch, 2 * beam_size] + step_fin_scores = top_scores + ( + 1.0 - tf.cast(flags, dtype=tf.float32)) * tf.float32.min + # [batch, 3 * beam_size] + fin_flags = tf.concat([prev_fin_flags, flags], axis=1) + fin_scores = tf.concat([prev_fin_scores, step_fin_scores], axis=1) + # [batch, beam_size] + fin_scores, fin_indices = tf.nn.top_k(fin_scores, beam_size) + fin_flags = gather_2d(fin_flags, fin_indices) + pad_seqs = tf.fill([batch_size, beam_size, 1], + tf.constant(0, tf.int32)) + prev_fin_seqs = tf.concat([prev_fin_seqs, pad_seqs], axis=2) + fin_seqs = tf.concat([prev_fin_seqs, candidate_seqs], axis=1) + fin_seqs = gather_2d(fin_seqs, fin_indices) + + new_state = BeamSearchState( + inputs=(alive_seqs, alive_log_probs, alive_scores), + state=(alive_states_key, alive_states_val), + finish=(fin_flags, fin_seqs, fin_scores), + ) + + return time + 1, new_state + + def _is_finished(t, s): + log_probs = s.inputs[1] + finished_flags = s.finish[0] + finished_scores = s.finish[2] + max_lp = tf.pow( + ((5.0 + tf.cast(max_decoded_trg_len, dtype=tf.float32)) / 6.0), + lp_rate) + best_alive_score = log_probs[:, 0] / max_lp + worst_finished_score = tf.reduce_min( + input_tensor=finished_scores + * tf.cast(finished_flags, dtype=tf.float32), + axis=1) + add_mask = 1.0 - tf.cast( + tf.reduce_any(input_tensor=finished_flags, axis=1), + dtype=tf.float32) + worst_finished_score += tf.float32.min * add_mask + bound_is_met = tf.reduce_all( + input_tensor=tf.greater(worst_finished_score, + best_alive_score)) + + cond = tf.logical_and( + tf.less(t, max_decoded_trg_len), tf.logical_not(bound_is_met)) + + return cond + + def _loop_fn(t, s): + outs = _beam_search_step(t, s) + return outs + + time = tf.constant(0, name='time') + shape_invariants = BeamSearchState( + inputs=(tf.TensorShape([None, None, None]), + tf.TensorShape([None, None]), tf.TensorShape([None, + None])), + state=([ + tf.TensorShape([None, None, None, hidden_size]) + for layer in range(num_decoder_layers) + ], [ + tf.TensorShape([None, None, None, hidden_size]) + for layer in range(num_decoder_layers) + ]), + finish=(tf.TensorShape([None, + None]), tf.TensorShape([None, None, None]), + tf.TensorShape([None, None]))) + outputs = tf.while_loop( + cond=_is_finished, + body=_loop_fn, + loop_vars=[time, state], + shape_invariants=[tf.TensorShape([]), shape_invariants], + parallel_iterations=1, + back_prop=False) + + final_state = outputs[1] + alive_seqs = final_state.inputs[0] + alive_scores = final_state.inputs[2] + final_flags = final_state.finish[0] + final_seqs = final_state.finish[1] + final_scores = final_state.finish[2] + + alive_seqs.set_shape([None, beam_size, None]) + final_seqs.set_shape([None, beam_size, None]) + + final_seqs = tf.compat.v1.where( + tf.reduce_any(input_tensor=final_flags, axis=1), final_seqs, + alive_seqs) + final_scores = tf.compat.v1.where( + tf.reduce_any(input_tensor=final_flags, axis=1), final_scores, + alive_scores) + + final_seqs = final_seqs[:, :, fixed_length - 1:-1] + return final_seqs, final_scores + + +class BeamSearchState( + namedtuple('BeamSearchState', ('inputs', 'state', 'finish'))): + pass + + +def tile_to_beam_size(tensor, beam_size): + """Tiles a given tensor by beam_size. """ + tensor = tf.expand_dims(tensor, axis=1) + tile_dims = [1] * tensor.shape.ndims + tile_dims[1] = beam_size + + return tf.tile(tensor, tile_dims) + + +def infer_shape(x): + x = tf.convert_to_tensor(x) + + if x.shape.dims is None: + return tf.shape(x) + + static_shape = x.shape.as_list() + dynamic_shape = tf.shape(x) + + ret = [] + for i in range(len(static_shape)): + dim = static_shape[i] + if dim is None: + dim = dynamic_shape[i] + ret.append(dim) + + return ret + + +def split_first_two_dims(tensor, dim_0, dim_1): + shape = infer_shape(tensor) + new_shape = [dim_0] + [dim_1] + shape[1:] + return tf.reshape(tensor, new_shape) + + +def merge_first_two_dims(tensor): + shape = infer_shape(tensor) + shape[0] *= shape[1] + shape.pop(1) + return tf.reshape(tensor, shape) + + +def gather_2d(params, indices, name=None): + """ Gather the 2nd dimension given indices + :param params: A tensor with shape [batch_size, M, ...] + :param indices: A tensor with shape [batch_size, N] + :param name: An optional string + :return: A tensor with shape [batch_size, N, ...] + """ + batch_size = tf.shape(params)[0] + range_size = tf.shape(indices)[1] + batch_pos = tf.range(batch_size * range_size) // range_size + batch_pos = tf.reshape(batch_pos, [batch_size, range_size]) + indices = tf.stack([batch_pos, indices], axis=-1) + output = tf.gather_nd(params, indices, name=name) + + return output + + +def linear(inputs, output_size, bias, concat=True, dtype=None, scope=None): + with tf.compat.v1.variable_scope( + scope, default_name='linear', values=[inputs], dtype=dtype): + if not isinstance(inputs, (list, tuple)): + inputs = [inputs] + + input_size = [item.get_shape()[-1] for item in inputs] + + if len(inputs) != len(input_size): + raise RuntimeError('inputs and input_size unmatched!') + + output_shape = tf.concat([tf.shape(inputs[0])[:-1], [output_size]], + axis=0) + # Flatten to 2D + inputs = [tf.reshape(inp, [-1, inp.shape[-1]]) for inp in inputs] + + results = [] + if concat: + input_size = sum(input_size) + inputs = tf.concat(inputs, 1) + + shape = [input_size, output_size] + matrix = tf.compat.v1.get_variable('matrix', shape) + results.append(tf.matmul(inputs, matrix)) + else: + for i in range(len(input_size)): + shape = [input_size[i], output_size] + name = 'matrix_%d' % i + matrix = tf.compat.v1.get_variable(name, shape) + results.append(tf.matmul(inputs[i], matrix)) + + output = tf.add_n(results) + + if bias: + shape = [output_size] + bias = tf.compat.v1.get_variable('bias', shape) + output = tf.nn.bias_add(output, bias) + + output = tf.reshape(output, output_shape) + + return output + + +def layer_norm(inputs, epsilon=1e-6, name=None, reuse=None): + with tf.compat.v1.variable_scope( + name, default_name='layer_norm', values=[inputs], reuse=reuse): + channel_size = inputs.get_shape().as_list()[-1] + + scale = tf.compat.v1.get_variable( + 'layer_norm_scale', [channel_size], + initializer=tf.ones_initializer()) + + offset = tf.compat.v1.get_variable( + 'layer_norm_offset', [channel_size], + initializer=tf.zeros_initializer()) + + mean = tf.reduce_mean(inputs, -1, True) + variance = tf.reduce_mean(tf.square(inputs - mean), -1, True) + + norm_inputs = (inputs - mean) * tf.compat.v1.rsqrt(variance + epsilon) + + return norm_inputs * scale + offset + + +def _layer_process(x, mode): + if not mode or mode == 'none': + return x + elif mode == 'layer_norm': + return layer_norm(x) + else: + raise ValueError('Unknown mode %s' % mode) + + +def _residual_fn(x, y, keep_prob=None): + if keep_prob and keep_prob < 1.0: + y = tf.nn.dropout(y, rate=1 - (keep_prob)) + return x + y + + +def embedding_augmentation_layer(x, embedding_augmentation, params, name=None): + hidden_size = params['hidden_size'] + keep_prob = 1.0 - params['relu_dropout'] + with tf.compat.v1.variable_scope( + name, + default_name='embedding_augmentation_layer', + values=[x, embedding_augmentation]): + with tf.compat.v1.variable_scope('input_layer'): + hidden = linear(embedding_augmentation, hidden_size, True, True) + hidden = tf.nn.relu(hidden) + + if keep_prob and keep_prob < 1.0: + hidden = tf.nn.dropout(hidden, rate=1 - (keep_prob)) + + with tf.compat.v1.variable_scope('output_layer'): + output = linear(hidden, hidden_size, True, True) + + return x + output + + +def transformer_ffn_layer(x, params, name=None): + filter_size = params['filter_size'] + hidden_size = params['hidden_size'] + keep_prob = 1.0 - params['relu_dropout'] + with tf.compat.v1.variable_scope( + name, default_name='ffn_layer', values=[x]): + with tf.compat.v1.variable_scope('input_layer'): + hidden = linear(x, filter_size, True, True) + hidden = tf.nn.relu(hidden) + + if keep_prob and keep_prob < 1.0: + hidden = tf.nn.dropout(hidden, rate=1 - (keep_prob)) + + with tf.compat.v1.variable_scope('output_layer'): + output = linear(hidden, hidden_size, True, True) + + return output + + +def transformer_encoder(encoder_input, + encoder_self_attention_bias, + mask, + params={}, + name='encoder'): + num_encoder_layers = params['num_encoder_layers'] + hidden_size = params['hidden_size'] + num_heads = params['num_heads'] + residual_dropout = params['residual_dropout'] + attention_dropout = params['attention_dropout'] + layer_preproc = params['layer_preproc'] + layer_postproc = params['layer_postproc'] + x = encoder_input + mask = tf.expand_dims(mask, 2) + with tf.compat.v1.variable_scope(name, reuse=tf.compat.v1.AUTO_REUSE): + for layer in range(num_encoder_layers): + with tf.compat.v1.variable_scope('layer_%d' % layer): + max_relative_dis = params['max_relative_dis'] \ + if params['position_info_type'] == 'relative' else None + o, w = multihead_attention( + _layer_process(x, layer_preproc), + None, + encoder_self_attention_bias, + hidden_size, + hidden_size, + hidden_size, + num_heads, + attention_dropout, + max_relative_dis=max_relative_dis, + name='encoder_self_attention') + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + o = transformer_ffn_layer( + _layer_process(x, layer_preproc), params) + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + x = tf.multiply(x, mask) + + return _layer_process(x, layer_preproc) + + +def transformer_semantic_encoder(encoder_input, + encoder_self_attention_bias, + mask, + params={}, + name='mini_xlm_encoder'): + num_encoder_layers = params['num_semantic_encoder_layers'] + hidden_size = params['hidden_size'] + num_heads = params['num_heads'] + residual_dropout = params['residual_dropout'] + attention_dropout = params['attention_dropout'] + layer_preproc = params['layer_preproc'] + layer_postproc = params['layer_postproc'] + x = encoder_input + mask = tf.expand_dims(mask, 2) + with tf.compat.v1.variable_scope(name, reuse=tf.compat.v1.AUTO_REUSE): + for layer in range(num_encoder_layers): + with tf.compat.v1.variable_scope('layer_%d' % layer): + max_relative_dis = params['max_relative_dis'] + o, w = multihead_attention( + _layer_process(x, layer_preproc), + None, + encoder_self_attention_bias, + hidden_size, + hidden_size, + hidden_size, + num_heads, + attention_dropout, + max_relative_dis=max_relative_dis, + name='encoder_self_attention') + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + o = transformer_ffn_layer( + _layer_process(x, layer_preproc), params) + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + x = tf.multiply(x, mask) + + with tf.compat.v1.variable_scope( + 'pooling_layer', reuse=tf.compat.v1.AUTO_REUSE): + output = tf.reduce_sum( + input_tensor=x, axis=1) / tf.reduce_sum( + input_tensor=mask, axis=1) + output = linear( + tf.expand_dims(output, axis=1), hidden_size, True, True) + + return _layer_process(output, layer_preproc) + + +def transformer_decoder(decoder_input, + encoder_output, + decoder_self_attention_bias, + encoder_decoder_attention_bias, + states_key=None, + states_val=None, + embedding_augmentation=None, + params={}, + name='decoder'): + num_decoder_layers = params['num_decoder_layers'] + hidden_size = params['hidden_size'] + num_heads = params['num_heads'] + residual_dropout = params['residual_dropout'] + attention_dropout = params['attention_dropout'] + layer_preproc = params['layer_preproc'] + layer_postproc = params['layer_postproc'] + x = decoder_input + with tf.compat.v1.variable_scope(name, reuse=tf.compat.v1.AUTO_REUSE): + for layer in range(num_decoder_layers): + with tf.compat.v1.variable_scope('layer_%d' % layer): + max_relative_dis = params['max_relative_dis'] \ + if params['position_info_type'] == 'relative' else None + # continuous semantic augmentation + if embedding_augmentation is not None: + x = embedding_augmentation_layer( + x, _layer_process(embedding_augmentation, + layer_preproc), params) + x = _layer_process(x, layer_postproc) + o, w = multihead_attention( + _layer_process(x, layer_preproc), + None, + decoder_self_attention_bias, + hidden_size, + hidden_size, + hidden_size, + num_heads, + attention_dropout, + states_key=states_key, + states_val=states_val, + layer=layer, + max_relative_dis=max_relative_dis, + name='decoder_self_attention') + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + o, w = multihead_attention( + _layer_process(x, layer_preproc), + encoder_output, + encoder_decoder_attention_bias, + hidden_size, + hidden_size, + hidden_size, + num_heads, + attention_dropout, + max_relative_dis=max_relative_dis, + name='encdec_attention') + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + o = transformer_ffn_layer( + _layer_process(x, layer_preproc), params) + x = _residual_fn(x, o, 1.0 - residual_dropout) + x = _layer_process(x, layer_postproc) + + return _layer_process(x, layer_preproc), w + + +def add_timing_signal(x, min_timescale=1.0, max_timescale=1.0e4): + length = tf.shape(x)[1] + channels = tf.shape(x)[2] + position = tf.cast(tf.range(length), tf.float32) + num_timescales = channels // 2 + + log_timescale_increment = \ + (math.log(float(max_timescale) / float(min_timescale)) / (tf.cast(num_timescales, tf.float32) - 1)) + inv_timescales = min_timescale * tf.exp( + tf.cast(tf.range(num_timescales), tf.float32) + * -log_timescale_increment) + + scaled_time = \ + tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0) + signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1) + signal = tf.pad(signal, [[0, 0], [0, tf.compat.v1.mod(channels, 2)]]) + signal = tf.reshape(signal, [1, length, channels]) + + return x + tf.cast(signal, x.dtype) + + +def attention_bias(inputs, mode, inf=-1e9, dtype=None): + if dtype is None: + dtype = tf.float32 + + if dtype != tf.float32: + inf = dtype.min + + if mode == 'masking': + mask = inputs + ret = (1.0 - mask) * inf + ret = tf.expand_dims(tf.expand_dims(ret, 1), 1) + + elif mode == 'causal': + length = inputs + lower_triangle = tf.linalg.band_part( + tf.fill([length, length], 1.0), -1, 0) + ret = inf * (1.0 - lower_triangle) + ret = tf.reshape(ret, [1, 1, length, length]) + else: + raise ValueError('Unknown mode %s' % mode) + + return tf.cast(ret, dtype) + + +def split_heads(x, num_heads): + n = num_heads + old_shape = x.get_shape().dims + ndims = x.shape.ndims + + last = old_shape[-1] + new_shape = old_shape[:-1] + [n] + [last // n if last else None] + ret = tf.reshape(x, tf.concat([tf.shape(x)[:-1], [n, -1]], 0)) + ret.set_shape(new_shape) + perm = [0, ndims - 1] + [i for i in range(1, ndims - 1)] + [ndims] + return tf.transpose(ret, perm) + + +def dot_product_attention(q, + k, + v, + bias, + dropout_rate=0.0, + name=None, + rpr=None): + with tf.compat.v1.variable_scope( + name, default_name='dot_product_attention', values=[q, k, v]): + q_shape = tf.shape(q) + bs, hd, lq, dk = q_shape[0], q_shape[1], q_shape[2], q_shape[3] + lk = tf.shape(k)[2] + dv = tf.shape(v)[3] + + if rpr is not None: + rpr_k, rpr_v = rpr['rpr_k'], rpr[ + 'rpr_v'] # (lq, lk, dk), (lq, lk, dv) + + if rpr is None: + logits = tf.matmul(q, k, transpose_b=True) + else: # self-attention with relative position representaion + logits_part1 = tf.matmul(q, k, transpose_b=True) # bs, hd, lq, lk + + q = tf.reshape(tf.transpose(q, [2, 0, 1, 3]), + [lq, bs * hd, dk]) # lq, bs*hd, dk + logits_part2 = tf.matmul(q, + tf.transpose(rpr_k, + [0, 2, 1])) # lq, bs*hd, lk + logits_part2 = tf.reshape( + tf.transpose(logits_part2, [1, 0, 2]), [bs, hd, lq, lk]) + + logits = logits_part1 + logits_part2 # bs, hd, lq, lk + + if bias is not None: + logits += bias + + weights = tf.nn.softmax(logits, name='attention_weights') + + if dropout_rate > 0.0: + weights = tf.nn.dropout(weights, 1.0 - dropout_rate) + + if rpr is None: + return tf.matmul(weights, v), weights + else: + outputs_part1 = tf.matmul(weights, v) # bs, hd, lq, dv + + weights = tf.reshape( + tf.transpose(weights, [2, 0, 1, 3]), + [lq, bs * hd, lk]) # lq, bs*hd, lk + outputs_part2 = tf.matmul(weights, rpr_v) # lq, bs*hd, dv + outputs_part2 = tf.reshape( + tf.transpose(outputs_part2, [1, 0, 2]), [bs, hd, lq, dv]) + + outputs = outputs_part1 + outputs_part2 # bs, hd, lq, dv + weights = tf.reshape( + tf.transpose(weights, [1, 0, 2]), + [bs, hd, lq, lk]) # bs, hd, lq, lk + + return outputs, weights + + +def combine_heads(x): + x = tf.transpose(x, [0, 2, 1, 3]) + old_shape = x.get_shape().dims + a, b = old_shape[-2:] + new_shape = old_shape[:-2] + [a * b if a and b else None] + x = tf.reshape(x, tf.concat([tf.shape(x)[:-2], [-1]], 0)) + x.set_shape(new_shape) + + return x + + +def create_rpr(orginal_var, + length_q, + length_kv, + max_relative_dis, + name='create_rpr'): + with tf.name_scope(name): + idxs = tf.reshape(tf.range(length_kv), [-1, 1]) # only self-attention + idys = tf.reshape(tf.range(length_kv), [1, -1]) + ids = idxs - idys + ids = ids + max_relative_dis + ids = tf.maximum(ids, 0) + ids = tf.minimum(ids, 2 * max_relative_dis) + ids = ids[-length_q:, :] + rpr = tf.gather(orginal_var, ids) + return rpr + + +def multihead_attention(queries, + memories, + bias, + key_depth, + value_depth, + output_depth, + num_heads, + dropout_rate, + states_key=None, + states_val=None, + layer=0, + max_relative_dis=None, + name=None): + if key_depth % num_heads != 0: + raise ValueError( + 'Key size (%d) must be divisible by the number of attention heads (%d).' + % (key_size, num_heads)) + + if value_depth % num_heads != 0: + raise ValueError( + 'Value size (%d) must be divisible by the number of attention heads (%d).' + % (value_size, num_heads)) + + with tf.compat.v1.variable_scope( + name, default_name='multihead_attention', + values=[queries, memories]): + if memories is None: + # self attention + combined = linear( + queries, + key_depth * 2 + value_depth, + True, + True, + scope='qkv_transform') + q, k, v = tf.split( + combined, [key_depth, key_depth, value_depth], axis=2) + else: + q = linear(queries, key_depth, True, True, scope='q_transform') + combined = linear( + memories, + key_depth + value_depth, + True, + True, + scope='kv_transform') + k, v = tf.split(combined, [key_depth, value_depth], axis=2) + + if states_key is not None: + k = states_key[layer] = tf.concat([states_key[layer], k], axis=1) + if states_val is not None: + v = states_val[layer] = tf.concat([states_val[layer], v], axis=1) + + q = split_heads(q, num_heads) + k = split_heads(k, num_heads) + v = split_heads(v, num_heads) + + key_depth_per_head = key_depth // num_heads + q *= key_depth_per_head**-0.5 + + length_q = tf.shape(q)[2] + length_kv = tf.shape(k)[2] + + # relative position representation (only in self-attention) + if memories is None and max_relative_dis is not None: + rpr_k = tf.compat.v1.get_variable( + 'rpr_k', [2 * max_relative_dis + 1, key_depth // num_heads]) + rpr_v = tf.compat.v1.get_variable( + 'rpr_v', [2 * max_relative_dis + 1, value_depth // num_heads]) + rpr_k = create_rpr(rpr_k, length_q, length_kv, max_relative_dis) + rpr_v = create_rpr(rpr_v, length_q, length_kv, max_relative_dis) + rpr = {'rpr_k': rpr_k, 'rpr_v': rpr_v} + x, w = dot_product_attention(q, k, v, bias, dropout_rate, rpr=rpr) + else: + x, w = dot_product_attention(q, k, v, bias, dropout_rate) + x = combine_heads(x) + w = tf.reduce_mean(w, 1) + x = linear(x, output_depth, True, True, scope='output_transform') + return x, w + + +def get_initializer(params): + if params['initializer'] == 'uniform': + max_val = params['initializer_scale'] + return tf.compat.v1.random_uniform_initializer(-max_val, max_val) + elif params['initializer'] == 'normal': + return tf.compat.v1.random_normal_initializer( + 0.0, params['initializer_scale']) + elif params['initializer'] == 'normal_unit_scaling': + return tf.compat.v1.variance_scaling_initializer( + params['initializer_scale'], mode='fan_avg', distribution='normal') + elif params['initializer'] == 'uniform_unit_scaling': + return tf.compat.v1.variance_scaling_initializer( + params['initializer_scale'], + mode='fan_avg', + distribution='uniform') + else: + raise ValueError('Unrecognized initializer: %s' + % params['initializer']) + + +def get_learning_rate_decay(learning_rate, global_step, params): + if params['learning_rate_decay'] in ['linear_warmup_rsqrt_decay', 'noam']: + step = tf.cast(global_step, dtype=tf.float32) + warmup_steps = tf.cast(params['warmup_steps'], dtype=tf.float32) + multiplier = params['hidden_size']**-0.5 + decay = multiplier * tf.minimum((step + 1) * (warmup_steps**-1.5), + (step + 1)**-0.5) + return learning_rate * decay + elif params['learning_rate_decay'] == 'piecewise_constant': + return tf.compat.v1.train.piecewise_constant( + tf.cast(global_step, dtype=tf.int32), + params['learning_rate_boundaries'], params['learning_rate_values']) + elif params['learning_rate_decay'] == 'none': + return learning_rate + else: + raise ValueError('Unknown learning_rate_decay') + + +def average_gradients(tower_grads): + average_grads = [] + for grad_and_vars in zip(*tower_grads): + grads = [] + for g, _ in grad_and_vars: + expanded_g = tf.expand_dims(g, 0) + grads.append(expanded_g) + grad = tf.concat(axis=0, values=grads) + grad = tf.reduce_mean(grad, 0) + v = grad_and_vars[0][1] + grad_and_var = (grad, v) + average_grads.append(grad_and_var) + return average_grads + + +_ENGINE = None + + +def all_reduce(tensor): + if _ENGINE is None: + return tensor + + return _ENGINE.allreduce(tensor, compression=_ENGINE.Compression.fp16) + + +class MultiStepOptimizer(tf.compat.v1.train.Optimizer): + + def __init__(self, + optimizer, + step=1, + use_locking=False, + name='MultiStepOptimizer'): + super(MultiStepOptimizer, self).__init__(use_locking, name) + self._optimizer = optimizer + self._step = step + self._step_t = tf.convert_to_tensor(step, name='step') + + def _all_reduce(self, tensor): + with tf.name_scope(self._name + '_Allreduce'): + if tensor is None: + return tensor + + if isinstance(tensor, tf.IndexedSlices): + tensor = tf.convert_to_tensor(tensor) + + return all_reduce(tensor) + + def compute_gradients(self, + loss, + var_list=None, + gate_gradients=tf.compat.v1.train.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None): + grads_and_vars = self._optimizer.compute_gradients( + loss, var_list, gate_gradients, aggregation_method, + colocate_gradients_with_ops, grad_loss) + + grads, var_list = list(zip(*grads_and_vars)) + + # Do not create extra variables when step is 1 + if self._step == 1: + grads = [self._all_reduce(t) for t in grads] + return list(zip(grads, var_list)) + + first_var = min(var_list, key=lambda x: x.name) + iter_var = self._create_non_slot_variable( + initial_value=0 if self._step == 1 else 1, + name='iter', + colocate_with=first_var) + + new_grads = [] + + for grad, var in zip(grads, var_list): + grad_acc = self._zeros_slot(var, 'grad_acc', self._name) + + if isinstance(grad, tf.IndexedSlices): + grad_acc = tf.scatter_add( + grad_acc, + grad.indices, + grad.values, + use_locking=self._use_locking) + else: + grad_acc = tf.assign_add( + grad_acc, grad, use_locking=self._use_locking) + + def _acc_grad(): + return grad_acc + + def _avg_grad(): + return self._all_reduce(grad_acc / self._step) + + grad = tf.cond(tf.equal(iter_var, 0), _avg_grad, _acc_grad) + new_grads.append(grad) + + return list(zip(new_grads, var_list)) + + def apply_gradients(self, grads_and_vars, global_step=None, name=None): + if self._step == 1: + return self._optimizer.apply_gradients( + grads_and_vars, global_step, name=name) + + grads, var_list = list(zip(*grads_and_vars)) + + def _pass_gradients(): + return tf.group(*grads) + + def _apply_gradients(): + op = self._optimizer.apply_gradients( + zip(grads, var_list), global_step, name) + with tf.control_dependencies([op]): + zero_ops = [] + for var in var_list: + grad_acc = self.get_slot(var, 'grad_acc') + zero_ops.append( + grad_acc.assign( + tf.zeros_like(grad_acc), + use_locking=self._use_locking)) + zero_op = tf.group(*zero_ops) + return tf.group(*[op, zero_op]) + + iter_var = self._get_non_slot_variable('iter', tf.get_default_graph()) + update_op = tf.cond( + tf.equal(iter_var, 0), _apply_gradients, _pass_gradients) + + with tf.control_dependencies([update_op]): + iter_op = iter_var.assign( + tf.mod(iter_var + 1, self._step_t), + use_locking=self._use_locking) + + return tf.group(*[update_op, iter_op]) + + +def shard_features(x, num_datashards): + x = tf.convert_to_tensor(x) + batch_size = tf.shape(x)[0] + size_splits = [] + + with tf.device('/cpu:0'): + for i in range(num_datashards): + size_splits.append( + tf.cond( + tf.greater( + tf.compat.v1.mod(batch_size, num_datashards), + i), lambda: batch_size // num_datashards + 1, + lambda: batch_size // num_datashards)) + + return tf.split(x, size_splits, axis=0) diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/__init__.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..08b184e5e427e4b915a190cb8f5002703d2cca0f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/__init__.py @@ -0,0 +1,53 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import DebertaV2Config + from .tokenization import DebertaV2Tokenizer + from .tokenization_fast import DebertaV2TokenizerFast + from .backbone import ( + DebertaV2Model, + DebertaV2PreTrainedModel, + ) + from .fill_mask import DebertaV2ForMaskedLM + +else: + _import_structure = { + 'configuration': ['DebertaV2Config'], + 'tokenization': ['DebertaV2Tokenizer'], + 'tokenization_fast': ['DebertaV2TokenizerFast'], + 'backbone': [ + 'DebertaV2Model', + 'DebertaV2PreTrainedModel', + ], + 'fill_mask': [ + 'DebertaV2ForMaskedLM', + ] + } + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__) diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/backbone.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecb3190fccb7eb50b3c5da447b62abe7bc8591a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/backbone.py @@ -0,0 +1,1222 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 Microsoft and the Hugging Face Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch DeBERTa-v2 model.""" + +from collections.abc import Sequence +from typing import Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import LayerNorm +from transformers.activations import ACT2FN +from transformers.modeling_utils import PreTrainedModel +from transformers.pytorch_utils import softmax_backward_data + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .configuration import DebertaV2Config + +logger = logging.get_logger() + + +# Copied from transformers.models.deberta.modeling_deberta.ContextPooler +class ContextPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.pooler_hidden_size, + config.pooler_hidden_size) + self.dropout = StableDropout(config.pooler_dropout) + self.config = config + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + + context_token = hidden_states[:, 0] + context_token = self.dropout(context_token) + pooled_output = self.dense(context_token) + pooled_output = ACT2FN[self.config.pooler_hidden_act](pooled_output) + return pooled_output + + @property + def output_dim(self): + return self.config.hidden_size + + +# Copied from transformers.models.deberta.modeling_deberta.XSoftmax with deberta->deberta_v2 +class XSoftmax(torch.autograd.Function): + """ + Masked Softmax which is optimized for saving memory + + Args: + input (`torch.tensor`): The input tensor that will apply softmax. + mask (`torch.IntTensor`): + The mask matrix where 0 indicate that element will be ignored in the softmax calculation. + dim (int): The dimension that will apply softmax + + Example: + + >>> import torch + >>> from transformers.models.deberta_v2.modeling_deberta_v2 import XSoftmax + + >>> # Make a tensor + >>> x = torch.randn([4, 20, 100]) + + >>> # Create a mask + >>> mask = (x > 0).int() + + >>> # Specify the dimension to apply softmax + >>> dim = -1 + + >>> y = XSoftmax.apply(x, mask, dim) + """ + + @staticmethod + def forward(self, input, mask, dim): + self.dim = dim + rmask = ~(mask.to(torch.bool)) + + output = input.masked_fill(rmask, + torch.tensor(torch.finfo(input.dtype).min)) + output = torch.softmax(output, self.dim) + output.masked_fill_(rmask, 0) + self.save_for_backward(output) + return output + + @staticmethod + def backward(self, grad_output): + (output, ) = self.saved_tensors + inputGrad = softmax_backward_data(self, grad_output, output, self.dim, + output) + return inputGrad, None, None + + @staticmethod + def symbolic(g, self, mask, dim): + import torch.onnx.symbolic_helper as sym_help + from torch.onnx.symbolic_opset9 import masked_fill, softmax + + mask_cast_value = g.op( + 'Cast', mask, to_i=sym_help.cast_pytorch_to_onnx['Long']) + r_mask = g.op( + 'Cast', + g.op('Sub', + g.op('Constant', value_t=torch.tensor(1, dtype=torch.int64)), + mask_cast_value), + to_i=sym_help.cast_pytorch_to_onnx['Byte'], + ) + output = masked_fill( + g, self, r_mask, + g.op( + 'Constant', + value_t=torch.tensor(torch.finfo(self.type().dtype()).min))) + output = softmax(g, output, dim) + return masked_fill( + g, output, r_mask, + g.op('Constant', value_t=torch.tensor(0, dtype=torch.uint8))) + + +# Copied from transformers.models.deberta.modeling_deberta.DropoutContext +class DropoutContext(object): + + def __init__(self): + self.dropout = 0 + self.mask = None + self.scale = 1 + self.reuse_mask = True + + +# Copied from transformers.models.deberta.modeling_deberta.get_mask +def get_mask(input, local_context): + if not isinstance(local_context, DropoutContext): + dropout = local_context + mask = None + else: + dropout = local_context.dropout + dropout *= local_context.scale + mask = local_context.mask if local_context.reuse_mask else None + + if dropout > 0 and mask is None: + mask = (1 - torch.empty_like(input).bernoulli_(1 - dropout)).to( + torch.bool) + + if isinstance(local_context, DropoutContext): + if local_context.mask is None: + local_context.mask = mask + + return mask, dropout + + +# Copied from transformers.models.deberta.modeling_deberta.XDropout +class XDropout(torch.autograd.Function): + """Optimized dropout function to save computation and memory by using mask operation instead of multiplication.""" + + @staticmethod + def forward(ctx, input, local_ctx): + mask, dropout = get_mask(input, local_ctx) + ctx.scale = 1.0 / (1 - dropout) + if dropout > 0: + ctx.save_for_backward(mask) + return input.masked_fill(mask, 0) * ctx.scale + else: + return input + + @staticmethod + def backward(ctx, grad_output): + if ctx.scale > 1: + (mask, ) = ctx.saved_tensors + return grad_output.masked_fill(mask, 0) * ctx.scale, None + else: + return grad_output, None + + @staticmethod + def symbolic(g: torch._C.Graph, input: torch._C.Value, + local_ctx: Union[float, DropoutContext]) -> torch._C.Value: + from torch.onnx import symbolic_opset12 + + dropout_p = local_ctx + if isinstance(local_ctx, DropoutContext): + dropout_p = local_ctx.dropout + # StableDropout only calls this function when training. + train = True + # TODO: We should check if the opset_version being used to export + # is > 12 here, but there's no good way to do that. As-is, if the + # opset_version < 12, export will fail with a CheckerError. + # Once https://github.com/pytorch/pytorch/issues/78391 is fixed, do something like: + # if opset_version < 12: + # return torch.onnx.symbolic_opset9.dropout(g, input, dropout_p, train) + return symbolic_opset12.dropout(g, input, dropout_p, train) + + +# Copied from transformers.models.deberta.modeling_deberta.StableDropout +class StableDropout(nn.Module): + """ + Optimized dropout module for stabilizing the training + + Args: + drop_prob (float): the dropout probabilities + """ + + def __init__(self, drop_prob): + super().__init__() + self.drop_prob = drop_prob + self.count = 0 + self.context_stack = None + + def forward(self, x): + """ + Call the module + + Args: + x (`torch.tensor`): The input tensor to apply dropout + """ + if self.training and self.drop_prob > 0: + return XDropout.apply(x, self.get_context()) + return x + + def clear_context(self): + self.count = 0 + self.context_stack = None + + def init_context(self, reuse_mask=True, scale=1): + if self.context_stack is None: + self.context_stack = [] + self.count = 0 + for c in self.context_stack: + c.reuse_mask = reuse_mask + c.scale = scale + + def get_context(self): + if self.context_stack is not None: + if self.count >= len(self.context_stack): + self.context_stack.append(DropoutContext()) + ctx = self.context_stack[self.count] + ctx.dropout = self.drop_prob + self.count += 1 + return ctx + else: + return self.drop_prob + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaSelfOutput with DebertaLayerNorm->LayerNorm +class DebertaV2SelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps) + self.dropout = StableDropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaAttention with Deberta->DebertaV2 +class DebertaV2Attention(nn.Module): + + def __init__(self, config): + super().__init__() + self.self = DisentangledSelfAttention(config) + self.output = DebertaV2SelfOutput(config) + self.config = config + + def forward( + self, + hidden_states, + attention_mask, + output_attentions=False, + query_states=None, + relative_pos=None, + rel_embeddings=None, + ): + self_output = self.self( + hidden_states, + attention_mask, + output_attentions, + query_states=query_states, + relative_pos=relative_pos, + rel_embeddings=rel_embeddings, + ) + if output_attentions: + self_output, att_matrix = self_output + if query_states is None: + query_states = hidden_states + attention_output = self.output(self_output, query_states) + + if output_attentions: + return (attention_output, att_matrix) + else: + return attention_output + + +# Copied from transformers.models.bert.modeling_bert.BertIntermediate with Bert->DebertaV2 +class DebertaV2Intermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaOutput with DebertaLayerNorm->LayerNorm +class DebertaV2Output(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps) + self.dropout = StableDropout(config.hidden_dropout_prob) + self.config = config + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaLayer with Deberta->DebertaV2 +class DebertaV2Layer(nn.Module): + + def __init__(self, config): + super().__init__() + self.attention = DebertaV2Attention(config) + self.intermediate = DebertaV2Intermediate(config) + self.output = DebertaV2Output(config) + + def forward( + self, + hidden_states, + attention_mask, + query_states=None, + relative_pos=None, + rel_embeddings=None, + output_attentions=False, + ): + attention_output = self.attention( + hidden_states, + attention_mask, + output_attentions=output_attentions, + query_states=query_states, + relative_pos=relative_pos, + rel_embeddings=rel_embeddings, + ) + if output_attentions: + attention_output, att_matrix = attention_output + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + if output_attentions: + return (layer_output, att_matrix) + else: + return layer_output + + +class ConvLayer(nn.Module): + + def __init__(self, config): + super().__init__() + kernel_size = getattr(config, 'conv_kernel_size', 3) + groups = getattr(config, 'conv_groups', 1) + self.conv_act = getattr(config, 'conv_act', 'tanh') + self.conv = nn.Conv1d( + config.hidden_size, + config.hidden_size, + kernel_size, + padding=(kernel_size - 1) // 2, + groups=groups) + self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps) + self.dropout = StableDropout(config.hidden_dropout_prob) + self.config = config + + def forward(self, hidden_states, residual_states, input_mask): + out = self.conv(hidden_states.permute(0, 2, 1).contiguous()).permute( + 0, 2, 1).contiguous() + rmask = (1 - input_mask).bool() + out.masked_fill_(rmask.unsqueeze(-1).expand(out.size()), 0) + out = ACT2FN[self.conv_act](self.dropout(out)) + + layer_norm_input = residual_states + out + output = self.LayerNorm(layer_norm_input).to(layer_norm_input) + + if input_mask is None: + output_states = output + else: + if input_mask.dim() != layer_norm_input.dim(): + if input_mask.dim() == 4: + input_mask = input_mask.squeeze(1).squeeze(1) + input_mask = input_mask.unsqueeze(2) + + input_mask = input_mask.to(output.dtype) + output_states = output * input_mask + + return output_states + + +class DebertaV2Encoder(nn.Module): + """Modified BertEncoder with relative position bias support""" + + def __init__(self, config): + super().__init__() + + self.layer = nn.ModuleList( + [DebertaV2Layer(config) for _ in range(config.num_hidden_layers)]) + self.relative_attention = getattr(config, 'relative_attention', False) + + if self.relative_attention: + self.max_relative_positions = getattr(config, + 'max_relative_positions', -1) + if self.max_relative_positions < 1: + self.max_relative_positions = config.max_position_embeddings + + self.position_buckets = getattr(config, 'position_buckets', -1) + pos_ebd_size = self.max_relative_positions * 2 + + if self.position_buckets > 0: + pos_ebd_size = self.position_buckets * 2 + + self.rel_embeddings = nn.Embedding(pos_ebd_size, + config.hidden_size) + + self.norm_rel_ebd = [ + x.strip() + for x in getattr(config, 'norm_rel_ebd', 'none').lower().split('|') + ] + + if 'layer_norm' in self.norm_rel_ebd: + self.LayerNorm = LayerNorm( + config.hidden_size, + config.layer_norm_eps, + elementwise_affine=True) + + self.conv = ConvLayer(config) if getattr(config, 'conv_kernel_size', + 0) > 0 else None + self.gradient_checkpointing = False + + def get_rel_embedding(self): + rel_embeddings = self.rel_embeddings.weight if self.relative_attention else None + if rel_embeddings is not None and ('layer_norm' in self.norm_rel_ebd): + rel_embeddings = self.LayerNorm(rel_embeddings) + return rel_embeddings + + def get_attention_mask(self, attention_mask): + if attention_mask.dim() <= 2: + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + attention_mask = extended_attention_mask * extended_attention_mask.squeeze( + -2).unsqueeze(-1) + attention_mask = attention_mask.byte() + elif attention_mask.dim() == 3: + attention_mask = attention_mask.unsqueeze(1) + + return attention_mask + + def get_rel_pos(self, hidden_states, query_states=None, relative_pos=None): + if self.relative_attention and relative_pos is None: + q = query_states.size( + -2) if query_states is not None else hidden_states.size(-2) + relative_pos = build_relative_position( + q, + hidden_states.size(-2), + bucket_size=self.position_buckets, + max_position=self.max_relative_positions) + return relative_pos + + def forward( + self, + hidden_states, + attention_mask, + output_hidden_states=True, + output_attentions=False, + query_states=None, + relative_pos=None, + return_dict=True, + ): + if attention_mask.dim() <= 2: + input_mask = attention_mask + else: + input_mask = (attention_mask.sum(-2) > 0).byte() + attention_mask = self.get_attention_mask(attention_mask) + relative_pos = self.get_rel_pos(hidden_states, query_states, + relative_pos) + + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + if isinstance(hidden_states, Sequence): + next_kv = hidden_states[0] + else: + next_kv = hidden_states + rel_embeddings = self.get_rel_embedding() + output_states = next_kv + for i, layer_module in enumerate(self.layer): + + if output_hidden_states: + all_hidden_states = all_hidden_states + (output_states, ) + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + output_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + next_kv, + attention_mask, + query_states, + relative_pos, + rel_embeddings, + ) + else: + output_states = layer_module( + next_kv, + attention_mask, + query_states=query_states, + relative_pos=relative_pos, + rel_embeddings=rel_embeddings, + output_attentions=output_attentions, + ) + + if output_attentions: + output_states, att_m = output_states + + if i == 0 and self.conv is not None: + output_states = self.conv(hidden_states, output_states, + input_mask) + + if query_states is not None: + query_states = output_states + if isinstance(hidden_states, Sequence): + next_kv = hidden_states[i + 1] if i + 1 < len( + self.layer) else None + else: + next_kv = output_states + + if output_attentions: + all_attentions = all_attentions + (att_m, ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (output_states, ) + + if not return_dict: + return tuple( + v for v in [output_states, all_hidden_states, all_attentions] + if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=output_states, + hidden_states=all_hidden_states, + attentions=all_attentions) + + +def make_log_bucket_position(relative_pos, bucket_size, max_position): + sign = torch.sign(relative_pos) + mid = bucket_size // 2 + abs_pos = torch.where( + (relative_pos < mid) & (relative_pos > -mid), + torch.tensor(mid - 1).type_as(relative_pos), + torch.abs(relative_pos), + ) + log_pos = ( + torch.ceil( + torch.log(abs_pos / mid) + / torch.log(torch.tensor( + (max_position - 1) / mid)) * (mid - 1)) + mid) + bucket_pos = torch.where(abs_pos <= mid, relative_pos.type_as(log_pos), + log_pos * sign) + return bucket_pos + + +def build_relative_position(query_size, + key_size, + bucket_size=-1, + max_position=-1): + """ + Build relative position according to the query and key + + We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key + \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q - + P_k\\) + + Args: + query_size (int): the length of query + key_size (int): the length of key + bucket_size (int): the size of position bucket + max_position (int): the maximum allowed absolute position + + Return: + `torch.LongTensor`: A tensor with shape [1, query_size, key_size] + + """ + q_ids = torch.arange(0, query_size) + k_ids = torch.arange(0, key_size) + rel_pos_ids = q_ids[:, None] - k_ids[None, :] + if bucket_size > 0 and max_position > 0: + rel_pos_ids = make_log_bucket_position(rel_pos_ids, bucket_size, + max_position) + rel_pos_ids = rel_pos_ids.to(torch.long) + rel_pos_ids = rel_pos_ids[:query_size, :] + rel_pos_ids = rel_pos_ids.unsqueeze(0) + return rel_pos_ids + + +@torch.jit.script +# Copied from transformers.models.deberta.modeling_deberta.c2p_dynamic_expand +def c2p_dynamic_expand(c2p_pos, query_layer, relative_pos): + return c2p_pos.expand([ + query_layer.size(0), + query_layer.size(1), + query_layer.size(2), + relative_pos.size(-1) + ]) + + +@torch.jit.script +# Copied from transformers.models.deberta.modeling_deberta.p2c_dynamic_expand +def p2c_dynamic_expand(c2p_pos, query_layer, key_layer): + return c2p_pos.expand([ + query_layer.size(0), + query_layer.size(1), + key_layer.size(-2), + key_layer.size(-2) + ]) + + +@torch.jit.script +# Copied from transformers.models.deberta.modeling_deberta.pos_dynamic_expand +def pos_dynamic_expand(pos_index, p2c_att, key_layer): + return pos_index.expand(p2c_att.size()[:2] + + (pos_index.size(-2), key_layer.size(-2))) + + +class DisentangledSelfAttention(nn.Module): + """ + Disentangled self-attention module + + Parameters: + config (`DebertaV2Config`): + A model config class instance with the configuration to build a new model. The schema is similar to + *BertConfig*, for more details, please refer [`DebertaV2Config`] + + """ + + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + self.num_attention_heads = config.num_attention_heads + _attention_head_size = config.hidden_size // config.num_attention_heads + self.attention_head_size = getattr(config, 'attention_head_size', + _attention_head_size) + self.all_head_size = self.num_attention_heads * self.attention_head_size + self.query_proj = nn.Linear( + config.hidden_size, self.all_head_size, bias=True) + self.key_proj = nn.Linear( + config.hidden_size, self.all_head_size, bias=True) + self.value_proj = nn.Linear( + config.hidden_size, self.all_head_size, bias=True) + + self.share_att_key = getattr(config, 'share_att_key', False) + self.pos_att_type = config.pos_att_type if config.pos_att_type is not None else [] + self.relative_attention = getattr(config, 'relative_attention', False) + + if self.relative_attention: + self.position_buckets = getattr(config, 'position_buckets', -1) + self.max_relative_positions = getattr(config, + 'max_relative_positions', -1) + if self.max_relative_positions < 1: + self.max_relative_positions = config.max_position_embeddings + self.pos_ebd_size = self.max_relative_positions + if self.position_buckets > 0: + self.pos_ebd_size = self.position_buckets + + self.pos_dropout = StableDropout(config.hidden_dropout_prob) + + if not self.share_att_key: + if 'c2p' in self.pos_att_type: + self.pos_key_proj = nn.Linear( + config.hidden_size, self.all_head_size, bias=True) + if 'p2c' in self.pos_att_type: + self.pos_query_proj = nn.Linear(config.hidden_size, + self.all_head_size) + + self.dropout = StableDropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, attention_heads): + new_x_shape = x.size()[:-1] + (attention_heads, -1) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3).contiguous().view(-1, x.size(1), + x.size(-1)) + + def forward( + self, + hidden_states, + attention_mask, + output_attentions=False, + query_states=None, + relative_pos=None, + rel_embeddings=None, + ): + """ + Call the module + + Args: + hidden_states (`torch.FloatTensor`): + Input states to the module usually the output from previous layer, it will be the Q,K and V in + *Attention(Q,K,V)* + + attention_mask (`torch.ByteTensor`): + An attention mask matrix of shape [*B*, *N*, *N*] where *B* is the batch size, *N* is the maximum + sequence length in which element [i,j] = *1* means the *i* th token in the input can attend to the *j* + th token. + + output_attentions (`bool`, optional): + Whether return the attention matrix. + + query_states (`torch.FloatTensor`, optional): + The *Q* state in *Attention(Q,K,V)*. + + relative_pos (`torch.LongTensor`): + The relative position encoding between the tokens in the sequence. It's of shape [*B*, *N*, *N*] with + values ranging in [*-max_relative_positions*, *max_relative_positions*]. + + rel_embeddings (`torch.FloatTensor`): + The embedding of relative distances. It's a tensor of shape [\\(2 \\times + \\text{max_relative_positions}\\), *hidden_size*]. + + + """ + if query_states is None: + query_states = hidden_states + query_layer = self.transpose_for_scores( + self.query_proj(query_states), self.num_attention_heads) + key_layer = self.transpose_for_scores( + self.key_proj(hidden_states), self.num_attention_heads) + value_layer = self.transpose_for_scores( + self.value_proj(hidden_states), self.num_attention_heads) + + rel_att = None + # Take the dot product between "query" and "key" to get the raw attention scores. + scale_factor = 1 + if 'c2p' in self.pos_att_type: + scale_factor += 1 + if 'p2c' in self.pos_att_type: + scale_factor += 1 + scale = torch.sqrt( + torch.tensor(query_layer.size(-1), dtype=torch.float) + * scale_factor) + attention_scores = torch.bmm(query_layer, key_layer.transpose( + -1, -2)) / torch.tensor( + scale, dtype=query_layer.dtype) + if self.relative_attention: + rel_embeddings = self.pos_dropout(rel_embeddings) + rel_att = self.disentangled_attention_bias(query_layer, key_layer, + relative_pos, + rel_embeddings, + scale_factor) + + if rel_att is not None: + attention_scores = attention_scores + rel_att + attention_scores = attention_scores + attention_scores = attention_scores.view(-1, self.num_attention_heads, + attention_scores.size(-2), + attention_scores.size(-1)) + + # bsz x height x length x dimension + attention_probs = XSoftmax.apply(attention_scores, attention_mask, -1) + attention_probs = self.dropout(attention_probs) + context_layer = torch.bmm( + attention_probs.view(-1, attention_probs.size(-2), + attention_probs.size(-1)), value_layer) + context_layer = ( + context_layer.view(-1, self.num_attention_heads, + context_layer.size(-2), + context_layer.size(-1)).permute(0, 2, 1, + 3).contiguous()) + new_context_layer_shape = context_layer.size()[:-2] + (-1, ) + context_layer = context_layer.view(new_context_layer_shape) + if output_attentions: + return (context_layer, attention_probs) + else: + return context_layer + + def disentangled_attention_bias(self, query_layer, key_layer, relative_pos, + rel_embeddings, scale_factor): + if relative_pos is None: + q = query_layer.size(-2) + relative_pos = build_relative_position( + q, + key_layer.size(-2), + bucket_size=self.position_buckets, + max_position=self.max_relative_positions) + if relative_pos.dim() == 2: + relative_pos = relative_pos.unsqueeze(0).unsqueeze(0) + elif relative_pos.dim() == 3: + relative_pos = relative_pos.unsqueeze(1) + # bsz x height x query x key + elif relative_pos.dim() != 4: + raise ValueError( + f'Relative position ids must be of dim 2 or 3 or 4. {relative_pos.dim()}' + ) + + att_span = self.pos_ebd_size + relative_pos = relative_pos.long().to(query_layer.device) + + rel_embeddings = rel_embeddings[0:att_span * 2, :].unsqueeze(0) + if self.share_att_key: + pos_query_layer = self.transpose_for_scores( + self.query_proj(rel_embeddings), + self.num_attention_heads).repeat( + query_layer.size(0) // self.num_attention_heads, 1, 1) + pos_key_layer = self.transpose_for_scores( + self.key_proj(rel_embeddings), + self.num_attention_heads).repeat( + query_layer.size(0) // self.num_attention_heads, 1, 1) + else: + if 'c2p' in self.pos_att_type: + pos_key_layer = self.transpose_for_scores( + self.pos_key_proj(rel_embeddings), + self.num_attention_heads).repeat( + query_layer.size(0) // self.num_attention_heads, 1, + 1) # .split(self.all_head_size, dim=-1) + if 'p2c' in self.pos_att_type: + pos_query_layer = self.transpose_for_scores( + self.pos_query_proj(rel_embeddings), + self.num_attention_heads).repeat( + query_layer.size(0) // self.num_attention_heads, 1, + 1) # .split(self.all_head_size, dim=-1) + + score = 0 + # content->position + if 'c2p' in self.pos_att_type: + scale = torch.sqrt( + torch.tensor(pos_key_layer.size(-1), dtype=torch.float) + * scale_factor) + c2p_att = torch.bmm(query_layer, pos_key_layer.transpose(-1, -2)) + c2p_pos = torch.clamp(relative_pos + att_span, 0, att_span * 2 - 1) + c2p_att = torch.gather( + c2p_att, + dim=-1, + index=c2p_pos.squeeze(0).expand([ + query_layer.size(0), + query_layer.size(1), + relative_pos.size(-1) + ]), + ) + score += c2p_att / torch.tensor(scale, dtype=c2p_att.dtype) + + # position->content + if 'p2c' in self.pos_att_type: + scale = torch.sqrt( + torch.tensor(pos_query_layer.size(-1), dtype=torch.float) + * scale_factor) + if key_layer.size(-2) != query_layer.size(-2): + r_pos = build_relative_position( + key_layer.size(-2), + key_layer.size(-2), + bucket_size=self.position_buckets, + max_position=self.max_relative_positions, + ).to(query_layer.device) + r_pos = r_pos.unsqueeze(0) + else: + r_pos = relative_pos + + p2c_pos = torch.clamp(-r_pos + att_span, 0, att_span * 2 - 1) + p2c_att = torch.bmm(key_layer, pos_query_layer.transpose(-1, -2)) + p2c_att = torch.gather( + p2c_att, + dim=-1, + index=p2c_pos.squeeze(0).expand([ + query_layer.size(0), + key_layer.size(-2), + key_layer.size(-2) + ]), + ).transpose(-1, -2) + score += p2c_att / torch.tensor(scale, dtype=p2c_att.dtype) + + return score + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaEmbeddings with DebertaLayerNorm->LayerNorm +class DebertaV2Embeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + pad_token_id = getattr(config, 'pad_token_id', 0) + self.embedding_size = getattr(config, 'embedding_size', + config.hidden_size) + self.word_embeddings = nn.Embedding( + config.vocab_size, self.embedding_size, padding_idx=pad_token_id) + + self.position_biased_input = getattr(config, 'position_biased_input', + True) + if not self.position_biased_input: + self.position_embeddings = None + else: + self.position_embeddings = nn.Embedding( + config.max_position_embeddings, self.embedding_size) + + if config.type_vocab_size > 0: + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + self.embedding_size) + + if self.embedding_size != config.hidden_size: + self.embed_proj = nn.Linear( + self.embedding_size, config.hidden_size, bias=False) + self.LayerNorm = LayerNorm(config.hidden_size, config.layer_norm_eps) + self.dropout = StableDropout(config.hidden_dropout_prob) + self.config = config + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + mask=None, + inputs_embeds=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, :seq_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + if self.position_embeddings is not None: + position_embeddings = self.position_embeddings(position_ids.long()) + else: + position_embeddings = torch.zeros_like(inputs_embeds) + + embeddings = inputs_embeds + if self.position_biased_input: + embeddings += position_embeddings + if self.config.type_vocab_size > 0: + token_type_embeddings = self.token_type_embeddings(token_type_ids) + embeddings += token_type_embeddings + + if self.embedding_size != self.config.hidden_size: + embeddings = self.embed_proj(embeddings) + + embeddings = self.LayerNorm(embeddings) + + if mask is not None: + if mask.dim() != embeddings.dim(): + if mask.dim() == 4: + mask = mask.squeeze(1).squeeze(1) + mask = mask.unsqueeze(2) + mask = mask.to(embeddings.dtype) + + embeddings = embeddings * mask + + embeddings = self.dropout(embeddings) + return embeddings + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaPreTrainedModel with Deberta->DebertaV2 +class DebertaV2PreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = DebertaV2Config + base_model_prefix = 'deberta' + _keys_to_ignore_on_load_missing = ['position_ids'] + _keys_to_ignore_on_load_unexpected = ['position_embeddings'] + supports_gradient_checkpointing = True + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights.""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, DebertaV2Encoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + ponet_config = DebertaV2Config(**kwargs) + model = cls(ponet_config) + else: + model = super( + Model, + cls).from_pretrained(pretrained_model_name_or_path=model_dir) + return model + + +@MODELS.register_module(Tasks.backbone, module_name=Models.deberta_v2) +# Copied from transformers.models.deberta.modeling_deberta.DebertaModel with Deberta->DebertaV2 +class DebertaV2Model(DebertaV2PreTrainedModel): + """The bare DeBERTa_v2 Model transformer outputting raw hidden-states without any specific head on top. + + The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled + Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build + on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two + improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data. + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config (`DebertaV2Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. + """ + + def __init__(self, config, **kwargs): + super().__init__(config) + + self.embeddings = DebertaV2Embeddings(config) + self.encoder = DebertaV2Encoder(config) + self.z_steps = 0 + self.config = config + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, new_embeddings): + self.embeddings.word_embeddings = new_embeddings + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + raise NotImplementedError( + 'The prune function is not implemented in DeBERTa model.') + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, AttentionBackboneModelOutput]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`): + Indices of input sequence tokens in the vocabulary. + + attention_mask (`torch.FloatTensor` of shape `('batch_size, sequence_length')`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + position_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + `[0,config.max_position_embeddings - 1]`. + + inputs_embeds (`torch.FloatTensor` of shape `('batch_size, sequence_length', hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert *input_ids* indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a dataclass instead of a plain tuple. + + Returns: + Returns `modelscope.outputs.AttentionBackboneModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_debertav2_fill-mask_chinese-lite', task='backbone') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_debertav2_fill-mask_chinese-lite') + >>> print(model(**preprocessor('这是个测试'))) + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + embedding_output = self.embeddings( + input_ids=input_ids, + token_type_ids=token_type_ids, + position_ids=position_ids, + mask=attention_mask, + inputs_embeds=inputs_embeds, + ) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask, + output_hidden_states=True, + output_attentions=output_attentions, + return_dict=return_dict, + ) + encoded_layers = encoder_outputs[1] + + if self.z_steps > 1: + hidden_states = encoded_layers[-2] + layers = [self.encoder.layer[-1] for _ in range(self.z_steps)] + query_states = encoded_layers[-1] + rel_embeddings = self.encoder.get_rel_embedding() + attention_mask = self.encoder.get_attention_mask(attention_mask) + rel_pos = self.encoder.get_rel_pos(embedding_output) + for layer in layers[1:]: + query_states = layer( + hidden_states, + attention_mask, + output_attentions=False, + query_states=query_states, + relative_pos=rel_pos, + rel_embeddings=rel_embeddings, + ) + encoded_layers.append(query_states) + + sequence_output = encoded_layers[-1] + + if not return_dict: + return (sequence_output, ) + encoder_outputs[ + (1 if output_hidden_states else 2):] + + return AttentionBackboneModelOutput( + last_hidden_state=sequence_output, + hidden_states=encoder_outputs.hidden_states + if output_hidden_states else None, + attentions=encoder_outputs.attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/configuration.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..351621f65952804e8e5171ce21d8f00692fd7ac3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/configuration.py @@ -0,0 +1,128 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020, Microsoft and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" DeBERTa-v2 model configuration, mainly copied from :class:`~transformers.DeBERTaV2Config""" + +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class DebertaV2Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`DebertaV2Model`]. It is used to instantiate a + DeBERTa-v2 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the DeBERTa + [microsoft/deberta-v2-xlarge](https://huggingface.co/microsoft/deberta-v2-xlarge) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Arguments: + vocab_size (`int`, *optional*, defaults to 128100): + Vocabulary size of the DeBERTa-v2 model. Defines the number of different tokens that can be represented by + the `inputs_ids` passed when calling [`DebertaV2Model`]. + hidden_size (`int`, *optional*, defaults to 1536): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 24): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 24): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 6144): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"`, `"gelu"`, `"tanh"`, `"gelu_fast"`, `"mish"`, `"linear"`, `"sigmoid"` and `"gelu_new"` + are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 0): + The vocabulary size of the `token_type_ids` passed when calling [`DebertaModel`] or [`TFDebertaModel`]. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-7): + The epsilon used by the layer normalization layers. + relative_attention (`bool`, *optional*, defaults to `True`): + Whether use relative position encoding. + max_relative_positions (`int`, *optional*, defaults to -1): + The range of relative positions `[-max_position_embeddings, max_position_embeddings]`. Use the same value + as `max_position_embeddings`. + pad_token_id (`int`, *optional*, defaults to 0): + The value used to pad input_ids. + position_biased_input (`bool`, *optional*, defaults to `False`): + Whether add absolute position embedding to content embedding. + pos_att_type (`List[str]`, *optional*): + The type of relative position attention, it can be a combination of `["p2c", "c2p"]`, e.g. `["p2c"]`, + `["p2c", "c2p"]`, `["p2c", "c2p"]`. + layer_norm_eps (`float`, optional, defaults to 1e-12): + The epsilon used by the layer normalization layers. + """ + model_type = 'deberta_v2' + + def __init__(self, + vocab_size=128100, + hidden_size=1536, + num_hidden_layers=24, + num_attention_heads=24, + intermediate_size=6144, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=0, + initializer_range=0.02, + layer_norm_eps=1e-7, + relative_attention=False, + max_relative_positions=-1, + pad_token_id=0, + position_biased_input=True, + pos_att_type=None, + pooler_dropout=0, + pooler_hidden_act='gelu', + **kwargs): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.relative_attention = relative_attention + self.max_relative_positions = max_relative_positions + self.pad_token_id = pad_token_id + self.position_biased_input = position_biased_input + + # Backwards compatibility + if type(pos_att_type) == str: + pos_att_type = [x.strip() for x in pos_att_type.lower().split('|')] + + self.pos_att_type = pos_att_type + self.vocab_size = vocab_size + self.layer_norm_eps = layer_norm_eps + + self.pooler_hidden_size = kwargs.get('pooler_hidden_size', hidden_size) + self.pooler_dropout = pooler_dropout + self.pooler_hidden_act = pooler_hidden_act diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/fill_mask.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..e8adf1b5b2535a5c211f3604967c16a7ccbe3111 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/fill_mask.py @@ -0,0 +1,230 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 Microsoft and the Hugging Face Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionFillMaskModelOutput +from modelscope.utils.constant import Tasks +from .backbone import DebertaV2Model, DebertaV2PreTrainedModel + + +# Copied from transformers.models.deberta.modeling_deberta.DebertaForMaskedLM with Deberta->DebertaV2 +@MODELS.register_module(Tasks.fill_mask, module_name=Models.deberta_v2) +class DebertaV2ForMaskedLM(DebertaV2PreTrainedModel): + r"""DeBERTa_v2 Model with a `language modeling` head on top. + + The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled + Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build + on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two + improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data. + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Preprocessor: + This is the fill_mask model of Deberta_v2, the preprocessor of this model + is `modelscope.preprocessors.FillMaskTransformersPreprocessor`. + + Parameters: + config (`DebertaV2Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. + """ + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config, **kwargs): + super().__init__(config) + + self.deberta = DebertaV2Model(config) + self.cls = DebertaV2OnlyMLMHead(config) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, AttentionFillMaskModelOutput]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`): + Indices of input sequence tokens in the vocabulary. + + attention_mask (`torch.FloatTensor` of shape `('batch_size, sequence_length')`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + position_ids (`torch.LongTensor` of shape `('batch_size, sequence_length')`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range `[0, config.max_position_embeddings - 1]`. + + inputs_embeds (`torch.FloatTensor` of shape `('batch_size, sequence_length', hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert *input_ids* indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a dataclass instead of a plain tuple. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + + Returns: + Returns `modelscope.outputs.AttentionFillMaskModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_debertav2_fill-mask_chinese-lite') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_debertav2_fill-mask_chinese-lite') + >>> # Call the model, return some tensors + >>> print(model(**preprocessor('你师父差得动你,你师父可[MASK]不动我。'))) + >>> # Call the pipeline + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('fill-mask', model=model, preprocessor=preprocessor) + >>> print(pipeline_ins('你师父差得动你,你师父可[MASK]不动我。')) + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.deberta( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + if not return_dict: + output = (prediction_scores, ) + outputs[1:] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return AttentionFillMaskModelOutput( + loss=masked_lm_loss, + logits=prediction_scores, + input_ids=input_ids, + attentions=outputs.attentions, + hidden_states=outputs.hidden_states) + + +# copied from transformers.models.bert.BertPredictionHeadTransform with bert -> deberta +class DebertaV2PredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +# copied from transformers.models.bert.BertLMPredictionHead with bert -> deberta +class DebertaV2LMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = DebertaV2PredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +# copied from transformers.models.bert.BertOnlyMLMHead with bert -> deberta +class DebertaV2OnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = DebertaV2LMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..adb60288661b6a4cf5c6a2630b45346751155274 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization.py @@ -0,0 +1,546 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 Microsoft and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for DeBERTa. mainly copied from :module:`~transformers.tokenization_deberta`""" + +import os +import unicodedata +from typing import Any, Dict, List, Optional, Tuple + +import sentencepiece as sp +from transformers.tokenization_utils import PreTrainedTokenizer + +PRETRAINED_VOCAB_FILES_MAP = {'vocab_file': {}} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} + +PRETRAINED_INIT_CONFIGURATION = {} + +VOCAB_FILES_NAMES = {'vocab_file': 'spm.model'} + + +class DebertaV2Tokenizer(PreTrainedTokenizer): + r""" + Constructs a DeBERTa-v2 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece) + and [jieba](https://github.com/fxsjy/jieba). + + Args: + vocab_file (`str`): + [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that + contains the vocabulary necessary to instantiate a tokenizer. + do_lower_case (`bool`, *optional*, defaults to `False`): + Whether or not to lowercase the input when tokenizing. + bos_token (`string`, *optional*, defaults to `"[CLS]"`): + The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token. + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + eos_token (`string`, *optional*, defaults to `"[SEP]"`): + The end of sequence token. When building a sequence using special tokens, this is not the token that is + used for the end of sequence. The token used is the `sep_token`. + unk_token (`str`, *optional*, defaults to `"[UNK]"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + sep_token (`str`, *optional*, defaults to `"[SEP]"`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + pad_token (`str`, *optional*, defaults to `"[PAD]"`): + The token used for padding, for example when batching sequences of different lengths. + cls_token (`str`, *optional*, defaults to `"[CLS]"`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + mask_token (`str`, *optional*, defaults to `"[MASK]"`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + sp_model_kwargs (`dict`, *optional*): + Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for + SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, + to set: + + - `enable_sampling`: Enable subword regularization. + - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. + + - `nbest_size = {0,1}`: No sampling is performed. + - `nbest_size > 1`: samples from the nbest_size results. + - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) + using forward-filtering-and-backward-sampling algorithm. + + - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for + BPE-dropout. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, + vocab_file, + do_lower_case=False, + split_by_punct=False, + split_chinese=True, + bos_token='[CLS]', + eos_token='[SEP]', + unk_token='[UNK]', + sep_token='[SEP]', + pad_token='[PAD]', + cls_token='[CLS]', + mask_token='[MASK]', + sp_model_kwargs: Optional[Dict[str, Any]] = None, + **kwargs) -> None: + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + + super().__init__( + do_lower_case=do_lower_case, + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + split_by_punct=split_by_punct, + split_chinese=split_chinese, + sp_model_kwargs=self.sp_model_kwargs, + **kwargs, + ) + + if not os.path.isfile(vocab_file): + raise ValueError( + f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained" + ' model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' + ) + self.do_lower_case = do_lower_case + self.split_by_punct = split_by_punct + self.split_chinese = split_chinese + self.vocab_file = vocab_file + self._tokenizer = SPMTokenizer( + vocab_file, + split_by_punct=split_by_punct, + sp_model_kwargs=self.sp_model_kwargs) + self.jieba = None + if self.split_chinese: + try: + import jieba + except ImportError: + raise ImportError( + 'You need to install jieba to split chinese and use DebertaV2Tokenizer. ' + 'See https://pypi.org/project/jieba/ for installation.') + self.jieba = jieba + + @property + def vocab_size(self): + return len(self.vocab) + + @property + def vocab(self): + return self._tokenizer.vocab + + def get_vocab(self): + vocab = self.vocab.copy() + vocab.update(self.get_added_vocab()) + return vocab + + def _tokenize(self, text: str) -> List[str]: + """Take as input a string and return a list of strings (tokens) for words/sub-words""" + if self.do_lower_case: + text = text.lower() + if self.split_chinese: + seg_list = [x for x in self.jieba.cut(text)] + text = ' '.join(seg_list) + return self._tokenizer.tokenize(text) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self._tokenizer.spm.PieceToId(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self._tokenizer.spm.IdToPiece( + index) if index < self.vocab_size else self.unk_token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + return self._tokenizer.decode(tokens) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A DeBERTa sequence has the following format: + + - single sequence: [CLS] X [SEP] + - pair of sequences: [CLS] A [SEP] B [SEP] + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, + token_ids_0, + token_ids_1=None, + already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ( + [0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences(self, + token_ids_0, + token_ids_1=None): + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. A DeBERTa + sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + + sep) * [1] + + def prepare_for_tokenization(self, + text, + is_split_into_words=False, + **kwargs): + add_prefix_space = kwargs.pop('add_prefix_space', False) + if is_split_into_words or add_prefix_space: + text = ' ' + text + return (text, kwargs) + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + return self._tokenizer.save_pretrained( + save_directory, filename_prefix=filename_prefix) + + +class SPMTokenizer: + r""" + Constructs a tokenizer based on [SentencePiece](https://github.com/google/sentencepiece). + + Args: + vocab_file (`str`): + [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that + contains the vocabulary necessary to instantiate a tokenizer. + sp_model_kwargs (`dict`, *optional*): + Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for + SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, + to set: + + - `enable_sampling`: Enable subword regularization. + - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. + + - `nbest_size = {0,1}`: No sampling is performed. + - `nbest_size > 1`: samples from the nbest_size results. + - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) + using forward-filtering-and-backward-sampling algorithm. + + - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for + BPE-dropout. + """ + + def __init__(self, + vocab_file, + split_by_punct=False, + sp_model_kwargs: Optional[Dict[str, Any]] = None): + self.split_by_punct = split_by_punct + self.vocab_file = vocab_file + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + spm = sp.SentencePieceProcessor(**self.sp_model_kwargs) + if not os.path.exists(vocab_file): + raise FileNotFoundError(f'{vocab_file} does not exist!') + spm.load(vocab_file) + bpe_vocab_size = spm.GetPieceSize() + # Token map + # 0+1 + # 1+1 + # 2+1 + self.vocab = {spm.IdToPiece(i): i for i in range(bpe_vocab_size)} + self.ids_to_tokens = [spm.IdToPiece(i) for i in range(bpe_vocab_size)] + # self.vocab['[PAD]'] = 0 + # self.vocab['[CLS]'] = 1 + # self.vocab['[SEP]'] = 2 + # self.vocab['[UNK]'] = 3 + + self.spm = spm + + def __getstate__(self): + state = self.__dict__.copy() + state['spm'] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + + # for backward compatibility + if not hasattr(self, 'sp_model_kwargs'): + self.sp_model_kwargs = {} + + self.spm = sp.SentencePieceProcessor(**self.sp_model_kwargs) + self.spm.Load(self.vocab_file) + + def tokenize(self, text): + return self._encode_as_pieces(text) + + def convert_ids_to_tokens(self, ids): + tokens = [] + for i in ids: + tokens.append(self.ids_to_tokens[i]) + return tokens + + def decode(self, tokens, start=-1, end=-1, raw_text=None): + if raw_text is None: + return self.spm.decode_pieces([t for t in tokens]) + else: + words = self.split_to_words(raw_text) + word_tokens = [self.tokenize(w) for w in words] + token2words = [0] * len(tokens) + tid = 0 + for i, w in enumerate(word_tokens): + for k, t in enumerate(w): + token2words[tid] = i + tid += 1 + word_start = token2words[start] + word_end = token2words[end] if end < len(tokens) else len(words) + text = ''.join(words[word_start:word_end]) + return text + + def add_special_token(self, token): + if token not in self.special_tokens: + self.special_tokens.append(token) + if token not in self.vocab: + self.vocab[token] = len(self.vocab) - 1 + self.ids_to_tokens.append(token) + return self.id(token) + + def part_of_whole_word(self, token, is_bos=False): + if is_bos: + return True + if (len(token) == 1 and (_is_whitespace(list(token)[0]))): + return False + if _is_control(list(token)[0]): + return False + if _is_punctuation(list(token)[0]): + return False + if token in self.add_special_token: + return False + + word_start = b'\xe2\x96\x81'.decode('utf-8') + return not token.startswith(word_start) + + def pad(self): + return '[PAD]' + + def bos(self): + return '[CLS]' + + def eos(self): + return '[SEP]' + + def unk(self): + return '[UNK]' + + def mask(self): + return '[MASK]' + + def sym(self, id): + return self.ids_to_tokens[id] + + def id(self, sym): + return self.vocab[sym] if sym in self.vocab else 1 + + def _encode_as_pieces(self, text): + text = convert_to_unicode(text) + if self.split_by_punct: + words = self._run_split_on_punc(text) + pieces = [self.spm.encode(w, out_type=str) for w in words] + return [p for w in pieces for p in w] + else: + return self.spm.encode(text, out_type=str) + + def split_to_words(self, text): + pieces = self._encode_as_pieces(text) + word_start = b'\xe2\x96\x81'.decode('utf-8') + words = [] + offset = 0 + prev_end = 0 + for i, p in enumerate(pieces): + if p.startswith(word_start): + if offset > prev_end: + words.append(text[prev_end:offset]) + prev_end = offset + w = p.replace(word_start, '') + else: + w = p + try: + s = text.index(w, offset) + pn = '' + k = i + 1 + while k < len(pieces): + pn = pieces[k].replace(word_start, '') + if len(pn) > 0: + break + k += 1 + + if len(pn) > 0 and pn in text[offset:s]: + offset = offset + 1 + else: + offset = s + len(w) + except Exception: + offset = offset + 1 + + if prev_end < offset: + words.append(text[prev_end:offset]) + + return words + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize('NFD', text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == 'Mn': + continue + output.append(char) + return ''.join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return [''.join(x) for x in output] + + def save_pretrained(self, path: str, filename_prefix: str = None): + filename = VOCAB_FILES_NAMES[list(VOCAB_FILES_NAMES.keys())[0]] + if filename_prefix is not None: + filename = filename_prefix + '-' + filename + full_path = os.path.join(path, filename) + with open(full_path, 'wb') as fs: + fs.write(self.spm.serialized_model_proto()) + return (full_path, ) + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically control characters but we treat them + # as whitespace since they are generally considered as such. + if char == ' ' or char == '\t' or char == '\n' or char == '\r': + return True + cat = unicodedata.category(char) + if cat == 'Zs': + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == '\t' or char == '\n' or char == '\r': + return False + cat = unicodedata.category(char) + if cat.startswith('C'): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or ( + cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126): + return True + cat = unicodedata.category(char) + if cat.startswith('P'): + return True + return False + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode('utf-8', 'ignore') + else: + raise ValueError(f'Unsupported string type: {type(text)}') diff --git a/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization_fast.py b/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..c37b18d9dbb9f9a7bae0a5c16ad7aa72a7660b19 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/deberta_v2/tokenization_fast.py @@ -0,0 +1,241 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2020 Microsoft and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fast Tokenization class for model DeBERTa.""" + +import os +from shutil import copyfile +from typing import Optional, Tuple + +from transformers.file_utils import is_sentencepiece_available +from transformers.tokenization_utils_fast import PreTrainedTokenizerFast + +from modelscope.utils import logger as logging + +if is_sentencepiece_available(): + from .tokenization import DebertaV2Tokenizer +else: + DebertaV2Tokenizer = None + +logger = logging.get_logger() + +VOCAB_FILES_NAMES = { + 'vocab_file': 'spm.model', + 'tokenizer_file': 'tokenizer.json' +} + +PRETRAINED_VOCAB_FILES_MAP = {'vocab_file': {}} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {} + +PRETRAINED_INIT_CONFIGURATION = {} + + +class DebertaV2TokenizerFast(PreTrainedTokenizerFast): + r""" + Constructs a DeBERTa-v2 fast tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece) + and [rjieba-py](https://github.com/messense/rjieba-py). + + Args: + vocab_file (`str`): + [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that + contains the vocabulary necessary to instantiate a tokenizer. + do_lower_case (`bool`, *optional*, defaults to `False`): + Whether or not to lowercase the input when tokenizing. + bos_token (`string`, *optional*, defaults to `"[CLS]"`): + The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token. + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + eos_token (`string`, *optional*, defaults to `"[SEP]"`): + The end of sequence token. When building a sequence using special tokens, this is not the token that is + used for the end of sequence. The token used is the `sep_token`. + unk_token (`str`, *optional*, defaults to `"[UNK]"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + sep_token (`str`, *optional*, defaults to `"[SEP]"`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + pad_token (`str`, *optional*, defaults to `"[PAD]"`): + The token used for padding, for example when batching sequences of different lengths. + cls_token (`str`, *optional*, defaults to `"[CLS]"`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + mask_token (`str`, *optional*, defaults to `"[MASK]"`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + sp_model_kwargs (`dict`, *optional*): + Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for + SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, + to set: + + - `enable_sampling`: Enable subword regularization. + - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. + + - `nbest_size = {0,1}`: No sampling is performed. + - `nbest_size > 1`: samples from the nbest_size results. + - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) + using forward-filtering-and-backward-sampling algorithm. + + - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for + BPE-dropout. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + slow_tokenizer_class = DebertaV2Tokenizer + + def __init__(self, + vocab_file=None, + tokenizer_file=None, + do_lower_case=False, + split_by_punct=False, + split_chinese=True, + bos_token='[CLS]', + eos_token='[SEP]', + unk_token='[UNK]', + sep_token='[SEP]', + pad_token='[PAD]', + cls_token='[CLS]', + mask_token='[MASK]', + **kwargs) -> None: + super().__init__( + vocab_file, + tokenizer_file=tokenizer_file, + do_lower_case=do_lower_case, + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + split_by_punct=split_by_punct, + split_chinese=split_chinese, + **kwargs, + ) + + self.do_lower_case = do_lower_case + self.split_by_punct = split_by_punct + self.split_chinese = split_chinese + self.vocab_file = vocab_file + self.can_save_slow_tokenizer = False if not self.vocab_file else True + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A DeBERTa sequence has the following format: + + - single sequence: [CLS] X [SEP] + - pair of sequences: [CLS] A [SEP] B [SEP] + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, + token_ids_0, + token_ids_1=None, + already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ( + [0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences(self, + token_ids_0, + token_ids_1=None): + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. A DeBERTa + sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + + sep) * [1] + + def save_vocabulary(self, + save_directory: str, + filename_prefix: Optional[str] = None) -> Tuple[str]: + if not self.can_save_slow_tokenizer: + raise ValueError( + 'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow ' + 'tokenizer.') + + if not os.path.isdir(save_directory): + logger.error( + f'Vocabulary path ({save_directory}) should be a directory') + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + '-' if filename_prefix else '') + + VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file, ) diff --git a/runtime/Lib/site_packages/models/nlp/dgds/__init__.py b/runtime/Lib/site_packages/models/nlp/dgds/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..911d8038f87b091fbfbea9d985d38050733afbb7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/dgds/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .document_grounded_dialog_generate import DocumentGroundedDialogGenerateModel + from .document_grounded_dialog_rerank import DocumentGroundedDialogRerankModel + from .document_grounded_dialog_retrieval import DocumentGroundedDialogRetrievalModel +else: + _import_structure = { + 'document_grounded_dialog_generate': + ['DocumentGroundedDialogGenerateModel'], + 'document_grounded_dialog_rerank': + ['DocumentGroundedDialogRerankModel'], + 'document_grounded_dialog_retrieval': + ['DocumentGroundedDialogRetrievalModel'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/dgds/backbone.py b/runtime/Lib/site_packages/models/nlp/dgds/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..17e3c5746bc544172cc702e3a13f9b48cacb9f16 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/dgds/backbone.py @@ -0,0 +1,190 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function +import os.path + +import torch +import torch.nn.functional as F +from torch import nn +from torch.utils.checkpoint import checkpoint +from transformers import (AutoConfig, DPRConfig, DPRQuestionEncoder, + MT5ForConditionalGeneration, RagTokenForGeneration, + XLMRobertaForSequenceClassification, XLMRobertaModel, + XLMRobertaTokenizer) + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class Wrapper(nn.Module): + + def __init__(self, encoder): + super(Wrapper, self).__init__() + self.encoder = encoder + + def forward(self, input_ids, attention_mask, dummy_tensor): + return self.encoder(input_ids, attention_mask).pooler_output + + +class DPRModel(nn.Module): + + def __init__(self, model_dir, config): + super().__init__() + self.config = config + + qry_encoder = XLMRobertaModel( + config=AutoConfig.from_pretrained( + os.path.join(model_dir, 'qry_encoder'))) + ctx_encoder = XLMRobertaModel( + config=AutoConfig.from_pretrained( + os.path.join(model_dir, 'ctx_encoder'))) + self.qry_encoder = Wrapper(qry_encoder) + self.ctx_encoder = Wrapper(ctx_encoder) + self.loss_fct = nn.CrossEntropyLoss() + + @staticmethod + def encode(model, input_ids, attention_mask, gck_segment=32): + dummy_tensor = torch.ones(1, dtype=torch.float32, requires_grad=True) + pooled_output = [] + for mini_batch in range(0, input_ids.shape[0], gck_segment): + mini_batch_input_ids = input_ids[mini_batch:mini_batch + + gck_segment] + mini_batch_attention_mask = attention_mask[mini_batch:mini_batch + + gck_segment] + mini_batch_pooled_output = checkpoint(model, mini_batch_input_ids, + mini_batch_attention_mask, + dummy_tensor) + pooled_output.append(mini_batch_pooled_output) + return torch.cat(pooled_output, dim=0) + + def forward(self, + query_input_ids, + query_attention_mask, + context_input_ids, + context_attention_mask, + labels, + gck_segment=32): + query_vector = self.encode(self.qry_encoder, query_input_ids, + query_attention_mask, gck_segment) + context_vector = self.encode(self.ctx_encoder, context_input_ids, + context_attention_mask, gck_segment) + logits = torch.matmul(query_vector, context_vector.T) + loss = self.loss_fct(logits, labels) + return loss, logits + + +class ClassifyRerank(nn.Module): + + def __init__(self, model_dir): + super().__init__() + self.base_model = XLMRobertaForSequenceClassification.from_pretrained( + model_dir) + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + *args, + **kwargs): + outputs = self.base_model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + return outputs + + +class Rerank(nn.Module): + + def __init__(self, encoder, top_k): + super().__init__() + self.encoder = encoder + self.top_k = top_k + + def forward(self, inputs): + model = self.encoder + logits = F.log_softmax(model(**inputs)[0], dim=-1)[:, 1] + logits = logits.view(-1, self.top_k) + logprobs = F.log_softmax(logits, dim=-1) + return logprobs + + +class Re2GModel(nn.Module): + + def __init__(self, model_dir, config): + super(Re2GModel, self).__init__() + self.config = config + self.top_k = self.config['top_k'] + encoder = XLMRobertaForSequenceClassification( + config=AutoConfig.from_pretrained( + os.path.join(model_dir, 'rerank'))) + generator = MT5ForConditionalGeneration( + config=AutoConfig.from_pretrained( + os.path.join(model_dir, 'generation'))) + + self.rerank = Rerank(encoder, self.top_k) + + dpr_config = DPRConfig() + dpr_config.vocab_size = encoder.config.vocab_size + rag_model = RagTokenForGeneration( + question_encoder=DPRQuestionEncoder(dpr_config), + generator=generator) + rag_model.rag.question_encoder = None + self.generator = rag_model + + def forward(self, rerank_input_ids, input_ids, attention_mask, label_ids): + doc_scores = self.rerank(rerank_input_ids) + + outputs = self.generator( + labels=label_ids, + context_input_ids=input_ids, + context_attention_mask=attention_mask, + doc_scores=doc_scores, + n_docs=self.top_k) + return outputs + + def generate(self, rerank_input_ids, input_ids, attention_mask): + doc_scores = self.rerank(rerank_input_ids) + + beam_search_output = self.generator.generate( + n_docs=self.top_k, + encoder_input_ids=input_ids, + context_input_ids=input_ids, + context_attention_mask=attention_mask, + doc_scores=doc_scores, + num_beams=self.config['num_beams'], + max_length=self.config['target_sequence_length'], + early_stopping=True, + no_repeat_ngram_size=self.config['no_repeat_ngram_size'], + return_dict_in_generate=True, + output_scores=True) + generated_ids = beam_search_output.detach().cpu().numpy() + + return generated_ids diff --git a/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_generate.py b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_generate.py new file mode 100644 index 0000000000000000000000000000000000000000..27902b6709d5b51569138fd5412e13db009e69a0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_generate.py @@ -0,0 +1,49 @@ +import os +from typing import Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import Re2GModel + + +@MODELS.register_module( + Tasks.document_grounded_dialog_generate, module_name=Models.doc2bot) +class DocumentGroundedDialogGenerateModel(TorchModel): + _backbone_prefix = '' + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + self.model = Re2GModel(model_dir, self.config) + state_dict = torch.load( + os.path.join(self.model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location='cpu') + compatible_position_ids( + state_dict, 'rerank.encoder.roberta.embeddings.position_ids') + self.model.load_state_dict(state_dict) + + def forward(self, input: Dict[str, Tensor]): + rerank_input_ids = input['rerank_input_ids'] + input_ids = input['input_ids'] + attention_mask = input['attention_mask'] + label_ids = input['label_ids'] + + outputs = self.model(rerank_input_ids, input_ids, attention_mask, + label_ids) + return outputs + + def generate(self, input: Dict[str, Tensor]): + rerank_input_ids = input['rerank_input_ids'] + input_ids = input['input_ids'] + attention_mask = input['attention_mask'] + outputs = self.model.generate(rerank_input_ids, input_ids, + attention_mask) + return outputs diff --git a/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_rerank.py b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_rerank.py new file mode 100644 index 0000000000000000000000000000000000000000..2b6fc5f68ac1c4f17f8f1b1ccbfc1107ac938cd9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_rerank.py @@ -0,0 +1,34 @@ +import os +from typing import Dict + +import torch +from torch import nn + +from modelscope.metainfo import Models +from modelscope.models.base import Model, Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import ClassifyRerank + + +@MODELS.register_module( + Tasks.document_grounded_dialog_rerank, module_name=Models.doc2bot) +class DocumentGroundedDialogRerankModel(TorchModel): + _backbone_prefix = '' + + def __init__(self, model_dir, **kwargs): + super().__init__(model_dir, **kwargs) + self.model = ClassifyRerank(model_dir) + + def forward(self, input: Dict[str, Tensor]): + outputs = self.model( + input_ids=input['input_ids'], + attention_mask=input['attention_mask']) + return outputs + + def resize_token_embeddings(self, size): + self.model.base_model.resize_token_embeddings(size) + + def save_pretrained(self, addr): + self.model.base_model.save_pretrained(addr) diff --git a/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_retrieval.py b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_retrieval.py new file mode 100644 index 0000000000000000000000000000000000000000..43051b578dfb60ed22e0eb8479d69c85bc898b57 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/dgds/document_grounded_dialog_retrieval.py @@ -0,0 +1,58 @@ +import os +from typing import Dict + +import torch + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import DPRModel + + +@MODELS.register_module( + Tasks.document_grounded_dialog_retrieval, module_name=Models.doc2bot) +class DocumentGroundedDialogRetrievalModel(TorchModel): + _backbone_prefix = '' + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + self.model = DPRModel(model_dir, self.config) + state_dict = torch.load( + os.path.join(self.model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location='cpu') + compatible_position_ids(state_dict, + 'ctx_encoder.encoder.embeddings.position_ids') + compatible_position_ids(state_dict, + 'qry_encoder.encoder.embeddings.position_ids') + self.model.load_state_dict(state_dict) + + def forward(self, input: Dict[str, Tensor], gck_segment=32): + query_input_ids = input['query_input_ids'] + query_attention_mask = input['query_attention_mask'] + context_input_ids = input['context_input_ids'] + context_attention_mask = input['context_attention_mask'] + labels = input['labels'] + outputs = self.model(query_input_ids, query_attention_mask, + context_input_ids, context_attention_mask, labels, + gck_segment) + return outputs + + def encode_query(self, input: Dict[str, Tensor]): + query_input_ids = input['query_input_ids'] + query_attention_mask = input['query_attention_mask'] + query_vector = self.model.qry_encoder(query_input_ids, + query_attention_mask, None) + return query_vector + + def encode_context(self, input: Dict[str, Tensor]): + context_input_ids = input['context_input_ids'] + context_attention_mask = input['context_attention_mask'] + context_vector = self.model.ctx_encoder(context_input_ids, + context_attention_mask, None) + return context_vector diff --git a/runtime/Lib/site_packages/models/nlp/fid_T5/__init__.py b/runtime/Lib/site_packages/models/nlp/fid_T5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e94b7d3f9e239872b72f369474a16398b0325c49 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_T5/__init__.py @@ -0,0 +1,34 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .text_generation import (T5Chat, FIDT5Chat) +else: + _import_structure = { + 'text_generation': ['T5Chat', 'FIDT5Chat'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/fid_T5/text_generation.py b/runtime/Lib/site_packages/models/nlp/fid_T5/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..c9fc938f20bba5dd3f13d5f6cb82fef66f71f64f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_T5/text_generation.py @@ -0,0 +1,207 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +import os + +import torch +from transformers import AutoConfig + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.nlp.T5 import T5ForConditionalGeneration +from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput +from modelscope.utils.constant import Tasks + +WEIGHTS_NAME = 'pytorch_model.bin' + + +class T5Chat(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + # init model + config = AutoConfig.from_pretrained(model_dir) + self.backbone = T5ForConditionalGeneration(config) + # load weights + pretrained_model_path = os.path.join(model_dir, WEIGHTS_NAME) + with io.open(pretrained_model_path, 'rb') as f: + print('before torch.load..') + state_dict = torch.load(f, map_location='cpu') + state_dict = { + k.replace('module.', ''): v + for k, v in state_dict.items() + } + print('after torch.load..') + model_state_dict = self.backbone.state_dict() + model_keys = model_state_dict.keys() + + for old_name in list(state_dict.keys()): + if old_name.startswith('module.'): + new_name = old_name.replace('module.', '') + state_dict[new_name] = state_dict.pop(old_name) + old_name = new_name + if old_name.startswith('backbone.encoder.encoder'): + # wrap encoder name map + new_name = old_name.replace('backbone.encoder.encoder', + 'encoder').replace( + 'module.layer', 'layer') + state_dict[new_name] = state_dict.pop(old_name) + elif old_name.startswith('backbone'): + new_name = old_name[old_name.index('.') + + 1:] # remove backbone. prefix + state_dict[new_name] = state_dict.pop(old_name) + + missing_keys = [key for key in model_keys if key not in state_dict] + unexpected_keys = [ + key for key in state_dict if key not in model_keys + ] + mismatched_keys = [ + key for key in model_keys if key in state_dict + and state_dict[key].shape != model_state_dict[key].shape + ] + for key in mismatched_keys: + del state_dict[key] + + self.backbone.load_state_dict(state_dict, strict=False) + print( + f'| Weights loaded for {self.backbone.__class__.__name__} from {pretrained_model_path}.' + ) + if missing_keys: + print('Missing keys:\n|\t' + '\n|\t'.join(missing_keys)) + if unexpected_keys: + print('Unexpected keys:\n|\t' + '\n|\t'.join(unexpected_keys)) + if mismatched_keys: + print('Mismatched keys:\n|\t' + '\n|\t'.join(mismatched_keys)) + # self.backbone.half() + + def generate(self, input_ids, *args, **kwargs) -> TokenGeneratorOutput: + self.backbone.eval() + with torch.no_grad(): + response = self.backbone.generate(input_ids, *args, **kwargs) + return response + + def forward(self, input_ids, decoder_input_ids, *args, **kwargs): + attention_mask = input_ids.ne(0).long() + return self.backbone.forward( + input_ids=input_ids, + attention_mask=attention_mask, + labels=decoder_input_ids, + *args, + **kwargs) + + +class EncoderWrapper(torch.nn.Module): + """ + Encoder Wrapper for T5 Wrapper to obtain a Fusion-in-Decoder model. + """ + + def __init__(self, encoder): + super().__init__() + self.main_input_name = 'input_ids' + self.encoder = encoder + self.n_passages = None + + def set_n_passages(self, n_passages): + self.n_passages = n_passages + + def forward( + self, + input_ids=None, + attention_mask=None, + *args, + **kwargs, + ): + # total_length = n_passages * passage_length + bsz, total_length = input_ids.shape + passage_length = total_length // self.n_passages + input_ids = input_ids.view(bsz * self.n_passages, passage_length) + if attention_mask is not None: + attention_mask = attention_mask.view(bsz * self.n_passages, + passage_length) + outputs = self.encoder(input_ids, attention_mask, *args, **kwargs) + if isinstance(outputs, tuple): + outputs = (outputs[0].view(bsz, self.n_passages * passage_length, + -1), ) + outputs[1:] + else: + outputs.last_hidden_state = outputs.last_hidden_state.view( + bsz, self.n_passages * passage_length, -1) + return outputs + + +@MODELS.register_module(Tasks.fid_dialogue, module_name=Models.fid_T5) +class FIDT5Chat(T5Chat): + """ + T5 model with FID(fuse-in-decoder) structure, mainly for dialogue tasks + + Parameters: + model_dir: A path to a `directory` containing a configuration files to build model + """ + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.wrap_encoder() + + def wrap_encoder(self): + """ + Wrap T5 encoder to obtain a Fusion-in-Decoder model. + """ + self.backbone.encoder = EncoderWrapper(self.backbone.encoder) + + def unwrap_encoder(self): + """ + Unwrap Fusion-in-Decoder encoder, useful to load T5 weights. + """ + self.backbone.encoder = self.backbone.encoder.encoder + block = [] + for mod in self.backbone.encoder.block: + block.append(mod.module) + block = torch.nn.ModuleList(block) + self.backbone.encoder.block = block + + def load(self, + pretrained_model_path, + from_tf=False): # only invoked when model is not onnx format + self.unwrap_encoder() + super().load(pretrained_model_path) + self.wrap_encoder() + + def generate(self, inputs, *args, **kwargs) -> TokenGeneratorOutput: + input_ids = inputs.get('input_ids') + n_passages = input_ids.size(1) + + self.backbone.eval() + with torch.no_grad(): + self.backbone.encoder.set_n_passages(n_passages) + input_ids = input_ids.view(input_ids.size(0), -1) + response = super().generate(input_ids, *args, **kwargs) + return response + + def forward(self, input_ids, decoder_input_ids, *args, **kwargs): + """ + The forward function of the model. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)` + or `(batch_size, n_passages, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + decoder_input_ids (`torch.LongTensor` of shape `(batch_size,target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + Returns: + Returns `modelscope.outputs.nlp_outputs.TextGenerationModelOutput` + """ + attention_mask = input_ids.ne(0).long() + if input_ids is not None: + # inputs might have already be resized in the generate method + if input_ids.dim() == 3: + self.backbone.encoder.set_n_passages(input_ids.size(1)) + input_ids = input_ids.view(input_ids.size(0), -1) + attention_mask = attention_mask.view(attention_mask.size(0), -1) + seq2seq_lm_output = self.backbone.forward( + input_ids=input_ids, + attention_mask=attention_mask, + labels=decoder_input_ids, + *args, + **kwargs) + return TextGenerationModelOutput( + loss=seq2seq_lm_output.loss, logits=seq2seq_lm_output.logits) diff --git a/runtime/Lib/site_packages/models/nlp/fid_plug/__init__.py b/runtime/Lib/site_packages/models/nlp/fid_plug/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fe455e8415ae7f27fba4920171d49e11584057d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_plug/__init__.py @@ -0,0 +1,36 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import PlugConfig + from .text_generation import (PlugV2Chat, PlugV2FidChat) +else: + _import_structure = { + 'configuration': ['PlugConfig'], + 'text_generation': ['PlugV2Chat', 'PlugV2FidChat'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/fid_plug/backbone.py b/runtime/Lib/site_packages/models/nlp/fid_plug/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..5dcddcc15bdef0a80239ba4bfe491d1e1f95f622 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_plug/backbone.py @@ -0,0 +1,1149 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import math +import os +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor, nn +from torch.nn.init import xavier_uniform_ +from transformers import (BertConfig, BertModel, BertTokenizer, RobertaConfig, + RobertaModel, RobertaTokenizer) +from transformers.activations import ACT2FN +from transformers.modeling_utils import PreTrainedModel + +from modelscope.utils import logger as logging +from .configuration import PlugConfig + +CONFIG_NAME = 'config.json' +WEIGHTS_NAME = 'pytorch_model.bin' + + +class MultiHeadedAttention(nn.Module): # SelfAttention + """ + Multi-Head Attention module from + "Attention is All You Need" + :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`. + + Similar to standard `dot` attention but uses + multiple attention distributions simulataneously + to select relevant items. + + .. mermaid:: + + graph BT + A[key] + B[value] + C[query] + O[output] + subgraph Attn + D[Attn 1] + E[Attn 2] + F[Attn N] + end + A --> D + C --> D + A --> E + C --> E + A --> F + C --> F + D --> O + E --> O + F --> O + B --> O + + Also includes several additional tricks. + + Args: + head_count (int): number of parallel heads + model_dim (int): the dimension of keys/values/queries, + must be divisible by head_count + dropout (float): dropout parameter + """ + + def __init__(self, + head_count, + model_dim, + dropout=0.1, + use_final_linear=True): + assert model_dim % head_count == 0 + self.dim_per_head = model_dim // head_count + self.model_dim = model_dim + + super().__init__() + self.head_count = head_count + + self.linear_keys = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_values = nn.Linear(model_dim, + head_count * self.dim_per_head) + self.linear_query = nn.Linear(model_dim, + head_count * self.dim_per_head) + self.softmax = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + self.use_final_linear = use_final_linear + if (self.use_final_linear): + self.final_linear = nn.Linear(model_dim, model_dim) + + def forward(self, + key, + value, + query, + mask=None, + layer_cache=None, + type=None, + predefined_graph_1=None, + return_attn=False): + """ + Compute the context vector and the attention vectors. + + Args: + key (`FloatTensor`): set of `key_len` + key vectors `[batch, key_len, dim]` + value (`FloatTensor`): set of `key_len` + value vectors `[batch, key_len, dim]` + query (`FloatTensor`): set of `query_len` + query vectors `[batch, query_len, dim]` + mask: binary mask indicating which keys have + non-zero attention `[batch, query_len, key_len]` + Returns: + (`FloatTensor`, `FloatTensor`) : + + * output context vectors `[batch, query_len, dim]` + * one of the attention vectors `[batch, query_len, key_len]` + """ + + batch_size = key.size(0) + dim_per_head = self.dim_per_head + head_count = self.head_count + + def shape(x): + """ projection """ + return x.view(batch_size, -1, head_count, dim_per_head) \ + .transpose(1, 2) + + def unshape(x): + """ compute context """ + return x.transpose(1, 2).contiguous() \ + .view(batch_size, -1, head_count * dim_per_head) + + # 1) Project key, value, and query. + if layer_cache is not None: + if type == 'self': + query, key, value = self.linear_query(query), self.linear_keys( + query), self.linear_values(query) + + key = shape(key) + value = shape(value) + + device = key.device + if layer_cache['self_keys'] is not None: + key = torch.cat((layer_cache['self_keys'].to(device), key), + dim=2) + if layer_cache['self_values'] is not None: + value = torch.cat( + (layer_cache['self_values'].to(device), value), dim=2) + layer_cache['self_keys'] = key + layer_cache['self_values'] = value + elif type == 'context': + query = self.linear_query(query) + if layer_cache['memory_keys'] is None: + key, value = self.linear_keys(key), self.linear_values( + value) + key = shape(key) + value = shape(value) + else: + key, value = layer_cache['memory_keys'], layer_cache[ + 'memory_values'] + layer_cache['memory_keys'] = key + layer_cache['memory_values'] = value + else: + key = self.linear_keys(key) + value = self.linear_values(value) + query = self.linear_query(query) + key = shape(key) + value = shape(value) + + query = shape(query) + + # 2) Calculate and scale scores. + query = query / math.sqrt(dim_per_head) + scores = torch.matmul(query, key.transpose(2, 3)) + + if mask is not None: + mask = mask.unsqueeze(1).expand_as(scores) + scores = scores.masked_fill(mask, float('-inf')) + + # 3) Apply attention dropout and compute context vectors. + + attn = self.softmax(scores) + + if (predefined_graph_1 is not None): + attn_masked = attn[:, -1] * predefined_graph_1 + attn_masked = attn_masked / ( + torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9) + + attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1) + + drop_attn = self.dropout(attn) + if (self.use_final_linear): + context = unshape(torch.matmul(drop_attn, value)) + output = self.final_linear(context) + if return_attn: + return output, attn + else: + return output + else: + context = torch.matmul(drop_attn, value) + if return_attn: + return context, attn + else: + return context + + +class PositionwiseFeedForward(nn.Module): # Output + """ A two-layer Feed-Forward-Network with residual layer norm. + + Args: + d_model (int): the size of input for the first-layer of the FFN. + d_ff (int): the hidden layer size of the second-layer + of the FNN. + dropout (float): dropout probability in :math:`[0, 1)`. + """ + + def __init__(self, d_model, d_ff, dropout=0.1): + super().__init__() + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.w_1 = nn.Linear(d_model, d_ff) + self.actv = ACT2FN['gelu_new'] + self.dropout_1 = nn.Dropout(dropout) + self.w_2 = nn.Linear(d_ff, d_model) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x): + inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x)))) + output = self.dropout_2(self.w_2(inter)) + return output + x + + +class TransformerDecoderLayer(nn.Module): # Layer + """ + Args: + d_model (int): the dimension of keys/values/queries in + MultiHeadedAttention, also the input size of + the first-layer of the PositionwiseFeedForward. + heads (int): the number of heads for MultiHeadedAttention. + d_ff (int): the second-layer of the PositionwiseFeedForward. + dropout (float): dropout probability(0-1.0). + self_attn_type (string): type of self-attention scaled-dot, average + """ + MAX_SIZE = 5000 + + def __init__(self, d_model, heads, d_ff, dropout): + super().__init__() + + self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + + self.context_attn = MultiHeadedAttention( + heads, d_model, dropout=dropout) + self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout) + self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6) + self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6) + self.drop = nn.Dropout(dropout) + mask = self._get_attn_subsequent_mask(self.MAX_SIZE) + # Register self.mask as a buffer in TransformerDecoderLayer, so + # it gets TransformerDecoderLayer's cuda behavior automatically. + self.register_buffer('mask', mask) + + def forward(self, + inputs, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=None, + layer_cache=None, + step=None): + """ + Args: + inputs (`FloatTensor`): `[batch_size x 1 x model_dim]` + memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]` + src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]` + tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]` + + Returns: + (`FloatTensor`, `FloatTensor`, `FloatTensor`): + + * output `[batch_size x 1 x model_dim]` + * attn `[batch_size x 1 x src_len]` + * all_input `[batch_size x current_step x model_dim]` + + """ + dec_mask = torch.gt( + tgt_pad_mask.type(torch.uint8) + + self.mask[:, :tgt_pad_mask.size(1), :tgt_pad_mask.size(1)].type( + torch.uint8), 0) + input_norm = self.layer_norm_1(inputs) + all_input = input_norm + if previous_input is not None: + all_input = torch.cat((previous_input, input_norm), dim=1) + dec_mask = None + + query = self.self_attn( + all_input, + all_input, + input_norm, + mask=dec_mask, + layer_cache=layer_cache, + type='self') + + query = self.drop(query) + inputs + + query_norm = self.layer_norm_2(query) + mid, attn = self.context_attn( + memory_bank, + memory_bank, + query_norm, + mask=src_pad_mask, + layer_cache=layer_cache, + type='context', + return_attn=True) + output = self.feed_forward(self.drop(mid) + query) + + return output, attn, all_input + + def _get_attn_subsequent_mask(self, size): + """ + Get an attention mask to avoid using the subsequent info. + + Args: + size: int + + Returns: + (`LongTensor`): + + * subsequent_mask `[1 x size x size]` + """ + attn_shape = (1, size, size) + subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8') + subsequent_mask = torch.from_numpy(subsequent_mask) + return subsequent_mask + + +class PositionalEncoding(nn.Module): + + def __init__(self, dropout, dim, max_len=5000): + super().__init__() + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) + * -(math.log(10000.0) / dim))) + pe[:, 0::2] = torch.sin(position.float() * div_term) + pe[:, 1::2] = torch.cos(position.float() * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + self.dropout = nn.Dropout(dropout) + self.dim = dim + + def forward(self, emb, step=None): + emb = emb * math.sqrt(self.dim) + if (step): + emb = emb + self.pe[:, step][:, None, :] + + else: + emb = emb + self.pe[:, :emb.size(1)] + emb = self.dropout(emb) + return emb + + def get_emb(self, emb): + return self.pe[:, :emb.size(1)] + + +class TransformerDecoderState: + + def __init__(self, src: Tensor, cache_num_layers: int = -1): + self.src: Tensor = src + self.previous_input: Tensor = None + self.previous_layer_inputs: Tensor = None + self.cache: Optional[Dict[str, Any]] = None + if cache_num_layers != -1: + self._init_cache(cache_num_layers) + + def update_state(self, new_input, previous_layer_inputs): + self.previous_input = new_input + self.previous_layer_inputs = previous_layer_inputs + self.cache = None + + def _init_cache(self, num_layers): + self.cache = {} + for layer in range(num_layers): + layer_cache = {'memory_keys': None, 'memory_values': None} + layer_cache['self_keys'] = None + layer_cache['self_values'] = None + self.cache['layer_{}'.format(layer)] = layer_cache + + def map_batch_fn(self, fn): + + def _recursive_map(struct, batch_dim=0): + for k, v in struct.items(): + if v is not None: + if isinstance(v, dict): + _recursive_map(v) + else: + struct[k] = fn(v, batch_dim) + + self.src = fn(self.src, 0) + if self.cache is not None: + _recursive_map(self.cache) + + +class TransformerDecoder(nn.Module): # Decoder + """ + The Transformer decoder from "Attention is All You Need". + + + .. mermaid:: + + graph BT + A[input] + B[multi-head self-attn] + BB[multi-head src-attn] + C[feed forward] + O[output] + A --> B + B --> BB + BB --> C + C --> O + + + Args: + num_layers (int): number of encoder layers. + d_model (int): size of the model + heads (int): number of heads + d_ff (int): size of the inner FF layer + dropout (float): dropout parameters + embeddings (:obj:`onmt.modules.Embeddings`): + embeddings to use, should have positional encodings + attn_type (str): if using a seperate copy attention + """ + decoder_type = 'transformer' + + def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings): + super().__init__() + + # Basic attributes. + self.num_layers = num_layers + self.embeddings = embeddings + self.pos_emb = PositionalEncoding(dropout, + self.embeddings.embedding_dim) + + # Build TransformerDecoder. + self.transformer_layers = nn.ModuleList([ + TransformerDecoderLayer(d_model, heads, d_ff, dropout) + for _ in range(num_layers) + ]) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.state = None + + def forward(self, + state: TransformerDecoderState, + tgt: Tensor, + memory_bank: Tensor, + step: int = None, + memory_masks: Tensor = None): + src_words = state.src + tgt_words = tgt + src_batch, src_len = src_words.size() + tgt_batch, tgt_len = tgt_words.size() + + # Run the forward pass of the TransformerDecoder. + # emb = self.embeddings(tgt, step=step) + emb = self.embeddings(tgt) + assert emb.dim() == 3 # len x batch x embedding_dim + output = self.pos_emb(emb, step) + + src_memory_bank = memory_bank + padding_idx = self.embeddings.padding_idx + tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \ + .expand(tgt_batch, tgt_len, tgt_len) + + if (memory_masks is not None): + src_len = memory_masks.size(-1) + src_pad_mask = memory_masks.expand(src_batch, tgt_len, src_len) + else: + src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \ + .expand(src_batch, tgt_len, src_len) + + if state.cache is None: + saved_inputs = [] + attns = [] + for i in range(self.num_layers): + prev_layer_input = None + if state.cache is None: + if state.previous_input is not None: + prev_layer_input = state.previous_layer_inputs[i] + output, attn, all_input \ + = self.transformer_layers[i]( + output, src_memory_bank, + src_pad_mask, tgt_pad_mask, + previous_input=prev_layer_input, + layer_cache=state.cache['layer_{}'.format(i)] + if state.cache is not None else None, + step=step) + if state.cache is None: + saved_inputs.append(all_input) + attns.append(attn) + + if state.cache is None: + saved_inputs = torch.stack(saved_inputs) + + output = self.layer_norm(output) + + # Process the result and update the attentions. + if state.cache is None: + state.update_state(tgt, saved_inputs) + + return output, attns, state + + +class PlugPointerGenerator(nn.Module): + + def __init__(self, hidden_size, vocab_size): + super().__init__() + self.dense = nn.Linear(hidden_size, vocab_size) + self.gen_func = nn.LogSoftmax(-1) + + def forward(self, x): + x = self.dense(x) + x = self.gen_func(x) + return x + + +class PlugPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = PlugConfig + base_model_prefix = 'plug' + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: Optional[Union[str, + os.PathLike]]): + config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) + config = PlugConfig.from_json_file(config_file) if os.path.isfile( + config_file) else PlugConfig() + config.encoder_pth = os.path.join(pretrained_model_name_or_path, + config.encoder_pth) + checkpoint_file = os.path.join(pretrained_model_name_or_path, + WEIGHTS_NAME) + checkpoint = torch.load(checkpoint_file) if os.path.isfile( + checkpoint_file) else None + return cls(config, checkpoint) + + +class PlugModel(PlugPreTrainedModel): # Model + + def __init__(self, config, checkpoint=None): + super().__init__(config) + self.config = config + if config.encoder == 'bert' or config.encoder == 'zh_bert': + self.bert = BertModel( + BertConfig.from_pretrained(config.encoder_pth)) + elif config.encoder == 'roberta': + self.bert = RobertaModel( + RobertaConfig.from_pretrained(config.encoder_pth)) + + if (config.max_pos > 512): + my_pos_embeddings = nn.Embedding( + config.max_pos, self.bert.model.config.hidden_size) + my_pos_embeddings.weight.data[: + 512] = self.bert.embeddings.position_embeddings.weight.data + my_pos_embeddings.weight.data[ + 512:] = self.bert.embeddings.position_embeddings.weight.data[ + -1][None, :].repeat(config.max_pos - 512, 1) + self.bert.model.embeddings.position_embeddings = my_pos_embeddings + self.vocab_size = self.bert.config.vocab_size + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=1 if config.encoder == 'roberta' else 0) + + if config.share_emb: + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight) + self.decoder = TransformerDecoder( + config.dec_layers, + config.dec_hidden_size, + heads=config.dec_heads, + d_ff=config.dec_ff_size, + dropout=config.dec_dropout, + embeddings=tgt_embeddings) + self.generator = PlugPointerGenerator(config.dec_hidden_size, + self.vocab_size) + self.generator.dense.weight = self.decoder.embeddings.weight + + if checkpoint is not None: + for key in list(checkpoint['model'].keys()): + if key.startswith('module.'): + checkpoint['model'][key.replace( + 'module.', '')] = checkpoint['model'][key] + checkpoint['model'].pop(key) + if key.startswith('plug.'): + checkpoint['model'][key.replace( + 'plug.', '')] = checkpoint['model'][key] + checkpoint['model'].pop(key) + msg = self.load_state_dict(checkpoint['model'], strict=False) + print(msg) + else: + for module in self.decoder.modules(): + if isinstance(module, (nn.Linear, nn.Embedding)): + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + for p in self.generator.parameters(): + if p.dim() > 1: + xavier_uniform_(p) + else: + p.data.zero_() + if config.use_bert_emb: + if config.encoder == 'roberta': + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=1) + else: + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=0) + tgt_embeddings.weight = copy.deepcopy( + self.bert.embeddings.word_embeddings.weight) + self.decoder.embeddings = tgt_embeddings + self.generator.dense.weight = self.decoder.embeddings.weight + + def forward(self, src, tgt, mask_src, token_type_ids): + top_vec, _ = self.bert( + src, mask_src, token_type_ids=token_type_ids, return_dict=False) + state = TransformerDecoderState(src) + decoder_outputs, attns, _ = self.decoder(state, tgt[:, :-1], top_vec) + return decoder_outputs, attns[-1], top_vec + + +class LabelSmoothingLoss(nn.Module): + """ + With label smoothing, + KL-divergence between q_{smoothed ground truth prob.}(w) + and p_{prob. computed by model}(w) is minimized. + """ + + def __init__(self, label_smoothing, tgt_vocab_size, ignore_index=-100): + assert 0.0 < label_smoothing <= 1.0 + self.padding_idx = ignore_index + super(LabelSmoothingLoss, self).__init__() + + smoothing_value = label_smoothing / (tgt_vocab_size - 2) + one_hot = torch.full((tgt_vocab_size, ), smoothing_value) + one_hot[self.padding_idx] = 0 + self.register_buffer('one_hot', one_hot.unsqueeze(0)) + self.confidence = 1.0 - label_smoothing + + def forward(self, output, target): + """ + output (FloatTensor): batch_size x n_classes + target (LongTensor): batch_size + """ + model_prob = self.one_hot.repeat(target.size(0), 1) + model_prob.scatter_(1, target.unsqueeze(1), self.confidence) + model_prob.masked_fill_((target == self.padding_idx).unsqueeze(1), 0) + + return F.kl_div(output, model_prob, reduction='sum') + + +class NMTLossCompute(nn.Module): + """ + Standard NMT Loss Computation. + """ + + def __init__(self, generator, symbols, vocab_size, label_smoothing=0.0): + super().__init__() + self.generator = generator + self.padding_idx = symbols['PAD'] + if label_smoothing > 0: + self.criterion = LabelSmoothingLoss( + label_smoothing, vocab_size, ignore_index=self.padding_idx) + else: + self.criterion = nn.NLLLoss( + ignore_index=self.padding_idx, reduction='sum') + + def _bottle(self, _v): + return _v.view(-1, _v.size(2)) + + def _unbottle(self, _v, batch_size): + return _v.view(-1, batch_size, _v.size(1)) + + def forward(self, tgt, output): + target = tgt[:, 1:] + batch_size, decoder_length = target.size(0), target.size(1) + normalization = target.ne(self.padding_idx).sum() + bottled_output = self._bottle(output) + scores = self.generator(bottled_output) + gtruth = target.contiguous().view(-1) + loss = self.criterion(scores, gtruth) + loss = loss.div(float(normalization)) + return loss, scores.view(batch_size, decoder_length, -1) + + +class PlugForConditionalGeneration(PlugPreTrainedModel): + + @dataclass + class Batch: + batch_size: int + src: torch.Tensor + tgt: torch.Tensor + mask_src: torch.Tensor + token_type_ids: torch.Tensor + query_id: List[None] = None + src_str: List[List[str]] = None + tgt_str: List[str] = None + + def __init__(self, config, checkpoint=None, dataset: str = 'default'): + super().__init__(config) + self.logger = logging.get_logger() + self.config = config + if config.encoder == 'roberta': + tokenizer = RobertaTokenizer.from_pretrained( + config.encoder_pth, do_lower_case=False) + symbols = { + 'BOS': tokenizer.cls_token_id, + 'EOS': tokenizer.sep_token_id, + 'PAD': tokenizer.pad_token_id, + 'EOQ': tokenizer.unk_token_id + } + elif config.encoder == 'bert' or config.encoder == 'zh_bert': + tokenizer = BertTokenizer.from_pretrained( + config.encoder_pth, do_lower_case=True) + symbols = { + 'BOS': tokenizer.vocab['[CLS]'], + 'EOS': tokenizer.vocab['[SEP]'], + 'PAD': tokenizer.vocab['[PAD]'], + 'EOQ': tokenizer.vocab['[unused2]'] + } + self.tokenizer = tokenizer + self.symbols = symbols + self.plug = PlugModel(config, checkpoint) + self.loss = NMTLossCompute(self.plug.generator, symbols, + self.plug.vocab_size, + config.label_smoothing) + # for generation + self.config.dataset = dataset + self.start_token = self.symbols['BOS'] + self.end_token = self.symbols['EOS'] + + def forward(self, src, tgt, mask_src=None, token_type_ids=None): + if mask_src is None: + mask_src = src.ne(self.symbols['PAD']).long() + output = self.plug(src, tgt, mask_src, token_type_ids)[0] + loss = self.loss(tgt, output) + return loss + + def translate_batch(self, + batch: 'Batch', + fast: bool = False, + *args, + **kwargs): + """ + Translate a batch of sentences. + + Mostly a wrapper around :obj:`Beam`. + + Args: + batch (:obj:`Batch`): a batch from a dataset object + data (:obj:`Dataset`): the dataset object + fast (bool): enables fast beam search (may not support all features) + + Todo: + Shouldn't need the original dataset. + """ + self.plug.eval() + with torch.no_grad(): + return self._fast_translate_batch(batch, *args, **kwargs) + + def _tile(self, x, count, dim=0): + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = x.view(batch, -1) \ + .transpose(0, 1) \ + .repeat(count, 1) \ + .transpose(0, 1) \ + .contiguous() \ + .view(*out_size) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + def _top_k_top_p_filtering(self, + logits, + top_k=10, + top_p=1.0, + filter_value=-float('Inf'), + min_tokens_to_keep=1): + if top_k > 0: + top_k = min(max(top_k, min_tokens_to_keep), + logits.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = filter_value + return logits + + def _fast_translate_batch(self, + batch: 'Batch', + max_length: int = 80, + min_length: int = 10, + bad_words_ids=None, + early_stopping=True, + num_beams=3, + length_penalty=1.2, + repetition_penalty=1.2, + no_repeat_ngram_size=4, + do_sample=False, + temperature=1.0, + top_k=0, + top_p=1.0, + *args, + **kwargs): + # TODO: faster code path for beam_size == 1. + # TODO: support these blacklisted features. + + num_beams = num_beams + batch_size = batch.batch_size + src = batch.src + mask_src = batch.mask_src + token_type_ids = batch.token_type_ids + + src_features, _ = self.plug.bert( + src, mask_src, token_type_ids=token_type_ids, return_dict=False) + state = TransformerDecoderState(src, self.plug.decoder.num_layers) + device = src_features.device + + # Tile states and memory beam_size times. + state.map_batch_fn( + lambda state, dim: self._tile(state, num_beams, dim=dim)) + src_features = self._tile(src_features, num_beams, dim=0) + batch_offset = torch.arange( + batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, + batch_size * num_beams, + step=num_beams, + dtype=torch.long, + device=device) + alive_seq = torch.full([batch_size * num_beams, 1], + self.start_token, + dtype=torch.long, + device=device) + + # cal bad_words_ids pre dict + bad_words_prefix_dict = {} + bad_words_prefix_len = set([]) + if bad_words_ids is not None: + for bw_id in bad_words_ids: + key = tuple(bw_id[:-1]) + value = bw_id[-1] + bad_words_prefix_dict[key] = bad_words_prefix_dict.get( + key, []) + [value] + bad_words_prefix_len.add(len(key)) + + # Give full probability to the first beam on the first step. + topk_log_probs = ( + torch.tensor( + [0.0] + [float('-inf')] * (num_beams - 1), + device=device).repeat(batch_size)) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results['predictions'] = [[] for _ in range(batch_size)] # noqa: F812 + results['scores'] = [[] for _ in range(batch_size)] # noqa: F812 + results['gold_score'] = [0] * batch_size + results['batch'] = batch + + for step in range(max_length): + # self.logger.info(f'step: {step + 1} / {max_length}') + decoder_input = alive_seq[:, -1].view(1, -1) + + # Decoder forward. + decoder_input = decoder_input.transpose(0, 1) + dec_out, attns, state = self.plug.decoder( + state, decoder_input, src_features, step=step) + + # Generator forward. + log_probs = self.plug.generator.forward( + dec_out.transpose(0, 1).squeeze(0)) + vocab_size = log_probs.size(-1) + + if step < min_length: + log_probs[:, self.end_token] = -1e20 + + # filter bad word + if len(bad_words_prefix_dict) > 0: + # cal bad word banned token: batch_size * num_beams + num_hypos = alive_seq.size(0) + bad_word_banned_token = [] + for i in range(num_hypos): + curr_banned_token = [] + for pre_len in bad_words_prefix_len: + pre_key = tuple(alive_seq[i, step + 1 - pre_len:step + + 1].cpu().numpy().tolist()) + curr_banned_token += bad_words_prefix_dict.get( + pre_key, []) + bad_word_banned_token.append(set(curr_banned_token)) + # set banned word prob=-1e20 + assert log_probs.size(0) == num_hypos + for i in range(num_hypos): + for banned_token in bad_word_banned_token[i]: + log_probs[i, banned_token] = -1e20 + + # do repetition_penalty + if repetition_penalty > 1.0: + """repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858). """ + # calculate prev_output_tokens for repetition_penalty: batch_size * num_beams + prev_output_tokens = self.calc_banned_tokens( + alive_seq, alive_seq.size(0), no_repeat_ngram_size, + step + 1) + # batch_size * num_beams + for i in range(log_probs.size(0)): + for previous_token in set(prev_output_tokens[i]): + if log_probs[i, previous_token] < 0: + log_probs[i, previous_token] *= repetition_penalty + else: + log_probs[i, previous_token] /= repetition_penalty + + # Multiply probs by the beam probability. + + curr_length_penalty = (step + 1)**length_penalty + # ''' + if do_sample: + _scores = log_probs / temperature + _scores = self._top_k_top_p_filtering( + _scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=1 + ) # (batch_size * num_beams, vocab_size) + # Sample 2 next words for each beam (so we have some spare tokens + # and match output of greedy beam search) + topk_ids = torch.multinomial( + F.softmax(_scores, dim=-1), + num_samples=1) # (batch_size * num_beams, 2) + # Compute next scores + _scores = F.log_softmax( + _scores, dim=1) # (batch_size * num_beams, vocab_size) + + _scores += topk_log_probs.view(-1).unsqueeze(1) + _scores = _scores / curr_length_penalty + topk_scores = torch.gather( + _scores, -1, topk_ids) # (batch_size * num_beams, 2) + # log_probs += # (batch_size * num_beams, 2) + # Match shape of greedy beam search + topk_ids = topk_ids.view( + -1, num_beams) # (batch_size, 2 * num_beams) + topk_scores = topk_scores.view( + -1, num_beams) # (batch_size, 2 * num_beams) + # ''' + else: + log_probs += topk_log_probs.view(-1).unsqueeze(1) + curr_scores = log_probs / curr_length_penalty + + curr_scores = curr_scores.reshape(-1, num_beams * vocab_size) + topk_scores, topk_ids = curr_scores.topk(num_beams, dim=-1) + if (self.config.block_trigram): + cur_len = alive_seq.size(1) + if (cur_len > 3): + for i in range(alive_seq.size(0)): + fail = False + words = [int(w) for w in alive_seq[i]] + if self.config.encoder == 'roberta': + # words = [self.vocab.convert_ids_to_tokens[w] for w in words] + words = self.tokenizer.decode( + words).strip().split() + else: + words = [ + self.tokenizer.ids_to_tokens[w] for w in words + ] + words = ' '.join(words).replace(' ##', '').split() + if (len(words) <= 3): + continue + trigrams = [(words[i - 1], words[i], words[i + 1]) + for i in range(1, + len(words) - 1)] + trigram = tuple(trigrams[-1]) + if trigram in trigrams[:-1]: + fail = True + if fail: + curr_scores[i] = -10e20 + # Recover log probs. + topk_log_probs = topk_scores * curr_length_penalty + + # Resolve beam origin and true word ids. + # topk_beam_index = topk_ids.div(vocab_size) + topk_beam_index = topk_ids // vocab_size + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = ( + topk_beam_index + + beam_offset[:topk_beam_index.size(0)].unsqueeze(1)) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat([ + alive_seq.index_select(0, select_indices), + topk_ids.view(-1, 1) + ], -1) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(self.end_token) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, num_beams, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(self.end_token) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append( + (topk_scores[i, j], predictions[i, j, 1:])) + if early_stopping and len(hypotheses) == num_beams: + end_condition[i] = True + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted( + hypotheses[b], key=lambda x: x[0], reverse=True) + if self.config.dataset == 'qg_ranking_test' or ( + self.config.dataset == 'paraphrase' + and not self.config.sample_topk): + for each in best_hyp[:num_beams]: + score, pred = each + results['scores'][b].append(score) + results['predictions'][b].append(pred) + else: + score, pred = best_hyp[0] + results['scores'][b].append(score) + results['predictions'][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished) \ + .view(-1, alive_seq.size(-1)) + + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + state.map_batch_fn( + lambda state, dim: state.index_select(dim, select_indices)) + + return results + + def calc_banned_tokens(self, prev_input_ids, num_hypos, + no_repeat_ngram_size, cur_len): + # Copied from fairseq for no_repeat_ngram in beam_search""" + if cur_len + 1 < no_repeat_ngram_size: + # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet + return [[] for _ in range(num_hypos)] + generated_ngrams = [{} for _ in range(num_hypos)] + for idx in range(num_hypos): + gen_tokens = prev_input_ids[idx].cpu().numpy().tolist() + generated_ngram = generated_ngrams[idx] + for ngram in zip( + *[gen_tokens[i:] for i in range(no_repeat_ngram_size)]): + prev_ngram_tuple = tuple(ngram[:-1]) + generated_ngram[prev_ngram_tuple] = generated_ngram.get( + prev_ngram_tuple, []) + [ngram[-1]] + + def _get_generated_ngrams(hypo_idx): + # Before decoding the next token, prevent decoding of ngrams that have already appeared + start_idx = cur_len + 1 - no_repeat_ngram_size + ngram_idx = tuple( + prev_input_ids[hypo_idx, + start_idx:cur_len].cpu().numpy().tolist()) + return generated_ngrams[hypo_idx].get(ngram_idx, []) + + banned_tokens = [ + _get_generated_ngrams(hypo_idx) for hypo_idx in range(num_hypos) + ] + return banned_tokens + + def translate(self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor = None, + token_type_ids=None, + *args, + **kwargs) -> Dict[str, torch.Tensor]: + if attention_mask is None: + attention_mask = input_ids.ne(self.symbols['PAD']).long() + batch = self.Batch( + batch_size=input_ids.size()[0], + src=input_ids, + tgt=None, + token_type_ids=token_type_ids, + mask_src=attention_mask) + translation_batch = self.translate_batch(batch, *args, **kwargs) + + preds = translation_batch['predictions'] + return {'predictions': preds} diff --git a/runtime/Lib/site_packages/models/nlp/fid_plug/configuration.py b/runtime/Lib/site_packages/models/nlp/fid_plug/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..38a53f5ce353e03875ee696edbf811dcb4d475c1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_plug/configuration.py @@ -0,0 +1,101 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PLUG model configuration """ +from transformers.configuration_utils import PretrainedConfig + + +class PlugConfig(PretrainedConfig): + r""" + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `optional`, defaults to 2): + The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layernorm_epsilon (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + dec_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer decoder. + attn_separate (:obj:`bool`, `optional`, defaults to false): + Whether or not to separate the q, k, v of attention. + + Examples:: + + >>> import PlugModel, PlugConfig + >>> configuration = PlugConfig() + + >>> # Initializing a model from the configuration + >>> model = PlugModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + model_type = 'plug' + + def __init__(self, + encoder='roberta', + encoder_pth='roberta-base', + max_pos=512, + share_emb=False, + dec_layers=12, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=3072, + dec_dropout=0.2, + use_bert_emb=True, + label_smoothing=0.1, + block_trigram=False, + **kwargs): + super().__init__(**kwargs) + self.encoder = encoder + self.encoder_pth = encoder_pth + self.max_pos = max_pos + self.share_emb = share_emb + self.dec_layers = dec_layers + self.dec_hidden_size = dec_hidden_size + self.dec_heads = dec_heads + self.dec_ff_size = dec_ff_size + self.dec_dropout = dec_dropout + self.use_bert_emb = use_bert_emb + self.label_smoothing = label_smoothing + # Translator + self.block_trigram = block_trigram diff --git a/runtime/Lib/site_packages/models/nlp/fid_plug/text_generation.py b/runtime/Lib/site_packages/models/nlp/fid_plug/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..8aac28c73f3f4852b681317f0852344a45c53963 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/fid_plug/text_generation.py @@ -0,0 +1,175 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +import os + +import torch +from transformers.modeling_outputs import Seq2SeqLMOutput + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import PlugForConditionalGeneration +from .configuration import PlugConfig + +CONFIG_NAME = 'config.json' +WEIGHTS_NAME = 'pytorch_model.bin' + + +class PlugV2Chat(TorchModel): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + # init model + plug_config_file = os.path.join(model_dir, CONFIG_NAME) + plug_config = PlugConfig.from_json_file(plug_config_file) + self.backbone = PlugForConditionalGeneration(plug_config) + # load weights + pretrained_model_path = os.path.join(model_dir, WEIGHTS_NAME) + with io.open(pretrained_model_path, 'rb') as f: + checkpoint = torch.load(f, map_location='cpu') + if 'model' in checkpoint: + checkpoint = checkpoint['model'] + for key in list(checkpoint.keys()): + # for old plugv2 version + if key.startswith('translator'): + checkpoint.pop(key) + continue + if key.startswith('module.'): + checkpoint[key.replace('module.', '')] = checkpoint[key] + checkpoint.pop(key) + if key.startswith('backbone.plug.bert.bert.'): + checkpoint[key.replace('backbone.plug.bert.bert.', + 'bert.')] = checkpoint[key] + checkpoint.pop(key) + elif key.startswith('backbone.plug.'): + checkpoint[key.replace('backbone.plug.', + '')] = checkpoint[key] + checkpoint.pop(key) + msg = self.backbone.plug.load_state_dict(checkpoint, strict=False) + print(f'| {msg}') + + def generate(self, input_ids, token_type_ids=None, *args, **kwargs): + pred_result = self.backbone.translate( + input_ids=input_ids, + token_type_ids=token_type_ids, + *args, + **kwargs)['predictions'] + response = [x[0].tolist() for x in pred_result] + response = torch.tensor(response) + return response + + def forward(self, + input_ids, + decoder_input_ids, + token_type_ids=None, + *args, + **kwargs): + loss = self.backbone.forward( + src=input_ids, + tgt=decoder_input_ids, + token_type_ids=token_type_ids, + **kwargs) + return Seq2SeqLMOutput(loss=loss[0], logits=loss[1]) + + +class PlugV2EncoderWrapper(torch.nn.Module): + + def __init__(self, bert): + super().__init__() + + self.bert = bert + self.n_passages = None + + def set_n_passages(self, n_passages): + self.n_passages = n_passages + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + *args, + **kwargs): + # total_length = n_passages * passage_length + bsz, total_length = input_ids.shape + passage_length = total_length // self.n_passages + input_ids = input_ids.view(bsz * self.n_passages, passage_length) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(bsz * self.n_passages, + passage_length) + if attention_mask is not None: + attention_mask = attention_mask.view(bsz * self.n_passages, + passage_length) + outputs = self.bert( + input_ids, + attention_mask, + token_type_ids=token_type_ids, + *args, + **kwargs) + if isinstance(outputs, tuple): + outputs = (outputs[0].view(bsz, self.n_passages * passage_length, + -1), ) + outputs[1:] + else: + outputs.last_hidden_state = outputs.last_hidden_state.view( + bsz, self.n_passages * passage_length, -1) + return outputs + + +@MODELS.register_module(Tasks.fid_dialogue, module_name=Models.fid_plug) +class PlugV2FidChat(PlugV2Chat): + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.wrap_encoder() + + def wrap_encoder(self): + self.backbone.plug.bert = PlugV2EncoderWrapper(self.backbone.plug.bert) + + def unwrap_encoder(self): + self.backbone.plug.bert = self.backbone.plug.bert.bert + + def load(self, + pretrained_model_path, + from_tf=False): # only invoked when model is not onnx format + self.unwrap_encoder() + super().load(pretrained_model_path) + self.wrap_encoder() + + def generate(self, inputs, *args, **kwargs): + input_ids = inputs.get('input_ids') + token_type_ids = inputs.get('token_type_ids', None) + n_passages = input_ids.size(1) + self.backbone.plug.bert.set_n_passages(n_passages) + input_ids = input_ids.view(input_ids.size(0), -1) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(token_type_ids.size(0), -1) + response = super().generate( + input_ids, token_type_ids=token_type_ids, *args, **kwargs) + return TokenGeneratorOutput(sequences=response) + + def forward(self, + input_ids, + decoder_input_ids, + token_type_ids=None, + *args, + **kwargs): + if input_ids is not None: + # inputs might have already be resized in the generate method + if input_ids.dim() == 3: + n_passages = input_ids.size(1) + self.backbone.plug.bert.set_n_passages(n_passages) + input_ids = input_ids.view(input_ids.size(0), -1) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(input_ids.size(0), -1) + seq2seq_lm_output = super().forward( + input_ids, + decoder_input_ids=decoder_input_ids, + token_type_ids=token_type_ids, + *args, + **kwargs) + return TextGenerationModelOutput( + loss=seq2seq_lm_output.loss, logits=seq2seq_lm_output.logits) diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/__init__.py b/runtime/Lib/site_packages/models/nlp/glm_130b/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f75eb87026a4b1ea4a3e348ee5ad5e2835cb8734 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/__init__.py @@ -0,0 +1,20 @@ +# Modified by Zhipu.AI +# Original Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING, Union + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .text_generation import GLM130bForTextGeneration +else: + _import_structure = {'text_generation': ['GLM130bForTextGeneration']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/generation/__init__.py b/runtime/Lib/site_packages/models/nlp/glm_130b/generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1b9ad83a1b30f4da637da93d9ac3851485a39ad --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/generation/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2022 Zhipu.AI +from .strategies import BaseStrategy, BeamSearchStrategy diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/generation/strategies.py b/runtime/Lib/site_packages/models/nlp/glm_130b/generation/strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..92ef97e3600fc014c111b6e3d692f211b2a869ec --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/generation/strategies.py @@ -0,0 +1,240 @@ +# Copyright (c) 2022 Zhipu.AI +import numpy as np +import torch +import torch.nn.functional as F +from SwissArmyTransformer.generation.sampling_strategies.base_strategy import \ + top_k_logits + + +class BaseStrategy: + + def __init__(self, + batch_size, + invalid_slices=[], + temperature=1., + top_k=200, + eps=1e-4, + top_p=0.0, + end_tokens=None): + self.batch_size = batch_size + self.invalid_slices = invalid_slices + self.temperature = temperature + self.topk = top_k + self.top_p = top_p + self.eps = eps + if end_tokens is None: + end_tokens = [] + self.end_tokens = end_tokens + self._is_done = np.zeros(self.batch_size, dtype=bool) + + @property + def is_done(self) -> bool: + return self._is_done.all() + + def forward(self, logits, tokens, mems, temperature=None): + logits = logits.view(-1, logits.size(-1)) + batch_size = tokens.shape[0] + if temperature is None: + temperature = self.temperature + logits = logits / temperature + for invalid_slice in self.invalid_slices: + logits[..., invalid_slice] = -65504 + + logits = top_k_logits(logits, self.topk, self.top_p) + probs = F.softmax( + logits.float(), + dim=-1) # float is essetial, due to a bug in Pytorch + pred = torch.multinomial(probs, num_samples=1) + for i in range(self.batch_size): + if i >= batch_size: + self._is_done[i] = True + elif self._is_done[i]: + pred[i] = -1 + elif pred[i].item() in self.end_tokens: + self._is_done[i] = True + tokens = torch.cat((tokens, pred.view(tokens.shape[:-1] + (1, ))), + dim=-1) + return tokens, mems + + def finalize(self, tokens, mems): + self._is_done = np.zeros(self.batch_size, dtype=bool) + return tokens, mems + + +class BeamSearchStrategy: + + def __init__( + self, + batch_size, + num_beams, + length_penalty=1.0, + consider_end=False, + end_tokens=[], + invalid_slices=[], + no_repeat_ngram_size=0, + min_gen_length=0, + deterministic=False, + ): + self.batch_size = batch_size + self.num_beams = num_beams + self.length_penalty = length_penalty + self.end_tokens = end_tokens + self.ngram = no_repeat_ngram_size + self.min_gen_length = min_gen_length + self.invalid_slices = invalid_slices + self.consider_end = consider_end + self.deterministic = deterministic + self._init_cache() + + def _init_cache(self): + self.end_beams = [[] for _ in range(self.batch_size) + ] # list of LongTensors + self.end_beams_penalized_scores = [[] for _ in range(self.batch_size) + ] # list of LongTensors + self.cached_beam_scores = 0 # [batch_size] + self.cached_beam_ngram_bans = [[{} for _ in range(self.num_beams)] + for _ in range(self.batch_size)] + self.length_generated = 0 + self._is_done = np.zeros(self.batch_size, dtype=bool) + + def _add_end_beams(self, score, beam, batch_idx): + score = score / ((5.0 + len(beam)) + / 6)**self.length_penalty # Magic number for OpenNMT + for i in range(len(self.end_beams[batch_idx]), -1, -1): + if i == 0 or score < self.end_beams_penalized_scores[batch_idx][ + i - 1]: + break + self.end_beams[batch_idx].insert(i, beam) + self.end_beams_penalized_scores[batch_idx].insert(i, score) + + self.end_beams[batch_idx] = self.end_beams[batch_idx][:self.num_beams] + self.end_beams_penalized_scores[ + batch_idx] = self.end_beams_penalized_scores[batch_idx][:self. + num_beams] + + @property + def is_done(self) -> bool: + return self._is_done.all() + + def forward(self, logits, tokens, mems): + batch_size, num_beams, vocab_size = logits.shape + seq_len = tokens.shape[-1] + logits = logits.float() + for invalid_slice in self.invalid_slices: + logits[..., invalid_slice] = -65504 + if self.min_gen_length > self.length_generated: + for end_token in self.end_tokens: + logits[..., end_token] = -65504 + if self.ngram > 0 and seq_len > self.ngram: + for batch_idx in range(batch_size): + for i in range(num_beams): + ngram_prefix = tokens[batch_idx, i, + -(self.ngram + - 1):].tolist() # TODO ngram=1 + for banned_index in self.cached_beam_ngram_bans[batch_idx][ + i].get(tuple(ngram_prefix), []): + logits[batch_idx, i, banned_index] = -65504 + + next_token_scores = F.log_softmax( + logits, dim=-1) # [batch_size, vocab_size] + prev_scores = self.cached_beam_scores + if isinstance(prev_scores, torch.Tensor): + prev_scores = prev_scores[..., None].expand_as(next_token_scores) + next_token_scores = next_token_scores + prev_scores + + next_token_scores = next_token_scores.view(batch_size, + num_beams * vocab_size) + + probs = F.softmax(next_token_scores, dim=-1) + if num_beams < self.num_beams: # First token + probs = probs[..., :vocab_size] + if self.deterministic: + next_tokens = torch.topk( + probs, k=(max(1, len(self.end_tokens)) + 1) + * self.num_beams).indices # [2*nb] + else: + next_tokens = torch.multinomial( + probs, + num_samples=(max(1, len(self.end_tokens)) + 1) + * self.num_beams) # [2*nb] + next_token_scores = next_token_scores[ + torch.arange(batch_size).unsqueeze(1), next_tokens] + next_token_scores, _indices = torch.sort( + next_token_scores, descending=True, dim=1) + next_tokens = next_tokens[torch.arange(batch_size).unsqueeze(1), + _indices] + + next_indices = torch.div( + next_tokens, vocab_size, rounding_mode='trunc') + next_tokens = next_tokens % vocab_size + + # select out end beams or continue beams + beam_continue_batch, score_continue_batch, mems_continue_batch = [], [], [] + for batch_idx in range(batch_size): + beam_continue = [] + scores_continue = [] + bans_continue = [] + mems_contiue = [] + for i in range(len(next_tokens[batch_idx])): + beam = torch.cat( + (tokens[batch_idx, next_indices[batch_idx, + i]], next_tokens[batch_idx, + i:i + 1])) + if not self._is_done[batch_idx] and int( + next_tokens[batch_idx, i]) in self.end_tokens: + self._add_end_beams(next_token_scores[batch_idx, i], beam, + batch_idx) + elif len(beam_continue) < self.num_beams: + beam_continue.append(beam) + mems_contiue.append(mems[:, batch_idx, + next_indices[batch_idx, i]]) + # update caches + scores_continue.append(next_token_scores[batch_idx, i]) + if self.ngram > 0: + bans = self.cached_beam_ngram_bans[batch_idx][ + next_indices[batch_idx, i]].copy() + # TODO ngram=1 + ngram_prefix = tuple( + tokens[batch_idx, next_indices[batch_idx, i], + -(self.ngram - 1):].tolist()) + bans[ngram_prefix] = bans.get( + ngram_prefix, tuple()) + (next_tokens[batch_idx, + i], ) + bans_continue.append(bans) + else: + break + beam_continue_batch.append(torch.stack(beam_continue)) + mems_continue_batch.append(torch.stack(mems_contiue, dim=1)) + score_continue_batch.append(scores_continue) + self.cached_beam_ngram_bans[batch_idx] = bans_continue + tokens = torch.stack(beam_continue_batch) + mems = torch.stack(mems_continue_batch, dim=1) + self.cached_beam_scores = torch.tensor( + score_continue_batch, device=logits.device) + self.length_generated += 1 + for batch_idx in range(self.batch_size): + if batch_idx >= batch_size: + self._is_done[batch_idx] = True + elif (len(self.end_beams[batch_idx]) == self.num_beams + and self.end_beams_penalized_scores[batch_idx][-1] >= # noqa + self.cached_beam_scores[batch_idx].max() / # noqa + ((5.0 + (seq_len + 1)) / 6)**self.length_penalty): # noqa + self._is_done[batch_idx] = True + + return tokens, mems + + def finalize(self, tokens, mems): + if self.consider_end: + batch_size, num_beams = tokens.shape[:2] + for batch_idx in range(batch_size): + if not self._is_done[batch_idx]: + for i in range(num_beams): + self._add_end_beams( + self.cached_beam_scores[batch_idx, i], + tokens[batch_idx, i], batch_idx) + mems = None + ret = self.end_beams[:batch_size] + else: + ret = tokens + self._init_cache() + return ret, mems diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/initialize.py b/runtime/Lib/site_packages/models/nlp/glm_130b/initialize.py new file mode 100644 index 0000000000000000000000000000000000000000..03c8f84979ec04d491b57f2b3faaa39d345b7630 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/initialize.py @@ -0,0 +1,161 @@ +# Copyright (c) 2022 Zhipu.AI + +import argparse +import time + +import torch +from SwissArmyTransformer import get_args, get_tokenizer +from SwissArmyTransformer.arguments import initialize_distributed +from SwissArmyTransformer.model import GLM130B +from SwissArmyTransformer.mpu import (get_model_parallel_group, + get_model_parallel_rank, + get_model_parallel_world_size) +from SwissArmyTransformer.training import load_checkpoint + +from .quantization import quantize + + +def add_bminf_args(parser): + """Arguments for BMInf""" + group = parser.add_argument_group('BMInf') + + group.add_argument( + '--bminf', + action='store_true', + help='Use BMInf to support low resource evaluation') + group.add_argument( + '--bminf-memory-limit', + type=int, + default=20, + help='Max memory for model per GPU (in GB)') + return parser + + +def add_quantization_args(parser): + group = parser.add_argument_group('Quantization') + + group.add_argument('--quantization-bit-width', type=int, default=4) + group.add_argument( + '--from-quantized-checkpoint', + type=bool, + default=True, + help='Loading from a quantized checkpoint') + + +def add_initialization_args(parser): + group = parser.add_argument_group('Initialization') + + group.add_argument( + '--sequential-initialization', + action='store_true', + help= + 'Initialize sequentially in tensor parallel group (reduce CPU RAM for initialization)', + ) + + +def set_up_model_args(args): + args.model_parallel_size = 4 + args.num_layers = 70 + args.hidden_size = 12288 + args.inner_hidden_size = 32768 + args.vocab_size = 150528 + args.num_attention_heads = 96 + args.max_sequence_length = 2048 + args.tokenizer_type = 'icetk-glm-130B' + args.layernorm_order = 'post' + args.skip_init = True + args.fp16 = True + args.mode = 'inference' + return args + + +def initialize(extra_args_provider): + parser = argparse.ArgumentParser(add_help=False) + add_bminf_args(parser) + add_quantization_args(parser) + add_initialization_args(parser) + GLM130B.add_model_specific_args(parser) + extra_args_provider(parser) + known, args_list = parser.parse_known_args() + args_list += ['--model-parallel-size', '4', '--mode', 'inference'] + args = get_args(args_list) + args = set_up_model_args(args) + args = argparse.Namespace(**vars(args), **vars(known)) + args.do_train = False + initialize_distributed(args) + return args + + +def initialize_model_and_tokenizer(args): + tokenizer = get_tokenizer(args) + + torch.distributed.barrier() + start = time.time() + + for i in range(get_model_parallel_world_size()): + if get_model_parallel_rank() == i: + # Initialize model + model = GLM130B(args).half() + + if args.from_quantized_checkpoint: + assert args.quantization_bit_width is not None + # Quantize model before moving to GPU + model = quantize(model, args.quantization_bit_width) + + # Load checkpoint + load_checkpoint(model, args) + + if args.quantization_bit_width is not None and not args.from_quantized_checkpoint: + # Quantize model before moving to GPU + model = quantize(model, args.quantization_bit_width) + + if args.bminf: + import bminf + + if torch.distributed.get_rank() == 0: + print( + f'> BMInf activated, memory limit: {args.bminf_memory_limit} GB' + ) + with torch.cuda.device(args.device): + model = bminf.wrapper( + model, + quantization=False, + memory_limit=args.bminf_memory_limit << 30) + else: + model = model.to(args.device) + if args.sequential_initialization: + torch.distributed.barrier(group=get_model_parallel_group()) + + torch.distributed.barrier() + if torch.distributed.get_rank() == 0: + print(f'> Model initialized in {time.time() - start:.1f}s') + + torch.cuda.empty_cache() + model.eval() + + # generate rotary embedding cache + original_parallel_output = model.transformer.parallel_output + model.transformer.parallel_output = True + with torch.no_grad(): + _, *_ = model( + torch.ones( + 1, + args.max_sequence_length, + device=torch.cuda.current_device(), + dtype=torch.int64), + torch.arange( + args.max_sequence_length, + device=torch.cuda.current_device(), + dtype=torch.int64).view(1, -1), + torch.randn( + 1, + 1, + args.max_sequence_length, + args.max_sequence_length, + device=torch.cuda.current_device(), + ) < 0.5, + ) + model.transformer.parallel_output = original_parallel_output + torch.distributed.barrier() + + return model, tokenizer diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/kernels/__init__.py b/runtime/Lib/site_packages/models/nlp/glm_130b/kernels/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70f4b34b09c56f801a618edb30167318631a1170 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/kernels/__init__.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022 Zhipu.AI +import ctypes +from typing import List + +import pkg_resources +import torch +from cpm_kernels.kernels.base import (KernelFunction, LazyKernelCModule, + round_up) + +RESOURCE_PACKAGE_NAME = __name__ + + +class Kernel: + + def __init__(self, filename: str, function_names: List[str]): + filename = filename + '.fatbin' + if not pkg_resources.resource_exists(RESOURCE_PACKAGE_NAME, filename): + raise RuntimeError('File `%s` not found in `%s`' % + (filename, RESOURCE_PACKAGE_NAME)) + self.filename = filename + self.code = pkg_resources.resource_string(RESOURCE_PACKAGE_NAME, + filename) + self._function_names = function_names + self._cmodule = LazyKernelCModule(self.code) + + for name in self._function_names: + setattr(self, name, KernelFunction(self._cmodule, name)) + + +kernels = Kernel( + 'quantization', + [ + 'int4WeightCompression', + 'int4WeightExtractionFloat', + 'int4WeightExtractionHalf', + 'int8WeightExtractionFloat', + 'int8WeightExtractionHalf', + ], +) + + +def compress_int4_weight(weight: torch.Tensor): # (n, m) + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + assert m % 2 == 0 + m = m // 2 + out = torch.empty(n, m, dtype=torch.int8, device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + kernels.int4WeightCompression( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m) + ], + ) + return out + + +def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, + source_bit_width: int): + if source_bit_width == 8: + func = kernels.int8WeightExtractionHalf + elif source_bit_width == 4: + func = kernels.int4WeightExtractionHalf + else: + assert False, 'Unsupported bit-width' + + with torch.cuda.device(weight.device): + n, m = weight.size(0), weight.size(1) + out = torch.empty( + n, m * (8 // source_bit_width), dtype=torch.half, device='cuda') + stream = torch.cuda.current_stream() + + gridDim = (n, 1, 1) + blockDim = (min(round_up(m, 32), 1024), 1, 1) + + func( + gridDim, + blockDim, + 0, + stream, + [ + ctypes.c_void_p(weight.data_ptr()), + ctypes.c_void_p(scale_list.data_ptr()), + ctypes.c_void_p(out.data_ptr()), + ctypes.c_int32(n), + ctypes.c_int32(m), + ], + ) + return out + + +if __name__ == '__main__': + weight = torch.randn(4, 32).to(torch.int8).cuda() + scale = torch.ones(weight.size(0)).to(torch.half).cuda() + + print(weight) + b = compress_int4_weight(weight) + print(b) + + a = extract_weight_to_half(b, scale, source_bit_width=4) + print(a) diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/__init__.py b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba8fb8af4264e0bbce1849d76a943a6710f79354 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/__init__.py @@ -0,0 +1,67 @@ +# Copyright (c) 2022 Zhipu.AI +import torch + +from .layers import QuantizedColumnParallelLinear, QuantizedRowParallelLinear + + +def quantize(model, weight_bit_width): + """Replace fp16 linear with quantized linear""" + + if torch.distributed.get_rank() == 0: + print(f'> Quantizing model weight to {weight_bit_width} bits') + + for layer in model.transformer.layers: + layer.attention.query_key_value = QuantizedColumnParallelLinear( + weight_bit_width=weight_bit_width, + weight=layer.attention.query_key_value.weight.to( + torch.cuda.current_device()), + input_size=layer.attention.query_key_value.input_size, + output_size=layer.attention.query_key_value.output_size, + bias=True, + gather_output=False, + params_dtype=torch.half, + name='query_key_value', + skip_init=True, + device=layer.attention.query_key_value.weight.device, + ) + layer.attention.dense = QuantizedRowParallelLinear( + weight_bit_width=weight_bit_width, + weight=layer.attention.dense.weight.to( + torch.cuda.current_device()), + input_size=layer.attention.dense.input_size, + output_size=layer.attention.dense.output_size, + bias=True, + input_is_parallel=True, + params_dtype=torch.half, + name='dense', + skip_init=True, + device=layer.attention.dense.weight.device, + ) + layer.mlp.dense_h_to_4h = QuantizedColumnParallelLinear( + weight_bit_width=weight_bit_width, + weight=layer.mlp.dense_h_to_4h.weight.to( + torch.cuda.current_device()), + input_size=layer.mlp.dense_h_to_4h.input_size, + output_size=layer.mlp.dense_h_to_4h.output_size, + bias=True, + gather_output=False, + params_dtype=torch.half, + name='dense_h_to_4h', + skip_init=True, + device=layer.mlp.dense_h_to_4h.weight.device, + ) + layer.mlp.dense_4h_to_h = QuantizedRowParallelLinear( + weight_bit_width=weight_bit_width, + weight=layer.mlp.dense_4h_to_h.weight.to( + torch.cuda.current_device()), + input_size=layer.mlp.dense_4h_to_h.input_size, + output_size=layer.mlp.dense_4h_to_h.output_size, + bias=True, + input_is_parallel=True, + params_dtype=torch.half, + name='dense_h_to_4h', + skip_init=True, + device=layer.mlp.dense_4h_to_h.weight.device, + ) + + return model diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/functional.py b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..7d6297ad498561a4dda07a489c8b9720c7a67f93 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/functional.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022 Zhipu.AI +import torch + +from ..kernels import extract_weight_to_half + + +class W8A16Linear(torch.autograd.Function): + + @staticmethod + def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, + scale_w: torch.Tensor, weight_bit_width): + ctx.inp_shape = inp.size() + ctx.weight_shape = quant_w.size() + ctx.weight_bit_width = weight_bit_width + out_features = quant_w.size(0) + inp = inp.contiguous().view(-1, inp.size(-1)) + weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width) + output = inp.mm(weight.t()) + ctx.save_for_backward(inp, quant_w, scale_w) + return output.view(*(ctx.inp_shape[:-1] + (out_features, ))) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + inp, quant_w, scale_w = ctx.saved_tensors + weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width) + grad_output = grad_output.contiguous().view(-1, weight.size(0)) + grad_input = grad_output.mm(weight) + grad_weight = grad_output.t().mm(inp) + return grad_input.view(ctx.inp_shape), grad_weight.view( + ctx.weight_shape), None diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/layers.py b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..c0693fd235172ab9b738378227e6afc25f6ed22a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/quantization/layers.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022 Zhipu.AI +import torch +from SwissArmyTransformer.mpu import (ColumnParallelLinear, RowParallelLinear, + copy_to_model_parallel_region, + gather_from_model_parallel_region, + reduce_from_model_parallel_region, + scatter_to_model_parallel_region) +from torch.nn.parameter import Parameter + +from ..kernels import compress_int4_weight +from .functional import W8A16Linear + + +class QuantizedColumnParallelLinear(ColumnParallelLinear): + + def __init__(self, weight_bit_width: int, weight=None, *args, **kwargs): + super(QuantizedColumnParallelLinear, self).__init__(*args, **kwargs) + self.weight_bit_width = weight_bit_width + + shape = self.weight.shape + del self.weight + + if weight is None: + self.weight = torch.empty( + shape[0], + shape[1] * weight_bit_width // 8, + dtype=torch.int8, + device=kwargs['device']) + self.weight_scale = torch.empty( + shape[0], + dtype=kwargs['params_dtype'], + device=kwargs['device']) + else: + self.weight_scale = ( + weight.abs().max(dim=-1).values / ( # noqa + (2**(weight_bit_width - 1)) - 1)).half() # noqa + self.weight = torch.round(weight / self.weight_scale[:, None]).to( + torch.int8) + if weight_bit_width == 4: + self.weight = compress_int4_weight(self.weight) + + self.weight = Parameter( + self.weight.to(kwargs['device']), requires_grad=False) + self.weight_scale = Parameter( + self.weight_scale.to(kwargs['device']), requires_grad=False) + + def forward(self, input_): + # Set up backprop all-reduce. + input_parallel = copy_to_model_parallel_region(input_) + # Matrix multiply. + output_parallel = W8A16Linear.apply(input_parallel, self.weight, + self.weight_scale, + self.weight_bit_width) + if self.bias is not None: + output_parallel = output_parallel + self.bias + if self.gather_output: + # All-gather across the partitions. + output = gather_from_model_parallel_region(output_parallel) + else: + output = output_parallel + return output + + +class QuantizedRowParallelLinear(RowParallelLinear): + + def __init__(self, weight_bit_width: int, weight=None, *args, **kwargs): + super(QuantizedRowParallelLinear, self).__init__(*args, **kwargs) + self.weight_bit_width = weight_bit_width + + shape = self.weight.shape + del self.weight + + if weight is None: + self.weight = torch.empty( + shape[0], + shape[1] * weight_bit_width // 8, + dtype=torch.int8, + device=kwargs['device']) + self.weight_scale = torch.empty( + shape[0], + dtype=kwargs['params_dtype'], + device=kwargs['device']) + else: + self.weight_scale = ( + weight.abs().max(dim=-1).values / ( # noqa + (2**(weight_bit_width - 1)) - 1)).half() # noqa + self.weight = torch.round(weight / self.weight_scale[:, None]).to( + torch.int8) + if weight_bit_width == 4: + self.weight = compress_int4_weight(self.weight) + + self.weight = Parameter( + self.weight.to(kwargs['device']), requires_grad=False) + self.weight_scale = Parameter( + self.weight_scale.to(kwargs['device']), requires_grad=False) + + def forward(self, input_): + # Set up backprop all-reduce. + if self.input_is_parallel: + input_parallel = input_ + else: + input_parallel = scatter_to_model_parallel_region(input_) + # Matrix multiply. + output_parallel = W8A16Linear.apply(input_parallel, self.weight, + self.weight_scale, + self.weight_bit_width) + # All-reduce across all the partitions. + output_ = reduce_from_model_parallel_region(output_parallel) + if self.bias is not None: + output = output_ + self.bias + else: + output = output_ + return output diff --git a/runtime/Lib/site_packages/models/nlp/glm_130b/text_generation.py b/runtime/Lib/site_packages/models/nlp/glm_130b/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..a0f6411c4e54cb74b75b2dc6d3ff388526801609 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/glm_130b/text_generation.py @@ -0,0 +1,354 @@ +# Copyright (c) 2022 Zhipu.AI +import copy +import os +import random +import re +import stat +import sys +import time +from functools import partial +from typing import Any, Dict, List, Tuple + +import torch +from SwissArmyTransformer import mpu +from SwissArmyTransformer.generation.autoregressive_sampling import ( + get_masks_and_position_ids_default, update_mems) +from SwissArmyTransformer.generation.utils import (generate_continually, + timed_name) + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .generation import BaseStrategy, BeamSearchStrategy +from .initialize import initialize, initialize_model_and_tokenizer + +torch.set_num_threads(24) + +logger = get_logger() + + +def batch_filling_sequence( + model, + seqs, + context_lengths, + strategy, + max_memory_length=100000, + get_masks_and_position_ids=get_masks_and_position_ids_default, + mems=None, + **kw_args): + ''' + seq: [2, 3, 5, ..., -1(to be generated), -1, ...] + mems: [num_layers, batch_size, len_mems(index), mem_hidden_size] + cache, should be first mems.shape[1] parts of context_tokens. + mems are the first-level citizens here, but we don't assume what is memorized. + input mems are used when multi-phase generation. + ''' + assert len(seqs.shape) == 2 + + # building the initial tokens, attention_mask, and position_ids + batch_size, context_length = seqs.shape + seqs, attention_mask, position_ids = get_masks_and_position_ids(seqs) + tokens = seqs[..., :context_length] + if attention_mask.dtype != torch.bool: + attention_mask = attention_mask.type_as(next( + model.parameters())) # if fp16 + # initialize generation + counter = context_length - 1 # Last fixed index is ``counter'' + index = 0 if mems is None else mems.shape[ + 2] # Next forward starting index, also the length of cache. + num_beams = 1 + # step-by-step generation + while counter < seqs.shape[1] - 1: + # Now, we want to generate seq[counter + 1], + # token[:, index: counter+1] needs forwarding. + # forward + tokens = tokens.reshape(batch_size * num_beams, -1) + mems = mems.reshape(mems.shape[0], batch_size + * num_beams, mems.shape[-2], + mems.shape[-1]) if mems is not None else None + logits, *output_per_layers = model( + tokens[:, index:], + position_ids[..., index:counter + 1], + attention_mask[..., + index:counter + 1, :counter + 1], # TODO memlen + mems=mems, + **kw_args) + mem_kv = [o['mem_kv'] for o in output_per_layers] + mems = update_mems(mem_kv, mems, max_memory_length=max_memory_length) + if counter == context_length - 1: + logits = logits[torch.arange(batch_size), context_lengths - 1] + else: + logits = logits[:, -1] + counter += 1 + index = counter + + # sampling + logits = logits.reshape(batch_size, num_beams, -1) + tokens = tokens.reshape(batch_size, num_beams, -1) + mems = mems.reshape(mems.shape[0], batch_size, num_beams, + mems.shape[-2], mems.shape[-1]) + tokens, mems = strategy.forward(logits, tokens, mems) + if len(tokens.shape) == 3 and num_beams == 1: + num_beams = tokens.shape[1] + position_ids = position_ids.unsqueeze(1).expand( + batch_size, num_beams, -1).reshape(batch_size * num_beams, -1) + attention_mask_shape = attention_mask.shape[-3:] + attention_mask = attention_mask.unsqueeze(1).expand( + batch_size, num_beams, -1, -1, + -1).reshape(batch_size * num_beams, *attention_mask_shape) + if strategy.is_done: + break + return strategy.finalize(tokens, mems) + + +def add_generation_specific_args(parser): + parser.add_argument( + '--sampling-strategy', + type=str, + default='BaseStrategy', + help='Type of sampling strategy.') + parser.add_argument( + '--min-gen-length', + type=int, + default=0, + help='The minimum length each blank should generate.') + parser.add_argument( + '--print-all-beams', + action='store_true', + help='Print all output generated by beam search strategy.') + + +def isEnglish(s): + try: + s.encode(encoding='utf-8').decode('ascii') + except UnicodeDecodeError: + return False + else: + return True + + +def get_masks_and_position_ids(seq, + mask_position, + max_gen_length, + gmask=False): + context_length = seq.shape[1] + tokens = torch.nn.functional.pad( + seq, (0, max_gen_length), mode='constant', value=-1) + attention_mask = torch.ones((1, tokens.shape[-1], tokens.shape[-1]), + device=tokens.device) + attention_mask.tril_() + attention_mask[..., :context_length - 1] = 1 + attention_mask.unsqueeze_(1) + attention_mask = (attention_mask < 0.5).bool() + + position_ids = torch.arange( + tokens.shape[-1], dtype=torch.long, device=tokens.device) + if not gmask: + position_ids[context_length - 1:] = mask_position + + position_ids = position_ids.unsqueeze(0) + + return tokens, attention_mask, position_ids + + +def fill_blanks(args, raw_text: str, model, tokenizer, + strategy) -> Tuple[List[str], List[str], List[List[str]]]: + # add MASK + generation_mask = '[gMASK]' + if '[MASK]' in raw_text: + generation_mask = '[MASK]' + elif '[sMASK]' in raw_text: + generation_mask = '[sMASK]' + use_gmask = '[MASK]' not in raw_text and '[sMASK]' not in raw_text + + mask_pattern = r'\[[sg]?MASK\]' + text_list = re.split(mask_pattern, raw_text) + pattern_list = re.compile(mask_pattern).findall(raw_text) + seq = [] + for i in range(len(pattern_list)): + pattern = pattern_list[i] + sub_text = text_list[i] + seq.extend(tokenizer.tokenize(sub_text)) + seq.append(tokenizer.get_command(pattern)) + + seq.extend(tokenizer.tokenize(text_list[-1])) + + if 'MASK]' not in raw_text: + seq += [tokenizer.get_command(generation_mask)] + raw_text += ' ' + generation_mask + if not raw_text.endswith('MASK]'): + seq = seq + [tokenizer.get_command('eos')] + if mpu.get_model_parallel_rank() == 0: + logger.info('\nInput: {}\n'.format(raw_text)) + if len(seq) > args.max_sequence_length: + raise ValueError('text too long.') + + # generation + is_english = isEnglish(raw_text) + output_list = [seq] + num_output = args.num_beams if args.sampling_strategy == 'BeamSearchStrategy' else 1 + last_pos, answers, answers_with_style, blanks = ( + [0] * num_output, + ['' for _ in range(num_output)], + ['' for _ in range(num_output)], + [[] for _ in range(num_output)], + ) + + # continually detect the first mark position + while True: + seq = output_list[0] + # detect mask position + mask_token = tokenizer.get_command(generation_mask) + if mask_token not in seq: + break + mask_position = seq.index(mask_token) + + output_list = [] + + input_seq = torch.cuda.LongTensor( + [seq + [tokenizer.get_command('sop')]], + device=args.device, + ) + output, _ = batch_filling_sequence( + model, + input_seq, + torch.cuda.LongTensor([input_seq.shape[-1]], device=args.device), + strategy=strategy, + get_masks_and_position_ids=partial( + get_masks_and_position_ids, + mask_position=mask_position, + max_gen_length=args.out_seq_length - input_seq.shape[-1], + gmask=use_gmask, + ), + ) + if isinstance(output, torch.Tensor): # different strategies + output = output.tolist() + output = output[0] # batch_size = 1 + output_list.extend(output) + + # clip -1s and fill back generated things into seq + for i in range(len(output_list)): + output = output_list[i].tolist() if isinstance( + output_list[i], torch.Tensor) else output_list[i] + try: + unfinished = output.index(-1) + except ValueError: + unfinished = len(output) + if output[unfinished - 1] in strategy.end_tokens: + unfinished -= 1 + bog = output.index(tokenizer.get_command('sop')) + + prefix = tokenizer.detokenize(output[last_pos[i]:mask_position]) + blank = tokenizer.detokenize(output[bog + 1:unfinished]) + answers_with_style[i] += ( + prefix + (' ' if is_english else '') + # noqa + ('\033[4m' if use_gmask else '\x1b[0;32m\033[4m') + blank + + # noqa + ('\033[0m' if use_gmask else '\033[0m\x1b[0m') + # noqa + (' ' if is_english else '')) # noqa + blanks[i].append(blank) + last_pos[i] = mask_position + unfinished - (bog + 1) + output_list[i] = output[:mask_position] + output[ + bog + 1:unfinished] + output[mask_position + 1:bog] + + for i, output in enumerate(output_list): + if output[-1] == tokenizer.get_command('eos'): + output = output[:-1] + answers_with_style[i] += tokenizer.detokenize(output[last_pos[i]:]) + answers[i] = tokenizer.detokenize(output) + + return answers, answers_with_style, blanks + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.glm130b) +class GLM130bForTextGeneration(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + # """initialize the glm130b model from the `model_dir` path. + + # Args: + # model_dir (str): the model path. + # """ + super().__init__(model_dir, *args, **kwargs) + self.cfg = Config.from_file(model_dir + '/' + ModelFile.CONFIGURATION) + args = initialize(extra_args_provider=add_generation_specific_args) + args.seed = random.randint(1, sys.maxsize - 1) + args.sampling_strategy = self.cfg.model.sampling_strategy + args.out_seq_length = self.cfg.model.out_seq_length + args.min_gen_length = self.cfg.model.min_gen_length + args.num_beams = self.cfg.model.num_beams + args.length_penalty = self.cfg.model.length_penalty + args.no_repeat_ngram_size = self.cfg.model.no_repeat_ngram_size + args.temperature = self.cfg.model.temperature + args.top_k = self.cfg.model.top_k + args.top_p = self.cfg.model.top_p + args.load = model_dir + + logger.info('Loading model and tokenizer ...') + self.model, self.tokenizer = initialize_model_and_tokenizer(args) + + end_tokens = [ + self.tokenizer.get_command('eop'), + self.tokenizer.get_command('eos') + ] + + if args.sampling_strategy == 'BaseStrategy': + self.strategy = BaseStrategy( + batch_size=1, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + end_tokens=end_tokens) + elif args.sampling_strategy == 'BeamSearchStrategy': + self.strategy = BeamSearchStrategy( + 1, + args.num_beams, + length_penalty=args.length_penalty, + consider_end=True, + end_tokens=end_tokens, + no_repeat_ngram_size=args.no_repeat_ngram_size, + min_gen_length=args.min_gen_length, + ) + else: + raise ValueError(f'unknown strategy {args.sampling_strategy}') + + self.args = args + + def func(self, raw_text): + answers, answers_with_style, blanks = fill_blanks( + self.args, raw_text, self.model, self.tokenizer, self.strategy) + + if mpu.get_model_parallel_rank() == 0: + logger.info('Output:' + str(answers_with_style[0])) + + return str(answers_with_style[0]) + + def forward(self, input: str) -> Dict[str, str]: + raw_text, is_stop = '', False + if torch.distributed.get_rank() == 0: + raw_text = input + if not raw_text: + return {OutputKeys.TEXT: 'Query should not be empty!'} + if raw_text == 'stop': + is_stop = True + torch.distributed.broadcast_object_list([raw_text, is_stop]) + else: + info = [raw_text, is_stop] + torch.distributed.broadcast_object_list(info) + raw_text, is_stop = info + if is_stop: + return + try: + start_time = time.time() + res = self.func(raw_text) + if torch.distributed.get_rank() == 0: + logger.info('\nTaken time {:.2f}\n'.format(time.time() + - start_time)) + except (ValueError, FileNotFoundError) as e: + return {OutputKeys.TEXT: str(e)} + logger.info('Generation finished.') + return {OutputKeys.TEXT: res} diff --git a/runtime/Lib/site_packages/models/nlp/gpt2/__init__.py b/runtime/Lib/site_packages/models/nlp/gpt2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c344d439359ef8c461cbc461c1bf992a21b2ca9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt2/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import GPT2Model +else: + _import_structure = { + 'backbone': ['GPT2Model'], + } + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/gpt2/backbone.py b/runtime/Lib/site_packages/models/nlp/gpt2/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..8f8f88f6f408ab4017d9799005b2bf0c44ed8589 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt2/backbone.py @@ -0,0 +1,15 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from transformers import GPT2Config +from transformers import GPT2Model as GPT2ModelTransform + +from modelscope.metainfo import Models +from modelscope.models.builder import BACKBONES +from modelscope.utils.constant import Tasks + + +@BACKBONES.register_module(group_key=Tasks.backbone, module_name=Models.gpt2) +class GPT2Model(GPT2ModelTransform): + + def __init__(self, **kwargs): + config = GPT2Config(**kwargs) + super().__init__(config) diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/__init__.py b/runtime/Lib/site_packages/models/nlp/gpt3/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..347e53bf1a965e61f791e08610d0fb009728e913 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import GPT3Config + from .backbone import GPT3Model + from .text_generation import GPT3ForTextGeneration + from .tokenizer import JiebaBPETokenizer + from .distributed_gpt3 import DistributedGPT3 +else: + _import_structure = { + 'configuration': ['GPT3Config'], + 'backbone': ['GPT3Model'], + 'text_generation': ['GPT3ForTextGeneration'], + 'tokenizer': ['JiebaBPETokenizer'], + 'distributed_gpt3': ['DistributedGPT3'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/backbone.py b/runtime/Lib/site_packages/models/nlp/gpt3/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..89e1948008df4be67a9daf25ee5d259451f1d6e2 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/backbone.py @@ -0,0 +1,429 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os +from typing import Optional, Union + +import addict +import torch +from torch import nn +from torch.nn import functional as F +from transformers.modeling_utils import PreTrainedModel + +from modelscope.outputs import TokenGeneratorOutput +from modelscope.utils.constant import ModelFile +from .configuration import GPT3Config +from .distributed_gpt3 import sample + + +class GPT3SelfAttention(nn.Module): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__(self, config): + super().__init__() + + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + # Per attention head + self.hidden_size_per_attention_head = \ + self.hidden_size // self.num_attention_heads + + self.query_key_value = nn.Linear(self.hidden_size, + 3 * self.hidden_size) + self.softmax = nn.Softmax(dim=-1) + self.attention_dropout = nn.Dropout( + config.attention_probs_dropout_prob) + + # Output. + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + self.output_dropout = nn.Dropout(config.hidden_dropout_prob) + + def _transpose_for_scores(self, tensor): + """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with + size [b, np, s, hn]. + """ + new_tensor_shape = tensor.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head) + tensor = tensor.view(*new_tensor_shape) + return tensor.permute(0, 2, 1, 3) + + def _split_tensor_along_last_dim(self, + tensor, + num_partitions, + contiguous_split_chunks=False): + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = tensor.size()[last_dim] // num_partitions + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + def forward(self, hidden_states, ltor_mask, is_infer=False): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Attention heads. [b, s, hp] + tgt_len = hidden_states.size(1) + ltor_mask = torch.reshape(ltor_mask, [1, 1, tgt_len, tgt_len]) + mixed_x_layer = self.query_key_value(hidden_states) + (mixed_query_layer, mixed_key_layer, mixed_value_layer) = \ + self._split_tensor_along_last_dim(mixed_x_layer, 3) + + # Reshape and transpose [b, np, s, hn] + query_layer = self._transpose_for_scores(mixed_query_layer) + key_layer = self._transpose_for_scores(mixed_key_layer) + value_layer = self._transpose_for_scores(mixed_value_layer) + + previous_type = value_layer.type() + + # Raw attention scores. [b, np, s, s] + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.hidden_size_per_attention_head) + # Apply the left to right attention mask. + if is_infer: + src_len = key_layer.size(2) + ltor_mask = torch.tril( + torch.ones((1, tgt_len, src_len), + device=hidden_states.device)).view( + 1, 1, tgt_len, src_len).type(previous_type) + converted_mask = 10000.0 * (1.0 - ltor_mask) + attention_scores = (torch.mul(attention_scores, ltor_mask) + - converted_mask).type(previous_type) + + # Attention probabilities. [b, np, s, s] + attention_probs = self.softmax(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.attention_dropout(attention_probs) + + # Context layer. + # [b, np, s, hn] + context_layer = torch.matmul(attention_probs, value_layer) + # [b, s, np, hn] + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size, ) + # [b, s, hp] + context_layer = context_layer.view(*new_context_layer_shape) + + # Output. [b, s, h] + output = self.dense(context_layer) + output = self.output_dropout(output) + + return output + + +class GPT3MLP(nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__(self, config): + super().__init__() + + hidden_size = config.hidden_size + # Project to 4h. + self.dense_h_to_4h = nn.Linear(hidden_size, 4 * hidden_size) + self.activation_func = F.gelu + # Project back to h. + self.dense_4h_to_h = nn.Linear(4 * hidden_size, hidden_size) + + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states): + + # [s, b, 4hp] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output = self.dense_4h_to_h(intermediate_parallel) + output = self.dropout(output) + return output + + +class GPT3TransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__(self, config): + super().__init__() + + # Layernorm on the input data. + self.input_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + # Self attention. + self.attention = GPT3SelfAttention(config) + + # Layernorm on the attention output + self.post_attention_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + # MLP + self.mlp = GPT3MLP(config) + + def forward(self, hidden_states, ltor_mask): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Layer norm at the begining of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output = self.attention(layernorm_output, ltor_mask) + # Residual connection. + layernorm_input = hidden_states + attention_output + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + # MLP. + mlp_output = self.mlp(layernorm_output) + # Second residual connection. + output = layernorm_input + mlp_output + + return output + + +class GPT3Transformer(nn.Module): + """Transformer class.""" + + def __init__(self, config): + super().__init__() + + self.input_tensor = None + + # Number of layers. + self.num_layers = config.num_hidden_layers + + self.layers = torch.nn.ModuleList( + [GPT3TransformerLayer(config) for _ in range(self.num_layers)]) + + # Final layer norm before output. + self.final_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def forward(self, hidden_states, attention_mask): + # hidden_states: [s, b, h] + + for index in range(self.num_layers): + layer = self._get_layer(index) + hidden_states = layer(hidden_states, attention_mask) + + # Final layer norm. + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class GPT3TransformerLanguageModel(nn.Module): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, config): + super().__init__() + + # Embeddings. + self.word_embeddings = nn.Embedding(config.vocab_size, + config.hidden_size) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.embedding_dropout = nn.Dropout(config.hidden_dropout_prob) + + # Transformer. + self.transformer = GPT3Transformer(config) + + def forward(self, input_ids, attention_mask, position_ids): + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + + embeddings = words_embeddings + position_embeddings + transformer_input = self.embedding_dropout(embeddings) + transformer_output = self.transformer(transformer_input, + attention_mask) + + logits = F.linear(transformer_output, self.word_embeddings.weight) + return logits + + +class GPT3Model(PreTrainedModel): + + config_class = GPT3Config + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def __init__(self, config): + super().__init__(config) + self.language_model = GPT3TransformerLanguageModel(config) + + def forward(self, + input_ids, + attention_mask=None, + position_ids=None, + labels=None, + **kwargs): + seq_length = input_ids.size(1) + attention_mask = torch.tril( + torch.ones((1, 1, seq_length, seq_length), + dtype=torch.long, + device=input_ids.device)) + if position_ids is None: + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + + logits = self.language_model(input_ids, attention_mask, position_ids) + loss = None + if labels is not None: + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.config.vocab_size), labels.view(-1)) + return addict.Dict(loss=loss, logits=logits) + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: Optional[Union[str, + os.PathLike]]): + config = cls.config_class.from_pretrained( + pretrained_model_name_or_path) + model = cls(config) + state_dict_file = os.path.join(pretrained_model_name_or_path, + ModelFile.TORCH_MODEL_BIN_FILE) + state_dict = torch.load(state_dict_file) + if 'state_dict' in state_dict: + state_dict = state_dict['state_dict'] + state_dict = { + k.replace('model.language_model', 'language_model'): v + for k, v in state_dict.items() + } + model.load_state_dict(state_dict) + return model + + def streaming_generate(self, tokens, temperature=1.0, **kwargs): + top_k = kwargs.pop('top_k', self.config.top_k) + top_p = kwargs.pop('top_p', self.config.top_p) + max_length = kwargs.pop('max_length', tokens.size(1) + 100) + + batch_size = tokens.size(0) + lengths = kwargs.pop( + 'prompt_length', + torch.tensor([tokens.size(1)], device=tokens.device)) + + min_prompt_length = lengths.min().item() + max_sequence_length = min(max_length, + self.config.max_position_embeddings) + + # If the context is too big, this happens + if min_prompt_length >= max_sequence_length: + raise ValueError('context length too large') + + pad_length = max_sequence_length - tokens.size(1) + if pad_length > 0: + pads = torch.zeros( + batch_size, pad_length, device=tokens.device).long() + tokens = torch.cat((tokens, pads), dim=-1) + + # Added termination_id to support the case that we want to terminate the + # generation once that id is generated. + termination_id = self.config.eod_id + + # Whether we have reached a termination id. + is_generation_done = torch.zeros( + batch_size, dtype=torch.uint8, device=tokens.device) + + with torch.no_grad(): + for context_length in range(min_prompt_length, + max_sequence_length): + + # Pick the slice that we need to pass through the network. + tokens2use = tokens[:, :context_length] + + # logits will be meanigful only in the last pipeline stage. + logits = self(tokens2use).logits + + # Sample. + last_token_logits = logits[:, -1, :] + new_sample = sample( + last_token_logits, + top_k=top_k, + top_p=top_p, + temperature=temperature, + vocab_size=self.config.vocab_size) + + # If a prompt length is smaller or equal th current context + # length, it means we have started generating tokens + started = lengths <= context_length + # Update the tokens. + tokens[started, context_length] = new_sample[started] + + yield TokenGeneratorOutput(sequences=tokens[:, :(context_length + + 1)]) + + done_token = (new_sample == termination_id).byte() & \ + started.byte() + + is_generation_done = is_generation_done | done_token + done = torch.all(is_generation_done) + + if done: + break + + def generate(self, tokens, temperature=1.0, **kwargs): + last_output = None + for output in self.streaming_generate(tokens, temperature, **kwargs): + last_output = output + return last_output diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/configuration.py b/runtime/Lib/site_packages/models/nlp/gpt3/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..0a1d16d75f13c05942d6824d5fafdde0fc986ef9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/configuration.py @@ -0,0 +1,190 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger() + + +class GPT3Config(PretrainedConfig): + r""" + Configuration classes for GPT-3 model. + + Class attributes: + + - **model_type** (`str`) -- An identifier for the model type, serialized into the JSON file, can be used to recreate + the correct object in [`~transformers.AutoConfig`]. + + Args: + vocab_size (`int`, *optional*, defaults to 25600): + Vocabulary size of the GPT model. Defines the number of different + tokens that can be represented by the `inputs_ids` passed when + calling [`GPT3Model`]. + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the decoder layers and the pooler layer. + ffn_hidden_size (`int`, *optional*, defaults to None): + Dimensionality of the ffn layer, None defaults to four times the hidden_size. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the + Transformer decoder. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) + layer in the Transformer decoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the + decoder and pooler. If string, `"gelu"`, `"relu"`, `"silu"` and + `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the + embeddings, decoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. + Typically set this to something large just in case (e.g., 512 or + 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling + [`GPT3Model`]. + layernorm_epsilon (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + bias_gelu_fusion (`bool`, *optional*, defaults to True): + Whether to use gelu activation function when mixing bias. + fp32_residual_connection (`bool`, *optional*, defaults to False): + Whether to use fp32 for residual connection + between layers to improve accuracy. + sequence_parallel (`bool`, *optional*, defaults to False): + Whether to use sequence parallel during training. + bf16 (`bool`, *optional*, defaults to `False`): + Whether to use bf16 16-bit (mixed) precision training instead of 32-bit training. + Requires Ampere or higher NVIDIA architecture or using CPU (no_cuda). + This is an experimental API and it may change. + fp16 (`bool`, *optional*, defaults to `False`): + Whether to use fp16 16-bit (mixed) precision training instead of 32-bit training. + apply_query_key_layer_scaling (`bool`, *optional*, defaults to `True`): + Whether to scale query and key layer parameters during training. + init_method_std (`float`, *optional*, defaults to `0.02`): + The standard deviation of the normal distribution for initialization process. + eod_id (`int`, *optional*, defaults to `1`): + The end of text label for tokenizer, also indicates the end of the generation. + tokens_to_generate (`int`, *optional*, defaults to 100): + Number of tokens to generate. + top_k (`int`, *optional*, defaults to 0): + Number of highest probability vocabulary tokens to keep for + top-k-filtering that will be used by default in + the `generate` method of the model. + top_p (`float`, *optional*, defaults to 0.9): + Value that will be used by default in the `generate` method of the model + for `top_p`. If set to float < 1, + only the most probable tokens with probabilities that add up to `top_p` + or higher are kept for generation. + temperature (`float`, *optional*, defaults to 1.0): + The value used to module the next token probabilities that will be used + by default in the `generate` method of the model. Must be strictly positive. + """ + + model_type = 'gpt3' + + def __init__( + self, + vocab_size=25600, + hidden_size=768, + ffn_hidden_size=None, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=2048, + type_vocab_size=2, + layernorm_epsilon=1e-12, + bias_gelu_fusion=True, + fp32_residual_connection=False, + sequence_parallel=False, + fp16=False, + bf16=False, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=False, + kv_channels=None, + masked_softmax_fusion=True, + attention_dropout=0.1, + bias_dropout_fusion=True, + apply_residual_connection_post_layernorm=False, + hidden_dropout=0.1, + init_method_std=0.02, + # generate + eod_id=1, + tokens_to_generate=100, + top_k=0, + top_p=0.9, + temperature=1.0, + **kwargs): + super().__init__(layer_norm_eps=layernorm_epsilon, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = 4 * hidden_size \ + if ffn_hidden_size is None else ffn_hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.layernorm_epsilon = layernorm_epsilon + self.bias_gelu_fusion = bias_gelu_fusion + self.fp32_residual_connection = fp32_residual_connection + self.sequence_parallel = sequence_parallel + self.fp16 = fp16 + self.bf16 = bf16 + assert not (fp16 and bf16) + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + if kv_channels is None: + assert hidden_size % num_attention_heads == 0 + self.kv_channels = hidden_size // num_attention_heads + self.masked_softmax_fusion = masked_softmax_fusion + self.attention_dropout = attention_dropout + self.bias_dropout_fusion = bias_dropout_fusion + self.apply_residual_connection_post_layernorm = \ + apply_residual_connection_post_layernorm + self.hidden_dropout = hidden_dropout + self.init_method_std = init_method_std + self.eod_id = eod_id + self.tokens_to_generate = tokens_to_generate + self.top_k = top_k + self.top_p = top_p + self.temperature = temperature + + TORCH_MAJOR = int(torch.__version__.split('.')[0]) + TORCH_MINOR = int(torch.__version__.split('.')[1]) + self.no_persist_layer_norm = \ + TORCH_MAJOR < 1 or (TORCH_MAJOR == 1 and TORCH_MINOR < 11) + + @property + def params_dtype(self): + if self.fp16: + return torch.half + elif self.bf16: + return torch.bfloat16 + else: + return torch.float diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/distributed_gpt3.py b/runtime/Lib/site_packages/models/nlp/gpt3/distributed_gpt3.py new file mode 100644 index 0000000000000000000000000000000000000000..0a4473acc18901f4258cbbd7e2c9e141ac48cc72 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/distributed_gpt3.py @@ -0,0 +1,1349 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os +from collections import OrderedDict +from typing import Callable, Dict, List, Optional, Union + +import torch +from megatron_util import get_args, mpu +from megatron_util.global_vars import get_global_memory_buffer +from megatron_util.model import (AttnMaskType, Float16Module, LayerNorm, + bias_gelu_impl) +from megatron_util.model.fused_softmax import FusedScaleMaskSoftmax +from torch import nn +from torch.nn import functional as F +from transformers.modeling_utils import PreTrainedModel + +from modelscope.models import TorchModel +from modelscope.models.nlp.gpt3 import GPT3Config +from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput +from modelscope.utils.megatron_utils import init_megatron_util +from modelscope.utils.nlp.load_checkpoint import pre_load +from modelscope.utils.streaming_output import StreamingOutputMixin + + +class GPT3ParallelMLP(nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__(self, config, init_method, output_layer_init_method): + super().__init__() + + # Project to 4h. + self.dense_h_to_4h = mpu.ColumnParallelLinear( + config.hidden_size, + config.ffn_hidden_size, + gather_output=False, + init_method=init_method, + skip_bias_add=True) + + self.bias_gelu_fusion = config.bias_gelu_fusion + self.activation_func = F.gelu + + # Project back to h. + self.dense_4h_to_h = mpu.RowParallelLinear( + config.ffn_hidden_size, + config.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True) + + def forward(self, hidden_states): + + # [s, b, 4hp] + intermediate_parallel, bias_parallel = self.dense_h_to_4h( + hidden_states) + + if self.bias_gelu_fusion: + intermediate_parallel = \ + bias_gelu_impl(intermediate_parallel, bias_parallel) + else: + intermediate_parallel = \ + self.activation_func(intermediate_parallel + bias_parallel) + + # [s, b, h] + output, output_bias = self.dense_4h_to_h(intermediate_parallel) + return output, output_bias + + +class GPT3Embedding(nn.Module): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + init_method: weight initialization method + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, config, init_method): + super().__init__() + + self.hidden_size = config.hidden_size + self.init_method = init_method + + # Word embeddings (parallel). + self.word_embeddings = mpu.VocabParallelEmbedding( + config.vocab_size, self.hidden_size, init_method=self.init_method) + + # Position embedding (serial). + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + self.hidden_size) + # Initialize the position embeddings. + self.init_method(self.position_embeddings.weight) + + self.fp32_residual_connection = config.fp32_residual_connection + self.sequence_parallel = config.sequence_parallel + # Embeddings dropout + self.embedding_dropout = nn.Dropout(config.hidden_dropout) + + def zero_parameters(self): + """Zero out all parameters in embedding.""" + self.word_embeddings.weight.data.fill_(0) + self.word_embeddings.weight.shared = True + self.position_embeddings.weight.data.fill_(0) + self.position_embeddings.weight.shared = True + + def forward(self, input_ids, position_ids): + # Embeddings. + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + embeddings = words_embeddings + position_embeddings + + # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. + embeddings = embeddings.transpose(0, 1).contiguous() + + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + + # Dropout. + if self.sequence_parallel: + embeddings = mpu.scatter_to_sequence_parallel_region(embeddings) + with mpu.get_cuda_rng_tracker().fork(): + embeddings = self.embedding_dropout(embeddings) + else: + embeddings = self.embedding_dropout(embeddings) + return embeddings + + +class NoopTransformerLayer(nn.Module): + + def __init__(self, layer_number): + super().__init__() + self.layer_number = layer_number + + def forward(self, + hidden_states, + attention_mask, + encoder_output=None, + enc_dec_attn_mask=None, + inference_params=None): + return hidden_states.clone() + + +def attention_mask_func(attention_scores, attention_mask): + attention_scores.masked_fill_(attention_mask, -10000.0) + return attention_scores + + +class GPT3CoreAttention(nn.Module): + + def __init__(self, + config, + layer_number, + attn_mask_type=AttnMaskType.padding): + super().__init__() + self.fp16 = config.fp16 + self.bf16 = config.bf16 + + self.apply_query_key_layer_scaling = config.apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = config.attention_softmax_in_fp32 + if self.apply_query_key_layer_scaling: + self.attention_softmax_in_fp32 = True + self.layer_number = max(1, layer_number) + self.attn_mask_type = attn_mask_type + self.sequence_parallel = config.sequence_parallel + + projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + world_size = mpu.get_tensor_model_parallel_world_size() + self.hidden_size_per_partition = mpu.divide(projection_size, + world_size) + self.hidden_size_per_attention_head = mpu.divide( + projection_size, config.num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + config.num_attention_heads, world_size) + + coeff = None + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + if self.apply_query_key_layer_scaling: + coeff = self.layer_number + self.norm_factor *= coeff + + self.scale_mask_softmax = FusedScaleMaskSoftmax( + self.fp16, self.bf16, self.attn_mask_type, + config.masked_softmax_fusion, attention_mask_func, + self.attention_softmax_in_fp32, coeff) + + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = nn.Dropout(config.attention_dropout) + + def forward(self, query_layer, key_layer, value_layer, attention_mask): + + # =================================== + # Raw attention scores. [b, np, s, s] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view(output_size[2], + output_size[0] * output_size[1], -1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], + output_size[0] * output_size[1], -1) + + # preallocting input tensor: [b * np, sq, sk] + matmul_input_buffer = get_global_memory_buffer().get_tensor( + (output_size[0] * output_size[1], output_size[2], output_size[3]), + query_layer.dtype, 'mpu') + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.baddbmm( + matmul_input_buffer, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / self.norm_factor)) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + # =========================== + # Attention probs and dropout + # =========================== + + # attention scores and attention mask [b, np, sq, sk] + attention_probs = self.scale_mask_softmax(attention_scores, + attention_mask) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + + if not self.sequence_parallel: + with mpu.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + else: + attention_probs = self.attention_dropout(attention_probs) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + + # change view [sk, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], + output_size[2], -1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size_per_partition,) + context_layer = context_layer.view(*new_context_layer_shape) + + return context_layer + + +class GPT3ParallelAttention(nn.Module): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__(self, config, init_method, output_layer_init_method, + layer_number): + super().__init__() + self.layer_number = max(1, layer_number) + self.params_dtype = config.params_dtype + + projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + world_size = mpu.get_tensor_model_parallel_world_size() + self.hidden_size_per_attention_head = mpu.divide( + projection_size, config.num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + config.num_attention_heads, world_size) + + # Strided linear layer. + self.query_key_value = mpu.ColumnParallelLinear( + config.hidden_size, + 3 * projection_size, + gather_output=False, + init_method=init_method) + + self.core_attention = GPT3CoreAttention(config, self.layer_number) + + # Output. + self.dense = mpu.RowParallelLinear( + projection_size, + config.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True) + + def _allocate_memory(self, inference_max_sequence_len, batch_size): + return torch.empty( + inference_max_sequence_len, + batch_size, + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + dtype=self.params_dtype, + device=torch.cuda.current_device()) + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [sq, b, h] + + # ================================================= + # Pre-allocate memory for key-values for inference. + # ================================================= + if inference_params: + if self.layer_number not in inference_params.key_value_memory_dict: + inf_max_seq_len = inference_params.max_sequence_len + inf_max_batch_size = inference_params.max_batch_size + inference_key_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size) + inference_value_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size) + inference_params.key_value_memory_dict[self.layer_number] = ( + inference_key_memory, inference_value_memory) + else: + inference_key_memory, inference_value_memory = \ + inference_params.key_value_memory_dict[self.layer_number] + + # ===================== + # Query, Key, and Value + # ===================== + # Attention heads [sq, b, h] --> [sq, b, (np * 3 * hn)] + mixed_x_layer, _ = self.query_key_value(hidden_states) + + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn] + new_tensor_shape = mixed_x_layer.size()[:-1] + \ + (self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head) + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) + + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] + (query_layer, key_layer, + value_layer) = mpu.split_tensor_along_last_dim(mixed_x_layer, 3) + + # ================================== + # Adjust key and value for inference + # ================================== + + if inference_params: + batch_start = inference_params.batch_size_offset + batch_end = batch_start + key_layer.size(1) + assert batch_end <= inference_key_memory.size(1) + sequence_start = inference_params.sequence_len_offset + sequence_end = sequence_start + key_layer.size(0) + assert sequence_end <= inference_key_memory.size(0) + # Copy key and values. + inference_key_memory[sequence_start:sequence_end, + batch_start:batch_end, ...] = key_layer + inference_value_memory[sequence_start:sequence_end, + batch_start:batch_end, ...] = value_layer + key_layer = inference_key_memory[:sequence_end, + batch_start:batch_end, ...] + value_layer = inference_value_memory[:sequence_end, + batch_start:batch_end, ...] + + # ================================== + # core attention computation + # ================================== + + context_layer = self.core_attention(query_layer, key_layer, + value_layer, attention_mask) + + # ================= + # Output. [sq, b, h] + # ================= + + output, bias = self.dense(context_layer) + + return output, bias + + +class nullcontext: + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass + + +def bias_dropout_add(x, bias, residual, prob, training): + # type: (Tensor, Tensor, Tensor, float, bool) -> Tensor + out = F.dropout(x + bias, p=prob, training=training) + out = residual + out + return out + + +def get_bias_dropout_add(training): + + def _bias_dropout_add(x, bias, residual, prob): + return bias_dropout_add(x, bias, residual, prob, training) + + return _bias_dropout_add + + +@torch.jit.script +def bias_dropout_add_fused_train(x: torch.Tensor, bias: torch.Tensor, + residual: torch.Tensor, + prob: float) -> torch.Tensor: + return bias_dropout_add(x, bias, residual, prob, True) + + +@torch.jit.script +def bias_dropout_add_fused_inference(x: torch.Tensor, bias: torch.Tensor, + residual: torch.Tensor, + prob: float) -> torch.Tensor: + return bias_dropout_add(x, bias, residual, prob, False) + + +class GPT3ParallelTransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__(self, config, init_method, output_layer_init_method, + layer_number): + + super().__init__() + self.layer_number = layer_number + + self.apply_residual_connection_post_layernorm \ + = config.apply_residual_connection_post_layernorm + + self.bf16 = config.bf16 + self.fp32_residual_connection = config.fp32_residual_connection + + # Layernorm on the input data. + self.input_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + # Self attention. + self.self_attention = GPT3ParallelAttention(config, init_method, + output_layer_init_method, + layer_number) + self.hidden_dropout = config.hidden_dropout + self.bias_dropout_fusion = config.bias_dropout_fusion + + # Layernorm on the attention output + self.post_attention_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + # MLP + self.mlp = GPT3ParallelMLP(config, init_method, + output_layer_init_method) + + # Set bias+dropout+add fusion grad_enable execution handler. + TORCH_MAJOR = int(torch.__version__.split('.')[0]) + TORCH_MINOR = int(torch.__version__.split('.')[1]) + use_nvfuser = TORCH_MAJOR > 1 or (TORCH_MAJOR == 1 + and TORCH_MINOR >= 10) + self.bias_dropout_add_exec_handler = \ + nullcontext if use_nvfuser else torch.enable_grad + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [s, b, h] + + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output, attention_bias = \ + self.self_attention( + layernorm_output, + attention_mask, + inference_params=inference_params) + # Residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + if self.bias_dropout_fusion: + if self.training: + bias_dropout_add_func = bias_dropout_add_fused_train + else: + bias_dropout_add_func = bias_dropout_add_fused_inference + else: + bias_dropout_add_func = get_bias_dropout_add(self.training) + + with self.bias_dropout_add_exec_handler(): + layernorm_input = bias_dropout_add_func( + attention_output, attention_bias.expand_as(residual), residual, + self.hidden_dropout) + + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + + # MLP. + mlp_output, mlp_bias = self.mlp(layernorm_output) + # Second residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + with self.bias_dropout_add_exec_handler(): + output = bias_dropout_add_func(mlp_output, + mlp_bias.expand_as(residual), + residual, self.hidden_dropout) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = mpu.make_viewless_tensor( + inp=output, requires_grad=output.requires_grad, keep_graph=True) + + return output + + +class GPT3ParallelTransformer(nn.Module): + """Transformer class.""" + + def __init__(self, + config, + init_method, + output_layer_init_method, + post_layer_norm=True, + pre_process=True, + post_process=True): + super().__init__() + + self.bf16 = config.bf16 + self.fp32_residual_connection = config.fp32_residual_connection + self.post_layer_norm = post_layer_norm + self.pre_process = pre_process + self.post_process = post_process + self.input_tensor = None + + self.sequence_parallel = config.sequence_parallel + + # Number of layers. + self.num_layers = config.num_hidden_layers + + # Transformer layers. + def build_layer(layer_number): + return GPT3ParallelTransformerLayer(config, init_method, + output_layer_init_method, + layer_number) + + if self.num_layers == 0: + self.num_layers = 1 + self.layers = torch.nn.ModuleList([NoopTransformerLayer(1)]) + else: + self.layers = torch.nn.ModuleList( + [build_layer(i + 1) for i in range(self.num_layers)]) + + if self.post_process and self.post_layer_norm: + # Final layer norm before output. + self.final_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [s, b, h] + + if not self.pre_process: + # See set_input_tensor() + hidden_states = self.input_tensor + + # Viewless tensor. + # - We only need to create a viewless tensor in the case of micro batch + # size (mbs) == 1, since in this case, 'hidden_states.transpose()' + # above creates a view tensor, and '.contiguous()' is a pass-through. + # For mbs >= 2, '.contiguous()' creates a new tensor, eliminating + # the need to make it viewless. + # + # However, we don't explicitly check mbs == 1 here because + # make_viewless_tensor() has negligible overhead when its input + # is already viewless. + # + # - For the 'else' case above, calling make_viewless_tensor() here is + # likely redundant, since p2p_communication.py (likely originator) + # already creates viewless tensors. That said, make_viewless_tensor() + # is called here to be future-proof and corner-case-proof. + hidden_states = mpu.make_viewless_tensor( + hidden_states, + requires_grad=True, + keep_graph=True, + ) + + if self.sequence_parallel: + rng_context = mpu.get_cuda_rng_tracker().fork() + else: + rng_context = nullcontext() + + with rng_context: + # Forward pass. + for index in range(self.num_layers): + layer = self._get_layer(index) + hidden_states = layer( + hidden_states, + attention_mask, + inference_params=inference_params) + + # Final layer norm. + if self.post_process and self.post_layer_norm: + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class GPT3TransformerLanguageModel(nn.Module): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, config, init_method, output_layer_init_method): + super().__init__() + + self.hidden_size = config.hidden_size + self.init_method = init_method + self.encoder_hidden_state = None + + # Embeddings. + self.embedding = GPT3Embedding(config, self.init_method) + + # Transformer. + self.encoder = GPT3ParallelTransformer( + config, + self.init_method, + output_layer_init_method, + ) + + def forward(self, + enc_input_ids, + enc_position_ids, + enc_attn_mask, + inference_params=None, + enc_hidden_states=None): + + # Encoder embedding. + encoder_input = self.embedding(enc_input_ids, enc_position_ids) + + # Run encoder. + if enc_hidden_states is None: + if self.encoder is not None: + encoder_output = self.encoder( + encoder_input, + enc_attn_mask, + inference_params=inference_params) + else: + encoder_output = self.encoder_hidden_state + else: + encoder_output = enc_hidden_states.to(encoder_input.dtype) + + return encoder_output + + +def init_method_normal(sigma): + """Init method based on N(0, sigma).""" + + def init_(tensor): + return nn.init.normal_(tensor, mean=0.0, std=sigma) + + return init_ + + +def scaled_init_method_normal(sigma, num_layers): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +class GPT3Model(PreTrainedModel): + + config_class = GPT3Config + + def __init__(self, config): + super().__init__(config) + + self.language_model = GPT3TransformerLanguageModel( + config, init_method_normal(config.init_method_std), + scaled_init_method_normal(config.init_method_std, + config.num_hidden_layers)) + + def word_embeddings_weight(self): + return self.language_model.embedding.word_embeddings.weight + + @staticmethod + def build_attention_mask_and_position_ids(tokens): + seq_length = tokens.size(1) + attention_mask = torch.tril( + torch.ones((1, 1, seq_length, seq_length), device=tokens.device)) + attention_mask = (attention_mask < 0.5) + + position_ids = torch.arange( + seq_length, dtype=torch.long, device=tokens.device) + position_ids = position_ids.unsqueeze(0).expand_as(tokens) + + return attention_mask, position_ids + + def forward(self, + input_ids, + attention_mask=None, + position_ids=None, + inference_params=None, + labels=None, + **kwargs): + if attention_mask is None and position_ids is None: + attention_mask, position_ids = \ + self.build_attention_mask_and_position_ids(input_ids) + + lm_output = self.language_model( + input_ids, + position_ids, + attention_mask, + inference_params=inference_params) + + logits_parallel = mpu.LinearWithGradAccumulationAndAsyncCommunication.apply( + lm_output, self.word_embeddings_weight(), None, False, True, + self.config.sequence_parallel) + + losses = None + if labels is not None: + # [b s] => [s b] + labels = labels.transpose(0, 1).contiguous() + losses = mpu.vocab_parallel_cross_entropy( + logits_parallel.clone().float(), labels) + # [s b] => [b s] + losses = losses.transpose(0, 1).contiguous() + + # Gather if needed. + logits = mpu.gather_from_tensor_model_parallel_region(logits_parallel) + # [s b h] => [b s h] + logits = logits.transpose(0, 1).contiguous() + + return logits, losses + + +def modify_logits_for_top_k_filtering(logits, top_k): + """Set the logits for none top-k values to -inf.""" + + filter_ = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits.masked_fill_(filter_, float('-Inf')) + + +def modify_logits_for_top_p_filtering(logits, top_p): + """Set the logits for none top-p values to -inf.""" + + # First sort and calculate cumulative sum of probabilities. + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Filteration based on the cumulative sum. + filter_ = cumulative_probs > top_p + # This shift by 1 is weird and I cannot justify it. This existed + # in the original implementation: + # https://github.com/ari-holtzman/degen/blob/master/gen.py + # and I guess it is needed so keeping it for now. + filter_[:, 1:] = filter_[:, :-1].clone() + # Make sure we at least have one token to select from. + filter_[..., 0] = 0 + + # Fill in the filtered part + filter_ = filter_.scatter(1, sorted_indices, filter_) + logits.masked_fill_(filter_, float('-Inf')) + + +def sample(logits, top_k=0, top_p=0.0, temperature=1.0, vocab_size=None): + """ Sample and generate a token. + Note: logits has the dimension [b, v] where b is the batch size + and v is the vocabulary size. + If vocab_size is provided, we will make sure the sample that is + generated is in [0, vocab-size). This will avoid out of vocabulary + generations due to padding. + """ + + # Check logits for consistency. + assert logits.ndim == 2, 'expected the logits to be of [b, v] shape.' + + # Greedy is just simple argmax. + if top_k == 1: + assert top_p == 0.0, 'cannot set both greedy and top-p samplings.' + samples = torch.argmax(logits, dim=-1) + + # Top-k or top-p sampling. + else: + # Clone so we do not modify the inputs, + logits = logits.clone() + # Apply temperature in place. + if temperature != 1.0: + logits.div_(temperature) + + if top_k > 1: + assert top_p == 0.0, 'cannot set both top-k and top-p samplings.' + assert top_k <= logits.size(1), 'top-k is larger than logit size.' + if vocab_size: + assert top_k < vocab_size, 'top-k is larger than vocab size.' + modify_logits_for_top_k_filtering(logits, top_k) + + elif top_p > 0.0: + assert top_p <= 1.0, 'top-p should be in (0, 1].' + modify_logits_for_top_p_filtering(logits, top_p) + + # After filtering, we need to recalculate the distribution. + probs = logits.softmax(dim=-1) + samples = torch.multinomial(probs, num_samples=1).view(-1) + + # If vocab size is provided, make sure the samples are in + # in the range [0, vocab-size). + if vocab_size: + samples = torch.clamp(samples, min=0, max=(vocab_size - 1)) + + return samples + + +class InferenceParams: + """Inference parameters that are passed to the main model in order + to efficienly calculate and store the context during inference.""" + + def __init__(self, max_batch_size, max_sequence_len): + """Note that offsets are set to zero and we always set the + flag to allocate memory. After the first call, make sure to + set this flag to False.""" + self.max_sequence_len = max_sequence_len + self.max_batch_size = max_batch_size + self.sequence_len_offset = 0 + self.batch_size_offset = 0 + self.key_value_memory_dict = {} + + def swap_key_value_dict(self, batch_idx): + 'swap between batches' + if len(self.key_value_memory_dict) == 0: + raise ValueError('should not swap when dict in empty') + + for layer_number in self.key_value_memory_dict.keys(): + inference_key_memory, inference_value_memory = self.key_value_memory_dict[ + layer_number] + assert len(batch_idx) == inference_key_memory.shape[ + 1] # make sure batch size is the same + new_inference_key_memory = inference_key_memory[:, batch_idx] + new_inference_value_memory = inference_value_memory[:, batch_idx] + self.key_value_memory_dict[layer_number] = ( + new_inference_key_memory, new_inference_value_memory) + + +def split_into_partitions(tensor, num_partitions, partition_dim, stride): + per_partition_size = mpu.utils.divide( + tensor.size(partition_dim), num_partitions) + per_partition_per_stride_size = mpu.utils.divide(per_partition_size, + stride) + partitions_list = torch.split( + tensor, per_partition_per_stride_size, dim=partition_dim) + partitions = [] + for i in range(num_partitions): + partition = torch.cat( + partitions_list[i::num_partitions], dim=partition_dim) + partitions.append(partition) + return partitions + + +def split_state_dict(state_dict: Dict[str, torch.Tensor], model: GPT3Model, + partitions: int) -> Dict[str, torch.Tensor]: + if partitions == 1: + return state_dict + rank: int = mpu.get_tensor_model_parallel_rank() + for name, parameters in model.named_parameters(): + if parameters.shape == state_dict[name].shape: + continue + dim = max(parameters.partition_dim, 0) + stride = parameters.partition_stride + state_dict[name] = split_into_partitions(state_dict[name], partitions, + dim, stride)[rank] + return state_dict + + +class DistributedGPT3(TorchModel, StreamingOutputMixin): + + def __init__(self, + model_dir, + rank, + path_load_tag='model', + *args, + megatron_cfg=None, + **kwargs): + super().__init__(model_dir, *args, **kwargs) + init_megatron_util(megatron_cfg, model_dir, rank=rank) + + self.config = GPT3Config.from_pretrained(model_dir) + # Build model. + model = GPT3Model(self.config) + + for param in model.parameters(): + mpu.set_defaults_if_not_set_tensor_model_parallel_attributes(param) + + # GPU allocation. + model.cuda(torch.cuda.current_device()) + + # Fp16 conversion. + if self.config.fp16 or self.config.bf16: + model = Float16Module(model, self.config) + + self.dist_model = model + + tensor_ws = mpu.get_tensor_model_parallel_world_size() + ckpt_ws = get_args().get('checkpoint_tensor_model_parallel_size', None) + ckpt_ws = tensor_ws if ckpt_ws is None else ckpt_ws + ckpt_rank = mpu.get_tensor_model_parallel_rank() * ckpt_ws // tensor_ws + load_model = pre_load(ckpt_rank, model_dir, tag=path_load_tag) + load_model = split_state_dict(load_model, model, tensor_ws // ckpt_ws) + + self.dist_model.load_state_dict( + load_model, strict=kwargs.get('strict', True)) + + self.inference_params = None + + def train(self, mode: bool = True): + if mode: + self.inference_params = None + return super().train(mode) + + def forward(self, + tokens, + attention_mask=None, + position_ids=None, + labels=None, + prompts_len=None, + inputs_len=None): + + logits, losses = self.dist_model( + tokens, + attention_mask, + position_ids, + inference_params=self.inference_params, + labels=labels) + + loss = None + if labels is None: + self.inference_params.sequence_len_offset += tokens.size(1) + else: + loss_mask = torch.ones( + labels.size(), dtype=torch.float, device=tokens.device) + if inputs_len is None: + for i, l in enumerate(prompts_len): + loss_mask[i, l:] = 0 + else: + for i, l in enumerate(inputs_len): + loss_mask[i, l - 1:] = 0 + for i, l in enumerate(prompts_len): + loss_mask[i, :l - 1] = 0 + + losses = losses.float() + loss_mask = loss_mask.view(-1).float() + mask_sum = loss_mask.sum() + if mask_sum == 0: + loss = torch.sum(losses.view(-1)).zero_() + else: + loss = torch.sum(losses.view(-1) * loss_mask) / mask_sum + + return TextGenerationModelOutput(logits=logits, loss=loss) + + def sample(self, + tokens, + prompts_len=None, + use_eod_token_for_early_termination=True, + stop_on_double_eol=False, + stop_on_eol=False, + **kwargs): + top_k = kwargs.pop('top_k', self.config.top_k) + top_p = kwargs.pop('top_p', self.config.top_p) + temperature = kwargs.pop('temperature', self.config.temperature) + max_length = kwargs.pop( + 'max_length', + tokens.size(1) + self.config.tokens_to_generate) + + batch_size = tokens.size(0) + lengths = prompts_len + if lengths is None: + lengths = torch.tensor([tokens.size(1)], device=tokens.device) + + min_prompt_length = lengths.min().item() + max_sequence_length = min(max_length, + self.config.max_position_embeddings) + + # If the context is too big, this happens + if min_prompt_length >= max_sequence_length: + raise ValueError('context length + tokens_to_generate too large') + + pad_length = max_sequence_length - tokens.size(1) + if pad_length > 0: + pads = torch.zeros( + batch_size, pad_length, device=tokens.device).long() + tokens = torch.cat((tokens, pads), dim=-1) + + # Initialize inference parameters. + self.inference_params = InferenceParams(batch_size, + max_sequence_length) + + # Added termination_id to support the case that we want to terminate the + # generation once that id is generated. + termination_id = self.config.eod_id + + # Whether we have reached a termination id. + is_generation_done = torch.zeros( + batch_size, dtype=torch.uint8, device=torch.cuda.current_device()) + + # ============= + # Run infernece + # ============= + + attention_mask, position_ids = \ + GPT3Model.build_attention_mask_and_position_ids(tokens) + prev_context_length = 0 + for context_length in range(min_prompt_length, max_sequence_length): + + # Pick the slice that we need to pass through the network. + tokens2use = tokens[:, prev_context_length:context_length] + positions2use = position_ids[:, prev_context_length:context_length] + attention_mask2use = attention_mask[ + ..., prev_context_length:context_length, :context_length] + + # logits will be meanigful only in the last pipeline stage. + logits = self(tokens2use, attention_mask2use, positions2use).logits + + # Sample. + last_token_logits = logits[:, -1, :] + new_sample = sample( + last_token_logits, + top_k=top_k, + top_p=top_p, + temperature=temperature, + vocab_size=self.config.vocab_size) + + # If a prompt length is smaller or equal th current context + # length, it means we have started generating tokens + started = lengths <= context_length + # Update the tokens. + tokens[started, context_length] = new_sample[started] + + # streaming output + yield TokenGeneratorOutput(sequences=tokens[:, :(context_length + + 1)]) + + # Update the context length for the next token generation. + prev_context_length = context_length + + # instead tokenization should be in the inference loop so stop sequences can be used + if stop_on_double_eol: + hit_double_eol = (new_sample == 628).byte() & started.byte() + hit_two_eols = (new_sample == 198).byte() & ( + tokens[:, + context_length - 1] == 198).byte() & started.byte() + done_token = hit_double_eol | hit_two_eols + elif stop_on_eol: + hit_double_eol = (new_sample == 628).byte() & started.byte() + hit_eol = (new_sample == 198).byte() & started.byte() + done_token = hit_double_eol | hit_eol + else: + done_token = (new_sample == termination_id).byte() & \ + started.byte() + + is_generation_done = is_generation_done | done_token + done = torch.all(is_generation_done) + + if use_eod_token_for_early_termination and done: + break + + def beam_search(self, tokens, beam_size=5, num_return_gen=1, **kwargs): + batch_size = tokens.size(0) + assert (batch_size == 1) + prompt_length = kwargs.pop( + 'prompt_length', + torch.tensor([tokens.size(1)], device=tokens.device)).item() + stop_token = self.config.eod_id + pads = torch.ones( + 1, self.config.tokens_to_generate, + device=tokens.device).long() * stop_token + tokens = torch.cat((tokens, pads), dim=-1) + final_sequence_length = tokens.size(1) + final_sequence_length = min(final_sequence_length, + self.config.max_position_embeddings) + + # If the context is too big, this happens + if prompt_length >= final_sequence_length: + raise ValueError('context length + tokens_to_generate too large') + + # Initialize inference parameters. + self.inference_params = InferenceParams(beam_size, + final_sequence_length) + + beam_hyp = BeamHypotheses(beam_size) + done = False + scores = torch.zeros( + beam_size, dtype=torch.float32, + device=torch.cuda.current_device()).unsqueeze(1) + + # ============= + # Run infernece + # ============= + tokens = tokens.repeat(beam_size, 1) + attention_mask, position_ids = \ + GPT3Model.build_attention_mask_and_position_ids(tokens) + prev_context_length = 0 + for context_length in range(prompt_length, final_sequence_length): + + # Pick the slice that we need to pass through the network. + tokens2use = tokens[:, prev_context_length:context_length] + positions2use = position_ids[:, prev_context_length:context_length] + attention_mask2use = attention_mask[ + ..., prev_context_length:context_length, :context_length] + + # logits will be meanigful only in the last pipeline stage. + logits = self(tokens2use, attention_mask2use, positions2use).logits + + vocab_size = logits.size(2) + log_probs = F.log_softmax(logits, dim=2) + new_scores = log_probs[:, -1, :] + scores + + if context_length == prompt_length: # if this is the first one + sorted_scores, indices = torch.sort( + new_scores[0, :], descending=True) + else: + sorted_scores, indices = torch.sort( + new_scores.view(-1), descending=True) + + best_beam_ids = torch.div(indices[:2 * beam_size], + vocab_size).trunc().long() + best_words = indices[:2 * beam_size] % vocab_size + best_scores = sorted_scores[:2 * beam_size] + + next_beams = [] + for beam_token_rank, (token_id, beam_score, beam_id) in enumerate( + zip(best_words, best_scores, best_beam_ids)): + if token_id.item() == stop_token: + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= beam_size + if is_beam_token_worse_than_top_num_beams: + continue + beam_hyp.add(tokens[beam_id].clone(), beam_score, + context_length + 1 - prompt_length) + else: + # add next predicted token since it is not eos_token + next_beams.append((token_id, beam_score, beam_id)) + + if len(next_beams) == beam_size: + break + + if beam_hyp.is_done(best_scores.max().item(), + context_length + 1 - prompt_length): + done = True + break + + best_batches = tokens.new([item[2] for item in next_beams]) + tokens = tokens[best_batches, :] + tokens[:, context_length] = tokens.new( + [item[0] for item in next_beams]) + scores = scores.new([item[1] for item in next_beams]).unsqueeze(1) + + # set inference key values to make it consistent with best beam index + self.inference_params.swap_key_value_dict(best_batches) + + # Update the context length for the next token generation. + prev_context_length = context_length + + # if cannot find stop token, add open beams to hyps + if not done: + for beam_id in range(beam_size): + beam_hyp.add(tokens[beam_id].clone(), scores[beam_id], + context_length + 1 - prompt_length) + + # rank based on scores + sorted_hyps = sorted(beam_hyp.beams, key=lambda x: x[0], reverse=True) + num_return_gen = min(num_return_gen, len(sorted_hyps)) + scores = [sorted_hyps[i][0] for i in range(num_return_gen)] + tokens = [sorted_hyps[i][1] for i in range(num_return_gen)] + scores = torch.stack(scores, dim=0) + tokens = torch.stack(tokens, dim=0) + + return TokenGeneratorOutput(sequences=tokens, scores=scores) + + @torch.no_grad() + def generate(self, tokens, do_sample=True, *args, **kwargs): + if do_sample: + last_output = None + for output in self.sample(tokens, *args, **kwargs): + last_output = output + return last_output + else: + return self.beam_search(tokens, *args, **kwargs) + + @torch.no_grad() + def stream_generate(self, tokens, *args, **kwargs): + return self.sample(tokens, *args, **kwargs) + + def state_dict(self, destination=None, prefix='', keep_vars=False): + return self.dist_model.state_dict(destination, prefix, keep_vars) + + def load_state_dict(self, + state_dict: 'OrderedDict[str, torch.Tensor]', + strict: bool = True): + return self.dist_model.load_state_dict(state_dict, strict) + + def save_pretrained(self, + target_folder: Union[str, os.PathLike], + save_checkpoint_names: Union[str, List[str]] = None, + save_function: Callable = None, + config: Optional[dict] = None, + **kwargs): + # DistributedPipeline type is different from task name + config['pipeline']['type'] = 'gpt3-generation' + + config['model'].pop('rank', None) + config['model'].pop('megatron_cfg', None) + config['megatron'].pop('rank', None) + config['megatron'].pop('checkpoint_tensor_model_parallel_size', None) + tp_size = get_args().tensor_model_parallel_size + pp_size = get_args().pipeline_model_parallel_size + config['megatron']['world_size'] = tp_size * pp_size + + return super().save_pretrained(target_folder, save_checkpoint_names, + save_function, config, **kwargs) + + +class BeamHypotheses: + + def __init__(self, + num_beams: int, + length_penalty: float = 1.0, + early_stopping: bool = False): + """ + Initialize n-best list of hypotheses. + """ + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.num_beams = num_beams + self.beams = [] + self.worst_score = 1e9 + + def __len__(self): + """ + Number of hypotheses in the list. + """ + return len(self.beams) + + def add(self, + hyp: torch.LongTensor, + sum_logprobs: float, + beam_indices: Optional[torch.LongTensor] = None): + """ + Add a new hypothesis to the list. + """ + score = sum_logprobs / (hyp.shape[-1]**self.length_penalty) + if len(self) < self.num_beams or score > self.worst_score: + self.beams.append((score, hyp, beam_indices)) + if len(self) > self.num_beams: + sorted_next_scores = sorted([ + (s, idx) for idx, (s, _, _) in enumerate(self.beams) + ]) + del self.beams[sorted_next_scores[0][1]] + self.worst_score = sorted_next_scores[1][0] + else: + self.worst_score = min(score, self.worst_score) + + def is_done(self, best_sum_logprobs: float, cur_len: int) -> bool: + """ + If there are enough hypotheses and that none of the hypotheses being generated can become better than the worst + one in the heap, then we are done with this sentence. + """ + + if len(self) < self.num_beams: + return False + elif self.early_stopping: + return True + else: + cur_score = best_sum_logprobs / cur_len**self.length_penalty + ret = self.worst_score >= cur_score + return ret diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/text_generation.py b/runtime/Lib/site_packages/models/nlp/gpt3/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..2bb36218dd8dfea9a5aa5a7bce165780c45d54da --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/text_generation.py @@ -0,0 +1,86 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from collections import OrderedDict +from typing import Dict, Generator + +import torch +from transformers import BertTokenizer + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.nlp.gpt3 import GPT3Model +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import read_config +from modelscope.utils.streaming_output import StreamingOutputMixin + +__all__ = ['GPT3ForTextGeneration'] + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.gpt3) +class GPT3ForTextGeneration(TorchModel, StreamingOutputMixin): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the text generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + # Temporarily compatible with DistributedGPT3 and GPT3Model, + # the base/large model based on GPT3Model will be replaced in the future, + # and GPT3Model will be deprecated + if 'megatron' in read_config(model_dir): + from modelscope.models.nlp import DistributedGPT3 + self.model = DistributedGPT3(model_dir, **kwargs) + else: + self.model = GPT3Model.from_pretrained(model_dir) + self.tokenizer = BertTokenizer.from_pretrained(model_dir) + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + + Example: + >>> { + >>> 'logits': Tensor([[0.54, 0.32...])]), # logits + >>> } + """ + return self.model(**input) + + def generate(self, inputs: Dict[str, Tensor], + **kwargs) -> Dict[str, Tensor]: + if not isinstance(self.model, GPT3Model): + return self.model.generate(**inputs, **kwargs) + + tokens = inputs['input_ids'] + lengths = self._get_length(inputs['attention_mask']) + return self.model.generate(tokens, prompt_length=lengths, **kwargs) + + @staticmethod + def _get_length(attention_mask: torch.Tensor) -> Tensor: + return attention_mask.sum(-1) - 1 + + def save_pretrained(self, *args, **kwargs): + if not isinstance(self.model, GPT3Model): + return self.model.save_pretrained(*args, **kwargs) + return super().save_pretrained(*args, **kwargs) + + def state_dict(self, destination=None, prefix='', keep_vars=False): + return self.model.state_dict(destination, prefix, keep_vars) + + def load_state_dict(self, + state_dict: 'OrderedDict[str, Tensor]', + strict: bool = True): + return self.model.load_state_dict(state_dict, strict) + + def stream_generate(self, inputs, **kwargs) -> Generator: + tokens = inputs['input_ids'] + lengths = self._get_length(inputs['attention_mask']) + return self.model.streaming_generate( + tokens, prompt_length=lengths, **kwargs) diff --git a/runtime/Lib/site_packages/models/nlp/gpt3/tokenizer.py b/runtime/Lib/site_packages/models/nlp/gpt3/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..eed3f80fafea2807e429814ab7b8e8d6fd495a18 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt3/tokenizer.py @@ -0,0 +1,75 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +from tokenizers import Tokenizer + + +class JiebaBPETokenizer: + """SentencePiece BPE tokenizer with Jieba integration""" + + def __init__(self, tokenizer_json_file): + self.name = 'Jieba BPE Tokenizer' + + self.tokenizer = Tokenizer.from_file(tokenizer_json_file) + self.eod_id = self.tokenizer.token_to_id('<|endoftext|>') + try: + import jieba + import logging + jieba.setLogLevel(logging.INFO) + except ImportError: + raise ImportError( + 'You need to install jieba to use JiebaTokenizer. ' + 'See https://pypi.org/project/jieba/ for installation.') + self.jieba = jieba + self.new_line = self.vocab['\n'] + self.sep_token = self.vocab[''] + + @property + def vocab_size(self): + return self.tokenizer.get_vocab_size(with_added_tokens=True) + + @property + def vocab(self): + return self.tokenizer.get_vocab(with_added_tokens=True) + + @property + def inv_vocab(self): + vocab = self.vocab + inv_vocab = dict() + for key, val in vocab.items(): + inv_vocab[val] = key + return inv_vocab + + def tokenize(self, text: str, is_code: bool = False) -> List[int]: + """ + """ + if not is_code: + seg_list = [x for x in self.jieba.cut(text)] + return self.tokenizer.encode( + seg_list, is_pretokenized=True, add_special_tokens=True).ids + else: + return self.tokenizer.encode( + text, is_pretokenized=False, add_special_tokens=True).ids + + def detokenize(self, token_ids: List[int], early_stop: bool = True) -> str: + if early_stop and self.sep_token in token_ids: + token_ids = token_ids[:token_ids.index(self.sep_token)] + text = self.tokenizer.decode(token_ids, skip_special_tokens=True) + return text + + @property + def eod(self): + return self.eod_id diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/__init__.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5508b9a35483b299a0f4d87b5fd3d7c39b5c6edc --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import GPTMoEConfig + from .backbone import GPTMoEModel + from .text_generation import GPTMoEForTextGeneration + from .tokenizer import JiebaBPETokenizer + from .distributed_gpt_moe import DistributedGPTMoE +else: + _import_structure = { + 'configuration': ['GPTMoEConfig'], + 'backbone': ['GPTMoEModel'], + 'text_generation': ['GPTMoEForTextGeneration'], + 'tokenizer': ['JiebaBPETokenizer'], + 'distributed_gpt_moe': ['DistributedGPTMoE'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/backbone.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..cea37432b93bba12698a13493a27aa6b319c1b61 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/backbone.py @@ -0,0 +1,355 @@ +# Copyright 2021-2022 The Alibaba PAI Team Authors. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os +from typing import Optional, Union + +import addict +import torch +from torch import nn +from torch.nn import functional as F +from transformers.modeling_utils import PreTrainedModel + +from modelscope.utils.constant import ModelFile +from .configuration import GPTMoEConfig + + +class GPTMoESelfAttention(nn.Module): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__(self, config): + super().__init__() + + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + # Per attention head + self.hidden_size_per_attention_head = \ + self.hidden_size // self.num_attention_heads + + self.query_key_value = nn.Linear(self.hidden_size, + 3 * self.hidden_size) + self.softmax = nn.Softmax(dim=-1) + self.attention_dropout = nn.Dropout( + config.attention_probs_dropout_prob) + + # Output. + self.dense = nn.Linear(self.hidden_size, self.hidden_size) + self.output_dropout = nn.Dropout(config.hidden_dropout_prob) + + def _transpose_for_scores(self, tensor): + """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with + size [b, np, s, hn]. + """ + new_tensor_shape = tensor.size()[:-1] + ( + self.num_attention_heads, self.hidden_size_per_attention_head) + tensor = tensor.view(*new_tensor_shape) + return tensor.permute(0, 2, 1, 3) + + def _split_tensor_along_last_dim(self, + tensor, + num_partitions, + contiguous_split_chunks=False): + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = tensor.size()[last_dim] // num_partitions + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + def forward(self, hidden_states, ltor_mask, is_infer=False): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Attention heads. [b, s, hp] + tgt_len = hidden_states.size(1) + ltor_mask = torch.reshape(ltor_mask, [1, 1, tgt_len, tgt_len]) + mixed_x_layer = self.query_key_value(hidden_states) + (mixed_query_layer, mixed_key_layer, mixed_value_layer) = \ + self._split_tensor_along_last_dim(mixed_x_layer, 3) + + # Reshape and transpose [b, np, s, hn] + query_layer = self._transpose_for_scores(mixed_query_layer) + key_layer = self._transpose_for_scores(mixed_key_layer) + value_layer = self._transpose_for_scores(mixed_value_layer) + + previous_type = value_layer.type() + + # Raw attention scores. [b, np, s, s] + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.hidden_size_per_attention_head) + # Apply the left to right attention mask. + if is_infer: + src_len = key_layer.size(2) + ltor_mask = torch.tril( + torch.ones((1, tgt_len, src_len), + device=hidden_states.device)).view( + 1, 1, tgt_len, src_len).type(previous_type) + converted_mask = 10000.0 * (1.0 - ltor_mask) + attention_scores = (torch.mul(attention_scores, ltor_mask) + - converted_mask).type(previous_type) + + # Attention probabilities. [b, np, s, s] + attention_probs = self.softmax(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.attention_dropout(attention_probs) + + # Context layer. + # [b, np, s, hn] + context_layer = torch.matmul(attention_probs, value_layer) + # [b, s, np, hn] + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size, ) + # [b, s, hp] + context_layer = context_layer.view(*new_context_layer_shape) + + # Output. [b, s, h] + output = self.dense(context_layer) + output = self.output_dropout(output) + + return output + + +class GPTMoEMLP(nn.Module): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__(self, config): + super().__init__() + + hidden_size = config.hidden_size + # Project to 4h. + self.dense_h_to_4h = nn.Linear(hidden_size, 4 * hidden_size) + self.activation_func = F.gelu + # Project back to h. + self.dense_4h_to_h = nn.Linear(4 * hidden_size, hidden_size) + + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states): + + # [s, b, 4hp] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output = self.dense_4h_to_h(intermediate_parallel) + output = self.dropout(output) + return output + + +class GPTMoETransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__(self, config): + super().__init__() + + # Layernorm on the input data. + self.input_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + # Self attention. + self.attention = GPTMoESelfAttention(config) + + # Layernorm on the attention output + self.post_attention_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + # MLP + self.mlp = GPTMoEMLP(config) + + def forward(self, hidden_states, ltor_mask): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Layer norm at the begining of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output = self.attention(layernorm_output, ltor_mask) + # Residual connection. + layernorm_input = hidden_states + attention_output + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + # MLP. + mlp_output = self.mlp(layernorm_output) + # Second residual connection. + output = layernorm_input + mlp_output + + return output + + +class GPTMoETransformer(nn.Module): + """Transformer class.""" + + def __init__(self, config): + super().__init__() + + self.input_tensor = None + + # Number of layers. + self.num_layers = config.num_hidden_layers + + self.layers = torch.nn.ModuleList( + [GPTMoETransformerLayer(config) for _ in range(self.num_layers)]) + + # Final layer norm before output. + self.final_layernorm = nn.LayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def forward(self, hidden_states, attention_mask): + # hidden_states: [s, b, h] + + for index in range(self.num_layers): + layer = self._get_layer(index) + hidden_states = layer(hidden_states, attention_mask) + + # Final layer norm. + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class GPTMoETransformerLanguageModel(nn.Module): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, config): + super().__init__() + + # Embeddings. + self.word_embeddings = nn.Embedding(config.vocab_size, + config.hidden_size) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.embedding_dropout = nn.Dropout(config.hidden_dropout_prob) + + # Transformer. + self.transformer = GPTMoETransformer(config) + + def forward(self, input_ids, attention_mask, position_ids): + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + + embeddings = words_embeddings + position_embeddings + transformer_input = self.embedding_dropout(embeddings) + transformer_output = self.transformer(transformer_input, + attention_mask) + + logits = F.linear(transformer_output, self.word_embeddings.weight) + return logits + + +class GPTMoEModel(PreTrainedModel): + + config_class = GPTMoEConfig + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def __init__(self, config): + super().__init__(config) + self.language_model = GPTMoETransformerLanguageModel(config) + + def forward(self, + input_ids, + attention_mask=None, + position_ids=None, + labels=None, + **kwargs): + seq_length = input_ids.size(1) + attention_mask = torch.tril( + torch.ones((1, 1, seq_length, seq_length), + dtype=torch.long, + device=input_ids.device)) + if position_ids is None: + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + + logits = self.language_model(input_ids, attention_mask, position_ids) + loss = None + if labels is not None: + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.config.vocab_size), labels.view(-1)) + return addict.Dict(loss=loss, logits=logits) + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: Optional[Union[str, + os.PathLike]]): + config = cls.config_class.from_pretrained( + pretrained_model_name_or_path) + model = cls(config) + state_dict_file = os.path.join(pretrained_model_name_or_path, + ModelFile.TORCH_MODEL_BIN_FILE) + state_dict = torch.load(state_dict_file) + if 'state_dict' in state_dict: + state_dict = state_dict['state_dict'] + state_dict = { + k.replace('model.language_model', 'language_model'): v + for k, v in state_dict.items() + } + model.load_state_dict(state_dict) + return model + + def prepare_inputs_for_generation(self, input_ids, *args, **kwargs): + return {'input_ids': input_ids} diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/checkpointing.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/checkpointing.py new file mode 100644 index 0000000000000000000000000000000000000000..08e5aaefe2b7d01921dcc9107d7ce87a1ec07891 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/checkpointing.py @@ -0,0 +1,131 @@ +# Copyright 2021-2022 The Alibaba PAI Team Authors. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from megatron_util import mpu +from megatron_util.model import Float16Module +from megatron_util.utils import unwrap_model +from torch.nn.parallel import DistributedDataParallel as torchDDP + +from .configuration import logger +from .moe.layer import MoE + + +def get_checkpoint_names(checkpoints_path, + path_load_tag, + num_experts, + tensor_rank=None, + expp_rank=None): + """Determine the directory name for this rank's checkpoint.""" + if tensor_rank is None: + tensor_rank = mpu.get_tensor_model_parallel_rank() + + common_path = os.path.join(checkpoints_path, path_load_tag, + f'mp_rank_{tensor_rank:02d}') + + if num_experts[0] > 0: + model_name = os.path.join(common_path, 'model_rng.pt') + optim_name = os.path.join( + checkpoints_path, path_load_tag, + f'expp_rank_{expp_rank}_mp_rank_{tensor_rank:02d}_optim_states.pt') + else: + model_name = optim_name = os.path.join(common_path, + 'model_optim_rng.pt') + + return model_name, optim_name + + +def _get_expert_ckpt_name(checkpoints_path, layer_id, expert_id): + mp_rank = mpu.get_tensor_model_parallel_rank() + ckpt_name = os.path.join( + os.path.join(checkpoints_path, 'model'), + f'layer_{layer_id}_expert_{expert_id}_mp_rank_{mp_rank:02d}_model_states.pt' + ) + return ckpt_name + + +def _load_base_checkpoint(load_dir, path_load_tag=None, num_experts=None): + """ Load the base state_dict from the given directory + + If rank0 is true, just loads rank 0 checkpoint, ignoring arguments. + """ + largest_group_name = mpu.get_max_expert_size_name() + expp_rank = mpu.get_expert_parallel_rank(largest_group_name) + checkpoint_names = get_checkpoint_names( + load_dir, + path_load_tag=path_load_tag, + num_experts=num_experts, + expp_rank=expp_rank) + model_checkpoint_name, optim_checkpoint_name = checkpoint_names + + logger.info(f'Loading model checkpoint from {model_checkpoint_name}') + model_state_dict = torch.load(model_checkpoint_name, map_location='cpu') + + return model_state_dict + + +def load_checkpoint(model, + load_dir, + num_experts=None, + strict=True, + path_load_tag='model', + load_ds_ckpts=True): + model = unwrap_model(model, (torchDDP, Float16Module)) + + model_state_dict = _load_base_checkpoint( + load_dir, path_load_tag=path_load_tag, num_experts=num_experts) + + assert model_state_dict is not None + + if load_ds_ckpts: + load_moe_checkpoint(model, model_state_dict['module'], load_dir) + else: + load_moe_checkpoint(model, model_state_dict['model'], load_dir) + + if load_ds_ckpts: + model.load_state_dict(model_state_dict['module'], strict=strict) + else: + model.load_state_dict(model_state_dict['model'], strict=strict) + + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + +def load_moe_checkpoint(model, state_dict, load_dir): + moe_layer_id = 0 + for n_module, module in model.named_modules(): + if isinstance(module, MoE): # and torch.distributed.get_rank() == 0: + group_name = module.expert_group_name + num_local_experts = module.num_local_experts + expp_rank = mpu.get_expert_parallel_rank(group_name) + # loop all local_experts + for local_expert_id in range(num_local_experts): + global_expert_id = expp_rank * num_local_experts + local_expert_id + moe_load_path = _get_expert_ckpt_name(load_dir, moe_layer_id, + global_expert_id) + logger.info(f'Loading expert states from {moe_load_path}') + expert_state_dict = torch.load( + moe_load_path, map_location=torch.device('cpu')) + # Updating global -> local expert ids + moe_str_prefix = '.deepspeed_moe.experts.deepspeed_experts.' + for key in list(expert_state_dict.keys()): + local_key = key.replace( + f'{moe_str_prefix}{global_expert_id}', + f'{moe_str_prefix}{local_expert_id}') + expert_state_dict[local_key] = expert_state_dict.pop(key) + state_dict.update(expert_state_dict) + moe_layer_id += 1 diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/configuration.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..7dd43ec5ccfc7dea14a9e77165d071ff5544090b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/configuration.py @@ -0,0 +1,128 @@ +# Copyright 2021-2022 The Alibaba PAI Team Authors. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger() + + +class GPTMoEConfig(PretrainedConfig): + + model_type = 'gpt-moe' + + def __init__( + self, + vocab_size=25600, + hidden_size=768, + ffn_hidden_size=None, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=2048, + type_vocab_size=2, + layernorm_epsilon=1e-12, + bias_gelu_fusion=True, + fp32_residual_connection=False, + sequence_parallel=False, + fp16=False, + bf16=False, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=False, + kv_channels=None, + masked_softmax_fusion=True, + attention_dropout=0.1, + bias_dropout_fusion=True, + apply_residual_connection_post_layernorm=False, + hidden_dropout=0.1, + init_method_std=0.02, + # generate + eod_id=7, + tokens_to_generate=100, + top_k=0, + top_p=0.9, + num_experts=[0], + use_tutel=False, + top_k_linear_strategy='standard', + use_expert_residual_network=False, + load_ds_ckpts=False, + model_dir=None, + **kwargs): + super().__init__(layer_norm_eps=layernorm_epsilon, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = 4 * hidden_size \ + if ffn_hidden_size is None else ffn_hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.layernorm_epsilon = layernorm_epsilon + self.bias_gelu_fusion = bias_gelu_fusion + self.fp32_residual_connection = fp32_residual_connection + self.sequence_parallel = sequence_parallel + self.fp16 = fp16 + self.bf16 = bf16 + assert not (fp16 and bf16) + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + if kv_channels is None: + assert hidden_size % num_attention_heads == 0 + self.kv_channels = hidden_size // num_attention_heads + self.masked_softmax_fusion = masked_softmax_fusion + self.attention_dropout = attention_dropout + self.bias_dropout_fusion = bias_dropout_fusion + self.apply_residual_connection_post_layernorm = \ + apply_residual_connection_post_layernorm + self.hidden_dropout = hidden_dropout + self.init_method_std = init_method_std + self.eod_id = eod_id + self.tokens_to_generate = tokens_to_generate + self.top_k = top_k + self.top_p = top_p + self.num_experts = num_experts + self.use_tutel = use_tutel + self.top_k_linear_strategy = top_k_linear_strategy + self.use_expert_residual_network = use_expert_residual_network + self.load_ds_ckpts = load_ds_ckpts + self.model_dir = model_dir + + if self.num_experts[0] > torch.cuda.device_count(): + self.moe_expert_parallel_size = torch.cuda.device_count() + else: + self.moe_expert_parallel_size = self.num_experts[0] + + TORCH_MAJOR = int(torch.__version__.split('.')[0]) + TORCH_MINOR = int(torch.__version__.split('.')[1]) + self.no_persist_layer_norm = \ + TORCH_MAJOR < 1 or (TORCH_MAJOR == 1 and TORCH_MINOR < 11) + + @property + def params_dtype(self): + if self.fp16: + return torch.half + elif self.bf16: + return torch.bfloat16 + else: + return torch.float diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/distributed_gpt_moe.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/distributed_gpt_moe.py new file mode 100644 index 0000000000000000000000000000000000000000..9c238d6f576780f755e01d08570ca71eea64cda7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/distributed_gpt_moe.py @@ -0,0 +1,1288 @@ +# Copyright 2021-2022 The Alibaba PAI Team Authors. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +from megatron_util import mpu +from megatron_util.global_vars import get_global_memory_buffer +from megatron_util.model import (AttnMaskType, Float16Module, LayerNorm, + bias_gelu_impl) +from megatron_util.model.fused_softmax import FusedScaleMaskSoftmax +from torch import nn +from torch.nn import functional as F +from transformers.modeling_utils import PreTrainedModel + +from modelscope.models import TorchModel +from modelscope.models.nlp.gpt_moe import GPTMoEConfig +from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput +from modelscope.utils.megatron_utils import init_megatron_util +from .checkpointing import load_checkpoint +from .moe.layer import MoE + + +class GPTMoEParallelMLP(nn.Module): + + def __init__(self, + config, + init_method, + output_layer_init_method, + moe=False, + enable_expert_tensor_parallelism=False): + super().__init__() + # Project to 4h. + self.dense_h_to_4h = mpu.ColumnParallelLinear( + config.hidden_size, + config.ffn_hidden_size, + gather_output=False, + init_method=init_method, + skip_bias_add=True, + moe=moe, + enable_expert_tensor_parallelism=enable_expert_tensor_parallelism) + + self.bias_gelu_fusion = config.bias_gelu_fusion + self.activation_func = F.gelu + # Project back to h. + self.dense_4h_to_h = mpu.RowParallelLinear( + config.ffn_hidden_size, + config.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True, + moe=moe, + enable_expert_tensor_parallelism=enable_expert_tensor_parallelism) + + def forward(self, hidden_states): + + # [s, b, 4hp] + intermediate_parallel, bias_parallel = self.dense_h_to_4h( + hidden_states) + + if self.bias_gelu_fusion: + intermediate_parallel = \ + bias_gelu_impl(intermediate_parallel, bias_parallel) + else: + intermediate_parallel = \ + self.activation_func(intermediate_parallel + bias_parallel) + + # [s, b, h] + output, output_bias = self.dense_4h_to_h(intermediate_parallel) + return output, output_bias + + +class GPTMoEEmbedding(nn.Module): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + init_method: weight initialization method + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, config, init_method): + super().__init__() + + self.hidden_size = config.hidden_size + self.init_method = init_method + + # Word embeddings (parallel). + self.word_embeddings = mpu.VocabParallelEmbedding( + config.vocab_size, self.hidden_size, init_method=self.init_method) + self._word_embeddings_key = 'word_embeddings' + + # Position embedding (serial). + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + self.hidden_size) + self._position_embeddings_key = 'position_embeddings' + # Initialize the position embeddings. + self.init_method(self.position_embeddings.weight) + + self.fp32_residual_connection = config.fp32_residual_connection + self.sequence_parallel = config.sequence_parallel + # Embeddings dropout + self.embedding_dropout = nn.Dropout(config.hidden_dropout) + + def zero_parameters(self): + """Zero out all parameters in embedding.""" + self.word_embeddings.weight.data.fill_(0) + self.word_embeddings.weight.shared = True + self.position_embeddings.weight.data.fill_(0) + self.position_embeddings.weight.shared = True + + def forward(self, input_ids, position_ids): + # Embeddings. + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + embeddings = words_embeddings + position_embeddings + + # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. + embeddings = embeddings.transpose(0, 1).contiguous() + + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + + # Dropout. + if self.sequence_parallel: + embeddings = mpu.scatter_to_sequence_parallel_region(embeddings) + with mpu.get_cuda_rng_tracker().fork(): + embeddings = self.embedding_dropout(embeddings) + else: + embeddings = self.embedding_dropout(embeddings) + return embeddings + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + # Word embedding. + if self._word_embeddings_key in state_dict: + state_dict_ = state_dict[self._word_embeddings_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'word_embeddings' in key: + state_dict_[key.split('word_embeddings.')[1]] \ + = state_dict[key] + self.word_embeddings.load_state_dict(state_dict_, strict=strict) + + # Position embedding. + if self._position_embeddings_key in state_dict: + state_dict_ = state_dict[self._position_embeddings_key] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'position_embeddings' in key: + state_dict_[key.split('position_embeddings.')[1]] \ + = state_dict[key] + self.position_embeddings.load_state_dict(state_dict_, strict=strict) + + +class NoopTransformerLayer(nn.Module): + + def __init__(self, layer_number): + super().__init__() + self.layer_number = layer_number + + def forward(self, + hidden_states, + attention_mask, + encoder_output=None, + enc_dec_attn_mask=None, + inference_params=None): + return hidden_states.clone() + + +def attention_mask_func(attention_scores, attention_mask): + attention_scores.masked_fill_(attention_mask, -10000.0) + return attention_scores + + +class GPTMoECoreAttention(nn.Module): + + def __init__(self, + config, + layer_number, + attn_mask_type=AttnMaskType.padding): + super().__init__() + self.fp16 = config.fp16 + self.bf16 = config.bf16 + + self.apply_query_key_layer_scaling = config.apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = config.attention_softmax_in_fp32 + if self.apply_query_key_layer_scaling: + self.attention_softmax_in_fp32 = True + self.layer_number = max(1, layer_number) + self.attn_mask_type = attn_mask_type + self.sequence_parallel = config.sequence_parallel + + projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + world_size = mpu.get_tensor_model_parallel_world_size() + self.hidden_size_per_partition = mpu.divide(projection_size, + world_size) + self.hidden_size_per_attention_head = mpu.divide( + projection_size, config.num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + config.num_attention_heads, world_size) + + coeff = None + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + if self.apply_query_key_layer_scaling: + coeff = self.layer_number + self.norm_factor *= coeff + + self.scale_mask_softmax = FusedScaleMaskSoftmax( + self.fp16, self.bf16, self.attn_mask_type, + config.masked_softmax_fusion, attention_mask_func, + self.attention_softmax_in_fp32, coeff) + + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = nn.Dropout(config.attention_dropout) + + def forward(self, query_layer, key_layer, value_layer, attention_mask): + + # =================================== + # Raw attention scores. [b, np, s, s] + # =================================== + + # [b, np, sq, sk] + output_size = (query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0)) + + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view(output_size[2], + output_size[0] * output_size[1], -1) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], + output_size[0] * output_size[1], -1) + + # preallocting input tensor: [b * np, sq, sk] + matmul_input_buffer = get_global_memory_buffer().get_tensor( + (output_size[0] * output_size[1], output_size[2], output_size[3]), + query_layer.dtype, 'mpu') + + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.baddbmm( + matmul_input_buffer, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / self.norm_factor)) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + # =========================== + # Attention probs and dropout + # =========================== + + # attention scores and attention mask [b, np, sq, sk] + attention_probs = self.scale_mask_softmax(attention_scores, + attention_mask) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + + if not self.sequence_parallel: + with mpu.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + else: + attention_probs = self.attention_dropout(attention_probs) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = (value_layer.size(1), value_layer.size(2), + query_layer.size(0), value_layer.size(3)) + + # change view [sk, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view(output_size[0] * output_size[1], + output_size[2], -1) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size_per_partition,) + context_layer = context_layer.view(*new_context_layer_shape) + + return context_layer + + +class GPTMoEParallelAttention(nn.Module): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__(self, config, init_method, output_layer_init_method, + layer_number): + super().__init__() + self.layer_number = max(1, layer_number) + self.params_dtype = config.params_dtype + + projection_size = config.kv_channels * config.num_attention_heads + + # Per attention head and per partition values. + world_size = mpu.get_tensor_model_parallel_world_size() + self.hidden_size_per_attention_head = mpu.divide( + projection_size, config.num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + config.num_attention_heads, world_size) + + # Strided linear layer. + self.query_key_value = mpu.ColumnParallelLinear( + config.hidden_size, + 3 * projection_size, + gather_output=False, + init_method=init_method) + + self.core_attention = GPTMoECoreAttention(config, self.layer_number) + + # Output. + self.dense = mpu.RowParallelLinear( + projection_size, + config.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True) + + def _allocate_memory(self, inference_max_sequence_len, batch_size): + return torch.empty( + inference_max_sequence_len, + batch_size, + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + dtype=self.params_dtype, + device=torch.cuda.current_device()) + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [sq, b, h] + + # ================================================= + # Pre-allocate memory for key-values for inference. + # ================================================= + if inference_params: + if self.layer_number not in inference_params.key_value_memory_dict: + inf_max_seq_len = inference_params.max_sequence_len + inf_max_batch_size = inference_params.max_batch_size + inference_key_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size) + inference_value_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size) + inference_params.key_value_memory_dict[self.layer_number] = ( + inference_key_memory, inference_value_memory) + else: + inference_key_memory, inference_value_memory = \ + inference_params.key_value_memory_dict[self.layer_number] + + # ===================== + # Query, Key, and Value + # ===================== + # Attention heads [sq, b, h] --> [sq, b, (np * 3 * hn)] + mixed_x_layer, _ = self.query_key_value(hidden_states) + + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn] + new_tensor_shape = mixed_x_layer.size()[:-1] + \ + (self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head) + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) + + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] + (query_layer, key_layer, + value_layer) = mpu.split_tensor_along_last_dim(mixed_x_layer, 3) + + # ================================== + # Adjust key and value for inference + # ================================== + + if inference_params: + batch_start = inference_params.batch_size_offset + batch_end = batch_start + key_layer.size(1) + assert batch_end <= inference_key_memory.size(1) + sequence_start = inference_params.sequence_len_offset + sequence_end = sequence_start + key_layer.size(0) + assert sequence_end <= inference_key_memory.size(0) + # Copy key and values. + inference_key_memory[sequence_start:sequence_end, + batch_start:batch_end, ...] = key_layer + inference_value_memory[sequence_start:sequence_end, + batch_start:batch_end, ...] = value_layer + key_layer = inference_key_memory[:sequence_end, + batch_start:batch_end, ...] + value_layer = inference_value_memory[:sequence_end, + batch_start:batch_end, ...] + + # ================================== + # core attention computation + # ================================== + + context_layer = self.core_attention(query_layer, key_layer, + value_layer, attention_mask) + + # ================= + # Output. [sq, b, h] + # ================= + + output, bias = self.dense(context_layer) + + return output, bias + + +class nullcontext: + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass + + +def bias_dropout_add(x, bias, residual, prob, training): + # type: (Tensor, Tensor, Tensor, float, bool) -> Tensor + out = torch.nn.functional.dropout(x + bias, p=prob, training=training) + out = residual + out + return out + + +def get_bias_dropout_add(training): + + def _bias_dropout_add(x, bias, residual, prob): + return bias_dropout_add(x, bias, residual, prob, training) + + return _bias_dropout_add + + +@torch.jit.script +def bias_dropout_add_fused_train(x: torch.Tensor, bias: torch.Tensor, + residual: torch.Tensor, + prob: float) -> torch.Tensor: + return bias_dropout_add(x, bias, residual, prob, True) + + +@torch.jit.script +def bias_dropout_add_fused_inference(x: torch.Tensor, bias: torch.Tensor, + residual: torch.Tensor, + prob: float) -> torch.Tensor: + return bias_dropout_add(x, bias, residual, prob, False) + + +class GPTMoEParallelTransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__(self, + config, + init_method, + output_layer_init_method, + layer_number, + num_experts=1): + + super().__init__() + self.layer_number = layer_number + + self.apply_residual_connection_post_layernorm \ + = config.apply_residual_connection_post_layernorm + + self.bf16 = config.bf16 + self.fp32_residual_connection = config.fp32_residual_connection + + # Layernorm on the input data. + self.input_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + # Self attention. + self.self_attention = GPTMoEParallelAttention( + config, init_method, output_layer_init_method, layer_number) + self.hidden_dropout = config.hidden_dropout + self.bias_dropout_fusion = config.bias_dropout_fusion + + # Layernorm on the attention output + self.post_attention_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + # MLP + self.num_experts = num_experts + if self.num_experts == 1: + self.mlp = GPTMoEParallelMLP(config, init_method, + output_layer_init_method) + else: + enable_expert_tensor_parallelism = config.enable_expert_tensor_parallelism + self.mlp = MoE( + config.hidden_size, + GPTMoEParallelMLP( + config, + init_method, + output_layer_init_method=output_layer_init_method, + moe=True, + enable_expert_tensor_parallelism= + enable_expert_tensor_parallelism), + num_experts=self.num_experts, + ep_size=config.moe_expert_parallel_size, + k=1, + use_residual=False, + capacity_factor=1.0, + eval_capacity_factor=1.0, + noisy_gate_policy=None, + min_capacity=1, + drop_tokens=True, + use_tutel=config.use_tutel, + top_k_linear_strategy=config.top_k_linear_strategy, + use_expert_residual_network=config.use_expert_residual_network) + + # Set bias+dropout+add fusion grad_enable execution handler. + TORCH_MAJOR = int(torch.__version__.split('.')[0]) + TORCH_MINOR = int(torch.__version__.split('.')[1]) + use_nvfuser = TORCH_MAJOR > 1 or (TORCH_MAJOR == 1 + and TORCH_MINOR >= 10) + self.bias_dropout_add_exec_handler = \ + nullcontext if use_nvfuser else torch.enable_grad + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [s, b, h] + + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output, attention_bias = \ + self.self_attention( + layernorm_output, + attention_mask, + inference_params=inference_params) + # Residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + if self.bias_dropout_fusion: + if self.training: + bias_dropout_add_func = bias_dropout_add_fused_train + else: + bias_dropout_add_func = bias_dropout_add_fused_inference + else: + bias_dropout_add_func = get_bias_dropout_add(self.training) + + with self.bias_dropout_add_exec_handler(): + layernorm_input = bias_dropout_add_func( + attention_output, attention_bias.expand_as(residual), residual, + self.hidden_dropout) + + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + + moe_loss = torch.tensor( + 0.0, device=layernorm_output.device, dtype=layernorm_output.dtype) + mlp_bias = torch.tensor( + 0.0, device=layernorm_output.device, dtype=layernorm_output.dtype) + + # MLP. + if self.num_experts == 1: + mlp_output, mlp_bias = self.mlp(layernorm_output) + else: + mlp_output, moe_loss, _ = self.mlp(layernorm_output) + + # Second residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + with self.bias_dropout_add_exec_handler(): + output = bias_dropout_add_func(mlp_output, + mlp_bias.expand_as(residual), + residual, self.hidden_dropout) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = mpu.make_viewless_tensor( + inp=output, requires_grad=output.requires_grad, keep_graph=True) + + return output, moe_loss + + +class GPTMoEParallelTransformer(nn.Module): + """Transformer class.""" + + def __init__(self, + config, + init_method, + output_layer_init_method, + post_layer_norm=True, + pre_process=True, + post_process=True, + num_experts=[0]): + super().__init__() + + self.bf16 = config.bf16 + self.fp32_residual_connection = config.fp32_residual_connection + self.post_layer_norm = post_layer_norm + self.pre_process = pre_process + self.post_process = post_process + self.input_tensor = None + + self.sequence_parallel = config.sequence_parallel + + # Number of layers. + self.num_layers = config.num_hidden_layers + + # Transformer layers. + def build_layer(layer_number, n_e=1): + return GPTMoEParallelTransformerLayer( + config, + init_method, + output_layer_init_method, + layer_number, + num_experts=n_e) + + offset = 0 + if len(num_experts) == 1 and num_experts[0] > 0: + num_experts = num_experts * (self.num_layers // 2) + + if self.num_layers == 0: + self.num_layers = 1 + self.layers = torch.nn.ModuleList([NoopTransformerLayer(1)]) + else: + if num_experts[0] == 0: + self.layers = torch.nn.ModuleList([ + build_layer(i + 1 + offset) for i in range(self.num_layers) + ]) + + else: + self.layers = [] + # Build the layers + for i in range(self.num_layers): + layer_num = i + 1 + offset + if layer_num % 2 == 0: + n_e = num_experts[(layer_num - 1) // 2] + else: + n_e = 1 + self.layers.append(build_layer(layer_num, n_e)) + self.layers = torch.nn.ModuleList(self.layers) + + if self.post_process and self.post_layer_norm: + # Final layer norm before output. + self.final_layernorm = LayerNorm( + config.hidden_size, + eps=config.layernorm_epsilon, + no_persist_layer_norm=config.no_persist_layer_norm, + sequence_parallel=config.sequence_parallel) + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def forward(self, hidden_states, attention_mask, inference_params=None): + # hidden_states: [s, b, h] + + if not self.pre_process: + # See set_input_tensor() + hidden_states = self.input_tensor + + # Viewless tensor. + # - We only need to create a viewless tensor in the case of micro batch + # size (mbs) == 1, since in this case, 'hidden_states.transpose()' + # above creates a view tensor, and '.contiguous()' is a pass-through. + # For mbs >= 2, '.contiguous()' creates a new tensor, eliminating + # the need to make it viewless. + # + # However, we don't explicitly check mbs == 1 here because + # make_viewless_tensor() has negligible overhead when its input + # is already viewless. + # + # - For the 'else' case above, calling make_viewless_tensor() here is + # likely redundant, since p2p_communication.py (likely originator) + # already creates viewless tensors. That said, make_viewless_tensor() + # is called here to be future-proof and corner-case-proof. + hidden_states = mpu.make_viewless_tensor( + hidden_states, + requires_grad=True, + keep_graph=True, + ) + + if self.sequence_parallel: + rng_context = mpu.get_cuda_rng_tracker().fork() + else: + rng_context = nullcontext() + + with rng_context: + # Forward pass. + moe_losses = [] + for index in range(self.num_layers): + layer = self._get_layer(index) + hidden_states, moe_loss = layer( + hidden_states, + attention_mask, + inference_params=inference_params) + moe_losses.append(moe_loss) + + # Final layer norm. + if self.post_process and self.post_layer_norm: + hidden_states = self.final_layernorm(hidden_states) + return (hidden_states, *moe_losses) + + +class GPTMoETransformerLanguageModel(nn.Module): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__(self, + config, + init_method, + output_layer_init_method, + num_experts=None): + super().__init__() + + self.hidden_size = config.hidden_size + self.init_method = init_method + self.encoder_hidden_state = None + self.num_experts = num_experts + + # Embeddings. + self.embedding = GPTMoEEmbedding(config, self.init_method) + + # Transformer. + self.encoder = GPTMoEParallelTransformer( + config, + self.init_method, + output_layer_init_method, + num_experts=self.num_experts) + + def forward(self, + enc_input_ids, + enc_position_ids, + enc_attn_mask, + inference_params=None, + enc_hidden_states=None): + + # Encoder embedding. + encoder_input = self.embedding(enc_input_ids, enc_position_ids) + + # Run encoder. + if enc_hidden_states is None: + if self.encoder is not None: + encoder_output, *moe_losses = self.encoder( + encoder_input, + enc_attn_mask, + inference_params=inference_params) + else: + encoder_output = self.encoder_hidden_state + else: + encoder_output = enc_hidden_states.to(encoder_input.dtype) + + return (encoder_output, *moe_losses) + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + # Embedding. + + if 'embedding' in state_dict: + state_dict_ = state_dict['embedding'] + else: + # for backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if '_embeddings' in key: + state_dict_[key] = state_dict[key] + self.embedding.load_state_dict(state_dict_, strict=strict) + + # Encoder. + if True: + if 'encoder' in state_dict: + state_dict_ = state_dict['encoder'] + # For backward compatibility. + elif 'transformer' in state_dict: + state_dict_ = state_dict['transformer'] + else: + # For backward compatibility. + state_dict_ = {} + for key in state_dict.keys(): + if 'transformer.' in key: + state_dict_[key.split('transformer.') + [1]] = state_dict[key] + + # For backward compatibility. + state_dict_self_attention = {} + encoder_state_dict_keys = list(self.encoder.state_dict().keys()) + for key in state_dict_.keys(): + if '.attention.' in key and key not in encoder_state_dict_keys: + state_dict_self_attention[key.replace( + '.attention.', '.self_attention.')] = state_dict_[key] + # to load pai bert-1.3B + elif '.self_attention.' in key and key not in encoder_state_dict_keys: + state_dict_self_attention[key.replace( + '.self_attention.', '.attention.')] = state_dict_[key] + else: + state_dict_self_attention[key] = state_dict_[key] + state_dict_ = state_dict_self_attention + + # Gather encoder MoE states + if 'moe_state_dict' in state_dict: + for key in list(state_dict['moe_state_dict'].keys()): + if 'encoder' in key: + key_list = key.split('.') + while key_list[0] != 'encoder': + key_list.pop(0) + key_list.pop(0) + actual_key = '.'.join(key_list) + state_dict_[actual_key] = state_dict[ + 'moe_state_dict'].pop(key) + if len(state_dict['moe_state_dict']) == 0: + del state_dict['moe_state_dict'] + + self.encoder.load_state_dict(state_dict_, strict=strict) + + +def init_method_normal(sigma): + """Init method based on N(0, sigma).""" + + def init_(tensor): + return nn.init.normal_(tensor, mean=0.0, std=sigma) + + return init_ + + +def scaled_init_method_normal(sigma, num_layers): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +class GPTMoEModel(PreTrainedModel): + + config_class = GPTMoEConfig + + def __init__(self, config, parallel_output=False): + super().__init__(config) + + self.parallel_output = parallel_output + self.language_model = GPTMoETransformerLanguageModel( + config, + init_method_normal(config.init_method_std), + scaled_init_method_normal(config.init_method_std, + config.num_hidden_layers), + num_experts=config.num_experts) + + def word_embeddings_weight(self): + return self.language_model.embedding.word_embeddings.weight + + @staticmethod + def build_attention_mask_and_position_ids(tokens): + seq_length = tokens.size(1) + attention_mask = torch.tril( + torch.ones((1, 1, seq_length, seq_length), + dtype=torch.long, + device=tokens.device)) + attention_mask = (attention_mask < 0.5) + + position_ids = torch.arange( + seq_length, dtype=torch.long, device=tokens.device) + position_ids = position_ids.unsqueeze(0).expand_as(tokens) + + return attention_mask, position_ids + + @staticmethod + def post_language_model_processing(input_, labels, word_embeddings_weight, + sequence_parallel): + + # Output. Format [s b h] + # Parallel logits. + input_parallel = input_ + # Matrix multiply. + logits_parallel = mpu.LinearWithGradAccumulationAndAsyncCommunication.apply( + input_parallel, word_embeddings_weight, None, False, False, + sequence_parallel) + + output = logits_parallel + + if labels is None: + # [s b h] => [b s h] + return output.transpose(0, 1).contiguous() + else: + # [b s] => [s b] + labels = labels.transpose(0, 1).contiguous() + loss = mpu.vocab_parallel_cross_entropy(output.float(), labels) + # [s b] => [b, s] + loss = loss.transpose(0, 1).contiguous() + return loss + + def forward(self, + input_ids, + attention_mask=None, + position_ids=None, + inference_params=None, + labels=None, + **kwargs): + if attention_mask is None and position_ids is None: + attention_mask, position_ids = \ + self.build_attention_mask_and_position_ids(input_ids) + + lm_output, *moe_losses = self.language_model( + input_ids, + position_ids, + attention_mask, + inference_params=inference_params) + + lm_output = self.post_language_model_processing( + lm_output, labels, self.word_embeddings_weight(), + self.config.sequence_parallel) + + return (lm_output, *moe_losses) + + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + # Gather MoE states and move under language model + moe_state_dict = {} + for key in list(state_dict.keys()): + if 'expert' in key and 'moe.gate.wg.weight' not in key: + moe_state_dict[key] = state_dict.pop(key) + + if 'language_model' in state_dict: + state_dict = state_dict['language_model'] + if len(moe_state_dict) > 0: + state_dict['moe_state_dict'] = moe_state_dict + self.language_model.load_state_dict(state_dict, strict=strict) + + +def modify_logits_for_top_k_filtering(logits, top_k): + """Set the logits for none top-k values to -inf.""" + + filter_ = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits.masked_fill_(filter_, float('-Inf')) + + +def modify_logits_for_top_p_filtering(logits, top_p): + """Set the logits for none top-p values to -inf.""" + + # First sort and calculate cumulative sum of probabilities. + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Filteration based on the cumulative sum. + filter_ = cumulative_probs > top_p + # This shift by 1 is weird and I cannot justify it. This existed + # in the original implementation: + # https://github.com/ari-holtzman/degen/blob/master/gen.py + # and I guess it is needed so keeping it for now. + filter_[:, 1:] = filter_[:, :-1].clone() + # Make sure we at least have one token to select from. + filter_[..., 0] = 0 + + # Fill in the filtered part + filter_ = filter_.scatter(1, sorted_indices, filter_) + logits.masked_fill_(filter_, float('-Inf')) + + +def sample(logits, top_k=0, top_p=0.0, temperature=1.0, vocab_size=None): + """ Sample and generate a token. + Note: logits has the dimension [b, v] where b is the batch size + and v is the vocabulary size. + If vocab_size is provided, we will make sure the sample that is + generated is in [0, vocab-size). This will avoid out of vocabulary + generations due to padding. + """ + + # Check logits for consistency. + assert logits.ndim == 2, 'expected the logits to be of [b, v] shape.' + assert logits.type() == 'torch.cuda.FloatTensor', \ + 'input logits should be floats.' + + # Greedy is just simple argmax. + if top_k == 1: + assert top_p == 0.0, 'cannot set both greedy and top-p samplings.' + samples = torch.argmax(logits, dim=-1) + + # Top-k or top-p sampling. + else: + # Clone so we do not modify the inputs, + logits = logits.clone() + # Apply temperature in place. + if temperature != 1.0: + logits.div_(temperature) + + if top_k > 1: + assert top_p == 0.0, 'cannot set both top-k and top-p samplings.' + assert top_k <= logits.size(1), 'top-k is larger than logit size.' + if vocab_size: + assert top_k < vocab_size, 'top-k is larger than vocab size.' + modify_logits_for_top_k_filtering(logits, top_k) + + elif top_p > 0.0: + assert top_p <= 1.0, 'top-p should be in (0, 1].' + modify_logits_for_top_p_filtering(logits, top_p) + + # After filtering, we need to recalculate the distribution. + probs = logits.softmax(dim=-1) + samples = torch.multinomial(probs, num_samples=1).view(-1) + + # If vocab size is provided, make sure the samples are in + # in the range [0, vocab-size). + if vocab_size: + samples = torch.clamp(samples, min=0, max=(vocab_size - 1)) + + return samples + + +class InferenceParams: + """Inference parameters that are passed to the main model in order + to efficienly calculate and store the context during inference.""" + + def __init__(self, max_batch_size, max_sequence_len): + """Note that offsets are set to zero and we always set the + flag to allocate memory. After the first call, make sure to + set this flag to False.""" + self.max_sequence_len = max_sequence_len + self.max_batch_size = max_batch_size + self.sequence_len_offset = 0 + self.batch_size_offset = 0 + self.key_value_memory_dict = {} + + def swap_key_value_dict(self, batch_idx): + 'swap between batches' + if len(self.key_value_memory_dict) == 0: + raise ValueError('should not swap when dict in empty') + + for layer_number in self.key_value_memory_dict.keys(): + inference_key_memory, inference_value_memory = self.key_value_memory_dict[ + layer_number] + assert len(batch_idx) == inference_key_memory.shape[ + 1] # make sure batch size is the same + new_inference_key_memory = inference_key_memory[:, batch_idx] + new_inference_value_memory = inference_value_memory[:, batch_idx] + self.key_value_memory_dict[layer_number] = ( + new_inference_key_memory, new_inference_value_memory) + + +class DistributedGPTMoE(TorchModel): + + def __init__(self, + model_dir, + rank, + path_load_tag='model', + *args, + **kwargs): + super().__init__(model_dir, *args, **kwargs) + + init_megatron_util(model_dir=model_dir, rank=rank) + + self.config = GPTMoEConfig.from_pretrained(model_dir) + if self.config.num_experts[0] > 0: + mpu.create_expert_and_data_parallel( + self.config.moe_expert_parallel_size) + + # Build model. + model = GPTMoEModel(self.config) + + for param in model.parameters(): + mpu.set_defaults_if_not_set_tensor_model_parallel_attributes(param) + + # GPU allocation. + model.cuda(torch.cuda.current_device()) + + # Fp16 conversion. + if self.config.fp16 or self.config.bf16: + model = Float16Module(model, self.config) + + self.dist_model = model + if self.config.model_dir is not None: + model_dir = self.config.model_dir + load_checkpoint( + self.dist_model, + model_dir, + num_experts=self.config.num_experts, + path_load_tag=path_load_tag, + load_ds_ckpts=self.config.load_ds_ckpts) + self.inference_params = None + + def train(self, mode: bool = True): + if mode: + self.inference_params = None + return super().train(mode) + + def forward(self, + tokens, + attention_mask=None, + position_ids=None, + labels=None, + prompt_length=None, + is_pair=(False, )): + + outputs, *other_losses = self.dist_model( + tokens, + attention_mask, + position_ids, + inference_params=self.inference_params, + labels=labels) + + if labels is None: + self.inference_params.sequence_len_offset += tokens.size(1) + return TextGenerationModelOutput(logits=outputs) + else: + + moe_losses = [] + for moe_loss in other_losses: + if moe_loss is not None: + moe_losses.append(moe_loss) + moe_loss = sum(moe_losses) * 0.01 + + loss_mask = torch.ones( + tokens.size(), dtype=torch.float, device=tokens.device) + + losses = outputs.float() + loss_mask = loss_mask.view(-1).float() + loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() + + loss = loss + moe_loss + + return TextGenerationModelOutput(loss=loss) + + def generate(self, + tokens, + temperature=1.0, + use_eod_token_for_early_termination=True, + stop_on_double_eol=False, + stop_on_eol=False, + **kwargs): + batch_size = tokens.size(0) + lengths = kwargs.pop( + 'prompt_length', + torch.tensor([tokens.size(1)], device=tokens.device)) + pads = torch.ones( + batch_size, self.config.tokens_to_generate, + device=tokens.device).long() * self.config.eod_id + tokens = torch.cat((tokens, pads), dim=-1) + + min_prompt_length = lengths.min().item() + max_sequence_length = tokens.size(1) + max_sequence_length = min(max_sequence_length, + self.config.max_position_embeddings) + + # If the context is too big, this happens + if min_prompt_length >= max_sequence_length: + raise ValueError('context length + tokens_to_generate too large') + + # Initialize inference parameters. + self.inference_params = InferenceParams(batch_size, + max_sequence_length) + + # Added termination_id to support the case that we want to terminate the + # generation once that id is generated. + termination_id = self.config.eod_id + + # Whether we have reached a termination id. + is_generation_done = torch.zeros( + batch_size, dtype=torch.uint8, device=torch.cuda.current_device()) + + # ============= + # Run infernece + # ============= + + with torch.no_grad(): + attention_mask, position_ids = \ + GPTMoEModel.build_attention_mask_and_position_ids(tokens) + + prev_context_length = 0 + for context_length in range(min_prompt_length, + max_sequence_length): + + # Pick the slice that we need to pass through the network. + tokens2use = tokens[:, prev_context_length:context_length] + positions2use = position_ids[:, prev_context_length: + context_length] + attention_mask2use = attention_mask[ + ..., prev_context_length:context_length, :context_length] + + # logits will be meanigful only in the last pipeline stage. + logits = self(tokens2use, attention_mask2use, + positions2use).logits + + # Sample. + last_token_logits = logits[:, -1, :] + new_sample = sample( + last_token_logits, + top_k=self.config.top_k, + top_p=self.config.top_p, + temperature=temperature, + vocab_size=self.config.vocab_size) + + # If a prompt length is smaller or equal th current context + # length, it means we have started generating tokens + started = lengths <= context_length + # Update the tokens. + tokens[started, context_length] = new_sample[started] + + # Update the context length for the next token generation. + prev_context_length = context_length + + # instead tokenization should be in the inference loop so stop sequences can be used + if stop_on_double_eol: + hit_double_eol = (new_sample + == 628).byte() & started.byte() + hit_two_eols = (new_sample == 198).byte() & ( + tokens[:, context_length - 1] + == 198).byte() & started.byte() + done_token = hit_double_eol | hit_two_eols + elif stop_on_eol: + hit_double_eol = (new_sample + == 628).byte() & started.byte() + hit_eol = (new_sample == 198).byte() & started.byte() + done_token = hit_double_eol | hit_eol + else: + done_token = (new_sample == termination_id).byte() & \ + started.byte() + + is_generation_done = is_generation_done | done_token + done = torch.all(is_generation_done) + + if use_eod_token_for_early_termination and done: + break + + tokens = tokens[:, :(context_length + 1)] + return TokenGeneratorOutput(sequences=tokens) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/__init__.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/experts.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/experts.py new file mode 100644 index 0000000000000000000000000000000000000000..b559b0b9c18e6760cc21ca5c19fe7c63c31a1ff1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/experts.py @@ -0,0 +1,36 @@ +''' +Copyright 2020 The Microsoft DeepSpeed Team +''' + +import copy + +import torch + + +class Experts(torch.nn.Module): + + def __init__(self, expert, num_local_experts=1, expert_group_name=None): + super(Experts, self).__init__() + + self.deepspeed_experts = torch.nn.ModuleList( + [copy.deepcopy(expert) for i in range(num_local_experts)]) + self.num_local_experts = num_local_experts + + # TODO: revisit allreduce for moe.gate... + for expert in self.deepspeed_experts: + # TODO: Create param groups to handle expert + data case (e.g. param.group = moe_group) + for name, param in expert.named_parameters(): + param.allreduce = False + param.group_name = expert_group_name + + def forward(self, inputs): + chunks = inputs.chunk(self.num_local_experts, dim=1) + expert_outputs = [] + for chunk, expert in zip(chunks, self.deepspeed_experts): + out = expert(chunk) + if type(out) is tuple: + out = out[0] # Ignore the bias term for now + expert_outputs += [out] + + expert_output = torch.cat(expert_outputs, dim=1) + return expert_output diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/layer.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/layer.py new file mode 100644 index 0000000000000000000000000000000000000000..10650f00e45ef78402b814a175c9569534f85ed1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/layer.py @@ -0,0 +1,98 @@ +''' +Copyright 2020 The Microsoft DeepSpeed Team +''' + +import typing + +import torch +from megatron_util import mpu + +from .experts import Experts +from .sharded_moe import MOELayer, TopKGate + + +class MoE(torch.nn.Module): + + def __init__(self, + hidden_size, + expert, + num_experts=1, + ep_size=1, + k=1, + capacity_factor=1., + eval_capacity_factor=1., + min_capacity=4, + use_residual=False, + noisy_gate_policy: typing.Optional[str] = None, + drop_tokens: bool = True, + use_rts=True, + use_tutel: bool = False, + top_k_linear_strategy: str = 'normal', + use_expert_residual_network: bool = False): + super(MoE, self).__init__() + self.use_residual = use_residual + assert num_experts % ep_size == 0, f'Number of experts ({num_experts}) should ' \ + f'be divisible by expert parallel size ({ep_size})' + self.ep_size = ep_size + self.expert_group_name = f'ep_size_{self.ep_size}' + self.num_experts = num_experts + self.num_local_experts = num_experts // self.ep_size + + assert noisy_gate_policy is None or noisy_gate_policy in ['None', 'Jitter', 'RSample'], \ + 'Unsupported noisy_gate_policy: ' + noisy_gate_policy + + experts = Experts(expert, self.num_local_experts, + self.expert_group_name) + self.deepspeed_moe = MOELayer( + TopKGate( + hidden_size, + num_experts, + k, + capacity_factor, + eval_capacity_factor, + min_capacity, + noisy_gate_policy, + drop_tokens, + use_rts, + top_k_linear_strategy=top_k_linear_strategy), + experts, + self.expert_group_name, + self.ep_size, + self.num_local_experts, + use_tutel=use_tutel, + use_expert_residual_network=use_expert_residual_network) + + self.deepspeed_moe._set_ep_group( + mpu.get_expert_parallel_group(self.expert_group_name)) + + if self.use_residual: + self.mlp = expert + # coefficient is used for weighted sum of the output of expert and mlp + self.coefficient = torch.nn.Linear(hidden_size, 2) + + def forward(self, hidden_states, used_token=None): + """ MoE forward + + Arguments: + hidden_states (Tensor): input to the layer + used_token (Tensor, optional): default: None, mask only used tokens + + Returns: + A tuple including output, gate loss, and expert count. + + * output (Tensor): output of the model + + * l_aux (Tensor): gate loss value + + * exp_counts (int): expert count + """ + output = self.deepspeed_moe(hidden_states, used_token) + if self.use_residual: + # Residual MoE + output_mlp = self.mlp(hidden_states) + if type(output_mlp) is tuple: + output_mlp = output_mlp[0] # Ignore the bias term for now + coef = self.coefficient(hidden_states) + coef = torch.nn.functional.softmax(coef, dim=1) + output = output * coef[..., 0:1] + output_mlp * coef[..., 1:] + return output, self.deepspeed_moe.l_aux, self.deepspeed_moe.exp_counts diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/mappings.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/mappings.py new file mode 100644 index 0000000000000000000000000000000000000000..011c4c2a0dec6f5bac4fda49825f9ed2bc765e2e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/mappings.py @@ -0,0 +1,87 @@ +''' +Copyright 2020 The Microsoft DeepSpeed Team +''' + +import torch +from megatron_util import mpu + + +def _gather_tokens(input_, dim=0): + """Gather tensors and concatenate them along a dimension""" + + input_ = input_.contiguous() + # Size and dimension. + rank = mpu.get_tensor_model_parallel_rank() + + tensor_list = [ + torch.empty_like(input_) + for _ in range(mpu.get_tensor_model_parallel_world_size()) + ] + tensor_list[rank] = input_ + torch.distributed.all_gather( + tensor_list, input_, group=mpu.get_tensor_model_parallel_group()) + + # Note: torch.cat already creates a contiguous tensor. + output = torch.cat(tensor_list, dim=dim).contiguous() + + return output + + +def _drop_tokens(input_, dim=0): + """Divide a tensor among the tensor parallel ranks""" + total_chunks = mpu.get_tensor_model_parallel_world_size() + this_chunk = mpu.get_tensor_model_parallel_rank() + assert input_.shape[ + dim] % total_chunks == 0, f'input dimension {dim} ({input_.shape[dim]}) ' \ + f'is not divisible by tensor parallel world size ({total_chunks})' + chunk_size = input_.shape[dim] // total_chunks + + return torch.narrow(input_, dim, this_chunk * chunk_size, chunk_size) + + +class _GatherTokens(torch.autograd.Function): + """All gather tokens among the tensor parallel ranks""" + + @staticmethod + def symbolic(graph, input_, dim): + return _gather_tokens(input_, dim) + + @staticmethod + def forward(ctx, input_, dim): + ctx.dim = dim + return _gather_tokens(input_, dim) + + @staticmethod + def backward(ctx, grad_output): + return _drop_tokens(grad_output, ctx.dim), None + + +class _DropTokens(torch.autograd.Function): + 'Divide tokens equally among the tensor parallel ranks' + + @staticmethod + def symbolic(graph, input_, dim): + return _drop_tokens(input_, dim) + + @staticmethod + def forward(ctx, input_, dim): + ctx.dim = dim + return _drop_tokens(input_, dim) + + @staticmethod + def backward(ctx, input_): + return _gather_tokens(input_, ctx.dim), None + + +def gather_tokens(input_, dim=0): + if mpu is None or mpu.get_tensor_model_parallel_world_size() == 1: + # no tensor parallelism for non-experts + return input_ + return _GatherTokens.apply(input_, dim) + + +def drop_tokens(input_, dim=0): + if mpu is None or mpu.get_tensor_model_parallel_world_size() == 1: + # no tensor parallelism for non-experts + return input_ + return _DropTokens.apply(input_, dim) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/sharded_moe.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/sharded_moe.py new file mode 100644 index 0000000000000000000000000000000000000000..f8eed00fd6d561f81865a705554e25018957bd98 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/sharded_moe.py @@ -0,0 +1,655 @@ +''' +Copyright 2021 The Microsoft DeepSpeed Team +''' +# The file has been adapted from two fairscale files: +# (1) https://github.com/facebookresearch/fairscale/blob/master/fairscale/nn/moe/moe_layer.py +# (2) https://github.com/facebookresearch/fairscale/blob/master/fairscale/nn/moe/top2gate.py +# Git commit hash: 34df606902a240567a0d898037ece55c2f1336cf +# We retain the following license from the original files: + +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from megatron_util import mpu +from scipy.special import binom +from torch import Tensor, nn +from torch.nn import Module + +from ..configuration import logger +from .mappings import drop_tokens, gather_tokens + +try: + # To enable Tutel MoE optimizations: + # python3 -m pip install --user --upgrade git+https://github.com/microsoft/tutel@v0.1.x + from tutel import moe as tutel_moe + TUTEL_INSTALLED = True +except ImportError: + # Fail silently so we don't spam logs unnecessarily if user isn't using tutel + TUTEL_INSTALLED = False + +try: + from apex.normalization import FusedLayerNorm as _FusedLayerNorm + + has_fused_layernorm = True + + class FusedLayerNorm(_FusedLayerNorm): + + @torch.jit.unused + def forward(self, x): + if not x.is_cuda: + return super().forward(x) + else: + with torch.cuda.device(x.device): + return super().forward(x) +except ImportError: + has_fused_layernorm = False + +if TYPE_CHECKING: + Base = Module[Tensor] +else: + Base = Module + +uniform_map: Dict[torch.device, Callable] = {} +gumbel_map: Dict[torch.device, Callable] = {} +exp_selection_uniform_map: Dict[torch.device, Callable] = {} + + +def multiplicative_jitter(x, device: torch.device, epsilon=1e-2): + """ + Modified from switch transformer paper. mesh transformers + Multiply values by a random number between 1-epsilon and 1+epsilon. + Makes models more resilient to rounding errors introduced by bfloat16. + This seems particularly important for logits. + Args: + x: a torch.tensor + device: torch.device + epsilon: a floating point value + Returns: + a jittered x. + """ + if epsilon == 0: + return x + uniform = uniform_map.get(device) + if uniform is None: + uniform = torch.distributions.uniform.Uniform( + low=torch.tensor(1.0 - epsilon, device=device), + high=torch.tensor(1.0 + epsilon, + device=device)).rsample # type: ignore + uniform_map[device] = uniform + return x * uniform(x.shape) + + +def gumbel_rsample(shape: Tuple, device: torch.device) -> Tensor: + gumbel = gumbel_map.get(device) + if gumbel is None: + one = torch.tensor(1.0, device=device) + zero = torch.tensor(0.0, device=device) + gumbel = torch.distributions.gumbel.Gumbel(zero, + one).rsample # type: ignore + gumbel_map[device] = gumbel + return gumbel(shape) + + +# Based on https://github.com/pytorch/pytorch/pull/40762 +class _AllToAll(torch.autograd.Function): + + @staticmethod + def forward(ctx: Any, group: dist.ProcessGroup, + input: Tensor) -> Tensor: # type: ignore + ctx.group = group + input = input.contiguous() + output = torch.empty_like(input) + dist.all_to_all_single(output, input, group=group) + return output + + @staticmethod + def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor]: + return (None, _AllToAll.apply(ctx.group, *grad_output)) + + +# einsum rewrites are on par or more performant +# switch can be bubbled up in future +USE_EINSUM = True + + +# einsum dimensions: (g)roup, (s)equence, (e)xpert, (m)odel, (c)apacity +# See https://arxiv.org/pdf/2006.16668.pdf for details. +def einsum(rule, a, b): + if USE_EINSUM: + return torch.einsum(rule, a, b) + elif rule == 's,se->se': + return a.reshape(a.shape[0], -1) * b + elif rule == 'se,sc->sec': + return a.unsqueeze(2) * b.unsqueeze(1) + elif rule == 'se,se->s': + return torch.bmm(a.unsqueeze(1), b.unsqueeze(2)).reshape(-1) + elif rule == 'sec,sm->ecm': + s = a.shape[0] + e = a.shape[1] + c = a.shape[2] + m = b.shape[1] + return torch.matmul(a.reshape(s, -1).t(), b).reshape(e, c, m) + elif rule == 'sec,ecm->sm': + return torch.matmul( + a.reshape(a.shape[0], -1), b.reshape(-1, b.shape[-1])) + elif rule == 'ks,ksm->sm': + k = b.shape[0] + s = b.shape[1] + m = b.shape[2] + # [k, s] -> [s, k] -> [s, 1, k] + a = a.t().unsqueeze(1) + # [k,s,m] -> [k, sm] -> [sm, k] -> [s, m, k] + b = b.reshape(k, -1).t().reshape(s, m, k) + # bmm([s, 1, k], [s, m, k]^t) -> [s, m, 1] + return torch.bmm(a, b.transpose(1, 2)).squeeze(2) + else: + return torch.einsum(rule, a, b) + + +# The following functions are extracted and scripted +# because otherwise during a torch.jit.trace, the non-Tensor +# values used in the calculations get recorded as constants. +# torch.jit.script coerces them into Tensors and preserves +# their dynamic shapes. This enables ONNX export. +# We can't script the entire top1gating function because it +# includes stateful caching logic which is incompatible with ONNX. + + +@torch.jit.script +def _capacity(gates: Tensor, capacity_factor: Tensor, + min_capacity: Tensor) -> Tensor: + # gates has shape of SE + num_tokens = gates.shape[0] + num_experts = gates.shape[1] + # to(torch.int64) works around a bug in torch.onnx.export: + # it should cast k to int64 when converting torch.topk but it doesn't. + capacity = torch.ceil( + (num_tokens / num_experts) * capacity_factor).to(torch.int64) + if capacity < min_capacity: + capacity = min_capacity.to(torch.int64) + return capacity + + +@torch.jit.script +def _top_idx(source, k): + return torch.topk(source, k=k, dim=0)[1] + + +@torch.jit.script +def _one_hot_to_float(x, num_classes): + return F.one_hot(x, num_classes=num_classes).float() + + +def top1gating( + logits: Tensor, + capacity_factor: float, + min_capacity: int, + used_token: Tensor = None, + noisy_gate_policy: Optional[str] = None, + drop_tokens: bool = True, + use_rts: bool = True, + use_tutel: bool = False) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Implements Top1Gating on logits.""" + if noisy_gate_policy == 'RSample': + logits_w_noise = logits + gumbel_rsample( + logits.shape, device=logits.device) + # everything is in fp32 in this function + gates = F.softmax(logits, dim=1) + + capacity = _capacity(gates, torch.tensor(capacity_factor), + torch.tensor(min_capacity)) + + # Create a mask for 1st's expert per token + # noisy gating + indices1_s = torch.argmax( + logits_w_noise if noisy_gate_policy == 'RSample' else gates, dim=1) + num_experts = int(gates.shape[1]) + mask1 = F.one_hot(indices1_s, num_classes=num_experts) + + # mask only used tokens + if used_token is not None: + mask1 = einsum('s,se->se', used_token, mask1) + + # gating decisions + exp_counts = torch.sum(mask1, dim=0).detach().to('cpu') + + # if we don't want to drop any tokens + if not drop_tokens: + new_capacity = torch.max(exp_counts).to(logits.device) + dist.all_reduce( + new_capacity, op=dist.ReduceOp.MAX, group=dist.group.WORLD) + capacity = new_capacity + + # Compute l_aux + alpha = torch.max(gates, dim=1).values.unsqueeze(1) + me = torch.mean(gates, dim=0) + ce = torch.mean(mask1.float(), dim=0) + l_aux = torch.sum(me * ce) * num_experts + + # Random Token Selection + if use_rts: + uniform = exp_selection_uniform_map.get(logits.device) + if uniform is None: + uniform = torch.distributions.uniform.Uniform( + low=torch.tensor(0.0, device=logits.device), + high=torch.tensor(1.0, device=logits.device)).rsample + exp_selection_uniform_map[logits.device] = uniform + + mask1_rand = mask1 * uniform(mask1.shape) + else: + mask1_rand = mask1 + + assert logits.shape[0] >= min_capacity, \ + 'No. of tokens (batch-size) should be greater than min_capacity. ' \ + 'Either set min_capacity to 0 or increase your batch size.' + + top_idx = _top_idx(mask1_rand, capacity) + + new_mask1 = mask1 * torch.zeros_like(mask1).scatter_(0, top_idx, 1) + mask1 = new_mask1 + + if use_tutel: + # Tutel doesn't support index values masked with zero + # so we need to replace masked indices with -1 + indices_mask = mask1.sum(dim=1) * num_experts - 1 + indices1_s = torch.min(indices1_s, indices_mask) + + # Compute locations in capacity buffer + if use_tutel: + locations1 = tutel_moe.fast_cumsum_sub_one(mask1) + else: + locations1 = torch.cumsum(mask1, dim=0) - 1 + + if use_tutel: + gates1_s = (gates * mask1).sum(dim=1) + locations1_s = torch.sum(locations1 * mask1, dim=1) + return l_aux, capacity, num_experts, [ + indices1_s, + ], [ + locations1_s, + ], [ + gates1_s, + ], exp_counts, alpha + + # Store the capacity location for each token + locations1_s = torch.sum(locations1 * mask1, dim=1) + + # Normalize gate probabilities + mask1_float = mask1.float() + gates = gates * mask1_float + + locations1_sc = _one_hot_to_float(locations1_s, capacity) + combine_weights = einsum('se,sc->sec', gates, locations1_sc) + + dispatch_mask = combine_weights.bool() + + return l_aux, combine_weights, dispatch_mask, exp_counts, alpha + + +class TopKGate(Module): + """Gate module which implements Top2Gating as described in Gshard_. + :: + + gate = TopKGate(model_dim, num_experts) + l_aux, combine_weights, dispatch_mask = gate(input) + + .. Gshard_: https://arxiv.org/pdf/2006.16668.pdf + + Args: + model_dim (int): + size of model embedding dimension + num_experts (ints): + number of experts in model + """ + + wg: torch.nn.Linear + + def __init__(self, + model_dim: int, + num_experts: int, + k: int = 1, + capacity_factor: float = 1.0, + eval_capacity_factor: float = 1.0, + min_capacity: int = 8, + noisy_gate_policy: Optional[str] = None, + drop_tokens: bool = True, + use_rts: bool = True, + top_k_linear_strategy: str = 'standard') -> None: + super().__init__() + + # Only top-1 are supported at the moment. + if k != 1: + raise ValueError('Only top-1 gatings are supported.') + if top_k_linear_strategy == 'standard': + self.wg = torch.nn.Linear( + model_dim, num_experts, bias=False).float() + elif top_k_linear_strategy == 'lsoftmax': + self.wg = LSoftmaxLinearLayer( + model_dim, num_experts, margin=1).float() + else: + raise ValueError( + 'Only standard or lsoftmax top-k-linear-strategy are supported.' + ) + + self.k = k + self.capacity_factor = capacity_factor + self.eval_capacity_factor = eval_capacity_factor + self.min_capacity = min_capacity + self.noisy_gate_policy = noisy_gate_policy + self.wall_clock_breakdown = False + self.gate_time = 0.0 + self.drop_tokens = drop_tokens + self.use_rts = use_rts + self.top_k_linear_strategy = top_k_linear_strategy + + def forward( + self, + input: torch.Tensor, + used_token: torch.Tensor = None, + use_tutel: bool = False + ) -> Tuple[Tensor, Tensor, Tensor]: # type: ignore + + if self.wall_clock_breakdown: + self.timers('TopKGate').start() + + if self.top_k_linear_strategy == 'standard': + if self.wg.weight.dtype != torch.float32: + self.wg = self.wg.float() + elif self.top_k_linear_strategy == 'lsoftmax': + if self.wg.weight.weight.dtype != torch.float32: + self.wg.weight = self.wg.weight.float() + + input_fp32 = input.float() + # input jittering + if self.noisy_gate_policy == 'Jitter' and self.training: + input_fp32 = multiplicative_jitter(input_fp32, device=input.device) + + if self.k == 1: + if self.top_k_linear_strategy == 'standard': + logits = self.wg(input_fp32) + elif self.top_k_linear_strategy == 'lsoftmax': + logits = self.wg(input_fp32, input_fp32.device, self.training) + + gate_output = top1gating( + logits, self.capacity_factor if self.training else + self.eval_capacity_factor, self.min_capacity, used_token, + self.noisy_gate_policy if self.training else None, + self.drop_tokens, self.use_rts, use_tutel) + + if self.wall_clock_breakdown: + self.timers('TopKGate').stop() + self.gate_time = self.timers('TopKGate').elapsed( + reset=False) * 1000 + + return gate_output + + +class MOELayer(Base): + """MOELayer module which implements MixtureOfExperts as described in Gshard_. + :: + + gate = TopKGate(model_dim, num_experts) + moe = MOELayer(gate, expert) + output = moe(input) + l_aux = moe.l_aux + + .. Gshard_: https://arxiv.org/pdf/2006.16668.pdf + + Args: + gate (torch.nn.Module): + gate network + expert (torch.nn.Module): + expert network + """ + + def __init__(self, + gate: Module, + experts: Module, + ep_group_name, + ep_size, + num_local_experts: int, + use_tutel: bool = False, + use_expert_residual_network: bool = False) -> None: + super().__init__() + self.gate = gate + self.experts = experts + self.ep_group = None + self.ep_size = ep_size + self.ep_group_name = ep_group_name + self.num_local_experts = num_local_experts + + self.wall_clock_breakdown = False + self.use_expert_residual_network = use_expert_residual_network + + if self.use_expert_residual_network: + self.expert_network = nn.Sequential(*([ + ExpertResidualLayer(self.gate.model_dim) for _ in range(6) + ])) # noqa + + self.use_tutel = use_tutel and TUTEL_INSTALLED + + if self.use_tutel: + logger.info('Using Tutel optimizations.') + elif use_tutel and not TUTEL_INSTALLED: + logger.info( + 'Tutel optimization requested but not installed Proceeding without Tutel.' + ) + + def _set_ep_group(self, ep_group): + self.ep_group = ep_group + + def forward(self, *input: Tensor, **kwargs: Any) -> Tensor: + + if self.wall_clock_breakdown: + self.timers('moe').start() + + # Implement Algorithm 2 from GShard paper. + d_model = input[0].shape[-1] + # Initial implementation -> Reshape into S tokens by dropping sequence dimension. + # Reshape into G groups so that each group can distribute tokens equally + # group_size = kwargs['group_size'] if 'group_size' in kwargs.keys() else 1 + reshaped_input = input[0].reshape(-1, d_model) + + if self.use_tutel: + self.l_aux, C, E, indices_, locations_, gates_, self.exp_counts, alpha = self.gate( + reshaped_input, input[1], True) + _, M = reshaped_input.size(0), reshaped_input.size(1) + + if not hasattr(self, '_tutel_dispatcher'): + self._tutel_dispatcher = tutel_moe.fast_dispatcher( + E, C, M, dispatch_dtype=reshaped_input.dtype) + self._tutel_dispatcher.update( + indices_, locations_, gates_, capacity=C) + dispatched_input = self._tutel_dispatcher.encode(reshaped_input) + else: + self.l_aux, combine_weights, dispatch_mask, self.exp_counts, alpha = self.gate( + reshaped_input, input[1]) + dispatched_input = einsum('sec,sm->ecm', + dispatch_mask.type_as(input[0]), + reshaped_input) + + if self.wall_clock_breakdown: + self.timers('falltoall').start() + + if mpu.get_expert_model_parallel_world_size() == 1: + # If the non-expert is tensor-parallel, it will create + # duplicate tokens on the tensor-parallel ranks. + # Since our experts are not tensor-parallel, these duplicates + # need to be dropped to ensure correctness. + # this also doubles up as a communication optimization as we are + # reducing the all-to-all communication volume. + if self.use_tutel: + # reshape tutel's output from [e*c,m] to [e,c,m] + dispatched_input = dispatched_input.reshape( + self.ep_size * self.num_local_experts, -1, d_model) + dispatched_input = drop_tokens(dispatched_input, dim=1) + + dispatched_input = _AllToAll.apply(self.ep_group, dispatched_input) + + if self.wall_clock_breakdown: + self.timers('falltoall').stop() + self.time_falltoall = self.timers('falltoall').elapsed( + reset=False) * 1000 + + # Re-shape after all-to-all: ecm -> gecm + dispatched_input = dispatched_input.reshape(self.ep_size, + self.num_local_experts, -1, + d_model) + + expert_output = self.experts(dispatched_input) + + if self.wall_clock_breakdown: + self.timers('salltoall').start() + + expert_output = _AllToAll.apply(self.ep_group, expert_output) + + if self.wall_clock_breakdown: + self.timers('salltoall').stop() + self.time_salltoall = self.timers('salltoall').elapsed( + reset=False) * 1000 + + # Re-shape back: gecm -> ecm + expert_output = expert_output.reshape( + self.ep_size * self.num_local_experts, -1, d_model) + + if mpu.get_expert_model_parallel_world_size() == 1: + # the dropped duplicate tokens need to be gathered on each + # tensor parallel rank again for the tensor-parallel + # non-expert of the next layer. + expert_output = gather_tokens(expert_output, dim=1) + + if self.use_tutel: + combined_output = self._tutel_dispatcher.decode( + expert_output.view(E * C, M)) + else: + combined_output = einsum('sec,ecm->sm', + combine_weights.type_as(input[0]), + expert_output) + + if self.use_expert_residual_network: + combined_output = alpha * self.expert_network(combined_output) + ( + 1 - alpha) * combined_output + + a = combined_output.reshape(input[0].shape) + + if self.wall_clock_breakdown: + self.timers('moe').stop() + self.time_moe = self.timers('moe').elapsed(reset=False) * 1000 + + return a + + +class LSoftmaxLinearLayer(torch.nn.Module): + + def __init__(self, input_features, output_features, margin): + super().__init__() + self.input_dim = input_features # number of input feature i.e. output of the last fc layer + self.output_dim = output_features # number of output = class numbers + self.margin = margin # m + self.beta = 100 + self.beta_min = 0 + self.scale = 0.99 + self.num_experts = output_features + # Initialize L-Softmax parameters + self.weight = torch.nn.Linear( + input_features, output_features, bias=False).float() + self.divisor = math.pi / self.margin # pi/m + self.C_m_2n = torch.Tensor(binom(margin, range(0, margin + 1, + 2))) # C_m{2n} + self.cos_powers = torch.Tensor(range(self.margin, -1, -2)) # m - 2n + self.sin2_powers = torch.Tensor(range(len(self.cos_powers))) # n + self.signs = torch.ones(margin // 2 + 1) # 1, -1, 1, -1, ... + self.signs[1::2] = -1 + + def calculate_cos_m_theta(self, cos_theta, device): + sin2_theta = 1 - cos_theta**2 + cos_terms = cos_theta.unsqueeze(1)**self.cos_powers.to( + device).unsqueeze(0) # cos^{m - 2n} + sin2_terms = ( + sin2_theta.unsqueeze(1)**self.sin2_powers.to(device).unsqueeze(0)) + + cos_m_theta = (self.signs.to(device).unsqueeze(0) + * self.C_m_2n.to(device).unsqueeze(0) * cos_terms + * sin2_terms).sum(1) # summation of all terms + + return cos_m_theta + + def reset_parameters(self): + nn.init.kaiming_normal_(self.weight.data.t()) + + def find_k(self, cos): + # to account for acos numerical errors + eps = 1e-7 + cos = torch.clamp(cos, -1 + eps, 1 - eps) + acos = cos.acos() + k = (acos / self.divisor).floor().detach() + return k + + def forward(self, input, device, training): + if training: + x, w = input, self.weight.float() + beta = max(self.beta, self.beta_min) + logit = w(x) + indexes = range(logit.size(0)) + # target = torch.fmod(torch.randperm(logit.size(0)), self.num_experts) + target = torch.fmod( + torch.range(0, + logit.size(0) - 1), self.num_experts).long() + logit_target = logit[indexes, target] + + # cos(theta) = w * x / ||w||*||x|| + w_target_norm = w.weight[:, target].norm(p=2, dim=0) + + x_norm = x.norm(p=2, dim=1) + cos_theta_target = logit_target / (w_target_norm * x_norm + 1e-10) + + # equation 7 + cos_m_theta_target = self.calculate_cos_m_theta( + cos_theta_target, device) + + # find k in equation 6 + k = self.find_k(cos_theta_target) + + # f_y_i + logit_target_updated = w_target_norm * x_norm * (( + (-1)**k * cos_m_theta_target) - 2 * k) + logit_target_updated_beta = (logit_target_updated + beta + * logit[indexes, target]) / (1 + beta) + + logit[indexes, target] = logit_target_updated_beta + self.beta *= self.scale + return logit + else: + return self.weight(input) + + +def LayerNorm(normalized_shape, + eps=1e-5, + elementwise_affine=True, + export=False): + if torch.jit.is_scripting() or torch.jit.is_tracing(): + export = True + if not export and torch.cuda.is_available() and has_fused_layernorm: + return FusedLayerNorm(normalized_shape, eps, elementwise_affine) + return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine) + + +class ExpertResidualLayer(torch.nn.Module): + + def __init__(self, embed_dim): + super().__init__() + self.norm = LayerNorm(embed_dim, export=False) + self.ff1 = torch.nn.Linear(embed_dim, embed_dim * 4) + self.ff2 = torch.nn.Linear(embed_dim * 4, embed_dim) + self.ff2.weight.data.zero_() + + def forward(self, xs): + return xs + self.ff2(torch.nn.functional.relu(self.ff1(self.norm(xs)))) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/utils.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b6d64d5bf9f8eb12c1b643ce1216dae01fc70fe6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/moe/utils.py @@ -0,0 +1,125 @@ +''' +Copyright 2020 The Microsoft DeepSpeed Team +''' + +from typing import Dict, List, Tuple + +import torch + +from .layer import MoE + + +def has_moe_layers(m): + has_moe = False + num_experts = 0 + for _, module in m.named_modules(): + if isinstance(module, MoE): + has_moe = True + num_experts = module.num_experts + break + return has_moe, num_experts + + +def is_moe_param(param: torch.Tensor) -> bool: + if hasattr(param, 'allreduce') and not param.allreduce: + return True + return False + + +def split_params_into_shared_and_expert_params( + params: List[torch.nn.Parameter] +) -> Tuple[torch.nn.Parameter, torch.nn.Parameter]: + shared_params, expert_params = [], [] + for p in params: + if is_moe_param(p): + expert_params.append(p) + else: + shared_params.append(p) + return shared_params, expert_params + + +def split_params_grads_into_shared_and_expert_params( + group: List[torch.nn.Parameter] +) -> Tuple[torch.nn.Parameter, torch.nn.Parameter]: + """Split grad of parameters into grads of non-expert params + and grads of expert params. This is useful while computing + grad-norms for clipping and overflow detection + + group (List[torch.nn.Parameter]): + Args: + The group of parameters to split + + Returns: + Tuple[List[torch.nn.Parameter], List[torch.nn.Parameter]]: + list of gradients for non MoE params, list of gradients of MoE params + """ + expert_grads = [] + shared_grads = [] + for p in group: + if p.grad is not None: + if is_moe_param(p): + expert_grads.append(p.grad.to(p.dtype)) + else: + shared_grads.append(p.grad.to(p.dtype)) + return shared_grads, expert_grads + + +def split_params_into_different_moe_groups_for_optimizer( + param_groups: Tuple[Dict]) -> Tuple[Dict]: + """Split parameters into different MoE groups for optimizer + + Args: + param_groups (Tuple[Dict]): + The list of parameter groups to split + + Returns: + Tuple[Dict]: + list of MoE/non-MoE groups for optimizer + """ + if isinstance(param_groups, tuple): + param_groups = list(param_groups) # Tuple cannot be modified + elif isinstance(param_groups, dict): + param_groups = [param_groups] + elif not isinstance(param_groups, list): + raise ValueError(f'Unknown param group type of {type(param_groups)}') + + # gather all data parallel group names + data_parallel_group_names = set() + for param_group in param_groups: + for param in param_group['params']: + if is_moe_param(param): + data_parallel_group_names.add(param.group_name) + data_parallel_group_names = list(data_parallel_group_names) + group_moe = {} + # Create the param MoE groups, leave param assign to next step + for param_group in param_groups: + group_moe[param_group['name']] = {} + for key in data_parallel_group_names: + group_moe[param_group['name']][key] = {} + group_moe[param_group['name']][key]['name'] = key + group_moe[param_group['name']][key]['moe'] = True + for ori_key in param_group.keys(): + if ori_key != 'name': + if ori_key == 'params': + group_moe[param_group['name']][key][ori_key] = [] + else: + group_moe[param_group['name']][key][ + ori_key] = param_group[ori_key] + # Assign param + for param_group in param_groups: + new_params = [] + for param in param_group['params']: + if is_moe_param(param): + group_moe[param_group['name']][ + param.group_name]['params'].append(param) + # param_group['params'].remove(param) + else: + new_params.append(param) + param_group['params'] = new_params + + # Flatten the moe groups + for k, v in group_moe.items(): + for k1, v1 in v.items(): + param_groups.append(v1) + + return tuple(param_groups) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/text_generation.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..917ac5b21fe65e21dfcb2344f02e4fe3b6a09357 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/text_generation.py @@ -0,0 +1,70 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +from transformers import BertTokenizer + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.nlp.gpt_moe import GPTMoEModel +from modelscope.utils.constant import Tasks + +__all__ = ['GPTMoEForTextGeneration'] + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.gpt_moe) +class GPTMoEForTextGeneration(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the text generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + # Temporarily compatible with DistributedGPT3 and GPT3Model, + # the base/large model based on GPT3Model will be replaced in the future, + # and GPT3Model will be deprecated + if 'model_parallel_size' in kwargs: + from modelscope.models.nlp import DistributedGPTMoE + self.model = DistributedGPTMoE(model_dir, **kwargs) + else: + self.model = GPTMoEModel.from_pretrained(model_dir) + self.tokenizer = BertTokenizer.from_pretrained(model_dir) + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'logits': Tensor([[0.54, 0.32...])]), # logits + } + """ + return self.model(**input) + + def generate(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + if not isinstance(self.model, GPTMoEModel): + return self.model.generate(**input) + assert 'input_ids' in input, "generate function must accept 'input_ids' key" + input_ids = input['input_ids'] + if 'attention_mask' in input: + attention_mask = input['attention_mask'] + input_ids = input_ids[0][attention_mask[0].nonzero()] \ + .squeeze().unsqueeze(0) + # remove sep token at the end of tokenizer output + input_ids = input_ids[:, :-1] + + gen_params = dict() + gen_params['inputs'] = input_ids + gen_params['do_sample'] = input.pop('do_sample', True) + gen_params['max_length'] = input.pop('max_length', 128) + gen_params['top_k'] = input.pop('top_k', 10) + gen_params['top_p'] = input.pop('top_p', None) + sample_output = self.model.generate(**gen_params) + return {'sequences': sample_output[0]} diff --git a/runtime/Lib/site_packages/models/nlp/gpt_moe/tokenizer.py b/runtime/Lib/site_packages/models/nlp/gpt_moe/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..a290b8468c789817692d00e48bf2370036345b55 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_moe/tokenizer.py @@ -0,0 +1,67 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tokenizers import Tokenizer + + +class JiebaBPETokenizer: + """SentencePiece BPE tokenizer with Jieba integration""" + + def __init__(self, tokenizer_json_file): + self.name = 'Jieba BPE Tokenizer' + + self.tokenizer = Tokenizer.from_file(tokenizer_json_file) + self.eod_id = self.tokenizer.token_to_id('<|endoftext|>') + try: + import jieba + except ImportError: + raise ImportError( + 'You need to install rjieba to use JiebaTokenizer. ' + 'See https://pypi.org/project/rjieba/ for installation.') + self.jieba = jieba + self.new_line = self.vocab['\n'] + self.sep_token = self.vocab[''] + + @property + def vocab_size(self): + return self.tokenizer.get_vocab_size(with_added_tokens=True) + + @property + def vocab(self): + return self.tokenizer.get_vocab(with_added_tokens=True) + + @property + def inv_vocab(self): + vocab = self.vocab + inv_vocab = dict() + for key, val in vocab.items(): + inv_vocab[val] = key + return inv_vocab + + def tokenize(self, text, is_code=False): + if not is_code: + seg_list = [x for x in self.jieba.cut(text)] + return self.tokenizer.encode( + seg_list, is_pretokenized=True, add_special_tokens=True).ids + else: + return self.tokenizer.encode( + text, is_pretokenized=False, add_special_tokens=True).ids + + def detokenize(self, token_ids): + text = self.tokenizer.decode(token_ids, skip_special_tokens=False) + return text + + @property + def eod(self): + return self.eod_id diff --git a/runtime/Lib/site_packages/models/nlp/gpt_neo/__init__.py b/runtime/Lib/site_packages/models/nlp/gpt_neo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ef5fdee5112f963ee1980f27095d30db2e073f62 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_neo/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import GPTNeoModel +else: + _import_structure = { + 'backbone': ['GPTNeoModel'], + } + import sys + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/gpt_neo/backbone.py b/runtime/Lib/site_packages/models/nlp/gpt_neo/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..a809bcde1a3cb47eca2830f0d141c47c3d69ec79 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/gpt_neo/backbone.py @@ -0,0 +1,16 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from transformers import GPTNeoConfig +from transformers import GPTNeoModel as GPTNeoModelTransform + +from modelscope.metainfo import Models +from modelscope.models.builder import BACKBONES +from modelscope.utils.constant import Tasks + + +@BACKBONES.register_module( + group_key=Tasks.backbone, module_name=Models.gpt_neo) +class GPTNeoModel(GPTNeoModelTransform): + + def __init__(self, **kwargs): + config = GPTNeoConfig(**kwargs) + super().__init__(config) diff --git a/runtime/Lib/site_packages/models/nlp/heads/__init__.py b/runtime/Lib/site_packages/models/nlp/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e0bafd400f8e741f0cc3276b1f7a5753d6de7091 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .text_classification_head import TextClassificationHead + from .torch_pretrain_head import BertMLMHead, RobertaMLMHead +else: + _import_structure = { + 'text_classification_head': ['TextClassificationHead'], + 'torch_pretrain_head': ['BertMLMHead', 'RobertaMLMHead'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/heads/crf_head.py b/runtime/Lib/site_packages/models/nlp/heads/crf_head.py new file mode 100644 index 0000000000000000000000000000000000000000..edccd7dec267e66618dbc1c18aa2f6917eba8e72 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/crf_head.py @@ -0,0 +1,563 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Optional + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import (AttentionTokenClassificationModelOutput, + ModelOutputBase, OutputKeys, + TokenClassificationModelOutput) +from modelscope.utils.constant import Tasks + + +@HEADS.register_module(Tasks.token_classification, module_name=Heads.lstm_crf) +@HEADS.register_module( + Tasks.named_entity_recognition, module_name=Heads.lstm_crf) +@HEADS.register_module(Tasks.word_segmentation, module_name=Heads.lstm_crf) +@HEADS.register_module(Tasks.part_of_speech, module_name=Heads.lstm_crf) +class LSTMCRFHead(TorchHead): + + def __init__(self, hidden_size=100, num_labels=None, **kwargs): + super().__init__(hidden_size=hidden_size, num_labels=num_labels) + assert num_labels is not None + self.ffn = nn.Linear(hidden_size * 2, num_labels) + self.crf = CRF(num_labels, batch_first=True) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + label=None, + label_mask=None, + offset_mapping=None, + **kwargs): + logits = self.ffn(inputs.last_hidden_state) + + return TokenClassificationModelOutput( + loss=None, + logits=logits, + ) + + def decode(self, logits, label_mask): + seq_lens = label_mask.sum(-1).long() + mask = torch.arange( + label_mask.shape[1], + device=seq_lens.device)[None, :] < seq_lens[:, None] + predicts = self.crf.decode(logits, mask).squeeze(0) + return predicts + + +@HEADS.register_module( + Tasks.transformer_crf, module_name=Heads.transformer_crf) +@HEADS.register_module( + Tasks.token_classification, module_name=Heads.transformer_crf) +@HEADS.register_module( + Tasks.named_entity_recognition, module_name=Heads.transformer_crf) +@HEADS.register_module( + Tasks.word_segmentation, module_name=Heads.transformer_crf) +@HEADS.register_module(Tasks.part_of_speech, module_name=Heads.transformer_crf) +class TransformersCRFHead(TorchHead): + + def __init__(self, hidden_size, num_labels, **kwargs): + super().__init__( + hidden_size=hidden_size, num_labels=num_labels, **kwargs) + self.linear = nn.Linear(hidden_size, num_labels) + self.crf = CRF(num_labels, batch_first=True) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + label=None, + label_mask=None, + offset_mapping=None, + **kwargs): + logits = self.linear(inputs.last_hidden_state) + if label_mask is not None: + mask = label_mask + masked_lengths = mask.sum(-1).long() + masked_logits = torch.zeros_like(logits) + for i in range(mask.shape[0]): + masked_logits[ + i, :masked_lengths[i], :] = logits[i].masked_select( + mask[i].unsqueeze(-1)).view(masked_lengths[i], -1) + logits = masked_logits + + return AttentionTokenClassificationModelOutput( + loss=None, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def decode(self, logits, label_mask): + seq_lens = label_mask.sum(-1).long() + mask = torch.arange( + label_mask.shape[1], + device=seq_lens.device)[None, :] < seq_lens[:, None] + predicts = self.crf.decode(logits, mask).squeeze(0) + return predicts + + +class CRF(nn.Module): + """Conditional random field. + This module implements a conditional random field [LMP01]_. The forward computation + of this class computes the log likelihood of the given sequence of tags and + emission score tensor. This class also has `~CRF.decode` method which finds + the best tag sequence given an emission score tensor using `Viterbi algorithm`_. + Args: + num_tags: Number of tags. + batch_first: Whether the first dimension corresponds to the size of a minibatch. + Attributes: + start_transitions (`~torch.nn.Parameter`): Start transition score tensor of size + ``(num_tags,)``. + end_transitions (`~torch.nn.Parameter`): End transition score tensor of size + ``(num_tags,)``. + transitions (`~torch.nn.Parameter`): Transition score tensor of size + ``(num_tags, num_tags)``. + .. [LMP01] Lafferty, J., McCallum, A., Pereira, F. (2001). + "Conditional random fields: Probabilistic models for segmenting and + labeling sequence data". *Proc. 18th International Conf. on Machine + Learning*. Morgan Kaufmann. pp. 282–289. + .. _Viterbi algorithm: https://en.wikipedia.org/wiki/Viterbi_algorithm + + """ + + def __init__(self, num_tags: int, batch_first: bool = False) -> None: + if num_tags <= 0: + raise ValueError(f'invalid number of tags: {num_tags}') + super().__init__() + self.num_tags = num_tags + self.batch_first = batch_first + self.start_transitions = nn.Parameter(torch.empty(num_tags)) + self.end_transitions = nn.Parameter(torch.empty(num_tags)) + self.transitions = nn.Parameter(torch.empty(num_tags, num_tags)) + + self.reset_parameters() + + def reset_parameters(self) -> None: + """Initialize the transition parameters. + The parameters will be initialized randomly from a uniform distribution + between -0.1 and 0.1. + """ + nn.init.uniform_(self.start_transitions, -0.1, 0.1) + nn.init.uniform_(self.end_transitions, -0.1, 0.1) + nn.init.uniform_(self.transitions, -0.1, 0.1) + + def __repr__(self) -> str: + return f'{self.__class__.__name__}(num_tags={self.num_tags})' + + def forward(self, + emissions: torch.Tensor, + tags: torch.LongTensor, + mask: Optional[torch.ByteTensor] = None, + reduction: str = 'mean') -> torch.Tensor: + """Compute the conditional log likelihood of a sequence of tags given emission scores. + Args: + emissions (`~torch.Tensor`): Emission score tensor of size + ``(seq_length, batch_size, num_tags)`` if ``batch_first`` is ``False``, + ``(batch_size, seq_length, num_tags)`` otherwise. + tags (`~torch.LongTensor`): Sequence of tags tensor of size + ``(seq_length, batch_size)`` if ``batch_first`` is ``False``, + ``(batch_size, seq_length)`` otherwise. + mask (`~torch.ByteTensor`): Mask tensor of size ``(seq_length, batch_size)`` + if ``batch_first`` is ``False``, ``(batch_size, seq_length)`` otherwise. + reduction: Specifies the reduction to apply to the output: + ``none|sum|mean|token_mean``. ``none``: no reduction will be applied. + ``sum``: the output will be summed over batches. ``mean``: the output will be + averaged over batches. ``token_mean``: the output will be averaged over tokens. + Returns: + `~torch.Tensor`: The log likelihood. This will have size ``(batch_size,)`` if + reduction is ``none``, ``()`` otherwise. + """ + if reduction not in ('none', 'sum', 'mean', 'token_mean'): + raise ValueError(f'invalid reduction: {reduction}') + if mask is None: + mask = torch.ones_like(tags, dtype=torch.uint8, device=tags.device) + if mask.dtype != torch.uint8: + mask = mask.byte() + self._validate(emissions, tags=tags, mask=mask) + + if self.batch_first: + emissions = emissions.transpose(0, 1) + tags = tags.transpose(0, 1) + mask = mask.transpose(0, 1) + + # shape: (batch_size,) + numerator = self._compute_score(emissions, tags, mask) + # shape: (batch_size,) + denominator = self._compute_normalizer(emissions, mask) + # shape: (batch_size,) + llh = numerator - denominator + + if reduction == 'none': + return llh + if reduction == 'sum': + return llh.sum() + if reduction == 'mean': + return llh.mean() + return llh.sum() / mask.float().sum() + + def decode(self, + emissions: torch.Tensor, + mask: Optional[torch.ByteTensor] = None, + nbest: Optional[int] = None, + pad_tag: Optional[int] = None) -> List[List[List[int]]]: + """Find the most likely tag sequence using Viterbi algorithm. + Args: + emissions (`~torch.Tensor`): Emission score tensor of size + ``(seq_length, batch_size, num_tags)`` if ``batch_first`` is ``False``, + ``(batch_size, seq_length, num_tags)`` otherwise. + mask (`~torch.ByteTensor`): Mask tensor of size ``(seq_length, batch_size)`` + if ``batch_first`` is ``False``, ``(batch_size, seq_length)`` otherwise. + nbest (`int`): Number of most probable paths for each sequence + pad_tag (`int`): Tag at padded positions. Often input varies in length and + the length will be padded to the maximum length in the batch. Tags at + the padded positions will be assigned with a padding tag, i.e. `pad_tag` + Returns: + A PyTorch tensor of the best tag sequence for each batch of shape + (nbest, batch_size, seq_length) + """ + if nbest is None: + nbest = 1 + if mask is None: + mask = torch.ones( + emissions.shape[:2], + dtype=torch.uint8, + device=emissions.device) + if mask.dtype != torch.uint8: + mask = mask.byte() + self._validate(emissions, mask=mask) + + if self.batch_first: + emissions = emissions.transpose(0, 1) + mask = mask.transpose(0, 1) + + if nbest == 1: + return self._viterbi_decode(emissions, mask, pad_tag).unsqueeze(0) + return self._viterbi_decode_nbest(emissions, mask, nbest, pad_tag) + + def _validate(self, + emissions: torch.Tensor, + tags: Optional[torch.LongTensor] = None, + mask: Optional[torch.ByteTensor] = None) -> None: + if emissions.dim() != 3: + raise ValueError( + f'emissions must have dimension of 3, got {emissions.dim()}') + if emissions.size(2) != self.num_tags: + raise ValueError( + f'expected last dimension of emissions is {self.num_tags}, ' + f'got {emissions.size(2)}') + + if tags is not None: + if emissions.shape[:2] != tags.shape: + raise ValueError( + 'the first two dimensions of emissions and tags must match, ' + f'got {tuple(emissions.shape[:2])} and {tuple(tags.shape)}' + ) + + if mask is not None: + if emissions.shape[:2] != mask.shape: + raise ValueError( + 'the first two dimensions of emissions and mask must match, ' + f'got {tuple(emissions.shape[:2])} and {tuple(mask.shape)}' + ) + no_empty_seq = not self.batch_first and mask[0].all() + no_empty_seq_bf = self.batch_first and mask[:, 0].all() + if not no_empty_seq and not no_empty_seq_bf: + raise ValueError('mask of the first timestep must all be on') + + def _compute_score(self, emissions: torch.Tensor, tags: torch.LongTensor, + mask: torch.ByteTensor) -> torch.Tensor: + # emissions: (seq_length, batch_size, num_tags) + # tags: (seq_length, batch_size) + # mask: (seq_length, batch_size) + seq_length, batch_size = tags.shape + mask = mask.float() + + # Start transition score and first emission + # shape: (batch_size,) + score = self.start_transitions[tags[0]] + score += emissions[0, torch.arange(batch_size), tags[0]] + + for i in range(1, seq_length): + # Transition score to next tag, only added if next timestep is valid (mask == 1) + # shape: (batch_size,) + score += self.transitions[tags[i - 1], tags[i]] * mask[i] + + # Emission score for next tag, only added if next timestep is valid (mask == 1) + # shape: (batch_size,) + score += emissions[i, torch.arange(batch_size), tags[i]] * mask[i] + + # End transition score + # shape: (batch_size,) + seq_ends = mask.long().sum(dim=0) - 1 + # shape: (batch_size,) + last_tags = tags[seq_ends, torch.arange(batch_size)] + # shape: (batch_size,) + score += self.end_transitions[last_tags] + + return score + + def _compute_normalizer(self, emissions: torch.Tensor, + mask: torch.ByteTensor) -> torch.Tensor: + # emissions: (seq_length, batch_size, num_tags) + # mask: (seq_length, batch_size) + seq_length = emissions.size(0) + + # Start transition score and first emission; score has size of + # (batch_size, num_tags) where for each batch, the j-th column stores + # the score that the first timestep has tag j + # shape: (batch_size, num_tags) + score = self.start_transitions + emissions[0] + + for i in range(1, seq_length): + # Broadcast score for every possible next tag + # shape: (batch_size, num_tags, 1) + broadcast_score = score.unsqueeze(2) + + # Broadcast emission score for every possible current tag + # shape: (batch_size, 1, num_tags) + broadcast_emissions = emissions[i].unsqueeze(1) + + # Compute the score tensor of size (batch_size, num_tags, num_tags) where + # for each sample, entry at row i and column j stores the sum of scores of all + # possible tag sequences so far that end with transitioning from tag i to tag j + # and emitting + # shape: (batch_size, num_tags, num_tags) + next_score = broadcast_score + self.transitions + broadcast_emissions + + # Sum over all possible current tags, but we're in score space, so a sum + # becomes a log-sum-exp: for each sample, entry i stores the sum of scores of + # all possible tag sequences so far, that end in tag i + # shape: (batch_size, num_tags) + next_score = torch.logsumexp(next_score, dim=1) + + # Set score to the next score if this timestep is valid (mask == 1) + # shape: (batch_size, num_tags) + score = torch.where(mask[i].unsqueeze(1), next_score, score) + + # End transition score + # shape: (batch_size, num_tags) + score += self.end_transitions + + # Sum (log-sum-exp) over all possible tags + # shape: (batch_size,) + return torch.logsumexp(score, dim=1) + + def _viterbi_decode(self, + emissions: torch.FloatTensor, + mask: torch.ByteTensor, + pad_tag: Optional[int] = None) -> List[List[int]]: + # emissions: (seq_length, batch_size, num_tags) + # mask: (seq_length, batch_size) + # return: (batch_size, seq_length) + if pad_tag is None: + pad_tag = 0 + + device = emissions.device + seq_length, batch_size = mask.shape + + # Start transition and first emission + # shape: (batch_size, num_tags) + score = self.start_transitions + emissions[0] + history_idx = torch.zeros((seq_length, batch_size, self.num_tags), + dtype=torch.long, + device=device) + oor_idx = torch.zeros((batch_size, self.num_tags), + dtype=torch.long, + device=device) + oor_tag = torch.full((seq_length, batch_size), + pad_tag, + dtype=torch.long, + device=device) + + # - score is a tensor of size (batch_size, num_tags) where for every batch, + # value at column j stores the score of the best tag sequence so far that ends + # with tag j + # - history_idx saves where the best tags candidate transitioned from; this is used + # when we trace back the best tag sequence + # - oor_idx saves the best tags candidate transitioned from at the positions + # where mask is 0, i.e. out of range (oor) + + # Viterbi algorithm recursive case: we compute the score of the best tag sequence + # for every possible next tag + for i in range(1, seq_length): + # Broadcast viterbi score for every possible next tag + # shape: (batch_size, num_tags, 1) + broadcast_score = score.unsqueeze(2) + + # Broadcast emission score for every possible current tag + # shape: (batch_size, 1, num_tags) + broadcast_emission = emissions[i].unsqueeze(1) + + # Compute the score tensor of size (batch_size, num_tags, num_tags) where + # for each sample, entry at row i and column j stores the score of the best + # tag sequence so far that ends with transitioning from tag i to tag j and emitting + # shape: (batch_size, num_tags, num_tags) + next_score = broadcast_score + self.transitions + broadcast_emission + + # Find the maximum score over all possible current tag + # shape: (batch_size, num_tags) + next_score, indices = next_score.max(dim=1) + + # Set score to the next score if this timestep is valid (mask == 1) + # and save the index that produces the next score + # shape: (batch_size, num_tags) + score = torch.where(mask[i].unsqueeze(-1).bool(), next_score, + score) + indices = torch.where(mask[i].unsqueeze(-1).bool(), indices, + oor_idx) + history_idx[i - 1] = indices + + # End transition score + # shape: (batch_size, num_tags) + end_score = score + self.end_transitions + _, end_tag = end_score.max(dim=1) + + # shape: (batch_size,) + seq_ends = mask.long().sum(dim=0) - 1 + + # insert the best tag at each sequence end (last position with mask == 1) + history_idx = history_idx.transpose(1, 0).contiguous() + history_idx.scatter_( + 1, + seq_ends.view(-1, 1, 1).expand(-1, 1, self.num_tags), + end_tag.view(-1, 1, 1).expand(-1, 1, self.num_tags)) + history_idx = history_idx.transpose(1, 0).contiguous() + + # The most probable path for each sequence + best_tags_arr = torch.zeros((seq_length, batch_size), + dtype=torch.long, + device=device) + best_tags = torch.zeros(batch_size, 1, dtype=torch.long, device=device) + for idx in range(seq_length - 1, -1, -1): + best_tags = torch.gather(history_idx[idx], 1, best_tags) + best_tags_arr[idx] = best_tags.data.view(batch_size) + + return torch.where(mask.bool(), best_tags_arr, oor_tag).transpose(0, 1) + + def _viterbi_decode_nbest( + self, + emissions: torch.FloatTensor, + mask: torch.ByteTensor, + nbest: int, + pad_tag: Optional[int] = None) -> List[List[List[int]]]: + # emissions: (seq_length, batch_size, num_tags) + # mask: (seq_length, batch_size) + # return: (nbest, batch_size, seq_length) + if pad_tag is None: + pad_tag = 0 + + device = emissions.device + seq_length, batch_size = mask.shape + + # Start transition and first emission + # shape: (batch_size, num_tags) + score = self.start_transitions + emissions[0] + history_idx = torch.zeros( + (seq_length, batch_size, self.num_tags, nbest), + dtype=torch.long, + device=device) + oor_idx = torch.zeros((batch_size, self.num_tags, nbest), + dtype=torch.long, + device=device) + oor_tag = torch.full((seq_length, batch_size, nbest), + pad_tag, + dtype=torch.long, + device=device) + + # + score is a tensor of size (batch_size, num_tags) where for every batch, + # value at column j stores the score of the best tag sequence so far that ends + # with tag j + # + history_idx saves where the best tags candidate transitioned from; this is used + # when we trace back the best tag sequence + # - oor_idx saves the best tags candidate transitioned from at the positions + # where mask is 0, i.e. out of range (oor) + + # Viterbi algorithm recursive case: we compute the score of the best tag sequence + # for every possible next tag + for i in range(1, seq_length): + if i == 1: + broadcast_score = score.unsqueeze(-1) + broadcast_emission = emissions[i].unsqueeze(1) + # shape: (batch_size, num_tags, num_tags) + next_score = broadcast_score + self.transitions + broadcast_emission + else: + broadcast_score = score.unsqueeze(-1) + broadcast_emission = emissions[i].unsqueeze(1).unsqueeze(2) + # shape: (batch_size, num_tags, nbest, num_tags) + next_score = broadcast_score + self.transitions.unsqueeze( + 1) + broadcast_emission + + # Find the top `nbest` maximum score over all possible current tag + # shape: (batch_size, nbest, num_tags) + next_score, indices = next_score.view(batch_size, -1, + self.num_tags).topk( + nbest, dim=1) + + if i == 1: + score = score.unsqueeze(-1).expand(-1, -1, nbest) + indices = indices * nbest + + # convert to shape: (batch_size, num_tags, nbest) + next_score = next_score.transpose(2, 1) + indices = indices.transpose(2, 1) + + # Set score to the next score if this timestep is valid (mask == 1) + # and save the index that produces the next score + # shape: (batch_size, num_tags, nbest) + score = torch.where(mask[i].unsqueeze(-1).bool().unsqueeze(-1), + next_score, score) + indices = torch.where(mask[i].unsqueeze(-1).unsqueeze(-1).bool(), + indices, oor_idx) + history_idx[i - 1] = indices + + # End transition score shape: (batch_size, num_tags, nbest) + end_score = score + self.end_transitions.unsqueeze(-1) + _, end_tag = end_score.view(batch_size, -1).topk(nbest, dim=1) + + # shape: (batch_size,) + seq_ends = mask.long().sum(dim=0) - 1 + + # insert the best tag at each sequence end (last position with mask == 1) + history_idx = history_idx.transpose(1, 0).contiguous() + history_idx.scatter_( + 1, + seq_ends.view(-1, 1, 1, 1).expand(-1, 1, self.num_tags, nbest), + end_tag.view(-1, 1, 1, nbest).expand(-1, 1, self.num_tags, nbest)) + history_idx = history_idx.transpose(1, 0).contiguous() + + # The most probable path for each sequence + best_tags_arr = torch.zeros((seq_length, batch_size, nbest), + dtype=torch.long, + device=device) + best_tags = torch.arange(nbest, dtype=torch.long, device=device) \ + .view(1, -1).expand(batch_size, -1) + for idx in range(seq_length - 1, -1, -1): + best_tags = torch.gather(history_idx[idx].view(batch_size, -1), 1, + best_tags) + best_tags_arr[idx] = best_tags.data.view(batch_size, -1) // nbest + + return torch.where(mask.unsqueeze(-1), best_tags_arr, + oor_tag).permute(2, 1, 0) diff --git a/runtime/Lib/site_packages/models/nlp/heads/fill_mask_head.py b/runtime/Lib/site_packages/models/nlp/heads/fill_mask_head.py new file mode 100644 index 0000000000000000000000000000000000000000..216fc7faa2283fee66740370abcbe4ba7cf3ad34 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/fill_mask_head.py @@ -0,0 +1,200 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN, gelu + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import (AttentionFillMaskModelOutput, ModelOutputBase, + OutputKeys) +from modelscope.utils.constant import Tasks + + +@HEADS.register_module(Tasks.fill_mask, module_name=Heads.bert_mlm) +@HEADS.register_module(Tasks.fill_mask, module_name=Heads.fill_mask) +class BertFillMaskHead(TorchHead): + + def __init__(self, + hidden_size=768, + hidden_act='gelu', + layer_norm_eps=1e-12, + vocab_size=30522, + **kwargs): + super().__init__( + hidden_size=hidden_size, + hidden_act=hidden_act, + layer_norm_eps=layer_norm_eps, + vocab_size=vocab_size) + self.cls = BertOnlyMLMHead(self.config) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + logits = self.cls(inputs.last_hidden_state) + loss = None + if labels is not None: + loss = self.compute_loss(logits, labels) + return AttentionFillMaskModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def compute_loss(self, logits: torch.Tensor, labels) -> torch.Tensor: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + logits.view(-1, self.config.vocab_size), labels.view(-1)) + return masked_lm_loss + + +@HEADS.register_module(Tasks.fill_mask, module_name=Heads.xlm_roberta_mlm) +class XlmRobertaMaskHead(TorchHead): + _keys_to_ignore_on_load_missing = [ + r'lm_head.decoder.weight', 'lm_head.decoder.bias' + ] + + def __init__(self, + hidden_size=1024, + hidden_act='gelu', + layer_norm_eps=1e-05, + vocab_size=274701, + **kwargs): + super().__init__( + hidden_size=hidden_size, + hidden_act=hidden_act, + layer_norm_eps=layer_norm_eps, + vocab_size=vocab_size) + self.lm_head = XLMRobertaLMHead(self.config) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + logits = self.lm_head(inputs.last_hidden_state) + loss = None + if labels is not None: + loss = self.compute_loss(logits, labels) + return AttentionFillMaskModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def compute_loss(self, logits: torch.Tensor, labels) -> torch.Tensor: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct( + logits.view(-1, self.config.vocab_size), labels.view(-1)) + return masked_lm_loss + + def get_output_embeddings(self): + return self.lm_head.decoder + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = BertLMPredictionHead(config) + + def forward(self, sequence_output: torch.Tensor) -> torch.Tensor: + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class XLMRobertaLMHead(nn.Module): + """Roberta Head for masked language modeling.""" + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.layer_norm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + self.decoder = nn.Linear(config.hidden_size, config.vocab_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.decoder.bias = self.bias + + def forward(self, features, **kwargs): + x = self.dense(features) + x = gelu(x) + x = self.layer_norm(x) + + # project back to size of vocabulary with bias + x = self.decoder(x) + + return x + + def _tie_weights(self): + # To tie those two weights if they get disconnected (on TPU or when the bias is resized) + # For accelerate compatibility and to not break backward compatibility + if self.decoder.bias.device.type == 'meta': + self.decoder.bias = self.bias + else: + self.bias = self.decoder.bias diff --git a/runtime/Lib/site_packages/models/nlp/heads/infromation_extraction_head.py b/runtime/Lib/site_packages/models/nlp/heads/infromation_extraction_head.py new file mode 100644 index 0000000000000000000000000000000000000000..e3cc58cd48a9d9a08b594dfef743e023998298a8 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/infromation_extraction_head.py @@ -0,0 +1,115 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch +from torch import nn + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import InformationExtractionOutput, ModelOutputBase +from modelscope.utils.constant import Tasks + + +@HEADS.register_module( + Tasks.information_extraction, module_name=Heads.information_extraction) +@HEADS.register_module( + Tasks.relation_extraction, module_name=Heads.information_extraction) +class InformationExtractionHead(TorchHead): + + def __init__(self, hidden_size=768, labels=None, **kwargs): + super().__init__(hidden_size=hidden_size, labels=labels) + assert labels is not None + self.labels = labels + self.s_layer = nn.Linear(hidden_size, 2) # head, tail, bce + self.o_layer = nn.Linear(2 * hidden_size, 2) # head, tail, bce + self.p_layer = nn.Linear(hidden_size, len(self.labels)) # label, ce + self.mha = nn.MultiheadAttention(hidden_size, 4) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + text=None, + offsets=None, + threshold=0.5, + **kwargs) -> InformationExtractionOutput: + + assert text is not None + assert offsets is not None + sequence_output = inputs.last_hidden_state + spos = [] + s_head_logits, s_tail_logits = self.s_layer(sequence_output).split( + 1, dim=-1) # (b, seq_len, 2) + s_head_logits = s_head_logits[0, :, 0].sigmoid() # (seq_len) + s_tail_logits = s_tail_logits[0, :, 0].sigmoid() # (seq_len) + s_masks, subjects = self._get_masks_and_mentions( + text, offsets, s_head_logits, s_tail_logits, None, threshold) + for s_mask, subject in zip(s_masks, subjects): + masked_sequence_output = sequence_output * s_mask.unsqueeze( + 0).unsqueeze(-1) # (b, s, h) + subjected_sequence_output = self.mha( + sequence_output.permute(1, 0, 2), + masked_sequence_output.permute(1, 0, 2), + masked_sequence_output.permute(1, 0, + 2))[0].permute(1, 0, + 2) # (b, s, h) + cat_sequence_output = torch.cat( + (sequence_output, subjected_sequence_output), dim=-1) + o_head_logits, o_tail_logits = self.o_layer( + cat_sequence_output).split( + 1, dim=-1) + o_head_logits = o_head_logits[0, :, 0].sigmoid() # (seq_len) + o_tail_logits = o_tail_logits[0, :, 0].sigmoid() # (seq_len) + so_masks, objects = self._get_masks_and_mentions( + text, offsets, o_head_logits, o_tail_logits, s_mask, threshold) + for so_mask, object in zip(so_masks, objects): + masked_sequence_output = ( + sequence_output * so_mask.unsqueeze(0).unsqueeze(-1)).sum( + 1) # (b, h) + lengths = so_mask.unsqueeze(0).sum(-1, keepdim=True) # (b, 1) + pooled_subject_object = masked_sequence_output / lengths # (b, h) + label = self.p_layer(pooled_subject_object).sigmoid().squeeze( + 0) + for i in range(label.size(-1)): + if label[i] > threshold: + predicate = self.labels[i] + spos.append((subject, predicate, object)) + + return InformationExtractionOutput(spo_list=spos) + + def _get_masks_and_mentions(self, + text, + offsets, + heads, + tails, + init_mask=None, + threshold=0.5): + ''' + text: str + heads: tensor (len(heads)) + tails: tensor (len(tails)) + ''' + seq_len = heads.size(-1) + potential_heads = [] + for i in range(seq_len - 1): + if heads[i] > threshold: + potential_heads.append(i) + potential_heads.append(seq_len - 1) + masks = [] + mentions = [] + for i in range(len(potential_heads) - 1): + head_index = potential_heads[i] + tail_index, max_val = None, 0 + for j in range(head_index, potential_heads[i + 1]): + if tails[j] > max_val and tails[j] > threshold: + tail_index = j + max_val = tails[j] + if tail_index is not None: + mask = torch.zeros_like( + heads) if init_mask is None else init_mask.clone() + mask[head_index:tail_index + 1] = 1 + masks.append(mask) # (seq_len) + char_head = offsets[head_index][0] + char_tail = offsets[tail_index][1] + mention = text[char_head:char_tail] + mentions.append(mention) + return masks, mentions diff --git a/runtime/Lib/site_packages/models/nlp/heads/text_classification_head.py b/runtime/Lib/site_packages/models/nlp/heads/text_classification_head.py new file mode 100644 index 0000000000000000000000000000000000000000..427b7785ce626cc1e805de23d8445370bae68c14 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/text_classification_head.py @@ -0,0 +1,59 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import (AttentionTextClassificationModelOutput, + ModelOutputBase, OutputKeys) +from modelscope.utils.constant import Tasks + + +@HEADS.register_module( + Tasks.text_classification, module_name=Heads.text_classification) +@HEADS.register_module( + Tasks.sentence_similarity, module_name=Heads.text_classification) +@HEADS.register_module(Tasks.nli, module_name=Heads.text_classification) +@HEADS.register_module( + Tasks.sentiment_classification, module_name=Heads.text_classification) +class TextClassificationHead(TorchHead): + + def __init__(self, + hidden_size=768, + classifier_dropout=0.1, + num_labels=None, + **kwargs): + super().__init__( + hidden_size=hidden_size, + classifier_dropout=classifier_dropout, + num_labels=num_labels, + ) + assert num_labels is not None + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(hidden_size, num_labels) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + pooler_output = inputs.pooler_output + pooler_output = self.dropout(pooler_output) + logits = self.classifier(pooler_output) + loss = None + if labels is not None: + loss = self.compute_loss(logits, labels) + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def compute_loss(self, logits: torch.Tensor, labels) -> torch.Tensor: + return F.cross_entropy(logits, labels) diff --git a/runtime/Lib/site_packages/models/nlp/heads/text_generation_head.py b/runtime/Lib/site_packages/models/nlp/heads/text_generation_head.py new file mode 100644 index 0000000000000000000000000000000000000000..169760b638f69f25ea5ffce258c022ad9c3996a7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/text_generation_head.py @@ -0,0 +1,33 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.utils.constant import Tasks + + +@HEADS.register_module( + Tasks.text_generation, module_name=Heads.text_generation) +class TextGenerationHead(TorchHead): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + config = self.config + self.linear = nn.Linear( + config['hidden_size'], config['vocab_size'], bias=False) + + def get_output_embeddings(self): + return self.linear + + def forward(self, inputs=None, **kwargs): + logits = self.linear(inputs) + return logits + + def compute_loss(self, logits: torch.Tensor, + labels) -> Dict[str, torch.Tensor]: + return F.cross_entropy(logits, labels) diff --git a/runtime/Lib/site_packages/models/nlp/heads/text_ranking_head.py b/runtime/Lib/site_packages/models/nlp/heads/text_ranking_head.py new file mode 100644 index 0000000000000000000000000000000000000000..9b52d0cc141387142d3dd421a6810b6f4b02a1af --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/text_ranking_head.py @@ -0,0 +1,60 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +from torch import nn + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import (AttentionTextClassificationModelOutput, + ModelOutputBase, OutputKeys) +from modelscope.utils.constant import Tasks + + +@HEADS.register_module(Tasks.text_ranking, module_name=Heads.text_ranking) +class TextRankingHead(TorchHead): + + def __init__(self, + hidden_size=768, + classifier_dropout=0.1, + num_labels=1, + neg_sample=4, + **kwargs): + super().__init__( + hidden_size=hidden_size, + classifier_dropout=classifier_dropout, + num_labels=num_labels, + neg_sample=neg_sample, + ) + self.neg_sample = neg_sample + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(hidden_size, num_labels) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + pooler_output = inputs.pooler_output + pooler_output = self.dropout(pooler_output) + logits = self.classifier(pooler_output) + loss = None + if self.training: + loss = self.compute_loss(logits) + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions, + ) + + def compute_loss(self, logits: torch.Tensor) -> torch.Tensor: + scores = logits.view(-1, self.neg_sample + 1) + batch_size = scores.size(0) + loss_fct = torch.nn.CrossEntropyLoss() + target_label = torch.zeros( + batch_size, dtype=torch.long, device=scores.device) + loss = loss_fct(scores, target_label) + return loss diff --git a/runtime/Lib/site_packages/models/nlp/heads/token_classification_head.py b/runtime/Lib/site_packages/models/nlp/heads/token_classification_head.py new file mode 100644 index 0000000000000000000000000000000000000000..66f5bcd13ec1b04be84a0dd2787a0ed4ba7b4378 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/token_classification_head.py @@ -0,0 +1,70 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.outputs import (AttentionTokenClassificationModelOutput, + ModelOutputBase, OutputKeys) +from modelscope.utils.constant import Tasks + + +@HEADS.register_module( + Tasks.token_classification, module_name=Heads.token_classification) +@HEADS.register_module( + Tasks.named_entity_recognition, module_name=Heads.token_classification) +@HEADS.register_module( + Tasks.part_of_speech, module_name=Heads.token_classification) +class TokenClassificationHead(TorchHead): + + def __init__(self, + hidden_size=768, + classifier_dropout=0.1, + num_labels=None, + **kwargs): + super().__init__( + num_labels=num_labels, + classifier_dropout=classifier_dropout, + hidden_size=hidden_size, + ) + assert num_labels is not None + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(hidden_size, num_labels) + + def forward(self, + inputs: ModelOutputBase, + attention_mask=None, + labels=None, + **kwargs): + sequence_output = inputs.last_hidden_state + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + loss = None + if labels is not None: + loss = self.compute_loss(logits, attention_mask, labels) + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=inputs.hidden_states, + attentions=inputs.attentions) + + def compute_loss(self, logits: torch.Tensor, attention_mask, + labels) -> torch.Tensor: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + return loss diff --git a/runtime/Lib/site_packages/models/nlp/heads/torch_pretrain_head.py b/runtime/Lib/site_packages/models/nlp/heads/torch_pretrain_head.py new file mode 100644 index 0000000000000000000000000000000000000000..e477533f463eafc5ba7307a46199a6aadaca556f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/heads/torch_pretrain_head.py @@ -0,0 +1,27 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +from transformers.models.bert.modeling_bert import BertOnlyMLMHead +from transformers.models.roberta.modeling_roberta import RobertaLMHead + +from modelscope.metainfo import Heads +from modelscope.models.base import TorchHead +from modelscope.models.builder import HEADS +from modelscope.utils.constant import Tasks + + +# @HEADS.register_module(Tasks.fill_mask, module_name=Heads.bert_mlm) +class BertMLMHead(BertOnlyMLMHead, TorchHead): + + def compute_loss(self, outputs: Dict[str, torch.Tensor], + labels) -> Dict[str, torch.Tensor]: + raise NotImplementedError() + + +@HEADS.register_module(Tasks.fill_mask, module_name=Heads.roberta_mlm) +class RobertaMLMHead(RobertaLMHead, TorchHead): + + def compute_loss(self, outputs: Dict[str, torch.Tensor], + labels) -> Dict[str, torch.Tensor]: + raise NotImplementedError() diff --git a/runtime/Lib/site_packages/models/nlp/hf_transformers/__init__.py b/runtime/Lib/site_packages/models/nlp/hf_transformers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..71acd6939f2f567ab6af7e7e2822303eed0a4e77 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/hf_transformers/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import TransformersModel +else: + _import_structure = { + 'backbone': ['TransformersModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/hf_transformers/backbone.py b/runtime/Lib/site_packages/models/nlp/hf_transformers/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..5b9a396517b617ceed0ac4ab27730e0db08904a0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/hf_transformers/backbone.py @@ -0,0 +1,118 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model. """ + +from transformers import AutoConfig, AutoModel +from transformers.modeling_utils import PreTrainedModel + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order + +logger = get_logger() + + +def _get_model_class(config, model_mapping): + supported_models = model_mapping[type(config)] + if not isinstance(supported_models, (list, tuple)): + return supported_models + + name_to_model = {model.__name__: model for model in supported_models} + architectures = getattr(config, 'architectures', []) + for arch in architectures: + if arch in name_to_model: + return name_to_model[arch] + elif f'TF{arch}' in name_to_model: + return name_to_model[f'TF{arch}'] + elif f'Flax{arch}' in name_to_model: + return name_to_model[f'Flax{arch}'] + + # If not architecture is set in the config or match the supported models, the first element of the tuple is the + # defaults. + return supported_models[0] + + +@MODELS.register_module( + group_key=Tasks.backbone, module_name=Models.transformers) +class TransformersModel(TorchModel, PreTrainedModel): + """The Bert Model transformer outputting raw hidden-states without any + specific head on top. + + This model inherits from [`PreTrainedModel`]. Check the superclass + documentation for the generic methods the library implements for all its + model (such as downloading or saving, resizing the input embeddings, pruning + heads etc.) + + This model is also a PyTorch + [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch + documentation for all matter related to general usage and behavior. + + Parameters: + config ([`BertConfig`]): Model configuration class with all the + parameters of the model. + Initializing with a config file does not load the weights associated + with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a + decoder, in which case a layer of cross-attention is added between the + self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam + Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz + Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the + `is_decoder` argument of the configuration set to `True`. To be used in a + Seq2Seq model, the model needs to initialized with both `is_decoder` + argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` + is then expected as an input to the forward pass. + + + """ + + @classmethod + def _instantiate(cls, model_dir=None, **config): + init_backbone = config.pop('init_backbone', False) + + # return the model with pretrained weights + if init_backbone: + model = AutoModel.from_pretrained(model_dir) + return model + + # return the model only + config, kwargs = AutoConfig.from_pretrained( + model_dir, + return_unused_kwargs=True, + trust_remote_code=False, + **config) + + model_mapping = AutoModel._model_mapping + if type(config) in model_mapping.keys(): + model_class = _get_model_class(config, model_mapping) + model = model_class(config) + model.model_dir = model_dir + return model + + raise ValueError( + f'Unrecognized configuration class {config.__class__} for the AutoModel' + f"Model type should be one of {', '.join(c.__name__ for c in model_mapping.keys())}." + ) diff --git a/runtime/Lib/site_packages/models/nlp/llama/__init__.py b/runtime/Lib/site_packages/models/nlp/llama/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d5b6fd19ec212930d28ec84e9b542d3afb0fa137 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from transformers.models.llama import (LlamaConfig, LlamaTokenizer, + LlamaTokenizerFast) + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import LlamaModel + from .text_generation import LlamaForTextGeneration +else: + _import_structure = { + 'backbone': ['LlamaModel'], + 'text_generation': ['LlamaForTextGeneration'], + } + _extra_objects = { + 'LlamaConfig': LlamaConfig, + 'LlamaTokenizer': LlamaTokenizer, + 'LlamaTokenizerFast': LlamaTokenizerFast, + } + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects=_extra_objects, + ) diff --git a/runtime/Lib/site_packages/models/nlp/llama/backbone.py b/runtime/Lib/site_packages/models/nlp/llama/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..dd22da016aa85f4b22ae9d60841480159b5f1a7c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama/backbone.py @@ -0,0 +1,71 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch LLaMA model.""" +from transformers.models.llama import LlamaConfig +from transformers.models.llama import LlamaModel as LlamaModelHF +from transformers.models.llama import \ + LlamaPreTrainedModel as LlamaPreTrainedModelHF + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class MsModelMixin: + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels not supplied. + If num_labels is not found, the model will use the default setting (2 classes). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + model_dir = kwargs.pop('model_dir', None) + device = kwargs.pop('device', None) + if model_dir is None: + config = LlamaConfig(**kwargs) + model = cls(config) + else: + model = super(MsModelMixin, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model if 'device_map' in kwargs \ + or device is None else model.to(device) + + +class LlamaPreTrainedModel(MsModelMixin, LlamaPreTrainedModelHF, TorchModel): + pass + + +@MODELS.register_module(Tasks.backbone, module_name=Models.llama2) +@MODELS.register_module(Tasks.backbone, module_name=Models.llama) +class LlamaModel(MsModelMixin, LlamaModelHF, TorchModel): + pass diff --git a/runtime/Lib/site_packages/models/nlp/llama/convert_llama_weights_to_hf.py b/runtime/Lib/site_packages/models/nlp/llama/convert_llama_weights_to_hf.py new file mode 100644 index 0000000000000000000000000000000000000000..8c0c8568bf185ffed09bb10f930c573d06100228 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama/convert_llama_weights_to_hf.py @@ -0,0 +1,310 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2022 EleutherAI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import gc +import math +import os +import shutil + +import json +import torch +from transformers.models.llama import LlamaConfig + +from .text_generation import LlamaForTextGeneration + +# This file is mainly copied from the llama code of transformers +INTERMEDIATE_SIZE_MAP = { + '7B': 11008, + '13B': 13824, + '30B': 17920, + '65B': 22016, +} +NUM_SHARDS = { + '7B': 1, + '13B': 2, + '30B': 4, + '65B': 8, +} + + +def compute_intermediate_size(n): + return int(math.ceil(n * 8 / 3) + 255) // 256 * 256 + + +def read_json(path): + with open(path, 'r') as f: + return json.load(f) + + +def write_json(text, path): + with open(path, 'w') as f: + json.dump(text, f) + + +def write_model(model_path, input_base_path, model_size): + os.makedirs(model_path, exist_ok=True) + tmp_model_path = os.path.join(model_path, 'tmp') + os.makedirs(tmp_model_path, exist_ok=True) + + params = read_json(os.path.join(input_base_path, 'params.json')) + num_shards = NUM_SHARDS[model_size] + n_layers = params['n_layers'] + n_heads = params['n_heads'] + n_heads_per_shard = n_heads // num_shards + dim = params['dim'] + dims_per_head = dim // n_heads + base = 10000.0 + inv_freq = 1.0 / ( + base**(torch.arange(0, dims_per_head, 2).float() / dims_per_head)) + + # permute for sliced rotary + def permute(w): + return w.view(n_heads, dim // n_heads // 2, 2, + dim).transpose(1, 2).reshape(dim, dim) + + print(f'Fetching all parameters from the checkpoint at {input_base_path}.') + # Load weights + if model_size == '7B': + # Not shared + # (The sharded implementation would also work, but this is simpler.) + loaded = torch.load( + os.path.join(input_base_path, 'consolidated.00.pth'), + map_location='cpu') + else: + # Sharded + loaded = [ + torch.load( + os.path.join(input_base_path, f'consolidated.{i:02d}.pth'), + map_location='cpu') for i in range(num_shards) + ] + param_count = 0 + index_dict = {'weight_map': {}} + for layer_i in range(n_layers): + filename = f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin' + if model_size == '7B': + # Unsharded + state_dict = { + f'model.layers.{layer_i}.self_attn.q_proj.weight': + permute(loaded[f'layers.{layer_i}.attention.wq.weight']), + f'model.layers.{layer_i}.self_attn.k_proj.weight': + permute(loaded[f'layers.{layer_i}.attention.wk.weight']), + f'model.layers.{layer_i}.self_attn.v_proj.weight': + loaded[f'layers.{layer_i}.attention.wv.weight'], + f'model.layers.{layer_i}.self_attn.o_proj.weight': + loaded[f'layers.{layer_i}.attention.wo.weight'], + f'model.layers.{layer_i}.mlp.gate_proj.weight': + loaded[f'layers.{layer_i}.feed_forward.w1.weight'], + f'model.layers.{layer_i}.mlp.down_proj.weight': + loaded[f'layers.{layer_i}.feed_forward.w2.weight'], + f'model.layers.{layer_i}.mlp.up_proj.weight': + loaded[f'layers.{layer_i}.feed_forward.w3.weight'], + f'model.layers.{layer_i}.input_layernorm.weight': + loaded[f'layers.{layer_i}.attention_norm.weight'], + f'model.layers.{layer_i}.post_attention_layernorm.weight': + loaded[f'layers.{layer_i}.ffn_norm.weight'], + } + else: + # Sharded + # Note that in the 13B checkpoint, not cloning the two following weights will result in the checkpoint + # becoming 37GB instead of 26GB for some reason. + state_dict = { + f'model.layers.{layer_i}.input_layernorm.weight': + loaded[0][f'layers.{layer_i}.attention_norm.weight'].clone(), + f'model.layers.{layer_i}.post_attention_layernorm.weight': + loaded[0][f'layers.{layer_i}.ffn_norm.weight'].clone(), + } + state_dict[ + f'model.layers.{layer_i}.self_attn.q_proj.weight'] = permute( + torch.cat( + [ + loaded[i] + [f'layers.{layer_i}.attention.wq.weight'].view( + n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim)) + state_dict[ + f'model.layers.{layer_i}.self_attn.k_proj.weight'] = permute( + torch.cat( + [ + loaded[i] + [f'layers.{layer_i}.attention.wk.weight'].view( + n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim)) + state_dict[ + f'model.layers.{layer_i}.self_attn.v_proj.weight'] = torch.cat( + [ + loaded[i] + [f'layers.{layer_i}.attention.wv.weight'].view( + n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim) # noqa + + state_dict[ + f'model.layers.{layer_i}.self_attn.o_proj.weight'] = torch.cat( + [ + loaded[i][f'layers.{layer_i}.attention.wo.weight'] + for i in range(num_shards) + ], + dim=1) + state_dict[ + f'model.layers.{layer_i}.mlp.gate_proj.weight'] = torch.cat( + [ + loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] + for i in range(num_shards) + ], + dim=0) + state_dict[ + f'model.layers.{layer_i}.mlp.down_proj.weight'] = torch.cat( + [ + loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] + for i in range(num_shards) + ], + dim=1) + state_dict[ + f'model.layers.{layer_i}.mlp.up_proj.weight'] = torch.cat( + [ + loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] + for i in range(num_shards) + ], + dim=0) + + state_dict[ + f'model.layers.{layer_i}.self_attn.rotary_emb.inv_freq'] = inv_freq + for k, v in state_dict.items(): + index_dict['weight_map'][k] = filename + param_count += v.numel() + torch.save(state_dict, os.path.join(tmp_model_path, filename)) + + filename = f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin' + if model_size == '7B': + # Unsharded + state_dict = { + 'model.embed_tokens.weight': loaded['tok_embeddings.weight'], + 'model.norm.weight': loaded['norm.weight'], + 'lm_head.weight': loaded['output.weight'], + } + else: + state_dict = { + 'model.norm.weight': + loaded[0]['norm.weight'], + 'model.embed_tokens.weight': + torch.cat([ + loaded[i]['tok_embeddings.weight'] for i in range(num_shards) + ], + dim=1), # noqa + 'lm_head.weight': + torch.cat([loaded[i]['output.weight'] for i in range(num_shards)], + dim=0), + } + + for k, v in state_dict.items(): + index_dict['weight_map'][k] = filename + param_count += v.numel() + torch.save(state_dict, os.path.join(tmp_model_path, filename)) + + # Write configs + index_dict['metadata'] = {'total_size': param_count * 2} + write_json(index_dict, + os.path.join(tmp_model_path, 'pytorch_model.bin.index.json')) + + config = LlamaConfig( + hidden_size=dim, + intermediate_size=compute_intermediate_size(dim), + num_attention_heads=params['n_heads'], + num_hidden_layers=params['n_layers'], + rms_norm_eps=params['norm_eps'], + ) + config.save_pretrained(tmp_model_path) + + # Make space so we can load the model properly now. + del state_dict + del loaded + gc.collect() + + print('Loading the checkpoint in a Llama model.') + model = LlamaForTextGeneration.from_pretrained( + tmp_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + # Avoid saving this as part of the config. + del model.config._name_or_path + + print('Saving in the Transformers format.') + model.save_pretrained(model_path) + shutil.rmtree(tmp_model_path) + + +def write_tokenizer(tokenizer_path, input_tokenizer_path): + print(f'Fetching the tokenizer from {input_tokenizer_path}.') + os.makedirs(tokenizer_path, exist_ok=True) + write_json({}, os.path.join(tokenizer_path, 'special_tokens_map.json')) + write_json( + { + 'bos_token': '', + 'eos_token': '', + 'model_max_length': int(1e30), + 'tokenizer_class': 'LlamaTokenizer', + 'unk_token': '', + }, + os.path.join(tokenizer_path, 'tokenizer_config.json'), + ) + shutil.copyfile(input_tokenizer_path, + os.path.join(tokenizer_path, 'tokenizer.model')) + + +def main(): + """ + Sample usage: + + ``` + python src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path + ``` + """ + + parser = argparse.ArgumentParser() + parser.add_argument( + '--input_dir', + help= + 'Location of LLaMA weights, which contains tokenizer.model and model folders', + ) + parser.add_argument( + '--model_size', + choices=['7B', '13B', '30B', '65B', 'tokenizer_only'], + ) + parser.add_argument( + '--output_dir', + help='Location to write HF model and tokenizer', + ) + args = parser.parse_args() + if args.model_size != 'tokenizer_only': + write_model( + model_path=args.output_dir, + input_base_path=os.path.join(args.input_dir, args.model_size), + model_size=args.model_size, + ) + write_tokenizer( + tokenizer_path=args.output_dir, + input_tokenizer_path=os.path.join(args.input_dir, 'tokenizer.model'), + ) + + +if __name__ == '__main__': + main() diff --git a/runtime/Lib/site_packages/models/nlp/llama/text_generation.py b/runtime/Lib/site_packages/models/nlp/llama/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..45b9d5f0b6d1386680bc0fdc807bc9376ec2064c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama/text_generation.py @@ -0,0 +1,121 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Optional, Tuple, Union + +import torch +from transformers.models.llama import LlamaForCausalLM + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import Tasks +from .backbone import MsModelMixin + + +def get_chat_prompt(system: str, text: str, history: List[Tuple[str, str]], + max_length: int, tokenizer): + system_prompt = f'[INST] <>\n{system}\n<>\n\n' + system_ids = tokenizer( + system_prompt, add_special_tokens=False, return_tensors='pt').input_ids + + text_prompt = f'{text.strip()} [/INST]' + text_ids = tokenizer( + text_prompt, add_special_tokens=False, return_tensors='pt').input_ids + + prompt_length = system_ids.shape[-1] + text_ids.shape[-1] + if prompt_length > max_length: + raise RuntimeError( + f'prepend prompt length {prompt_length} is bigger than max_length {max_length}' + ) + + history_prompt = '' + history_ids_list = [] + # traverse history in reverse order + for user, bot in history[::-1]: + assert isinstance(user, str) + assert isinstance(bot, str) + round_prompt = f'{user.strip()} [/INST] {bot.strip()} [INST] ' + round_ids = tokenizer( + round_prompt, add_special_tokens=False, + return_tensors='pt').input_ids + if prompt_length + round_ids.shape[-1] > max_length: + # excess history should not be appended to the prompt + break + else: + history_prompt = round_prompt + history_prompt + history_ids_list = [round_ids] + history_ids_list + prompt_length += round_ids.shape[-1] + + prompt_list = [system_prompt, history_prompt, text_prompt] + prompt_ids_list = [system_ids] + history_ids_list + [text_ids] + + return ''.join(prompt_list), torch.cat(prompt_ids_list, dim=1) + + +# This file is mainly copied from the llama code of transformers +@MODELS.register_module(Tasks.chat, module_name=Models.llama2) +@MODELS.register_module(Tasks.chat, module_name=Models.llama) +@MODELS.register_module(Tasks.text_generation, module_name=Models.llama2) +@MODELS.register_module(Tasks.text_generation, module_name=Models.llama) +class LlamaForTextGeneration(MsModelMixin, LlamaForCausalLM, TorchModel): + + def chat(self, input: Dict, tokenizer) -> Dict: + import copy + gen_kwargs = copy.copy(input) + if 'text' not in input: + text: str = '' + else: + text: str = input['text'] + gen_kwargs.pop('text') + + if 'system' not in input: + system: str = '' + else: + system: str = input['system'] + gen_kwargs.pop('system') + + if 'history' not in input: + history = [] + else: + history: List[Tuple] = copy.copy(input['history']) + gen_kwargs.pop('history') + + if 'max_length' not in gen_kwargs: + gen_kwargs['max_length'] = 4096 + + prompt, prompt_ids = get_chat_prompt( + system=system, + text=text, + history=history, + max_length=gen_kwargs['max_length'], + tokenizer=tokenizer) + input_ids = prompt_ids.to(self.device) + generate_ids = self.generate(input_ids, **gen_kwargs) + # remove input tokens + generate_ids = generate_ids[:, input_ids.shape[1]:] + response = tokenizer.batch_decode( + generate_ids, + skip_special_tokens=True, + clean_up_tokenization_spaces=False)[0] + response = response.strip() + history.append((text, response)) + + return {OutputKeys.RESPONSE: response, OutputKeys.HISTORY: history} diff --git a/runtime/Lib/site_packages/models/nlp/llama2/__init__.py b/runtime/Lib/site_packages/models/nlp/llama2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..82bed042dd6c40af77c0de00aa1e117a43c593a7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama2/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.models.nlp.llama import LlamaConfig as Llama2Config +from modelscope.models.nlp.llama import LlamaTokenizer as Llama2Tokenizer +from modelscope.models.nlp.llama import \ + LlamaTokenizerFast as Llama2TokenizerFast +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import Llama2Model + from .text_generation import Llama2ForTextGeneration +else: + _import_structure = { + 'backbone': ['Llama2Model'], + 'text_generation': ['Llama2ForTextGeneration'], + } + _extra_objects = { + 'Llama2Config': Llama2Config, + 'Llama2Tokenizer': Llama2Tokenizer, + 'Llama2TokenizerFast': Llama2TokenizerFast, + } + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects=_extra_objects, + ) diff --git a/runtime/Lib/site_packages/models/nlp/llama2/backbone.py b/runtime/Lib/site_packages/models/nlp/llama2/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..3627efef7912b972a4073f69e41d5daf9c3616b1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama2/backbone.py @@ -0,0 +1,4 @@ +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.llama import LlamaModel as LlamaModel2 +from modelscope.utils.constant import Tasks diff --git a/runtime/Lib/site_packages/models/nlp/llama2/text_generation.py b/runtime/Lib/site_packages/models/nlp/llama2/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..4d77d3f19ad078b2e5aa8797dd5963b38793fc47 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/llama2/text_generation.py @@ -0,0 +1,5 @@ +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.llama import \ + LlamaForTextGeneration as Llama2ForTextGeneration +from modelscope.utils.constant import Tasks diff --git a/runtime/Lib/site_packages/models/nlp/lstm/__init__.py b/runtime/Lib/site_packages/models/nlp/lstm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..61e94ee547aad1a22b3d18b7ae9958def196fb4a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/lstm/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import LSTMModel + from .token_classification import LSTMForTokenClassificationWithCRF +else: + _import_structure = { + 'backbone': ['LSTM'], + 'token_classification': ['LSTMForTokenClassificationWithCRF'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/lstm/backbone.py b/runtime/Lib/site_packages/models/nlp/lstm/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..137bb921cbe79373b22dfadf1e3f208e23214908 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/lstm/backbone.py @@ -0,0 +1,41 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""PyTorch LSTM model. """ + +import torch.nn as nn + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import BackboneModelOutput +from modelscope.utils.constant import Tasks + + +@MODELS.register_module(group_key=Tasks.backbone, module_name=Models.lstm) +class LSTMModel(TorchModel): + + def __init__(self, vocab_size, embed_width, hidden_size=100, **kwargs): + super().__init__() + hidden_size = kwargs.get('lstm_hidden_size', hidden_size) + self.embedding = Embedding(vocab_size, embed_width) + self.lstm = nn.LSTM( + embed_width, + hidden_size, + num_layers=1, + bidirectional=True, + batch_first=True) + + def forward(self, input_ids, **kwargs) -> BackboneModelOutput: + embedding = self.embedding(input_ids) + lstm_output, _ = self.lstm(embedding) + return BackboneModelOutput(last_hidden_state=lstm_output) + + +class Embedding(nn.Module): + + def __init__(self, vocab_size, embed_width): + super(Embedding, self).__init__() + + self.embedding = nn.Embedding(vocab_size, embed_width) + + def forward(self, input_ids): + return self.embedding(input_ids) diff --git a/runtime/Lib/site_packages/models/nlp/lstm/token_classification.py b/runtime/Lib/site_packages/models/nlp/lstm/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..5ec2cc09d0869a340f861f0b7394aca232b3d2f3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/lstm/token_classification.py @@ -0,0 +1,49 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from modelscope.metainfo import Heads, Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models import ( + ModelForTokenClassification, ModelForTokenClassificationWithCRF) +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.token_classification, module_name=Models.lcrf) +@MODELS.register_module( + Tasks.named_entity_recognition, module_name=Models.lcrf) +@MODELS.register_module(Tasks.part_of_speech, module_name=Models.lcrf) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.lcrf) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.lcrf_wseg) +class LSTMForTokenClassificationWithCRF(ModelForTokenClassificationWithCRF): + r"""Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks, word-segmentation. + + """ + override_base_model_type = True + base_model_type = Models.lstm + head_type = Heads.lstm_crf + + def parse_head_cfg(self): + head_cfg = super(ModelForTokenClassification, self).parse_head_cfg() + head_cfg['hidden_size'] = ( + head_cfg.hidden_size + if hasattr(head_cfg, 'hidden_size') else head_cfg.lstm_hidden_size) + head_cfg['num_labels'] = self.config.num_labels + return head_cfg diff --git a/runtime/Lib/site_packages/models/nlp/megatron_bert/__init__.py b/runtime/Lib/site_packages/models/nlp/megatron_bert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c0aa427d331cbeda61021be5a8ffe66c30c7516f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/megatron_bert/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import MegatronBertConfig + from .backbone import MegatronBertModel + from .fill_mask import MegatronBertForMaskedLM +else: + _import_structure = { + 'configuration': ['MegatronBertConfig'], + 'backbone': ['MegatronBertModel'], + 'fill_mask': ['MegatronBertForMaskedLM'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/megatron_bert/backbone.py b/runtime/Lib/site_packages/models/nlp/megatron_bert/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..92bc7475f79316fd281f42fc2ebf38ed1abf95c8 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/megatron_bert/backbone.py @@ -0,0 +1,897 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch MegatronBERT model.""" + +import math + +import torch +import torch.utils.checkpoint +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import MegatronBertConfig + +logger = get_logger() + +_CONFIG_FOR_DOC = 'MegatronBertConfig' + + +class MegatronBertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + + # In Megatron, layer-norm is applied after the 1st dropout. + # self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + + def forward( + self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + ) -> torch.Tensor: + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + + # Megatron BERT moves that layer norm after the drop-out (and to each layer). + # embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +# Copied from transformers.models.bert.modeling_bert.BertSelfAttention with Bert->MegatronBert +class MegatronBertSelfAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = position_embedding_type or getattr( + config, 'position_embedding_type', 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor: + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in MegatronBertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +# Based transformers.models.bert.modeling_bert.BertSelfOutput. Moved LayerNorm to MegatronBertAttention below. +class MegatronBertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, residual): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + return residual + hidden_states + + +# Based transformers.models.bert.modeling_bert.BertAttention. Added LayerNorm. +class MegatronBertAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.ln = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.self = MegatronBertSelfAttention(config) + self.output = MegatronBertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + ln_outputs = self.ln(hidden_states) + self_outputs = self.self( + ln_outputs, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +# Copied from transformers.models.bert.modeling_bert.BertIntermediate with Bert->MegatronBert +class MegatronBertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +# Based on transformers.models.bert.modeling_bert.BertOutput. Moved LayerNorm to MegatronBertLayer below. +class MegatronBertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states: torch.Tensor, + input_tensor: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + return input_tensor + hidden_states + + +# Based on transformers.models.bert.modeling_bert.BertLayer. Added LayerNorm. +class MegatronBertLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = MegatronBertAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise TypeError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = MegatronBertAttention(config) + self.ln = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.intermediate = MegatronBertIntermediate(config) + self.output = MegatronBertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise AttributeError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers' + ' by setting `config.add_cross_attention=True`') + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + ln_output = self.ln(attention_output) + intermediate_output = self.intermediate(ln_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class MegatronBertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList([ + MegatronBertLayer(config) for _ in range(config.num_hidden_layers) + ]) + + # The final layer norm. We removed the 1st LN, moved LN to each hidden layer and this one + # is simply the final LN (Transformer's BERT has it attached to each hidden layer). + self.ln = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + # Because we moved the layer-norm at the end of the hidden layer, we have non-normali- + # zed data here. If that's really needed, we must apply LN to match Transformer's BERT. + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + # Finalize the hidden states. + hidden_states = self.ln(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +# Copied from transformers.models.bert.modeling_bert.BertPooler with Bert->MegatronBert +class MegatronBertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class MegatronBertPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = MegatronBertConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, MegatronBertEncoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels not supplied. + If num_labels is not found, the model will use the default setting (2 classes). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + if model_dir is None: + config = MegatronBertConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + model.model_dir = model_dir + return model + + +@MODELS.register_module( + group_key=Tasks.backbone, module_name=Models.megatron_bert) +class MegatronBertModel(MegatronBertPreTrainedModel): + """The bare MegatronBert Model transformer outputting raw hidden-states without any specific head on top.", + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MegatronBertConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set + to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and + `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = MegatronBertEmbeddings(config) + self.encoder = MegatronBertEncoder(config) + + self.pooler = MegatronBertPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs) -> AttentionBackboneModelOutput: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`BertTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the + inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position + embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, + num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask + values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, + *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a + plain tuple. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, + sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the + encoder. Used in the cross-attention if the model is configured as a + decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, + sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of + the encoder input. This mask is used in the cross-attention if the + model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length + `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention + blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only + the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead + of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned + and can be used to speed up decoding (see `past_key_values`). + Others (**kwargs) + some additional parameters might passed in from upstream pipeline, + which not influence the results. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return AttentionBackboneModelOutput( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/megatron_bert/configuration.py b/runtime/Lib/site_packages/models/nlp/megatron_bert/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..85769ad6ddc55338b45c281bda021c7101796cb7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/megatron_bert/configuration.py @@ -0,0 +1,125 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MEGATRON_BERT model configuration""" + +from collections import OrderedDict +from typing import Mapping + +from transformers.configuration_utils import PretrainedConfig +from transformers.onnx import OnnxConfig + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class MegatronBertConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`MegatronBertModel`]. It is used to instantiate a + MEGATRON_BERT model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the MEGATRON_BERT + [nvidia/megatron-bert-uncased-345m](https://huggingface.co/nvidia/megatron-bert-uncased-345m) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 29056): + Vocabulary size of the MEGATRON_BERT model. Defines the number of different tokens that can be represented + by the `inputs_ids` passed when calling [`MegatronBertModel`]. + hidden_size (`int`, *optional*, defaults to 1024): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 24): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 4096): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling [`MegatronBertModel`]. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (`str`, *optional*, defaults to `"absolute"`): + Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For + positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to + [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). + For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models + with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + + Examples: + + >>> from transformers import MegatronBertConfig, MegatronBertModel + + >>> # Initializing a MEGATRON_BERT bert-base-uncased style configuration + >>> configuration = MegatronBertConfig() + + >>> # Initializing a model (with random weights) from the bert-base-uncased style configuration + >>> model = MegatronBertModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + model_type = 'megatron-bert' + + def __init__(self, + vocab_size=29056, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type='absolute', + use_cache=True, + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache diff --git a/runtime/Lib/site_packages/models/nlp/megatron_bert/fill_mask.py b/runtime/Lib/site_packages/models/nlp/megatron_bert/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..652cf1ed1ff3abc9bddf89d312d5a4f69c353675 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/megatron_bert/fill_mask.py @@ -0,0 +1,287 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionFillMaskModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import MegatronBertModel, MegatronBertPreTrainedModel +from .configuration import MegatronBertConfig + +logger = logging.get_logger() + + +# Copied from transformers.models.bert.modeling_bert.BertPredictionHeadTransform with Bert->MegatronBert +class MegatronBertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->MegatronBert +class MegatronBertLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = MegatronBertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertOnlyMLMHead with Bert->MegatronBert +class MegatronBertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = MegatronBertLMPredictionHead(config) + + def forward(self, sequence_output: torch.Tensor) -> torch.Tensor: + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +# Copied from transformers.models.bert.modeling_bert.BertOnlyNSPHead with Bert->MegatronBert +class MegatronBertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +# Copied from transformers.models.bert.modeling_bert.BertPreTrainingHeads with Bert->MegatronBert +class MegatronBertPreTrainingHeads(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = MegatronBertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +@MODELS.register_module(Tasks.fill_mask, module_name=Models.megatron_bert) +class MegatronBertForMaskedLM(MegatronBertPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r'pooler', r'seq_relationship'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config: MegatronBertConfig, **kwargs): + super().__init__(config) + + if config.is_decoder: + logger.warning( + 'If you want to use `MegatronBertForMaskedLM` make sure `config.is_decoder=False` for ' + 'bi-directional self-attention.') + + self.bert = MegatronBertModel(config, add_pooling_layer=False) + self.cls = MegatronBertOnlyMLMHead(config) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + `What are input IDs? <../glossary.html#input-ids>`__ + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + `What are attention masks? <../glossary.html#attention-mask>`__ + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0,config.max_position_embeddings - 1]``. + + `What are position IDs? <../glossary.html#position-ids>`_ + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, + *optional*): + Labels for computing the masked language modeling loss. Indices + should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` + docstring) Tokens with indices set to `-100` are ignored (masked), + the loss is only computed for the tokens with labels in `[0, ..., + config.vocab_size]` + + Returns: + Returns `modelscope.outputs.AttentionFillMaskModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_megatronbert_backbone_base_std') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_megatronbert_backbone_base_std') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return AttentionFillMaskModelOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + input_ids=input_ids, + ) + + def prepare_inputs_for_generation(self, + input_ids, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + if self.config.pad_token_id is None: + raise ValueError('The PAD token should be defined for generation') + attention_mask_new_zeros = attention_mask.new_zeros( + (attention_mask.shape[0], 1)) + attention_mask = torch.cat([attention_mask, attention_mask_new_zeros], + dim=-1) + dummy_token = torch.full((effective_batch_size, 1), + self.config.pad_token_id, + dtype=torch.long, + device=input_ids.device) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {'input_ids': input_ids, 'attention_mask': attention_mask} diff --git a/runtime/Lib/site_packages/models/nlp/mglm/__init__.py b/runtime/Lib/site_packages/models/nlp/mglm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c96ac4a00ffd3636c89e31e96065fed24d16506 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/__init__.py @@ -0,0 +1,22 @@ +# Modified by Zhipu.AI +# Original Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .mglm_for_text_summarization import MGLMForTextSummarization +else: + _import_structure = { + 'mglm_for_text_summarization': ['MGLMForTextSummarization'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/arguments.py b/runtime/Lib/site_packages/models/nlp/mglm/arguments.py new file mode 100644 index 0000000000000000000000000000000000000000..4fa33c65aabd3f77791e65ef0e8bc2934c2e7a73 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/arguments.py @@ -0,0 +1,793 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""argparser configuration""" + +import argparse +import os + +import deepspeed +import json +import torch + +from .utils import get_hostname + + +def add_model_config_args(parser): + """Model arguments""" + + group = parser.add_argument_group('model', 'model configuration') + + group.add_argument( + '--transformer-xl', + action='store_true', + help='use transformer-xl for training') + group.add_argument( + '--pretrained-bert', + action='store_true', + help='use a pretrained bert-large-uncased model instead' + 'of initializing from scratch. See ' + '--tokenizer-model-type to specify which pretrained ' + 'BERT model to use') + group.add_argument( + '--encoder-decoder', + action='store_true', + help='use the encoder-decoder architecture for blocklm') + group.add_argument( + '--attention-dropout', + type=float, + default=0.1, + help='dropout probability for attention weights') + group.add_argument( + '--num-attention-heads', + type=int, + default=16, + help='num of transformer attention heads') + group.add_argument( + '--hidden-size', type=int, default=1024, help='tansformer hidden size') + group.add_argument( + '--intermediate-size', + type=int, + default=None, + help='transformer embedding dimension for FFN' + 'set to 4*`--hidden-size` if it is None') + group.add_argument( + '--num-layers', type=int, default=24, help='num decoder layers') + group.add_argument( + '--layernorm-epsilon', + type=float, + default=1e-5, + help='layer norm epsilon') + group.add_argument( + '--hidden-dropout', + type=float, + default=0.1, + help='dropout probability for hidden state transformer') + group.add_argument( + '--output-dropout', + type=float, + default=0.1, + help='dropout probability for pooled output') + group.add_argument( + '--max-position-embeddings', + type=int, + default=512, + help='maximum number of position embeddings to use') + group.add_argument( + '--vocab-size', + type=int, + default=250112, + help='vocab size to use for non-character-level ' + 'tokenization. This value will only be used when ' + 'creating a tokenizer') + group.add_argument( + '--deep-init', + action='store_true', + help='initialize bert model similar to gpt2 model.' + 'scales initialization of projection layers by a ' + 'factor of 1/sqrt(2N). Necessary to train bert ' + 'models larger than BERT-Large.') + group.add_argument( + '--make-vocab-size-divisible-by', + type=int, + default=128, + help='Pad the vocab size to be divisible by this value.' + 'This is added for computational efficieny reasons.') + group.add_argument( + '--cpu-optimizer', action='store_true', help='Run optimizer on CPU') + group.add_argument( + '--cpu_torch_adam', + action='store_true', + help='Use Torch Adam as optimizer on CPU.') + + return parser + + +def add_fp16_config_args(parser): + """Mixed precision arguments.""" + + group = parser.add_argument_group('fp16', 'fp16 configurations') + + group.add_argument( + '--fp16', action='store_true', help='Run model in fp16 mode') + group.add_argument( + '--fp32-embedding', action='store_true', help='embedding in fp32') + group.add_argument( + '--fp32-layernorm', action='store_true', help='layer norm in fp32') + group.add_argument( + '--fp32-tokentypes', + action='store_true', + help='embedding token types in fp32') + group.add_argument( + '--fp32-allreduce', action='store_true', help='all-reduce in fp32') + group.add_argument( + '--hysteresis', + type=int, + default=2, + help='hysteresis for dynamic loss scaling') + group.add_argument( + '--loss-scale', + type=float, + default=None, + help='Static loss scaling, positive power of 2 ' + 'values can improve fp16 convergence. If None, dynamic' + 'loss scaling is used.') + group.add_argument( + '--loss-scale-window', + type=float, + default=1000, + help='Window over which to raise/lower dynamic scale') + group.add_argument( + '--min-scale', + type=float, + default=1, + help='Minimum loss scale for dynamic loss scale') + group.add_argument('--attention-scale', type=float, default=1.0) + return parser + + +def add_training_args(parser): + """Training arguments.""" + + group = parser.add_argument_group('train', 'training configurations') + + group.add_argument( + '--experiment-name', + type=str, + default='gpt-345M', + help='The experiment name for summary and checkpoint') + group.add_argument( + '--batch-size', type=int, default=4, help='Data Loader batch size') + group.add_argument( + '--gradient-accumulation-steps', + type=int, + default=1, + help='Data Loader batch size') + group.add_argument( + '--weight-decay', + type=float, + default=0.01, + help='weight decay coefficient for L2 regularization') + group.add_argument( + '--checkpoint-activations', + action='store_true', + help='checkpoint activation to allow for training ' + 'with larger models and sequences') + group.add_argument( + '--checkpoint-num-layers', + type=int, + default=1, + help='chunk size (number of layers) for checkpointing') + group.add_argument( + '--deepspeed-activation-checkpointing', + action='store_true', + help='uses activation checkpointing from deepspeed') + group.add_argument( + '--epochs', + type=int, + default=None, + help='Number of finetunning epochs. Zero results in evaluation only.') + group.add_argument( + '--clip-grad', type=float, default=1.0, help='gradient clipping') + group.add_argument( + '--train-iters', + type=int, + default=0, + help='total number of iterations to train over all training runs') + group.add_argument('--label-smoothing', type=float, default=0.0) + group.add_argument( + '--log-interval', type=int, default=100, help='report interval') + group.add_argument( + '--summary-dir', + type=str, + default='', + help='The directory to store the summary') + group.add_argument('--seed', type=int, default=1234, help='random seed') + # Batch producer arguments + group.add_argument( + '--reset-position-ids', + action='store_true', + help='Reset posistion ids after end-of-document token.') + group.add_argument( + '--reset-attention-mask', + action='store_true', + help='Reset self attention maske after ' + 'end-of-document token.') + + # Learning rate. + group.add_argument( + '--lr-decay-iters', + type=int, + default=None, + help='number of iterations to decay LR over,' + ' If None defaults to `--train-iters`*`--epochs`') + group.add_argument( + '--lr-decay-style', + type=str, + default='linear', + choices=['constant', 'linear', 'cosine', 'exponential'], + help='learning rate decay function') + group.add_argument('--lr-decay-ratio', type=float, default=0.1) + group.add_argument( + '--lr', type=float, default=1.0e-4, help='initial learning rate') + group.add_argument( + '--warmup', + type=float, + default=0.01, + help='percentage of data to warmup on (.01 = 1% of all ' + 'training iters). Default 0.01') + group.add_argument( + '--switch-linear', + action='store_true', + help='Switch to linear decay for cosine decay') + # model checkpointing + group.add_argument( + '--save', + type=str, + default=None, + help='Output directory to save checkpoints to.') + group.add_argument('--new-save-directory', action='store_true') + group.add_argument( + '--save-epoch', + type=int, + default=1, + help='number of epochs between saves') + group.add_argument( + '--save-interval', + type=int, + default=5000, + help='number of iterations between saves') + group.add_argument( + '--no-save-optim', + action='store_true', + help='Do not save current optimizer.') + group.add_argument( + '--no-save-rng', + action='store_true', + help='Do not save current rng state.') + group.add_argument( + '--load', + type=str, + default=None, + help='Path to a directory containing a model checkpoint.') + group.add_argument( + '--no-load-optim', + action='store_true', + help='Do not load optimizer when loading checkpoint.') + group.add_argument( + '--no-load-rng', + action='store_true', + help='Do not load rng state when loading checkpoint.') + group.add_argument( + '--no-load-lr-scheduler', + action='store_true', + help='Do not load lr scheduler when loading checkpoint.') + group.add_argument( + '--no-deepspeed-load', + action='store_true', + help='Not use deepspeed when loading checkpoint') + group.add_argument( + '--finetune', + action='store_true', + help='Load model for finetuning. Do not load optimizer ' + 'or rng state from checkpoint and set iteration to 0. ' + 'Assumed when loading a release checkpoint.') + group.add_argument( + '--resume-dataloader', + action='store_true', + help='Resume the dataloader when resuming training. ' + 'Does not apply to tfrecords dataloader, try resuming' + 'with a different seed in this case.') + # distributed training args + group.add_argument( + '--distributed-backend', + default='nccl', + help= + 'which backend to use for distributed training. One of [gloo, nccl]', + choices=['nccl', 'gloo']) + group.add_argument( + '--DDP-impl', + default='torch', + choices=['local', 'torch', 'none'], + help='which DistributedDataParallel implementation to use.') + + group.add_argument( + '--local_rank', + type=int, + default=None, + help='local rank passed from distributed launcher') + # BlockLM training args + group.add_argument( + '--block-lm', + action='store_true', + help='whether use the BlockLM pre-training') + group.add_argument( + '--masked-lm', + action='store_true', + help='whether to use the mlm objective') + group.add_argument('--bert-prob', type=float, default=0.5) + group.add_argument('--gpt-infill-prob', type=float, default=0.5) + group.add_argument('--gpt-min-ratio', type=float, default=0.5) + group.add_argument('--gap-sentence-prob', type=float, default=0.0) + group.add_argument('--gap-sentence-ratio', type=float, default=0.15) + group.add_argument('--avg-block-length', type=int, default=3) + group.add_argument('--short-seq-prob', type=float, default=0.0) + group.add_argument('--single-span-prob', type=float, default=0.0) + group.add_argument( + '--task-mask', + action='store_true', + help='Use different mask for generation and blank filling') + group.add_argument( + '--no-shuffle-block', + action='store_true', + help='not shuffle the blocks when filling the blank') + group.add_argument( + '--no-block-position', + action='store_true', + help='Use (rough) absolute positions instead of block positions') + group.add_argument( + '--sentinel-token', + action='store_true', + help='Use sentinel (mask) tokens to replace 2d position encoding') + group.add_argument('--block-mask-prob', type=float, default=0.0) + group.add_argument('--context-mask-ratio', type=float, default=0.0) + group.add_argument( + '--random-position', + action='store_true', + help='Use random start position to cover all the position embeddings') + return parser + + +def add_evaluation_args(parser): + """Evaluation arguments.""" + + group = parser.add_argument_group('validation', + 'validation configurations') + + group.add_argument( + '--eval-batch-size', + type=int, + default=None, + help='Data Loader batch size for evaluation datasets.' + 'Defaults to `--batch-size`') + group.add_argument( + '--eval-iters', + type=int, + default=100, + help='number of iterations to run for evaluation' + 'validation/test for') + group.add_argument( + '--eval-interval', + type=int, + default=1000, + help='interval between running evaluation on validation set') + group.add_argument( + '--eval-epoch', + type=int, + default=1, + help='epoch between running evaluation on validation set') + group.add_argument( + '--eval-seq-length', + type=int, + default=None, + help='Maximum sequence length to process for ' + 'evaluation. Defaults to `--seq-length`') + group.add_argument( + '--eval-max-preds-per-seq', + type=int, + default=None, + help='Maximum number of predictions to use for ' + 'evaluation. Defaults to ' + 'math.ceil(`--eval-seq-length`*.15/10)*10') + group.add_argument('--overlapping-eval', type=int, default=32) + + return parser + + +def add_text_generate_args(parser): + """Text generate arguments.""" + + group = parser.add_argument_group('Text generation', 'configurations') + group.add_argument('--temperature', type=float, default=1.0) + group.add_argument('--top_p', type=float, default=0.0) + group.add_argument('--top_k', type=int, default=0) + group.add_argument('--out-seq-length', type=int, default=256) + group.add_argument('--num-beams', type=int, default=1) + group.add_argument('--length-penalty', type=float, default=0.0) + group.add_argument('--no-repeat-ngram-size', type=int, default=0) + group.add_argument('--min-tgt-length', type=int, default=0) + group.add_argument('--select-topk', action='store_true') + group.add_argument('--blank-maskratio', type=float, default=0.1) + return parser + + +def add_data_args(parser): + """Train/valid/test data arguments.""" + + group = parser.add_argument_group('data', 'data configurations') + + group.add_argument( + '--model-parallel-size', + type=int, + default=1, + help='size of the model parallel.') + group.add_argument( + '--shuffle', + action='store_true', + help='Shuffle data. Shuffling is deterministic ' + 'based on seed and current epoch.') + group.add_argument('--filter-english', action='store_true') + group.add_argument( + '--train-data', + nargs='+', + default=None, + help='Whitespace separated filenames or corpora names ' + 'for training.') + group.add_argument( + '--valid-data', + nargs='*', + default=None, + help="""Filename for validation data.""") + group.add_argument( + '--test-data', + nargs='*', + default=None, + help="""Filename for testing""") + group.add_argument( + '--data-dir', + type=str, + default=None, + help='The data path to all the data files') + group.add_argument( + '--input-data-sizes-file', + type=str, + default='sizes.txt', + help='the filename containing all the shards sizes') + + group.add_argument( + '--delim', default=',', help='delimiter used to parse csv data files') + group.add_argument( + '--text-key', + default='sentence', + help='key to use to extract text from json/csv') + group.add_argument( + '--eval-text-key', + default=None, + help='key to use to extract text from ' + 'json/csv evaluation datasets') + group.add_argument( + '--split', + default='1000,1,1', + help='comma-separated list of proportions for training,' + ' validation, and test split') + + group.add_argument( + '--no-lazy-loader', + action='store_true', + help='whether to lazy read the data set') + group.add_argument('--half-lazy-loader', action='store_true') + group.add_argument( + '--loader-scatter', + type=int, + default=None, + help='Number of scatters to use for dataloaders') + group.add_argument( + '--loose-json', + action='store_true', + help='Use loose json (one json-formatted string per ' + 'newline), instead of tight json (data file is one ' + 'json string)') + group.add_argument( + '--presplit-sentences', + action='store_true', + help='Dataset content consists of documents where ' + 'each document consists of newline separated sentences') + group.add_argument( + '--num-workers', + type=int, + default=2, + help="""Number of workers to use for dataloading""") + group.add_argument( + '--tokenizer-model-type', + type=str, + default=None, + help="Model type to use for sentencepiece tokenization \ + (one of ['bpe', 'char', 'unigram', 'word']) or \ + bert vocab to use for BertWordPieceTokenizer (one of \ + ['bert-large-uncased', 'bert-large-cased', etc.])") + group.add_argument( + '--tokenizer-path', + type=str, + default='tokenizer.model', + help='path used to save/load sentencepiece tokenization ' + 'models') + group.add_argument( + '--tokenizer-type', + type=str, + default='BertWordPieceTokenizer', + choices=[ + 'CharacterLevelTokenizer', 'SentencePieceTokenizer', + 'BertWordPieceTokenizer', 'GPT2BPETokenizer', 'ChineseSPTokenizer' + ], + help='what type of tokenizer to use') + group.add_argument('--no-pre-tokenize', action='store_true') + group.add_argument( + '--cache-dir', + default=None, + type=str, + help='Where to store pre-trained BERT downloads') + group.add_argument( + '--use-tfrecords', + action='store_true', + help='load `--train-data`, `--valid-data`, ' + '`--test-data` from BERT tf records instead of ' + 'normal data pipeline') + group.add_argument( + '--seq-length', + type=int, + default=512, + help='Maximum sequence length to process') + group.add_argument( + '--mem-length', + type=int, + default=0, + help='The memory length to preserve') + group.add_argument( + '--max-preds-per-seq', + type=int, + default=None, + help='Maximum number of predictions to use per sequence.' + 'Defaults to math.ceil(`--seq-length`*.15/10)*10.' + 'MUST BE SPECIFIED IF `--use-tfrecords` is True.') + group.add_argument('--non-sentence-start', type=float, default=0.0) + group.add_argument( + '--sample-one-document', + action='store_true', + help='only sample one document in one sample') + group.add_argument( + '--load-splits', + type=str, + default=None, + help='The path to load split indices from') + group.add_argument( + '--save-splits', + type=str, + default=None, + help='The path to save split indices to') + group.add_argument( + '--save-test-data', + type=str, + default=None, + help='The path to save the test data') + group.add_argument( + '--multi-task-data', + nargs='*', + default=None, + help='Downsteam task names for multi-task pre-training') + group.add_argument( + '--multi-task-ratio', + type=float, + default=0.0, + help='Ratio for multi-task pre-training') + group.add_argument('--multi-seq-length', type=int, default=None) + group.add_argument('--multi-batch-size', type=int, default=None) + return parser + + +def add_finetune_config_args(parser): + group = parser.add_argument_group('finetune', 'finetune configurations') + group.add_argument('--task', type=str, help='Task name.') + group.add_argument( + '--load-pretrained', + type=str, + help='Load pretrained model', + default=None) + group.add_argument( + '--pool-token', + type=str, + choices=['start', 'pad', 'cls'], + help='The token to pool the sequence representation', + default='cls') + group.add_argument( + '--cloze-eval', + action='store_true', + help='Evaluation dataset with cloze task') + group.add_argument( + '--multi-token', + action='store_true', + help='Use multi token for cloze evaluation') + group.add_argument( + '--segment-length', + type=int, + default=0, + help='The maximum segment length for cloze evaluation') + group.add_argument( + '--loss-func', + type=str, + choices=['cross_entropy', 'hinge', 'generative', 'mix'], + default='cross_entropy') + group.add_argument('--block-lm-ratio', type=float, default=0.0) + group.add_argument( + '--adapet', + action='store_true', + help='Use the decoupled cross entropy loss in AdaPET') + group.add_argument('--pattern-id', type=int, default=0) + group.add_argument( + '--fast-decode', + action='store_true', + help= + 'Fast decode for multi-token cloze. Can only be used without checkpoint activation.' + ) + group.add_argument('--few-superglue', action='store_true') + group.add_argument( + '--eval-valid', + action='store_true', + help='Whether evaluate on the valid set') + group.add_argument('--validation-metric', type=str, default=None) + group.add_argument( + '--unidirectional', + action='store_true', + help='Use the left to right language model') + group.add_argument('--src-seq-length', type=int, default=None) + group.add_argument('--tgt-seq-length', type=int, default=None) + group.add_argument('--adam-beta1', type=float, default=0.9) + group.add_argument('--adam-beta2', type=float, default=0.999) + group.add_argument('--adam-eps', type=float, default=1e-8) + group.add_argument( + '--optimizer', type=str, choices=['adam', 'adafactor'], default='adam') + group.add_argument('--wsc-negative', action='store_true') + group.add_argument('--overwrite', action='store_true') + group.add_argument('--no-validation', action='store_true') + # Continuous prompt arguments + group.add_argument( + '--continuous-prompt', + action='store_true', + help='Use continuous prompt for PET') + group.add_argument('--num-prompt-tokens', type=int, default=0) + group.add_argument( + '--prompt-func', default='lstm', choices=['lstm', 'mlp', 'none']) + group.add_argument( + '--freeze-transformer', action='store_true', default=False) + group.add_argument('--tune-prefix-layers', type=int, default=None) + group.add_argument('--prefix-prompt', type=int, default=0) + group.add_argument('--prompt-init', action='store_true', default=False) + return parser + + +def get_args(): + """Parse all the args.""" + + parser = argparse.ArgumentParser(description='PyTorch BERT Model') + parser = add_model_config_args(parser) + parser = add_fp16_config_args(parser) + parser = add_training_args(parser) + parser = add_evaluation_args(parser) + parser = add_text_generate_args(parser) + parser = add_data_args(parser) + parser = add_finetune_config_args(parser) + + # Include DeepSpeed configuration arguments + parser = deepspeed.add_config_arguments(parser) + + args = parser.parse_args(args=[]) + if not args.train_data and not args.data_dir: + print('WARNING: No training data specified') + + args.cuda = torch.cuda.is_available() + + args.rank = int(os.getenv('RANK', '0')) + args.world_size = int(os.getenv('WORLD_SIZE', '1')) + if hasattr(args, 'deepspeed_mpi') and args.deepspeed_mpi: + mpi_define_env(args) + elif os.getenv('OMPI_COMM_WORLD_LOCAL_RANK'): + # We are using (OpenMPI) mpirun for launching distributed data parallel processes + local_rank = int(os.getenv('OMPI_COMM_WORLD_LOCAL_RANK')) + local_size = int(os.getenv('OMPI_COMM_WORLD_LOCAL_SIZE')) + + # Possibly running with Slurm + num_nodes = int(os.getenv('SLURM_JOB_NUM_NODES', '1')) + nodeid = int(os.getenv('SLURM_NODEID', '0')) + + args.local_rank = local_rank + args.rank = nodeid * local_size + local_rank + args.world_size = num_nodes * local_size + + args.model_parallel_size = min(args.model_parallel_size, args.world_size) + if args.rank == 0: + print('using world size: {} and model-parallel size: {} '.format( + args.world_size, args.model_parallel_size)) + + args.dynamic_loss_scale = False + if args.loss_scale is None: + args.dynamic_loss_scale = True + if args.rank == 0: + print(' > using dynamic loss scaling') + + # The args fp32_* or fp16_* meant to be active when the + # args fp16 is set. So the default behaviour should all + # be false. + if not args.fp16: + args.fp32_embedding = False + args.fp32_tokentypes = False + args.fp32_layernorm = False + + if hasattr(args, 'deepspeed' + ) and args.deepspeed and args.deepspeed_config is not None: + with open(args.deepspeed_config, encoding='utf-8') as file: + deepspeed_config = json.load(file) + if 'train_micro_batch_size_per_gpu' in deepspeed_config: + args.batch_size = deepspeed_config[ + 'train_micro_batch_size_per_gpu'] + if 'gradient_accumulation_steps' in deepspeed_config: + args.gradient_accumulation_steps = deepspeed_config[ + 'gradient_accumulation_steps'] + else: + args.gradient_accumulation_steps = 1 + if 'optimizer' in deepspeed_config: + optimizer_params_config = deepspeed_config['optimizer'].get( + 'params', {}) + args.lr = optimizer_params_config.get('lr', args.lr) + args.weight_decay = optimizer_params_config.get( + 'weight_decay', args.weight_decay) + return args + + +def mpi_define_env(args): + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + world_size = comm.Get_size() + + master_addr = None + if rank == 0: + master_addr = get_hostname() + master_addr = comm.bcast(master_addr, root=0) + + # Determine local rank by assuming hostnames are unique + proc_name = MPI.Get_processor_name() + all_procs = comm.allgather(proc_name) + local_rank = sum([i == proc_name for i in all_procs[:rank]]) + + os.environ['RANK'] = str(rank) + os.environ['WORLD_SIZE'] = str(world_size) + args.local_rank = local_rank + args.world_size = world_size + args.rank = rank + os.environ['MASTER_ADDR'] = master_addr + os.environ[ + 'MASTER_PORT'] = '29500' # TORCH_DISTRIBUTED_DEFAULT_PORT = 29500 + + print( + 'Discovered MPI settings of world_rank={}, local_rank={}, world_size={}, master_addr={}, master_port={}' + .format(os.environ['RANK'], args.local_rank, os.environ['WORLD_SIZE'], + os.environ['MASTER_ADDR'], os.environ['MASTER_PORT'])) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/blocklm_utils.py b/runtime/Lib/site_packages/models/nlp/mglm/blocklm_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e75aea923b1d1143d4f6034e6894763d89168f5b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/blocklm_utils.py @@ -0,0 +1,620 @@ +# Copyright (c) 2022 Zhipu.AI + +import copy +import math +import random + +import numpy as np +import torch +import torch.utils.data +from megatron_util import mpu, print_rank_0 +from scipy.stats import poisson + + +def rindex(lst, val, start=None): + if start is None: + start = len(lst) - 1 + for i in range(start, -1, -1): + if lst[i] == val: + return i + return -1 + + +def index_in_list(lst, val, start=None): + if start is None: + start = 0 + for i in range(start, len(lst)): + if lst[i] == val: + return i + return -1 + + +class ConstructBlockStrategy: + + def __init__(self, + args, + tokenizer, + max_seq_length, + bert_prob=1.0, + gap_sentence_prob=0.0, + gpt_infill_prob=0.5, + gpt_min_ratio=0.5, + bert_ratio=0.15, + gap_sentence_ratio=0.15, + average_block_length=3, + max_block_length=40, + block_mask_prob=0.0, + context_mask_ratio=0.0, + context_mask_range=3, + short_seq_prob=0.0, + single_span_prob=0.0, + block_position_encoding=True, + encoder_decoder=False, + shuffle_blocks=True, + sentinel_token=False, + task_mask=False, + random_position=False, + masked_lm=False): + self.eod_token = args.eod_token + self.tokenizer = tokenizer + self.count = 0 + self.max_seq_length = max_seq_length + self.rank = mpu.get_data_parallel_rank() + self.world_size = mpu.get_data_parallel_world_size() + # self.rank = 0 + # self.world_size = 1 + assert 0.0 <= bert_prob <= 1.0 + self.bert_prob = bert_prob + self.gap_sentence_prob = gap_sentence_prob + self.gpt_prob = 1 - bert_prob - gap_sentence_prob + assert self.gpt_prob >= -1e-10 + self.infill_prob = gpt_infill_prob + self.gpt_min_ratio = gpt_min_ratio + self.bert_ratio = bert_ratio + self.gap_sentence_ratio = gap_sentence_ratio + self.block_length_distribution = [ + poisson.pmf(i, average_block_length) + for i in range(1, max_block_length) + ] + self.block_mask_prob = block_mask_prob + self.context_mask_ratio = context_mask_ratio + self.context_mask_range = context_mask_range + self.short_seq_prob = short_seq_prob + self.single_span_prob = single_span_prob + self.block_position_encoding = block_position_encoding + self.encoder_decoder = encoder_decoder + self.shuffle_blocks = shuffle_blocks + self.sentinel_token = sentinel_token + self.generation_mask = 'gMASK' if task_mask else 'MASK' + self.generation_mask = self.tokenizer.get_command( + self.generation_mask).Id + self.gap_sentence_mask = 'sMASK' if task_mask else 'MASK' + self.gap_sentence_mask = self.tokenizer.get_command( + self.gap_sentence_mask).Id + self.random_position = random_position + self.masked_lm = masked_lm + print_rank_0( + f'BERT prob {self.bert_prob}, gap sent prob {self.gap_sentence_prob}, GPT prob {self.gpt_prob}, infill prob {self.infill_prob}' # noqa + ) + print_rank_0( + f'generation min ratio {self.gpt_min_ratio}, block ratio {self.bert_ratio}, gap sent ratio {self.gap_sentence_ratio}' # noqa + ) + print_rank_0( + f'block length distribution {self.block_length_distribution}') + print_rank_0( + f'block mask prob {self.block_mask_prob}, context mask ratio {self.context_mask_ratio}' + ) + + def contains_sentence_end(self, tok): + tok = self.tokenizer.IdToToken(tok) + if '.' in tok: + return True + if '?' in tok: + return True + if '!' in tok: + return True + if ';' in tok: + return True + if ':' in tok: + return True + if '。' in tok: + return True + if '?' in tok: + return True + if '!' in tok: + return True + if ';' in tok: + return True + if '…' in tok: + return True + if '\n' in tok: + return True + return False + + @staticmethod + def sample_spans(span_lengths, total_length, rng, offset=0): + blank_length = total_length - sum(span_lengths) + m = blank_length - len(span_lengths) + 1 + places = [rng.randrange(m + 1) for _ in range(len(span_lengths))] + places.sort() + spans = [] + for place, span_length in zip(places, span_lengths): + start = offset + place + end = offset + place + span_length + spans.append((start, end)) + offset += span_length + 1 + return spans + + def sample_span_in_document(self, tokens, masked_lengths, rng): + rng.shuffle(masked_lengths) + mask_spans = [] + mask_index = 0 + indices = [-1] + np.where(tokens == self.eod_token)[0].tolist() + last_index = len(tokens) + documents = [] + for index in reversed(indices): + start_index = index + if start_index + 1 < len(tokens) and tokens[ + start_index + 1] == self.tokenizer.get_command('ENC').Id: + start_index += 1 + length = last_index - start_index - 1 + if last_index == len(tokens) and length > 0: + length -= 1 + documents.append((start_index + 1, length)) + last_index = index + documents.sort(key=lambda x: x[1]) + for i, (offset, length) in enumerate(documents): + if i == len(documents) - 1: + current_masked_length, current_count = 0, 0 + while mask_index + current_count < len( + masked_lengths + ) and masked_lengths[ + mask_index + # noqa + current_count] + current_masked_length + current_count <= length: + current_masked_length += masked_lengths[mask_index + + current_count] + current_count += 1 + if current_count > 0: + spans = self.sample_spans( + masked_lengths[mask_index:mask_index + current_count], + length, + rng, + offset=offset) + mask_spans += spans + if mask_index + current_count < len(masked_lengths) - 1: + print(length, masked_lengths[mask_index:], + masked_lengths[:mask_index], indices) + else: + current_masked_total = int(length * self.bert_ratio) + current_masked_length, current_count = 0, 0 + while mask_index + current_count < len( + masked_lengths + ) and masked_lengths[ + mask_index + # noqa + current_count] + current_masked_length <= current_masked_total: + current_masked_length += masked_lengths[mask_index + + current_count] + current_count += 1 + if current_count > 0: + spans = self.sample_spans( + masked_lengths[mask_index:mask_index + current_count], + length, + rng, + offset=offset) + mask_spans += spans + mask_index += current_count + return mask_spans + + def make_masked_data(self, + tokens, + loss_masks, + attention_mask, + block_spans, + rng, + task='bert'): + position_ids = np.arange(len(tokens), dtype=int) + targets = copy.deepcopy(tokens) + mask_id = self.tokenizer.get_command('MASK').Id + mlm_masks = np.zeros(len(tokens), dtype=int) + for start, end in block_spans: + for idx in range(start, end): + tokens[idx] = mask_id + mlm_masks[start:end] = 1 + loss_masks = loss_masks * mlm_masks + return tokens, targets, loss_masks, position_ids + + def make_block_data(self, + tokens, + loss_masks, + attention_mask, + block_spans, + rng, + task='bert'): + text_length = len(tokens) + position_ids = np.ones(len(tokens), dtype=int) + for start, end in block_spans: + position_ids[start + 1:end] = 0 + position_ids = np.cumsum(position_ids) - 1 + if self.random_position and position_ids[-1] < self.max_seq_length - 1: + position_bias = self.max_seq_length - position_ids[-1] + position_bias = rng.randrange(0, position_bias) + position_ids = position_ids + position_bias + if self.encoder_decoder or not self.shuffle_blocks: + block_spans.sort(key=lambda x: x[0]) + else: + rng.shuffle(block_spans) + if self.sentinel_token: + block_spans = [(start, end, idx) + for idx, (start, end) in enumerate(block_spans)] + else: + block_spans = [(start, end, 0) for start, end in block_spans] + target_tokens, target_position_ids, target_block_position_ids, targets = [], [], [], [] + for start, end, idx in block_spans: + sop_token = 'sop' if idx == 0 else f'sop{idx}' + target_tokens.append([self.tokenizer.get_command(sop_token).Id]) + span_tokens = copy.deepcopy(tokens[start:end]) + if self.block_mask_prob > 0.0 and task == 'bert': + for sub_idx in range(len(span_tokens)): + if random.random() < self.block_mask_prob: + span_tokens[sub_idx] = self.tokenizer.get_command( + 'dBLOCK').Id + target_tokens.append(span_tokens) + targets.append(tokens[start:end]) + targets.append([self.tokenizer.get_command('eop').Id]) + if not self.sentinel_token: + target_position_id = position_ids[start:end] + target_position_ids.append(target_position_id) + target_position_ids.append([target_position_id[0]]) + else: + target_position_ids.append([self.max_seq_length] * # noqa + (end - start + 1)) + if self.block_position_encoding: + target_block_position_ids.append( + np.arange(1, end - start + 2, dtype=int)) + else: + target_block_position_ids.append([1] * (end - start + 1)) + block_spans.sort(key=lambda x: x[0]) + source_tokens, source_position_ids, local_spans = [], [], [] + last, current_length = 0, 0 + for start, end, idx in block_spans: + if task == 'generation': + mask_id = self.generation_mask + elif task == 'gap_sentence': + mask_id = self.gap_sentence_mask + else: + mask_token = 'MASK' if idx == 0 else f'MASK{idx}' + mask_id = self.tokenizer.get_command(mask_token).Id + local_spans.append((current_length, current_length + start - last)) + source_tokens.append(tokens[last:start]) + source_tokens.append([mask_id]) + source_position_ids.append(position_ids[last:start]) + source_position_ids.append([position_ids[start]]) + current_length += start - last + 1 + last = end + if last < len(tokens): + local_spans.append( + (current_length, current_length + len(tokens) - last)) + source_tokens.append(tokens[last:]) + source_position_ids.append(position_ids[last:]) + source_length = sum(map(len, source_tokens)) + if attention_mask is not None: + assert source_length == attention_mask + if target_tokens and self.eod_token in np.concatenate( + target_tokens).tolist(): + print('Found EOS in target', self.tokenizer.DecodeIds(tokens)) + raise RuntimeError + if self.encoder_decoder: + target_tokens = target_tokens + [ + self.tokenizer.get_command('eop').Id + ] + loss_masks = np.ones(len(target_tokens), dtype=int) + return source_tokens, target_tokens, loss_masks + else: + tokens = np.concatenate(source_tokens + target_tokens) + if task == 'bert' and self.context_mask_ratio > 0: + mask_candidates = set() + for start, end in local_spans: + if start != 0: + local_end = min(end, start + self.context_mask_range) + mask_candidates.update(range(start, local_end)) + if end != 0: + local_start = max(start, end - self.context_mask_range) + mask_candidates.update(range(local_start, end)) + mask_pos = rng.sample( + mask_candidates, + int(self.context_mask_ratio * text_length)) + for pos in mask_pos: + tokens[pos] = self.tokenizer.get_command('dBLOCK').Id + targets = np.concatenate(source_tokens + targets) + loss_masks = np.ones(len(tokens), dtype=int) + loss_masks[:source_length] = 0 + position_ids = np.concatenate(source_position_ids + + target_position_ids) + block_position_ids = np.concatenate( + [np.zeros(source_length, dtype=int)] + + target_block_position_ids) + position_ids = np.stack([position_ids, block_position_ids], axis=0) + if attention_mask is not None: + return tokens, targets, loss_masks, position_ids + else: + return tokens, targets, loss_masks, position_ids, source_length + + def generate_blank_data(self, + sample, + masked_lengths, + attention_mask, + rng, + task='bert'): + rng.shuffle(masked_lengths) + tokens, loss_masks = sample['text'], sample['loss_mask'] + assert tokens[0] == self.tokenizer.get_command('ENC').Id + block_spans = self.sample_span_in_document(tokens, masked_lengths, rng) + if len(block_spans) < len(masked_lengths): + return None + if self.masked_lm: + data = self.make_masked_data(tokens, loss_masks, attention_mask, + block_spans, rng) + else: + data = self.make_block_data( + tokens, + loss_masks, + attention_mask, + block_spans, + rng, + task=task) + return data + + def split_samples(self, samples, rng): + target_length = rng.randrange(32, self.max_seq_length - 1) + num_splits = (self.max_seq_length - 1) // target_length + new_samples = [] + cls_id = self.tokenizer.get_command('ENC').Id + eos_id = self.tokenizer.get_command('eos').Id + for sample in samples: + tokens, loss_masks = sample['text'][1:], sample['loss_mask'][1:] + for _ in range(num_splits): + if target_length >= len(tokens): + new_tokens, new_loss_masks = tokens, loss_masks + else: + random_start = rng.randrange(0, + len(tokens) - target_length) + while random_start > 0 and ( + tokens[random_start] == eos_id or # noqa + not (self.contains_sentence_end( # noqa + tokens[random_start - 1]) or # noqa + tokens[random_start - 1] == eos_id)): # noqa + random_start -= 1 + random_end = random_start + target_length + while random_end > random_start and not ( + self.contains_sentence_end(tokens[random_end - 1]) + or tokens[random_end - 1] == eos_id): + random_end -= 1 + if random_end - random_start < target_length // 2: + random_end = random_start + target_length + new_tokens, new_loss_masks = tokens[ + random_start:random_end], loss_masks[ + random_start:random_end] + new_tokens = np.concatenate(([cls_id], new_tokens)) + new_loss_masks = np.concatenate(([0], new_loss_masks)) + new_samples.append({ + 'text': new_tokens, + 'loss_mask': new_loss_masks + }) + return new_samples + + def construct_blocks(self, samples): + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None: + worker_id, num_workers = worker_info.id, worker_info.num_workers + else: + worker_id, num_workers = 0, 1 + rng = random.Random((self.count * num_workers + worker_id) + * self.world_size + self.rank) + self.count += 1 + token_batch, target_batch, loss_mask_batch, position_id_batch = [], [], [], [] + source_batch, target_batch = [], [] + if rng.random() < self.short_seq_prob: + samples = self.split_samples(samples, rng) + rand = rng.random() + single_span = rand < self.single_span_prob + rand = 0.0 if single_span else rng.random() + attention_mask = [] + if rand < self.bert_prob: + mode = 'bert' + for sample in samples: + if single_span: + masked_lengths = [ + rng.choices( + range(1, + len(self.block_length_distribution) + 1), + weights=self.block_length_distribution)[0] + ] + masked_count = masked_lengths[0] + else: + masked_lengths, masked_count = [], 0 + while masked_count < int( + self.bert_ratio * len(sample['text'])): + block_length = rng.choices( + range(1, + len(self.block_length_distribution) + 1), + weights=self.block_length_distribution)[0] + masked_lengths.append(block_length) + masked_count += block_length + if self.masked_lm: + sep = len(sample['text']) + else: + sep = len( + sample['text']) - masked_count + len(masked_lengths) + data = self.generate_blank_data( + sample, masked_lengths, sep, rng, task='bert') + if data is not None: + if self.encoder_decoder: + source_tokens, target_tokens, loss_masks = data + source_batch.append(source_tokens) + target_batch.append(target_tokens) + loss_mask_batch.append(loss_masks) + else: + tokens, targets, loss_masks, position_ids = data + token_batch.append(tokens) + target_batch.append(targets) + loss_mask_batch.append(loss_masks) + position_id_batch.append(position_ids) + attention_mask.append(sep) + + elif rand < self.bert_prob + self.gap_sentence_prob: + mode = 'sentence' + for sample in samples: + tokens, loss_masks = sample['text'], sample['loss_mask'] + sentence_spans = [] + last_index = 1 if tokens[0] == self.tokenizer.get_command( + 'ENC').Id else 0 + for i in range(len(tokens)): + if self.contains_sentence_end(tokens[i]): + if last_index < i + 1: + sentence_spans.append((last_index, i + 1)) + last_index = i + 1 + elif tokens[i] == self.tokenizer.get_command('eos').Id: + last_index = i + 1 + if last_index < len(tokens): + sentence_spans.append((last_index, len(tokens))) + if not sentence_spans and torch.distributed.get_rank() == 0: + try: + print(self.tokenizer.DecodeIds(tokens[1:])) + except IndexError: + print(tokens[1:]) + rng.shuffle(sentence_spans) + block_spans, block_length = [], 0 + for start, end in sentence_spans: + block_spans.append((start, end)) + block_length += end - start + if block_length >= int( + self.gap_sentence_ratio * len(tokens)): + break + data = self.make_block_data( + tokens, + loss_masks, + None, + block_spans, + rng, + task='gap_sentence') + tokens, targets, loss_masks, position_ids, sep = data + token_batch.append(tokens) + target_batch.append(targets) + loss_mask_batch.append(loss_masks) + position_id_batch.append(position_ids) + attention_mask.append(sep) + else: + # start_indices = [index_in_list(sample['loss_mask'], 1) for sample in samples] + # end_indices = [rindex(sample['loss_mask'], 1) for sample in samples] + # start_index, end_index = max(start_indices), min(end_indices) - self.min_generation_length + # if end_index < start_index + 1: + # end_index = start_index + 1 + # division = rng.randrange(start_index, end_index) + mode = 'gpt' + max_generation_length = rng.randint( + int(self.gpt_min_ratio + * min(map(lambda x: len(x['text']), samples))), + max(map(lambda x: len(x['text']), samples)) - 2) + for sample in samples: + generation_length = min(max_generation_length, + len(sample['text']) - 2) + attention_mask.append( + len(sample['text']) - generation_length + 1) + multiple_doc = index_in_list( + sample['text'], + self.tokenizer.get_command('eos').Id) not in [ + -1, len(sample['text']) - 1 + ] # noqa + if multiple_doc or rng.random() < self.infill_prob: + division = len(sample['text']) - generation_length + tokens, loss_masks = sample['text'], sample['loss_mask'] + source_tokens, target_tokens = tokens[:division], tokens[ + division:] + target_masks = loss_masks[division:] + tokens = np.concatenate((source_tokens, [ + self.generation_mask, + self.tokenizer.get_command('sop').Id + ], target_tokens[:-1])) + targets = np.concatenate( + (source_tokens, [self.generation_mask], target_tokens)) + loss_masks = np.concatenate( + (np.zeros(len(source_tokens) + 1, + dtype=int), target_masks)) + token_batch.append(tokens) + target_batch.append(targets) + loss_mask_batch.append(loss_masks) + position_ids = np.arange( + len(source_tokens) + len(target_tokens) + 1, dtype=int) + position_ids[len(source_tokens) + 1:] = len(source_tokens) + if self.block_position_encoding: + block_position_ids = np.concatenate( + (np.zeros(len(source_tokens), dtype=int), + np.arange(len(target_tokens) + 1, dtype=int))) + else: + block_position_ids = np.concatenate( + (np.zeros(len(source_tokens) + 1, dtype=int), + np.ones(len(target_tokens) + 1, dtype=int))) + position_id_batch.append( + np.stack([position_ids, block_position_ids], axis=0)) + else: + tokens, targets, loss_masks, position_ids = self.generate_blank_data( + sample, [generation_length], + attention_mask[-1], + rng, + task='generation') + token_batch.append(tokens) + target_batch.append(targets) + loss_mask_batch.append(loss_masks) + position_id_batch.append(position_ids) + if tokens is None: + print(sample, generation_length, multiple_doc) + if self.encoder_decoder: + return { + 'text': torch.tensor(source_batch, dtype=torch.long), + 'target': torch.tensor(target_batch, dtype=torch.long), + 'loss_mask': torch.tensor(loss_mask_batch, dtype=torch.long) + } + else: + token_batch, target_batch, loss_mask_batch, position_id_batch = self.pad_batch( + token_batch, target_batch, loss_mask_batch, position_id_batch) + return { + 'text': torch.tensor(token_batch, dtype=torch.long), + 'target': torch.tensor(target_batch, dtype=torch.long), + 'loss_mask': torch.tensor(loss_mask_batch, dtype=torch.long), + 'position_id': + torch.tensor(position_id_batch, dtype=torch.long), + 'attention_mask': + torch.tensor(attention_mask, dtype=torch.long), + 'mode': mode + } + + @staticmethod + def pad_batch(token_batch, target_batch, loss_mask_batch, + position_id_batch): + seq_lengths = list(map(len, token_batch)) + if seq_lengths.count(seq_lengths[0]) != len(seq_lengths): + max_length = max(seq_lengths) + token_batch = [ + np.concatenate( + (tokens, np.zeros(max_length - len(tokens), dtype=int))) + for tokens in token_batch + ] + target_batch = [ + np.concatenate( + (targets, np.zeros(max_length - len(targets), dtype=int))) + for targets in target_batch + ] + loss_mask_batch = [ + np.concatenate( + (loss_masks, + np.zeros(max_length - len(loss_masks), dtype=int))) + for loss_masks in loss_mask_batch + ] + position_id_batch = [ + np.concatenate( + (position_ids, + np.zeros( + (2, max_length - position_ids.shape[1]), dtype=int)), + axis=1) for position_ids in position_id_batch + ] + return token_batch, target_batch, loss_mask_batch, position_id_batch diff --git a/runtime/Lib/site_packages/models/nlp/mglm/configure_data.py b/runtime/Lib/site_packages/models/nlp/mglm/configure_data.py new file mode 100644 index 0000000000000000000000000000000000000000..2761fec17daa7b19dec9d92af5441cae67a65c4f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/configure_data.py @@ -0,0 +1,513 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""parses arguments and preps data loader""" + +import copy +import os +import random +from bisect import bisect_right +from itertools import accumulate + +import numpy as np +import torch +import torch.utils.data +from megatron_util import mpu, print_rank_0 + +from . import data_utils +from .blocklm_utils import ConstructBlockStrategy +from .data_utils.tokenization import make_tokenizer + + +class MultiTaskDataset(torch.utils.data.Dataset): + + def __init__(self, + tasks, + datasets, + reweight=True, + temperature=0.8, + max_limit=200000): + super(MultiTaskDataset, self).__init__() + self.tasks = tasks + self.datasets = datasets + self.reweight = reweight + self.temperature = temperature + self.lens = [len(dataset) for dataset in datasets] + self.weights = np.array( + [min(length, max_limit)**temperature for length in self.lens]) + self.total_len = sum(self.lens) + self.cumulative_lens = list(accumulate(self.lens)) + if self.reweight: + print_rank_0(list(zip(self.tasks, self.lens, self.weights))) + else: + print_rank_0(list(zip(self.tasks, self.lens))) + self.weights /= self.weights.sum() + + def __len__(self): + return self.total_len * 1000 + + @staticmethod + def pet_wrapper(data): + text = data['text'] + loss_mask = data['logit_mask'] + target = data['target'] + attention_mask = data['mask'] + position_id = data['position'] + label = data['label'] + if len(text.shape) == 2: + text = text[label] + loss_mask = loss_mask[label] + target = target[label] + attention_mask = attention_mask[label] + position_id = position_id[label] + else: + target = target[label] + if not target.shape: + target = target.repeat(len(text)) + return { + 'text': text, + 'target': target, + 'loss_mask': loss_mask, + 'position_id': position_id, + 'attention_mask': attention_mask + } + + def __getitem__(self, idx): + if self.reweight: + rng = random.Random(idx) + rng = np.random.RandomState( + seed=[rng.randint(0, 2**32 - 1) for _ in range(16)]) + dataset_idx = rng.choice( + np.arange(len(self.datasets)), p=self.weights) + dataset = self.datasets[dataset_idx] + sample_idx = rng.choice(np.arange(len(dataset))) + item = self.datasets[dataset_idx][sample_idx] + else: + dataset_idx = bisect_right(self.cumulative_lens, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_lens[dataset_idx - 1] + item = self.datasets[dataset_idx][sample_idx] + item = self.pet_wrapper(item) + return item + + +class DataConfig: + + def __init__(self, defaults=None): + super(DataConfig, self).__init__() + if defaults is None: + defaults = {} + self.defaults = defaults + + def apply(self, args, tokenizer): + if torch.distributed.get_rank() == 0: + print('configuring data') + self.apply_defaults(args) + return make_loaders(args, tokenizer) + + def set_defaults(self, **kwargs): + for k, v in kwargs.items(): + self.defaults[k] = v + + def apply_defaults(self, args): + for k, v in self.defaults.items(): + k = k.replace('-', '_') + if not hasattr(args, k): + setattr(args, k, v) + + +def prepare_tokenizer(args): + add_sentinel_token = 0 + if args.sentinel_token: + add_sentinel_token = args.max_position_embeddings + tokenizer = make_tokenizer( + args.tokenizer_type, + None, + args.tokenizer_path, + args.vocab_size, + args.tokenizer_model_type, + add_block_symbols=args.block_lm, + cache_dir=args.cache_dir, + add_sentinel_token=add_sentinel_token, + add_task_mask=args.task_mask, + add_decoder_mask=args.block_mask_prob > 0.0 + or args.context_mask_ratio > 0.0) + if mpu.get_model_parallel_rank() == 0: + num_tokens = tokenizer.num_tokens + eod_token = tokenizer.get_command('eos').Id + assert eod_token == tokenizer.get_command('pad').Id + before = num_tokens + after = before + multiple = args.make_vocab_size_divisible_by + while (after % multiple) != 0: + after += 1 + print_rank_0('> padded vocab (size: {}) with {} dummy ' + 'tokens (new size: {})'.format(before, after - before, + after)) + print_rank_0('> found end-of-document token: {}'.format(eod_token)) + token_counts = torch.cuda.LongTensor([after, eod_token]) + else: + token_counts = torch.cuda.LongTensor([0, 0]) + # Broadcast num tokens. + torch.distributed.broadcast( + token_counts, + mpu.get_model_parallel_src_rank(), + group=mpu.get_model_parallel_group()) + num_tokens = token_counts[0].item() + eod_token = token_counts[1].item() + args.vocab_size, args.eod_token = num_tokens, eod_token + return tokenizer + + +def make_data_loader(dataset, + tokenizer, + batch_size, + num_iters, + args, + shuffle=False, + block_collate=False): + world_size = torch.distributed.get_world_size( + group=mpu.get_data_parallel_group()) + rank = torch.distributed.get_rank(group=mpu.get_data_parallel_group()) + if args.loader_scatter is not None: + rank = rank // args.loader_scatter + world_size = world_size // args.loader_scatter + batch_size = batch_size // args.loader_scatter + distributed = world_size > 1 + if args.transformer_xl: + batch_sampler = data_utils.samplers.DistributedSequentialSampler( + len(dataset), num_iters, batch_size, rank, world_size) + else: + if shuffle: + sampler = data_utils.samplers.RandomSampler( + dataset, + replacement=True, + num_samples=batch_size * args.train_iters + * args.gradient_accumulation_steps) + else: + sampler = torch.utils.data.SequentialSampler(dataset) + drop_last = distributed + # the GPUs in the same model parallel group receive the same data + if distributed: + batch_sampler = data_utils.samplers.DistributedBatchSampler( + sampler, + batch_size, + drop_last, + rank, + world_size, + gradient_accumulation_steps=args.gradient_accumulation_steps) + else: + batch_sampler = torch.utils.data.BatchSampler( + sampler, batch_size, drop_last) + collate_fn = None + if block_collate: + collate_fn = ConstructBlockStrategy( + args, + tokenizer, + args.seq_length, + bert_prob=args.bert_prob, + gap_sentence_prob=args.gap_sentence_prob, + gap_sentence_ratio=args.gap_sentence_ratio, + gpt_infill_prob=args.gpt_infill_prob, + average_block_length=args.avg_block_length, + gpt_min_ratio=args.gpt_min_ratio, + block_mask_prob=args.block_mask_prob, + context_mask_ratio=args.context_mask_ratio, + short_seq_prob=args.short_seq_prob, + single_span_prob=args.single_span_prob, + shuffle_blocks=not args.no_shuffle_block, + block_position_encoding=not args.no_block_position, + sentinel_token=args.sentinel_token, + encoder_decoder=args.encoder_decoder, + task_mask=args.task_mask, + random_position=args.random_position, + masked_lm=args.masked_lm).construct_blocks + data_loader = torch.utils.data.DataLoader( + dataset, + batch_sampler=batch_sampler, + num_workers=args.num_workers, + pin_memory=True, + collate_fn=collate_fn) + + return data_loader + + +def make_tfrecord_loaders(args): + """Load train/val/test dataset from shuffled TFRecords""" + + import data_utils.tf_dl + data_set_args = { + 'batch_size': args.batch_size, + 'max_seq_len': args.seq_length, + 'max_preds_per_seq': args.max_preds_per_seq, + 'train': True, + 'num_workers': max(args.num_workers, 1), + 'seed': args.seed + args.rank + 1, + 'threaded_dl': args.num_workers > 0 + } + train = data_utils.tf_dl.TFRecordDataLoader(args.train_data, + **data_set_args) + data_set_args['train'] = False + if args.eval_seq_length is not None: + data_set_args['max_seq_len'] = args.eval_seq_length + if args.eval_max_preds_per_seq is not None: + data_set_args['max_preds_per_seq'] = args.eval_max_preds_per_seq + valid = None + if args.valid_data is not None: + valid = data_utils.tf_dl.TFRecordDataLoader(args.valid_data, + **data_set_args) + test = None + if args.test_data is not None: + test = data_utils.tf_dl.TFRecordDataLoader(args.test_data, + **data_set_args) + tokenizer = data_utils.make_tokenizer( + args.tokenizer_type, + train, + args.tokenizer_path, + args.vocab_size, + args.tokenizer_model_type, + cache_dir=args.cache_dir) + + return (train, valid, test), tokenizer + + +def make_loaders(args, tokenizer): + """makes training/val/test""" + + if args.use_tfrecords: + return make_tfrecord_loaders(args) + world_size = torch.distributed.get_world_size( + group=mpu.get_data_parallel_group()) + if args.loader_scatter is not None: + assert world_size % args.loader_scatter == 0 + batch_size = args.batch_size * world_size + eval_batch_size = batch_size + if args.eval_batch_size is not None: + eval_batch_size = args.eval_batch_size * world_size + seq_length = args.seq_length + if seq_length < 0: + seq_length = seq_length * world_size + eval_seq_length = args.eval_seq_length + if eval_seq_length is not None and eval_seq_length < 0: + eval_seq_length = eval_seq_length * world_size + split = get_split(args) + data_set_args = { + 'path': args.train_data, + 'seq_length': seq_length, + 'mem_length': args.mem_length, + 'delim': args.delim, + 'text_key': args.text_key, + 'label_key': 'label', + 'ds_type': args.data_set_type, + 'split': split, + 'loose': args.loose_json, + 'max_preds_per_seq': args.max_preds_per_seq, + 'presplit_sentences': args.presplit_sentences, + 'sample_one_document': args.sample_one_document, + 'filter_english': args.filter_english, + 'pre_tokenize': not args.no_pre_tokenize, + 'tokenizer': tokenizer, + 'save_splits': args.save_splits, + 'load_splits': args.load_splits, + 'save_test_data': args.save_test_data, + 'no_lazy_loader': args.no_lazy_loader, + 'loader_scatter': args.loader_scatter, + 'data_parallel_rank': mpu.get_data_parallel_rank(), + 'non_sentence_start': args.non_sentence_start, + 'half_lazy_loader': args.half_lazy_loader + } + + eval_set_args = copy.copy(data_set_args) + eval_set_args['split'] = [1.] + # if optional eval args were set then replace their + # equivalent values in the arg dict + if eval_seq_length: + eval_set_args['seq_length'] = eval_seq_length + if args.eval_max_preds_per_seq: + eval_set_args['max_preds_per_seq'] = args.eval_max_preds_per_seq + if args.eval_text_key is not None: + eval_set_args['text_key'] = args.eval_text_key + + # make datasets splits and tokenizer + train, valid, test = None, None, None + + if args.train_data is not None: + train = data_utils.make_dataset(**data_set_args) + if data_utils.should_split(split): + train, valid, test = train + eval_set_args['tokenizer'] = tokenizer + + # make training and val dataset if necessary + if valid is None and args.valid_data is not None: + eval_set_args['path'] = args.valid_data + valid = data_utils.make_dataset(**eval_set_args) + eval_set_args['tokenizer'] = tokenizer + if test is None and args.test_data is not None: + eval_set_args['path'] = args.test_data + test = data_utils.make_dataset(**eval_set_args) + + # wrap datasets with data loader + use_block = args.block_lm or args.encoder_decoder + + if train is not None and args.batch_size > 0: + train = make_data_loader( + train, + tokenizer, + batch_size, + args.train_iters, + args, + shuffle=args.shuffle, + block_collate=use_block) + args.do_train = True + else: + args.do_train = False + eval_batch_size = eval_batch_size if eval_batch_size != 0 else batch_size + if valid is not None: + valid = make_data_loader( + valid, + tokenizer, + eval_batch_size, + args.train_iters, + args, + shuffle=args.shuffle, + block_collate=use_block) + args.do_valid = True + else: + args.do_valid = False + if test is not None: + test = make_data_loader( + test, + tokenizer, + eval_batch_size, + len(test) // eval_batch_size + 1, + args, + shuffle=args.shuffle, + block_collate=use_block) + args.do_test = True + else: + args.do_test = False + + return train, valid, test + + +def build_multi_task_dataset(args, tokenizer): + task_dirs = { + 'mnli': 'MNLI', + 'cola': 'CoLA', + 'mrpc': 'MRPC', + 'qnli': 'QNLI', + 'qqp': 'QQP', + 'sst2': 'SST-2', + 'agnews': 'Agnews', + 'yelp-polarity': 'yelp_review_polarity_csv', + 'yelp-full': 'yelp_review_full_csv', + 'yahoo': 'Yahoo', + 'squad': 'SQuAD', + 'race': 'RACE' + } + train, valid = None, None + if mpu.get_model_parallel_rank() == 0: + multi_seq_length = args.seq_length + if args.multi_seq_length is not None: + multi_seq_length = args.multi_seq_length + train_datasets, valid_datasets = [], [] + for task in args.multi_task_data: + task = task.lower() + data_dir = os.path.join(args.data_dir, task_dirs[task]) + train_datasets.append( + SuperGlueDataset( + args, + task, + data_dir, + multi_seq_length, + 'train', + tokenizer, + pattern_ensemble=True)) + valid_datasets.append( + SuperGlueDataset( + args, + task, + data_dir, + multi_seq_length, + 'dev', + tokenizer, + pattern_ensemble=True)) + train = MultiTaskDataset(args.multi_task_data, train_datasets) + valid = MultiTaskDataset(args.multi_task_data, valid_datasets) + world_size = torch.distributed.get_world_size( + group=mpu.get_data_parallel_group()) + multi_batch_size = args.batch_size * world_size + if args.multi_batch_size is not None: + multi_batch_size = args.multi_batch_size * world_size + train = make_data_loader( + train, + tokenizer, + multi_batch_size, + args.train_iters, + args, + shuffle=True) + valid = make_data_loader( + valid, + tokenizer, + multi_batch_size, + args.train_iters, + args, + shuffle=True) + return train, valid + + +def get_split(args): + """ + Get dataset splits from comma separated string list + """ + splits = [] + if args.split.find(',') != -1: + splits = [float(s) for s in args.split.split(',')] + elif args.split.find('/') != -1: + splits = [float(s) for s in args.split.split('/')] + else: + splits = [float(args.split)] + split_total = sum(splits) + if split_total < 1.: + splits.append(1 - split_total) + while len(splits) < 3: + splits.append(0.) + splits = splits[:3] + if args.valid_data is not None: + splits[1] = 0. + if args.test_data is not None: + splits[2] = 0. + final_sum = sum(splits) + return [s / final_sum for s in splits] + + +def configure_data(): + """add cmdline flags for configuring datasets""" + # These are options that are used by data_utils, but are either + # deprecated or not meant to be exposed to the command line user. + # These options are intneded to be set in code by specific scripts. + defaults = { + 'world_size': 1, + 'rank': -1, + 'persist_state': 0, + 'lazy': False, + 'transpose': False, + 'data_set_type': 'supervised', + 'seq_length': 256, + 'eval_seq_length': 256, + 'samples_per_shard': 100 + } + + return DataConfig(defaults=defaults) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/__init__.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67d465c5c2c8e66f8b6d8f8f1ec612bfe7cf8975 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/__init__.py @@ -0,0 +1,341 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""utils for creating datasets""" +import math +import os +import random +import time + +import torch + +from . import corpora +from .datasets import (BertSentencepairDataset, BlockDataset, ConcatDataset, + GPT2Dataset, ShuffleDataset, SplitDataset, XLDataset, + split_ds) +from .lazy_loader import (LazyLoader, LazyWriter, exists_lazy, exists_scatter, + get_scatter_path) +from .samplers import DistributedBatchSampler +from .tokenization import (BertWordPieceTokenizer, CharacterLevelTokenizer, + CommandToken, GPT2BPETokenizer, Tokenization, + Tokenizer, make_tokenizer) + +TRAIN_DATA = 0 +VAL_DATA = 1 +TEST_DATA = 2 + + +def should_split(split): + """ + given split proportions checks if should split + Examples: + >>> should_split([10,0,0]) + >>> False + >>> should_split([1,.1,.2]) + >>> True + """ + return max(split) / sum(split) != 1. + + +def get_ext(path): + """gets path extension""" + return os.path.splitext(path)[1] + + +def get_dataset(name, + tokenizer, + pre_tokenize, + data_parallel_rank, + loader_scatter=None, + no_lazy_loader=False, + half_lazy_loader=False): + """gets dataset object based on keyword args and file at `path`""" + global_rank = torch.distributed.get_rank() + if not supported_corpus(name): + raise NotImplementedError('dataset %s is not supported' % name) + dataset = corpora.NAMED_CORPORA[name] + path = dataset.PATH + if issubclass(dataset, corpora.PromptReader): + if not (exists_lazy(path, data_type='prompt') + and exists_lazy(path, data_type='text')) and not ( + loader_scatter is not None and exists_scatter( + path, data_type='prompt', scatter_num=loader_scatter) + and exists_scatter( + path, data_type='text', scatter_num=loader_scatter)): + # create cached version of dataset for lazy loading if it doesn't exist + if global_rank == 0: + print(f'Creating lazy loader for dataset {name}') + prompt_writer = LazyWriter( + path, data_type='prompt', is_array=pre_tokenize) + text_writer = LazyWriter( + path, data_type='text', is_array=pre_tokenize) + writers = {'prompt': prompt_writer, 'text': text_writer} + reader = dataset( + writers=writers, + tokenizer=tokenizer, + tokenize=pre_tokenize) + reader.process() + prompt_writer.close() + text_writer.close() + else: + while not os.path.exists( + LazyWriter.get_len_path(path, data_type='prompt')): + time.sleep(1) + map_fn = (lambda x: x.tolist()) if pre_tokenize else None + if loader_scatter is not None: + if not (exists_scatter( + path, data_type='prompt', scatter_num=loader_scatter) + and exists_scatter( + path, data_type='text', scatter_num=loader_scatter)): + if global_rank == 0: + print(f'Creating scatter loader for dataset {name}') + prompts = LazyLoader( + path, + data_type='prompt', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize) + texts = LazyLoader( + path, + data_type='text', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize) + indices = list(range(len(texts))) + random.shuffle(indices) + segment_length = (len(indices) - 1) // loader_scatter + 1 + for i in range(loader_scatter): + scatter_path = get_scatter_path(path, scatter_rank=i) + prompt_writer = LazyWriter( + scatter_path, + data_type='prompt', + is_array=pre_tokenize) + text_writer = LazyWriter( + scatter_path, + data_type='text', + is_array=pre_tokenize) + for idx in indices[i * segment_length:(i + 1) + * segment_length]: + prompt_writer.write(prompts[idx]) + text_writer.write(texts[idx]) + prompt_writer.close() + text_writer.close() + else: + while not (exists_scatter( + path, data_type='prompt', + scatter_num=loader_scatter) and exists_scatter( + path, + data_type='text', + scatter_num=loader_scatter)): + time.sleep(1) + scatter_path = get_scatter_path( + path, scatter_rank=data_parallel_rank % loader_scatter) + print(f'Rank {global_rank} is using scatter from {scatter_path}') + prompts = LazyLoader( + scatter_path, + data_type='prompt', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize, + load_memory=no_lazy_loader, + half_load=half_lazy_loader) + texts = LazyLoader( + scatter_path, + data_type='text', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize, + load_memory=no_lazy_loader, + half_load=half_lazy_loader) + else: + prompts = LazyLoader( + path, + data_type='prompt', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize, + load_memory=no_lazy_loader, + half_load=half_lazy_loader) + texts = LazyLoader( + path, + data_type='text', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize, + load_memory=no_lazy_loader, + half_load=half_lazy_loader) + text = corpora.PromptDataset( + prompt_loader=prompts, + text_loader=texts, + tokenizer=tokenizer, + to_tokenize=not pre_tokenize) + if loader_scatter is None: + if global_rank == 0: + print(f'Create dataset {name} with {len(text)} documents') + for i in range(10): + rand_id = i if i < 5 else random.randrange(len(text)) + sample_tokens = text[rand_id]['tokens'][:1024] + print(sample_tokens) + print(tokenizer.DecodeIds(sample_tokens).encode('utf-8')) + else: + for scatter_id in range(loader_scatter): + if data_parallel_rank % loader_scatter == scatter_id and data_parallel_rank // loader_scatter == 0: + print( + f'Create dataset {name} at scatter {scatter_id} with {len(text)} documents' + ) + for i in range(10): + sample_tokens = text[i]['tokens'][:1024] + print(sample_tokens) + print(tokenizer.DecodeIds(sample_tokens)) + torch.distributed.barrier() + return text + elif issubclass(dataset, corpora.KeyReader): + if not (exists_lazy(path, data_type='text') + and exists_lazy(path, data_type='mask')): + # create cached version of dataset for lazy loading if it doesn't exist + if global_rank == 0: + text_writer = LazyWriter( + path, data_type='text', is_array=pre_tokenize) + mask_writer = LazyWriter(path, data_type='mask', is_array=True) + writers = {'mask': mask_writer, 'text': text_writer} + dataset( + writers=writers, + tokenizer=tokenizer, + tokenize=pre_tokenize) + mask_writer.close() + text_writer.close() + else: + while not os.path.exists( + LazyWriter.get_len_path(path, data_type='mask')): + time.sleep(1) + map_fn = (lambda x: x.tolist()) if pre_tokenize else None + masks = LazyLoader( + path, data_type='mask', map_fn=map_fn, mem_map=True, is_array=True) + texts = LazyLoader( + path, + data_type='text', + map_fn=map_fn, + mem_map=True, + is_array=pre_tokenize) + text = corpora.KeyDataset( + mask_loader=masks, + text_loader=texts, + tokenizer=tokenizer, + to_tokenize=not pre_tokenize) + return text + + +def supported_corpus(corpus_name): + """checks if corpus name is defined in `corpora.py`""" + return corpus_name in corpora.NAMED_CORPORA + + +def make_dataset(path, + seq_length, + mem_length, + shuffle=True, + split=None, + tokenizer=None, + sample_one_document=False, + pre_tokenize=False, + ds_type='', + save_splits=None, + load_splits=None, + save_test_data=None, + no_lazy_loader=False, + loader_scatter=None, + data_parallel_rank=None, + filter_english=False, + non_sentence_start=0.0, + half_lazy_loader=False, + **kwargs): + """function to create datasets+tokenizers for common options""" + if split is None: + split = [1.] + + # get one or multiple datasets and concatenate + if isinstance(path, str): + ds = get_dataset( + path, + tokenizer=tokenizer, + pre_tokenize=pre_tokenize, + no_lazy_loader=no_lazy_loader, + loader_scatter=loader_scatter, + data_parallel_rank=data_parallel_rank, + half_lazy_loader=half_lazy_loader) + else: + ds = [ + get_dataset( + p, + tokenizer=tokenizer, + pre_tokenize=pre_tokenize, + no_lazy_loader=no_lazy_loader, + loader_scatter=loader_scatter, + data_parallel_rank=data_parallel_rank, + half_lazy_loader=half_lazy_loader) for p in path + ] + ds = ConcatDataset(ds) + + # Split dataset into train/val/test (and wrap bert dataset) + def wrap_dataset(dataset): + if ds_type.lower() == 'bert': + presplit_sentences = kwargs[ + 'presplit_sentences'] if 'presplit_sentences' in kwargs else False + dataset = BertSentencepairDataset( + dataset, + max_seq_len=seq_length, + presplit_sentences=presplit_sentences) + elif ds_type.lower() == 'gpt-xl': + assert pre_tokenize + dataset = XLDataset( + dataset, + tokenizer, + max_seq_len=seq_length, + mem_len=mem_length, + sample_across_doc=not sample_one_document) + elif ds_type.lower() == 'gpt2': + dataset = GPT2Dataset( + dataset, + tokenizer, + max_seq_len=seq_length, + sample_across_doc=not sample_one_document) + elif ds_type.lower() == 'block': + dataset = BlockDataset( + dataset, + tokenizer, + max_seq_len=seq_length, + sample_across_doc=not sample_one_document, + filter_english=filter_english, + non_sentence_start=non_sentence_start) + return dataset + + if should_split(split): + ds = split_ds( + ds, + split, + shuffle=shuffle, + save_splits=save_splits, + load_splits=load_splits) + if save_test_data is not None and torch.distributed.get_rank() == 0: + test_ds = ds[-1] + with open(save_test_data, 'w', encoding='utf-8') as output: + for data in test_ds: + text = data['tokens'] + text = tokenizer.DecodeIds(text) + output.write(text) + output.write('\n') + print(f'Write test data to {save_test_data}') + ds = [wrap_dataset(d) if d is not None else None for d in ds] + else: + ds = wrap_dataset(ds) + return ds diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/corpora.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/corpora.py new file mode 100644 index 0000000000000000000000000000000000000000..cf756c0aaca8065c532350081cd56847f3b97ab6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/corpora.py @@ -0,0 +1,583 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""several datasets with preset arguments""" +import os +import random +from collections import defaultdict +from multiprocessing import Process, Queue +from queue import Empty + +import json +import tqdm +from torch.utils import data + +from modelscope.models.nlp.mglm.utils import print_rank_0 +from .datasets import csv_dataset, json_dataset +from .lazy_loader import LazyLoader + +NUM_PROCESSES = 100 + + +def punctuation_standardization(string: str): + punctuation_dict = { + '\u201c': "\"", + '\u201d': "\"", + '\u2019': "'", + '\u2018': "'", + '\u2013': '-' + } + for key, value in punctuation_dict.items(): + string = string.replace(key, value) + return string + + +class KeyDataset(data.Dataset): + + def __init__(self, text_loader, mask_loader, **kwargs): + self.texts = text_loader + self.masks = mask_loader + self.is_lazy = False + if isinstance(self.texts, LazyLoader) and isinstance( + self.masks, LazyLoader): + self.text_lens = self.texts.lens + self.is_lazy = True + + def get_text_len(self, idx): + return self.text_lens[idx] + + def __getitem__(self, index): + text = self.texts[index] + mask_length = self.masks[index] + mask = [] + for i, length in enumerate(mask_length): + if i % 2 == 0: + mask += [0] * length + else: + mask += [1] * length + assert len(text) == len(mask) + return {'tokens': text, 'loss_masks': mask} + + def __len__(self): + return len(self.texts) + + +class PromptDataset(data.Dataset): + + def __init__(self, + prompt_loader, + text_loader, + tokenizer=None, + to_tokenize=False, + **kwargs): + self.prompts = prompt_loader + self.texts = text_loader + self.tokenizer = tokenizer + self.to_tokenize = to_tokenize + if isinstance(self.prompts, LazyLoader) and isinstance( + self.texts, LazyLoader): + self.prompt_lens = self.prompts.lens + self.text_lens = self.texts.lens + self.is_lazy = True + + def get_text_len(self, idx): + return self.prompt_lens[idx] + self.text_lens[idx] + + def __getitem__(self, index): + prompt = self.prompts[index] + text = self.texts[index] + if self.to_tokenize: + prompt = self.tokenizer.EncodeAsIds(prompt).tokenization + text = self.tokenizer.EncodeAsIds(text).tokenization + return { + 'tokens': prompt + text, + 'loss_masks': [0] * len(prompt) + [1] * len(text) + } + + def __len__(self): + return len(self.prompts) + + +class DataReader: + PATH = None + assert_str = None + reserve_punct = False + split_row = True + TASK_QUEUE_LIMIT = 10000000 + DONE_QUEUE_LIMIT = 10000000 + + def tokenize_worker(self, input, output, info, tokenizer, tokenize): + raise NotImplementedError + + def print_info(self, info): + pass + + def __init__(self, writers, tokenizer=None, tokenize=False, **kwargs): + print(self.PATH) + print(self.assert_str) + assert os.path.exists(self.PATH), self.assert_str + print_rank_0(f'Creating dataset from {self.PATH}') + self.tokenizer = tokenizer + self.tokenize = tokenize + self.writers = writers + + def process(self): + if os.path.isdir(self.PATH): + paths = [ + os.path.join(top, name) for top, _, names in os.walk(self.PATH) + for name in names + ] + # paths = [entry.path for entry in os.scandir(self.PATH) if + # not entry.is_dir() and not entry.name.endswith("bz2")] + else: + paths = [self.PATH] + task_queue, done_queue, info_queue = Queue( + maxsize=self.TASK_QUEUE_LIMIT), Queue( + maxsize=self.DONE_QUEUE_LIMIT), Queue() + processes = [] + for i in range(NUM_PROCESSES): + process = Process( + target=self.tokenize_worker, + args=(task_queue, done_queue, info_queue, self.tokenizer, + self.tokenize)) + process.start() + processes.append(process) + + def read_input_to_queue(): + for path in paths: + print_rank_0(f'Start reading {path}') + with open(path, encoding='utf-8') as file: + items = json.load(file) + for item in items: + task_queue.put(item) + # if self.split_row: + # for row in file: + # task_queue.put(row) + # else: + # items = json.load(file) + # for item in items["RECORDS"]: + # task_queue.put(item) + print_rank_0('Read input complete') + for i in range(len(processes)): + task_queue.put('STOP') + + process = Process(target=read_input_to_queue) + process.start() + count = len(processes) + progress_bar = tqdm.tqdm() + while True: + data = done_queue.get() + if data == 'COMPLETE': + count -= 1 + if count == 0: + break + else: + self.write_result(data, self.writers) + progress_bar.update() + progress_bar.close() + self.print_info(info_queue) + + @staticmethod + def write_result(data, writers): + raise NotImplementedError + + @staticmethod + def get_token_count(contents): + return sum(map(len, contents)) + + @classmethod + def process_sample(cls, text, tokenizer, tokenize): + if isinstance(text, str) and tokenize: + if not cls.reserve_punct: + text = punctuation_standardization(text) + text = tokenizer.EncodeAsIds(text).tokenization if text else [] + return text + + @staticmethod + def trim_field(content, max_length): + if len(content) > max_length: + content = content[:max_length] + content += '......' + return content + + def process_line(self, data, tokenizer, tokenize): + raise NotImplementedError + + +class PromptReader(DataReader): + is_json = True + + def tokenize_worker(self, input, output, info, tokenizer, tokenize): + for row in iter(input.get, 'STOP'): + if row: + if self.is_json: + row = row.rstrip() + row = json.loads(row) + prompts, texts = self.process_line(row, tokenizer, tokenize) + for prompt, text in zip(prompts, texts): + output.put((prompt, text)) + output.put('COMPLETE') + + @staticmethod + def write_result(data, writers): + prompt, text = data + writers['prompt'].write(prompt) + writers['text'].write(text) + + +class KeyReader(DataReader): + PATH = '/root/data/wikipedia/wiki-key.txt' + assert_str = 'make sure to set PATH for wikipedia data_utils/corpora.py' + + def process_line(self, data, tokenizer, tokenize): + keys, contents = data['key'], data['content'] + assert len(keys) == len(contents) + for i in range(1, len(keys)): + keys[i] = ' ' + keys[i] + contents = [' ' + content for content in contents] + keys = [tokenizer.EncodeAsIds(key).tokenization for key in keys] + contents = [ + tokenizer.EncodeAsIds(content).tokenization for content in contents + ] + summary = sum(keys, []) + summary_prefix = self.process_sample('Summary: ', tokenizer, tokenize) + summary_mask = [len(summary_prefix), len(summary)] + summary = summary_prefix + summary + text, text_mask = [], [] + for key, content in zip(keys, contents): + content = content + [tokenizer.get_command('eop').Id] + text += key + text += content + text_mask.append(len(key)) + text_mask.append(len(content)) + return (summary, summary_mask), (text, text_mask) + + def tokenize_worker(self, input, output, info, tokenizer, tokenize): + for row in iter(input.get, 'STOP'): + data = json.loads(row) + summary, content = self.process_line(data, tokenizer, tokenize) + output.put((summary, content)) + output.put('COMPLETE') + + @staticmethod + def write_result(data, writers): + summary, content = data + writers['text'].write(summary[0]) + writers['mask'].write(summary[1]) + writers['text'].write(content[0]) + writers['mask'].write(content[1]) + + +class zhihu(PromptReader): + PATH = '/dataset/fd5061f6/data/tokenize_data/zhihu.lazy' + reserve_punct = True + assert_str = 'make sure to set PATH for zhihu data_utils/corpora.py' + qtitle_prefix = '问题:' + qcontent_prefix = '问题描述:' + user_prefix = '回答用户:' + answer_prefix = ' 回答:' + + # qtitle_prefix = [] + # qcontent_prefix = [] + # user_prefix = [] + # answer_prefix = [] + + def process_line(self, data, tokenizer, tokenize): + prompts, texts = [], [] + ans_length = len(data.get('ans-content', '')) + ans_up = data.get('ans-up-num', '') + ans_up = int(ans_up) if ans_up else 0 + if ans_length > 100 or ans_up > 1000: + qtitle = data['q_title'] + qcontent = data['q-content'] + if qcontent is None: + qcontent = '' + qcontent = self.trim_field(qcontent, max_length=100) + user = data.get('user-signature', '') + prompt = self.qtitle_prefix + qtitle + self.qcontent_prefix + qcontent + self.user_prefix + user + self.answer_prefix # noqa + text = data['ans-content'] + prompt, text = self.process_sample(prompt, tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + prompts.append(prompt) + texts.append(text) + # prompt = data["q_title"] + data["q-content"] + data["user-signature"] + # text = data["ans-content"] + # prompts.append(prompt) + # texts.append(text) + return prompts, texts + + +class zhidao(PromptReader): + PATH = '/root/data/zhidao/zhidao' + reserve_punct = True + assert_str = 'make sure to set PATH for zhidao data_utils/corpora.py' + qtitle_prefix = '问题:' + qcontent_prefix = '问题描述:' + answer_prefix = '回答:' + + def process_line(self, data, tokenizer, tokenize): + if 'title' not in data: + return [], [] + prompts, texts = [], [] + qtitle = data['title'] + qcontent = data.get('content', '') + qcontent = self.trim_field(qcontent, max_length=100) + prompt = self.qtitle_prefix + qtitle + self.qcontent_prefix + qcontent + self.answer_prefix + prompt = self.process_sample(prompt, tokenizer, tokenize) + if 'best_answer' in data: + text = data['best_answer']['content'] + if len(text) > 10: + text = self.process_sample(text, tokenizer, tokenize) + prompts.append(prompt) + texts.append(text) + for answer in data.get('other_answers', []): + text = answer['content'] + if len(text) > 100: + text = self.process_sample(text, tokenizer, tokenize) + prompts.append(prompt) + texts.append(text) + return prompts, texts + + +class baike(PromptReader): + PATH = '/dataset/fd5061f6/data/tokenize_data/baike.lazy' + reserve_punct = True + assert_str = 'make sure to set PATH for baike data_utils/corpora.py' + + def process_line(self, data, tokenizer, tokenize): + prompts, texts = [], [] + text = data.get('title', '') + data.get('abstract', '') + data.get( + 'content', '') + if text: + p, t = self.process_sample('', tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + prompts.append(p) + texts.append(t) + return prompts, texts + + +class wikipedia(PromptReader): + """ + dataset for wikipedia with arguments configured for convenience + + command line usage: `--train-data wikipedia` + """ + # PATH = '/dataset/data/wiki.txt' + PATH = '/root/data/bert_data/wiki.txt' + assert_str = 'make sure to set PATH for wikipedia data_utils/corpora.py' + + def process_line(self, data, tokenizer, tokenize): + text = data['text'] + prompt, text = self.process_sample('', tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + return [prompt], [text] + + +class TestDataset(PromptReader): + PATH = '/root/data/test.json' + assert_str = 'make sure to set PATH for wikipedia data_utils/corpora.py' + + def process_line(self, data, tokenizer, tokenize): + prompt, text = data['prompt'], data['text'] + prompt, text = self.process_sample(prompt, tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + return [prompt], [text] + + +class OpenWebText(PromptReader): + PATH = '/dataset/fd5061f6/english_data/openwebtext2' + assert_str = 'make sure to set PATH for openwebtext data_utils/corpora.py' + + def __init__(self, *args, **kwargs): + import fasttext + super().__init__(*args, **kwargs) + self.model = fasttext.load_model( + '/dataset/fd5061f6/english_data/lid.176.bin') + print_rank_0('Load language detection model') + + def process_line(self, data, tokenizer, tokenize): + text = data['text'] + if len(text) > 100: + lang = self.model.predict(text.replace('\n', ''))[0][0] + if lang == '__label__en': + prompt, text = self.process_sample( + '', tokenizer, + tokenize), self.process_sample(text, tokenizer, tokenize) + return [prompt], [text] + return [], [] + + +class CCNews(PromptReader): + PATH = '/mnt/cc_news.json' + assert_str = 'make sure to set PATH for cc-news data_utils/corpora.py' + + def process_line(self, data, tokenizer, tokenize): + text = '' + title = data.get('title', None) + description = data.get('description', None) + maintext = data.get('maintext', None) + if title: + text += title.strip() + ' ' + if description and (not maintext + or not maintext.startswith(description)): + text += description.strip() + ' ' + if maintext: + text += maintext + if len(text) > 100: + prompt, text = self.process_sample('', tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + return [prompt], [text] + else: + return [], [] + + +class BertData(PromptReader): + is_json = False + PATH = '/dataset/fd5061f6/english_data/wikibook' + + def process_line(self, data, tokenizer, tokenize): + if data: + prompt, text = '', data + prompt, text = self.process_sample(prompt, tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + return [prompt], [text] + else: + return [], [] + + +class Pile(PromptReader): + is_json = True + PATH = '/mnt/train' + filtered_sources = [ + 'Github', 'StackExchange', 'DM Mathematics', 'Ubuntu IRC', 'EuroParl', + 'YoutubeSubtitles', 'Enron Emails' + ] + downsample_sources = {'PubMed Central': 0.3, 'ArXiv': 0.3, 'FreeLaw': 0.3} + + def print_info(self, info): + total_dict = defaultdict(int) + while True: + try: + source_dict = info.get(block=False) + for source, length in source_dict.items(): + total_dict[source] += length + except Empty: + break + print_rank_0(total_dict) + + def tokenize_worker(self, input, output, info, tokenizer, tokenize): + source_dict = defaultdict(int) + for row in iter(input.get, 'STOP'): + row = row.rstrip() + if row: + if self.is_json: + row = json.loads(row) + prompts, texts, source = self.process_line( + row, tokenizer, tokenize) + length = 0 + for prompt, text in zip(prompts, texts): + length += len(text) + output.put((prompt, text)) + if source: + source_dict[source] += length + output.put('COMPLETE') + info.put(source_dict) + + def process_line(self, data, tokenizer, tokenize): + source = data['meta'].get('pile_set_name', None) + text = data.get('text', None) + if source and text: + if source in self.filtered_sources: + return [], [], None + elif source in self.downsample_sources and random.random( + ) > self.downsample_sources[source]: + return [], [], None + else: + prompt, text = self.process_sample( + '', tokenizer, + tokenize), self.process_sample(text, tokenizer, tokenize) + return [prompt], [text], source + else: + return [], [], None + + +class Stories(PromptReader): + is_json = True + PATH = '/dataset/fd5061f6/english_data/stories_31G.jsonl' + + def process_line(self, data, tokenizer, tokenize): + text = data.get('text', None) + if text: + prompt, text = self.process_sample('', tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + return [prompt], [text] + else: + return [], [] + + +class BertBaseData(BertData): + PATH = '/root/data/formatted_one_article_per_line' + + +class BertLargeData(BertData): + PATH = '/dataset/c07bd62b/cognitive/zhengxiao/formatted_one_article_per_line_large' + + +class WuDaoCorpus(PromptReader): + # PATH = "/dataset/fd5061f6/chinese_data/WuDao" + PATH = '/wudao' + is_json = False + reserve_punct = True + split_row = False + + def process_line(self, item, tokenizer, tokenize): + prompts, texts = [], [] + text = '' + title = item.get('title', None) + content = item.get('content', None) + if title: + text += title.strip() + ' ' + if content: + text += content + if len(text) > 100: + prompt, text = self.process_sample('', tokenizer, + tokenize), self.process_sample( + text, tokenizer, tokenize) + prompts.append(prompt) + texts.append(text) + return prompts, texts + + +NAMED_CORPORA = { + 'wikipedia': wikipedia, + 'wikipedia-key': KeyReader, + 'openwebtext': OpenWebText, + 'zhihu': zhihu, + 'zhidao': zhidao, + 'baike': baike, + 'test': TestDataset, + 'wikibook': BertData, + 'bert-base': BertBaseData, + 'bert-large': BertLargeData, + 'cc-news': CCNews, + 'pile': Pile, + 'stories': Stories, + 'wudao': WuDaoCorpus +} diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/datasets.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..37bfbcc26ffa8377a60077500d3d505a4e6d2127 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/datasets.py @@ -0,0 +1,1244 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""dataset objects for jsons, csvs, and BERT datasets""" + +import csv +import math +import os +import random +import time +from bisect import bisect_right +from itertools import accumulate +from operator import itemgetter + +import json +import nltk +import numpy as np +import pandas as pd +import torch +import tqdm +from nltk import tokenize +from torch.utils import data + +from modelscope.models.nlp.mglm.utils import print_rank_0 +from .lazy_loader import LazyLoader, exists_lazy + + +class ShuffleDataset(data.Dataset): + + def __init__(self, ds): + self.ds = ds + self.shuffle_ids = list(range(len(self.ds))) + random.shuffle(self.shuffle_ids) + self.is_lazy = hasattr(ds, 'is_lazy') and ds.is_lazy + if self.is_lazy: + self.prompt_lens = [ + self.ds.prompt_lens[idx] for idx in self.shuffle_ids + ] + self.text_lens = [ + self.ds.text_lens[idx] for idx in self.shuffle_ids + ] + + def __getitem__(self, idx): + return self.ds[self.shuffle_ids[idx]] + + def __len__(self): + return len(self.ds) + + +class ConcatDataset(data.Dataset): + """ + Dataset to concatenate multiple datasets. + Purpose: useful to assemble different existing datasets, possibly + large-scale datasets as the concatenation operation is done in an + on-the-fly manner. + Arguments: + datasets (sequence): List of datasets to be concatenated. + """ + + @staticmethod + def cumsum(sequence): + r, s = [], 0 + for e in sequence: + l = len(e) # noqa + r.append(l + s) + s += l + return r + + def __init__(self, datasets, **kwargs): + super(ConcatDataset, self).__init__() + assert len(datasets) > 0, 'datasets should not be an empty iterable' + self.datasets = list(datasets) + self.is_lazy = sum([ + isinstance(ds, LazyLoader) + or (hasattr(ds, 'is_lazy') and ds.is_lazy) for ds in self.datasets + ]) == len(self.datasets) + self.cumulative_sizes = self.cumsum(self.datasets) + self._X = None + self._Y = None + self._lens = None + + def get_text_len(self, idx): + dataset_idx = bisect_right(self.cumulative_sizes, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] + return self.datasets[dataset_idx].get_text_len(sample_idx) + + def SetTokenizer(self, tokenizer): + for ds in self.datasets: + ds.SetTokenizer(tokenizer) + + def GetTokenizer(self): + return self.datasets[0].GetTokenizer() + + def __len__(self): + return self.cumulative_sizes[-1] + + def __getitem__(self, idx): + dataset_idx = bisect_right(self.cumulative_sizes, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] + return self.datasets[dataset_idx][sample_idx] + + @property + def lens(self): + if self._lens is None: + self._lens = [] + if self.is_lazy: + for data in self.datasets: # noqa + self._lens.extend(data.lens) + else: + for data in self.datasets: # noqa + self._lens.extend([ + len(d['text']) if isinstance(d, dict) else len(d) + for d in data + ]) + return self._lens + + @property + def X(self): + if self._X is None: + self._X = [] + for data in self.datasets: # noqa + self._X.extend(data.X) + return self._X + + @property + def Y(self): + if self._Y is None: + self._Y = [] + for data in self.datasets: # noqa + self._Y.extend(list(data.Y)) + self._Y = np.array(self._Y) + return self._Y + + +class SplitDataset(data.Dataset): + """ + Dataset wrapper to access a subset of another dataset. + Purpose: useful to index into existing datasets, possibly + large-scale datasets as the subindexing operation is done in an + on-the-fly manner. + Arguments: + ds (Dataset or array-like): List of datasets to be subindexed + split_inds (1D array-like): List of indices part of subset + """ + + def __init__(self, ds, split_inds, **kwargs): + self.split_inds = list(split_inds) + self.wrapped_data = ds + self.is_lazy = isinstance(ds, LazyLoader) or (hasattr(ds, 'is_lazy') + and ds.is_lazy) + self._X = None + self._Y = None + + def __len__(self): + return len(self.split_inds) + + def get_text_len(self, idx): + return self.wrapped_data.get_text_len(self.split_inds[idx]) + + def __getitem__(self, index): + return self.wrapped_data[self.split_inds[index]] + + def SetTokenizer(self, tokenizer): + self.wrapped_data.SetTokenizer(tokenizer) + + def GetTokenizer(self): + return self.wrapped_data.GetTokenizer() + + @property + def X(self): + if self._X is None: + self._X = itemgetter(*self.split_inds)(self.wrapped_data.X) + return self._X + + @property + def Y(self): + if self._Y is None: + self._Y = np.array( + itemgetter(*self.split_inds)(self.wrapped_data.Y)) + return self._Y + + def __iter__(self): + for idx in self.split_inds: + yield self.wrapped_data[idx] + + +def split_ds(ds, split=None, shuffle=True, save_splits=None, load_splits=None): + """ + Split a dataset into subsets given proportions of how + much to allocate per split. If a split is 0% returns None for that split. + Purpose: Useful for creating train/val/test splits + Arguments: + ds (Dataset or array-like): Data to be split. + split (1D array-like): proportions to split `ds`. `sum(splits) != 0` + shuffle (boolean): Randomly split dataset. Default: True + save_splits: save split indices to file + load_splits: load split indices from file + """ + if split is None: + split = [.8, .2, .0] + split_sum = sum(split) + if split_sum == 0: + raise Exception('Split cannot sum to 0.') + split = np.array(split) + split /= split_sum + ds_len = len(ds) + inds = np.arange(ds_len) + if shuffle: + rng = np.random.RandomState(1234) + rng.shuffle(inds) + if load_splits is not None: + inds = np.load(load_splits) + assert len(inds) == ds_len + print_rank_0(f'Load split indices from {load_splits}') + elif save_splits is not None: + if torch.distributed.get_rank() == 0: + np.save(save_splits, inds) + print(f'Save split indices to {save_splits}') + start_idx = 0 + residual_idx = 0 + rtn_ds = [None] * len(split) + for i, f in enumerate(split): + if f != 0: + proportion = ds_len * split[i] + residual_idx += proportion % 1 + split_ = int(int(proportion) + residual_idx) + split_inds = inds[start_idx:start_idx + max(split_, 1)] + rtn_ds[i] = SplitDataset(ds, split_inds) + start_idx += split_ + residual_idx %= 1 + return rtn_ds + + +class csv_dataset(data.Dataset): + """ + Class for loading datasets from csv files. + Purpose: Useful for loading data for unsupervised modeling or transfer tasks + Arguments: + path (str): Path to csv file with dataset. + tokenizer (data_utils.Tokenizer): Tokenizer to use when processing text. Default: None + preprocess_fn (callable): Callable that process a string into desired format. + delim (str): delimiter for csv. Default: ',' + binarize_sent (bool): binarize label values to 0 or 1 if they\'re on a different scale. Default: False + drop_unlabeled (bool): drop rows with unlabelled values. Always fills remaining empty + columns with -1 (regardless if rows are dropped based on value) Default: False + text_key (str): key to get text from csv. Default: 'sentence' + label_key (str): key to get label from json dictionary. Default: 'label' + Attributes: + X (list): all strings from the csv file + Y (np.ndarray): labels to train with + """ + + def __init__(self, + path, + tokenizer=None, + preprocess_fn=None, + delim=',', + binarize_sent=False, + drop_unlabeled=False, + text_key='sentence', + label_key='label', + **kwargs): + self.is_lazy = False + self.preprocess_fn = preprocess_fn + self.SetTokenizer(tokenizer) + self.path = path + self.delim = delim + self.text_key = text_key + self.label_key = label_key + self.drop_unlabeled = drop_unlabeled + + if '.tsv' in self.path: + self.delim = '\t' + + self.X = [] + self.Y = [] + try: + cols = [text_key] + if isinstance(label_key, list): + cols += label_key + else: + cols += [label_key] + data = pd.read_csv( + self.path, sep=self.delim, usecols=cols, encoding='latin-1') + except: # noqa + data = pd.read_csv( + self.path, + sep=self.delim, + usecols=[text_key], + encoding='latin-1') + + data = data.dropna(axis=0) + + self.X = data[text_key].values.tolist() + try: + self.Y = data[label_key].values + except Exception as e: # noqa + self.Y = np.ones(len(self.X)) * -1 + + if binarize_sent: + self.Y = binarize_labels(self.Y, hard=binarize_sent) + + def SetTokenizer(self, tokenizer): + if tokenizer is None: + self.using_tokenizer = False + if not hasattr(self, '_tokenizer'): + self._tokenizer = tokenizer + else: + self.using_tokenizer = True + self._tokenizer = tokenizer + + def GetTokenizer(self): + return self._tokenizer + + @property + def tokenizer(self): + if self.using_tokenizer: + return self._tokenizer + return None + + def __len__(self): + return len(self.X) + + def __getitem__(self, index): + """process+tokenize string and return string,label,and stringlen""" + x = self.X[index] + if self.tokenizer is not None: + x = self.tokenizer.EncodeAsIds(x, self.preprocess_fn) + elif self.preprocess_fn is not None: + x = self.preprocess_fn(x) + y = self.Y[index] + if isinstance(y, str): + if self.tokenizer is not None: + y = self.tokenizer.EncodeAsIds(y, self.preprocess_fn) + elif self.preprocess_fn is not None: + y = self.preprocess_fn(y) + return {'text': x, 'length': len(x), 'label': y} + + def write(self, writer_gen=None, path=None, skip_header=False): + """ + given a generator of metrics for each of the data points X_i, + write the metrics, text, and labels to a csv file + """ + if path is None: + path = self.path + '.results' + print('generating csv at ' + path) + with open(path, 'w') as csvfile: + c = csv.writer(csvfile, delimiter=self.delim) + if writer_gen is not None: + # if first item of generator is a header of what the metrics mean then write header to csv file + if not skip_header: + header = (self.label_key, ) + tuple( + next(writer_gen)) + (self.text_key, ) + c.writerow(header) + for i, row in enumerate(writer_gen): + row = (self.Y[i], ) + tuple(row) + (self.X[i], ) + c.writerow(row) + else: + c.writerow([self.label_key, self.text_key]) + for row in zip(self.Y, self.X): + c.writerow(row) + + +class json_dataset(data.Dataset): + """ + Class for loading datasets from a json dump. + Purpose: Useful for loading data for unsupervised modeling or transfer tasks + Arguments: + path (str): path to json file with dataset. + tokenizer (data_utils.Tokenizer): Tokenizer to use when processing text. Default: None + preprocess_fn (callable): callable function that process a string into desired format. + Takes string, maxlen=None, encode=None as arguments. Default: process_str + text_key (str): key to get text from json dictionary. Default: 'sentence' + label_key (str): key to get label from json dictionary. Default: 'label' + Attributes: + all_strs (list): list of all strings from the dataset + all_labels (list): list of all labels from the dataset (if they have it) + """ + + def __init__(self, + path, + tokenizer=None, + preprocess_fn=None, + binarize_sent=False, + text_key='sentence', + label_key='label', + loose_json=False, + **kwargs): + self.is_lazy = False + self.preprocess_fn = preprocess_fn + self.path = path + self.SetTokenizer(tokenizer) + self.X = [] + self.Y = [] + self.text_key = text_key + self.label_key = label_key + self.loose_json = loose_json + + for j in self.load_json_stream(self.path): + s = j[text_key] + self.X.append(s) + self.Y.append(j[label_key]) + + if binarize_sent: + self.Y = binarize_labels(self.Y, hard=binarize_sent) + + def SetTokenizer(self, tokenizer): + if tokenizer is None: + self.using_tokenizer = False + if not hasattr(self, '_tokenizer'): + self._tokenizer = tokenizer + else: + self.using_tokenizer = True + self._tokenizer = tokenizer + + def GetTokenizer(self): + return self._tokenizer + + @property + def tokenizer(self): + if self.using_tokenizer: + return self._tokenizer + return None + + def __getitem__(self, index): + """gets the index'th string from the dataset""" + x = self.X[index] + if self.tokenizer is not None: + x = self.tokenizer.EncodeAsIds(x, self.preprocess_fn) + elif self.preprocess_fn is not None: + x = self.preprocess_fn(x) + y = self.Y[index] + if isinstance(y, str): + if self.tokenizer is not None: + y = self.tokenizer.EncodeAsIds(y, self.preprocess_fn) + elif self.preprocess_fn is not None: + y = self.preprocess_fn(y) + return {'text': x, 'length': len(x), 'label': y} + + def __len__(self): + return len(self.X) + + def write(self, writer_gen=None, path=None, skip_header=False): + """ + given a generator of metrics for each of the data points X_i, + write the metrics, text, and labels to a json file + """ + if path is None: + path = self.path + '.results' + + if writer_gen is not None: + # if first item of generator is a header of what the metrics mean then write header to csv file + def gen_helper(): + keys = {} + keys[0] = self.label_key + if not skip_header: + for idx, k in enumerate(tuple(next(writer_gen))): + keys[idx + 1] = k + for i, row in enumerate(writer_gen): + if i == 0 and skip_header: + for idx, _ in enumerate(row): + keys[idx + 1] = 'metric_%d' % (idx, ) + j = {} + for idx, v in enumerate((self.Y[i], ) + tuple(row)): + k = keys[idx] + j[k] = v + yield j + else: + + def gen_helper(): + for y in self.Y: + j = {} + j[self.label_key] = y + yield j + + def out_stream(): + for i, j in enumerate(gen_helper()): + j[self.text_key] = self.X[i] + yield j + + self.save_json_stream(path, out_stream()) + + def save_json_stream(self, save_path, json_stream): + if self.loose_json: + with open(save_path, 'w') as f: + for i, j in enumerate(json_stream): + write_string = '' + if i != 0: + write_string = '\n' + write_string += json.dumps(j) + f.write(write_string) + else: + jsons = [j for j in json_stream] + json.dump(jsons, open(save_path, 'w'), separators=(',', ':')) + + def load_json_stream(self, load_path): + if not self.loose_json: + jsons = json.load(open(load_path, 'r', encoding='utf-8')) + generator = iter(jsons) + else: + + def gen_helper(): + with open(load_path, 'r', encoding='utf-8') as f: + for row in f: + yield json.loads(row) + + generator = gen_helper() + + for j in generator: + if self.label_key not in j: + j[self.label_key] = -1 + yield j + + +class XLDataset(data.Dataset): + + def __init__(self, + ds, + tokenizer, + max_seq_len=1024, + mem_len=None, + sample_across_doc=True, + **kwargs): + self.ds = ds + self.tokenizer = tokenizer + self.max_seq_len = max_seq_len + if mem_len is None: + mem_len = max_seq_len + self.mem_len = mem_len + self.sample_across_doc = sample_across_doc + self.indices, self.num_samples = None, None + if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: + self.is_lazy = True + self.init_indices() + + def init_indices(self): + if self.is_lazy: + lens = np.array( + [self.ds.get_text_len(idx) for idx in range(len(self.ds))]) + else: + lens = np.array([ + len(d['prompt']) + + len(d['text']) if isinstance(d, dict) else len(d) + for d in self.ds + ]) + self.indices = list(accumulate(lens)) + print_rank_0( + f'Dataset document count {len(lens)}, token count {self.indices[-1]}' + ) + self.num_samples = self.indices[-1] // self.max_seq_len + 1 + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + tokens, targets, loss_mask, attention_mask = self.getidx(idx) + tokens = self.pad_seq(tokens) + targets = self.pad_seq(targets) + loss_mask = self.pad_seq(loss_mask, pad_id=0) + return { + 'text': np.array(tokens), + 'target': np.array(targets), + 'loss_mask': np.array(loss_mask), + 'attention_mask': np.array(attention_mask) + } + + def getidx(self, idx): + tokens, targets, loss_masks = [], [], [] + attention_mask = np.concatenate( + (np.zeros((self.max_seq_len, self.mem_len), dtype=int), + np.ones((self.max_seq_len, self.max_seq_len), dtype=int)), + axis=1) + sample_idx = bisect_right(self.indices, idx * self.max_seq_len) + last_end = 0 if sample_idx == 0 else self.indices[sample_idx - 1] + token_offset = idx * self.max_seq_len - last_end + if token_offset != 0: + history = min(self.mem_len, token_offset) + attention_mask[:, + -self.max_seq_len - history:-self.max_seq_len] = 1 + count = 0 + while len(tokens) < self.max_seq_len and sample_idx < len(self.ds): + item = self.ds[sample_idx] + text, masks = item['tokens'], item['loss_masks'] + text = text + [self.tokenizer.get_command('eos').Id] + end = min( + len(text) - 1, token_offset + self.max_seq_len - len(tokens)) + masks = masks + [1] + if count > 0: + current = len(tokens) + attention_mask[current:, :current + self.mem_len] = 0 + tokens += text[token_offset:end] + targets += text[token_offset + 1:end + 1] + loss_masks += masks[token_offset + 1:end + 1] + count += 1 + sample_idx += 1 + token_offset = 0 + return tokens, targets, loss_masks, attention_mask + + def pad_seq(self, seq, pad_id=None): + total_tokens = self.max_seq_len + num_pad_tokens = max(0, total_tokens - len(seq)) + seq += [ + self.tokenizer.get_command('pad').Id if pad_id is None else pad_id + ] * ( + num_pad_tokens) + return seq + + +class BlockDataset(data.Dataset): + + def __init__(self, + ds, + tokenizer, + max_seq_len=1024, + sample_across_doc=True, + non_sentence_start=0.0, + filter_english=False, + **kwargs): + """ + sentence_start: the stripped article must start with a complete sentence + """ + self.ds = ds + self.ds_len = len(self.ds) + self.num_samples = 1000 * self.ds_len + self.max_seq_len = max_seq_len + self.tokenizer = tokenizer + self.sample_across_doc = sample_across_doc + self.non_sentence_start = non_sentence_start + self.filter_english = filter_english + self.weighting, self.total_len = None, None + self.is_lazy = False + if self.filter_english: + import fasttext + self.model = fasttext.load_model('/mnt/lid.176.bin') + print_rank_0('Load language detection model') + if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: + self.is_lazy = True + self.init_weighting() + + def init_weighting(self): + if self.is_lazy: + lens = np.array( + [self.ds.get_text_len(idx) for idx in range(len(self.ds))]) + else: + lens = np.array([ + len(d['text']) if isinstance(d, dict) else len(d) + for d in self.ds + ]) + self.total_len = np.sum(lens) + print_rank_0( + f'Dataset document count {len(lens)}, token count {self.total_len}, non sentence start{self.non_sentence_start}' # noqa + ) + self.weighting = list(accumulate(lens)) + + def get_weighted_samples(self, np_rng): + while True: + idx = np_rng.randint(self.total_len) + data_idx = bisect_right(self.weighting, idx) + tokens, loss_mask = self.getidx(data_idx) + if self.filter_english: + text = self.tokenizer.DecodeIds(tokens[:1024]) + lang = self.model.predict(text.replace('\n', ''))[0][0] + if lang == '__label__en': + break + else: + break + return tokens, loss_mask + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + # init rng + rng = random.Random(idx) + rng = np.random.RandomState( + seed=[rng.randint(0, 2**32 - 1) for _ in range(16)]) + + # get possibly weighted random index from dataset + tokens, loss_mask = self.get_weighted_samples(rng) + # truncate or pad tokens + num_tokens = len(tokens) + tokens_to_strip = num_tokens - self.max_seq_len + 1 + + # randomly choose a position for start + if tokens_to_strip > 0: + move_count = 0 + strip_left_tokens = rng.randint(tokens_to_strip) + if rng.random() > self.non_sentence_start: + if rng.random() < 0.5: + while move_count < self.max_seq_len // 2 and strip_left_tokens > 0 and not self.contains_sentence_end( # noqa + tokens[strip_left_tokens - 1]): # noqa + strip_left_tokens -= 1 + move_count += 1 + else: + while move_count < self.max_seq_len // 2 and strip_left_tokens < len( + tokens) and not self.contains_sentence_end( + tokens[strip_left_tokens - 1]): + strip_left_tokens += 1 + move_count += 1 + tokens = [self.tokenizer.get_command('ENC').Id + ] + tokens[strip_left_tokens:] + loss_mask = [0] + loss_mask[strip_left_tokens:] + if len(tokens) == 2 and tokens[1] == self.tokenizer.get_command( + 'eos').Id: + tokens, loss_mask = [], [] + tokens, loss_mask = self.right_strip_seq(tokens, loss_mask, + self.max_seq_len) + else: + tokens = [self.tokenizer.get_command('ENC').Id] + tokens + loss_mask = [0] + loss_mask + # Sample multiple documents + if self.sample_across_doc: + while len(tokens) < self.max_seq_len: + new_tokens, new_loss_mask = self.get_weighted_samples(rng) + new_tokens = [self.tokenizer.get_command('ENC').Id + ] + new_tokens + new_loss_mask = [0] + new_loss_mask + is_last = len(new_tokens) >= self.max_seq_len - len(tokens) + new_tokens, new_loss_mask = self.right_strip_seq( + new_tokens, new_loss_mask, + self.max_seq_len - len(tokens)) + tokens += new_tokens + loss_mask += new_loss_mask + if is_last: + break + return {'text': np.array(tokens), 'loss_mask': np.array(loss_mask)} + + def right_strip_seq(self, tokens, loss_mask, seq_length): + strip_right_tokens = len(tokens) - seq_length + if strip_right_tokens > 0: + while strip_right_tokens < len( + tokens) - 1 and not self.contains_sentence_end( + tokens[-strip_right_tokens - 1]): + strip_right_tokens += 1 + if len(tokens) - strip_right_tokens < seq_length // 2: + strip_right_tokens = len(tokens) - seq_length + tokens = tokens[:-strip_right_tokens] + loss_mask = loss_mask[:-strip_right_tokens] + return tokens, loss_mask + + def getidx(self, data_idx): + data = self.ds[data_idx] + tokens, loss_masks = data['tokens'], data['loss_masks'] + tokens = tokens + [self.tokenizer.get_command('eos').Id] + loss_masks = loss_masks + [1] + return tokens, loss_masks + + def pad_seq(self, seq, pad_id=None): + total_tokens = self.max_seq_len + num_pad_tokens = max(0, total_tokens - len(seq)) + seq += [ + self.tokenizer.get_command('pad').Id if pad_id is None else pad_id + ] * ( + num_pad_tokens) + return seq + + # TODO: rewrite this function for chinese + def contains_sentence_end(self, tok): + tok = self.tokenizer.IdToToken(tok) + if '.' in tok: + return True + if '?' in tok: + return True + if '!' in tok: + return True + if ';' in tok: + return True + if ':' in tok: + return True + if '\n' in tok: + return True + return False + + +class GPT2Dataset(data.Dataset): + + def __init__(self, + ds, + tokenizer, + max_seq_len=1024, + num_samples=None, + weighted=True, + sample_across_doc=True, + random_across_doc_sampling=True, + sentence_start=False, + **kwargs): + """ + sentence_start: the stripped article must start with a complete sentence + """ + self.ds = ds + self.ds_len = len(self.ds) + self.num_samples = num_samples + if num_samples is None: + self.num_samples = 1000 * self.ds_len + self.max_seq_len = max_seq_len + self.tokenizer = tokenizer + self.weighted = weighted + self.sample_across_doc = sample_across_doc + self.random_across_doc_sampling = random_across_doc_sampling + self.sentence_start = sentence_start + self.weighting, self.total_len = None, None + self.is_lazy = False + if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: + self.is_lazy = True + self.init_weighting() + + def init_weighting(self): + if self.weighted: + if self.is_lazy: + lens = np.array( + [self.ds.get_text_len(idx) for idx in range(len(self.ds))]) + else: + lens = np.array([ + len(d['text']) if isinstance(d, dict) else len(d) + for d in self.ds + ]) + self.total_len = np.sum(lens) + print_rank_0( + f'Dataset document count {len(lens)}, token count {self.total_len}' + ) + self.weighting = list(accumulate(lens)) + else: + self.weighting = None + + def get_weighted_samples(self, np_rng): + if self.weighting is not None: + idx = np_rng.randint(self.total_len) + return bisect_right(self.weighting, idx) + else: + return np_rng.randint(self.ds_len) + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + # init rng + rng = random.Random(idx) + rng = np.random.RandomState( + seed=[rng.randint(0, 2**32 - 1) for _ in range(16)]) + + # get possibly weighted random index from dataset + data_idx = self.get_weighted_samples(rng) + # data_idx = rng.choice(self.ds_len, p=self.weighting) + tokens, loss_mask = self.getidx(data_idx) + + # truncate or pad tokens + num_tokens = len(tokens) + tokens_to_strip = num_tokens - self.max_seq_len - 1 + + # randomly choose a position for start + if tokens_to_strip > 0: + strip_left_tokens = rng.randint(tokens_to_strip + 1) + tokens = tokens[strip_left_tokens:] + loss_mask = loss_mask[strip_left_tokens:] + # if self.sentence_start: + # token_copy = list(tokens) + # not_done = True + # while (len(token_copy) > 0) and not_done: + # tok = token_copy.pop(0) + # if self.contains_sentence_end(tok): + # tokens = token_copy + # not_done = False + strip_right_rokens = len(tokens) - self.max_seq_len - 1 + if strip_right_rokens > 0: + tokens = tokens[:-strip_right_rokens] + loss_mask = loss_mask[:-strip_right_rokens] + # Sample multiple documents + if self.sample_across_doc: + while (len(tokens) < (self.max_seq_len + 1)): + if self.random_across_doc_sampling: + data_idx = self.get_weighted_samples(rng) + else: + data_idx = (data_idx + 1) % self.ds_len + new_tokens, new_loss_mask = self.getidx(data_idx) + tokens += new_tokens + loss_mask += new_loss_mask + tokens = tokens[:(self.max_seq_len + 1)] + loss_mask = loss_mask[:(self.max_seq_len + 1)] + + tokens = self.pad_seq(tokens) + loss_mask = self.pad_seq(loss_mask, pad_id=0) + return {'text': np.array(tokens), 'loss_mask': np.array(loss_mask)} + + def getidx(self, data_idx): + data = self.ds[data_idx] + tokens, loss_masks = data['tokens'], data['loss_masks'] + tokens = tokens + [self.tokenizer.get_command('eos').Id] + loss_masks = loss_masks + [1] + return tokens, loss_masks + + def pad_seq(self, seq, pad_id=None): + total_tokens = self.max_seq_len + 1 + num_pad_tokens = max(0, total_tokens - len(seq)) + seq += [ + self.tokenizer.get_command('pad').Id if pad_id is None else pad_id + ] * ( + num_pad_tokens) + return seq + + # TODO: rewrite this function for chinese + def contains_sentence_end(self, tok): + tok = self.tokenizer.IdToToken(tok) + if '.' in tok: + return True + if '?' in tok: + return True + if '!' in tok: + return True + return False + + +class BertSentencepairDataset(data.Dataset): + """ + Dataset containing sentencepairs for BERT training. Each index corresponds to a randomly generated sentence pair. + Arguments: + ds (Dataset or array-like): data corpus to use for training + max_seq_len (int): maximum sequence length to use for a sentence pair + mask_lm_prob (float): proportion of tokens to mask for masked LM + max_preds_per_seq (int): Maximum number of masked tokens per sentence pair. Default: math.ceil(max_seq_len*mask_lm_prob/10)*10 + short_seq_prob (float): Proportion of sentence pairs purposefully shorter than max_seq_len + dataset_size (int): number of random sentencepairs in the dataset. Default: len(ds)*(len(ds)-1) + + """ # noqa + + def __init__(self, + ds, + max_seq_len=512, + mask_lm_prob=.15, + max_preds_per_seq=None, + short_seq_prob=.01, + dataset_size=None, + presplit_sentences=False, + weighted=True, + **kwargs): + self.ds = ds + self.ds_len = len(self.ds) + self.tokenizer = self.ds.GetTokenizer() + self.vocab_words = list(self.tokenizer.text_token_vocab.values()) + self.ds.SetTokenizer(None) + self.max_seq_len = max_seq_len + self.mask_lm_prob = mask_lm_prob + if max_preds_per_seq is None: + max_preds_per_seq = math.ceil(max_seq_len * mask_lm_prob / 10) * 10 + self.max_preds_per_seq = max_preds_per_seq + self.short_seq_prob = short_seq_prob + self.dataset_size = dataset_size + if self.dataset_size is None: + self.dataset_size = self.ds_len * (self.ds_len - 1) + self.presplit_sentences = presplit_sentences + if not self.presplit_sentences: + nltk.download('punkt', download_dir='./nltk') + self.weighted = weighted + self.get_weighting() + + def get_weighting(self): + if self.weighted: + if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: + lens = np.array(self.ds.lens) + else: + lens = np.array([ + len(d['text']) if isinstance(d, dict) else len(d) + for d in self.ds + ]) + self.total_len = np.sum(lens) + self.weighting = list(accumulate(lens)) + else: + self.weighting = None + + def get_weighted_samples(self, np_rng): + if self.weighting is not None: + idx = np_rng.randint(self.total_len) + return bisect_right(self.weighting, idx) + else: + return np_rng.randint(self.ds_len) + + def __len__(self): + return self.dataset_size + + def __getitem__(self, idx): + # get rng state corresponding to index (allows deterministic random pair) + rng = random.Random(idx) + np_rng = np.random.RandomState( + seed=[rng.randint(0, 2**32 - 1) for _ in range(16)]) + # get seq length + target_seq_length = self.max_seq_len + short_seq = False # noqa + if rng.random() < self.short_seq_prob: + target_seq_length = rng.randint(2, target_seq_length) + short_seq = True # noqa + + # get sentence pair and label + is_random_next = None + lena = 0 + lenb = 0 + while (is_random_next is None) or (lena < 1) or (lenb < 1): + tokensa, tokensb, is_random_next = self.create_random_sentencepair( + target_seq_length, rng, np_rng) + lena = len(tokensa[0]) + lenb = len(tokensb[0]) + + # truncate sentence pair to max_seq_len + tokensa, tokensb = self.truncate_seq_pair(tokensa, tokensb, + self.max_seq_len, rng) + # join sentence pair, mask, and pad + tokens, mask, mask_labels, pad_mask = self.create_masked_lm_predictions( + tokensa, tokensb, self.mask_lm_prob, self.max_preds_per_seq, + self.vocab_words, rng) + sample = { + 'text': np.array(tokens[0]), + 'types': np.array(tokens[1]), + 'is_random': int(is_random_next), + 'mask': np.array(mask), + 'mask_labels': np.array(mask_labels), + 'pad_mask': np.array(pad_mask) + } + return sample + + def sentence_split(self, document): + """split document into sentences""" + lines = document.split('\n') + if self.presplit_sentences: + return [line for line in lines if line] + rtn = [] + for line in lines: + if line != '': + rtn.extend(tokenize.sent_tokenize(line)) + return rtn + + def sentence_tokenize(self, + sent, + sentence_num=0, + beginning=False, + ending=False): + """tokenize sentence and get token types""" + tokens = self.tokenizer.EncodeAsIds(sent).tokenization + str_type = 'str' + str(sentence_num) + token_types = [self.tokenizer.get_type(str_type).Id] * len(tokens) + return tokens, token_types + + def get_doc(self, idx): + """gets text of document corresponding to idx""" + rtn = self.ds[idx] + if isinstance(rtn, dict): + rtn = rtn['text'] + return rtn + + def create_random_sentencepair(self, target_seq_length, rng, np_rng): + """ + fetches a random sentencepair corresponding to rng state similar to + https://github.com/google-research/bert/blob/master/create_pretraining_data.py#L248-L294 + """ + is_random_next = None + + curr_strs = [] + curr_str_types = [] + curr_len = 0 + + while curr_len < 1: + curr_len = 0 + doc_a = None + while doc_a is None: + if self.weighted: + # doc_a_idx = np_rng.choice(self.ds_len, p=self.weighting) + doc_a_idx = self.get_weighted_samples(np_rng) + else: + doc_a_idx = rng.randint(0, self.ds_len - 1) + doc_a = self.sentence_split(self.get_doc(doc_a_idx)) + if not doc_a: + doc_a = None + + random_start_a = rng.randint(0, len(doc_a) - 1) + while random_start_a < len(doc_a): + sentence = doc_a[random_start_a] + sentence, sentence_types = self.sentence_tokenize( + sentence, 0, random_start_a == 0, + random_start_a == len(doc_a)) + curr_strs.append(sentence) + curr_str_types.append(sentence_types) + curr_len += len(sentence) + if random_start_a == len( + doc_a) - 1 or curr_len >= target_seq_length: + break + random_start_a = (random_start_a + 1) + + if curr_strs: + num_a = 1 + if len(curr_strs) >= 2: + num_a = rng.randint(0, len(curr_strs)) + + tokens_a = [] + token_types_a = [] + for j in range(num_a): + tokens_a.extend(curr_strs[j]) + token_types_a.extend(curr_str_types[j]) + + tokens_b = [] + token_types_b = [] + is_random_next = False + if len(curr_strs) == 1 or rng.random() < 0.5: + is_random_next = True + target_b_length = target_seq_length - len(tokens_a) + b_len = 0 + while b_len < 1: + doc_b = None + while doc_b is None: + doc_b_idx = rng.randint(0, self.ds_len - 2) + doc_b_idx += int(doc_b_idx >= doc_a_idx) + + doc_b = self.sentence_split(self.get_doc(doc_b_idx)) + if not doc_b: + doc_b = None + + random_start_b = rng.randint(0, len(doc_b) - 1) + while random_start_b < len(doc_b): + sentence_b = doc_b[random_start_b] + new_b_tokens, new_b_types = self.sentence_tokenize( + sentence_b, 1, random_start_b == 0, + random_start_b == len(doc_b)) + b_len += len(new_b_tokens) + tokens_b.extend(new_b_tokens) + token_types_b.extend(new_b_types) + if len(tokens_b) >= target_b_length: + break + random_start_b = (random_start_b + 1) + else: + is_random_next = False + for j in range(num_a, len(curr_strs)): + tokens_b.extend(curr_strs[j]) + token_types_b.extend(curr_str_types[j]) + + return (tokens_a, token_types_a), (tokens_b, + token_types_b), is_random_next + + def truncate_seq_pair(self, a, b, max_seq_len, rng): + """ + Truncate sequence pair according to original BERT implementation: + https://github.com/google-research/bert/blob/master/create_pretraining_data.py#L391 + """ + tokens_a, token_types_a = a + tokens_b, token_types_b = b + max_num_tokens = max_seq_len - 3 + while True: + len_a = len(tokens_a) + len_b = len(tokens_b) + total_length = len_a + len_b + if total_length <= max_num_tokens: + break + if len(tokens_a) > len(tokens_b): + trunc_tokens = tokens_a + trunc_types = token_types_a + else: + trunc_tokens = tokens_b + trunc_types = token_types_b + + assert len(trunc_tokens) >= 1 + + if rng.random() < 0.5: + trunc_tokens.pop(0) + trunc_types.pop(0) + else: + trunc_tokens.pop() + trunc_types.pop() + return (tokens_a, token_types_a), (tokens_b, token_types_b) + + def mask_token(self, idx, tokens, types, vocab_words, rng): + """ + helper function to mask `idx` token from `tokens` according to + section 3.3.1 of https://arxiv.org/pdf/1810.04805.pdf + """ + label = tokens[idx] + if rng.random() < 0.8: + new_label = self.tokenizer.get_command('MASK').Id + else: + if rng.random() < 0.5: + new_label = label + else: + new_label = rng.choice(vocab_words) + + tokens[idx] = new_label + + return label + + def pad_seq(self, seq): + """helper function to pad sequence pair""" + num_pad = max(0, self.max_seq_len - len(seq)) + pad_mask = [0] * len(seq) + [1] * num_pad + seq += [self.tokenizer.get_command('pad').Id] * num_pad + return seq, pad_mask + + def create_masked_lm_predictions(self, a, b, mask_lm_prob, + max_preds_per_seq, vocab_words, rng): + """ + Mask sequence pair for BERT training according to: + https://github.com/google-research/bert/blob/master/create_pretraining_data.py#L338 + """ + tokens_a, token_types_a = a + tokens_b, token_types_b = b + tokens = [self.tokenizer.get_command('ENC').Id] + tokens_a + [ + self.tokenizer.get_command('sep').Id + ] + tokens_b + [self.tokenizer.get_command('sep').Id] + token_types = [token_types_a[0]] + token_types_a + [ + token_types_a[0] + ] + token_types_b + [token_types_b[0]] + + len_a = len(tokens_a) + len_b = len(tokens_b) + + cand_indices = [idx + 1 for idx in range(len_a) + ] + [idx + 2 + len_a for idx in range(len_b)] + + rng.shuffle(cand_indices) + + output_tokens, pad_mask = self.pad_seq(list(tokens)) + output_types, _ = self.pad_seq(list(token_types)) + + num_to_predict = min(max_preds_per_seq, + max(1, int(round(len(tokens) * mask_lm_prob)))) + + mask = [0] * len(output_tokens) + mask_labels = [-1] * len(output_tokens) + + for idx in sorted(cand_indices[:num_to_predict]): + mask[idx] = 1 + label = self.mask_token(idx, output_tokens, output_types, + vocab_words, rng) + mask_labels[idx] = label + + return (output_tokens, output_types), mask, mask_labels, pad_mask diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/extraction.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..11e0ed6202ffcdd601fa35cf29138b27b18ce993 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/extraction.py @@ -0,0 +1,80 @@ +# Copyright (c) 2022 Zhipu.AI + +import glob +import os + +import json +import nltk + + +class NLTKSegmenter: + + def __init(self): + download_nltk() + + @staticmethod + def segment_string(article): + return nltk.tokenize.sent_tokenize(article) + + +def download_nltk(): + nltk.download('punkt') + wiki_path = 'data/extracted' + output_path = 'formatted/wiki-key.txt' + segmenter = NLTKSegmenter() + with open(output_path, 'w') as output: + for dirname in glob.glob( + os.path.join(wiki_path, '*'), recursive=False): + for filename in glob.glob( + os.path.join(dirname, 'wiki_*'), recursive=True): + print(filename) + article_lines = [] + article_open = False + with open( + filename, mode='r', newline='\n', + encoding='utf-8') as file: + for line in file: + line = line.rstrip() + if '' in line: + key_sentences, contents = [], [] + key, content = None, [] + for sentences in article_lines[1:]: + if len(sentences) > 1: + if key: + if len(content) > 0 or len( + contents) == 0: + key_sentences.append(key) + contents.append(content) + else: + contents[-1].append(key) + key, content = None, [] + key_sentences.append(sentences[0]) + contents.append(sentences[1:]) + elif len(sentences) > 0: + if key: + content.append(sentences[0]) + else: + key = sentences[0] + if key: + if len(content) > 0 or len(contents) == 0: + key_sentences.append(key) + contents.append(content) + else: + contents[-1].append(key) + contents = [ + ' '.join(content) for content in contents + ] + article = { + 'key': key_sentences, + 'content': contents + } + output.write(json.dumps(article)) + output.write('\n') + article_open = False + article_lines = [] + else: + if article_open and line: + sentences = segmenter.segment_string(line) + article_lines.append(sentences) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/file_utils.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..794e127afa8cf40c64ad55e7e83b57f919af4940 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/file_utils.py @@ -0,0 +1,256 @@ +# Modified by Zhipu.AI +# This file is provided as is from: +# https://github.com/huggingface/pytorch-pretrained-BERT +# Please refer to their repository for copyright. +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import logging +import os +import shutil +import sys +import tempfile +from functools import wraps +from hashlib import sha256 +from io import open +from urllib.parse import urlparse + +import boto3 +import json +import requests +from botocore.exceptions import ClientError +from tqdm import tqdm + +try: + from pathlib import Path + PYTORCH_PRETRAINED_BERT_CACHE = Path( + os.getenv('PYTORCH_PRETRAINED_BERT_CACHE', + Path.home() / '.pytorch_pretrained_bert')) +except (AttributeError, ImportError): + PYTORCH_PRETRAINED_BERT_CACHE = os.getenv( + 'PYTORCH_PRETRAINED_BERT_CACHE', + os.path.join(os.path.expanduser('~'), '.pytorch_pretrained_bert')) + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + +def url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + """ + url_bytes = url.encode('utf-8') + url_hash = sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode('utf-8') + etag_hash = sha256(etag_bytes) + filename += '.' + etag_hash.hexdigest() + + return filename + + +def filename_to_url(filename, cache_dir=None): + """ + Return the url and etag (which may be ``None``) stored for `filename`. + Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. + """ + if cache_dir is None: + cache_dir = PYTORCH_PRETRAINED_BERT_CACHE + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + cache_path = os.path.join(cache_dir, filename) + if not os.path.exists(cache_path): + raise EnvironmentError('file {} not found'.format(cache_path)) + + meta_path = cache_path + '.json' + if not os.path.exists(meta_path): + raise EnvironmentError('file {} not found'.format(meta_path)) + + with open(meta_path, encoding='utf-8') as meta_file: + metadata = json.load(meta_file) + url = metadata['url'] + etag = metadata['etag'] + + return url, etag + + +def cached_path(url_or_filename, cache_dir=None): + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + """ + if cache_dir is None: + cache_dir = PYTORCH_PRETRAINED_BERT_CACHE + if sys.version_info[0] == 3 and isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + parsed = urlparse(url_or_filename) + + if parsed.scheme in ('http', 'https', 's3'): + # URL, so get it from the cache (downloading if necessary) + return get_from_cache(url_or_filename, cache_dir) + elif os.path.exists(url_or_filename): + # File, and it exists. + return url_or_filename + elif parsed.scheme == '': + # File, but it doesn't exist. + raise EnvironmentError('file {} not found'.format(url_or_filename)) + else: + # Something unknown + raise ValueError( + 'unable to parse {} as a URL or as a local path'.format( + url_or_filename)) + + +def split_s3_path(url): + """Split a full s3 path into the bucket name and path.""" + parsed = urlparse(url) + if not parsed.netloc or not parsed.path: + raise ValueError('bad s3 path {}'.format(url)) + bucket_name = parsed.netloc + s3_path = parsed.path + # Remove '/' at beginning of path. + if s3_path.startswith('/'): + s3_path = s3_path[1:] + return bucket_name, s3_path + + +def s3_request(func): + """ + Wrapper function for s3 requests in order to create more helpful error + messages. + """ + + @wraps(func) + def wrapper(url, *args, **kwargs): + try: + return func(url, *args, **kwargs) + except ClientError as exc: + if int(exc.response['Error']['Code']) == 404: + raise EnvironmentError('file {} not found'.format(url)) + else: + raise + + return wrapper + + +@s3_request +def s3_etag(url): + """Check ETag on S3 object.""" + s3_resource = boto3.resource('s3') + bucket_name, s3_path = split_s3_path(url) + s3_object = s3_resource.Object(bucket_name, s3_path) + return s3_object.e_tag + + +@s3_request +def s3_get(url, temp_file): + """Pull a file directly from S3.""" + s3_resource = boto3.resource('s3') + bucket_name, s3_path = split_s3_path(url) + s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) + + +def http_get(url, temp_file): + req = requests.get(url, stream=True) + content_length = req.headers.get('Content-Length') + total = int(content_length) if content_length is not None else None + progress = tqdm(unit='B', total=total) + for chunk in req.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + temp_file.write(chunk) + progress.close() + + +def get_from_cache(url, cache_dir=None): + """ + Given a URL, look for the corresponding dataset in the local cache. + If it's not there, download it. Then return the path to the cached file. + """ + if cache_dir is None: + cache_dir = PYTORCH_PRETRAINED_BERT_CACHE + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + # Get eTag to add to filename, if it exists. + if url.startswith('s3://'): + etag = s3_etag(url) + else: + response = requests.head(url, allow_redirects=True) + if response.status_code != 200: + raise IOError( + 'HEAD request failed for url {} with status code {}'.format( + url, response.status_code)) + etag = response.headers.get('ETag') + + filename = url_to_filename(url, etag) + + # get cache path to put the file + cache_path = os.path.join(cache_dir, filename) + + if not os.path.exists(cache_path): + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with tempfile.NamedTemporaryFile() as temp_file: + logger.info('%s not found in cache, downloading to %s', url, + temp_file.name) + + # GET file object + if url.startswith('s3://'): + s3_get(url, temp_file) + else: + http_get(url, temp_file) + + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() + # shutil.copyfileobj() starts at the current position, so go to the start + temp_file.seek(0) + + logger.info('copying %s to cache at %s', temp_file.name, + cache_path) + with open(cache_path, 'wb') as cache_file: + shutil.copyfileobj(temp_file, cache_file) + + logger.info('creating metadata file for %s', cache_path) + meta = {'url': url, 'etag': etag} + meta_path = cache_path + '.json' + with open(meta_path, 'w', encoding='utf-8') as meta_file: + json.dump(meta, meta_file) + + logger.info('removing temp file %s', temp_file.name) + + return cache_path + + +def read_set_from_file(filename): + ''' + Extract a de-duped collection (set) of text from a file. + Expected file format is one item per line. + ''' + collection = set() + with open(filename, 'r', encoding='utf-8') as file_: + for line in file_: + collection.add(line.rstrip()) + return collection + + +def get_file_extension(path, dot=True, lower=True): + ext = os.path.splitext(path)[1] + ext = ext if dot else ext[1:] + return ext.lower() if lower else ext diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/lazy_loader.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/lazy_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..bf24556e5bb3f148e4c2ee1ec426fde1060fc3a0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/lazy_loader.py @@ -0,0 +1,286 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""utils for loading text from disk""" +import mmap +import os +import pickle as pkl +import time +from itertools import accumulate + +import numpy as np +import torch +from torch.multiprocessing import Lock + + +def get_lazy_path(path): + """ + Gets directory path where lazy files are stored. + """ + return os.path.splitext(path)[0] + '.lazy' + + +def exists_lazy(path, data_type='data'): + """ + Check if we've already made a lazy version of this file for the `data_type` field. + """ + if not os.path.exists(get_lazy_path(path)): + return False + contents = os.listdir(get_lazy_path(path)) + if data_type not in contents: + return False + if data_type + '.len.pkl' not in contents: + return False + return True + + +def get_scatter_path(path, scatter_rank): + path = os.path.splitext(path)[0] + '.scatter' + scatter_path = os.path.join(path, str(scatter_rank)) + return scatter_path + + +def exists_scatter(path, scatter_num=64, data_type='data'): + for i in range(scatter_num): + scatter_path = get_scatter_path(path, scatter_rank=i) + if not exists_lazy(scatter_path, data_type=data_type): + return False + return True + + +class LazyWriter: + + def __init__(self, + path, + data_type, + is_array=False, + array_data_type=np.int32): + lazypath = get_lazy_path(path) + if not os.path.exists(lazypath): + os.makedirs(lazypath) + self.datapath = os.path.join(lazypath, data_type) + self.lenpath = os.path.join(lazypath, data_type + '.len.pkl') + self.array_data_type = array_data_type + self.output = open(self.datapath, 'wb') + self.lengths = [] + self.is_array = is_array + + @staticmethod + def get_len_path(path, data_type): + lazypath = get_lazy_path(path) + return os.path.join(lazypath, data_type + '.len.pkl') + + def write(self, s): + if isinstance(s, dict): + s = s['text'] + if self.is_array: + encoded = np.array( + s, dtype=self.array_data_type).tobytes(order='C') + self.output.write(encoded) + self.lengths.append(len(s)) + else: + encoded = s.encode('utf-8') + self.output.write(encoded) + self.lengths.append(len(encoded)) + + def close(self): + self.output.close() + with open(self.lenpath, 'wb') as f: + pkl.dump(self.lengths, f) + + +def split_strings(strings, start, chr_lens): + """ + Split strings based on string lengths and given start. + """ + return [ + strings[i - start:j - start] + for i, j in zip([start] + chr_lens[:-1], chr_lens) + ] + + +class ProcessorTokenizer: + """ + callable class that runs a preprocessing, as well as tokenization step, + on input text. + """ + + def __init__(self, tokenizer, process_fn=None): + self.tokenizer = tokenizer + self.process_fn = process_fn + + def __call__(self, string): + if self.tokenizer is not None: + string = self.tokenizer(string, process_fn=self.process_fn) + elif self.process_fn is not None: + string = self.process_fn(string) + return string + + +class LazyLoader(object): + """ + Arguments: + path: path to directory where array entries are concatenated into one big string file + and the .len file are located + data_type (str): Some datsets have multiple fields that are stored in different paths. + `data_type` specifies which of these fields to load in this class + mem_map (boolean): Specifies whether to memory map file `path` + map_fn (callable): Fetched strings are passed through map_fn before being returned. + + Example of lazy loader directory structure: + file.json + file.lazy/ + data_type1 + data_type1.len.pkl + data_type2 + data_type2.len.pkl + """ + + def __init__(self, + path, + data_type='data', + mem_map=False, + map_fn=None, + is_array=False, + array_data_type=np.int32, + load_memory=False, + half_load=False): + lazypath = get_lazy_path(path) + datapath = os.path.join(lazypath, data_type) + # get file where array entries are concatenated into one big string + self._file = open(datapath, 'rb') + self.file = self._file + self.is_array = is_array + self.array_data_type = array_data_type + # memory map file if necessary + lenpath = os.path.join(lazypath, data_type + '.len.pkl') + self.lens = pkl.load(open(lenpath, 'rb')) + if half_load: + self.lens = self.lens[:2 * len(self.lens) // 3] + self.ends = list(accumulate(self.lens)) + self.dumb_ends = list(self.ends) + self.mem_map = mem_map + self.load_memory = load_memory + if self.load_memory: + data_type_size = np.dtype(self.array_data_type).itemsize + if half_load: + self.file = self.file.read(sum(self.lens) * data_type_size) + else: + self.file = self.file.read() + self.file = np.ndarray( + shape=(len(self.file) // data_type_size, ), + dtype=array_data_type, + buffer=self.file, + order='C') + elif self.mem_map: + if is_array: + if self.ends[-1] == 0: + self.file = np.array([], dtype=array_data_type) + else: + self.file = np.memmap( + self.file, dtype=array_data_type, mode='r', order='C') + else: + if self.ends[-1] == 0: + self.file = bytearray() + else: + self.file = mmap.mmap( + self.file.fileno(), 0, prot=mmap.PROT_READ) + self.read_lock = Lock() + self.process_fn = map_fn + self.map_fn = map_fn + self._tokenizer = None + self.is_lazy = True + + def SetTokenizer(self, tokenizer): + """ + logic to set and remove (set to None) tokenizer. + combines preprocessing/tokenization into one callable. + """ + if tokenizer is None: + if not hasattr(self, '_tokenizer'): + self._tokenizer = tokenizer + else: + self._tokenizer = tokenizer + self.map_fn = ProcessorTokenizer(tokenizer, self.process_fn) + + def GetTokenizer(self): + return self._tokenizer + + def __getitem__(self, index): + """ + read file and splice strings based on string ending array `self.ends` + """ + if not isinstance(index, slice): + if index == 0: + start = 0 + else: + start = self.ends[index - 1] + end = self.ends[index] + rtn = self.file_read(start, end) + if self.map_fn is not None: + rtn = self.map_fn(rtn) + else: + # if slice, fetch strings with 1 diskread and then splice in memory + chr_lens = self.ends[index] + if index.start == 0 or index.start is None: + start = 0 + else: + start = self.ends[index.start - 1] + stop = chr_lens[-1] + strings = self.file_read(start, stop) + rtn = split_strings(strings, start, chr_lens) + if self.map_fn is not None: + rtn = [self.map_fn(s) for s in rtn] + return rtn + + def __len__(self): + return len(self.ends) + + def file_read(self, start=0, end=None): + """read specified portion of file""" + data_type_size = np.dtype(self.array_data_type).itemsize + # atomic reads to avoid race conditions with multiprocess dataloader + self.read_lock.acquire() + if not self.mem_map and not self.load_memory: + # seek to start of file read + if self.is_array: + start = start * data_type_size + end = end * data_type_size if end is not None else None + self.file.seek(start) + # read to end of file if no end point provided + if end is None: + rtn = self.file.read() + # else read amount needed to reach end point + else: + rtn = self.file.read(end - start) + if self.is_array: + rtn = np.ndarray( + shape=(len(rtn) // data_type_size, ), + dtype=self.array_data_type, + buffer=rtn, + order='C') + else: + rtn = rtn.decode('utf-8', 'ignore') + else: + rtn = self.file[start:end] + if self.is_array: + rtn = rtn.copy() + else: + rtn = rtn.decode('utf-8', 'strict') + self.read_lock.release() + # TODO: @raulp figure out mem map byte string bug + # if mem map'd need to decode byte string to string + # # rtn = str(rtn) + # if self.mem_map: + # rtn = rtn.decode('unicode_escape') + return rtn diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/samplers.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..c0f6e1abfaa1b0d4074942fc811dcc5b7c23ec84 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/samplers.py @@ -0,0 +1,190 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""batch samplers that work with either random or sequential data samplers""" +import math +import os +import sys + +import numpy as np +import torch +from torch.utils import data + + +class RandomSampler(data.sampler.Sampler): + r""" + Based off of pytorch RandomSampler and DistributedSampler. Essentially a RandomSampler, + but this class lets the user set an epoch like DistributedSampler + Samples elements randomly. If without replacement, then sample from a shuffled dataset. + If with replacement, then user can specify ``num_samples`` to draw. + Arguments: + data_source (Dataset): dataset to sample from + num_samples (int): number of samples to draw, default=len(dataset) + replacement (bool): samples are drawn with replacement if ``True``, default=False + """ + + def __init__(self, data_source, replacement=False, num_samples=None): + super(RandomSampler, self).__init__(data_source) + self.data_source = data_source + self.replacement = replacement + self._num_samples = num_samples + self.epoch = -1 + + if self._num_samples is not None and replacement is False: + raise ValueError( + 'With replacement=False, num_samples should not be specified, ' + 'since a random permute will be performed.') + + if not isinstance(self.num_samples, int) or self.num_samples <= 0: + raise ValueError('num_samples should be a positive integer ' + 'value, but got num_samples={}'.format( + self.num_samples)) + if not isinstance(self.replacement, bool): + raise ValueError('replacement should be a boolean value, but got ' + 'replacement={}'.format(self.replacement)) + + @property + def num_samples(self): + # dataset size might change at runtime + if self._num_samples is None: + return len(self.data_source) + return self._num_samples + + def __iter__(self): + n = len(self.data_source) + g = torch.Generator() + if self.epoch >= 0: + g.manual_seed(self.epoch) + if self.replacement: + for _ in range(self.num_samples // 32): + yield from torch.randint( + high=n, size=(32, ), dtype=torch.int64, + generator=g).tolist() + yield from torch.randint( + high=n, + size=(self.num_samples % 32, ), + dtype=torch.int64, + generator=g).tolist() + else: + yield from torch.randperm(n, generator=self.generator).tolist() + + def __len__(self): + return self.num_samples + + def set_epoch(self, epoch): + self.epoch = epoch + + +class DistributedSequentialSampler(data.sampler.Sampler): + + def __init__(self, + num_samples, + train_iters, + batch_size, + rank=-1, + world_size=2): + super().__init__(num_samples) + if rank == -1: + rank = 0 + world_size = 1 + self.num_samples = num_samples + self.rank = rank + self.world_size = world_size + self.start_iter = 0 + self.train_iters = train_iters + self.batch_size = batch_size + self.batch_bias = [ + i * (num_samples // batch_size) for i in range(batch_size) + ] + + def __iter__(self): + for idx in range(self.start_iter, self.train_iters * 10): + batch = [(idx + bias) % self.num_samples + for bias in self.batch_bias] + tbatch = self._batch(batch) + yield tbatch + + def __len__(self): + return self.train_iters + + def _batch(self, batch): + """extracts samples only pertaining to this worker's batch""" + start = self.rank * self.batch_size // self.world_size + end = (self.rank + 1) * self.batch_size // self.world_size + return batch[start:end] + + +class DistributedBatchSampler(data.sampler.BatchSampler): + """ + similar to normal implementation of distributed sampler, except implementation is at the + batch sampler level, instead of just the sampler level. This allows wrapping of arbitrary + data samplers (sequential, random, WeightedRandomSampler, etc.) with this batch sampler. + """ + + def __init__(self, + sampler, + batch_size, + drop_last, + rank=-1, + world_size=2, + wrap_last=False, + gradient_accumulation_steps=None): + super(DistributedBatchSampler, self).__init__(sampler, batch_size, + drop_last) + if rank == -1: + assert False, 'should not be here' + self.rank = rank + self.world_size = world_size + self.sampler.wrap_around = 0 + self.wrap_around = 0 + self.wrap_last = wrap_last + self.start_iter = 0 + self.effective_batch_size = batch_size if gradient_accumulation_steps is None else batch_size * gradient_accumulation_steps # noqa + + def __iter__(self): + batch = [] + i = 0 + for idx in self.data_iterator(self.sampler, wrap_around=False): + batch.append(idx) + if len(batch) == self.batch_size: + tbatch = self._batch(batch) + if i >= self.start_iter * self.effective_batch_size: + yield tbatch + self.start_iter = 0 + i += len(batch) + batch = [] + batch_len = len(batch) + if batch_len > 0 and not self.drop_last: + if self.wrap_last: + self.sampler.wrap_around -= (self.batch_size) + self.wrap_around += (len(batch)) + self.wrap_around %= self.batch_size + yield self._batch(batch) + if self.wrap_last: + self.sampler.wrap_around += self.batch_size + + def data_iterator(self, _iter, wrap_around=False): + """iterates through data and handles wrap around""" + for i, idx in enumerate(_iter): + if i < self.wrap_around % self.batch_size: + continue + if wrap_around: + self.wrap_around += 1 + self.wrap_around %= self.batch_size + yield idx + + def _batch(self, batch): + """extracts samples only pertaining to this worker's batch""" + start = self.rank * self.batch_size // self.world_size + end = (self.rank + 1) * self.batch_size // self.world_size + return batch[start:end] diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/sp_tokenizer.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/sp_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..b4d1afe36dd2c2c25e8370a8f8480404a55aafbd --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/sp_tokenizer.py @@ -0,0 +1,158 @@ +# Modified by Zhipu.AI +""" +from https://github.com/openai/gpt-2/, changed for chinese +""" +import os # yapf: disable + + +""" +SentencePiece is an unsupervised text tokenizer and detokenizer mainly for Neural Network-based text generation +systems where the vocabulary size is predetermined prior to the neural model training. SentencePiece implements +subword units (e.g., byte-pair-encoding (BPE) [Sennrich et al.]) and unigram language model [Kudo.]) with the +extension of direct training from raw sentences. SentencePiece allows us to make a purely end-to-end +system that does not depend on language-specific pre/postprocessing. +https://github.com/google/sentencepiece + +pip install sentencepiece + +or git clone https://github.com/google/sentencepiece.git +python setup.py install + +""" + + +def get_pairs(word): + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class Encoder: + + def __init__(self, encoder, bpe_merges): + self.encoder = encoder + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + self.max_len = 0 + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + if not pairs: + return token + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: # noqa + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + return [self.encoder.get(token, 1) for token in self.tokenize(text)] + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + return text + + def tokenize(self, text): + bpe_tokens = [] + bpe_tokens.extend(bpe_token for bpe_token in self.bpe(text).split(' ')) + return bpe_tokens + + def convert_tokens_to_ids(self, tokens): + return [self.encoder.get(token, 1) for token in tokens] + + +class Encoder_SP: + + def __init__(self, model_path): + import sentencepiece as spm + self.sp = spm.SentencePieceProcessor() + self.sp.Load(model_path) + + def encode(self, text): + """ + text="...." + """ + return self.sp.EncodeAsIds(text) + + def decode(self, tokens): + """ + tokens=[x1,x2,...] + """ + text = [int(token) for token in tokens] + # print(text) + return self.sp.DecodeIds(text) + + def tokenize(self, text): + return self.sp.EncodeAsPieces(text) + + def convert_tokens_to_ids(self, tokens): + return [self.sp.PieceToId(token) for token in tokens] + + def convert_token_to_id(self, token): + return self.sp.PieceToId(token) + + def convert_id_to_token(self, idx): + return self.sp.IdToPiece(idx) + + +def get_encoder(encoder_file, bpe_file): + import json + filepath, filename = os.path.split(encoder_file) + shotname, extension = os.path.splitext(filename) + + if ('.model' == extension) and (bpe_file == ''): + return Encoder_SP(encoder_file) + else: + with open(encoder_file, 'r', encoding='utf-8') as f: + encoder = json.load(f) + with open(bpe_file, 'r', encoding='utf-8') as f: + bpe_data = f.read() + bpe_merges = [ + tuple(merge_str.split()) + for merge_str in bpe_data.split('\n')[1:-1] + ] + return Encoder( + encoder=encoder, + bpe_merges=bpe_merges, + ) + + +def from_pretrained(model_path): + return get_encoder(model_path + '/tokenizer/mglm250k/mglm250k-uni.model', + '') diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..c89cc37173099cb074217a295fac3b962c3b1636 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization.py @@ -0,0 +1,1396 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)""" +import csv +import itertools +import os +import random +from collections import namedtuple + +import nltk +import regex as re +import sentencepiece as spm +import torch +from nltk import tokenize as nltk_tokenize + +from . import sp_tokenizer +from .tokenization_gpt2 import GPT2Tokenizer +from .wordpiece import PRETRAINED_VOCAB_ARCHIVE_MAP, BertTokenizer + + +def make_tokenizer(tokenizer_type, + corpus, + model_path=None, + vocab_size=None, + model_type=None, + pad_token=0, + character_coverage=1.0, + command_tokens=None, + type_tokens=None, + **kwargs): + """ + Helper function to instantiate a tokenizer given common combinations of options. + """ + tokenizer_class = tokenizer_type + if isinstance(tokenizer_class, str): + tokenizer_class = eval(tokenizer_class) + if tokenizer_class is BertWordPieceTokenizer: + return BertWordPieceTokenizer(model_type, **kwargs) + elif tokenizer_class is GPT2BPETokenizer: + if model_type is None: + model_type = 'gpt2' + return GPT2BPETokenizer(model_type, **kwargs) + elif tokenizer_class is ChineseSPTokenizer: + return ChineseSPTokenizer(model_path, **kwargs) + text_tokenizer = tokenizer_class( + corpus=corpus, + vocab_size=vocab_size, + model_path=model_path, + model_type=model_type, + pad_token=pad_token, + character_coverage=character_coverage) + return Tokenizer(text_tokenizer, command_tokens, type_tokens) + + +class Tokenization(object): + """ + Tokenization object to hold tokenization, (processed text),and original + text. Can hold tokenization as Ids or tokens. + + It also holds command tokens (pad, unk, etc.) for the tokenization. + This allows functions to pad/operate on tokenizations without having + access to the full tokenizer, just the tokenization. + + Several standard array operations are implemented (insert, append, extend). + """ + + def __init__(self, + tokenization, + text=None, + original_text=None, + command_tokens=None, + asIds=True): + self.tokenization = tokenization + self.text = text + if self.text is None: + self.text = self.tokenization + self.original_text = original_text + if self.original_text is None: + self.original_text = self.text + self.command_tokens = command_tokens + self.asIds = asIds + self.parse_command_tokens() + + def set_command_tokens(self, command_tokens): + self.command_tokens = command_tokens + return self.parse_command_tokens() + + def parse_command_tokens(self): + if self.command_tokens is None: + return + for command_token in self.command_tokens: + if self.asIds: + setattr(self, command_token.name, command_token.Id) + else: + setattr(self, command_token.name, command_token.token) + + def __getitem__(self, index): + return self.tokenization[index] + + def __len__(self): + return len(self.tokenization) + + def insert(self, idx, other): + if isinstance(other, (CommandToken, TypeToken)): + self.tokenization.insert(idx, other.Id) + if idx == 0: + self.text = other.token + self.text + self.original_text = other.token + self.original_text + elif idx == len(self.tokenization) - 1: + self.text += other.token + self.original_text += other.token + elif isinstance(other, Tokenization): + self.tokenization = self.tokenization[: + idx] + other.tokenization + self.tokenization[ + idx:] + else: + self.tokenization = self.tokenization[: + idx] + other.tokenization + self.tokenization[ + idx:] + + def append(self, other): + if isinstance(other, (CommandToken, TypeToken)): + self.tokenization.append(other.Id) + self.text += other.token + self.original_text += other.token + elif isinstance(other, Tokenization): + self.tokenization.extend(other.tokenization) + self.text += other.text + self.original_text += other.original_text + else: + self.tokenization.append(other) + return self + + def extend(self, other): + if isinstance(other, (CommandToken, TypeToken)): + self.tokenization.append(other.Id) + self.text += other.token + self.original_text += other.token + elif isinstance(other, list) and isinstance(other[0], + (CommandToken, TypeToken)): + self.tokenization.extend([o.Id for o in other]) + self.text += [o.token for o in other] + self.original_text += [o.token for o in other] + elif isinstance(other, Tokenization): + self.tokenization.extend(other.tokenization) + self.text += other.text + self.original_text += other.original_text + else: + self.tokenization.extend(other) + return self + + +"""define some default command tokens for the tokenizer to use""" +token_format = '<{0}>' + +COMMAND_TUPLE = namedtuple('CommandToken', ('name', 'token', 'Id')) + + +def prep_command_tokens(tokenlist, token_format=token_format): + return [ + CommandToken(tok[0], token_format.format(tok[0]), tok[1]) + for tok in tokenlist + ] + + +class CommandToken(object): + + def __init__(self, name, token, Id, lstrip=False, rstrip=False): + self.name = name + self.token = token + self.Id = Id + self.lstrip = lstrip + self.rstrip = rstrip + + def __str__(self): + return str(COMMAND_TUPLE(self.name, self.token, self.Id)) + + +DEFAULT_COMMAND_TOKENS = [ + ('pad', 0), + ('eos', 1), + ('bos', 2), + ('unk', 3), + ('sep', 4), + ('L2R', 5), + ('ENC', 6), + ('MASK', 7), +] +DEFAULT_COMMAND_TOKENS = prep_command_tokens(DEFAULT_COMMAND_TOKENS) +"""define some default type tokens for bert training""" + +TYPE_TUPLE = namedtuple('TypeToken', ('name', 'token', 'Id')) + + +def prep_type_tokens(tokenlist, token_format=token_format): + return [ + TypeToken(tok[0], token_format.format(tok[0]), tok[1]) + for tok in tokenlist + ] + + +class TypeToken(object): + + def __init__(self, name, token, Id): + self.name = name + self.token = token + self.Id = Id + + def __str__(self): + return str(TYPE_TUPLE(self.name, self.token, self.Id)) + + +DEFAULT_TYPE_TOKENS = [ + ('function', 0), + ('command', 1), + ('str0', 2), + ('str1', 3), + ('str2', 4), + ('embedding0', 5), + ('embedding1', 6), + ('embedding2', 7), + ('arg0', 8), + ('arg1', 9), + ('arg2', 10), +] +DEFAULT_TYPE_TOKENS = prep_type_tokens(DEFAULT_TYPE_TOKENS) + + +class Tokenizer(object): + """ + Tokenizer object that handles text tokenization, command tokens, and type tokens. + + Command tokens and text tokens are stored together in one mapping of size + `len(text_tokenizer)+len(command_tokens)`. Command tokens are stored as first + `len(command_tokens)` tokens. Token idx is stored at `idx+len(command_tokens)`. + + Token types are stored in a separate mapping of size `len(type_tokens)`. + """ + + def __init__(self, text_tokenizer, command_tokens=None, type_tokens=None): + # set text tokenizer + self.text_tokenizer = text_tokenizer + if not hasattr(self, 'num_text_tokens'): + self.num_text_tokens = len(self.text_tokenizer) + + # set command tokens + if command_tokens is None: + command_tokens = DEFAULT_COMMAND_TOKENS + self._command_tokens = command_tokens + self.command_name_map = {tok.name: tok for tok in self._command_tokens} + self.command_token_map = { + tok.token: tok + for tok in self._command_tokens + } + self.command_id_map = {tok.Id: tok for tok in self._command_tokens} + if not hasattr(self, 'num_command_tokens'): + self.num_command_tokens = len(self._command_tokens) + if not hasattr(self, 'num_tokens'): + self.num_tokens = self.num_command_tokens + self.num_text_tokens + + # set type tokens + if type_tokens is None: + type_tokens = DEFAULT_TYPE_TOKENS + self.type_tokens = type_tokens + self.type_name_map = {tok.name: tok for tok in self.type_tokens} + self.type_token_map = {tok.token: tok for tok in self.type_tokens} + self.type_id_map = {tok.Id: tok for tok in self.type_tokens} + if not hasattr(self, 'num_type_tokens'): + self.num_type_tokens = len(self.type_tokens) + + # parse tokens and vocabs from tokenizer + self._tokens = list(self.command_token_map.keys()) + list( + self.text_tokenizer.tokens) + self._vocab = {t: Id for Id, t in self.command_id_map.items()} + self._vocab.update({ + t: Id + self.num_command_tokens + for t, Id in self.text_tokenizer.vocab.items() + }) + + self._text_tokens = list(self.text_tokenizer.tokens) + self._text_token_vocab = { + t: Id + self.num_command_tokens + for t, Id in self.text_tokenizer.vocab.items() + } + + self._command_token_tokens = list(self.command_token_map.keys()) + self._command_token_vocab = { + t: Id + for Id, t in self.command_id_map.items() + } + + self._token_types = list(self.type_token_map.keys()) + self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()} + + def __call__(self, text, process_fn=None): + """run preprocessing and encode text as Ids""" + return self.EncodeAsIds(text, process_fn=process_fn) + + def __len__(self): + """total number of tokens""" + return self.num_tokens + + def get_command(self, name): + """get command token corresponding to `name`""" + return self.command_name_map[name] + + def get_type(self, name): + """get type token corresponding to `name`""" + return self.type_name_map[name] + + @property + def tokens(self): + """list (or iterable) of all tokens for tokenizer""" + return self._tokens + + @property + def vocab(self): + """dictionary mapping tokens to ids for tokenizer""" + return self._vocab + + @property + def token_types(self): + """list (or iterable) of all token types for tokenizer""" + return self._token_types + + @property + def token_type_vocab(self): + """dictionary mapping token types to ids for tokenizer""" + return self._token_type_vocab + + @property + def command_tokens(self): + """list (or iterable) of all command tokens for tokenizer""" + return self._command_token_tokens + + @property + def command_token_vocab(self): + """dictionary mapping command tokens to ids for tokenizer""" + return self._command_token_vocab + + @property + def text_tokens(self): + """list (or iterable) of text tokens for text tokenizer""" + return self._text_tokens + + @property + def text_token_vocab(self): + """dictionary mapping text tokens to ids for text tokenizer""" + return self._text_token_vocab + + def EncodeAsIds(self, text, process_fn=None): + """ + encode text using text tokenizer and shift Id values for command tokens + """ + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + + def split_on_token(tok_extended: CommandToken, text): + result = [] + tok = tok_extended.token + split_text = text.split(tok) + for i, sub_text in enumerate(split_text): + # CommandToken can control whitespace stripping around them. + # We use them for GPT2 and Roberta to have different behavior depending on the special token + # Cf. https://github.com/huggingface/transformers/pull/2778 + # and https://github.com/huggingface/transformers/issues/3788 + # Strip white spaces on the right + if tok_extended.rstrip and i > 0: + # A bit counter-intuitive but we strip the left of the string + # since tok_extended.rstrip means the special token is eating all white spaces on its right + sub_text = sub_text.lstrip() + # Strip white spaces on the left + if tok_extended.lstrip and i < len(split_text) - 1: + sub_text = sub_text.rstrip() # Opposite here + + if i == 0 and not sub_text: + result.append(tok) + elif i == len(split_text) - 1: + if sub_text: + result.append(sub_text) + else: + pass + else: + if sub_text: + result.append(sub_text) + result.append(tok) + return result + + def split_on_tokens(tok_list, text): + if not text.strip(): + return [] + if not tok_list: + return self.text_tokenizer.encode(text) + + tokenized_text = [] + text_list = [text] + for tok in tok_list: + tokenized_text = [] + for sub_text in text_list: + if sub_text not in self._command_token_tokens: + tokenized_text.extend(split_on_token(tok, sub_text)) + else: + tokenized_text.append(sub_text) + text_list = tokenized_text + + return list( + itertools.chain.from_iterable( + (self._encode(token) + if token not in self._command_token_tokens else + [self.command_token_map[token].Id] + for token in tokenized_text))) + + no_split_tokens = self._command_tokens + Ids = split_on_tokens(no_split_tokens, processed_text) + tokenization = Tokenization(Ids, processed_text, text) + tokenization.set_command_tokens(self._command_tokens) + return tokenization + + def _encode(self, text): + raise NotImplementedError + + def EncodeAsTokens(self, text, process_fn=None): + """ + encode text as tokens using text tokenizer + """ + tokenization = self.text_tokenizer.EncodeAsTokens( + text, process_fn=process_fn) + tokenization.set_command_tokens(self._command_tokens) + return tokenization + + def IdToToken(self, Id, type_token=False): + """convert Id to token accounting for command and type tokens""" + if isinstance(Id, (TypeToken, CommandToken)): + return Id.token + if type_token: + return self.type_id_map[Id].token + if Id < self.num_command_tokens: + return self.command_id_map[Id].token + return self.text_tokenizer.IdToToken(Id - self.num_command_tokens) + + def TokenToId(self, token, type_token=False): + """convert token to Id accounting for command and type tokens""" + if isinstance(token, (TypeToken, CommandToken)): + return token.Id + if type_token: + return self.type_token_map[token].Id + if token in self.command_token_map: + return self.command_token_map[token].Id + return self.text_tokenizer.TokenToId(token) + self.num_command_tokens + + def DecodeIds(self, Ids, type_token=False): + """ + convert Ids to tokens accounting for command and type tokens, tokens + are joined and returned as a string. + """ + if type_token: + return ' '.join( + Id.token if isinstance(Id, TypeToken) else self. + type_id_map[Id].token for Id in Ids) + rtn_strs = [] + current_str = [] + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + for Id in Ids: + if isinstance(Id, CommandToken): + rtn_strs.append(self.text_tokenizer.DecodeIds(current_str)) + current_str = [] + rtn_strs.append(Id.token) + elif Id < self.num_command_tokens: + rtn_strs.append(self.text_tokenizer.DecodeIds(current_str)) + current_str = [] + rtn_strs.append(self.command_id_map[Id].token) + else: + current_str.append(Id - self.num_command_tokens) + if current_str != []: + rtn_strs.append(self.text_tokenizer.DecodeIds(current_str)) + return ' '.join(rtn_strs) + + def DecodeTokens(self, Tokens, type_token=False): + """ + convert tokens to a string accounting for command and type tokens. + """ + if type_token: + return ' '.join( + t.token if isinstance(t, TypeToken) else t for t in Tokens) + rtn_strs = [] + current_str = [] + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + for t in Tokens: + if isinstance(t, CommandToken): + rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) + current_str = [] + rtn_strs.append(t.token) + elif t in self.command_token_map: + rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) + current_str = [] + rtn_strs.append(t) + else: + current_str.append(t) + if current_str != []: + rtn_strs.append(self.text_tokenizer.DecodeTokens(current_str)) + return ' '.join(rtn_strs) + + +class TextTokenizer(object): + """ + Interface for text tokenizer + """ + + def __init__(self): + if not hasattr(self, 'num_text_tokens'): + self.num_text_tokens = 0 + if not hasattr(self, 'num_tokens'): + self.num_tokens = self.num_text_tokens + + def __call__(self, text, process_fn=None): + return self.EncodeAsIds(text, process_fn) + + def __len__(self): + return self.num_text_tokens + + @property + def tokens(self): + """list (or iterable) of text tokens for text tokenizer""" + raise NotImplementedError( + 'TextTokenizer tokens property not implemented') + + @property + def vocab(self): + """dictionary mapping tokens to ids""" + raise NotImplementedError( + 'TextTokenizer vocab property not implemented') + + @staticmethod + def exists(model_path): + """check if the filepath for a text tokenizer exists""" + raise NotImplementedError( + 'TextTokenizer exists method not implemented') + + def Train(self, corpus): + """train a tokenizer on a data corpus and save model for future use""" + raise NotImplementedError('TextTokenizer Train not implemented') + + def EncodeAsIds(self, text, process_fn=None): + """ + Preprocess text and encode as ids. Return a tokenization object with + original text, processed text, and id tokenization. + """ + raise NotImplementedError('TextTokenizer EncodeAsIds not implemented') + + def EncodeAsTokens(self, text, process_fn=None): + """ + Preprocess text and encode as tokens. Return a tokenization object with + original text, processed text, and token tokenization. + """ + raise NotImplementedError( + 'TextTokenizer EncodeAsTokens not implemented') + + def IdToToken(self, Id): + """Convert an Id to Token. Reverse lookup of self.vocab""" + raise NotImplementedError('TextTokenizer IdToToken not implemented') + + def TokenToId(self, token): + """Convert a Token to Id. Lookup of self.vocab""" + raise NotImplementedError('TextTokenizer TokenToId not implemented') + + def DecodeIds(self, Ids): + """Convert a list or tokenization object of Ids to a text string""" + raise NotImplementedError('TextTokenizer DecodeIds not implemented') + + def DecodeTokens(self, Tokens): + """Convert a list or tokenization object of tokens to a text string""" + raise NotImplementedError('TextTokenizer DecodeTokens not implemented') + + +class CharacterLevelTokenizer(TextTokenizer): + """ + Text tokenizer for ASCII-256 Character Level Tokenization. + """ + + def __init__(self, **kwargs): + self.num_text_tokens = 256 + super(CharacterLevelTokenizer, self).__init__() + self._tokens = [ + self.IdToToken(Id) for Id in range(self.num_text_tokens) + ] + self._vocab = {t: i for i, t in enumerate(self._tokens)} + + def __len__(self): + return 256 + + @staticmethod + def exists(model_path): + return True + + def Train(self, corpus): + pass + + @property + def tokens(self): + return self._tokens + + @property + def vocab(self): + return self._vocab + + def EncodeAsIds(self, text, process_fn=None): + """convert text to ascii 256 Ids""" + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + processed_text = str(processed_text) + tokens = [self.TokenToId(c) for c in processed_text] + return Tokenization(tokens, processed_text, text) + + def EncodeAsTokens(self, text, process_fn=None): + """convert text to ascii 256 characters""" + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + processed_text = str(processed_text) + tokens = [c for c in processed_text] + return Tokenization(tokens, processed_text, text, asIds=False) + + def IdToToken(self, Id): + """ascii index to character""" + return chr(Id) + + def TokenToId(self, token): + """ascii character to index""" + return ord(token) + + def DecodeIds(self, Ids): + """converts ascii ids to tokens before joining them into text""" + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + return ''.join([self.IdToToken(tok) for tok in Ids]) + + def DecodeTokens(self, Tokens): + """just concatenates ascii tokens into text""" + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + return ''.join(Tokens) + + +MAX_SENTENCEPIECE_SENTENCES = 100000000 + + +def get_corpus_freq(dataset, filepath, filetype='tsv'): + """ + Take corpus, split it into sentences, and extract word frequencies. + Write frequencies to `filepath` as a tsv. Only write the first + MAX_SENTENCEPIECE_SENTENCES most common words to the file. + """ + nltk.download('punkt', download_dir='./nltk') + if filetype == 'tsv': + delimiter = '\t' + else: + delimiter = ',' + + print('compute corpus frequency\n', flush=True) + + total_sentence_count = 0 + maxlen = 0 + freqs = {} + for entry in dataset: + if isinstance(entry, dict): + entry = entry['text'] + lines = entry.strip().split('\n') + for line in lines: + sentences = nltk_tokenize.sent_tokenize(line) + total_sentence_count += len(sentences) + for sentence in sentences: + maxlen = max(len(line), maxlen) + for word in sentence.split(): + if word not in freqs: + freqs[word] = 0 + freqs[word] += 1 + + print('length of freqs before truncating ' + str(len(freqs)), flush=True) + print('file path for freq ' + str(filepath), flush=True) + + freqs_sorted = {} + counter = 0 + for word, count in sorted(freqs.items(), key=lambda x: x[1], reverse=True): + if counter >= MAX_SENTENCEPIECE_SENTENCES: + break + counter += 1 + freqs_sorted[word] = count + + print( + 'length of freqs after trancating ' + str(len(freqs_sorted)), + flush=True) + + with open(filepath, 'w') as f: + writer = csv.writer(f, delimiter=delimiter) + for k, v in freqs_sorted.items(): + writer.writerow([str(k), str(v)]) + + return total_sentence_count, maxlen + + +class SentencePieceTokenizer(TextTokenizer): + """Trains and uses sentencepiece for text tokenization""" + + def __init__(self, + model_type='bpe', + vocab_size=None, + corpus=None, + model_path=None, + character_coverage=1.0, + **kwargs): + self.character_coverage = character_coverage + self.model_type = model_type.lower() + self.spm_model = model_path + self.num_text_tokens = vocab_size + make_train = not SentencePieceTokenizer.exists(self.spm_model) + if make_train: + assert corpus is not None and self.num_text_tokens is not None + self.Train(corpus, self.num_text_tokens) + self._tokens = [] + self._vocab = {} + self.load_spm_model() + super(SentencePieceTokenizer, self).__init__() + + def __len__(self): + return self.num_text_tokens + + @property + def tokens(self): + return self._tokens + + @property + def vocab(self): + return self._vocab + + @staticmethod + def exists(model_path): + if model_path is None: + return False + # check if path exists + dne = not os.path.exists(model_path) + # check if path.model exists + if dne and not model_path.endswith('.model'): + dne = not os.path.exists(model_path + '.model') + return not dne + + def load_spm_model(self): + """load sentencepiece model and parse vocab""" + if not os.path.exists( + self.spm_model) and not self.spm_model.endswith('.model'): + self.spm_model = self.spm_model + '.model' + self.sp = spm.SentencePieceProcessor() + self.sp.Load(self.spm_model) + self.vocab_size = self.num_text_tokens = len(self.sp) + self._tokens = [self.IdToToken(t) for t in range(self.vocab_size)] + self._vocab = {t: i for i, t in enumerate(self._tokens)} + + def Train(self, corpus, num_text_tokens): + """train sentencepiece model on corpus using word frequencies""" + self.num_text_tokens = num_text_tokens + use_model_path = self.spm_model + random_hash = str(random.randint(0, 2147483647)) + if use_model_path is None: + use_model_path = random_hash + if use_model_path.endswith('.model'): + use_model_path = use_model_path[:use_model_path.rfind('.model')] + input_path = use_model_path + '.tsv.' + random_hash + line_count, maxlenline = get_corpus_freq(corpus, input_path) + line_count = min(line_count, MAX_SENTENCEPIECE_SENTENCES) + print( + 'line count used as input_sentence_size ', line_count, flush=True) + print('training sentencepiece model', flush=True) + train_string = '--input={file_path} --model_prefix={model_prefix} --vocab_size={vocab_size}' \ + + ' --model_type={model_type} --character_coverage={character_coverage} ' \ + + '--input_sentence_size={input_sentence_size} ' \ + + '--input_format=tsv' + train_string = train_string.format( + file_path=input_path, + model_prefix=use_model_path, + vocab_size=num_text_tokens, + model_type=self.model_type, + character_coverage=self.character_coverage, + input_sentence_size=int(line_count)) # , #)#, + print( + 'calling spm.SentencePieceTrainer.Train(%s)' % (train_string), + flush=True) + spm.SentencePieceTrainer.Train(train_string) + os.remove(input_path) + self.spm_model = use_model_path + '.model' + print('sentencepiece model written to ' + self.spm_model, flush=True) + + def EncodeAsIds(self, text, process_fn=None): + """convert text to sentencepiece Ids""" + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + tokens = self.sp.EncodeAsIds(processed_text) + return Tokenization(tokens, processed_text, text) + + def EncodeAsTokens(self, text, process_fn=None): + """convert text to sentencepiece tokens""" + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + tokens = self.sp.EncodeAsTokens(processed_text) + return Tokenization(tokens, processed_text, text, asIds=False) + + def IdToToken(self, Id): + """convert Id to sentencpiece token""" + return self.sp.IdToPiece(Id) + + def TokenToId(self, token): + """convert sentencpiece token to Id""" + return self.sp.PieceToId(token) + + def DecodeIds(self, Ids): + """converts ids to a text string""" + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + return self.sp.DecodeIds(Ids) + + def DecodeTokens(self, Tokens): + """converts sentencepiece tokens to a text string""" + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + return self.sp.DecodeTokens(Tokens) + + +class BertWordPieceTokenizer(Tokenizer): + """ + Loads a pretrained WordPiece tokenizer from `cache_dir` for tokenization + in BERT training. Default to bert-large-uncased tokenizer. + """ + + def __init__(self, + tokenizer_model_type=None, + cache_dir=None, + add_block_symbols=False, + add_sentinel_token=0, + add_task_mask=False, + add_decoder_mask=False, + **kwargs): + # default to bert-large-uncased tokenizer + if tokenizer_model_type not in PRETRAINED_VOCAB_ARCHIVE_MAP: + tokenizer_model_type = 'bert-large-uncased' + if not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0: + print('loading BertWordPieceTokenizer (', tokenizer_model_type, + ') from cache_dir ', cache_dir) + do_lower_case = not ('-cased' in tokenizer_model_type + or 'chinese' in tokenizer_model_type) + self.text_tokenizer = BertTokenizer.from_pretrained( + tokenizer_model_type, + do_lower_case=do_lower_case, + cache_dir=cache_dir) + if not torch.distributed.is_initialized( + ) or torch.distributed.get_rank() == 0: + print('loaded', tokenizer_model_type) + # disable max len warnings by increasing max len + self.text_tokenizer.max_len = int(1e12) + + # set command tokens from wordpiece tokenizer values + self.num_command_tokens = 6 + self.num_tokens = len(self.text_tokenizer.vocab) + self.num_text_tokens = self.num_tokens - 5 + self.num_type_tokens = 2 + + self._command_tokens = [ + CommandToken('pad', '[PAD]', self.text_tokenizer.vocab['[PAD]']), + CommandToken('ENC', '[CLS]', self.text_tokenizer.vocab['[CLS]']), + CommandToken('MASK', '[MASK]', + self.text_tokenizer.vocab['[MASK]']), + CommandToken('unk', '[UNK]', self.text_tokenizer.vocab['[UNK]']), + CommandToken('sep', '[SEP]', self.text_tokenizer.vocab['[SEP]']), + CommandToken('eos', '[PAD]', self.text_tokenizer.vocab['[PAD]']), + ] + if add_block_symbols: + self._command_tokens.extend([ + CommandToken('sop', '<|startofpiece|>', self.num_tokens), + CommandToken('eop', '<|endofpiece|>', self.num_tokens + 1) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + if add_task_mask: + self._command_tokens.extend([ + CommandToken('gMASK', '[gMASK]', self.num_tokens), + CommandToken('sMASK', '[sMASK]', self.num_tokens + 1) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + if add_decoder_mask: + self._command_tokens.extend( + [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)]) + self.num_tokens += 1 + self.num_command_tokens += 1 + if add_sentinel_token > 0: + for i in range(1, add_sentinel_token): + self._command_tokens.extend([ + CommandToken(f'MASK{i}', f'[MASK{i}]', self.num_tokens), + CommandToken(f'sop{i}', f'<|startofpiece{i}|>', + self.num_tokens + 1) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + self.command_name_map = {tok.name: tok for tok in self._command_tokens} + self.command_token_map = { + tok.token: tok + for tok in self._command_tokens + } + self.command_id_map = {tok.Id: tok for tok in self._command_tokens} + + # set type tokens + self.type_tokens = [ + TypeToken('str0', '', 0), + TypeToken('str1', '', 1), + ] + self.type_name_map = {tok.name: tok for tok in self.type_tokens} + self.type_token_map = {tok.token: tok for tok in self.type_tokens} + self.type_id_map = {tok.Id: tok for tok in self.type_tokens} + + # parse tokens and vocabs from tokenizer + + self._tokens = list(self.text_tokenizer.vocab.keys()) + self._vocab = {k: v for k, v in self.text_tokenizer.vocab.items()} + + self._text_tokens = list(self._tokens) + self._text_token_vocab = { + k: v + for k, v in self.text_tokenizer.vocab.items() + } + + self._command_token_tokens = list(self.command_token_map.keys()) + self._command_token_vocab = { + t: Id + for Id, t in self.command_id_map.items() + } + + self._token_types = list(self.type_token_map.keys()) + self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()} + + def _encode(self, text): + tokens = self.text_tokenizer.tokenize(text) + ids = self.text_tokenizer.convert_tokens_to_ids(tokens) + return ids + + def EncodeAsTokens(self, text, process_fn=None): + """convert wordpiece token to Id""" + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + tokens = self.text_tokenizer.tokenize(processed_text) + return Tokenization(tokens, processed_text, text, asIds=False) + + def IdToToken(self, Id, type_token=False): + """convert Id to sentencpiece token""" + if isinstance(Id, (TypeToken, CommandToken)): + return Id.token + if type_token: + return self.type_id_map[Id].token + if Id in self.command_id_map: + return self.command_id_map[Id].token + return self.text_tokenizer.ids_to_tokens[Id] + + def TokenToId(self, token, type_token=False): + """convert sentencpiece token to Id""" + if isinstance(token, (TypeToken, CommandToken)): + return token.Id + if type_token: + return self.type_token_map[token].Id + return self.text_tokenizer.vocab[token] + + def DecodeIds(self, Ids, type_token=False): + """converts ids to wordpiece tokens and joins them as a text string""" + if type_token: + return ' '.join( + Id.token if isinstance(Id, TypeToken) else self. + type_id_map[Id].token for Id in Ids) + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + Tokens = [] + for Id in Ids: + if Id in self.command_id_map: + Tokens.append(self.command_id_map[Id].token) + elif Id in self.text_tokenizer.ids_to_tokens: + Tokens.append(self.text_tokenizer.ids_to_tokens[Id]) + new_tokens = [] + for token in Tokens: + if token.startswith('##') and len(new_tokens) > 0: + new_tokens[-1] += token[2:] + else: + new_tokens.append(token) + return ' '.join(new_tokens) + + def DecodeTokens(self, Tokens, type_token=False): + """converts wordpiece tokens to a text string""" + if type_token: + return ' '.join( + t.token if isinstance(t, TypeToken) else t for t in Tokens) + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + return ' '.join(Tokens) + + +class GPT2BPETokenizer(Tokenizer): + + def __init__(self, + model_type_or_path, + cache_dir=None, + add_block_symbols=False, + add_task_mask=False, + add_decoder_mask=False, + **kwargs): + self.text_tokenizer = GPT2Tokenizer.from_pretrained( + model_type_or_path, cache_dir=cache_dir) + + # disable max len warnings by increasing max len + self.text_tokenizer.max_len = int(1e12) + self.num_tokens = len(self.text_tokenizer.encoder) + self.num_type_tokens = 2 + if model_type_or_path.startswith('roberta'): + self.num_command_tokens = 6 + self.num_text_tokens = self.num_tokens - 3 + self._command_tokens = [ + CommandToken('pad', '<|endoftext|>', + self.text_tokenizer.encoder['']), + CommandToken('eos', '<|endoftext|>', + self.text_tokenizer.encoder['']), + CommandToken('sep', '[SEP]', + self.text_tokenizer.encoder['']), + CommandToken('ENC', '[CLS]', + self.text_tokenizer.encoder['']), + CommandToken( + 'MASK', + '[MASK]', + self.text_tokenizer.encoder[''], + lstrip=True), + CommandToken('unk', '[UNK]', + self.text_tokenizer.encoder['']) + ] + if add_block_symbols: + self._command_tokens.extend([ + CommandToken('sop', '<|startofpiece|>', self.num_tokens), + CommandToken('eop', '<|endofpiece|>', self.num_tokens + 1) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + else: + self.num_command_tokens = 2 + self.num_text_tokens = self.num_tokens - 1 + self._command_tokens = [ + CommandToken('pad', '<|endoftext|>', + self.text_tokenizer.encoder['<|endoftext|>']), + CommandToken('eos', '<|endoftext|>', + self.text_tokenizer.encoder['<|endoftext|>']) + ] + if add_block_symbols: + self._command_tokens.extend([ + CommandToken('sop', '<|startofpiece|>', self.num_tokens), + CommandToken('eop', '<|endofpiece|>', self.num_tokens + 1), + CommandToken('ENC', '[CLS]', self.num_tokens + 2), + CommandToken( + 'MASK', '[MASK]', self.num_tokens + 3, lstrip=True), + CommandToken('sep', '[SEP]', self.num_tokens + 4), + CommandToken('unk', '[UNK]', self.num_tokens + 5) + ]) + self.num_tokens += 6 + self.num_command_tokens += 6 + if add_block_symbols: + if add_task_mask: + self._command_tokens.extend([ + CommandToken( + 'gMASK', '[gMASK]', self.num_tokens, lstrip=True), + CommandToken( + 'sMASK', '[sMASK]', self.num_tokens + 1, lstrip=True) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + if add_decoder_mask: + self._command_tokens.extend( + [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)]) + self.num_tokens += 1 + self.num_command_tokens += 1 + self.command_name_map = {tok.name: tok for tok in self._command_tokens} + self.command_token_map = { + tok.token: tok + for tok in self._command_tokens + } + self.command_id_map = {tok.Id: tok for tok in self._command_tokens} + + self.type_tokens = [ + TypeToken('str0', '', 0), + TypeToken('str1', '', 1), + ] + self.type_name_map = {tok.name: tok for tok in self.type_tokens} + self.type_token_map = {tok.token: tok for tok in self.type_tokens} + self.type_id_map = {tok.Id: tok for tok in self.type_tokens} + + self._tokens = list(self.text_tokenizer.encoder.keys()) + self._vocab = {k: v for k, v in self.text_tokenizer.encoder.items()} + + self._text_tokens = list(self._tokens) + self._text_token_vocab = { + k: v + for k, v in self.text_tokenizer.encoder.items() + } + + self._command_token_tokens = list(self.command_token_map.keys()) + self._command_token_vocab = { + t: Id + for Id, t in self.command_id_map.items() + } + + self._token_types = list(self.type_token_map.keys()) + self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()} + + for idx, tok in self.command_id_map.items(): + self.text_tokenizer.decoder[idx] = tok.token + + def EncodeAsIds(self, text, process_fn=None): + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + + def split_on_token(tok_extended: CommandToken, text): + result = [] + tok = tok_extended.token + split_text = text.split(tok) + for i, sub_text in enumerate(split_text): + # CommandToken can control whitespace stripping around them. + # We use them for GPT2 and Roberta to have different behavior depending on the special token + # Cf. https://github.com/huggingface/transformers/pull/2778 + # and https://github.com/huggingface/transformers/issues/3788 + # Strip white spaces on the right + if tok_extended.rstrip and i > 0: + # A bit counter-intuitive but we strip the left of the string + # since tok_extended.rstrip means the special token is eating all white spaces on its right + sub_text = sub_text.lstrip() + # Strip white spaces on the left + if tok_extended.lstrip and i < len(split_text) - 1: + sub_text = sub_text.rstrip() # Opposite here + + if i == 0 and not sub_text: + result.append(tok) + elif i == len(split_text) - 1: + if sub_text: + result.append(sub_text) + else: + pass + else: + if sub_text: + result.append(sub_text) + result.append(tok) + return result + + def split_on_tokens(tok_list, text): + if not text.strip(): + return [] + if not tok_list: + return self.text_tokenizer.encode(text) + + tokenized_text = [] + text_list = [text] + for tok in tok_list: + tokenized_text = [] + for sub_text in text_list: + if sub_text not in self._command_token_tokens: + tokenized_text.extend(split_on_token(tok, sub_text)) + else: + tokenized_text.append(sub_text) + text_list = tokenized_text + + return list( + itertools.chain.from_iterable( + (self.text_tokenizer.encode(token) + if token not in self._command_token_tokens else + [self.command_token_map[token].Id] + for token in tokenized_text))) + + no_split_tokens = self._command_tokens + Ids = split_on_tokens(no_split_tokens, processed_text) + tokenization = Tokenization(Ids, processed_text, text) + tokenization.set_command_tokens(self._command_tokens) + return tokenization + + def _encode(self, text): + return self.text_tokenizer.encode(text) + + def EncodeAsTokens(self, text, process_fn=None): + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + tokens = [] + for token in re.findall(self.text_tokenizer.pat, processed_text): + token = ''.join(self.text_tokenizer.bye_encoder[b] + for b in token.encode('utf-8')) + tokens.extend( + bpe_token + for bpe_token in self.text_tokenizer.bpe(token).split(' ')) + tokenization = Tokenization(tokens, processed_text, text, asIds=False) + tokenization.set_command_tokens(self._command_tokens) + return tokenization + + def DecodeAsTokens(self, Ids): + return [self.IdToToken(x) for x in Ids] + + def IdToToken(self, Id, type_token=False): + if isinstance(Id, (TypeToken, CommandToken)): + return Id.token + if type_token: + return self.type_id_map[Id].token + if Id in self.command_id_map: + return self.command_id_map[Id].token + return self.text_tokenizer.decoder[Id] + + def TokenToId(self, token, type_token=False): + if isinstance(token, (TypeToken, CommandToken)): + return token.Id + if type_token: + return self.type_token_map[token].Id + return self.text_tokenizer.encoder[token] + + def DecodeIds(self, Ids, type_token=False): + if type_token: + return ' '.join( + Id.token if isinstance(Id, TypeToken) else self. + type_id_map[Id].token for Id in Ids) + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + return self.text_tokenizer.decode(Ids) + + def DecodeTokens(self, Tokens, type_token=False): + if type_token: + return ' '.join( + t.token if isinstance(t, TypeToken) else t for t in Tokens) + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + return self.text_tokenizer.decode( + [self.TokenToId(tok) for tok in Tokens]) + + +class ChineseSPTokenizer(Tokenizer): + + def __init__(self, + model_path, + add_block_symbols=False, + add_task_mask=False, + add_decoder_mask=False, + **kwargs): + self.text_tokenizer = sp_tokenizer.from_pretrained(model_path) + + self.num_command_tokens = 0 + self.num_text_tokens = self.text_tokenizer.sp.vocab_size() + self.num_tokens = self.num_text_tokens + self.num_type_tokens = 2 + + self._command_tokens = [ + CommandToken('pad', '<|endoftext|>', self.num_text_tokens), + CommandToken('eos', '<|endoftext|>', self.num_text_tokens), + CommandToken('sep', '[SEP]', self.num_text_tokens + 1), + CommandToken('ENC', '[CLS]', self.num_text_tokens + 2), + CommandToken( + 'MASK', '[MASK]', self.num_text_tokens + 3, lstrip=True), + CommandToken('unk', '[UNK]', self.num_text_tokens + 4) + ] + self.num_tokens += 5 + self.num_command_tokens += 6 + if add_block_symbols: + self._command_tokens.extend([ + CommandToken('sop', '<|startofpiece|>', self.num_tokens + 1), + CommandToken('eop', '<|endofpiece|>', self.num_tokens + 2) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + if add_task_mask: + self._command_tokens.extend([ + CommandToken( + 'gMASK', '[gMASK]', self.num_tokens, lstrip=True), + CommandToken( + 'sMASK', '[sMASK]', self.num_tokens + 1, lstrip=True) + ]) + self.num_tokens += 2 + self.num_command_tokens += 2 + if add_decoder_mask: + self._command_tokens.extend( + [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)]) + self.num_tokens += 1 + self.num_command_tokens += 1 + self.command_name_map = {tok.name: tok for tok in self._command_tokens} + self.command_token_map = { + tok.token: tok + for tok in self._command_tokens + } + self.command_id_map = {tok.Id: tok for tok in self._command_tokens} + + self.type_tokens = [ + TypeToken('str0', '', 0), + TypeToken('str1', '', 1), + ] + self.type_name_map = {tok.name: tok for tok in self.type_tokens} + self.type_token_map = {tok.token: tok for tok in self.type_tokens} + self.type_id_map = {tok.Id: tok for tok in self.type_tokens} + + # self._tokens = list(self.text_tokenizer.encoder.keys()) + # self._vocab = {k:v for k,v in self.text_tokenizer.encoder.items()} + # + # self._text_tokens = list(self._tokens) + # self._text_token_vocab = {k:v for k,v in self.text_tokenizer.encoder.items()} + + self._command_token_tokens = list(self.command_token_map.keys()) + self._command_token_vocab = { + t: Id + for Id, t in self.command_id_map.items() + } + + self._token_types = list(self.type_token_map.keys()) + self._token_type_vocab = {t: Id for Id, t in self.type_id_map.items()} + + def _encode(self, text): + ids = self.text_tokenizer.encode(text) + return ids + + def EncodeAsTokens(self, text, process_fn=None): + processed_text = text + if process_fn is not None: + processed_text = process_fn(processed_text) + tokens = self.text_tokenizer.tokenize(processed_text) + tokenization = Tokenization(tokens, processed_text, text, asIds=False) + tokenization.set_command_tokens(self._command_tokens) + return tokenization + # return Tokenization(tokens, processed_text, text, asIds=False) + + def IdToToken(self, Id, type_token=False): + if isinstance(Id, (TypeToken, CommandToken)): + return Id.token + if type_token: + return self.type_id_map[Id].token + if Id in self.command_id_map: + return self.command_id_map[Id].token + elif Id in self.type_id_map: + return self.type_id_map[Id].token + else: + return self.text_tokenizer.convert_id_to_token(int(Id)) + + def TokenToId(self, token, type_token=False): + if isinstance(token, (TypeToken, CommandToken)): + return token.Id + if type_token: + return self.type_token_map[token].Id + return self.text_tokenizer.convert_token_to_id(token) + + def DecodeIds(self, Ids, type_token=False): + if type_token: + return ' '.join( + Id.token if isinstance(Id, TypeToken) else self. + type_id_map[Id].token for Id in Ids) + if isinstance(Ids, Tokenization): + Ids = Ids.tokenization + Ids = list(map(int, Ids)) + pieces = [] + last = 0 + for i, token_id in enumerate(Ids): + if token_id in self.command_id_map: + pieces.append(Ids[last:i]) + pieces.append(token_id) + last = i + 1 + pieces.append(Ids[last:]) + text = '' + for piece in pieces: + if isinstance(piece, int): + text += self.command_id_map[piece].token + elif piece: + text += self.text_tokenizer.decode(piece) + return text + + def DecodeTokens(self, Tokens, type_token=False): + if type_token: + return ' '.join( + t.token if isinstance(t, TypeToken) else t for t in Tokens) + if isinstance(Tokens, Tokenization): + Tokens = Tokens.tokenization + return self.text_tokenizer.decode( + [self.TokenToId(tok) for tok in Tokens]) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization_gpt2.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization_gpt2.py new file mode 100644 index 0000000000000000000000000000000000000000..84947ee882ba79bb24d131194d9b2aa0ba58f647 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/tokenization_gpt2.py @@ -0,0 +1,359 @@ +# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for OpenAI GPT.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import logging +import os +import sys +from io import open + +import json +import regex as re + +from .file_utils import cached_path + +try: + from functools import lru_cache +except ImportError: + # Just a dummy decorator to get the checks to run on python2 + # because honestly I don't want to support a byte-level unicode BPE tokenizer on python 2 right now. + def lru_cache(): + return lambda func: func + + +logger = logging.getLogger(__name__) + +PRETRAINED_VOCAB_ARCHIVE_MAP = { + 'gpt2': '.pytorch_pretrained_bert/gpt2-vocab.json', + 'roberta': '.pytorch_pretrained_bert/roberta-vocab.json' +} +PRETRAINED_MERGES_ARCHIVE_MAP = { + 'gpt2': '.pytorch_pretrained_bert/gpt2-merges.txt', + 'roberta': '.pytorch_pretrained_bert/roberta-merges.txt' +} +PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP = { + 'gpt2': 1024, +} +VOCAB_NAME = 'vocab.json' +MERGES_NAME = 'merges.txt' +SPECIAL_TOKENS_NAME = 'special_tokens.txt' + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + _chr = unichr if sys.version_info[0] == 2 else chr + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [_chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class GPT2Tokenizer(object): + """ + GPT-2 BPE tokenizer. Peculiarities: + - Byte-level BPE + """ + + @classmethod + def from_pretrained(cls, + pretrained_model_name_or_path, + cache_dir=None, + *inputs, + **kwargs): + """ + Instantiate a PreTrainedBertModel from a pre-trained model file. + Download and cache the pre-trained model file if needed. + """ + if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: + vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[ + pretrained_model_name_or_path] + merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[ + pretrained_model_name_or_path] + special_tokens_file = None + else: + vocab_file = os.path.join(pretrained_model_name_or_path, + VOCAB_NAME) + merges_file = os.path.join(pretrained_model_name_or_path, + MERGES_NAME) + special_tokens_file = os.path.join(pretrained_model_name_or_path, + SPECIAL_TOKENS_NAME) + if not os.path.exists(special_tokens_file): + special_tokens_file = None + else: + logger.info('loading special tokens file {}'.format( + special_tokens_file)) + # redirect to the cache, if necessary + # try: + # resolved_vocab_file = cached_path(vocab_file, cache_dir=cache_dir) + # resolved_merges_file = cached_path(merges_file, cache_dir=cache_dir) + # except EnvironmentError: + # logger.error( + # "Model name '{}' was not found in model name list ({}). " + # "We assumed '{}' was a path or url but couldn't find files {} and {} " + # "at this path or url.".format( + # pretrained_model_name_or_path, + # ', '.join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()), + # pretrained_model_name_or_path, + # vocab_file, merges_file)) + # return None + # if resolved_vocab_file == vocab_file and resolved_merges_file == merges_file: + # logger.info("loading vocabulary file {}".format(vocab_file)) + # logger.info("loading merges file {}".format(merges_file)) + # else: + # logger.info("loading vocabulary file {} from cache at {}".format( + # vocab_file, resolved_vocab_file)) + # logger.info("loading merges file {} from cache at {}".format( + # merges_file, resolved_merges_file)) + resolved_vocab_file = vocab_file + resolved_merges_file = merges_file + logger.info('loading vocabulary file {}'.format(vocab_file)) + logger.info('loading merges file {}'.format(merges_file)) + if pretrained_model_name_or_path in PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP: + # if we're using a pretrained model, ensure the tokenizer wont index sequences longer + # than the number of positional embeddings + max_len = PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP[ + pretrained_model_name_or_path] + kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len) + # Instantiate tokenizer. + if special_tokens_file and 'special_tokens' not in kwargs: + special_tokens = open( + special_tokens_file, encoding='utf-8').read().split('\n')[:-1] + else: + special_tokens = kwargs.pop('special_tokens', []) + tokenizer = cls( + resolved_vocab_file, + resolved_merges_file, + special_tokens=special_tokens, + *inputs, + **kwargs) + return tokenizer + + def __init__(self, + vocab_file, + merges_file, + errors='replace', + special_tokens=None, + max_len=None): + self.max_len = max_len if max_len is not None else int(1e12) + self.encoder = json.load(open(vocab_file), encoding='utf-8') + self.decoder = {v: k for k, v in self.encoder.items()} + self.errors = errors # how to handle errors in decoding + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_data] + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + + # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions + self.pat = re.compile( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + + self.special_tokens = {} + self.special_tokens_decoder = {} + self.set_special_tokens(special_tokens) + + def __len__(self): + return len(self.encoder) + len(self.special_tokens) + + def set_special_tokens(self, special_tokens): + """ Add a list of additional tokens to the encoder. + The additional tokens are indexed starting from the last index of the + current vocabulary in the order of the `special_tokens` list. + """ + if not special_tokens: + self.special_tokens = {} + self.special_tokens_decoder = {} + return + self.special_tokens = dict((tok, len(self.encoder) + i) + for i, tok in enumerate(special_tokens)) + self.special_tokens_decoder = { + v: k + for k, v in self.special_tokens.items() + } + logger.info('Special tokens {}'.format(self.special_tokens)) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + + if not pairs: + return token + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: # noqa + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def tokenize(self, text): + """ Tokenize a string. """ + bpe_tokens = [] + for token in re.findall(self.pat, text): + if sys.version_info[0] == 2: + token = ''.join(self.byte_encoder[ord(b)] for b in token) + else: + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend( + bpe_token for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def convert_tokens_to_ids(self, tokens): + """ Converts a sequence of tokens into ids using the vocab. """ + ids = [] + if isinstance(tokens, str) or (sys.version_info[0] == 2 + and isinstance(tokens, unicode)): + if tokens in self.special_tokens: + return self.special_tokens[tokens] + else: + return self.encoder.get(tokens, 0) + for token in tokens: + if token in self.special_tokens: + ids.append(self.special_tokens[token]) + else: + ids.append(self.encoder.get(token, 0)) + if len(ids) > self.max_len: + logger.warning( + 'Token indices sequence length is longer than the specified maximum ' + ' sequence length for this OpenAI GPT model ({} > {}). Running this' + ' sequence through the model will result in indexing errors'. + format(len(ids), self.max_len)) + return ids + + def convert_ids_to_tokens(self, ids, skip_special_tokens=False): + """Converts a sequence of ids in BPE tokens using the vocab.""" + tokens = [] + for i in ids: + if i in self.special_tokens_decoder: + if not skip_special_tokens: + tokens.append(self.special_tokens_decoder[i]) + else: + tokens.append(self.decoder[i]) + return tokens + + def encode(self, text): + return self.convert_tokens_to_ids(self.tokenize(text)) + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors=self.errors) + return text + + def save_vocabulary(self, vocab_path): + """Save the tokenizer vocabulary and merge files to a directory.""" + if not os.path.isdir(vocab_path): + logger.error('Vocabulary path ({}) should be a directory'.format( + vocab_path)) + return + vocab_file = os.path.join(vocab_path, VOCAB_NAME) + merge_file = os.path.join(vocab_path, MERGES_NAME) + special_tokens_file = os.path.join(vocab_path, SPECIAL_TOKENS_NAME) + + with open(vocab_file, 'w', encoding='utf-8') as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + index = 0 + with open(merge_file, 'w', encoding='utf-8') as writer: + writer.write(u'#version: 0.2\n') + for bpe_tokens, token_index in sorted( + self.bpe_ranks.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + 'Saving vocabulary to {}: BPE merge indices are not consecutive.' + ' Please check that the tokenizer is not corrupted!'. + format(merge_file)) + index = token_index + writer.write(' '.join(bpe_tokens) + u'\n') + index += 1 + + index = len(self.encoder) + with open(special_tokens_file, 'w', encoding='utf-8') as writer: + for token, token_index in sorted( + self.special_tokens.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + 'Saving special tokens vocabulary to {}: BPE indices are not consecutive.' + ' Please check that the tokenizer is not corrupted!'. + format(special_tokens_file)) + index = token_index + writer.write(token + u'\n') + index += 1 + + return vocab_file, merge_file, special_tokens_file diff --git a/runtime/Lib/site_packages/models/nlp/mglm/data_utils/wordpiece.py b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/wordpiece.py new file mode 100644 index 0000000000000000000000000000000000000000..b2a3fd3d567d466d968c85585f3dd5e8c3db7b5d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/data_utils/wordpiece.py @@ -0,0 +1,408 @@ +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes. Provided as is from https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/tokenization.py""" # noqa + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import collections +import logging +import os +import unicodedata +from io import open + +from .file_utils import cached_path + +logger = logging.getLogger(__name__) + +PRETRAINED_VOCAB_ARCHIVE_MAP = { + 'bert-base-uncased': + '.pytorch_pretrained_bert/bert-base-uncased-vocab.txt', + 'bert-large-uncased': + '.pytorch_pretrained_bert/bert-large-uncased-vocab.txt', + 'bert-base-cased': + '.pytorch_pretrained_bert/bert-base-cased-vocab.txt', + 'bert-large-cased': + '.pytorch_pretrained_bert/bert-large-cased-vocab.txt', + 'bert-base-multilingual-uncased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt', + 'bert-base-multilingual-cased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt', + 'bert-base-chinese': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt', +} +PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP = { + 'bert-base-uncased': 512, + 'bert-large-uncased': 512, + 'bert-base-cased': 512, + 'bert-large-cased': 512, + 'bert-base-multilingual-uncased': 512, + 'bert-base-multilingual-cased': 512, + 'bert-base-chinese': 512, +} +VOCAB_NAME = 'vocab.txt' + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, 'r', encoding='utf-8') as reader: + while True: + token = reader.readline() + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class BertTokenizer(object): + """Runs end-to-end tokenization: punctuation splitting + wordpiece""" + + def __init__(self, + vocab_file, + do_lower_case=True, + max_len=None, + do_basic_tokenize=True, + never_split=('[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]')): + """Constructs a BertTokenizer. + + Args: + vocab_file: Path to a one-wordpiece-per-line vocabulary file + do_lower_case: Whether to lower case the input + Only has an effect when do_wordpiece_only=False + do_basic_tokenize: Whether to do basic tokenization before wordpiece. + max_len: An artificial maximum length to truncate tokenized sequences to; + Effective maximum length is always the minimum of this + value (if specified) and the underlying BERT model's + sequence length. + never_split: List of tokens which will never be split during tokenization. + Only has an effect when do_wordpiece_only=False + """ + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + 'model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' + .format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict([ + (ids, tok) for tok, ids in self.vocab.items() + ]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, never_split=never_split) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + self.max_len = max_len if max_len is not None else int(1e12) + + def tokenize(self, text): + if self.do_basic_tokenize: + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def convert_tokens_to_ids(self, tokens): + """Converts a sequence of tokens into ids using the vocab.""" + ids = [] + for token in tokens: + ids.append(self.vocab[token]) + if len(ids) > self.max_len: + logger.warning( + 'Token indices sequence length is longer than the specified maximum ' + ' sequence length for this BERT model ({} > {}). Running this' + ' sequence through BERT will result in indexing errors'.format( + len(ids), self.max_len)) + return ids + + def convert_ids_to_tokens(self, ids): + """Converts a sequence of ids in wordpiece tokens using the vocab.""" + tokens = [] + for i in ids: + tokens.append(self.ids_to_tokens[i]) + return tokens + + @classmethod + def from_pretrained(cls, + pretrained_model_name_or_path, + cache_dir=None, + *inputs, + **kwargs): + """ + Instantiate a PreTrainedBertModel from a pre-trained model file. + Download and cache the pre-trained model file if needed. + """ + if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: + vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[ + pretrained_model_name_or_path] + else: + vocab_file = pretrained_model_name_or_path + if os.path.isdir(vocab_file): + vocab_file = os.path.join(vocab_file, VOCAB_NAME) + # redirect to the cache, if necessary + try: + resolved_vocab_file = cached_path(vocab_file, cache_dir=cache_dir) + except EnvironmentError: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + 'associated to this path or url.'.format( + pretrained_model_name_or_path, + ', '.join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()), + vocab_file)) + return None + if resolved_vocab_file == vocab_file: + logger.info('loading vocabulary file {}'.format(vocab_file)) + else: + logger.info('loading vocabulary file {} from cache at {}'.format( + vocab_file, resolved_vocab_file)) + if pretrained_model_name_or_path in PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP: + # if we're using a pretrained model, ensure the tokenizer wont index sequences longer + # than the number of positional embeddings + max_len = PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP[ + pretrained_model_name_or_path] + kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len) + # Instantiate tokenizer. + tokenizer = cls(resolved_vocab_file, *inputs, **kwargs) + return tokenizer + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, + do_lower_case=True, + never_split=('[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]')): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + self.never_split = never_split + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = self._clean_text(text) + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case and token not in self.never_split: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(' '.join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize('NFD', text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == 'Mn': + continue + output.append(char) + return ''.join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + if text in self.never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return [''.join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(' ') + output.append(char) + output.append(' ') + else: + output.append(char) + return ''.join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # noqa + (cp >= 0x3400 and cp <= 0x4DBF) or # noqa + (cp >= 0x20000 and cp <= 0x2A6DF) or # noqa + (cp >= 0x2A700 and cp <= 0x2B73F) or # noqa + (cp >= 0x2B740 and cp <= 0x2B81F) or # noqa + (cp >= 0x2B820 and cp <= 0x2CEAF) or # noqa + (cp >= 0xF900 and cp <= 0xFAFF) or # noqa + (cp >= 0x2F800 and cp <= 0x2FA1F)): # noqa + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(' ') + else: + output.append(char) + return ''.join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token='[UNK]', max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + >>> input = "unaffable" + >>> output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = ''.join(chars[start:end]) + if start > 0: + substr = '##' + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == ' ' or char == '\t' or char == '\n' or char == '\r': + return True + cat = unicodedata.category(char) + if cat == 'Zs': + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == '\t' or char == '\n' or char == '\r': + return False + cat = unicodedata.category(char) + if cat.startswith('C'): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith('P'): + return True + return False diff --git a/runtime/Lib/site_packages/models/nlp/mglm/generation_utils.py b/runtime/Lib/site_packages/models/nlp/mglm/generation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6db75b2de91b794c19a9ab0a298e753d0ad6b1cf --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/generation_utils.py @@ -0,0 +1,483 @@ +# Copyright 2020 The HuggingFace Inc. team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from collections import UserDict +from typing import Iterable, List, Optional, Tuple + +import torch + +PROCESS_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_beams, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using any class inheriting from :class:`~transformers.PretrainedTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + `What are input IDs? <../glossary.html#input-ids>`__ + next_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, 2 * num_beams)`): + Current scores of the top :obj:`2 * num_beams` non-finished beam hypotheses. + next_tokens (:obj:`torch.LongTensor` of shape :obj:`(batch_size, 2 * num_beams)`): + :obj:`input_ids` of the tokens corresponding to the top :obj:`2 * num_beams` non-finished beam hypotheses. + next_indices (:obj:`torch.LongTensor` of shape :obj:`(batch_size, 2 * num_beams)`): + Beam indices indicating to which beam hypothesis the :obj:`next_tokens` correspond. + pad_token_id (:obj:`int`, `optional`): + The id of the `padding` token. + eos_token_id (:obj:`int`, `optional`): + The id of the `end-of-sequence` token. + + Return: + :obj:`UserDict`: A dictionary composed of the fields as defined above: + + - **next_beam_scores** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Updated + scores of all non-finished beams. + - **next_beam_tokens** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Next tokens + to be added to the non-finished beam_hypotheses. + - **next_beam_indices** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Beam indices + indicating to which beam the next tokens shall be added. + +""" + +FINALIZE_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_beams, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using any class inheriting from :class:`~transformers.PretrainedTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + `What are input IDs? <../glossary.html#input-ids>`__ + final_beam_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`): + The final scores of all non-finished beams. + final_beam_tokens (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`): + The last tokens to be added to the non-finished beam_hypotheses. + final_beam_indices (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`): + The beam indices indicating to which beam the :obj:`final_beam_tokens` shall be added. + pad_token_id (:obj:`int`, `optional`): + The id of the `padding` token. + eos_token_id (:obj:`int`, `optional`): + The id of the `end-of-sequence` token. + + Return: + :obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`: The generated + sequences. The second dimension (sequence_length) is either equal to :obj:`max_length` or shorter if all + batches finished early due to the :obj:`eos_token_id`. + +""" + + +class BeamScorer(ABC): + """ + Abstract base class for all beam scorers that are used for :meth:`~transformers.PretrainedModel.beam_search` and + :meth:`~transformers.PretrainedModel.beam_sample`. + """ + + @abstractmethod + def process(self, input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + **kwargs) -> Tuple[torch.Tensor]: + raise NotImplementedError('This is an abstract method.') + + @abstractmethod + def finalize(self, input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, **kwargs) -> torch.LongTensor: + raise NotImplementedError('This is an abstract method.') + + +class BeamSearchScorer(BeamScorer): + r""" + :class:`transformers.BeamScorer` implementing standard beam search decoding. + + Adapted in part from `Facebook's XLM beam search code + `__. + + Args: + batch_size (:obj:`int`): + Batch Size of :obj:`input_ids` for which beam search decoding is run in parallel. + max_length (:obj:`int`): + The maximum length of the sequence to be generated. + num_beams (:obj:`int`): + Number of beams for beam search. + device (:obj:`torch.device`): + Defines the device type (*e.g.*, :obj:`"cpu"` or :obj:`"cuda"`) on which this instance of + :obj:`BeamSearchScorer` will be allocated. + length_penalty (:obj:`float`, `optional`, defaults to 1.0): + Exponential penalty to the length. 1.0 means no penalty. Set to values < 1.0 in order to encourage the + model to generate shorter sequences, to a value > 1.0 in order to encourage the model to produce longer + sequences. + do_early_stopping (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to stop the beam search when at least ``num_beams`` sentences are finished per batch or not. + num_beam_hyps_to_keep (:obj:`int`, `optional`, defaults to 1): + The number of beam hypotheses that shall be returned upon calling + :meth:`~transformer.BeamSearchScorer.finalize`. + """ + + def __init__( + self, + batch_size: int, + max_length: int, + num_beams: int, + device: torch.device, + length_penalty: Optional[float] = 1.0, + do_early_stopping: Optional[bool] = False, + num_beam_hyps_to_keep: Optional[int] = 1, + ): + self.max_length = max_length + self.num_beams = num_beams + self.device = device + self.length_penalty = length_penalty + self.do_early_stopping = do_early_stopping + self.num_beam_hyps_to_keep = num_beam_hyps_to_keep + + self._is_init = False + self._beam_hyps = [ + BeamHypotheses( + num_beams=self.num_beams, + max_length=self.max_length, + length_penalty=self.length_penalty, + early_stopping=self.do_early_stopping, + ) for _ in range(batch_size) + ] + self._done = torch.tensor([False for _ in range(batch_size)], + dtype=torch.bool, + device=self.device) + + # if not isinstance(num_beams, int) or num_beams <= 1: + # raise ValueError( + # ) + + @property + def is_done(self) -> bool: + return self._done.all() + + def process(self, + input_ids: torch.LongTensor, + next_scores: torch.FloatTensor, + next_tokens: torch.LongTensor, + next_indices: torch.LongTensor, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + mems=None) -> Tuple[torch.Tensor]: + cur_len = input_ids.shape[-1] + batch_size = len(self._beam_hyps) + assert batch_size == (input_ids.shape[0] // self.num_beams) + if isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + device = next_scores.device + next_beam_scores = torch.zeros((batch_size, self.num_beams), + dtype=next_scores.dtype, + device=device) + next_beam_tokens = torch.zeros((batch_size, self.num_beams), + dtype=next_tokens.dtype, + device=device) + next_beam_indices = torch.zeros((batch_size, self.num_beams), + dtype=next_indices.dtype, + device=device) + + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + assert ( + len(beam_hyp) >= self.num_beams + ), 'Batch can only be done if at least {} beams have been generated'.format( + self.num_beams) + assert ( + eos_token_id is not None and pad_token_id is not None + ), 'generated beams >= num_beams -> eos_token_id and pad_token have to be defined' + # pad the batch + next_beam_scores[batch_idx, :] = 0 + next_beam_tokens[batch_idx, :] = pad_token_id + next_beam_indices[batch_idx, :] = 0 + continue + + # next tokens for this sentence + beam_idx = 0 + for beam_token_rank, (next_token, next_score, + next_index) in enumerate( + zip(next_tokens[batch_idx], + next_scores[batch_idx], + next_indices[batch_idx])): + batch_beam_idx = batch_idx * self.num_beams + next_index + # add to generated hypotheses if end of sentence + if (eos_token_id is not None) and (next_token.item() + in eos_token_id): + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.num_beams + if is_beam_token_worse_than_top_num_beams: + continue + beam_hyp.add( + input_ids[batch_beam_idx].clone(), + next_score.item(), + mems=[mem[[next_index.item()]] + for mem in mems] if mems else None) + else: + # add next predicted token since it is not eos_token + next_beam_scores[batch_idx, beam_idx] = next_score + next_beam_tokens[batch_idx, beam_idx] = next_token + next_beam_indices[batch_idx, beam_idx] = batch_beam_idx + beam_idx += 1 + + # once the beam for next step is full, don't add more tokens to it. + if beam_idx == self.num_beams: + break + + if beam_idx < self.num_beams: + raise ValueError( + f'At most {self.num_beams} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id: {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected.' # noqa + ) # noqa + + # Check if we are done so that we can save a pad step if all(done) + self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done( + next_scores[batch_idx].max().item(), cur_len) + + return UserDict({ + 'next_beam_scores': next_beam_scores.view(-1), + 'next_beam_tokens': next_beam_tokens.view(-1), + 'next_beam_indices': next_beam_indices.view(-1), + }) + + def finalize(self, + input_ids: torch.LongTensor, + final_beam_scores: torch.FloatTensor, + final_beam_tokens: torch.LongTensor, + final_beam_indices: torch.LongTensor, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + mems=None) -> Tuple[torch.LongTensor, List[torch.Tensor]]: + batch_size = len(self._beam_hyps) + + # finalize all open beam hypotheses and add to generated hypotheses + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + continue + + # need to add best num_beams hypotheses to generated hyps + for beam_id in range(self.num_beams): + batch_beam_idx = batch_idx * self.num_beams + beam_id + final_score = final_beam_scores[batch_beam_idx].item() + final_tokens = input_ids[batch_beam_idx] + beam_hyp.add( + final_tokens, + final_score, + mems=[mem[[batch_beam_idx]] + for mem in mems] if mems else None) + + # select the best hypotheses + sent_lengths = input_ids.new(batch_size * self.num_beam_hyps_to_keep) + best = [] + + # retrieve best hypotheses + for i, beam_hyp in enumerate(self._beam_hyps): + sorted_hyps = sorted(beam_hyp.beams, key=lambda x: x[0]) + for j in range(self.num_beam_hyps_to_keep): + best_hyp, mems = sorted_hyps.pop()[1:] + sent_lengths[self.num_beam_hyps_to_keep * i + + j] = len(best_hyp) + best.append((best_hyp, mems)) + + # prepare for adding eos + sent_max_len = min(sent_lengths.max().item(), self.max_length) + decoded: torch.LongTensor = input_ids.new( + batch_size * self.num_beam_hyps_to_keep, sent_max_len) + # shorter batches are padded if needed + if sent_lengths.min().item() != sent_lengths.max().item(): + assert pad_token_id is not None, '`pad_token_id` has to be defined' + decoded.fill_(pad_token_id) + + # fill with hypotheses and eos_token_id if the latter fits in + mems = [] + for i, (hypo, mem) in enumerate(best): + decoded[i, :sent_lengths[i]] = hypo + if sent_lengths[i] < sent_max_len: + decoded[i, sent_lengths[i]] = eos_token_id + mems.append(mem) + mems = [ + torch.cat([mem[i] for mem in mems], dim=0) + for i in range(len(mems[0])) + ] if mems and mems[0] else None + return decoded, mems + + +class BeamHypotheses: + + def __init__(self, num_beams: int, max_length: int, length_penalty: float, + early_stopping: bool): + """ + Initialize n-best list of hypotheses. + """ + self.max_length = max_length - 1 # ignoring bos_token + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.num_beams = num_beams + self.beams = [] + self.worst_score = 1e9 + + def __len__(self): + """ + Number of hypotheses in the list. + """ + return len(self.beams) + + def add(self, hyp: torch.LongTensor, sum_logprobs: float, mems=None): + """ + Add a new hypothesis to the list. + """ + score = sum_logprobs / (max(hyp.shape[-1], 1)**self.length_penalty) + if len(self) < self.num_beams or score > self.worst_score: + self.beams.append((score, hyp, mems)) + if len(self) > self.num_beams: + sorted_next_scores = sorted([ + (s, idx) for idx, (s, _, _) in enumerate(self.beams) + ]) + del self.beams[sorted_next_scores[0][1]] + self.worst_score = sorted_next_scores[1][0] + else: + self.worst_score = min(score, self.worst_score) + + def is_done(self, best_sum_logprobs: float, cur_len: int) -> bool: + """ + If there are enough hypotheses and that none of the hypotheses being generated can become better than the worst + one in the heap, then we are done with this sentence. + """ + + if len(self) < self.num_beams: + return False + elif self.early_stopping: + return True + else: + cur_score = best_sum_logprobs / cur_len**self.length_penalty + ret = self.worst_score >= cur_score + return ret + + +class LogitsProcessor(ABC): + """Abstract base class for all logit processors that can be applied during generation.""" + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + """Torch method for processing logits.""" + raise NotImplementedError( + f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.' + ) + + +class LogitsProcessorList(list): + """ + This class can be used to create a list of :class:`~transformers.LogitsProcessor` or + :class:`~transformers.LogitsWarper` to subsequently process a :obj:`scores` input tensor. This class inherits from + list and adds a specific `__call__` method to apply each :class:`~transformers.LogitsProcessor` or + :class:`~transformers.LogitsProcessor` to the inputs. + """ + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + for processor in self: + scores = processor(input_ids, scores) + return scores + + +class MinLengthLogitsProcessor(LogitsProcessor): + r""" + :class:`transformers.LogitsProcessor` enforcing a min-length by setting EOS probability to 0. + + Args: + min_length (:obj:`int`): + The minimum length below which the score of :obj:`eos_token_id` is set to :obj:`-float("Inf")`. + eos_token_id (:obj:`int`): + The id of the `end-of-sequence` token. + """ + + def __init__(self, min_length: int, eos_token_id: int): + if not isinstance(min_length, int) or min_length < 0: + raise ValueError( + f'`min_length` has to be a positive integer, but is {min_length}' + ) + + if not isinstance(eos_token_id, int) or eos_token_id < 0: + raise ValueError( + f'`eos_token_id` has to be a positive integer, but is {eos_token_id}' + ) + + self.min_length = min_length + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + cur_len = input_ids.shape[-1] + if cur_len < self.min_length: + scores[:, self.eos_token_id] = -float('inf') + return scores + + +class NoRepeatNGramLogitsProcessor(LogitsProcessor): + r""" + :class:`transformers.LogitsProcessor` that enforces no repetition of n-grams. See `Fairseq + `__. + + Args: + ngram_size (:obj:`int`): + All ngrams of size :obj:`ngram_size` can only occur once. + """ + + def __init__(self, ngram_size: int): + if not isinstance(ngram_size, int) or ngram_size <= 0: + raise ValueError( + f'`ngram_size` has to be a strictly positive integer, but is {ngram_size}' + ) + self.ngram_size = ngram_size + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + num_batch_hypotheses = scores.shape[0] + cur_len = input_ids.shape[-1] + banned_batch_tokens = self._calc_banned_ngram_tokens( + input_ids, num_batch_hypotheses, cur_len) + + for i, banned_tokens in enumerate(banned_batch_tokens): + scores[i, banned_tokens] = -float('inf') + + return scores + + def _calc_banned_ngram_tokens(self, prev_input_ids: torch.Tensor, + num_hypos: int, + cur_len: int) -> List[Iterable[int]]: + """Copied from fairseq for no_repeat_ngram in beam_search""" + if cur_len + 1 < self.ngram_size: + # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet + return [[] for _ in range(num_hypos)] + generated_ngrams = [{} for _ in range(num_hypos)] + for idx in range(num_hypos): + gen_tokens = prev_input_ids[idx].tolist() + generated_ngram = generated_ngrams[idx] + for ngram in zip(*[gen_tokens[i:] + for i in range(self.ngram_size)]): + prev_ngram_tuple = tuple(ngram[:-1]) + generated_ngram[prev_ngram_tuple] = generated_ngram.get( + prev_ngram_tuple, []) + [ngram[-1]] + + def _get_generated_ngrams(hypo_idx): + # Before decoding the next token, prevent decoding of ngrams that have already appeared + start_idx = cur_len + 1 - self.ngram_size + ngram_idx = tuple(prev_input_ids[hypo_idx, + start_idx:cur_len].tolist()) + return generated_ngrams[hypo_idx].get(ngram_idx, []) + + banned_tokens = [ + _get_generated_ngrams(hypo_idx) for hypo_idx in range(num_hypos) + ] + return banned_tokens diff --git a/runtime/Lib/site_packages/models/nlp/mglm/mglm_for_text_summarization.py b/runtime/Lib/site_packages/models/nlp/mglm/mglm_for_text_summarization.py new file mode 100644 index 0000000000000000000000000000000000000000..079cfd46d7be3a3ae28538a4a9827284fe165f2f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/mglm_for_text_summarization.py @@ -0,0 +1,428 @@ +# Copyright (c) 2022 Zhipu.AI + +import os +import random +from os import path as osp +from typing import Dict + +import numpy as np +import torch +import torch.nn.functional as F +from megatron_util import mpu + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import OutputKeys +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.megatron_utils import init_megatron_util +from .arguments import get_args +from .generation_utils import BeamSearchScorer +from .train_utils import get_model +from .utils import load_checkpoint + +__all__ = ['MGLMForTextSummarization'] + + +def setup_args(args): + args.block_lm = True + args.task_mask = True + args.cloze_eval = True + args.num_layers = 24 + args.hidden_size = 1536 + args.num_attention_heads = 16 + args.max_position_embeddings = 1024 + args.tokenizer_type = 'ChineseSPTokenizer' + args.load_pretrained = '' + args.DDP_impl = 'none' + args.model_parallel_size = 1 + args.fp16 = True + args.cache_dir = 'cache' + args.out_seq_length = 200 + args.seq_length = 512 + args.temperature = 0.9 + args.top_k = 2 + args.top_p = 0.8 + args.frequency_penalty = 0.1 + args.presence_penalty = 0.1 + args.mem_length = args.seq_length + args.mem_length - 1 + return args + + +def setup_model(args): + """Setup model and optimizer.""" + + model = get_model(args, model_type='generation') + + if args.load_pretrained is not None: + args.no_load_optim = True + args.load = args.load_pretrained + _ = load_checkpoint(model, None, None, args) + + return model + + +def get_masks_and_position_ids(data, + eod_token, + reset_position_ids, + reset_attention_mask, + loss_mask=None, + attention_mask=None, + set_loss_mask=False, + mem_length=None): + # Extract batch size and sequence length. + batch_size, seq_length = data.size() + + # Attention mask (lower triangular). + if mem_length: + if attention_mask is None: + attention_mask = torch.ones( + (1, seq_length, seq_length + mem_length), device=data.device) + attention_mask = torch.tril( + torch.triu(attention_mask, 1 - seq_length + mem_length), + mem_length) + else: + if reset_attention_mask: + att_mask_batch = batch_size + else: + att_mask_batch = 1 + if attention_mask is None: + attention_mask = torch.ones( + (att_mask_batch, seq_length, seq_length), device=data.device) + attention_mask = torch.tril(attention_mask) + attention_mask = attention_mask.unsqueeze(1) + + # Loss mask. + if loss_mask is None: + loss_mask = torch.ones( + data.size(), dtype=torch.float, device=data.device) + + # Position ids. + position_ids = torch.arange( + seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data) + if set_loss_mask: + loss_mask[data == eod_token] = 0.0 + # We need to clone as the ids will be modifed based on batch index. + if reset_position_ids: + position_ids = position_ids.clone() + + if reset_position_ids or reset_attention_mask: + # Loop through the batches: + for b in range(batch_size): + + # Find indecies where EOD token is. + eod_index = position_ids[b, data[b] == eod_token] + # Detach indecies from positions if going to modify positions. + if reset_position_ids: + eod_index = eod_index.clone() + + # Loop through EOD indecies: + prev_index = 0 + for j in range(eod_index.size()[0]): + i = eod_index[j] + # Mask attention loss. + if reset_attention_mask: + attention_mask[b, 0, (i + 1):, :(i + 1)] = 0 + # Reset positions. + if reset_position_ids: + position_ids[b, (i + 1):] -= (i + 1 - prev_index) + prev_index = i + 1 + + return attention_mask, loss_mask, position_ids + + +def get_batch(context_tokens, device, args): + tokens = context_tokens + tokens = tokens.view(args.batch_size, -1).contiguous() + tokens = tokens.to(device) + + # Get the masks and postition ids. + if args.block_lm: + attention_mask = torch.tensor([tokens.size(1)], + device=device, + dtype=torch.long) + position_ids = torch.arange( + tokens.size(1), device=device, dtype=torch.long) + if not args.no_block_position: + block_position_ids = torch.zeros( + tokens.size(1), device=device, dtype=torch.long) + position_ids = torch.stack((position_ids, block_position_ids), + dim=0) + position_ids = position_ids.unsqueeze(0) + else: + attention_mask, loss_mask, position_ids = get_masks_and_position_ids( + tokens, + args.eod_token, + reset_position_ids=False, + reset_attention_mask=False, + set_loss_mask=False, + mem_length=args.mem_length) + + return tokens, attention_mask, position_ids + + +def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')): + # This function has been mostly taken from huggingface conversational ai code at + # https://medium.com/huggingface/how-to-build-a-state-of-the-art-conversational-ai-with-transfer-learning-2d818ac26313 + + if top_k > 0: + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p > 0.0: + # convert to 1D + logits = logits.view(logits.size()[1]).contiguous() + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + indices_to_remove = sorted_indices[sorted_indices_to_remove] + logits[indices_to_remove] = filter_value + # going back to 2D + logits = logits.view(1, -1).contiguous() + + return logits + + +def sample_sequence(model, + tokenizer, + context_tokens, + context_length, + args, + device, + mems=None, + end_tokens=None): + if not args.block_lm: + context_tokens, attention_mask, position_ids = get_batch( + context_tokens, device, args) + tokens = torch.empty((args.num_beams, 0), + device=context_tokens.device, + dtype=torch.long) + else: + tokens = context_tokens.new_full((1, 1), + tokenizer.get_command('sop').Id) + counter = 0 + if mems is None: + mems = [] + if end_tokens is None: + end_tokens = [args.eod_token] + + last_beam_num = 1 + output_tokens_list = [] + generated_tokens_list = [] + + while counter < args.out_seq_length: + if counter == 0 and not args.block_lm: + next_token_logits, *mems = model(context_tokens, position_ids, + attention_mask, *mems) + else: + if args.block_lm: + if args.no_block_position: + position_ids = context_tokens.new_full( + (last_beam_num, 1), context_length + counter) + else: + position_ids = context_tokens.new_ones(last_beam_num, 2, 1) + position_ids[:, 0] = context_length + position_ids[:, 1] = counter + 1 + attention_mask = context_tokens.new_zeros( + [1], device=context_tokens.device, dtype=torch.long) + else: + position_ids = context_tokens.new_ones((last_beam_num, 1)) * ( + context_length + counter - 1) + attention_mask = context_tokens.new_ones( + last_beam_num, + 1, + 1, + args.mem_length + 1, + device=context_tokens.device, + dtype=torch.float) + last_token = tokens[:, -1:] + next_token_logits, *mems = model(last_token, position_ids, + attention_mask, *mems) + next_token_logits = next_token_logits[:, -1] + + next_token_logits /= args.temperature + frequency_count = torch.zeros(next_token_logits.shape) + for tk in output_tokens_list: + frequency_count[0][tk] += 1 + + next_token_logits -= (args.frequency_penalty + * frequency_count).to(device) + next_token_logits -= ( + args.presence_penalty * # noqa + (frequency_count > 0)).to(device) + + next_token_logits = top_k_logits( + next_token_logits, top_k=args.top_k, top_p=args.top_p) + log_probs = F.softmax(next_token_logits, dim=-1) + prev = torch.multinomial(log_probs, num_samples=1)[0] + is_end = prev.item() in end_tokens + if is_end: + break + decode_tokens = tokenizer.DecodeIds([prev.item()]) # noqa + generated_tokens_list.append(prev.item()) + prev = prev.view(1, 1) + tokens = prev if tokens is None else torch.cat((tokens, prev), dim=1) + counter += 1 + output_tokens_list = tokens.view(-1).contiguous() + return torch.cat((context_tokens, tokens), dim=1), mems + + +def read_context(tokenizer, args, context): + terminate_runs, skip_run = 0, 0 # noqa + if mpu.get_model_parallel_rank() == 0: + while True: + # raw_text = input("\nContext prompt (stop to exit) >>> ") + raw_text = context + if not raw_text: + print('Prompt should not be empty!') + break + # if raw_text == "stop": + # terminate_runs = 1 + # break + generation_mask = '[gMASK]' if args.task_mask else '[MASK]' + if args.block_lm and 'MASK]' not in raw_text: + raw_text += ' ' + generation_mask + # output.write(raw_text) + context_tokens = tokenizer.EncodeAsIds(raw_text).tokenization + if args.block_lm: + context_tokens = [tokenizer.get_command('ENC').Id + ] + context_tokens + if not raw_text.endswith('[gMASK]'): + context_tokens = context_tokens + [ + tokenizer.get_command('eos').Id + ] + context_length = len(context_tokens) + + if context_length >= args.seq_length: + print('\nContext length', context_length, + '\nPlease give smaller context than the window length!') + break + break + else: + context_length = 0 + + terminate_runs_tensor = torch.cuda.LongTensor([terminate_runs]) + torch.distributed.broadcast( + terminate_runs_tensor, + mpu.get_model_parallel_src_rank(), + group=mpu.get_model_parallel_group()) + terminate_runs = terminate_runs_tensor[0].item() + + if terminate_runs == 1: + return terminate_runs, None, None, None + + context_length_tensor = torch.cuda.LongTensor([context_length]) + + torch.distributed.broadcast( + context_length_tensor, + mpu.get_model_parallel_src_rank(), + group=mpu.get_model_parallel_group()) + context_length = context_length_tensor[0].item() + if mpu.get_model_parallel_rank() == 0: + context_tokens_tensor = torch.cuda.LongTensor(context_tokens) + else: + context_tokens_tensor = torch.cuda.LongTensor([0] * context_length) + torch.distributed.broadcast( + context_tokens_tensor, + mpu.get_model_parallel_src_rank(), + group=mpu.get_model_parallel_group()) + if mpu.get_model_parallel_rank() != 0: + raw_text = tokenizer.DecodeIds(context_tokens_tensor.tolist()) + return terminate_runs, raw_text, context_tokens_tensor, context_length + + +@MODELS.register_module(Tasks.text_summarization, module_name=Models.mglm) +class MGLMForTextSummarization(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the text summarization model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + + from .configure_data import prepare_tokenizer + # Disable CuDNN. + torch.backends.cudnn.enabled = False + # Arguments. + self.args = setup_args(get_args()) + self.args.load_pretrained = model_dir + + try: + init_megatron_util(model_dir=model_dir) + except AssertionError: + print('megatron initialized twice') + + # setting default batch size to 1 + self.args.batch_size = 1 + self.args.tokenizer_path = model_dir + self.tokenizer = prepare_tokenizer(self.args) + self.model = setup_model(self.args) + self.cfg = Config.from_file( + osp.join(model_dir, ModelFile.CONFIGURATION)) + + def forward(self, input: Dict[str, str]) -> Dict[str, str]: + pass + + def generate(self, input: Dict[str, str]) -> Dict[str, str]: + model = self.model + tokenizer = self.tokenizer + args = self.args + device = torch.cuda.current_device() + model.eval() + + context = input['text'] + self.cfg.model.prompt + with torch.no_grad(): + terminate_runs, raw_text, context_tokens_tensor, context_length = read_context( + tokenizer, args, context) + mems = [] + tokens, attention_mask, position_ids = get_batch( + context_tokens_tensor, device, args) + mask_tokens = ['MASK', 'sMASK', 'gMASK' + ] if args.task_mask else ['MASK'] + mask_tokens = [ + tokenizer.get_command(token).Id for token in mask_tokens + ] + end_tokens = [tokenizer.get_command('eop').Id, args.eod_token] + + mask_positions = [] + for token in mask_tokens: + mask_positions += (context_tokens_tensor == token).nonzero( + as_tuple=True)[0].tolist() + mask_positions.sort() + if args.no_block_position: + for mask_position in mask_positions: + position_ids[0, mask_position + 1:] += args.out_seq_length + _, *mems = model(tokens, position_ids, attention_mask, *mems) + for mask_position in mask_positions: + if args.no_block_position: + position = position_ids[0, mask_position].item() + else: + position = mask_position + tokens, mems, = sample_sequence( + model, + tokenizer, + tokens, + position, + args, + device, + mems=mems, + end_tokens=end_tokens) + output_tokens_list = tokens.view(-1).contiguous() + trim_decode_tokens = tokenizer.DecodeIds( + output_tokens_list.tolist()) + res = trim_decode_tokens.split('<|startofpiece|>')[-1] + print(res) + return {OutputKeys.TEXT: res} diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/__init__.py b/runtime/Lib/site_packages/models/nlp/mglm/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..84c55ae3a0c875504043719fee0aa9011cf06ef1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .distributed import (DistributedDataParallel, + PyTorchDistributedDataParallel) +from .downstream import (GLMForMultiTokenCloze, GLMForMultiTokenClozeFast, + GLMForSequenceClassification, GLMForSingleTokenCloze) +from .modeling_glm import (GLMModel, + glm_get_params_for_weight_decay_optimization) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/distributed.py b/runtime/Lib/site_packages/models/nlp/mglm/model/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..328721dc1974ac37078af2063edfb4e8765df5f1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/distributed.py @@ -0,0 +1,126 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed as dist +from megatron_util import mpu +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors +from torch.autograd import Variable +from torch.nn.modules import Module +from torch.nn.parallel.distributed import DistributedDataParallel as DDP + + +class PyTorchDistributedDataParallel(DDP): + + def named_parameters(self, prefix: str = '', recurse: bool = True): + return self.module.named_parameters(prefix=prefix, recurse=recurse) + + def state_dict(self, destination=None, prefix='', keep_vars=False): + sd = self.module.state_dict(destination, prefix, keep_vars) + return sd + + def load_state_dict(self, state_dict, strict=True): + return self.module.load_state_dict(state_dict, strict=strict) + + +class DistributedDataParallel(Module): + + def __init__(self, module): + super(DistributedDataParallel, self).__init__() + self.warn_on_half = True if dist._backend == dist.dist_backend.GLOO else False + + self.module = module + self.data_parallel_group = mpu.get_data_parallel_group() + src_rank = mpu.get_model_parallel_rank() + for p in self.module.parameters(): + if torch.is_tensor(p): + dist.broadcast(p, src_rank, group=self.data_parallel_group) + + def allreduce_params(reduce_after=True, + no_scale=False, + fp32_allreduce=False): + if (self.needs_reduction): + self.needs_reduction = False + buckets = {} + for name, param in self.module.named_parameters(): + if param.requires_grad and param.grad is not None: + tp = (param.data.type()) + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(param) + if self.warn_on_half: + if torch.cuda.HalfTensor in buckets: + print( + 'WARNING: gloo dist backend for half parameters may be extremely slow. It is recommended to use the NCCL backend in this case.' # noqa + ) + self.warn_on_half = False + for tp in buckets: + bucket = buckets[tp] + grads = [param.grad.data for param in bucket] + coalesced = _flatten_dense_tensors(grads) + if fp32_allreduce: + coalesced = coalesced.float() + if not no_scale and not reduce_after: + coalesced /= dist.get_world_size( + group=self.data_parallel_group) + dist.all_reduce(coalesced, group=self.data_parallel_group) + torch.cuda.synchronize() + if not no_scale and reduce_after: + coalesced /= dist.get_world_size( + group=self.data_parallel_group) + for buf, synced in zip( + grads, _unflatten_dense_tensors(coalesced, grads)): + buf.copy_(synced) + + self.hook_handles = [] + self.hooks = [] + for param in list(self.module.parameters()): + + def allreduce_hook(*unused): + Variable._execution_engine.queue_callback(allreduce_params) + + self.allreduce_params = allreduce_params + + def forward(self, *inputs, **kwargs): + self.needs_reduction = True + return self.module(*inputs, **kwargs) + + def state_dict(self, destination=None, prefix='', keep_vars=False): + sd = self.module.state_dict(destination, prefix, keep_vars) + return sd + + def load_state_dict(self, state_dict, strict=True): + return self.module.load_state_dict(state_dict, strict=strict) + + def named_parameters(self, prefix: str = '', recurse: bool = True): + return self.module.named_parameters(prefix=prefix, recurse=recurse) + + ''' + def _sync_buffers(self): + buffers = list(self.module._all_buffers()) + if len(buffers) > 0: + # cross-node buffer sync + flat_buffers = _flatten_dense_tensors(buffers) + dist.broadcast(flat_buffers, 0) + for buf, synced in zip(buffers, _unflatten_dense_tensors(flat_buffers, buffers)): + buf.copy_(synced) + def train(self, mode=True): + # Clear NCCL communicator and CUDA event cache of the default group ID, + # These cache will be recreated at the later call. This is currently a + # work-around for a potential NCCL deadlock. + if dist._backend == dist.dist_backend.NCCL: + dist._clear_group_cache() + super(DistributedDataParallel, self).train(mode) + self.module.train(mode) + ''' diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/downstream.py b/runtime/Lib/site_packages/models/nlp/mglm/model/downstream.py new file mode 100644 index 0000000000000000000000000000000000000000..61b1e807b58bff9ac4d4fa3fd9c996f4c5a929e0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/downstream.py @@ -0,0 +1,242 @@ +# Copyright (c) 2022 Zhipu.AI +"""Multiple choice model.""" + +import torch +import torch.nn + +from .modeling_glm import GLMModel + + +class GLMForMultiTokenCloze(torch.nn.Module): + + def __init__(self, + language_model: GLMModel, + take_softmax=True, + length_penalty=0.0): + super(GLMForMultiTokenCloze, self).__init__() + self.model = language_model + self.take_softmax = take_softmax + self.length_penalty = length_penalty + + def state_dict(self, destination=None, prefix='', keep_vars=False): + # [h.remove() for h in self.hook_handles] + sd = self.model.state_dict(destination, prefix, keep_vars) + return sd + + def load_state_dict(self, state_dict, strict=True): + return self.model.load_state_dict(state_dict, strict=strict) + + def named_parameters(self, prefix: str = '', recurse: bool = True): + return self.model.named_parameters(prefix=prefix, recurse=recurse) + + def forward(self, + input_ids, + position_ids, + attention_mask, + target_ids=None, + logit_mask=None, + prompt_pos=None): + if target_ids is None: + return self.model(input_ids, position_ids, attention_mask) + num_choices = None + if len(input_ids.shape) == 3: + batch_size, num_choices = input_ids.shape[:2] + input_ids = input_ids.reshape(-1, input_ids.size(-1)) + attention_mask = attention_mask.reshape(-1, + *attention_mask.size()[2:]) + position_ids = position_ids.reshape(-1, *position_ids.size()[2:]) + target_ids = target_ids.reshape(-1, target_ids.size(-1)) + logit_mask = logit_mask.reshape(-1, logit_mask.size(-1)) + if prompt_pos is not None: + prompt_pos = prompt_pos.reshape(-1, prompt_pos.size(-1)) + outputs, *mems = self.model( + input_ids, position_ids, attention_mask, prompt_pos=prompt_pos) + if self.take_softmax: + outputs = torch.nn.functional.log_softmax(outputs, dim=-1) + # select the target logits + batch_ids = torch.arange( + target_ids.size(0), dtype=torch.long, device=target_ids.device) + batch_ids = batch_ids.unsqueeze(1).expand_as(target_ids) + seq_ids = torch.arange( + target_ids.size(-1), dtype=torch.long, device=target_ids.device) + seq_ids = seq_ids.unsqueeze(0).expand_as(target_ids) + logits = outputs[batch_ids, seq_ids, target_ids] + logits = (logits * logit_mask).sum(dim=1) + if self.length_penalty > 0.0: + logits = logits / logit_mask.sum(dim=1)**self.length_penalty + if num_choices is not None: + logits = logits.view(-1, num_choices) + return (logits, *mems) + + +class GLMForMultiTokenClozeFast(torch.nn.Module): + + def __init__(self, language_model, take_softmax=True, length_penalty=0.0): + super(GLMForMultiTokenClozeFast, self).__init__() + self.model = language_model + self.take_softmax = take_softmax + self.length_penalty = length_penalty + + def forward(self, input_ids, position_ids, attention_mask, dec_input_ids, + dec_position_ids, dec_attention_mask, dec_target_ids, + dec_logit_mask): + # encoder + outputs, *mems = self.model( + input_ids, + position_ids, + attention_mask, + return_memory=True, + detach_memory=False) + batch_size, num_choices, max_dec_len = dec_input_ids.size() + max_enc_len = input_ids.size(-1) + + enc_mems = [] + for hidden in mems: + hidden = hidden.unsqueeze(1).expand(-1, num_choices, -1, + -1).reshape( + batch_size * num_choices, + *hidden.size()[1:]) + enc_mems.append(hidden) + + def build_dec_mask_matrix(seq_length, sep, memory_length=0): + m = enc_mems[0].new_ones((1, seq_length, seq_length)) + m = torch.tril(m) + + # sep = dec_attention_mask + ids = torch.arange( + memory_length, device=sep.device, dtype=sep.dtype).view(1, -1) + mask = ids < sep.view(-1, 1) # batch * mem + mask = mask.unsqueeze(1).float().expand(-1, seq_length, -1) + + m = m.expand(batch_size * num_choices, -1, -1) + m = torch.cat((mask, m), dim=2) + m = m.unsqueeze(1) + return m + + dec_input_ids = dec_input_ids.reshape(-1, max_dec_len) + dec_position_ids = dec_position_ids.reshape( + -1, + *dec_position_ids.size()[2:]) + # dec_attention_mask = dec_attention_mask.reshape(-1, *dec_attention_mask.size()[2:]).unsqueeze(1) + dec_attention_mask = build_dec_mask_matrix( + max_dec_len, dec_attention_mask.reshape(-1), max_enc_len) + dec_target_ids = dec_target_ids.reshape(-1, dec_target_ids.size(-1)) + dec_logit_mask = dec_logit_mask.reshape(-1, dec_logit_mask.size(-1)) + + outputs, *mems = self.model(dec_input_ids, dec_position_ids, + dec_attention_mask, *enc_mems) + if self.take_softmax: + outputs = torch.nn.functional.log_softmax(outputs, dim=-1) + + batch_ids = torch.arange( + dec_target_ids.size(0), + dtype=torch.long, + device=dec_target_ids.device) + batch_ids = batch_ids.unsqueeze(1).expand_as(dec_target_ids) + seq_ids = torch.arange( + dec_target_ids.size(-1), + dtype=torch.long, + device=dec_target_ids.device) + seq_ids = seq_ids.unsqueeze(0).expand_as(dec_target_ids) + logits = outputs[batch_ids, seq_ids, dec_target_ids] + logits = (logits * dec_logit_mask).sum(dim=1) + if self.length_penalty > 0.0: + logits = logits / dec_logit_mask.sum(dim=1)**self.length_penalty + if num_choices is not None: + logits = logits.view(-1, num_choices) + return (logits, *mems) + + +class GLMForSingleTokenCloze(torch.nn.Module): + + def __init__(self, language_model, take_softmax=False): + super().__init__() + self.model = language_model + self.take_softmax = take_softmax + + def state_dict(self, destination=None, prefix='', keep_vars=False): + # [h.remove() for h in self.hook_handles] + sd = self.model.state_dict(destination, prefix, keep_vars) + return sd + + def load_state_dict(self, state_dict, strict=True): + return self.model.load_state_dict(state_dict, strict=strict) + + def named_parameters(self, prefix: str = '', recurse: bool = True): + return self.model.named_parameters(prefix=prefix, recurse=recurse) + + def forward(self, + input_ids, + position_ids, + attention_mask, + target_ids=None, + logit_mask=None, + prompt_pos=None): + if target_ids is None: + return self.model(input_ids, position_ids, attention_mask) + assert len(input_ids.shape) == 2 + outputs, *mems = self.model( + input_ids, position_ids, attention_mask, prompt_pos=prompt_pos) + batch_ids = torch.arange( + outputs.size(0), + dtype=attention_mask.dtype, + device=attention_mask.device) + target_logits = outputs[batch_ids, attention_mask] + if self.take_softmax: + target_prob = torch.nn.functional.log_softmax( + target_logits, dim=-1) + else: + target_prob = target_logits + batch_ids = batch_ids.unsqueeze(1).expand_as(target_ids) + output = target_prob[batch_ids, target_ids] + + return (output, target_logits, *mems) + + +class GLMForSequenceClassification(torch.nn.Module): + + def __init__(self, + language_model, + hidden_size, + hidden_dropout, + pool_token, + num_class=1): + super().__init__() + self.pool_token = pool_token + self.model = language_model + self.num_class = num_class + # Multi-choice head. + self.pool_layer = torch.nn.Linear(hidden_size, hidden_size) + self.multichoice_dropout = torch.nn.Dropout(hidden_dropout) + self.multichoice_head = torch.nn.Linear(hidden_size, num_class) + + def forward(self, input_ids, position_ids, attention_mask): + num_choices = None + if len(input_ids.shape) == 3: + assert self.num_class == 1 + batch_size, num_choices = input_ids.shape[:2] + input_ids = input_ids.reshape(-1, input_ids.size(-1)) + attention_mask = attention_mask.reshape(-1, + *attention_mask.size()[2:]) + position_ids = position_ids.reshape(-1, *position_ids.size()[2:]) + outputs, *mems = self.model(input_ids, position_ids, attention_mask) + if self.pool_token == 'start': + output = outputs[torch.arange( + outputs.size(0), + dtype=attention_mask.dtype, + device=attention_mask.device), attention_mask] + elif self.pool_token == 'pad': + output = outputs[torch.arange( + outputs.size(0), + dtype=attention_mask.dtype, + device=attention_mask.device), attention_mask - 1] + elif self.pool_token == 'cls': + output = outputs[:, 0] + else: + raise NotImplementedError + output = torch.tanh(self.pool_layer(output)) + multichoice_output = self.multichoice_dropout(output) + logits = self.multichoice_head(multichoice_output) + if num_choices is not None: + logits = logits.view(-1, num_choices) + return (logits, *mems) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_bert.py b/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..28b5cd1ead00683500d42e7c4acb91a5b1dd8a81 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_bert.py @@ -0,0 +1,1560 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import copy +import logging +import math +import os +import shutil +import tarfile +import tempfile + +import json +import torch +import torch.nn.functional as F +from data_utils.file_utils import cached_path +from megatron_util import mpu +from torch import nn +from torch.nn import CrossEntropyLoss + +# from torch.utils.checkpoint import checkpoint + + +def normal_init_method(mean, std): + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=mean, std=std) + + return init_ + + +def scaled_init_method(mean, std, num_layers): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = std / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=mean, std=std) + + return init_ + + +def bert_extended_attention_mask(attention_mask): + # We create a 3D attention mask from a 2D tensor mask. + # [b, 1, s] + attention_mask_b1s = attention_mask.unsqueeze(1) + # [b, s, 1] + attention_mask_bs1 = attention_mask.unsqueeze(2) + # [b, s, s] + attention_mask_bss = attention_mask_b1s * attention_mask_bs1 + # [b, 1, s, s] + extended_attention_mask = attention_mask_bss.unsqueeze(1) + + return extended_attention_mask + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +PRETRAINED_MODEL_ARCHIVE_MAP = { + 'bert-base-uncased': + '/root/data/bert-base-uncased.tar.gz', + 'bert-large-uncased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased.tar.gz', + 'bert-base-cased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased.tar.gz', + 'bert-large-cased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased.tar.gz', + 'bert-base-multilingual-uncased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased.tar.gz', + 'bert-base-multilingual-cased': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased.tar.gz', + 'bert-base-chinese': + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese.tar.gz', +} +CONFIG_NAME = 'bert_config.json' +WEIGHTS_NAME = 'pytorch_model.bin' +TF_WEIGHTS_NAME = 'model.ckpt' + + +def load_tf_weights_in_bert(model, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + print( + 'Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see ' + 'https://www.tensorflow.org/install/ for installation instructions.' + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + print('Converting TensorFlow checkpoint from {}'.format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + print('Loading TF weight {} with shape {}'.format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ['adam_v', 'adam_m'] for n in name): + print('Skipping {}'.format('/'.join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) # noqa + else: + l = [m_name] # noqa + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + else: + pointer = getattr(pointer, l[0]) + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + print('Initialize PyTorch weight {}'.format(name)) + pointer.data = torch.from_numpy(array) + return model + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class BertConfig(object): + """Configuration class to store the configuration of a `BertModel`. + """ + + def __init__(self, + vocab_size_or_config_json_file, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + deep_init=False, + fp32_layernorm=False, + fp32_embedding=False, + fp32_tokentypes=False, + layernorm_epsilon=1e-12): + """Constructs BertConfig. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + if isinstance(vocab_size_or_config_json_file, str): + with open( + vocab_size_or_config_json_file, 'r', + encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.deep_init = deep_init + self.fp32_layernorm = fp32_layernorm + self.fp32_embedding = fp32_embedding + self.layernorm_epsilon = layernorm_epsilon + self.fp32_tokentypes = fp32_tokentypes + else: + raise ValueError( + 'First argument must be either a vocabulary size (int)' + 'or the path to a pretrained model config file (str)') + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' + + +try: + from apex.normalization.fused_layer_norm import FusedLayerNorm as BertLayerNorm +except ImportError: + print( + 'Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.' + ) + + class BertLayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(BertLayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(BertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, + config.hidden_size) + # self.word_embeddings = mpu.VocabParallelEmbedding( + # config.vocab_size, config.hidden_size, + # init_method=normal_init_method(mean=0.0, + # std=config.initializer_range)) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.fp32_layernorm = config.fp32_layernorm + self.fp32_embedding = config.fp32_embedding + self.fp32_tokentypes = config.fp32_tokentypes + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None): + seq_length = input_ids.size(1) + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + if not self.fp32_tokentypes: + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + if self.fp32_embedding and not self.fp32_layernorm: + embeddings = embeddings.half() + previous_type = embeddings.type() + if self.fp32_layernorm: + embeddings = embeddings.float() + embeddings = self.LayerNorm(embeddings) + if self.fp32_layernorm: + if self.fp32_embedding: + embeddings = embeddings.half() + else: + embeddings = embeddings.type(previous_type) + else: + embeddings = words_embeddings.float() + position_embeddings.float( + ) + token_type_embeddings.float() + if self.fp32_tokentypes and not self.fp32_layernorm: + embeddings = embeddings.half() + previous_type = embeddings.type() + if self.fp32_layernorm: + embeddings = embeddings.float() + embeddings = self.LayerNorm(embeddings) + if self.fp32_layernorm: + if self.fp32_tokentypes: + embeddings = embeddings.half() + else: + embeddings = embeddings.type(previous_type) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config): + super(BertSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + previous_type = attention_probs.type() # noqa + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super(BertSelfOutput, self).__init__() + if hasattr(config, 'deep_init') and config.deep_init: + init_method = scaled_init_method( + mean=0.0, + std=config.initializer_range, + num_layers=config.num_hidden_layers) + else: + init_method = normal_init_method( # noqa + mean=0.0, std=config.initializer_range) + self.dense = nn.Linear( + config.hidden_size, config.hidden_size, bias=True) + # self.dense = mpu.RowParallelLinear( + # input_size=config.hidden_size, + # output_size=config.hidden_size, + # bias=True, + # input_is_parallel=True, + # stride=1, + # init_method=init_method) + self.fp32_layernorm = config.fp32_layernorm + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + ln_input = hidden_states + input_tensor + previous_type = ln_input.type() + if self.fp32_layernorm: + ln_input = ln_input.float() + hidden_states = self.LayerNorm(ln_input) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config): + super(BertAttention, self).__init__() + self.self = BertSelfAttention(config) + # self.self = mpu.BertParallelSelfAttention( + # hidden_size=config.hidden_size, + # num_attention_heads=config.num_attention_heads, + # dropout_prob=config.attention_probs_dropout_prob, + # output_parallel=True, + # init_method=normal_init_method(mean=0.0, + # std=config.initializer_range)) + self.output = BertSelfOutput(config) + + def forward(self, input_tensor, attention_mask): + self_output = self.self(input_tensor, attention_mask) + attention_output = self.output(self_output, input_tensor) + return attention_output + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super(BertIntermediate, self).__init__() + self.dense = nn.Linear( + config.hidden_size, config.intermediate_size, bias=True) + # self.dense = mpu.ColumnParallelLinear( + # input_size=config.hidden_size, + # output_size=config.intermediate_size, + # bias=True, + # gather_output=False, + # stride=1, + # init_method=normal_init_method(mean=0.0, + # std=config.initializer_range)) + self.intermediate_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super(BertOutput, self).__init__() + if hasattr(config, 'deep_init') and config.deep_init: + init_method = scaled_init_method( + mean=0.0, + std=config.initializer_range, + num_layers=config.num_hidden_layers) + else: + init_method = normal_init_method( # noqa + mean=0.0, std=config.initializer_range) + self.dense = nn.Linear( + config.intermediate_size, config.hidden_size, bias=True) + # self.dense = mpu.RowParallelLinear( + # input_size=config.intermediate_size, + # output_size=config.hidden_size, + # bias=True, + # input_is_parallel=True, + # stride=1, + # init_method=init_method) + self.fp32_layernorm = config.fp32_layernorm + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + ln_input = hidden_states + input_tensor + previous_type = ln_input.type() + if self.fp32_layernorm: + ln_input = ln_input.float() + hidden_states = self.LayerNorm(ln_input) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config): + super(BertLayer, self).__init__() + self.attention = BertAttention(config) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward(self, hidden_states, attention_mask): + attention_output = self.attention(hidden_states, attention_mask) + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super(BertEncoder, self).__init__() + # layer = BertLayer(config) + # self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.num_hidden_layers)]) + self.layer = nn.ModuleList( + [BertLayer(config) for _ in range(config.num_hidden_layers)]) + + # def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True): + # all_encoder_layers = [] + # for layer_module in self.layer: + # hidden_states = layer_module(hidden_states, attention_mask) + # if output_all_encoded_layers: + # all_encoder_layers.append(hidden_states) + # if not output_all_encoded_layers: + # all_encoder_layers.append(hidden_states) + # return all_encoder_layers + def forward(self, + hidden_states, + attention_mask, + output_all_encoded_layers=True, + checkpoint_activations=False): + all_encoder_layers = [] + + def custom(start, end): + + def custom_forward(*inputs): + layers = self.layer[start:end] + x_ = inputs[0] + for layer in layers: + x_ = layer(x_, inputs[1]) + return x_ + + return custom_forward + + if checkpoint_activations: + l = 0 # noqa + num_layers = len(self.layer) + chunk_length = 1 # math.ceil(math.sqrt(num_layers)) + while l < num_layers: + hidden_states = mpu.checkpoint( + custom(l, l + chunk_length), hidden_states, + attention_mask * 1) + l += chunk_length # noqa + # decoder layers + else: + for i, layer_module in enumerate(self.layer): + hidden_states = layer_module(hidden_states, attention_mask) + + if output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + + if not output_all_encoded_layers or checkpoint_activations: + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BertPooler(nn.Module): + + def __init__(self, config): + super(BertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super(BertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.transform_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.fp32_layernorm = config.fp32_layernorm + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + previous_type = hidden_states.type() + if self.fp32_layernorm: + hidden_states = hidden_states.float() + hidden_states = self.LayerNorm(hidden_states) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + bert_model_embedding_weights.size(1), + bert_model_embedding_weights.size(0), + bias=False) + # self.decoder_weight = bert_model_embedding_weights + # self.bias = nn.Parameter(torch.zeros(bert_model_embedding_weights.size(0))) + # self.bias.model_parallel = True + self.fp32_embedding = config.fp32_embedding + self.fp32_layernorm = config.fp32_layernorm + + def convert_to_type(tensor): + if self.fp32_embedding: + return tensor.half() + else: + return tensor + + self.type_converter = convert_to_type + self.converted = False + + def forward(self, hidden_states): + if not self.converted: + self.converted = True + if self.fp32_embedding: + self.transform.half() + if self.fp32_layernorm: + self.transform.LayerNorm.float() + hidden_states = self.transform(self.type_converter(hidden_states)) + hidden_states = self.decoder(hidden_states) + self.bias + # hidden_states = mpu.copy_to_model_parallel_region(hidden_states) + # hidden_states = F.linear(self.type_converter(hidden_states), + # self.type_converter(self.decoder_weight), + # self.type_converter(self.bias)) + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertOnlyMLMHead, self).__init__() + self.predictions = BertLMPredictionHead(config, + bert_model_embedding_weights) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super(BertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertPreTrainingHeads, self).__init__() + self.predictions = BertLMPredictionHead(config, + bert_model_embedding_weights) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + for p in self.seq_relationship.parameters(): + if p is None: + continue + pooled_output = pooled_output.type_as(p) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class PreTrainedBertModel(nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedBertModel, self).__init__() + if not isinstance(config, BertConfig): + raise ValueError( + 'Parameter config in `{}(config)` should be an instance of class `BertConfig`. ' + 'To create a model from a Google pretrained model use ' + '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format( + self.__class__.__name__, self.__class__.__name__)) + self.config = config + + def init_bert_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + @classmethod + def from_pretrained(cls, + pretrained_model_name, + state_dict=None, + cache_dir=None, + fp32_layernorm=False, + fp32_embedding=False, + layernorm_epsilon=1e-12, + fp32_tokentypes=False, + *inputs, + **kwargs): + """ + Instantiate a PreTrainedBertModel from a pre-trained model file or a pytorch state dict. + Download and cache the pre-trained model file if needed. + + Params: + pretrained_model_name: either: + - a str with the name of a pre-trained model to load selected in the list of: + . `bert-base-uncased` + . `bert-large-uncased` + . `bert-base-cased` + . `bert-large-cased` + . `bert-base-multilingual-uncased` + . `bert-base-multilingual-cased` + . `bert-base-chinese` + - a path or url to a pretrained model archive containing: + . `bert_config.json` a configuration file for the model + . `pytorch_model.bin` a PyTorch dump of a BertForPreTraining instance + cache_dir: an optional path to a folder in which the pre-trained models will be cached. + state_dict: an optional state dictionnary (collections.OrderedDict object) to use instead of Google pre-trained models + *inputs, **kwargs: additional input for the specific Bert class + (ex: num_labels for BertForSequenceClassification) + """ # noqa + if pretrained_model_name in PRETRAINED_MODEL_ARCHIVE_MAP: + archive_file = PRETRAINED_MODEL_ARCHIVE_MAP[pretrained_model_name] + else: + archive_file = pretrained_model_name + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path( + archive_file, cache_dir=cache_dir) + except FileNotFoundError: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + 'associated to this path or url.'.format( + pretrained_model_name, + ', '.join(PRETRAINED_MODEL_ARCHIVE_MAP.keys()), + archive_file)) + return None + if resolved_archive_file == archive_file: + logger.info('loading archive file {}'.format(archive_file)) + else: + logger.info('loading archive file {} from cache at {}'.format( + archive_file, resolved_archive_file)) + tempdir = None + if os.path.isdir(resolved_archive_file): + serialization_dir = resolved_archive_file + else: + # Extract archive to temp dir + tempdir = tempfile.mkdtemp() + logger.info('extracting archive file {} to temp dir {}'.format( + resolved_archive_file, tempdir)) + with tarfile.open(resolved_archive_file, 'r:gz') as archive: + archive.extractall(tempdir) + serialization_dir = tempdir + # Load config + config_file = os.path.join(serialization_dir, CONFIG_NAME) + config = BertConfig.from_json_file(config_file) + config.fp32_layernorm = fp32_layernorm + config.fp32_embedding = fp32_embedding + config.layernorm_epsilon = layernorm_epsilon + config.fp32_tokentypes = fp32_tokentypes + logger.info('Model config {}'.format(config)) + # Instantiate model. + model = cls(config, *inputs, **kwargs) + if state_dict is None: + weights_path = os.path.join(serialization_dir, WEIGHTS_NAME) + state_dict = torch.load(weights_path) + + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, + True, missing_keys, unexpected_keys, + error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix='' if hasattr(model, 'bert') else 'bert.') + if len(missing_keys) > 0: + print('Weights of {} not initialized from pretrained model: {}'. + format(model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + print('Weights from pretrained model not used in {}: {}'.format( + model.__class__.__name__, unexpected_keys)) + if tempdir: + # Clean up temp dir + shutil.rmtree(tempdir) + return model + + +class BertModel(PreTrainedBertModel): + """BERT model ("Bidirectional Embedding Representations from a Transformer"). + + Params: + config: a BertConfig class instance with the configuration to build a new model + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `output_all_encoded_layers`: boolean which controls the content of the `encoded_layers` output as described below. Default: `True`. + + Outputs: Tuple of (encoded_layers, pooled_output) + `encoded_layers`: controled by `output_all_encoded_layers` argument: + - `output_all_encoded_layers=True`: outputs a list of the full sequences of encoded-hidden-states at the end + of each attention block (i.e. 12 full sequences for BERT-base, 24 for BERT-large), each + encoded-hidden-state is a torch.FloatTensor of size [batch_size, sequence_length, hidden_size], + - `output_all_encoded_layers=False`: outputs only the full sequence of hidden-states corresponding + to the last attention block of shape [batch_size, sequence_length, hidden_size], + `pooled_output`: a torch.FloatTensor of size [batch_size, hidden_size] which is the output of a + classifier pretrained on top of the hidden state associated to the first character of the + input (`CLF`) to train on the Next-Sentence task (see BERT's paper). + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = modeling.BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = modeling.BertModel(config=config) + >>> all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) + """ # noqa + + def __init__(self, config): + super(BertModel, self).__init__(config) + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder(config) + self.pooler = BertPooler(config) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + output_all_encoded_layers=True, + checkpoint_activations=False): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.encoder.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output = self.embeddings(input_ids, token_type_ids) + encoded_layers = self.encoder( + embedding_output, + extended_attention_mask, + output_all_encoded_layers=output_all_encoded_layers, + checkpoint_activations=checkpoint_activations) + sequence_output = encoded_layers[-1] + for p in self.pooler.parameters(): + if p is None: + continue + sequence_output = sequence_output.type_as(p) + break + pooled_output = self.pooler(sequence_output) + if not output_all_encoded_layers or checkpoint_activations: + encoded_layers = encoded_layers[-1] + return encoded_layers, pooled_output + + +class BertForPreTraining(PreTrainedBertModel): + """BERT model with pre-training heads. + This module comprises the BERT model followed by the two pre-training heads: + - the masked language modeling head, and + - the next sentence classification head. + + Params: + config: a BertConfig class instance with the configuration to build a new model. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `masked_lm_labels`: masked language modeling labels: torch.LongTensor of shape [batch_size, sequence_length] + with indices selected in [-1, 0, ..., vocab_size]. All labels set to -1 are ignored (masked), the loss + is only computed for the labels set in [0, ..., vocab_size] + `next_sentence_label`: next sentence classification loss: torch.LongTensor of shape [batch_size] + with indices selected in [0, 1]. + 0 => next sentence is the continuation, 1 => next sentence is a random sentence. + + Outputs: + if `masked_lm_labels` and `next_sentence_label` are not `None`: + Outputs the total_loss which is the sum of the masked language modeling loss and the next + sentence classification loss. + if `masked_lm_labels` or `next_sentence_label` is `None`: + Outputs a tuple comprising + - the masked language modeling logits of shape [batch_size, sequence_length, vocab_size], and + - the next sentence classification logits of shape [batch_size, 2]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = BertForPreTraining(config) + >>> masked_lm_logits_scores, seq_relationship_logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertForPreTraining, self).__init__(config) + self.bert = BertModel(config) + self.cls = BertPreTrainingHeads( + config, self.bert.embeddings.word_embeddings.weight) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + masked_lm_labels=None, + next_sentence_label=None, + checkpoint_activations=False): + sequence_output, pooled_output = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + prediction_scores, seq_relationship_score = self.cls( + sequence_output, pooled_output) + + if masked_lm_labels is not None and next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size).float(), + masked_lm_labels.view(-1)) + next_sentence_loss = loss_fct( + seq_relationship_score.view(-1, 2).float(), + next_sentence_label.view(-1)) + total_loss = masked_lm_loss + next_sentence_loss + return total_loss + else: + return prediction_scores, seq_relationship_score + + +class BertForMaskedLM(PreTrainedBertModel): + """BERT model with the masked language modeling head. + This module comprises the BERT model followed by the masked language modeling head. + + Params: + config: a BertConfig class instance with the configuration to build a new model. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `masked_lm_labels`: masked language modeling labels: torch.LongTensor of shape [batch_size, sequence_length] + with indices selected in [-1, 0, ..., vocab_size]. All labels set to -1 are ignored (masked), the loss + is only computed for the labels set in [0, ..., vocab_size] + + Outputs: + if `masked_lm_labels` is not `None`: + Outputs the masked language modeling loss. + if `masked_lm_labels` is `None`: + Outputs the masked language modeling logits of shape [batch_size, sequence_length, vocab_size]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = BertForMaskedLM(config) + >>> masked_lm_logits_scores = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertForMaskedLM, self).__init__(config) + self.bert = BertModel(config) + self.cls = BertOnlyMLMHead(config, + self.bert.embeddings.word_embeddings.weight) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + masked_lm_labels=None, + checkpoint_activations=False): + sequence_output, _ = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + prediction_scores = self.cls(sequence_output) + + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + masked_lm_labels.view(-1)) + return masked_lm_loss + else: + return prediction_scores + + +class BertForNextSentencePrediction(PreTrainedBertModel): + """BERT model with next sentence prediction head. + This module comprises the BERT model followed by the next sentence classification head. + + Params: + config: a BertConfig class instance with the configuration to build a new model. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `next_sentence_label`: next sentence classification loss: torch.LongTensor of shape [batch_size] + with indices selected in [0, 1]. + 0 => next sentence is the continuation, 1 => next sentence is a random sentence. + + Outputs: + if `next_sentence_label` is not `None`: + Outputs the total_loss which is the sum of the masked language modeling loss and the next + sentence classification loss. + if `next_sentence_label` is `None`: + Outputs the next sentence classification logits of shape [batch_size, 2]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = BertForNextSentencePrediction(config) + >>> seq_relationship_logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertForNextSentencePrediction, self).__init__(config) + self.bert = BertModel(config) + self.cls = BertOnlyNSPHead(config) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + next_sentence_label=None, + checkpoint_activations=False): + _, pooled_output = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + seq_relationship_score = self.cls(pooled_output) + + if next_sentence_label is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + next_sentence_loss = loss_fct( + seq_relationship_score.view(-1, 2), + next_sentence_label.view(-1)) + return next_sentence_loss + else: + return seq_relationship_score + + +class BertForSequenceClassification(PreTrainedBertModel): + """BERT model for classification. + This module is composed of the BERT model with a linear layer on top of + the pooled output. + + Params: + `config`: a BertConfig class instance with the configuration to build a new model. + `num_labels`: the number of classes for the classifier. Default = 2. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `labels`: labels for the classification output: torch.LongTensor of shape [batch_size] + with indices selected in [0, ..., num_labels]. + + Outputs: + if `labels` is not `None`: + Outputs the CrossEntropy classification loss of the output with the labels. + if `labels` is `None`: + Outputs the classification logits of shape [batch_size, num_labels]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> num_labels = 2 + + >>> model = BertForSequenceClassification(config, num_labels) + >>> logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config, num_labels=2): + super(BertForSequenceClassification, self).__init__(config) + self.num_labels = num_labels + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, num_labels) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + labels=None, + checkpoint_activations=False): + _, pooled_output = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + return loss + else: + return logits + + +class BertForMultipleChoice(PreTrainedBertModel): + """BERT model for multiple choice tasks. + This module is composed of the BERT model with a linear layer on top of + the pooled output. + + Params: + `config`: a BertConfig class instance with the configuration to build a new model. + `num_choices`: the number of classes for the classifier. Default = 2. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, num_choices, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, num_choices, sequence_length] + with the token types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` + and type 1 corresponds to a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, num_choices, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `labels`: labels for the classification output: torch.LongTensor of shape [batch_size] + with indices selected in [0, ..., num_choices]. + + Outputs: + if `labels` is not `None`: + Outputs the CrossEntropy classification loss of the output with the labels. + if `labels` is `None`: + Outputs the classification logits of shape [batch_size, num_labels]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[[31, 51, 99], [15, 5, 0]], [[12, 16, 42], [14, 28, 57]]]) + >>> input_mask = torch.LongTensor([[[1, 1, 1], [1, 1, 0]],[[1,1,0], [1, 0, 0]]]) + >>> token_type_ids = torch.LongTensor([[[0, 0, 1], [0, 1, 0]],[[0, 1, 1], [0, 0, 1]]]) + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> num_choices = 2 + + >>> model = BertForMultipleChoice(config, num_choices) + >>> logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertForMultipleChoice, self).__init__(config) + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + labels=None, + checkpoint_activations=False): + batch_size, num_choices = input_ids.shape[:2] + flat_input_ids = input_ids.reshape(-1, input_ids.size(-1)) + flat_token_type_ids = token_type_ids.reshape(-1, + token_type_ids.size(-1)) + flat_attention_mask = attention_mask.reshape(-1, + attention_mask.size(-1)) + _, pooled_output = self.bert( + flat_input_ids, + flat_token_type_ids, + flat_attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.reshape(-1, num_choices) + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + return loss + else: + return reshaped_logits + + +class BertForTokenClassification(PreTrainedBertModel): + """BERT model for token-level classification. + This module is composed of the BERT model with a linear layer on top of + the full hidden state of the last layer. + + Params: + `config`: a BertConfig class instance with the configuration to build a new model. + `num_labels`: the number of classes for the classifier. Default = 2. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `labels`: labels for the classification output: torch.LongTensor of shape [batch_size] + with indices selected in [0, ..., num_labels]. + + Outputs: + if `labels` is not `None`: + Outputs the CrossEntropy classification loss of the output with the labels. + if `labels` is `None`: + Outputs the classification logits of shape [batch_size, sequence_length, num_labels]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> num_labels = 2 + + >>> model = BertForTokenClassification(config, num_labels) + >>> logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config, num_labels=2): + super(BertForTokenClassification, self).__init__(config) + self.num_labels = num_labels + self.bert = BertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, num_labels) + # self.classifier = mpu.RowParallelLinear( + # input_size=config.hidden_size, + # output_size=num_labels, + # bias=True, + # input_is_parallel=True, + # stride=1, + # init_method=normal_init_method(mean=0.0, + # std=config.initializer_range)) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + labels=None, + checkpoint_activations=False): + sequence_output, _ = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + with mpu.get_cuda_rng_tracker().fork(): + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + return loss + else: + return logits + + +class BertForQuestionAnswering(PreTrainedBertModel): + """BERT model for Question Answering (span extraction). + This module is composed of the BERT model with a linear layer on top of + the sequence output that computes start_logits and end_logits + + Params: + `config`: a BertConfig class instance with the configuration to build a new model. + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `start_positions`: position of the first token for the labeled span: torch.LongTensor of shape [batch_size]. + Positions are clamped to the length of the sequence and position outside of the sequence are not taken + into account for computing the loss. + `end_positions`: position of the last token for the labeled span: torch.LongTensor of shape [batch_size]. + Positions are clamped to the length of the sequence and position outside of the sequence are not taken + into account for computing the loss. + + Outputs: + if `start_positions` and `end_positions` are not `None`: + Outputs the total_loss which is the sum of the CrossEntropy loss for the start and end token positions. + if `start_positions` or `end_positions` is `None`: + Outputs a tuple of start_logits, end_logits which are the logits respectively for the start and end + position tokens of shape [batch_size, sequence_length]. + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = BertForQuestionAnswering(config) + >>> start_logits, end_logits = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertForQuestionAnswering, self).__init__(config) + self.bert = BertModel(config) + # TODO check with Google if it's normal there is no dropout on the token classifier of SQuAD in the TF version + # self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.qa_outputs = nn.Linear(config.hidden_size, 2) + # self.qa_outputs = mpu.RowParallelLinear( + # input_size=config.hidden_size, + # output_size=2, + # bias=True, + # input_is_parallel=True, + # stride=1, + # init_method=normal_init_method(mean=0.0, + # std=config.initializer_range)) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + start_positions=None, + end_positions=None, + checkpoint_activations=False): + sequence_output, _ = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + return total_loss + else: + return start_logits, end_logits diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_glm.py b/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_glm.py new file mode 100644 index 0000000000000000000000000000000000000000..4f21943afa06a42dddbefef85e2675006fa287dc --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/modeling_glm.py @@ -0,0 +1,245 @@ +# Modified by Zhipu.AI +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GPT-2 model.""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from megatron_util import mpu, print_rank_0 + +from modelscope.models.nlp.mglm.model.prompt import PromptSpell +from .transformer import GPT2ParallelTransformer + + +def init_method_normal(std=0.02): + """Init method based on normal distribution. + + This is only used for embeddings. The transformer has its + own initializer. + """ + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +class GLMModel(torch.nn.Module): + """GLM Language model. + + The output of the forward method are the logits (parallel or + serial depending on the `parallel_output` flag. + """ + + def __init__( + self, + num_layers, + vocab_size, + hidden_size, + num_attention_heads, + embedding_dropout_prob, + attention_dropout_prob, + output_dropout_prob, + max_sequence_length, + max_memory_length, + checkpoint_activations, + checkpoint_num_layers=1, + parallel_output=True, + relative_encoding=False, + block_position_encoding=False, + output_predict=True, + spell_length=None, + spell_func='lstm', + attention_scale=1.0, + ): + + super(GLMModel, self).__init__() + + self.parallel_output = parallel_output + self.output_predict = output_predict + self.hidden_size = hidden_size + + init_method = init_method_normal(std=0.02) + + # Word embeddings (parallel). + self.word_embeddings = mpu.VocabParallelEmbedding( + vocab_size, hidden_size, init_method=init_method) + + # Transformer + self.transformer = GPT2ParallelTransformer( + num_layers, + hidden_size, + num_attention_heads, + max_sequence_length, + max_memory_length, + embedding_dropout_prob, + attention_dropout_prob, + output_dropout_prob, + checkpoint_activations, + checkpoint_num_layers, + attention_scale=attention_scale, + relative_encoding=relative_encoding, + block_position_encoding=block_position_encoding) + if spell_length is not None: + self.prompt_spell = PromptSpell(spell_length, self.hidden_size, + spell_func) + + def freeze_transformer(self, tune_prefix_layers=None): + log_str = 'Freeze transformer' + self.word_embeddings.requires_grad_(False) + self.transformer.requires_grad_(False) + if tune_prefix_layers is not None: + log_str += f' tune {tune_prefix_layers} prefix layers' + for i in range(tune_prefix_layers): + self.transformer.layers[i].requires_grad_(True) + print_rank_0(log_str) + + def forward(self, + input_ids, + position_ids, + attention_mask, + *mems, + return_memory=False, + detach_memory=True, + prompt_pos=None): + # Embeddings. + batch_size = input_ids.size(0) + words_embeddings = self.word_embeddings(input_ids) + embeddings = words_embeddings + if prompt_pos is not None: + embeddings = embeddings.clone() + prompt_embeds = self.prompt_spell() + batch_index = torch.arange( + batch_size, device=input_ids.device).unsqueeze(1) + embeddings[batch_index, prompt_pos] = prompt_embeds + # Transformer. + transformer_output = self.transformer( + embeddings, + position_ids, + attention_mask, + mems, + return_memory=return_memory, + detach_memory=detach_memory) + logits, hidden_layers = transformer_output + outputs = hidden_layers + + if self.output_predict: + # Parallel logits. + logits_parallel = mpu.copy_to_model_parallel_region(logits) + logits_parallel = F.linear(logits_parallel, + self.word_embeddings.weight) + + if self.parallel_output: + return (logits_parallel, *outputs) + + return (mpu.gather_from_model_parallel_region(logits_parallel), + *outputs) + else: + return (logits, *outputs) + + +class EncoderDecoder(torch.nn.Module): + """Seq2Seq Transformer Model + The output of the forward method are the logits (parallel or serial depending on the `parallel_output` flag). + """ + + def __init__(self, + num_layers, + vocab_size, + hidden_size, + num_attention_heads, + embedding_dropout_prob, + attention_dropout_prob, + output_dropout_prob, + max_sequence_length, + max_memory_length, + checkpoint_activations, + checkpoint_num_layers=1, + parallel_output=True, + output_predict=True): + super(EncoderDecoder, self).__init__() + + self.parallel_output = parallel_output + self.output_predict = output_predict + + init_method = init_method_normal(std=0.02) + + # Word embeddings (parallel). + self.word_embeddings = mpu.VocabParallelEmbedding( + vocab_size, hidden_size, init_method=init_method) + + # Transformer + self.encoder = GPT2ParallelTransformer( + num_layers, hidden_size, num_attention_heads, max_sequence_length, + max_memory_length, embedding_dropout_prob, attention_dropout_prob, + output_dropout_prob, checkpoint_activations, checkpoint_num_layers) + self.decoder = GPT2ParallelTransformer( + num_layers, + hidden_size, + num_attention_heads, + max_sequence_length, + max_memory_length, + embedding_dropout_prob, + attention_dropout_prob, + output_dropout_prob, + checkpoint_activations, + checkpoint_num_layers, + use_decoder_layer=True) + + def forward(self, source_ids, target_ids, source_position_ids, + target_position_ids, source_mask, target_mask): + # Embeddings. + source_embeddings = self.word_embeddings(source_ids) + target_embeddings = self.word_embeddings(target_ids) + + # Transformer. + encoder_output, _ = self.encoder(source_embeddings, + source_position_ids, source_mask) + decoder_output, _ = self.decoder(target_embeddings, + target_position_ids, target_mask) + if self.output_predict: + # Parallel logits. + output_parallel = mpu.copy_to_model_parallel_region(decoder_output) + logits_parallel = F.linear(output_parallel, + self.word_embeddings.weight) + + if self.parallel_output: + return (logits_parallel, ) + + return (mpu.gather_from_model_parallel_region(logits_parallel), ) + else: + return (decoder_output, ) + + +def glm_get_params_for_weight_decay_optimization(module): + weight_decay_params = {'params': []} + no_weight_decay_params = {'params': [], 'weight_decay': 0.0} + for module_ in module.modules(): + if isinstance(module_, (mpu.LayerNorm, torch.nn.LayerNorm)): + no_weight_decay_params['params'].extend([ + p for p in list(module_._parameters.values()) + if p is not None and p.requires_grad + ]) + else: + weight_decay_params['params'].extend([ + p for n, p in list(module_._parameters.items()) + if p is not None and p.requires_grad and n != 'bias' + ]) + no_weight_decay_params['params'].extend([ + p for n, p in list(module_._parameters.items()) + if p is not None and p.requires_grad and n == 'bias' + ]) + + return weight_decay_params, no_weight_decay_params diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/prompt.py b/runtime/Lib/site_packages/models/nlp/mglm/model/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..a29ceda0710585f17dc8b310b3c77481e57c66ea --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/prompt.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022 Zhipu.AI + +import random + +import torch + + +class PromptSpell(torch.nn.Module): + + def __init__(self, spell_length, hidden_size, spell_func): + super(PromptSpell, self).__init__() + self.spell_length = spell_length + self.hidden_size = hidden_size + self.spell_embeddings = torch.nn.Embedding(self.spell_length, + self.hidden_size) + self.spell_func = spell_func + if self.spell_func == 'lstm': + self.lstm_head = torch.nn.LSTM( + input_size=self.hidden_size, + hidden_size=self.hidden_size, + num_layers=2, + # dropout=self.lstm_dropout, + bidirectional=True, + batch_first=True) # .to(torch.device("cuda")) + self.mlp_head = torch.nn.Sequential( + torch.nn.Linear(2 * self.hidden_size, self.hidden_size), + torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.hidden_size)) + elif self.spell_func == 'mlp': + self.mlp_head = torch.nn.Sequential( + torch.nn.Linear(self.hidden_size, self.hidden_size), + torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.hidden_size)) + elif self.spell_func != 'none': + raise NotImplementedError('Prompt function ' + self.spell_func) + + def init_embedding(self, word_embeddings=None, task_tokens=None): + num_words = 5000 + with torch.no_grad(): + for i in range(self.spell_length): + rand_token = random.randrange(num_words) + if task_tokens is None: + target_embedding = word_embeddings[rand_token] + else: + word_embedding = word_embeddings[rand_token] + task_token = random.choice(task_tokens) + task_embedding = word_embeddings[task_token] + ratio = random.random() + target_embedding = word_embedding * ratio + task_embedding * ( + 1 - ratio) + self.spell_embeddings.weight.data[i] = target_embedding + + def forward(self): + prompt_embeds = self.spell_embeddings.weight.unsqueeze(0) + if self.spell_func == 'lstm': + prompt_embeds = self.lstm_head(prompt_embeds)[0] + if self.spell_func == 'lstm' or self.spell_func == 'mlp': + prompt_embeds = self.mlp_head(prompt_embeds) + return prompt_embeds diff --git a/runtime/Lib/site_packages/models/nlp/mglm/model/transformer.py b/runtime/Lib/site_packages/models/nlp/mglm/model/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..da944c768f45abd981cdc94e92b2cb3cbb594afc --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/model/transformer.py @@ -0,0 +1,1193 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Transformer.""" + +import math + +import deepspeed +import torch +import torch.nn.init as init +from apex.normalization.fused_layer_norm import FusedLayerNorm as LayerNorm +from megatron_util import mpu + + +class PositionalEmbedding(torch.nn.Module): + + def __init__(self, hidden_size): + super(PositionalEmbedding, self).__init__() + + self.hidden_size = hidden_size + + inv_freq = 1 / ( + 10000**(torch.arange(0.0, hidden_size, 2.0) / hidden_size)) # noqa + self.register_buffer('inv_freq', inv_freq) + + def forward(self, pos_seq, bsz=None): + sinusoid_inp = torch.ger(pos_seq, self.inv_freq) + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + + if bsz is not None: + return pos_emb[None, :, :].expand(bsz, -1, -1) + else: + return pos_emb[None, :, :] + + +class ParallelCrossAttention(torch.nn.Module): + """Parallel cross-attention layer for Transformer""" + + def __init__(self, + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + init_method, + output_layer_init_method=None): + super(ParallelCrossAttention, self).__init__() + # Set output layer initialization if not provided. + if output_layer_init_method is None: + output_layer_init_method = init_method + # Per attention head and per partition values. + world_size = mpu.get_model_parallel_world_size() + self.hidden_size_per_partition = mpu.divide(hidden_size, world_size) + self.hidden_size_per_attention_head = mpu.divide( + hidden_size, num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + num_attention_heads, world_size) + # Strided linear layer. + self.query = mpu.ColumnParallelLinear( + hidden_size, + hidden_size, + gather_output=False, + init_method=init_method) + self.key_value = mpu.ColumnParallelLinear( + hidden_size, + 2 * hidden_size, + stride=2, + gather_output=False, + init_method=init_method) + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = torch.nn.Dropout(attention_dropout_prob) + + # Output. + self.dense = mpu.RowParallelLinear( + hidden_size, + hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method) + self.output_dropout = torch.nn.Dropout(output_dropout_prob) + + if deepspeed.checkpointing.is_configured(): + mpu.get_cuda_rng_tracker = deepspeed.checkpointing.get_cuda_rng_tracker + mpu.checkpoint = deepspeed.checkpointing.checkpoint + + def _transpose_for_scores(self, tensor): + """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with + size [b, np, s, hn]. + """ + new_tensor_shape = tensor.size()[:-1] + \ + (self.num_attention_heads_per_partition, # noqa + self.hidden_size_per_attention_head) # noqa + tensor = tensor.view(*new_tensor_shape) + return tensor.permute(0, 2, 1, 3) + + def forward(self, hidden_states, encoder_states, cross_mask): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Attention heads. [b, s, hp] + mixed_query_layer = self.query(hidden_states) + mixed_x_layer = self.key_value(encoder_states) + (mixed_key_layer, mixed_value_layer) = mpu.split_tensor_along_last_dim( + mixed_x_layer, 2) + + # Reshape and transpose [b, np, s, hn] + query_layer = self._transpose_for_scores(mixed_query_layer) + key_layer = self._transpose_for_scores(mixed_key_layer) + value_layer = self._transpose_for_scores(mixed_value_layer) + # Raw attention scores. [b, np, s, s] + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.hidden_size_per_attention_head) + if cross_mask is not None: + # Apply the left to right attention mask. + attention_scores = torch.mul(attention_scores, cross_mask) - \ + 10000.0 * (1.0 - cross_mask) # noqa + + # Attention probabilities. [b, np, s, s] + attention_probs = torch.nn.Softmax(dim=-1)(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + with mpu.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + + # Context layer. + # [b, np, s, hn] + context_layer = torch.matmul(attention_probs, value_layer) + # [b, s, np, hn] + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size_per_partition,) # noqa + # [b, s, hp] + context_layer = context_layer.view(*new_context_layer_shape) + + # Output. [b, s, h] + output = self.dense(context_layer) + output = self.output_dropout(output) + + return output + + +class ParallelSelfAttention(torch.nn.Module): + """Parallel self-attention layer for GPT2. + + Self-attention layer takes input with size [b, s, h] where b is + the batch size, s is the sequence lenght, and h is the hidden size + and creates output of the same size. + Arguments: + hidden_size: total hidden size of the layer (h). + num_attention_heads: number of attention heads (n). Note that we + require n to be divisible by number of GPUs + used to parallelize the model. Also, we + require hidden size to be divisible by n. + attention_dropout_prob: dropout probability for the attention scores. + init_method: weight initialization. + output_layer_init_method: output layer initialization. If None, use + `init_method`. + We use the following notation: + h: hidden_size + n: num_attention_heads + p: number of partitions + np: n/p + hp: h/p + hn: h/n + b: batch size + s: sequence length + """ + + def __init__(self, + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + init_method, + output_layer_init_method=None, + relative_encoding=False, + performer=False, + attention_scale=1.0): + super(ParallelSelfAttention, self).__init__() + self.performer = performer + # Set output layer initialization if not provided. + if output_layer_init_method is None: + output_layer_init_method = init_method + # Per attention head and per partition values. + world_size = mpu.get_model_parallel_world_size() + self.hidden_size_per_partition = mpu.divide(hidden_size, world_size) + self.hidden_size_per_attention_head = mpu.divide( + hidden_size, num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + num_attention_heads, world_size) + self.relative_encoding = relative_encoding + self.attention_scale = attention_scale + # Strided linear layer. + self.query_key_value = mpu.ColumnParallelLinear( + hidden_size, + 3 * hidden_size, + stride=3, + gather_output=False, + init_method=init_method) + if relative_encoding: + self.relative = mpu.ColumnParallelLinear( + hidden_size, + hidden_size, + gather_output=False, + init_method=init_method) + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = torch.nn.Dropout(attention_dropout_prob) + + # Output. + self.dense = mpu.RowParallelLinear( + hidden_size, + hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method) + self.output_dropout = torch.nn.Dropout(output_dropout_prob) + + if deepspeed.checkpointing.is_configured(): + mpu.get_cuda_rng_tracker = deepspeed.checkpointing.get_cuda_rng_tracker + mpu.checkpoint = deepspeed.checkpointing.checkpoint + + def _transpose_for_scores(self, tensor): + """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with + size [b, np, s, hn]. + """ + new_tensor_shape = tensor.size()[:-1] + \ + (self.num_attention_heads_per_partition, # noqa + self.hidden_size_per_attention_head) # noqa + tensor = tensor.view(*new_tensor_shape) + return tensor.permute(0, 2, 1, 3) + + @staticmethod + def _rel_shift(x, zero_triu=False): + # ql x kl x bsz x h + # bsz x h x ql x kl + zero_pad = torch.zeros((*x.size()[:-2], x.size(-2), 1), + device=x.device, + dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=-1) + + x_padded = x_padded.view(*x.size()[:-2], x.size(-1) + 1, x.size(-2)) + + x = x_padded[:, :, 1:].view_as(x) + + if zero_triu: + ones = torch.ones((x.size(0), x.size(1))) + x = x * torch.tril(ones, x.size(1) - x.size(0))[:, :, None, None] + + return x + + def forward(self, + hidden_states, + ltor_mask, + position_embeddings=None, + r_w_bias=None, + r_r_bias=None, + mem=None): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Attention heads. [b, s, hp] + query_length = hidden_states.size(1) + + if mem is None: + mixed_x_layer = self.query_key_value(hidden_states) + (mixed_query_layer, mixed_key_layer, + mixed_value_layer) = mpu.split_tensor_along_last_dim( + mixed_x_layer, 3) + else: + cat = torch.cat((mem, hidden_states), 1) + mixed_x_layer = self.query_key_value(cat) + (mixed_query_layer, mixed_key_layer, + mixed_value_layer) = mpu.split_tensor_along_last_dim( + mixed_x_layer, 3) + mixed_query_layer = mixed_query_layer[:, -query_length:] + + # Reshape and transpose [b, np, s, hn] + query_layer = self._transpose_for_scores(mixed_query_layer) + key_layer = self._transpose_for_scores(mixed_key_layer) + value_layer = self._transpose_for_scores(mixed_value_layer) + if self.relative_encoding: + relative_layer = self.relative(position_embeddings) + relative_layer = self._transpose_for_scores( + relative_layer) # 1 (bsz) x n_head x klen x d_head + # Raw attention scores. [b, np, qs, ks] + rw_head_q = query_layer + r_w_bias.unsqueeze(1) + ac_score = torch.matmul(rw_head_q, key_layer.transpose(-1, -2)) + rr_head_q = query_layer + r_r_bias.unsqueeze(1) + bd_score = torch.matmul(rr_head_q, + relative_layer.transpose(-1, -2)) + bd_score = self._rel_shift(bd_score) # qlen x klen x bsz x n_head + # bd_score = bd_score.permute(2, 3, 0, 1) # bsz n_head qlen klen + + attention_scores = ac_score + bd_score + attention_scores = attention_scores / math.sqrt( + self.hidden_size_per_attention_head) + else: + if self.attention_scale > 1.0: + # Raw attention scores. [b, np, s, s] + attention_scores = torch.matmul( + query_layer / math.sqrt(self.attention_scale), + key_layer.transpose(-1, -2) + / math.sqrt(self.hidden_size_per_attention_head + * self.attention_scale)) + else: + attention_scores = torch.matmul( + query_layer, + key_layer.transpose(-1, -2) + / math.sqrt(self.hidden_size_per_attention_head)) + + # Apply the left to right attention mask. + attention_scores = torch.mul(attention_scores, ltor_mask) + if self.attention_scale > 1.0: + max_attention_scores = attention_scores.max( + dim=-1, keepdim=True)[0] + attention_scores -= max_attention_scores + attention_scores *= self.attention_scale + # if torch.distributed.get_rank() == 0: + # print(min_attention_scores, attention_scores.max().item()) + attention_scores = attention_scores + (-65504.0) * (1.0 - ltor_mask) + # Attention probabilities. [b, np, s, s] + attention_probs = torch.nn.Softmax(dim=-1)(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + with mpu.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + + # Context layer. + # [b, np, s, hn] + context_layer = torch.matmul(attention_probs, value_layer) + # [b, s, np, hn] + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + \ + (self.hidden_size_per_partition,) # noqa + # [b, s, hp] + context_layer = context_layer.view(*new_context_layer_shape) + + # Output. [b, s, h] + output = self.dense(context_layer) + output = self.output_dropout(output) + + return output + + +@torch.jit.script +def gelu_impl(x): + """OpenAI's gelu implementation.""" + return 0.5 * x * ( + 1.0 + torch.tanh(0.7978845608028654 * x * # noqa + (1.0 + 0.044715 * x * x))) # noqa + + +def gelu(x): + return gelu_impl(x) + + +class ParallelMLP(torch.nn.Module): + """MLP for GPT2. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform gelu transformation, and project the + state back into h hidden dimension. At the end, dropout is also + applied. + + Arguments: + hidden_size: The hidden size of the self attention. + output_dropout_prob: dropout probability for the outputs + after self attention and final output. + init_method: initialization method used for the weights. Note + that all biases are initialized to zero and + layernorm weight are initialized to one. + output_layer_init_method: output layer initialization. If None, + use `init_method`. + """ + + def __init__(self, + hidden_size, + output_dropout_prob, + init_method, + output_layer_init_method=None): + super(ParallelMLP, self).__init__() + # Set output layer initialization if not provided. + if output_layer_init_method is None: + output_layer_init_method = init_method + # Project to 4h. + self.dense_h_to_4h = mpu.ColumnParallelLinear( + hidden_size, + 4 * hidden_size, + gather_output=False, + init_method=init_method) + # Project back to h. + self.dense_4h_to_h = mpu.RowParallelLinear( + 4 * hidden_size, + hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method) + self.dropout = torch.nn.Dropout(output_dropout_prob) + + def forward(self, hidden_states): + # [b, s, 4hp] + intermediate_parallel = self.dense_h_to_4h(hidden_states) + intermediate_parallel = gelu(intermediate_parallel) + + # [b, s, h] + output = self.dense_4h_to_h(intermediate_parallel) + output = self.dropout(output) + return output + + +class ParallelDecoderLayer(torch.nn.Module): + """A single layer transformer for GPT2. + + We use the following notation: + h: hidden size + n: number of attention heads + b: batch size + s: sequence length + Transformore layer takes input with size [b, s, h] and returns an + output of the same size. + + Arguments: + hidden_size: The hidden size of the self attention. + num_attention_heads: number of attention head in the self + attention. + attention_dropout_prob: dropout probability of the attention + score in self attention. + output_dropout_prob: dropout probability for the outputs + after self attention and final output. + layernorm_epsilon: epsilon used in layernorm to avoid + division by zero. + init_method: initialization method used for the weights. Note + that all biases are initialized to zero and + layernorm weight are initialized to one. + output_layer_init_method: output layers (attention output and + mlp output) initialization. If None, + use `init_method`. + """ + + def __init__(self, + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + layernorm_epsilon, + init_method, + output_layer_init_method=None): + super(ParallelDecoderLayer, self).__init__() + # Set output layer initialization if not provided. + if output_layer_init_method is None: + output_layer_init_method = init_method + + # Layernorm on the input data. + self.input_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon) + + # Self attention. + self.self_attention = ParallelSelfAttention( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + init_method, + output_layer_init_method=output_layer_init_method) + + # Layernorm after the self attention. + self.post_self_layernorm = LayerNorm( + hidden_size, eps=layernorm_epsilon) + + self.cross_attention = ParallelCrossAttention( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + init_method, + output_layer_init_method=output_layer_init_method) + + # Layernorm after the cross attention. + self.post_attention_layernorm = LayerNorm( + hidden_size, eps=layernorm_epsilon) + + # MLP + self.mlp = ParallelMLP( + hidden_size, + output_dropout_prob, + init_method, + output_layer_init_method=output_layer_init_method) + + def forward(self, + hidden_states, + encoder_states, + ltor_mask, + cross_mask=None): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Layer norm at the begining of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + self_attention_output = self.self_attention(layernorm_output, + ltor_mask) + # Residual connection. + self_layernorm_input = hidden_states + self_attention_output + # Layer norm post the self attention. + self_layernorm_output = self.post_self_layernorm(self_layernorm_input) + # Cross attention + attention_output = self.cross_attention(self_layernorm_output, + encoder_states, cross_mask) + # Residual connection + layernorm_input = self_layernorm_input + attention_output + # Layer norm post the cross attention + layernorm_output = self.post_attention_layernorm(layernorm_input) + # MLP. + mlp_output = self.mlp(layernorm_output) + # Second residual connection. + output = layernorm_input + mlp_output + return output + + +class ParallelTransformerLayer(torch.nn.Module): + """A single layer transformer for GPT2. + + We use the following notation: + h: hidden size + n: number of attention heads + b: batch size + s: sequence length + Transformore layer takes input with size [b, s, h] and returns an + output of the same size. + + Arguments: + hidden_size: The hidden size of the self attention. + num_attention_heads: number of attention head in the self + attention. + attention_dropout_prob: dropout probability of the attention + score in self attention. + output_dropout_prob: dropout probability for the outputs + after self attention and final output. + layernorm_epsilon: epsilon used in layernorm to avoid + division by zero. + init_method: initialization method used for the weights. Note + that all biases are initialized to zero and + layernorm weight are initialized to one. + output_layer_init_method: output layers (attention output and + mlp output) initialization. If None, + use `init_method`. + """ + + def __init__(self, + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + layernorm_epsilon, + init_method, + output_layer_init_method=None, + relative_encoding=False, + performer=False, + attention_scale=1.0): + super(ParallelTransformerLayer, self).__init__() + # Set output layer initialization if not provided. + if output_layer_init_method is None: + output_layer_init_method = init_method + + # Layernorm on the input data. + self.input_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon) + + # Self attention. + self.attention = ParallelSelfAttention( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + init_method, + output_layer_init_method=output_layer_init_method, + relative_encoding=relative_encoding, + performer=performer, + attention_scale=attention_scale) + + # Layernorm on the input data. + self.post_attention_layernorm = LayerNorm( + hidden_size, eps=layernorm_epsilon) + + # MLP + self.mlp = ParallelMLP( + hidden_size, + output_dropout_prob, + init_method, + output_layer_init_method=output_layer_init_method) + + def forward(self, + hidden_states, + ltor_mask, + position_embeddings=None, + r_w_bias=None, + r_r_bias=None, + mem=None): + # hidden_states: [b, s, h] + # ltor_mask: [1, 1, s, s] + + # Layer norm at the begining of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + mem = self.input_layernorm(mem) if mem is not None else None + # Self attention. + attention_output = self.attention(layernorm_output, ltor_mask, + position_embeddings, r_w_bias, + r_r_bias, mem) + # Residual connection. + layernorm_input = hidden_states + attention_output + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + # MLP. + mlp_output = self.mlp(layernorm_output) + # Second residual connection. + output = layernorm_input + mlp_output + + return output + + +def unscaled_init_method(sigma): + """Init method based on N(0, sigma).""" + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=0.0, std=sigma) + + return init_ + + +def scaled_init_method(sigma, num_layers): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +class GPT2ParallelTransformer(torch.nn.Module): + """GPT-2 transformer. + + This module takes input from embedding layer and it's output can + be used directly by a logit layer. It consists of L (num-layers) + blocks of: + layer norm + self attention + residual connection + layer norm + mlp + residual connection + followed by a final layer norm. + + Arguments: + num_layers: Number of transformer layers. + hidden_size: The hidden size of the self attention. + num_attention_heads: number of attention head in the self + attention. + attention_dropout_prob: dropout probability of the attention + score in self attention. + output_dropout_prob: dropout probability for the outputs + after self attention and final output. + checkpoint_activations: if True, checkpoint activations. + checkpoint_num_layers: number of layers to checkpoint. This + is basically the chunk size in checkpoitning. + layernorm_epsilon: epsilon used in layernorm to avoid + division by zero. + init_method_std: standard deviation of the init method which has + the form N(0, std). + use_scaled_init_for_output_weights: If Ture use 1/sqrt(2*num_layers) + scaling for the output weights ( + output of self attention and mlp). + """ + + def __init__( + self, + num_layers, + hidden_size, + num_attention_heads, + max_sequence_length, + max_memory_length, + embedding_dropout_prob, + attention_dropout_prob, + output_dropout_prob, + checkpoint_activations, + checkpoint_num_layers=1, + layernorm_epsilon=1.0e-5, + init_method_std=0.02, + use_scaled_init_for_output_weights=True, + relative_encoding=False, + block_position_encoding=False, + performer=False, + use_decoder_layer=False, + attention_scale=1.0, + ): + super(GPT2ParallelTransformer, self).__init__() + self.hidden_size = hidden_size + # Store activation checkpoiting flag. + self.checkpoint_activations = checkpoint_activations + self.checkpoint_num_layers = checkpoint_num_layers + self.max_memory_length = max_memory_length + self.performer = performer + self.use_decoder_layer = use_decoder_layer + assert not (performer and relative_encoding) + + output_layer_init_method = None + if use_scaled_init_for_output_weights: + output_layer_init_method = scaled_init_method( + init_method_std, num_layers) + # Embeddings dropout + self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob) + self.relative_encoding = relative_encoding + self.block_position_encoding = block_position_encoding + if relative_encoding: + # Relative position embedding + self.position_embeddings = PositionalEmbedding(hidden_size) + # Per attention head and per partition values. + world_size = mpu.get_model_parallel_world_size() + self.hidden_size_per_attention_head = mpu.divide( + hidden_size, num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + num_attention_heads, world_size) + self.r_w_bias = torch.nn.Parameter( + torch.Tensor(self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head)) + self.r_w_bias.model_parallel = True + self.r_r_bias = torch.nn.Parameter( + torch.Tensor(self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head)) + self.r_r_bias.model_parallel = True + # Always initialize bias to zero. + with torch.no_grad(): + self.r_w_bias.zero_() + self.r_r_bias.zero_() + else: + # Position embedding (serial). + if block_position_encoding: + self.position_embeddings = torch.nn.Embedding( + max_sequence_length + 1, hidden_size) + self.block_position_embeddings = torch.nn.Embedding( + max_sequence_length + 1, hidden_size) + torch.nn.init.normal_( + self.block_position_embeddings.weight, + mean=0.0, + std=init_method_std) + else: + self.position_embeddings = torch.nn.Embedding( + max_sequence_length, hidden_size) + # Initialize the position embeddings. + torch.nn.init.normal_( + self.position_embeddings.weight, mean=0.0, std=init_method_std) + + def get_layer(): + if use_decoder_layer: + return ParallelDecoderLayer( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + layernorm_epsilon, + unscaled_init_method(init_method_std), + output_layer_init_method=output_layer_init_method) + else: + return ParallelTransformerLayer( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + layernorm_epsilon, + unscaled_init_method(init_method_std), + output_layer_init_method=output_layer_init_method, + relative_encoding=relative_encoding, + performer=performer, + attention_scale=attention_scale) + + # Transformer layers. + self.layers = torch.nn.ModuleList( + [get_layer() for _ in range(num_layers)]) + + # Final layer norm before output. + self.final_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon) + + if deepspeed.checkpointing.is_configured(): + mpu.get_cuda_rng_tracker = deepspeed.checkpointing.get_cuda_rng_tracker + mpu.checkpoint = deepspeed.checkpointing.checkpoint + + def forward(self, + hidden_states, + position_ids, + attention_mask, + memory_states=None, + encoder_states=None, + return_memory=False, + detach_memory=True): + batch_size, query_length = hidden_states.size()[:2] + memory_length = memory_states[0].size(1) if memory_states else 0 + key_length = query_length + memory_length + # attention mask is the beginning postion of B region, \in [0, query_len) + is_scalar = torch.numel(attention_mask) == 1 + is_sep = is_scalar or torch.numel(attention_mask) == batch_size + if self.performer: + assert is_scalar, 'attention_mask should be a scalar to indicate the seperation position.' + assert memory_length == 0, 'Do not support transformer-xl.' + if is_sep: + sep = attention_mask.item() if is_scalar else attention_mask + + # conventional transformer + def build_mask_matrix(seq_length, sep, memory_length=0): + m = hidden_states.new_ones((1, seq_length, seq_length)) + m = torch.tril(m) + if is_scalar: + m[0, :, :sep] = 1 + else: + m = m.expand(batch_size, -1, -1) + ids = torch.arange( + seq_length, device=sep.device, + dtype=sep.dtype).view(1, -1) + mask = ids < sep.view(-1, 1) + m = m.masked_fill(mask.unsqueeze(1).expand_as(m), 1) + if memory_length > 0: + m = m.expand(batch_size, -1, -1) + m = torch.cat( + (hidden_states.new_ones((batch_size, seq_length, + memory_length)), m), # noqa + dim=2) # noqa + m = m.unsqueeze(1) + return m + + if not self.performer: + attention_mask = build_mask_matrix( + query_length, sep, memory_length=memory_length) + else: + attention_mask = attention_mask[:, :, :, + -query_length - memory_length:] + + if self.relative_encoding: + position_sequence = torch.arange( + key_length - 1, + -1, + -1.0, + device=hidden_states.device, + dtype=hidden_states.dtype) + position_embeddings = self.position_embeddings(position_sequence) + # Apply dropout + position_embeddings = self.embedding_dropout(position_embeddings) + else: + if self.block_position_encoding: + position_ids, block_position_ids = position_ids[:, + 0], position_ids[:, + 1] + position_embeddings = self.position_embeddings(position_ids) + hidden_states = hidden_states + position_embeddings + if self.block_position_encoding: + block_position_embeddings = self.block_position_embeddings( + block_position_ids) + hidden_states = hidden_states + block_position_embeddings + hidden_states = self.embedding_dropout(hidden_states) + + def check_detach(_hidden_states): + if detach_memory: + return _hidden_states.detach() + return _hidden_states + + if self.max_memory_length > 0 or return_memory: + mem_layers = [check_detach(hidden_states)] + else: + mem_layers = [] + + def custom(start, end): + + def custom_forward(*inputs): + layers_ = self.layers[start:end] + x_, inputs = inputs[0], inputs[1:] + if self.relative_encoding: + inputs, mems_ = inputs[:4], inputs[4:] + else: + inputs, mems_ = inputs[:1], inputs[1:] + for i, layer in enumerate(layers_): + mem_i_ = mems_[i] if mems_ else None + x_ = layer(x_, *inputs, mem=mem_i_) + if self.max_memory_length > 0 or return_memory: + mem_layers.append(check_detach(x_)) + return x_ + + return custom_forward + + if self.checkpoint_activations: + l = 0 # noqa + num_layers = len(self.layers) + chunk_length = self.checkpoint_num_layers + while l < num_layers: + args = [hidden_states, attention_mask + ] if not self.use_decoder_layer else [ + hidden_states, + encoder_states, + attention_mask # noqa + ] # noqa + if self.relative_encoding: + args += [position_embeddings, self.r_w_bias, self.r_r_bias] + if memory_states: + args += memory_states[l:l + chunk_length] + hidden_states = mpu.checkpoint( + custom(l, l + chunk_length), *args) + l += chunk_length # noqa + else: + for i, layer in enumerate(self.layers): + args = [hidden_states, attention_mask + ] if not self.use_decoder_layer else [ + hidden_states, + encoder_states, + attention_mask # noqa + ] # noqa + if self.relative_encoding: + args += [position_embeddings, self.r_w_bias, self.r_r_bias] + mem_i = memory_states[i] if memory_states else None + hidden_states = layer(*args, mem=mem_i) + if self.max_memory_length > 0 or return_memory: + mem_layers.append(check_detach(hidden_states)) + + # Final layer norm. + output = self.final_layernorm(hidden_states) + if self.max_memory_length > 0 or return_memory: + mem_layers = self.update_mems( + mem_layers, memory_states, return_memory=return_memory) + + return (output, mem_layers) + + def update_mems(self, hiddens, mems, return_memory=False): + memory_length = mems[0].size(1) if mems else 0 + query_length = hiddens[0].size(1) + new_memory_length = memory_length + query_length + if not return_memory: + new_memory_length = min(self.max_memory_length, new_memory_length) + new_mems = [] + # with torch.no_grad(): + for i in range(len(hiddens)): + if new_memory_length <= query_length: + new_mems.append(hiddens[i][:, -new_memory_length:]) + else: + new_mems.append( + torch.cat((mems[i][:, -new_memory_length + query_length:], + hiddens[i]), + dim=1)) + return new_mems + + +class BertParallelSelfAttention(torch.nn.Module): + """Parallel self-attention layer for BERT. + + Self-attention layer takes input with size [b, s, h] where b is + the batch size, s is the sequence lenght, and h is the hidden size + and creates output of the same size. + Arguments: + hidden_size: total hidden size of the layer (h). + num_attention_heads: number of attention heads (n). Note that we + require n to be divisible by number of GPUs + used to parallelize the model. Also, we + require hidden size be divisible by n. + dropout_prob: dropout probability for the attention scores. + output_parallel: If true, no all-gather is done on the output and + the output values will be per partition. + We use the following notation: + h: hidden_size + n: num_attention_heads + p: number of partitions + np: n/p + hp: h/p + hn: h/n + b: batch size + s: sequence length + """ + + def __init__(self, + hidden_size, + num_attention_heads, + dropout_prob, + output_parallel=False, + init_method=init.xavier_normal_): + super(BertParallelSelfAttention, self).__init__() + # Input configuration. + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.dropout_prob = dropout_prob + self.output_parallel = output_parallel + # Per attention head and per partition values. + world_size = mpu.get_model_parallel_world_size() + self.hidden_size_per_partition = mpu.divide(hidden_size, world_size) + self.hidden_size_per_attention_head = mpu.divide( + hidden_size, num_attention_heads) + self.num_attention_heads_per_partition = mpu.divide( + num_attention_heads, world_size) + # Strided linear layer. + self.query_key_value = mpu.ColumnParallelLinear( + hidden_size, + 3 * hidden_size, + stride=3, + gather_output=False, + init_method=init_method) + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.dropout = torch.nn.Dropout(dropout_prob) + + if deepspeed.checkpointing.is_configured(): + mpu.get_cuda_rng_tracker = deepspeed.checkpointing.get_cuda_rng_tracker + mpu.checkpoint = deepspeed.checkpointing.checkpoint + + def _transpose_for_scores(self, tensor): + """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with + size [b, np, s, hn]. + """ + new_tensor_shape = tensor.size()[:-1] + \ + (self.num_attention_heads_per_partition, # noqa + self.hidden_size_per_attention_head) # noqa + tensor = tensor.view(*new_tensor_shape) + return tensor.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask): + + # Attention heads. [b, s, hp] + mixed_x_layer = self.query_key_value(hidden_states) + (mixed_query_layer, mixed_key_layer, + mixed_value_layer) = mpu.split_tensor_along_last_dim( + mixed_x_layer, 3) + + # Reshape and transpose [b, np, s, hn] + query_layer = self._transpose_for_scores(mixed_query_layer) + key_layer = self._transpose_for_scores(mixed_key_layer) + value_layer = self._transpose_for_scores(mixed_value_layer) + + # Raw attention scores. [b, np, s, s] + norm_factor = math.sqrt(math.sqrt(self.hidden_size_per_attention_head)) + attention_scores = torch.matmul( + query_layer / norm_factor, + key_layer.transpose(-1, -2) / norm_factor) + # Apply the attention mask. + attention_scores += attention_mask + + # Attention probabilities. [b, np, s, s] + attention_probs = torch.nn.Softmax(dim=-1)(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + with mpu.get_cuda_rng_tracker().fork(): + attention_probs = self.dropout(attention_probs) + + # Context layer. + # [b, np, s, hn] + context_layer = torch.matmul(attention_probs, value_layer) + # [b, s, np, hn] + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size_per_partition, ) # noqa + # [b, s, hp] + context_layer = context_layer.view(*new_context_layer_shape) + + # Output. [b, s, h] + if self.output_parallel: + output = context_layer + else: + output = mpu.gather_from_model_parallel_region(context_layer) + + return output + + +class BertParallelTransformerOutput(torch.nn.Module): + """The output layer used after self attention and intermediate + parts of transformer layer.""" + + def __init__(self, + input_size, + output_size, + dropout_prob, + layernorm_epsilon=1.0e-12, + input_is_parallel=False, + init_method=init.xavier_normal_): + super(BertParallelTransformerOutput, self).__init__() + # Components. + self.dense = mpu.RowParallelLinear( + input_size, + output_size, + input_is_parallel=input_is_parallel, + init_method=init_method) + self.dropout = torch.nn.Dropout(dropout_prob) + self.layernorm = LayerNorm(output_size, eps=layernorm_epsilon) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + layernorm_input = hidden_states + input_tensor + hidden_states = self.layernorm(layernorm_input) + return hidden_states + + +class BertParallelTransformerLayer(torch.nn.Module): + """A single layer transformer for Bert. + + We use the following notation: + h: hidden size + n: number of attention heads + b: batch size + s: sequence length + Transformore layer takes input with size [b, s, h] and returns an + output of the same size. + + Arguments: + hidden_size: The hidden size of the self attention. + intermediate_size: size of the intermediate state after + self attention. In both BERT and GPT + this is set to be 4 times the hidden + size. + num_attention_heads: number of attention head in the self + attention. + attention_dropout_prob: dropout probability of the attention + score in self attention. + output_dropout_prob: dropout probability for the outputs + after self attention and final output. + intermediate_activation_fn: activation function for output + of intermediate. + layernorm_epsilon: epsilon used in layernorm to avoid + division by zero. + init_method: initialization method used for the weights. Note + that all biases are initialized to zero and + layernorm weight are initialized to one. + """ + + def __init__(self, + hidden_size, + intermediate_size, + num_attention_heads, + attention_dropout_prob, + output_dropout_prob, + intermediate_activation_fn, + layernorm_epsilon, + init_method=init.xavier_normal_): + super(BertParallelTransformerLayer, self).__init__() + + # Self attention. + self.attention = BertParallelSelfAttention( + hidden_size, + num_attention_heads, + attention_dropout_prob, + output_parallel=True, + init_method=init_method) + # Self attention output. + self.self_output = BertParallelTransformerOutput( + hidden_size, + hidden_size, + output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + input_is_parallel=True, + init_method=init_method) + # Intermediate. + self.intermediate = mpu.ColumnParallelLinear( + hidden_size, + intermediate_size, + gather_output=False, + init_method=init_method) + self.intermediate_activation_fn = intermediate_activation_fn + # Output. + self.output = BertParallelTransformerOutput( + intermediate_size, + hidden_size, + output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + input_is_parallel=True, + init_method=init_method) + + def forward(self, hidden_states, attention_mask): + # [b, s, hp] + attention_output_parallel = self.attention(hidden_states, + attention_mask) + # [b, s, h] + attention_self_output = self.self_output(attention_output_parallel, + hidden_states) + # [b, s, ip] + intermediate_output_parallel = self.intermediate(attention_self_output) + intermediate_output_parallel = self.intermediate_activation_fn( + intermediate_output_parallel) + # [b, s, h] + layer_output = self.output(intermediate_output_parallel, + attention_self_output) + + return layer_output diff --git a/runtime/Lib/site_packages/models/nlp/mglm/process_grid.py b/runtime/Lib/site_packages/models/nlp/mglm/process_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d26cce13c2ea895fffac2eb6e1a40bf5201b7b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/process_grid.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022 Zhipu.AI + +import glob +import os +import statistics +import sys + +import json + +path_pattern = sys.argv[1] +target_type = sys.argv[2] +best_value, best_result, best_name = None, None, None +mean_result = {} +print(path_pattern) +for dir_path in glob.glob(path_pattern, recursive=True): + entry = os.path.basename(dir_path) + valid_result = None + test_found = os.path.exists(os.path.join(dir_path, 'test_results.json')) + valid_path = os.path.join(dir_path, 'results.json') + if os.path.exists(valid_path): + print(entry) + with open(valid_path, encoding='utf-8') as file: + valid_result = json.load(file) + else: + print(f'{entry} no validation results') + continue + if not test_found: + print(f'{entry} not tested yet') + if target_type == 'max': + metric = sys.argv[3] + metric_value = valid_result[metric] + if best_value is None or metric_value > best_value: + best_value = metric_value + best_result = valid_result + best_name = entry + elif target_type == 'mean' or target_type == 'median': + if mean_result: + for metric, value in valid_result.items(): + if metric not in ['type', 'epoch']: + mean_result[metric].append(value) + else: + mean_result = { + metric: [value] + for metric, value in valid_result.items() + if metric not in ['type', 'epoch'] + } + +if target_type == 'max': + print(f'Best result found at {best_name}: {best_result}') +elif target_type == 'mean': + mean_result = { + metric: sum(value) / len(value) + for metric, value in mean_result.items() + } + print(f'Mean result {mean_result}') +elif target_type == 'median': + mean_result = { + metric: statistics.median(value) + for metric, value in mean_result.items() + } + print(f'Mean result {mean_result}') diff --git a/runtime/Lib/site_packages/models/nlp/mglm/run_test.py b/runtime/Lib/site_packages/models/nlp/mglm/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..2f5682653cabc8a7bf02d10ddad805098367365a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/run_test.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022 Zhipu.AI + +import sys + +if sys.argv[1] == 'block': + from test.test_block import main + main() +elif sys.argv[1] == 'rel_shift': + from test.test_rel_shift import main + main() diff --git a/runtime/Lib/site_packages/models/nlp/mglm/test/__init__.py b/runtime/Lib/site_packages/models/nlp/mglm/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/nlp/mglm/test/test_block.py b/runtime/Lib/site_packages/models/nlp/mglm/test/test_block.py new file mode 100644 index 0000000000000000000000000000000000000000..eb630835151fc49ae204d1f0fad723c2f8c171f7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/test/test_block.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022 Zhipu.AI + +import random +from argparse import Namespace + +import numpy as np +from blocklm_utils import ConstructBlockStrategy + + +# rng = random.Random() +# span_lengths = [2, 3, 4, 2, 3, 4] +# length = 100 +# +# counts = np.array([0] * length) +# for _ in range(10000): +# rng.shuffle(span_lengths) +# spans = ConstructBlockStrategy.sample_spans(span_lengths, length, rng) +# for start, end in spans: +# counts[start: end] += 1 +# print(counts) +def main(): + args = Namespace() + args.seq_length = 10 + args.eod_token = 0 + + strategy = ConstructBlockStrategy( + args, None, bert_ratio=0.4, max_seq_length=128) + counts = np.array([0] * 10) + for _ in range(10000): + spans = strategy.sample_span_in_document( + np.array([1, 2, 3, 0, 4, 5, 6, 7, 9, 0], dtype=int), [1, 1], + random.Random()) + for start, end in spans: + counts[start:end] += 1 + + print(counts) diff --git a/runtime/Lib/site_packages/models/nlp/mglm/test/test_rel_shift.py b/runtime/Lib/site_packages/models/nlp/mglm/test/test_rel_shift.py new file mode 100644 index 0000000000000000000000000000000000000000..ad68b15ecb5b5b69e9881ebad1b5a64c90727d5e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/test/test_rel_shift.py @@ -0,0 +1,27 @@ +# Copyright (c) 2022 Zhipu.AI + +import matplotlib.pyplot as plt +import numpy as np +from learning_rates import AnnealingLR +from torch.nn.modules import Linear +from torch.optim import Adam + + +def main(): + model = Linear(10, 10) + optimizer = Adam(model.parameters()) + lr_scheduler = AnnealingLR( + optimizer, + start_lr=0.00015, + warmup_iter=3000, + num_iters=300000, + decay_style='cosine', + decay_ratio=0.1) + steps = np.arange(0, 400000, 10, dtype=int) + rates = [] + for step in steps: + lr_scheduler.num_iters = step + rates.append(lr_scheduler.get_lr()) + print(rates) + plt.plot(steps, rates) + plt.savefig('lr.pdf', format='pdf') diff --git a/runtime/Lib/site_packages/models/nlp/mglm/train_utils.py b/runtime/Lib/site_packages/models/nlp/mglm/train_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4f8bf8b1bf603d15fa80d74d41fb0aa7459d4472 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/train_utils.py @@ -0,0 +1,472 @@ +# Copyright (c) 2022 Zhipu.AI + +import deepspeed +import torch +from apex.optimizers import FusedAdam as Adam +from megatron_util import mpu +from megatron_util.fp16 import DynamicLossScaler, FP16_Module, FP16_Optimizer +from torch import distributed as dist + +from .model import DistributedDataParallel as LocalDDP +from .model import (GLMForMultiTokenCloze, GLMForMultiTokenClozeFast, + GLMForSequenceClassification, GLMForSingleTokenCloze, + GLMModel) +from .model import PyTorchDistributedDataParallel as TorchDDP +from .model import glm_get_params_for_weight_decay_optimization +from .utils import get_checkpoint_iteration, get_checkpoint_name, print_rank_0 + + +def load_pretrained(model, checkpoint_path, args, task_tokens=None): + load_dir, tag, release, success = get_checkpoint_iteration(checkpoint_path) + checkpoint_name = get_checkpoint_name(load_dir, tag, release) + if mpu.get_data_parallel_rank() == 0: + print('global rank {} is loading pretrained model {}'.format( + torch.distributed.get_rank(), checkpoint_name)) + # Load the checkpoint. + sd = torch.load(checkpoint_name, map_location='cpu') + if args.deepspeed: + model = model.module + if isinstance(model, TorchDDP): + model = model.module + if isinstance(model, FP16_Module): + model = model.module + if hasattr(model, 'model'): + model = model.model + + # Model. + def extend_embedding_weights(state_weights, model_weights): + original_length = state_weights.shape[0] + assert original_length <= args.max_position_embeddings + 1 + new_weights = model_weights.clone() + new_weights[:original_length] = state_weights + return new_weights + + if args.block_lm: + if 'transformer.block_position_embeddings.weight' in sd['module']: + position_weights = sd['module'][ + 'transformer.position_embeddings.weight'] + if args.max_position_embeddings + 1 > position_weights.shape[0]: + sd['module'][ + 'transformer.position_embeddings.weight'] = extend_embedding_weights( + position_weights, + model.state_dict() + ['transformer.position_embeddings.weight'].data) + print_rank_0( + f'Extend position embedding to {args.max_position_embeddings + 1}' + ) + if 'transformer.block_position_embeddings.weight' in sd['module']: + block_position_weights = sd['module'][ + 'transformer.block_position_embeddings.weight'] + if args.max_position_embeddings + 1 > block_position_weights.shape[ + 0]: + sd['module'][ + 'transformer.block_position_embeddings.weight'] = extend_embedding_weights( + block_position_weights, + model.state_dict() + ['transformer.block_position_embeddings.weight'].data) + print_rank_0( + f'Extend block position embedding to {args.max_position_embeddings + 1}' + ) + for key in list(model.state_dict().keys()): + print(key) + model.state_dict()[key.replace( + 'mixins.block_position_embedding.block_position_embeddings.weight', + 'transformer.block_position_embeddings.weight').replace( + 'transformer.word_embeddings.weight', + 'word_embeddings.weight')] = model.state_dict().pop(key) + + missing_keys, unexpected_keys = model.load_state_dict( + sd['module'], strict=False) + if missing_keys or unexpected_keys: + print_rank_0( + f'Missing keys {missing_keys}, unexpected keys {unexpected_keys}') + if args.continuous_prompt and args.prompt_init: + model.prompt_spell.init_embedding(model.word_embeddings.weight.data, + task_tokens) + + +def get_model(args, + model_type=None, + multi_token=True, + num_labels=None, + spell_length=None): + """Build the model.""" + print_rank_0('building GPT2 model ...') + if args.pretrained_bert: + if model_type == 'multiple_choice': + model = BertForMultipleChoice.from_pretrained( + args.tokenizer_model_type, + cache_dir=args.cache_dir, + fp32_layernorm=args.fp32_layernorm, + fp32_embedding=args.fp32_embedding, + layernorm_epsilon=args.layernorm_epsilon) + elif model_type == 'classification': + model = BertForSequenceClassification.from_pretrained( + args.tokenizer_model_type, + cache_dir=args.cache_dir, + fp32_layernorm=args.fp32_layernorm, + fp32_embedding=args.fp32_embedding, + layernorm_epsilon=args.layernorm_epsilon, + num_labels=num_labels) + else: + raise NotImplementedError + else: + output_predict, paralle_output = True, True + if (model_type == 'multiple_choice' + or model_type == 'classification') and not args.cloze_eval: + output_predict = False + if model_type is not None: + paralle_output = False + if spell_length is not None: + print_rank_0(f'Continuous spell length {spell_length}') + model = GLMModel( + num_layers=args.num_layers, + vocab_size=args.vocab_size, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + embedding_dropout_prob=args.hidden_dropout, + attention_dropout_prob=args.attention_dropout, + output_dropout_prob=args.hidden_dropout, + max_sequence_length=args.max_position_embeddings, + max_memory_length=args.mem_length, + checkpoint_activations=args.checkpoint_activations, + checkpoint_num_layers=args.checkpoint_num_layers, + parallel_output=paralle_output, + relative_encoding=args.transformer_xl, + block_position_encoding=args.block_lm and not args.masked_lm, + output_predict=output_predict, + spell_length=spell_length, + spell_func=args.prompt_func, + attention_scale=args.attention_scale) + if args.freeze_transformer: + model.freeze_transformer( + tune_prefix_layers=args.tune_prefix_layers) + if model_type is not None: + if model_type == 'multiple_choice': + if args.cloze_eval: + if multi_token: + if args.fast_decode: + model = GLMForMultiTokenClozeFast( + model, length_penalty=args.length_penalty) + else: + model = GLMForMultiTokenCloze( + model, length_penalty=args.length_penalty) + else: + model = GLMForSingleTokenCloze( + model, take_softmax=args.adapet) + else: + model = GLMForSequenceClassification( + model, + args.hidden_size, + args.output_dropout, + args.pool_token, + num_class=num_labels) + elif model_type == 'classification': + model = GLMForSequenceClassification( + model, + args.hidden_size, + args.output_dropout, + args.pool_token, + num_class=num_labels) + elif model_type == 'generation': + pass + else: + raise NotImplementedError(model_type) + + if mpu.get_data_parallel_rank() == 0: + print( + ' > number of parameters on model parallel rank {}: {}'.format( + mpu.get_model_parallel_rank(), + sum([p.nelement() for p in model.parameters()])), + flush=True) + + # To prevent OOM for model sizes that cannot fit in GPU memory in full precision + if args.fp16: + model.half() + + # GPU allocation. + model.cuda(torch.cuda.current_device()) + + # Fp16 conversion. + if args.fp16: + model = FP16_Module(model) + + # Wrap model for distributed training. + if not args.deepspeed and (args.train_iters or args.epochs): + if args.DDP_impl == 'torch': + i = torch.cuda.current_device() + model = TorchDDP( + model, + device_ids=[i], + output_device=i, + process_group=mpu.get_data_parallel_group()) + elif args.DDP_impl == 'local': + model = LocalDDP(model) + else: + print_rank_0('Skip DDP model') + return model + + +def get_optimizer_param_groups(model): + # Build parameter groups (weight decay and non-decay). + while isinstance(model, (LocalDDP, TorchDDP, FP16_Module)): + model = model.module + param_groups = glm_get_params_for_weight_decay_optimization(model) + + # Add model parallel attribute if it is not set. + for param_group in param_groups: + # print('## param_group', len(param_group['params'])) + for param in param_group['params']: + if not hasattr(param, 'model_parallel'): + param.model_parallel = False + + return param_groups + + +def get_optimizer(param_groups, args): + """Set up the optimizer.""" + if args.cpu_optimizer: + # Apex FusedAdam uses decoupled weight decay so use the same here + if args.cpu_torch_adam: + cpu_adam_optimizer = torch.optim.AdamW + else: + from deepspeed.ops.adam import DeepSpeedCPUAdam + cpu_adam_optimizer = DeepSpeedCPUAdam + optimizer = cpu_adam_optimizer( + param_groups, lr=args.lr, weight_decay=args.weight_decay) + else: + # Use FusedAdam. + if args.optimizer == 'adam': + optimizer = Adam( + param_groups, + lr=args.lr, + weight_decay=args.weight_decay, + betas=(args.adam_beta1, args.adam_beta2), + eps=args.adam_eps) + elif args.optimizer == 'adafactor': + from transformers import Adafactor + optimizer = Adafactor( + param_groups, + lr=args.lr, + relative_step=False, + warmup_init=False) + else: + raise NotImplementedError + + print(f'Optimizer = {optimizer.__class__.__name__}') + if hasattr(args, 'deepspeed') and args.deepspeed: + raise NotImplementedError + # fp16 wrapper is not required for DeepSpeed. + # return optimizer + + # Wrap into fp16 optimizer. + if args.fp16: + optimizer = FP16_Optimizer( + optimizer, + static_loss_scale=args.loss_scale, + dynamic_loss_scale=args.dynamic_loss_scale, + dynamic_loss_args={ + 'scale_window': args.loss_scale_window, + 'min_scale': args.min_scale, + 'delayed_shift': args.hysteresis + }) + + return optimizer + + +def get_learning_rate_scheduler(optimizer, args): + """Build the learning rate scheduler.""" + + # Add linear learning rate scheduler. + if args.lr_decay_iters is not None: + num_iters = args.lr_decay_iters + else: + num_iters = args.train_iters + if args.finetune: + num_iters = num_iters // args.gradient_accumulation_steps + num_iters = max(1, num_iters) + init_step = -1 + warmup_iter = args.warmup * num_iters + lr_scheduler = AnnealingLR( + optimizer, + start_lr=args.lr, + warmup_iter=warmup_iter, + num_iters=num_iters - warmup_iter, + decay_style=args.lr_decay_style, + last_iter=init_step, + decay_ratio=args.lr_decay_ratio) + + return lr_scheduler + + +def setup_model_and_optimizer(args, + model_type=None, + multi_token=True, + num_labels=None, + spell_length=None): + """Setup model and optimizer.""" + + model = get_model( + args, + model_type=model_type, + multi_token=multi_token, + num_labels=num_labels, + spell_length=spell_length) + param_groups = get_optimizer_param_groups(model) + + if args.train_data is not None or args.data_dir is not None and ( + args.epochs > 0 or args.train_iters > 0): + if args.deepspeed: + print_rank_0('DeepSpeed is enabled.') + + model, optimizer, _, _ = deepspeed.initialize( + model=model, + model_parameters=param_groups, + args=args, + mpu=mpu, + dist_init_required=False) + else: + optimizer = get_optimizer(param_groups, args) + lr_scheduler = get_learning_rate_scheduler(optimizer, args) + else: + optimizer, lr_scheduler = None, None + + return model, optimizer, lr_scheduler + + +def backward_step(optimizer, model, lm_loss, args, timers): + """Backward step.""" + + # Total loss. + loss = lm_loss + + # Backward pass. + if args.deepspeed: + model.backward(loss) + else: + # optimizer.zero_grad() + if args.fp16: + optimizer.backward(loss, update_master_grads=False) + else: + loss.backward() + + if args.deepspeed or args.DDP_impl == 'torch': + # DeepSpeed backward propagation already addressed all reduce communication. + # Reset the timer to avoid breaking timer logs below. + timers('allreduce').reset() + else: + timers('allreduce').start() + model.allreduce_params( + reduce_after=False, fp32_allreduce=args.fp32_allreduce) + timers('allreduce').stop() + + # Update master gradients. + if not args.deepspeed: + if args.fp16: + optimizer.update_master_grads() + + # Clipping gradients helps prevent the exploding gradient. + if args.clip_grad > 0: + if not args.fp16: + mpu.clip_grad_norm(model.parameters(), args.clip_grad) + else: + optimizer.clip_master_grads(args.clip_grad) + + return lm_loss + + +def see_memory_usage(message, force=False): + if not force: + return + dist.barrier() + if dist.get_rank() == 0: + print(message) + print('Memory Allocated ', + torch.cuda.memory_allocated() / (1024 * 1024 * 1024), + 'GigaBytes') + print('Max Memory Allocated ', + torch.cuda.max_memory_allocated() / (1024 * 1024 * 1024), + 'GigaBytes') + print('Cache Allocated ', + torch.cuda.memory_cached() / (1024 * 1024 * 1024), 'GigaBytes') + print('Max cache Allocated ', + torch.cuda.max_memory_cached() / (1024 * 1024 * 1024), + 'GigaBytes') + print(' ') + # input("Press Any Key To Continue ..") + + +def train_step(data_iterator, + model, + optimizer, + lr_scheduler, + args, + timers, + forward_step_func, + mems=None, + single_step=False): + """Single training step.""" + lm_loss_total, count = 0.0, 0 + mems = [] if mems is None else mems + if not args.deepspeed: + optimizer.zero_grad() + while True: + skipped_iter, complete = 0, False + # Forward model for one step. + timers('forward').start() + lm_loss, mems, _ = forward_step_func(data_iterator, model, args, + timers, mems) + timers('forward').stop() + # print_rank_0("Forward step") + if not args.deepspeed: + lm_loss /= args.gradient_accumulation_steps + + reduced_loss = lm_loss.detach().clone().view(1) + torch.distributed.all_reduce( + reduced_loss.data, group=mpu.get_data_parallel_group()) + reduced_loss.data = reduced_loss.data / ( + args.world_size / args.model_parallel_size) + + if not DynamicLossScaler._has_inf_or_nan(reduced_loss): + lm_loss_total += reduced_loss + count += 1 + + # Calculate gradients, reduce across processes, and clip. + timers('backward').start() + backward_step(optimizer, model, lm_loss, args, timers) + timers('backward').stop() + # print_rank_0("Backward step") + # Update parameters. + timers('optimizer').start() + if args.deepspeed: + if model.is_gradient_accumulation_boundary(): + model.step() + complete = True + if not (args.fp16 and optimizer.overflow): + lr_scheduler.step() + else: + skipped_iter = 1 + else: + model.step() + else: + if count == args.gradient_accumulation_steps: + optimizer.step() + complete = True + # Update learning rate. + if not (args.fp16 and optimizer.overflow): + lr_scheduler.step() + else: + skipped_iter = 1 + # print_rank_0("Optimizer step") + timers('optimizer').stop() + if complete: + break + else: + print_rank_0('Found NaN loss, skip backward') + del lm_loss, reduced_loss + mems = [] + if single_step: + break + if args.deepspeed: + lm_loss_total = lm_loss_total / count + return lm_loss_total, skipped_iter, mems diff --git a/runtime/Lib/site_packages/models/nlp/mglm/utils.py b/runtime/Lib/site_packages/models/nlp/mglm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecec588d691d709031a4955f09e81cbd0eedb90 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/mglm/utils.py @@ -0,0 +1,520 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for logging and serialization""" + +import os +import random +import subprocess +import time + +import json +import numpy as np +import torch +from megatron_util import mpu, print_rank_0 +from megatron_util.fp16 import FP16_Optimizer + +SUMMARY_WRITER_DIR_NAME = 'runs' + + +def get_log_dir(name, base): + return os.path.join(base, SUMMARY_WRITER_DIR_NAME, name) + + +def get_hostname(): + hostname_cmd = ['hostname -I'] + result = subprocess.check_output(hostname_cmd, shell=True) + master_addr = result.decode('utf-8').split()[0] + return master_addr + + +def get_spare_port(args): + if torch.distributed.get_rank() == 0: + port = subprocess.check_output(['shuf -n 1 -i 10000-65535'], + shell=True) + port = int(port.strip()) + if port == args.master_port: + port = subprocess.check_output(['shuf -n 1 -i 10000-65535'], + shell=True) + port = int(port.strip()) + port = torch.cuda.LongTensor([port]) + else: + port = torch.cuda.LongTensor([0]) + torch.distributed.broadcast(port, 0) + port = port.item() + return port + + +def print_and_save_args(args, verbose=True, log_dir=None): + """Print arguments.""" + if verbose: + print('arguments:', flush=True) + for arg in vars(args): + dots = '.' * (29 - len(arg)) + print( + ' {} {} {}'.format(arg, dots, getattr(args, arg)), flush=True) + if log_dir is not None: + json_file = os.path.join(log_dir, 'config.json') + with open(json_file, 'w') as output: + json.dump(vars(args), output, sort_keys=True) + if args.deepspeed and args.deepspeed_config is not None: + with open(args.deepspeed_config, encoding='utf-8') as file: + deepspeed_config = json.load(file) + deepspeed_json_file = os.path.join(log_dir, + 'config_gpt_large.json') + with open(deepspeed_json_file, 'w') as output: + json.dump(deepspeed_config, output) + + +def print_params_min_max_norm(optimizer, iteration): + """Print min, max, and norm of all parameters.""" + index = 0 + rank = torch.distributed.get_rank() + string = 'iteration, rank, index, model-parallel,min, max, norm\n' + optimizer_ = optimizer + if isinstance(optimizer, FP16_Optimizer): + optimizer_ = optimizer.optimizer + for param_group in optimizer_.param_groups: + for param in param_group['params']: + index += 1 + min_ = param.data.min() + max_ = param.data.max() + norm = param.data.norm() + string += '{:7d}, {:4d}, {:4d}, {:2d}, '.format( + iteration, rank, index, int(param.model_parallel)) + string += '{:.6E}, {:.6E}, {:.6E}\n'.format(min_, max_, norm) + print(string, flush=True) + + +class Timers: + """Group of timers.""" + + class Timer: + """Timer.""" + + def __init__(self, name): + self.name_ = name + self.elapsed_ = 0.0 + self.started_ = False + self.start_time = time.time() + + def start(self): + """Start the timer.""" + assert not self.started_, 'timer has already been started' + torch.cuda.synchronize() + self.start_time = time.time() + self.started_ = True + + def stop(self): + """Stop the timer.""" + assert self.started_, 'timer is not started' + torch.cuda.synchronize() + self.elapsed_ += (time.time() - self.start_time) + self.started_ = False + + def reset(self): + """Reset timer.""" + self.elapsed_ = 0.0 + self.started_ = False + + def elapsed(self, reset=True): + """Calculate the elapsed time.""" + started_ = self.started_ + # If the timing in progress, end it first. + if self.started_: + self.stop() + # Get the elapsed time. + elapsed_ = self.elapsed_ + # Reset the elapsed time + if reset: + self.reset() + # If timing was in progress, set it back. + if started_: + self.start() + return elapsed_ + + def __init__(self): + self.timers = {} + + def __call__(self, name): + if name not in self.timers: + self.timers[name] = self.Timer(name) + return self.timers[name] + + def log(self, names, normalizer=1.0, reset=True): + """Log a group of timers.""" + assert normalizer > 0.0 + string = 'time (ms)' + for name in names: + elapsed_time = self.timers[name].elapsed( + reset=reset) * 1000.0 / normalizer + string += ' | {}: {:.2f}'.format(name, elapsed_time) + print_rank_0(string) + + +def report_memory(name): + """Simple GPU memory report.""" + + mega_bytes = 1024.0 * 1024.0 + string = name + ' memory (MB)' + string += ' | allocated: {}'.format(torch.cuda.memory_allocated() + / mega_bytes) + string += ' | max allocated: {}'.format(torch.cuda.max_memory_allocated() + / mega_bytes) + string += ' | cached: {}'.format(torch.cuda.memory_cached() / mega_bytes) + string += ' | max cached: {}'.format(torch.cuda.memory_reserved() + / mega_bytes) + print_rank_0(string) + + +def get_checkpoint_name(checkpoints_path, + iteration, + release=False, + zero=False): + if release: + d = 'release' + else: + d = '{}'.format(iteration) + if zero: + dp_rank = mpu.get_data_parallel_rank() + d += '_zero_dp_rank_{}'.format(dp_rank) + return os.path.join( + checkpoints_path, d, + 'mp_rank_{:02d}_model_states.pt'.format(mpu.get_model_parallel_rank())) + + +def ensure_directory_exists(filename): + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname, exist_ok=True) + + +def get_checkpoint_tracker_filename(checkpoints_path): + return os.path.join(checkpoints_path, 'latest_checkpointed_iteration.txt') + + +def save_zero_checkpoint(args, iteration, optimizer): + zero_sd = { + 'iteration': iteration, + 'optimizer_state_dict': optimizer.state_dict() + } + zero_checkpoint_name = get_checkpoint_name(args.save, iteration, zero=True) + ensure_directory_exists(zero_checkpoint_name) + torch.save(zero_sd, zero_checkpoint_name) + print(' successfully saved {}'.format(zero_checkpoint_name)) + + +def save_checkpoint(iteration, + model, + optimizer, + lr_scheduler, + args, + tag=None, + barrier=True, + only_changed_parameters=False, + no_deepspeed=False, + no_save_optim=False): + """Save a model checkpoint.""" + if tag is None: + tag = str(iteration) + if args.deepspeed and not no_deepspeed: + save_ds_checkpoint(iteration, model, lr_scheduler, args, tag=tag) + else: + # Only rank zer0 of the data parallel writes to the disk. + + if mpu.get_data_parallel_rank() == 0: + checkpoint_name = get_checkpoint_name(args.save, tag) + print( + 'global rank {} is saving checkpoint at iteration {:7d} to {}'. + format(torch.distributed.get_rank(), iteration, + checkpoint_name)) + sd = {'iteration': iteration} + if args.deepspeed: + model = model.module + state_dict = model.state_dict() + if only_changed_parameters: + requires_grad_dict = {} + for name, parameter in model.named_parameters(): + requires_grad_dict[name] = parameter.requires_grad + state_dict = { + key: value + for key, value in state_dict.items() + if requires_grad_dict[key] + } + sd['module'] = state_dict + + # Optimizer stuff. + if not args.no_save_optim and not no_save_optim: + if optimizer is not None: + sd['optimizer'] = optimizer.state_dict() + if lr_scheduler is not None: + sd['lr_scheduler'] = lr_scheduler.state_dict() + + # rng states. + if not args.no_save_rng: + sd['random_rng_state'] = random.getstate() + sd['np_rng_state'] = np.random.get_state() + sd['torch_rng_state'] = torch.get_rng_state() + sd['cuda_rng_state'] = torch.cuda.get_rng_state() + sd['rng_tracker_states'] = mpu.get_cuda_rng_tracker( + ).get_states() + + ensure_directory_exists(checkpoint_name) + torch.save(sd, checkpoint_name) + print(' successfully saved {}'.format(checkpoint_name)) + + # Wait so everyone is done (necessary) + if barrier: + torch.distributed.barrier() + # And update the latest iteration + if torch.distributed.get_rank() == 0: + tracker_filename = get_checkpoint_tracker_filename(args.save) + with open(tracker_filename, 'w') as f: + f.write(tag) + + +def save_ds_checkpoint(iteration, model, lr_scheduler, args, tag): + """Save a model checkpoint.""" + + sd = {} + sd['iteration'] = iteration + if lr_scheduler is not None: + sd['client_lr_scheduler'] = lr_scheduler.state_dict() + # rng states. + if not args.no_save_rng: + sd['random_rng_state'] = random.getstate() + sd['np_rng_state'] = np.random.get_state() + sd['torch_rng_state'] = torch.get_rng_state() + sd['cuda_rng_state'] = torch.cuda.get_rng_state() + sd['rng_tracker_states'] = mpu.get_cuda_rng_tracker().get_states() + model.save_checkpoint(args.save, tag, client_state=sd) + + +def get_checkpoint_iteration(load_path): + # Read the tracker file and set the iteration. + tracker_filename = get_checkpoint_tracker_filename(load_path) + if not os.path.isfile(tracker_filename): + print_rank_0('WARNING: could not find the metadata file {} '.format( + tracker_filename)) + if os.path.isdir(load_path): + path = os.path.normpath(load_path) + load_dir, tag = os.path.split(path) + print_rank_0( + 'Try to directly load the checkpoint from the directory') + return load_dir, tag, False, True + print_rank_0(' will not load any checkpoints and will start from ' + 'random') + return load_path, 0, False, False + with open(tracker_filename, 'r', encoding='utf-8') as f: + metastring = f.read().strip() + release = metastring == 'release' + # try: + # iteration = int(metastring) + # except ValueError: + # release = metastring == 'release' + # if not release: + # print_rank_0('ERROR: Invalid metadata file {}. Exiting'.format( + # tracker_filename)) + # exit() + + # assert iteration > 0 or release, 'error parsing metadata file {}'.format( + # tracker_filename) + + return load_path, metastring, release, True + + +def load_checkpoint(model, + optimizer, + lr_scheduler, + args, + no_deepspeed=False, + no_load_optim=False): + """Load a model checkpoint.""" + + load_dir, tag, release, success = get_checkpoint_iteration(args.load) + + if not success: + return 0 + + if args.deepspeed and not no_deepspeed: + + checkpoint_name, sd = model.load_checkpoint( + load_dir, + tag, + load_optimizer_states=not args.no_load_optim and not no_load_optim, + load_lr_scheduler_states=not args.no_load_lr_scheduler) + if not args.no_load_lr_scheduler and 'client_lr_scheduler' in sd: + lr_scheduler.load_state_dict(sd['client_lr_scheduler']) + print_rank_0('Load lr scheduler state') + if checkpoint_name is None: + if mpu.get_data_parallel_rank() == 0: + print('Unable to load checkpoint.') + return tag + + else: + + # Checkpoint. + checkpoint_name = get_checkpoint_name(load_dir, tag, release) + + if mpu.get_data_parallel_rank() == 0: + print('global rank {} is loading checkpoint {}'.format( + torch.distributed.get_rank(), checkpoint_name)) + + # Load the checkpoint. + sd = torch.load(checkpoint_name, map_location='cpu') + + # Model. + if args.deepspeed: + model = model.module + missing_keys, unexpected_keys = model.load_state_dict( + sd['module'], strict=False) + if missing_keys or unexpected_keys: + print_rank_0( + f'Missing keys {missing_keys}, unexpected keys {unexpected_keys}' + ) + + # Optimizer. + if not release and not args.finetune and not args.no_load_optim and not no_load_optim: + try: + if optimizer is not None: + optimizer.load_state_dict(sd['optimizer']) + if lr_scheduler is not None: + lr_scheduler.load_state_dict(sd['lr_scheduler']) + except KeyError: + print_rank_0( + 'Unable to load optimizer from checkpoint {}, exiting. ' + 'Specify --no-load-optim or --finetune to prevent ' + 'attempting to load the optimizer ' + 'state.'.format(checkpoint_name)) + + # Iterations. + if args.finetune or release: + iteration = 0 + else: + try: + iteration = sd['iteration'] + except KeyError: + try: # Backward compatible with older checkpoints + iteration = sd['total_iters'] + except KeyError: + print_rank_0( + 'A metadata file exists but Unable to load iteration ' + ' from checkpoint {}, starting from 0 iteration'.format( + checkpoint_name)) + iteration = 0 + + # rng states. + if not release and not args.finetune and not args.no_load_rng: + try: + random.setstate(sd['random_rng_state']) + np.random.set_state(sd['np_rng_state']) + torch.set_rng_state(sd['torch_rng_state']) + torch.cuda.set_rng_state(sd['cuda_rng_state']) + mpu.get_cuda_rng_tracker().set_states(sd['rng_tracker_states']) + except KeyError: + print_rank_0( + 'Unable to load random state from checkpoint {}, exiting. ' + 'Specify --no-load-rng or --finetune to prevent ' + 'attempting to load the random ' + 'state.'.format(checkpoint_name)) + + if mpu.get_data_parallel_rank() == 0: + print(' successfully loaded {}'.format(checkpoint_name)) + + return iteration + + +def load_weights(src, dst, dst2src=False): + """ + Loads weights from src to dst via in place copy. + src is a huggingface gpt2model, while dst is one of our models. + dst2src=True loads parameters from our models into huggingface's. + ^dst2src is still untested + """ + conv_layer = 'Conv1D' in str(type(src)) + for n, p in src.named_parameters(): + if dst2src: + data = dst._parameters[n].data + load = p.data + else: + data = p.data + load = dst._parameters[n].data + if conv_layer and 'weight' in n: + data = data.t().contiguous() + load.copy_(data) + + +# dst._parameters[n].data.copy_(data) + + +def load_mlp(our, oai, dst2src=False): + load_weights(oai.c_fc, our.dense_h_to_4h, dst2src) + load_weights(oai.c_proj, our.dense_4h_to_h, dst2src) + + +def load_attention(our, oai, dst2src=False): + load_weights(oai.c_attn, our.query_key_value, dst2src) + load_weights(oai.c_proj, our.dense, dst2src) + + +def load_transformer_layer(our, oai, dst2src=False): + load_weights(oai.ln_1, our.input_layernorm, dst2src) + load_weights(oai.ln_2, our.post_attention_layernorm, dst2src) + load_mlp(our.mlp, oai.mlp, dst2src) + load_attention(our.attention, oai.attn, dst2src) + + +def move_weights(our, oai, dst2src=False): + """ + Loads weights from `oai` to `our` via in place copy. + `oai` is a huggingface gpt2model, while `our` is one of our models. + dst2src=True loads parameters from our models into huggingface's. + ^dst2src=True is still untested + """ + # while isinstance(our, (torchDDP, model.distributed.DistributedDataParallel, FP16_Module)): + # our=our.module + transformer_model = oai.transformer + load_weights(transformer_model.ln_f, our.transformer.final_layernorm, + dst2src) + load_weights(transformer_model.wte, our.word_embeddings, dst2src) + load_weights(transformer_model.wpe, our.position_embeddings, dst2src) + + for our_layer, oai_layer in zip(our.transformer.layers, oai.transformer.h): + load_transformer_layer(our_layer, oai_layer, dst2src) + + +def debug_finetune_data(local_vars, batch_id, tokenizer): + tokens, target_ids = local_vars['tokens'], local_vars['target_ids'] + attention_mask, logit_mask, position_ids = local_vars[ + 'attention_mask'], local_vars['logit_mask'], local_vars['position_ids'] + output_tokens = [] + sep = attention_mask[batch_id].item() + for i, token in enumerate(tokens[batch_id][:sep].tolist()): + token = tokenizer.IdToToken(token) + if token == '[MASK]': + token = f'[{position_ids[batch_id][0, i].item()}]' + output_tokens.append(token) + print(' '.join(output_tokens)) + target_positions = [] + for i in range(sep, tokens.size(-1)): + if logit_mask[batch_id][i]: + target_positions.append(i) + print(target_positions) + print(tokenizer.DecodeIds(tokens[batch_id][target_positions].tolist())) + if len(target_ids.shape) > 2: + print( + tokenizer.DecodeIds( + target_ids[batch_id][target_positions].tolist())) + else: + print(tokenizer.DecodeIds(target_ids[batch_id].tolist())) + print(position_ids[batch_id][:, target_positions]) diff --git a/runtime/Lib/site_packages/models/nlp/palm_v2/__init__.py b/runtime/Lib/site_packages/models/nlp/palm_v2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c3fef28a922deecc2df3ed29440d183995e655e3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/palm_v2/__init__.py @@ -0,0 +1,41 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import PalmConfig + from .text_generation import ( + AbsSummarizer, + PalmForTextGeneration, + Translator, + ) +else: + _import_structure = { + 'configuration': ['PalmConfig'], + 'text_generation': + ['AbsSummarizer', 'Translator', 'PalmForTextGeneration'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/palm_v2/configuration.py b/runtime/Lib/site_packages/models/nlp/palm_v2/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..9d033479a56ea7ca060c638d559fa52e0cc06c5a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/palm_v2/configuration.py @@ -0,0 +1,116 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PALM model configuration """ + +from transformers.configuration_utils import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class PalmConfig(PretrainedConfig): + r""" + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `optional`, defaults to 2): + The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layernorm_epsilon (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + dec_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer decoder. + attn_separate (:obj:`bool`, `optional`, defaults to false): + Whether or not to separate the q, k, v of attention. + + Examples: + + >>> from modelscope.models.nlp.palm_v2 import PalmForConditionalGeneration, PalmConfig + >>> configuration = PalmConfig() + + >>> # Initializing a model from the configuration + >>> model = PalmForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + model_type = 'palm' + + def __init__(self, + encoder='roberta', + encoder_pth='roberta-base', + max_pos=512, + share_emb=False, + dec_layers=12, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=3072, + dec_dropout=0.2, + use_bert_emb=True, + label_smoothing=0.1, + alpha=0.95, + beam_size=5, + min_length=40, + max_length=130, + sample_topk=False, + block_trigram=False, + **kwargs): + super().__init__(**kwargs) + self.encoder = encoder + self.encoder_pth = encoder_pth + self.max_pos = max_pos + self.share_emb = share_emb + self.dec_layers = dec_layers + self.dec_hidden_size = dec_hidden_size + self.dec_heads = dec_heads + self.dec_ff_size = dec_ff_size + self.dec_dropout = dec_dropout + self.use_bert_emb = use_bert_emb + self.label_smoothing = label_smoothing + # Translator + self.alpha = alpha + self.beam_size = beam_size + self.min_length = min_length + self.max_length = max_length + self.sample_topk = sample_topk + self.block_trigram = block_trigram diff --git a/runtime/Lib/site_packages/models/nlp/palm_v2/dureader_eval.py b/runtime/Lib/site_packages/models/nlp/palm_v2/dureader_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..db54f21d7a957e6e4a69e7aaa3e3e0ae42606a42 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/palm_v2/dureader_eval.py @@ -0,0 +1,872 @@ +# ============================================================================== +# Copyright 2017 Baidu.com, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +This module computes evaluation metrics for DuReader dataset. +""" + +import argparse +import copy +import math +import re +import sys +import zipfile +from collections import Counter, defaultdict + +import json +import numpy as np +from rouge import Rouge + +EMPTY = '' +YESNO_LABELS = set(['Yes', 'No', 'Depends']) + + +def my_lcs(string, sub): + """ + Calculates longest common subsequence for a pair of tokenized strings + :param string : list of str : tokens from a string split using whitespace + :param sub : list of str : shorter string, also split using whitespace + :returns: length (list of int): length of the longest common subsequence between the two strings + + Note: my_lcs only gives length of the longest common subsequence, not the actual LCS + """ + if (len(string) < len(sub)): + sub, string = string, sub + + lengths = [[0 for i in range(0, + len(sub) + 1)] + for j in range(0, + len(string) + 1)] + + for j in range(1, len(sub) + 1): + for i in range(1, len(string) + 1): + if (string[i - 1] == sub[j - 1]): + lengths[i][j] = lengths[i - 1][j - 1] + 1 + else: + lengths[i][j] = max(lengths[i - 1][j], lengths[i][j - 1]) + + return lengths[len(string)][len(sub)] + + +class Bleu: + + def __init__(self, n=4): + # default compute Blue score up to 4 + self._n = n + self._hypo_for_image = {} + self.ref_for_image = {} + + def compute_score(self, gts, res): + assert (list(gts.keys()) == list(res.keys())) + imgIds = list(gts.keys()) + + bleu_scorer = BleuScorer(n=self._n) + for id in imgIds: + hypo = res[id] + ref = gts[id] + + # Sanity check. + assert (type(hypo) is list) + assert (len(hypo) == 1) + assert (type(ref) is list) + assert (len(ref) >= 1) + + bleu_scorer += (hypo[0], ref) + + score, scores = bleu_scorer.compute_score(option='closest', verbose=1) + return score, scores + + def method(self): + return 'Bleu' + + +def precook(s, n=4, out=False): + """Takes a string as input and returns an object that can be given to + either cook_refs or cook_test. This is optional: cook_refs and cook_test + can take string arguments as well.""" + words = s.split() + counts = defaultdict(int) + for k in range(1, n + 1): + for i in range(len(words) - k + 1): + ngram = tuple(words[i:i + k]) + counts[ngram] += 1 + return (len(words), counts) + + +def cook_refs(refs, eff=None, n=4): # lhuang: oracle will call with "average" + '''Takes a list of reference sentences for a single segment + and returns an object that encapsulates everything that BLEU + needs to know about them.''' + + reflen = [] + maxcounts = {} + for ref in refs: + rl, counts = precook(ref, n) + reflen.append(rl) + for (ngram, count) in counts.items(): + maxcounts[ngram] = max(maxcounts.get(ngram, 0), count) + + # Calculate effective reference sentence length. + if eff == 'shortest': + reflen = min(reflen) + elif eff == 'average': + reflen = float(sum(reflen)) / len(reflen) + + # lhuang: N.B.: leave reflen computaiton to the very end!! + + # lhuang: N.B.: in case of "closest", keep a list of reflens!! (bad design) + + return reflen, maxcounts + + +def cook_test(test, xxx_todo_changeme, eff=None, n=4): + '''Takes a test sentence and returns an object that + encapsulates everything that BLEU needs to know about it.''' + (reflen, refmaxcounts) = xxx_todo_changeme + testlen, counts = precook(test, n, True) + + result = {} + + # Calculate effective reference sentence length. + + if eff == 'closest': + result['reflen'] = min((abs(ref - testlen), ref) for ref in reflen)[1] + else: # i.e., "average" or "shortest" or None + result['reflen'] = reflen + + result['testlen'] = testlen + + result['guess'] = [max(0, testlen - k + 1) for k in range(1, n + 1)] + + result['correct'] = [0] * n + for (ngram, count) in counts.items(): + result['correct'][len(ngram) - 1] += min( + refmaxcounts.get(ngram, 0), count) + + return result + + +class BleuScorer(object): + """Bleu scorer. + """ + + __slots__ = 'n', 'crefs', 'ctest', '_score', '_ratio', '_testlen', '_reflen', 'special_reflen' + + # special_reflen is used in oracle (proportional effective ref len for a node). + + def copy(self): + ''' copy the refs.''' + new = BleuScorer(n=self.n) + new.ctest = copy.copy(self.ctest) + new.crefs = copy.copy(self.crefs) + new._score = None + return new + + def __init__(self, test=None, refs=None, n=4, special_reflen=None): + ''' singular instance ''' + + self.n = n + self.crefs = [] + self.ctest = [] + self.cook_append(test, refs) + self.special_reflen = special_reflen + + def cook_append(self, test, refs): + '''called by constructor and __iadd__ to avoid creating new instances.''' + + if refs is not None: + self.crefs.append(cook_refs(refs)) + if test is not None: + cooked_test = cook_test(test, self.crefs[-1]) + self.ctest.append(cooked_test) # N.B.: -1 + else: + self.ctest.append( + None) # lens of crefs and ctest have to match + + self._score = None # need to recompute + + def ratio(self, option=None): + self.compute_score(option=option) + return self._ratio + + def score_ratio(self, option=None): + '''return (bleu, len_ratio) pair''' + return (self.fscore(option=option), self.ratio(option=option)) + + def score_ratio_str(self, option=None): + return '%.4f (%.2f)' % self.score_ratio(option) + + def reflen(self, option=None): + self.compute_score(option=option) + return self._reflen + + def testlen(self, option=None): + self.compute_score(option=option) + return self._testlen + + def retest(self, new_test): + if type(new_test) is str: + new_test = [new_test] + assert len(new_test) == len(self.crefs), new_test + self.ctest = [] + for t, rs in zip(new_test, self.crefs): + self.ctest.append(cook_test(t, rs)) + self._score = None + + return self + + def rescore(self, new_test): + ''' replace test(s) with new test(s), and returns the new score.''' + + return self.retest(new_test).compute_score() + + def size(self): + assert len(self.crefs) == len( + self.ctest), 'refs/test mismatch! %d<>%d' % (len( + self.crefs), len(self.ctest)) + return len(self.crefs) + + def __iadd__(self, other): + '''add an instance (e.g., from another sentence).''' + + if type(other) is tuple: + # avoid creating new BleuScorer instances + self.cook_append(other[0], other[1]) + else: + assert self.compatible(other), 'incompatible BLEUs.' + self.ctest.extend(other.ctest) + self.crefs.extend(other.crefs) + self._score = None # need to recompute + + return self + + def compatible(self, other): + return isinstance(other, BleuScorer) and self.n == other.n + + def single_reflen(self, option='average'): + return self._single_reflen(self.crefs[0][0], option) + + def _single_reflen(self, reflens, option=None, testlen=None): + + if option == 'shortest': + reflen = min(reflens) + elif option == 'average': + reflen = float(sum(reflens)) / len(reflens) + elif option == 'closest': + reflen = min((abs(ref - testlen), ref) for ref in reflens)[1] + else: + assert False, 'unsupported reflen option %s' % option + + return reflen + + def recompute_score(self, option=None, verbose=0): + self._score = None + return self.compute_score(option, verbose) + + def compute_score(self, option=None, verbose=0): + n = self.n + small = 1e-9 + tiny = 1e-15 # so that if guess is 0 still return 0 + bleu_list = [[] for _ in range(n)] + + if self._score is not None: + return self._score + + if option is None: + option = 'average' if len(self.crefs) == 1 else 'closest' + + self._testlen = 0 + self._reflen = 0 + totalcomps = { + 'testlen': 0, + 'reflen': 0, + 'guess': [0] * n, + 'correct': [0] * n + } + + # for each sentence + for comps in self.ctest: + testlen = comps['testlen'] + self._testlen += testlen + + if self.special_reflen is None: # need computation + reflen = self._single_reflen(comps['reflen'], option, testlen) + else: + reflen = self.special_reflen + + self._reflen += reflen + + for key in ['guess', 'correct']: + for k in range(n): + totalcomps[key][k] += comps[key][k] + + # append per image bleu score + bleu = 1. + for k in range(n): + bleu *= (float(comps['correct'][k]) + tiny) / ( + float(comps['guess'][k]) + small) + bleu_list[k].append(bleu**(1. / (k + 1))) + ratio = (testlen + tiny) / (reflen + small + ) # N.B.: avoid zero division + if ratio < 1: + for k in range(n): + bleu_list[k][-1] *= math.exp(1 - 1 / ratio) + + if verbose > 1: + print(comps, reflen) + + totalcomps['reflen'] = self._reflen + totalcomps['testlen'] = self._testlen + + bleus = [] + bleu = 1. + for k in range(n): + bleu *= float(totalcomps['correct'][k] + tiny) / ( + totalcomps['guess'][k] + small) + bleus.append(bleu**(1. / (k + 1))) + ratio = (self._testlen + tiny) / (self._reflen + small + ) # N.B.: avoid zero division + if ratio < 1: + for k in range(n): + bleus[k] *= math.exp(1 - 1 / ratio) + + if verbose > 0: + print(totalcomps) + print('ratio:', ratio) + + self._score = bleus + return self._score, bleu_list + + +def normalize(s): + """ + Normalize strings to space joined chars. + + Args: + s: a list of strings. + + Returns: + A list of normalized strings. + """ + if not s: + return s + normalized = [] + for ss in s: + tokens = [c for c in list(ss) if len(c.strip()) != 0] + normalized.append(' '.join(tokens)) + return normalized + + +def data_check(obj, task): + """ + Check data. + + Raises: + Raises AssertionError when data is not legal. + """ + assert 'question_id' in obj, "Missing 'question_id' field." + assert 'question_type' in obj, \ + "Missing 'question_type' field. question_id: {}".format(obj['question_type']) + + assert 'yesno_answers' in obj, \ + "Missing 'yesno_answers' field. question_id: {}".format(obj['question_id']) + assert isinstance(obj['yesno_answers'], list), \ + r"""'yesno_answers' field must be a list, if the 'question_type' is not + 'YES_NO', then this field should be an empty list. + question_id: {}""".format(obj['question_id']) + + assert 'entity_answers' in obj, \ + "Missing 'entity_answers' field. question_id: {}".format(obj['question_id']) + assert isinstance( + obj['entity_answers'], + list) and len(obj['entity_answers']) > 0, r"""'entity_answers' field + must be a list, and has at least one element, which can be a empty list. + question_id: {}""".format(obj['question_id']) + + +def read_file(file_name, task, is_ref=False): + """ + Read predict answers or reference answers from file. + + Args: + file_name: the name of the file containing predict result or reference + result. + + Returns: + A dictionary mapping question_id to the result information. The result + information itself is also a dictionary with has four keys: + - question_type: type of the query. + - yesno_answers: A list of yesno answers corresponding to 'answers'. + - answers: A list of predicted answers. + - entity_answers: A list, each element is also a list containing the entities + tagged out from the corresponding answer string. + """ + + def _open(file_name, mode, zip_obj=None): + if zip_obj is not None: + return zip_obj.open(file_name, mode) + return open(file_name, mode) + + results = {} + keys = ['answers', 'yesno_answers', 'entity_answers', 'question_type'] + if is_ref: + keys += ['source'] + + zf = zipfile.ZipFile(file_name, + 'r') if file_name.endswith('.zip') else None + file_list = [file_name] if zf is None else zf.namelist() + + for fn in file_list: + for line in _open(fn, 'r', zip_obj=zf): + try: + obj = json.loads(line.strip()) + except ValueError: + raise ValueError('Every line of data should be legal json') + data_check(obj, task) + qid = obj['question_id'] + assert qid not in results, 'Duplicate question_id: {}'.format(qid) + results[qid] = {} + for k in keys: + results[qid][k] = obj[k] + return results + + +def compute_bleu_rouge(pred_dict, ref_dict, bleu_order=4): + """ + Compute bleu and rouge scores. + """ + assert set(pred_dict.keys()) == set(ref_dict.keys()), \ + 'missing keys: {}'.format(set(ref_dict.keys()) - set(pred_dict.keys())) + scores = {} + bleu_scores, _ = Bleu(bleu_order).compute_score(ref_dict, pred_dict) + for i, bleu_score in enumerate(bleu_scores): + scores['Bleu-%d' % (i + 1)] = bleu_score + # rouge_score, _ = Rouge().compute_score(ref_dict, pred_dict) + rouge_score = Rouge().get_scores( + list(map(lambda x: x[0], pred_dict.values())), + list(map(lambda x: x[0], ref_dict.values()))) + rouge_score = sum([d['rouge-l']['f'] + for d in rouge_score]) / len(rouge_score) + scores['Rouge-L'] = rouge_score + return scores + + +def local_prf(pred_list, ref_list): + """ + Compute local precision recall and f1-score, + given only one prediction list and one reference list + """ + common = Counter(pred_list) & Counter(ref_list) + num_same = sum(common.values()) + if num_same == 0: + return 0, 0, 0 + p = 1.0 * num_same / len(pred_list) + r = 1.0 * num_same / len(ref_list) + f1 = (2 * p * r) / (p + r) + return p, r, f1 + + +def compute_prf(pred_dict, ref_dict): + """ + Compute precision recall and f1-score. + """ + # pred_question_ids = set(pred_dict.keys()) + ref_question_ids = set(ref_dict.keys()) + correct_preds, total_correct, total_preds = 0, 0, 0 + for question_id in ref_question_ids: + pred_entity_list = pred_dict.get(question_id, [[]]) + assert len(pred_entity_list) == 1, \ + 'the number of entity list for question_id {} is not 1.'.format(question_id) + pred_entity_list = pred_entity_list[0] + all_ref_entity_lists = ref_dict[question_id] + best_local_f1 = 0 + best_ref_entity_list = None + for ref_entity_list in all_ref_entity_lists: + local_f1 = local_prf(pred_entity_list, ref_entity_list)[2] + if local_f1 > best_local_f1: + best_ref_entity_list = ref_entity_list + best_local_f1 = local_f1 + if best_ref_entity_list is None: + if len(all_ref_entity_lists) > 0: + best_ref_entity_list = sorted( + all_ref_entity_lists, key=lambda x: len(x))[0] + else: + best_ref_entity_list = [] + gold_entities = set(best_ref_entity_list) + pred_entities = set(pred_entity_list) + correct_preds += len(gold_entities & pred_entities) + total_preds += len(pred_entities) + total_correct += len(gold_entities) + p = float(correct_preds) / total_preds if correct_preds > 0 else 0 + r = float(correct_preds) / total_correct if correct_preds > 0 else 0 + f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0 + return {'Precision': p, 'Recall': r, 'F1': f1} + + +def prepare_prf(pred_dict, ref_dict): + """ + Prepares data for calculation of prf scores. + """ + preds = {k: v['entity_answers'] for k, v in pred_dict.items()} + refs = {k: v['entity_answers'] for k, v in ref_dict.items()} + return preds, refs + + +def filter_dict(result_dict, key_tag): + """ + Filter a subset of the result_dict, where keys ends with 'key_tag'. + """ + filtered = {} + for k, v in result_dict.items(): + if k.endswith(key_tag): + filtered[k] = v + return filtered + + +def get_metrics(pred_result, ref_result, task, source): + """ + Computes metrics. + """ + metrics = {} + + ref_result_filtered = {} + pred_result_filtered = {} + if source == 'both': + ref_result_filtered = ref_result + pred_result_filtered = pred_result + else: + for question_id, info in ref_result.items(): + if info['source'] == source: + ref_result_filtered[question_id] = info + if question_id in pred_result: + pred_result_filtered[question_id] = pred_result[ + question_id] + + if task == 'main' or task == 'all' \ + or task == 'description': + pred_dict, ref_dict = prepare_bleu(pred_result_filtered, + ref_result_filtered, task) + metrics = compute_bleu_rouge(pred_dict, ref_dict) + elif task == 'yesno': + pred_dict, ref_dict = prepare_bleu(pred_result_filtered, + ref_result_filtered, task) + keys = ['Yes', 'No', 'Depends'] + preds = [filter_dict(pred_dict, k) for k in keys] + refs = [filter_dict(ref_dict, k) for k in keys] + + metrics = compute_bleu_rouge(pred_dict, ref_dict) + + for k, pred, ref in zip(keys, preds, refs): + m = compute_bleu_rouge(pred, ref) + k_metric = [(k + '|' + key, v) for key, v in m.items()] + metrics.update(k_metric) + + elif task == 'entity': + pred_dict, ref_dict = prepare_prf(pred_result_filtered, + ref_result_filtered) + pred_dict_bleu, ref_dict_bleu = prepare_bleu(pred_result_filtered, + ref_result_filtered, task) + metrics = compute_prf(pred_dict, ref_dict) + metrics.update(compute_bleu_rouge(pred_dict_bleu, ref_dict_bleu)) + else: + raise ValueError('Illegal task name: {}'.format(task)) + + return metrics + + +def prepare_bleu(pred_result, ref_result, task): + """ + Prepares data for calculation of bleu and rouge scores. + """ + pred_list, ref_list = [], [] + qids = ref_result.keys() + for qid in qids: + if task == 'main': + pred, ref = get_main_result(qid, pred_result, ref_result) + elif task == 'yesno': + pred, ref = get_yesno_result(qid, pred_result, ref_result) + elif task == 'all': + pred, ref = get_all_result(qid, pred_result, ref_result) + elif task == 'entity': + pred, ref = get_entity_result(qid, pred_result, ref_result) + elif task == 'description': + pred, ref = get_desc_result(qid, pred_result, ref_result) + else: + raise ValueError('Illegal task name: {}'.format(task)) + if pred and ref: + pred_list += pred + ref_list += ref + pred_dict = dict(pred_list) + ref_dict = dict(ref_list) + for qid, ans in ref_dict.items(): + ref_dict[qid] = normalize(ref_dict[qid]) + pred_dict[qid] = normalize(pred_dict.get(qid, [EMPTY])) + if not ans or ans == [EMPTY]: + del ref_dict[qid] + del pred_dict[qid] + + for k, v in pred_dict.items(): + assert len(v) == 1, \ + 'There should be only one predict answer. question_id: {}'.format(k) + return pred_dict, ref_dict + + +def get_main_result(qid, pred_result, ref_result): + """ + Prepare answers for task 'main'. + + Args: + qid: question_id. + pred_result: A dict include all question_id's result information read + from args.pred_file. + ref_result: A dict incluce all question_id's result information read + from args.ref_file. + Returns: + Two lists, the first one contains predict result, the second + one contains reference result of the same question_id. Each list has + elements of tuple (question_id, answers), 'answers' is a list of strings. + """ + ref_ans = ref_result[qid]['answers'] + if not ref_ans: + ref_ans = [EMPTY] + pred_ans = pred_result.get(qid, {}).get('answers', [])[:1] + if not pred_ans: + pred_ans = [EMPTY] + + return [(qid, pred_ans)], [(qid, ref_ans)] + + +def get_entity_result(qid, pred_result, ref_result): + """ + Prepare answers for task 'entity'. + + Args: + qid: question_id. + pred_result: A dict include all question_id's result information read + from args.pred_file. + ref_result: A dict incluce all question_id's result information read + from args.ref_file. + Returns: + Two lists, the first one contains predict result, the second + one contains reference result of the same question_id. Each list has + elements of tuple (question_id, answers), 'answers' is a list of strings. + """ + if ref_result[qid]['question_type'] != 'ENTITY': + return None, None + return get_main_result(qid, pred_result, ref_result) + + +def get_desc_result(qid, pred_result, ref_result): + """ + Prepare answers for task 'description'. + + Args: + qid: question_id. + pred_result: A dict include all question_id's result information read + from args.pred_file. + ref_result: A dict incluce all question_id's result information read + from args.ref_file. + Returns: + Two lists, the first one contains predict result, the second + one contains reference result of the same question_id. Each list has + elements of tuple (question_id, answers), 'answers' is a list of strings. + """ + if ref_result[qid]['question_type'] != 'DESCRIPTION': + return None, None + return get_main_result(qid, pred_result, ref_result) + + +def get_yesno_result(qid, pred_result, ref_result): + """ + Prepare answers for task 'yesno'. + + Args: + qid: question_id. + pred_result: A dict include all question_id's result information read + from args.pred_file. + ref_result: A dict incluce all question_id's result information read + from args.ref_file. + Returns: + Two lists, the first one contains predict result, the second + one contains reference result of the same question_id. Each list has + elements of tuple (question_id, answers), 'answers' is a list of strings. + """ + + def _uniq(li, is_ref): + uniq_li = [] + left = [] + keys = set() + for k, v in li: + if k not in keys: + uniq_li.append((k, v)) + keys.add(k) + else: + left.append((k, v)) + + if is_ref: + dict_li = dict(uniq_li) + for k, v in left: + dict_li[k] += v + uniq_li = [(k, v) for k, v in dict_li.items()] + return uniq_li + + def _expand_result(uniq_li): + expanded = uniq_li[:] + keys = set([x[0] for x in uniq_li]) + for k in YESNO_LABELS - keys: + expanded.append((k, [EMPTY])) + return expanded + + def _get_yesno_ans(qid, result_dict, is_ref=False): + if qid not in result_dict: + return [(str(qid) + '_' + k, v) for k, v in _expand_result([])] + yesno_answers = result_dict[qid]['yesno_answers'] + answers = result_dict[qid]['answers'] + lbl_ans = _uniq([(k, [v]) for k, v in zip(yesno_answers, answers)], + is_ref) + ret = [(str(qid) + '_' + k, v) for k, v in _expand_result(lbl_ans)] + return ret + + if ref_result[qid]['question_type'] != 'YES_NO': + return None, None + + ref_ans = _get_yesno_ans(qid, ref_result, is_ref=True) + pred_ans = _get_yesno_ans(qid, pred_result) + return pred_ans, ref_ans + + +def get_all_result(qid, pred_result, ref_result): + """ + Prepare answers for task 'all'. + + Args: + qid: question_id. + pred_result: A dict include all question_id's result information read + from args.pred_file. + ref_result: A dict incluce all question_id's result information read + from args.ref_file. + Returns: + Two lists, the first one contains predict result, the second + one contains reference result of the same question_id. Each list has + elements of tuple (question_id, answers), 'answers' is a list of strings. + """ + if ref_result[qid]['question_type'] == 'YES_NO': + return get_yesno_result(qid, pred_result, ref_result) + return get_main_result(qid, pred_result, ref_result) + + +def format_metrics(metrics, task, err_msg): + """ + Format metrics. 'err' field returns any error occured during evaluation. + + Args: + metrics: A dict object contains metrics for different tasks. + task: Task name. + err_msg: Exception raised during evaluation. + Returns: + Formatted result. + """ + result = {} + sources = ['both', 'search', 'zhidao'] + if err_msg is not None: + return {'errorMsg': str(err_msg), 'errorCode': 1, 'data': []} + data = [] + if task != 'all' and task != 'main': + sources = ['both'] + + if task == 'entity': + metric_names = ['Bleu-4', 'Rouge-L'] + metric_names_prf = ['F1', 'Precision', 'Recall'] + for name in metric_names + metric_names_prf: + for src in sources: + obj = { + 'name': name, + 'value': round(metrics[src].get(name, 0) * 100, 2), + 'type': src, + } + data.append(obj) + elif task == 'yesno': + metric_names = ['Bleu-4', 'Rouge-L'] + details = ['Yes', 'No', 'Depends'] + src = sources[0] + for name in metric_names: + obj = { + 'name': name, + 'value': round(metrics[src].get(name, 0) * 100, 2), + 'type': 'All', + } + data.append(obj) + for d in details: + obj = { + 'name': name, + 'value': round(metrics[src].get(d + '|' + name, 0) * 100, + 2), + 'type': d + } + data.append(obj) + else: + metric_names = ['Bleu-4', 'Rouge-L'] + for name in metric_names: + for src in sources: + obj = { + 'name': name, + 'value': round(metrics[src].get(name, 0) * 100, 2), + 'type': src + } + data.append(obj) + + result['data'] = data + result['errorCode'] = 0 + result['errorMsg'] = 'success' + + return result + + +def main(args): + """ + Do evaluation. + """ + err = None + metrics = {} + try: + pred_result = read_file(args.pred_file, args.task) + ref_result = read_file(args.ref_file, args.task, is_ref=True) + sources = ['both', 'search', 'zhidao'] + if args.task not in set(['main', 'all']): + sources = sources[:1] + for source in sources: + metrics[source] = get_metrics(pred_result, ref_result, args.task, + source) + except ValueError as ve: + err = ve + except AssertionError as ae: + err = ae + + print( + json.dumps( + format_metrics(metrics, args.task, err), + ensure_ascii=False).encode('utf8')) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('pred_file', help='predict file') + parser.add_argument('ref_file', help='reference file') + parser.add_argument( + 'task', help='task name: Main|Yes_No|All|Entity|Description') + + args = parser.parse_args() + args.task = args.task.lower().replace('_', '') + main(args) diff --git a/runtime/Lib/site_packages/models/nlp/palm_v2/text_generation.py b/runtime/Lib/site_packages/models/nlp/palm_v2/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..cd3ecdaf216e33c5a750bf0fd47d379f3b02631d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/palm_v2/text_generation.py @@ -0,0 +1,1372 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import copy +import math +import os +import subprocess +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import json +import numpy as np +import torch +import torch.nn.functional as F +from torch import Tensor, nn +from torch.nn.init import xavier_uniform_ +from transformers import (BertConfig, BertModel, BertTokenizer, RobertaConfig, + RobertaModel, RobertaTokenizer) +from transformers.activations import ACT2FN +from transformers.modeling_utils import PreTrainedModel + +from modelscope.metainfo import Models +from modelscope.models import Model +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import TextGenerationModelOutput, TokenGeneratorOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .configuration import PalmConfig +from .dureader_eval import compute_bleu_rouge, normalize + +CONFIG_NAME = 'config.json' +WEIGHTS_NAME = 'pytorch_model.bin' + + +class MultiHeadedAttention(nn.Module): # SelfAttention + """ + Multi-Head Attention module from + "Attention is All You Need" + :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`. + + Similar to standard `dot` attention but uses + multiple attention distributions simulataneously + to select relevant items. + + .. mermaid:: + + graph BT + A[key] + B[value] + C[query] + O[output] + subgraph Attn + D[Attn 1] + E[Attn 2] + F[Attn N] + end + A --> D + C --> D + A --> E + C --> E + A --> F + C --> F + D --> O + E --> O + F --> O + B --> O + + Also includes several additional tricks. + + Args: + head_count (int): number of parallel heads + model_dim (int): the dimension of keys/values/queries, + must be divisible by head_count + dropout (float): dropout parameter + """ + + def __init__(self, + head_count, + model_dim, + dropout=0.1, + use_final_linear=True): + assert model_dim % head_count == 0 + self.dim_per_head = model_dim // head_count + self.model_dim = model_dim + + super().__init__() + self.head_count = head_count + + self.linear_keys = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_values = nn.Linear(model_dim, + head_count * self.dim_per_head) + self.linear_query = nn.Linear(model_dim, + head_count * self.dim_per_head) + self.softmax = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + self.use_final_linear = use_final_linear + if (self.use_final_linear): + self.final_linear = nn.Linear(model_dim, model_dim) + + def forward(self, + key, + value, + query, + mask=None, + layer_cache=None, + type=None, + predefined_graph_1=None, + return_attn=False): + """ + Compute the context vector and the attention vectors. + + Args: + key (`FloatTensor`): set of `key_len` + key vectors `[batch, key_len, dim]` + value (`FloatTensor`): set of `key_len` + value vectors `[batch, key_len, dim]` + query (`FloatTensor`): set of `query_len` + query vectors `[batch, query_len, dim]` + mask: binary mask indicating which keys have + non-zero attention `[batch, query_len, key_len]` + Returns: + (`FloatTensor`, `FloatTensor`) : + + * output context vectors `[batch, query_len, dim]` + * one of the attention vectors `[batch, query_len, key_len]` + """ + + batch_size = key.size(0) + dim_per_head = self.dim_per_head + head_count = self.head_count + + def shape(x): + """ projection """ + return x.view(batch_size, -1, head_count, dim_per_head) \ + .transpose(1, 2) + + def unshape(x): + """ compute context """ + return x.transpose(1, 2).contiguous() \ + .view(batch_size, -1, head_count * dim_per_head) + + # 1) Project key, value, and query. + if layer_cache is not None: + if type == 'self': + query, key, value = self.linear_query(query), self.linear_keys( + query), self.linear_values(query) + + key = shape(key) + value = shape(value) + + device = key.device + if layer_cache['self_keys'] is not None: + key = torch.cat((layer_cache['self_keys'].to(device), key), + dim=2) + if layer_cache['self_values'] is not None: + value = torch.cat( + (layer_cache['self_values'].to(device), value), dim=2) + layer_cache['self_keys'] = key + layer_cache['self_values'] = value + elif type == 'context': + query = self.linear_query(query) + if layer_cache['memory_keys'] is None: + key, value = self.linear_keys(key), self.linear_values( + value) + key = shape(key) + value = shape(value) + else: + key, value = layer_cache['memory_keys'], layer_cache[ + 'memory_values'] + layer_cache['memory_keys'] = key + layer_cache['memory_values'] = value + else: + key = self.linear_keys(key) + value = self.linear_values(value) + query = self.linear_query(query) + key = shape(key) + value = shape(value) + + query = shape(query) + + # 2) Calculate and scale scores. + query = query / math.sqrt(dim_per_head) + scores = torch.matmul(query, key.transpose(2, 3)) + + if mask is not None: + mask = mask.unsqueeze(1).expand_as(scores) + scores = scores.masked_fill(mask, -1e18) + + # 3) Apply attention dropout and compute context vectors. + + attn = self.softmax(scores) + + if predefined_graph_1 is not None: + attn_masked = attn[:, -1] * predefined_graph_1 + attn_masked = attn_masked / ( + torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9) + + attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1) + + drop_attn = self.dropout(attn) + if self.use_final_linear: + context = unshape(torch.matmul(drop_attn, value)) + output = self.final_linear(context) + if return_attn: + return output, attn + else: + return output + else: + context = torch.matmul(drop_attn, value) + if return_attn: + return context, attn + else: + return context + + +class PositionwiseFeedForward(nn.Module): # Output + """ A two-layer Feed-Forward-Network with residual layer norm. + + Args: + d_model (int): the size of input for the first-layer of the FFN. + d_ff (int): the hidden layer size of the second-layer + of the FNN. + dropout (float): dropout probability in :math:`[0, 1)`. + """ + + def __init__(self, d_model, d_ff, dropout=0.1): + super().__init__() + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.w_1 = nn.Linear(d_model, d_ff) + self.actv = ACT2FN['gelu_new'] + self.dropout_1 = nn.Dropout(dropout) + self.w_2 = nn.Linear(d_ff, d_model) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x): + inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x)))) + output = self.dropout_2(self.w_2(inter)) + return output + x + + +class TransformerDecoderLayer(nn.Module): # Layer + """ + Args: + d_model (int): the dimension of keys/values/queries in + MultiHeadedAttention, also the input size of + the first-layer of the PositionwiseFeedForward. + heads (int): the number of heads for MultiHeadedAttention. + d_ff (int): the second-layer of the PositionwiseFeedForward. + dropout (float): dropout probability(0-1.0). + self_attn_type (string): type of self-attention scaled-dot, average + """ + MAX_SIZE = 5000 + + def __init__(self, d_model, heads, d_ff, dropout): + super().__init__() + + self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + + self.context_attn = MultiHeadedAttention( + heads, d_model, dropout=dropout) + self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout) + self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6) + self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6) + self.drop = nn.Dropout(dropout) + mask = self._get_attn_subsequent_mask(self.MAX_SIZE) + # Register self.mask as a buffer in TransformerDecoderLayer, so + # it gets TransformerDecoderLayer's cuda behavior automatically. + self.register_buffer('mask', mask) + + def forward(self, + inputs, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=None, + layer_cache=None, + step=None): + """ + Args: + inputs (`FloatTensor`): `[batch_size x 1 x model_dim]` + memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]` + src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]` + tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]` + + Returns: + (`FloatTensor`, `FloatTensor`, `FloatTensor`): + + * output `[batch_size x 1 x model_dim]` + * attn `[batch_size x 1 x src_len]` + * all_input `[batch_size x current_step x model_dim]` + + """ + dec_mask = torch.gt( + tgt_pad_mask.type(torch.uint8) + + self.mask[:, :tgt_pad_mask.size(1), :tgt_pad_mask.size(1)].type( + torch.uint8), 0) + input_norm = self.layer_norm_1(inputs) + all_input = input_norm + if previous_input is not None: + all_input = torch.cat((previous_input, input_norm), dim=1) + dec_mask = None + + query = self.self_attn( + all_input, + all_input, + input_norm, + mask=dec_mask, + layer_cache=layer_cache, + type='self') + + query = self.drop(query) + inputs + + query_norm = self.layer_norm_2(query) + mid, attn = self.context_attn( + memory_bank, + memory_bank, + query_norm, + mask=src_pad_mask, + layer_cache=layer_cache, + type='context', + return_attn=True) + output = self.feed_forward(self.drop(mid) + query) + + return output, attn, all_input + + def _get_attn_subsequent_mask(self, size): + """ + Get an attention mask to avoid using the subsequent info. + + Args: + size: int + + Returns: + (`LongTensor`): + + * subsequent_mask `[1 x size x size]` + """ + attn_shape = (1, size, size) + subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8') + subsequent_mask = torch.from_numpy(subsequent_mask) + return subsequent_mask + + +class PositionalEncoding(nn.Module): + + def __init__(self, dropout, dim, max_len=5000): + super().__init__() + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) + * -(math.log(10000.0) / dim))) + pe[:, 0::2] = torch.sin(position.float() * div_term) + pe[:, 1::2] = torch.cos(position.float() * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + self.dropout = nn.Dropout(dropout) + self.dim = dim + + def forward(self, emb, step=None): + emb = emb * math.sqrt(self.dim) + if (step): + emb = emb + self.pe[:, step][:, None, :] + + else: + emb = emb + self.pe[:, :emb.size(1)] + emb = self.dropout(emb) + return emb + + def get_emb(self, emb): + return self.pe[:, :emb.size(1)] + + +class TransformerDecoderState: + + def __init__(self, src: Tensor, cache_num_layers: int = -1): + self.src: Tensor = src + self.previous_input: Tensor = None + self.previous_layer_inputs: Tensor = None + self.cache: Optional[Dict[str, Any]] = None + if cache_num_layers != -1: + self._init_cache(cache_num_layers) + + def update_state(self, new_input, previous_layer_inputs): + self.previous_input = new_input + self.previous_layer_inputs = previous_layer_inputs + self.cache = None + + def _init_cache(self, num_layers): + self.cache = {} + for num in range(num_layers): + layer_cache = {'memory_keys': None, 'memory_values': None} + layer_cache['self_keys'] = None + layer_cache['self_values'] = None + self.cache['layer_{}'.format(num)] = layer_cache + + def map_batch_fn(self, fn): + + def _recursive_map(struct, batch_dim=0): + for k, v in struct.items(): + if v is not None: + if isinstance(v, dict): + _recursive_map(v) + else: + struct[k] = fn(v, batch_dim) + + self.src = fn(self.src, 0) + if self.cache is not None: + _recursive_map(self.cache) + + +class TransformerDecoder(nn.Module): # Decoder + """ + The Transformer decoder from "Attention is All You Need". + + + .. mermaid:: + + graph BT + A[input] + B[multi-head self-attn] + BB[multi-head src-attn] + C[feed forward] + O[output] + A --> B + B --> BB + BB --> C + C --> O + + + Args: + num_layers (int): number of encoder layers. + d_model (int): size of the model + heads (int): number of heads + d_ff (int): size of the inner FF layer + dropout (float): dropout parameters + embeddings (:obj:`onmt.modules.Embeddings`): + embeddings to use, should have positional encodings + attn_type (str): if using a seperate copy attention + """ + decoder_type = 'transformer' + + def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings): + super().__init__() + + # Basic attributes. + self.num_layers = num_layers + self.embeddings = embeddings + self.pos_emb = PositionalEncoding(dropout, + self.embeddings.embedding_dim) + + # Build TransformerDecoder. + self.transformer_layers = nn.ModuleList([ + TransformerDecoderLayer(d_model, heads, d_ff, dropout) + for _ in range(num_layers) + ]) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.state = None + + def forward(self, + state: TransformerDecoderState, + tgt: Tensor, + memory_bank: Tensor, + step: int = None, + memory_masks: Tensor = None): + src_words = state.src + tgt_words = tgt + src_batch, src_len = src_words.size() + tgt_batch, tgt_len = tgt_words.size() + + # Run the forward pass of the TransformerDecoder. + # emb = self.embeddings(tgt, step=step) + emb = self.embeddings(tgt) + assert emb.dim() == 3 # len x batch x embedding_dim + output = self.pos_emb(emb, step) + + src_memory_bank = memory_bank + padding_idx = self.embeddings.padding_idx + tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \ + .expand(tgt_batch, tgt_len, tgt_len) + + if memory_masks is not None: + src_len = memory_masks.size(-1) + src_pad_mask = memory_masks.expand(src_batch, tgt_len, src_len) + else: + src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \ + .expand(src_batch, tgt_len, src_len) + + if state.cache is None: + saved_inputs = [] + attns = [] + for i in range(self.num_layers): + prev_layer_input = None + if state.cache is None: + if state.previous_input is not None: + prev_layer_input = state.previous_layer_inputs[i] + output, attn, all_input \ + = self.transformer_layers[i]( + output, src_memory_bank, + src_pad_mask, tgt_pad_mask, + previous_input=prev_layer_input, + layer_cache=state.cache['layer_{}'.format(i)] + if state.cache is not None else None, + step=step) + if state.cache is None: + saved_inputs.append(all_input) + attns.append(attn) + + if state.cache is None: + saved_inputs = torch.stack(saved_inputs) + + output = self.layer_norm(output) + + # Process the result and update the attentions. + if state.cache is None: + state.update_state(tgt, saved_inputs) + + return output, attns, state + + +class PalmPointerGenerator(nn.Module): + + def __init__(self, hidden_size, vocab_size): + super().__init__() + self.dense = nn.Linear(hidden_size, vocab_size) + self.gen_func = nn.LogSoftmax(-1) + + def forward(self, x): + x = self.dense(x) + x = self.gen_func(x) + return x + + +class PalmPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = PalmConfig + base_model_prefix = 'palm' + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + @classmethod + def _from_pretrained( + cls, pretrained_model_name_or_path: Optional[Union[str, + os.PathLike]], + **kwargs): + config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) + config = PalmConfig.from_json_file(config_file) if os.path.isfile( + config_file) else PalmConfig() + config.encoder_pth = os.path.join(pretrained_model_name_or_path, + config.encoder_pth) + checkpoint_file = os.path.join(pretrained_model_name_or_path, + WEIGHTS_NAME) + checkpoint = torch.load(checkpoint_file) if os.path.isfile( + checkpoint_file) else None + return cls(config, checkpoint, **kwargs) + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels not supplied. + If num_labels is not found, the model will use the default setting (2 classes). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir') + model = cls._from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model + + +class AbsSummarizer(PalmPreTrainedModel): # Model + + def __init__(self, config, checkpoint=None, **kwargs): + super().__init__(config, **kwargs) + self.config = config + if config.encoder == 'bert' or config.encoder == 'zh_bert': + self.bert = BertModel( + BertConfig.from_pretrained(config.encoder_pth)) + elif config.encoder == 'roberta': + self.bert = RobertaModel( + RobertaConfig.from_pretrained(config.encoder_pth)) + + if config.max_pos > 512: + my_pos_embeddings = nn.Embedding( + config.max_pos, self.bert.model.config.hidden_size) + my_pos_embeddings.weight.data[: + 512] = self.bert.embeddings.position_embeddings.weight.data + my_pos_embeddings.weight.data[ + 512:] = self.bert.embeddings.position_embeddings.weight.data[ + -1][None, :].repeat(config.max_pos - 512, 1) + self.bert.model.embeddings.position_embeddings = my_pos_embeddings + self.vocab_size = self.bert.config.vocab_size + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=1 if config.encoder == 'roberta' else 0) + + if config.share_emb: + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight) + self.decoder = TransformerDecoder( + config.dec_layers, + config.dec_hidden_size, + heads=config.dec_heads, + d_ff=config.dec_ff_size, + dropout=config.dec_dropout, + embeddings=tgt_embeddings) + self.generator = PalmPointerGenerator(config.dec_hidden_size, + self.vocab_size) + self.generator.dense.weight = self.decoder.embeddings.weight + + if checkpoint is not None: + checkpoint = self._unwrap_checkpoint(checkpoint) + self.load_state_dict(checkpoint, strict=False) + else: + for module in self.decoder.modules(): + if isinstance(module, (nn.Linear, nn.Embedding)): + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + for p in self.generator.parameters(): + if p.dim() > 1: + xavier_uniform_(p) + else: + p.data.zero_() + if config.use_bert_emb: + if config.encoder == 'roberta': + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=1) + else: + tgt_embeddings = nn.Embedding( + self.vocab_size, + self.bert.config.hidden_size, + padding_idx=0) + tgt_embeddings.weight = copy.deepcopy( + self.bert.embeddings.word_embeddings.weight) + self.decoder.embeddings = tgt_embeddings + self.generator.dense.weight = self.decoder.embeddings.weight + + @staticmethod + def _unwrap_checkpoint(checkpoint: Dict): + wrap_names = ('model', 'palm') + for name in wrap_names: + if name in checkpoint: + checkpoint = checkpoint[name] + for name in wrap_names: + checkpoint = {(k[len(name) + 1:] if k.startswith(name) else k): v + for k, v in checkpoint.items()} + return checkpoint + + def forward(self, src, tgt, mask_src): + top_vec, _ = self.bert(src, mask_src, return_dict=False) + state = TransformerDecoderState(src) + decoder_outputs, attns, _ = self.decoder(state, tgt[:, :-1], top_vec) + return decoder_outputs, attns[-1], top_vec + + +class LabelSmoothingLoss(nn.Module): + """ + With label smoothing, + KL-divergence between q_{smoothed ground truth prob.}(w) + and p_{prob. computed by model}(w) is minimized. + """ + + def __init__(self, label_smoothing, tgt_vocab_size, ignore_index=-100): + assert 0.0 < label_smoothing <= 1.0 + self.padding_idx = ignore_index + super(LabelSmoothingLoss, self).__init__() + + smoothing_value = label_smoothing / (tgt_vocab_size - 2) + one_hot = torch.full((tgt_vocab_size, ), smoothing_value) + one_hot[self.padding_idx] = 0 + self.register_buffer('one_hot', one_hot.unsqueeze(0)) + self.confidence = 1.0 - label_smoothing + + def forward(self, output, target): + """ + output (FloatTensor): batch_size x n_classes + target (LongTensor): batch_size + """ + model_prob = self.one_hot.repeat(target.size(0), 1) + model_prob.scatter_(1, target.unsqueeze(1), self.confidence) + model_prob.masked_fill_((target == self.padding_idx).unsqueeze(1), 0) + + return F.kl_div(output, model_prob, reduction='sum') + + +class NMTLossCompute(nn.Module): + """ + Standard NMT Loss Computation. + """ + + def __init__(self, generator, symbols, vocab_size, label_smoothing=0.0): + super().__init__() + self.generator = generator + self.padding_idx = symbols['PAD'] + if label_smoothing > 0: + self.criterion = LabelSmoothingLoss( + label_smoothing, vocab_size, ignore_index=self.padding_idx) + else: + self.criterion = nn.NLLLoss( + ignore_index=self.padding_idx, reduction='sum') + + def _bottle(self, _v): + return _v.view(-1, _v.size(2)) + + def _unbottle(self, _v, batch_size): + return _v.view(-1, batch_size, _v.size(1)) + + def forward(self, tgt, output): + target = tgt[:, 1:] + normalization = target.ne(self.padding_idx).sum() + bottled_output = self._bottle(output) + scores = self.generator(bottled_output) + gtruth = target.contiguous().view(-1) + loss = self.criterion(scores, gtruth) + loss.div(float(normalization)) + return loss + + +class Translator(object): + """ + Uses a model to translate a batch of sentences. + """ + + @dataclass + class Batch: + batch_size: int + src: torch.Tensor + tgt: torch.Tensor + mask_src: torch.Tensor + query_id: List[None] = None + src_str: List[List[str]] = None + tgt_str: List[str] = None + + def __init__(self, model, dataset: str = 'cnn'): + super().__init__() + self.logger = logging.get_logger() + self.args = model.config + self.args.dataset = dataset + self.model = model.palm + self.generator = self.model.generator + self.vocab = model.tokenizer + self.symbols = model.symbols + self.start_token = self.symbols['BOS'] + self.end_token = self.symbols['EOS'] + self.alpha = self.args.alpha + self.beam_size = self.args.beam_size + + def from_batch(self, translation_batch): + batch = translation_batch['batch'] + assert (len(translation_batch['gold_score']) == len( + translation_batch['predictions'])) + batch_size = batch.batch_size + + preds, pred_score, tgt_str, src, src_str = translation_batch[ + 'predictions'], translation_batch[ + 'scores'], batch.tgt_str, batch.src, batch.src_str + query_id = batch.query_id + ''' + try: + query_id = batch.query_id + except: + query_id = None + ''' + translations = [] + for b in range(batch_size): + if self.args.dataset == 'qg_ranking_test': + if self.args.encoder == 'bert' or self.args.encoder == 'zh_bert': + pred_sents = [ + ' '.join( + self.vocab.convert_ids_to_tokens( + [int(n) for n in each])).replace(' ##', '') + for each in preds[b] + ] + elif self.args.encoder == 'roberta': + pred_sents = [ + self.vocab.decode([int(n) for n in each + ]).replace('', + '').replace('', '') + for each in preds[b] + ] + elif self.args.encoder == 'roberta': + pred_sents = self.vocab.decode([int(n) + for n in preds[b][0]]).replace( + '', + '').replace('', '') + elif self.args.encoder == 'bert': + pred_sents = self.vocab.convert_ids_to_tokens( + [int(n) for n in preds[b][0]]) + pred_sents = ' '.join(pred_sents).replace(' ##', '') + elif self.args.encoder == 'zh_bert' and self.args.dataset == 'paraphrase': + pred_sents = [ + self.vocab.convert_ids_to_tokens([int(n) for n in pred]) + for pred in preds[b] + ] + pred_sents = [ + ''.join(pred).replace(' ##', '') for pred in pred_sents + ] + elif self.args.encoder == 'zh_bert': + pred_sents = self.vocab.convert_ids_to_tokens( + [int(n) for n in preds[b][0]]) + pred_sents = ''.join(pred_sents).replace('##', '') + gold_sent = tgt_str[b] + + if self.args.encoder == 'roberta': + raw_src = self.vocab.decode([int(t) for t in src[b]]) + raw_src = ' '.join(src_str[b]) + else: + raw_src = [self.vocab.ids_to_tokens[int(t)] + for t in src[b]][:500] + raw_src = ' '.join(raw_src) + if self.args.dataset == 'faq': + translation = (pred_sents, gold_sent, src_str[b], query_id[b], + pred_score[b]) + else: + translation = (pred_sents, gold_sent, raw_src, query_id[b], + pred_score[b]) + # translation = (pred_sents[0], gold_sent) + translations.append(translation) + + return translations + + def translate(self, data_iter, step): + gold_path = self.args.result_path + '.%d.gold' % step + can_path = self.args.result_path + '.%d.candidate' % step + self.gold_out_file = codecs.open(gold_path, 'w', 'utf-8') + self.can_out_file = codecs.open(can_path, 'w', 'utf-8') + self.pred_json_score_out_file = codecs.open(can_path + '.sample', 'w', + 'utf-8') + if self.args.dataset == 'paraphrase' and self.args.encoder == 'roberta': + out = '\t'.join([ + 'query_id', 'source_query', 'target_query', 'predict_query' + ]) + '\n' + self.pred_json_score_out_file.write(out) + + raw_src_path = self.args.result_path + '.%d.raw_src' % step + self.src_out_file = codecs.open(raw_src_path, 'w', 'utf-8') + + pred_results, gold_results = [], [] + cnt = 0 + pred_dict, ref_dict = {}, {} + for i, batch in enumerate(data_iter): + self.logger.info(f'data: {i + 1} / {len(data_iter)}') + batch_data = self.translate_batch(batch) + translations = self.from_batch(batch_data) + + for trans in translations: + pred, gold, src, query_id, pred_score = trans + src = src.replace('', '').replace('##', '').strip() + if self.args.dataset == 'qg_ranking_test': + pred_str = '\t'.join([ + each.replace('[unused0]', '').replace( + '[PAD]', '').replace('[unused1]', '').replace( + r' +', ' ').replace('[SEP]', '').replace( + '[unused2]', + '').replace(r' +', ' ').replace( + '', + '').replace('', '').replace( + '', + '').replace('', '').replace( + '', ' ').strip() + for each in pred + ]) + else: + pred_str = pred.replace('[unused0]', '').replace( + '[PAD]', '').replace('[unused1]', '').replace( + r' +', ' ').replace('[SEP]', '').replace( + '[unused2]', '').replace('[CLS]', '').replace( + '[SEP]', '').replace('[UNK]', '').strip() + pred_str = pred_str.replace(r' +', ' ').replace( + '', + '').replace('', '').replace('', '').replace( + '', '').replace('', ' ').strip() + gold_str = gold.replace('', '').strip().replace( + '[UNK]', '').replace('[unused1]', '').replace( + '[unused2]', + '').replace('##', '').replace('[CLS]', '').replace( + '[SEP]', '').strip().replace('', '').replace( + '', '').replace('', ' ').strip() + if self.args.recall_eval: + _pred_str = '' + for sent in pred_str.split(''): + can_pred_str = _pred_str + '' + sent.strip() + if len(can_pred_str.split()) >= len( + gold_str.split()) + 10: + pred_str = _pred_str + break + else: + _pred_str = can_pred_str + + if self.args.dataset == 'marco' or self.args.dataset == 'squad' or self.args.dataset == 'qg_ranking': + pred_str = pred_str.replace('', ' ') + if query_id is not None: + pred_json = { + 'query_id': query_id, + 'answers': [pred_str] + } + gold_json = { + 'query_id': query_id, + 'answers': [gold_str] + } + pred_json_score = { + 'query_id': query_id, + 'answers': [pred_str], + 'scores': pred_score[0].cpu().numpy().tolist() + } + else: + pred_json = {'query_id': cnt, 'answers': [pred_str]} + gold_json = {'query_id': cnt, 'answers': [gold_str]} + pred_json_score = { + 'query_id': cnt, + 'answers': [pred_str], + 'scores': pred_score[0].cpu().numpy().tolist() + } + json.dump(pred_json, self.can_out_file) + self.can_out_file.write('\n') + json.dump(gold_json, self.gold_out_file) + self.gold_out_file.write('\n') + json.dump(pred_json_score, self.pred_json_score_out_file) + self.pred_json_score_out_file.write('\n') + self.src_out_file.write(src.strip() + '\n') + elif self.args.dataset == 'cnn': + self.can_out_file.write(pred_str + '\n') + self.gold_out_file.write(gold_str + '\n') + self.src_out_file.write(src.strip() + '\n') + elif self.args.dataset == 'dureader': + if query_id is None: + query_id = str(cnt) + pred_results.extend(normalize([pred_str])) + gold_results.extend(normalize([gold_str])) + self.can_out_file.write(pred_str + '\n') + self.gold_out_file.write('\t'.join([src[0], gold_str]) + + '\n') + + elif self.args.dataset == 'paraphrase': + if query_id is None: + query_id = str(cnt) + if self.args.encoder == 'roberta': + pred_str = [pred_str] + pred_dict[query_id] = normalize([pred_str[0]]) + ref_dict[query_id] = normalize([gold_str]) + self.pred_json_score_out_file.write( + '\t'.join([str(query_id), src, gold_str, pred_str[0]]) + + '\n') + elif self.args.dataset == 'faq': + if pred_score[0].cpu().numpy().tolist() < -3.5: + continue + self.can_out_file.write( + '\t'.join([str(query_id), src, pred_str]) + '\n') + self.gold_out_file.write( + '\t'.join([str(query_id), src, gold_str]) + '\n') + # passage, answer, question, score + self.pred_json_score_out_file.write('\t'.join([ + str(query_id), gold_str, src, pred_str, + str(pred_score[0].cpu().numpy().tolist()) + ]) + '\n') + elif self.args.dataset == 'qg_ranking_test': + self.can_out_file.write( + str(query_id) + '\t' + pred_str + '\n') + + cnt += 1 + self.can_out_file.flush() + self.gold_out_file.flush() + self.src_out_file.flush() + self.logger.info('cnt: %s' % cnt) + self.can_out_file.close() + self.gold_out_file.close() + self.src_out_file.close() + + if step != -1: + if self.args.dataset == 'marco' or self.args.dataset == 'squad' or self.args.dataset == 'qg_ranking': + cnn_results = subprocess.getoutput( + './run.sh %s %s' % (gold_path, can_path)) # run.sh ... + self.logger.info(cnn_results) + elif self.args.dataset == 'cnn': + self.logger.info('Calculating Rouge') + from rouge import Rouge + candidates = [ + line.strip() for line in open(can_path, encoding='utf-8') + ] + references = [ + line.strip() for line in open(gold_path, encoding='utf-8') + ] + rouge_score = Rouge().get_scores( + candidates, references, avg=True) + # self.logger.info('Rouges at step %d \n%s' % (step, rouge_results_to_str(rouges))) + print(rouge_score) + elif self.args.dataset == 'dureader' or self.args.dataset == 'paraphrase': + + def postprocess_text(preds, labels): + preds = [pred.strip().replace('.', '') for pred in preds] + labels = [label.strip() for label in labels] + while '' in preds: + idx = preds.index('') + preds[idx] = '。' + return preds, labels + + pred_results, gold_results = postprocess_text( + pred_results, gold_results) + pred_dict = {str(i): tmp for i, tmp in enumerate(pred_results)} + gold_dict = {str(i): tmp for i, tmp in enumerate(gold_results)} + bleu_rouge = compute_bleu_rouge(pred_dict, gold_dict) + print(bleu_rouge) + # unreachable + elif self.args.dataset == 'dureader' or self.args.dataset == 'paraphrase': + pred_results, gold_results = postprocess_text( + pred_results, gold_results) + bleu_score = cal_bleu(pred_results, gold_results) + from rouge import Rouge + rouge = Rouge() + rouge_score = rouge.get_scores( + pred_results, gold_results, avg=True) + print("'Dev eval result: Bleu-4={}, {}".format( + bleu_score, rouge_score)) + + def translate_batch(self, batch: 'Batch', fast: bool = False): + """ + Translate a batch of sentences. + + Mostly a wrapper around :obj:`Beam`. + + Args: + batch (:obj:`Batch`): a batch from a dataset object + data (:obj:`Dataset`): the dataset object + fast (bool): enables fast beam search (may not support all features) + + Todo: + Shouldn't need the original dataset. + """ + self.model.eval() + with torch.no_grad(): + return self._fast_translate_batch(batch) + + def _tile(self, x, count, dim=0): + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = x.view(batch, -1) \ + .transpose(0, 1) \ + .repeat(count, 1) \ + .transpose(0, 1) \ + .contiguous() \ + .view(*out_size) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + def _top_k_top_p_filtering(self, + logits, + top_k=10, + top_p=1.0, + filter_value=-float('Inf'), + min_tokens_to_keep=1): + if top_k > 0: + top_k = min(max(top_k, min_tokens_to_keep), + logits.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = filter_value + return logits + + def _fast_translate_batch(self, batch: 'Batch'): + # TODO: faster code path for beam_size == 1. + # TODO: support these blacklisted features. + + max_length = self.args.max_length + min_length = self.args.min_length + + beam_size = self.beam_size + batch_size = batch.batch_size + src = batch.src + mask_src = batch.mask_src + + src_features, _ = self.model.bert(src, mask_src, return_dict=False) + state = TransformerDecoderState(src, self.model.decoder.num_layers) + device = src_features.device + + # Tile states and memory beam_size times. + state.map_batch_fn( + lambda state, dim: self._tile(state, beam_size, dim=dim)) + src_features = self._tile(src_features, beam_size, dim=0) + batch_offset = torch.arange( + batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, + batch_size * beam_size, + step=beam_size, + dtype=torch.long, + device=device) + alive_seq = torch.full([batch_size * beam_size, 1], + self.start_token, + dtype=torch.long, + device=device) + + # Give full probability to the first beam on the first step. + topk_log_probs = ( + torch.tensor( + [0.0] + [float('-inf')] * (beam_size - 1), + device=device).repeat(batch_size)) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results['predictions'] = [[] for _ in range(batch_size)] # noqa: F812 + results['scores'] = [[] for _ in range(batch_size)] # noqa: F812 + results['gold_score'] = [0] * batch_size + results['batch'] = batch + + for step in range(max_length): + decoder_input = alive_seq[:, -1].view(1, -1) + + # Decoder forward. + decoder_input = decoder_input.transpose(0, 1) + dec_out, attns, state = self.model.decoder( + state, decoder_input, src_features, step=step) + + # Generator forward. + log_probs = self.generator.forward( + dec_out.transpose(0, 1).squeeze(0)) + vocab_size = log_probs.size(-1) + + if step < min_length: + log_probs[:, self.end_token] = -1e20 + + # Multiply probs by the beam probability. + + length_penalty = ((5.0 + (step + 1)) / 6.0)**self.alpha + if self.args.sample_topk: + temperature = self.args.temperature + _scores = log_probs / temperature + _scores = self._top_k_top_p_filtering( + _scores, + top_k=self.args.top_k, + top_p=self.args.top_p, + min_tokens_to_keep=1 + ) # (batch_size * num_beams, vocab_size) + # Sample 2 next words for each beam (so we have some spare tokens + # and match output of greedy beam search) + topk_ids = torch.multinomial( + F.softmax(_scores, dim=-1), + num_samples=1) # (batch_size * num_beams, 2) + # Compute next scores + _scores = F.log_softmax( + _scores, dim=1) # (batch_size * num_beams, vocab_size) + + _scores += topk_log_probs.view(-1).unsqueeze(1) + _scores = _scores / length_penalty + topk_scores = torch.gather( + _scores, -1, topk_ids) # (batch_size * num_beams, 2) + # Match shape of greedy beam search + topk_ids = topk_ids.view( + -1, beam_size) # (batch_size, 2 * num_beams) + topk_scores = topk_scores.view( + -1, beam_size) # (batch_size, 2 * num_beams) + else: + log_probs += topk_log_probs.view(-1).unsqueeze(1) + curr_scores = log_probs / length_penalty + + curr_scores = curr_scores.reshape(-1, beam_size * vocab_size) + topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1) + if self.args.block_trigram: + cur_len = alive_seq.size(1) + if cur_len > 3: + for i in range(alive_seq.size(0)): + fail = False + words = [int(w) for w in alive_seq[i]] + if self.args.encoder == 'roberta': + words = self.vocab.decode(words).strip().split() + else: + words = [ + self.vocab.ids_to_tokens[w] for w in words + ] + words = ' '.join(words).replace(' ##', '').split() + if len(words) <= 3: + continue + trigrams = [(words[i - 1], words[i], words[i + 1]) + for i in range(1, + len(words) - 1)] + trigram = tuple(trigrams[-1]) + if trigram in trigrams[:-1]: + fail = True + if fail: + curr_scores[i] = -10e20 + # Recover log probs. + topk_log_probs = topk_scores * length_penalty + + # Resolve beam origin and true word ids. + topk_beam_index = topk_ids // vocab_size + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = ( + topk_beam_index + + beam_offset[:topk_beam_index.size(0)].unsqueeze(1)) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat([ + alive_seq.index_select(0, select_indices), + topk_ids.view(-1, 1) + ], -1) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(self.end_token) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(self.end_token) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append( + (topk_scores[i, j], predictions[i, j, 1:])) + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted( + hypotheses[b], key=lambda x: x[0], reverse=True) + if self.args.dataset == 'qg_ranking_test' or ( + self.args.dataset == 'paraphrase' + and not self.args.sample_topk): + for each in best_hyp[:beam_size]: + score, pred = each + results['scores'][b].append(score) + results['predictions'][b].append(pred) + else: + score, pred = best_hyp[0] + results['scores'][b].append(score) + results['predictions'][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished) \ + .view(-1, alive_seq.size(-1)) + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + state.map_batch_fn( + lambda state, dim: state.index_select(dim, select_indices)) + + return results + + def __call__(self, input_ids: torch.Tensor, attention_mask: torch.Tensor, + **kwargs) -> Dict[str, torch.Tensor]: + batch = self.Batch( + batch_size=input_ids.size()[0], + src=input_ids, + tgt=None, + mask_src=attention_mask) + translation_batch = self.translate_batch(batch) + + preds = translation_batch['predictions'] + return {'predictions': preds} + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.palm) +class PalmForTextGeneration(PalmPreTrainedModel): + + def __init__(self, config, checkpoint=None, **kwargs): + super().__init__(config, **kwargs) + self.config = config + if config.encoder == 'roberta': + tokenizer = RobertaTokenizer.from_pretrained( + config.encoder_pth, do_lower_case=False) + symbols = { + 'BOS': tokenizer.cls_token_id, + 'EOS': tokenizer.sep_token_id, + 'PAD': tokenizer.pad_token_id, + 'EOQ': tokenizer.unk_token_id + } + elif config.encoder == 'bert' or config.encoder == 'zh_bert': + tokenizer = BertTokenizer.from_pretrained( + config.encoder_pth, do_lower_case=True) + symbols = { + 'BOS': tokenizer.vocab['[CLS]'], + 'EOS': tokenizer.vocab['[SEP]'], + 'PAD': tokenizer.vocab['[PAD]'], + 'EOQ': tokenizer.vocab['[unused2]'] + } + self.tokenizer = tokenizer + self.symbols = symbols + self.palm = AbsSummarizer(config, checkpoint) + self.loss = NMTLossCompute(self.palm.generator, symbols, + self.palm.vocab_size, + config.label_smoothing) + self.generator = Translator(self) + + def forward(self, input_ids, attention_mask, labels): + output = self.palm(src=input_ids, tgt=labels, mask_src=attention_mask) + loss = self.loss(labels, output[0]) + return TextGenerationModelOutput( + loss=loss, + logits=output[0], + ) + + def generate(self, input: Dict[str, Tensor], + **kwargs) -> TokenGeneratorOutput: + for k, v in kwargs.items(): + setattr(self.generator.args, k, v) + outputs = self.generator(**input) + preds = outputs['predictions'] + return TokenGeneratorOutput(sequences=[pred[0] for pred in preds]) diff --git a/runtime/Lib/site_packages/models/nlp/peer/__init__.py b/runtime/Lib/site_packages/models/nlp/peer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d51a6172434db483dba4042a3ee8a579ff4cf97 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/peer/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import PeerConfig + from .text_classification import PeerForSequenceClassification +else: + _import_structure = { + 'configuration': ['PeerConfig'], + 'text_classification': ['PeerForSequenceClassification'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/peer/backbone.py b/runtime/Lib/site_packages/models/nlp/peer/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..4bf376cddd8f22b8904e9790aad738c911abd973 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/peer/backbone.py @@ -0,0 +1,1256 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2019 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch PEER model. """ + +import math +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from transformers.activations import ACT2FN, get_activation +from transformers.file_utils import ModelOutput, add_start_docstrings +from transformers.modeling_outputs import \ + BaseModelOutputWithPastAndCrossAttentions +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.models import Model, TorchModel +from modelscope.utils import logger as logging +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import PeerConfig +from .sas_utils import SequenceSideInfo + +logger = logging.get_logger() + +PEER_PRETRAINED_MODEL_ARCHIVE_LIST = [ + 'google/peer-small-generator', + 'google/peer-base-generator', + 'google/peer-large-generator', + 'google/peer-small-discriminator', + 'google/peer-base-discriminator', + 'google/peer-large-discriminator', + # See all PEER models at https://huggingface.co/models?filter=peer +] + + +class PeerEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.embedding_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.embedding_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.embedding_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + ['absolute']) + if 'absolute_token_position_in_sentence' in self.position_embedding_type: + self.side_info_size = 16 + self.position_embeddings__token_position_in_sentence = nn.Embedding( + self.side_info_size, config.embedding_size) + + # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.forward + def forward( + self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + side_info_sets=dict(), + ): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if 'absolute' in self.position_embedding_type: + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + + if 'absolute_token_position_in_sentence' in self.position_embedding_type: + position_idx = torch.clamp( + side_info_sets['ss_token_position_in_sentence'], + min=0, + max=self.side_info_size - 1) + position_embeddings__token_position_in_sentence = self.position_embeddings__token_position_in_sentence( + position_idx) + embeddings += position_embeddings__token_position_in_sentence + + # Pass to attention layers to calcualte position-2-position attention scores + if 'absolute_self_only' in self.position_embedding_type: + if 'embeddings' not in side_info_sets: + side_info_sets['embeddings'] = dict() + side_info_sets['embeddings'][ + 'ss_token_position_in_sequence'] = self.position_embeddings( + position_ids) + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class PeerSelfAttention(nn.Module): + + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + ['absolute']) + + if 'relative_scalar_bias' in self.position_embedding_type: + self.max_relative_position_embeddings = config.max_position_embeddings // 4 + self.distance_embedding = nn.Embedding( + 2 * self.max_relative_position_embeddings, + self.num_attention_heads) + + elif 'relative_scalar_bias_with_side_info_token' in self.position_embedding_type: + self.max_relative_position_embeddings = config.max_position_embeddings // 4 + self.side_info_size = 16 # leverage the information of token_position_in_sentence + self.distance_embedding = nn.Embedding( + (2 * self.max_relative_position_embeddings) + * self.side_info_size, self.num_attention_heads) + + elif 'relative_scalar_bias_token_plus_sentence' in self.position_embedding_type: + self.max_relative_position_embeddings = config.max_position_embeddings // 4 + self.max_sen_relative_position_embeddings = self.max_relative_position_embeddings // 4 + + self.distance_embedding = nn.Embedding( + 2 * self.max_relative_position_embeddings, + self.num_attention_heads) + self.distance_embedding_sentence = nn.Embedding( + 2 * self.max_sen_relative_position_embeddings, + self.num_attention_heads) + + elif 'relative_scalar_bias_with_side_info_sentence' in self.position_embedding_type: + self.max_relative_position_embeddings = config.max_position_embeddings // 4 + self.max_sen_relative_position_embeddings = self.max_relative_position_embeddings // 4 + + vocab = (2 * self.max_relative_position_embeddings) * ( + 2 * self.max_sen_relative_position_embeddings) + self.distance_embedding = nn.Embedding(vocab, + self.num_attention_heads) + + elif 'relative_key' in self.position_embedding_type or 'relative_key_query' in self.position_embedding_type: + self.max_relative_position_embeddings = config.max_position_embeddings // 4 + self.distance_embedding = nn.Embedding( + 2 * self.max_relative_position_embeddings, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + side_info_sets=dict(), + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose( + -1, -2)) / math.sqrt(self.attention_head_size) + attention_scores_terms = 1 + + if 'absolute_self_only' in self.position_embedding_type: + attention_scores += side_info_sets[ + 'side_info_attention_scores'] # already normalized by sqrt(attention_head_size) + attention_scores_terms += 1 + + if 'relative_key' in self.position_embedding_type or 'relative_key_query' in self.position_embedding_type \ + or 'relative_scalar_bias' in self.position_embedding_type \ + or 'relative_scalar_bias_with_side_info_token' in self.position_embedding_type \ + or 'relative_scalar_bias_token_plus_sentence' in self.position_embedding_type \ + or 'relative_scalar_bias_with_side_info_sentence' in self.position_embedding_type: + + distance_idx = side_info_sets['distance_idx'] + + positional_embedding = self.distance_embedding(distance_idx) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if 'relative_scalar_bias' in self.position_embedding_type: + relative_scalar_bias = positional_embedding.permute( + [2, 0, 1]).unsqueeze(0) + attention_scores = attention_scores / math.sqrt( + attention_scores_terms) + relative_scalar_bias + + elif ('relative_scalar_bias_with_side_info_token' + in self.position_embedding_type + or 'relative_scalar_bias_with_side_info_sentence' + in self.position_embedding_type): + relative_scalar_bias = positional_embedding.permute( + [0, 3, 1, 2]) + attention_scores = attention_scores / math.sqrt( + attention_scores_terms) + relative_scalar_bias + + elif 'relative_scalar_bias_token_plus_sentence' in self.position_embedding_type: + relative_scalar_bias = positional_embedding.permute( + [2, 0, 1]).unsqueeze(0) + + distance_idx_sentence = side_info_sets['distance_idx_sentence'] + positional_embedding_sentence = self.distance_embedding_sentence( + distance_idx_sentence) + positional_embedding_sentence = positional_embedding_sentence.to( + dtype=query_layer.dtype) # fp16 compatibility + relative_scalar_bias_sentence = positional_embedding_sentence.permute( + [0, 3, 1, 2]) + + attention_scores = attention_scores / math.sqrt( + attention_scores_terms + ) + relative_scalar_bias + relative_scalar_bias_sentence + + elif 'relative_key' in self.position_embedding_type: + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, + positional_embedding) / math.sqrt(self.attention_head_size) + attention_scores_terms += 1 + attention_scores = (attention_scores + relative_position_scores + ) / math.sqrt(attention_scores_terms) + elif 'relative_key_query' in self.position_embedding_type: + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + relative_position_scores = ( + relative_position_scores_query + + relative_position_scores_key) / math.sqrt( + self.attention_head_size) + attention_scores_terms += 2 + attention_scores = (attention_scores + relative_position_scores + ) / math.sqrt(attention_scores_terms) + + else: + attention_scores = attention_scores / math.sqrt( + attention_scores_terms) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in PeerModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +class PeerSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class PeerAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.self = PeerSelfAttention(config) + self.output = PeerSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + side_info_sets=dict(), + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + side_info_sets, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class PeerIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class PeerOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class PeerLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = PeerAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + assert self.is_decoder, f'{self} should be used as a decoder model if cross attention is added' + self.crossattention = PeerAttention(config) + self.intermediate = PeerIntermediate(config) + self.output = PeerOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + side_info_sets=dict(), + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + side_info_sets=side_info_sets, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + assert hasattr( + self, 'crossattention' + ), f'If `encoder_hidden_states` are passed, {self} has to be instantiated \ + with cross-attention layers by setting `config.add_cross_attention=True`' + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class PeerEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [PeerLayer(config) for _ in range(config.num_hidden_layers)]) + + self.position_embedding_type = getattr(config, + 'position_embedding_type', + ['absolute']) + if 'absolute_self_only' in self.position_embedding_type: + # To be used/shared in all self-attention layers. Copy their dimensions here to be consistent. + self.self_attention = self.layer[0].attention.self + + self.num_attention_heads = self.self_attention.num_attention_heads + self.attention_head_size = self.self_attention.attention_head_size + self.all_head_size = self.self_attention.all_head_size + + self.pos_query = nn.Linear(self.self_attention.query.in_features, + self.self_attention.query.out_features) + self.pos_key = nn.Linear(self.self_attention.key.in_features, + self.self_attention.key.out_features) + + def get_position_attention_score(self, hidden_states): + query_layer = self.self_attention.transpose_for_scores( + self.pos_query(hidden_states)) + key_layer = self.self_attention.transpose_for_scores( + self.pos_key(hidden_states)) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + return attention_scores + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + side_info_sets=dict(), + return_dict=True, + ): + + if 'absolute_self_only' in self.position_embedding_type: + side_info_attention_scores = self.get_position_attention_score( + hidden_states=side_info_sets['embeddings'] + ['ss_token_position_in_sequence']) + side_info_sets[ + 'side_info_attention_scores'] = side_info_attention_scores + + if 'relative_key' in self.position_embedding_type or 'relative_key_query' in self.position_embedding_type \ + or 'relative_scalar_bias' in self.position_embedding_type \ + or 'relative_scalar_bias_with_side_info_token' in self.position_embedding_type \ + or 'relative_scalar_bias_token_plus_sentence' in self.position_embedding_type \ + or 'relative_scalar_bias_with_side_info_sentence' in self.position_embedding_type: + seq_length = hidden_states.shape[1] + batch_size = hidden_states.shape[0] + + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + max_relative_position_embeddings = self.layer[ + 0].attention.self.max_relative_position_embeddings + distance_idx = torch.clamp( + position_ids_l - position_ids_r + + max_relative_position_embeddings - 2, + min=0, + max=2 * max_relative_position_embeddings - 4) + distance_idx[ + 0, :] = 2 * max_relative_position_embeddings - 3 # CLS-to-others + distance_idx[:, + 0] = 2 * max_relative_position_embeddings - 2 # others-to-CLS + distance_idx[ + 0, 0] = 2 * max_relative_position_embeddings - 1 # CLS-to-CLS + distance_idx_max = 2 * max_relative_position_embeddings + + # token position-aware relative position + if 'relative_scalar_bias_with_side_info_token' in self.position_embedding_type: + idx1 = torch.clamp( + side_info_sets['ss_token_position_in_sentence'], + min=0, + max=self.layer[0].attention.self.side_info_size + - 1).unsqueeze(2).repeat(1, 1, seq_length) + idx2 = distance_idx.unsqueeze(0).repeat(batch_size, 1, 1) + distance_idx = idx1 * distance_idx_max + idx2 + # relative token position + relative sentence position + elif 'relative_scalar_bias_with_side_info_sentence' in self.position_embedding_type: + sen_position_ids_l = side_info_sets[ + 'ss_sentence_position_in_sequence'].view( + batch_size, -1, 1) + sen_position_ids_r = side_info_sets[ + 'ss_sentence_position_in_sequence'].view( + batch_size, 1, -1) + max_sen_relative_position_embeddings = self.layer[ + 0].attention.self.max_sen_relative_position_embeddings + idx1 = torch.clamp( + sen_position_ids_l - sen_position_ids_r + + max_sen_relative_position_embeddings, + min=0, + max=2 * max_sen_relative_position_embeddings - 1) + idx2 = distance_idx.unsqueeze(0).repeat(batch_size, 1, 1) + distance_idx = idx1 * distance_idx_max + idx2 + elif 'relative_scalar_bias_token_plus_sentence' in self.position_embedding_type: + sen_position_ids_l = side_info_sets[ + 'ss_sentence_position_in_sequence'].view( + batch_size, -1, 1) + sen_position_ids_r = side_info_sets[ + 'ss_sentence_position_in_sequence'].view( + batch_size, 1, -1) + max_sen_relative_position_embeddings = self.layer[ + 0].attention.self.max_sen_relative_position_embeddings + idx1 = torch.clamp( + sen_position_ids_l - sen_position_ids_r + + max_sen_relative_position_embeddings, + min=0, + max=2 * max_sen_relative_position_embeddings - 1) + side_info_sets['distance_idx_sentence'] = idx1 + + side_info_sets['distance_idx'] = distance_idx + + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + if getattr(self.config, 'gradient_checkpointing', False): + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + side_info_sets, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + side_info_sets, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class PeerDiscriminatorPredictions(nn.Module): + """Prediction module for the discriminator, made up of two dense layers.""" + + def __init__(self, config): + super().__init__() + + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.dense_prediction = nn.Linear(config.hidden_size, 1) + self.config = config + + def forward(self, discriminator_hidden_states): + hidden_states = self.dense(discriminator_hidden_states) + hidden_states = get_activation(self.config.hidden_act)(hidden_states) + logits = self.dense_prediction(hidden_states).squeeze(-1) + + return logits + + +class PeerGeneratorPredictions(nn.Module): + """Prediction module for the generator, made up of two dense layers.""" + + def __init__(self, config): + super().__init__() + + self.LayerNorm = nn.LayerNorm(config.embedding_size) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + + def forward(self, generator_hidden_states): + hidden_states = self.dense(generator_hidden_states) + hidden_states = get_activation('gelu')(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + + return hidden_states + + +class PeerPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = PeerConfig + base_model_prefix = 'teams1_shared_bottom' + _keys_to_ignore_on_load_missing = [r'position_ids'] + _keys_to_ignore_on_load_unexpected = [ + r'peer\.embeddings_project\.weight', r'peer\.embeddings_project\.bias' + ] + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + + if model_dir is None: + config = PeerConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + return model + + +@dataclass +class PeerForRTDOutput(ModelOutput): + """ + Output type of :class:`~transformers.PeerForRTD`. + + Args: + loss (`optional`, returned when ``labels`` is provided, ``torch.FloatTensor`` of shape :obj:`(1,)`): + Total loss of the PEER objective. + logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): + Prediction scores of the head (scores for each token before SoftMax). + hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, + returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): + Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) + of shape :obj:`(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, + returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): + Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, + sequence_length, sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +@dataclass +class PeerForPreTrainingOutput(ModelOutput): + """ + Output type of :class:`~transformers.PeerForPreTraining`. + + Args: + loss (`optional`, returned when ``labels`` is provided, ``torch.FloatTensor`` of shape :obj:`(1,)`): + Total loss of the PEER objective. + logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): + Prediction scores of the head (scores for each token before SoftMax). + hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, + returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): + Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) + of shape :obj:`(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, + returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): + Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, + sequence_length, sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[torch.FloatTensor] = None + mlm_loss: Optional[torch.FloatTensor] = None + rtd_loss: Optional[torch.FloatTensor] = None + mlm_logits: torch.FloatTensor = None + rtd_logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +PEER_START_DOCSTRING = r""" + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config (:class:`~transformers.PeerConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. +""" + +PEER_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~transformers.PeerTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + `What are input IDs? <../glossary.html#input-ids>`__ + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + `What are attention masks? <../glossary.html#attention-mask>`__ + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + + `What are position IDs? <../glossary.html#position-ids>`_ + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. +""" + + +@add_start_docstrings( + 'The bare Peer Model transformer outputting raw hidden-states without any specific head on top. Identical to ' + 'the BERT model except that it uses an additional linear layer between the embedding layer and the encoder if the ' + 'hidden size and embedding size are different.' + '' + 'Both the generator and discriminator checkpoints may be loaded into this model.', + PEER_START_DOCSTRING, +) +class PeerModel(PeerPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + self.embeddings = PeerEmbeddings(config) + + if config.embedding_size != config.hidden_size: + self.embeddings_project = nn.Linear(config.embedding_size, + config.hidden_size) + + self.encoder = PeerEncoder(config) + self.config = config + self.init_weights() + + if self.config.seq_side_info_embeddings: + self.input_sequence_side_info = dict() + self.sequence_side_info = SequenceSideInfo() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def update_seq_side_info(self, side_info_sets, input_ids): + + device = input_ids.device + if 'input_sequence_side_info' not in side_info_sets or len( + side_info_sets['input_sequence_side_info']) == 0: + input_sequence_side_info = self.sequence_side_info.generate_seq_side_info( + self.config.seq_side_info_embeddings, input_ids) + + else: + # Save compute in PEER pre-training + # (Save the extra side info into cpu in the first epoch; Directly retrieve it from cpu in later epochs) + input_sequence_side_info = side_info_sets[ + 'input_sequence_side_info'] + + for ss in input_sequence_side_info.keys(): + input_sequence_side_info[ss] = input_sequence_side_info[ss].to( + device=device).long() + side_info_sets = {**side_info_sets, **input_sequence_side_info} + return side_info_sets + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + side_info_sets=dict(), + return_dict=None, + ): + if self.config.seq_side_info_embeddings: + side_info_sets = self.update_seq_side_info(side_info_sets, + input_ids) + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape, device) + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + hidden_states = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + side_info_sets=side_info_sets, + ) + + if hasattr(self, 'embeddings_project'): + hidden_states = self.embeddings_project(hidden_states) + + hidden_states = self.encoder( + hidden_states, + attention_mask=extended_attention_mask, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + side_info_sets=side_info_sets, + return_dict=return_dict, + ) + + return hidden_states + + +class PeerTopModel(PeerPreTrainedModel): + + def __init__(self, config): + super().__init__(config) + + self.encoder = PeerEncoder(config) + self.config = config + self.init_weights() + + if self.config.seq_side_info_embeddings: + self.input_sequence_side_info = dict() + self.sequence_side_info = SequenceSideInfo() + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def update_seq_side_info(self, side_info_sets, input_ids): + + device = input_ids.device + if 'input_sequence_side_info' not in side_info_sets or len( + side_info_sets['input_sequence_side_info']) == 0: + input_sequence_side_info = self.sequence_side_info.generate_seq_side_info( + self.config.seq_side_info_embeddings, input_ids) + + else: + # Save compute in PEER pre-training + # (Save the extra side info into cpu in the first epoch; Directly retrieve it from cpu in later epochs) + input_sequence_side_info = side_info_sets[ + 'input_sequence_side_info'] + + for ss in input_sequence_side_info.keys(): + input_sequence_side_info[ss] = input_sequence_side_info[ss].to( + device=device).long() + side_info_sets = {**side_info_sets, **input_sequence_side_info} + return side_info_sets + + def forward( + self, + hidden_states, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + side_info_sets=dict(), + return_dict=None, + ): + + if self.config.seq_side_info_embeddings: + side_info_sets = self.update_seq_side_info(side_info_sets, + input_ids) + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape, device) + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + hidden_states = self.encoder( + hidden_states, + attention_mask=extended_attention_mask, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + side_info_sets=side_info_sets, + return_dict=return_dict, + ) + + return hidden_states + + +class PeerClassificationHead(nn.Module): + """Head for sentence-level classification tasks.""" + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.out_proj = nn.Linear(config.hidden_size, config.num_labels) + + def forward(self, features, **kwargs): + x = features[:, 0, :] # take token (equiv. to [CLS]) + x = self.dropout(x) + x = self.dense(x) + x = get_activation('gelu')( + x + ) # although BERT uses tanh here, it seems Peer authors used gelu here + x = self.dropout(x) + x = self.out_proj(x) + return x diff --git a/runtime/Lib/site_packages/models/nlp/peer/configuration.py b/runtime/Lib/site_packages/models/nlp/peer/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..794b89f722957beb5a514dafada551fbaeec7728 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/peer/configuration.py @@ -0,0 +1,224 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PEER model configuration """ + +# modified the path according to the structure in my directory csssl_4_15/cssl/ and its env +from transformers.configuration_utils import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class PeerConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a :class:`~transformers.PeerModel` or a + :class:`~transformers.TFPeerModel`. It is used to instantiate a PEER model according to the specified + arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar + configuration to that of the PEER `google/peer-small-discriminator + `__ architecture. + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `onal`, defaults to 30522) + Vocabulary size of the PEER model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.PeerModel` or + :class:`~transformers.TFPeerModel`. + embedding_size (:obj:`int`, `onal`, defaults to 128) + Dimensionality of the encoder layers and the pooler layer. + hidden_size (:obj:`int`, `onal`, defaults to 256) + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `onal`, defaults to 12) + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `onal`, defaults to 4) + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `onal`, defaults to 1024) + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `onal`, defaults to :obj:`"gelu"`) + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `onal`, defaults to 0.1) + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `onal`, defaults to 0.1) + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `onal`, defaults to 512) + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `onal`, defaults to 2) + The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.PeerModel` or + :class:`~transformers.TFPeerModel`. + initializer_range (:obj:`float`, `onal`, defaults to 0.02) + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (:obj:`float`, `onal`, defaults to 1e-12) + The epsilon used by the layer normalization layers. + summary_type (:obj:`str`, `onal`, defaults to :obj:`"first"`) + Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. + + Has to be one of the following ons + + - :obj:`"last"`: Take the last token hidden state (like XLNet). + - :obj:`"first"`: Take the first token hidden state (like BERT). + - :obj:`"mean"`: Take the mean of all tokens hidden states. + - :obj:`"cls_index"`: Supply a Tensor of classification token position (like GPT/GPT-2). + - :obj:`"attn"`: Not implemented now, use multi-head attention. + summary_use_proj (:obj:`bool`, `onal`, defaults to :obj:`True`) + Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. + + Whether or not to add a projection after the vector extraction. + summary_activation (:obj:`str`, `onal`) + Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. + + Pass :obj:`"gelu"` for a gelu activation to the output, any other value will result in no activation. + summary_last_dropout (:obj:`float`, `onal`, defaults to 0.0) + Argument used when doing sequence summary. Used in the sequence classification and multiple choice models. + + The dropout ratio to be used after the projection and activation. + position_embedding_type (:obj:`str`, `onal`, defaults to :obj:`"absolute"`) + Type of position embedding. Choose one of :obj:`"absolute"`, :obj:`"relative_key"`, + :obj:`"relative_key_query"`. For positional embeddings use :obj:`"absolute"`. For more information on + :obj:`"relative_key"`, please refer to `Self-Attention with Relative Position Representations (Shaw et al.) + `__. For more information on :obj:`"relative_key_query"`, please refer to + `Method 4` in `Improve Transformer Models with Better Relative Position Embeddings (Huang et al.) + `__. + + Examples:: + + >>> from transformers import PeerModel, PeerConfig + + >>> # Initializing a PEER peer-base-uncased style configuration + >>> configuration = PeerConfig() + + >>> # Initializing a model from the peer-base-uncased style configuration + >>> model = PeerModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + model_type = 'peer' + + def __init__(self, + vocab_size=30522, + embedding_size=128, + hidden_size=256, + num_hidden_layers=12, + num_hidden_layers_shared=3, + num_hidden_layers_gen=6, + num_attention_heads=4, + intermediate_size=1024, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + summary_type='first', + summary_use_proj=True, + summary_activation='gelu', + summary_last_dropout=0.1, + pad_token_id=0, + position_embedding_type='absolute', + gen_weight=1, + dis_weight=50, + dis_weight_scheduler=1, + augmentation_copies=1, + augmentation_temperature=1, + absolute_position_embedding=1, + relative_position_embedding=32, + seq_side_info_embeddings=0, + cold_start_epochs=1.25, + debug_config=dict(), + rtd_levels=2, + rtd_level_thresholds='', + ranking_start_epoch=1.0, + real_token_rank_for_good_estimate=5, + rank_sampl_prop=0.3, + rank_sampl_range=100, + rank_delta_factor=0.0, + rank_level_compare_method=0, + weight_loss_low_levels=1.0, + weight_loss_low_levels_setting='1.0-1.0', + weight_loss_low_levels_scheduler=0, + weight_loss_level_compos=1, + mask_da=0, + mask_da_start_epoch=0.0, + mask_da_mlm_topk_val=0, + mask_ratio_setting='0.15-0.15', + mask_ratio_scheduler=0, + mask_ratio_stage1_epochs=0.0, + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_hidden_layers_shared = num_hidden_layers_shared + self.num_hidden_layers_gen = num_hidden_layers_gen + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_last_dropout = summary_last_dropout + if type(position_embedding_type) == str: + position_embedding_type = position_embedding_type.split('+') + self.position_embedding_type = position_embedding_type + self.augmentation_temperature = augmentation_temperature + + self.gen_weight = gen_weight + self.dis_weight = dis_weight + self.dis_weight_scheduler = dis_weight_scheduler + self.augmentation_copies = augmentation_copies + + self.absolute_position_embedding = absolute_position_embedding + self.relative_position_embedding = relative_position_embedding + self.seq_side_info_embeddings = seq_side_info_embeddings + + self.cold_start_epochs = cold_start_epochs + self.debug_config = debug_config + + self.rtd_levels = rtd_levels + self.rtd_level_thresholds = rtd_level_thresholds + self.ranking_start_epoch = ranking_start_epoch + self.real_token_rank_for_good_estimate = real_token_rank_for_good_estimate + self.rank_sampl_prop = rank_sampl_prop + self.rank_sampl_range = rank_sampl_range + self.rank_delta_factor = rank_delta_factor + self.rank_level_compare_method = rank_level_compare_method + self.weight_loss_low_levels = weight_loss_low_levels + self.weight_loss_low_levels_setting = weight_loss_low_levels_setting + self.weight_loss_low_levels_scheduler = weight_loss_low_levels_scheduler + self.weight_loss_level_compos = weight_loss_level_compos + + self.mask_da = mask_da + self.mask_da_start_epoch = mask_da_start_epoch + self.mask_da_mlm_topk_val = mask_da_mlm_topk_val + + self.mask_ratio_setting = mask_ratio_setting + self.mask_ratio_scheduler = mask_ratio_scheduler + self.mask_ratio_stage1_epochs = mask_ratio_stage1_epochs diff --git a/runtime/Lib/site_packages/models/nlp/peer/sas_utils.py b/runtime/Lib/site_packages/models/nlp/peer/sas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..da947e4d8046e5da3e75670b1f74da20c889a0f2 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/peer/sas_utils.py @@ -0,0 +1,173 @@ +import random + +import nltk +import numpy as np +import torch + + +def get_random_states(device=None): + random_states = {} + + random_states['rng_state_torch'] = torch.get_rng_state() + random_states['rng_state_np'] = np.random.get_state() + random_states['rng_state_rnd'] = random.getstate() + if device is not None and device.type == 'cuda': + random_states['rng_state_torch_cuda'] = torch.cuda.get_rng_state( + device) + + return random_states + + +def set_random_states(random_states, device=None): + + torch.set_rng_state(random_states['rng_state_torch']) + np.random.set_state(random_states['rng_state_np']) + random.setstate(random_states['rng_state_rnd']) + if device is not None and device.type == 'cuda': + torch.cuda.set_rng_state(random_states['rng_state_torch_cuda']) + + +# Check any nan or inf in the data. Return an array of two elements for nan and inf, respectively. +# Inputs +# data: a tensor or a tuple of multiple tensors +# Outputs: +# results: Each element shows the # of tensors that includes nan or inf. +# If data is a "tuple" (instead of a single tensor), +# we add 10 to the count if any nan or inf is detected. +def check_nan_inf(data): + if data is None: + return None + + result = [0, 0] + if torch.is_tensor(data): + if torch.isnan(data).any(): + result[0] = 1 + if torch.isinf(data).any(): + result[1] = 1 + + elif type(data) is tuple: + for i in range(len(data)): + if torch.is_tensor(data[i]): + if torch.isnan(data[i]).any(): + result[0] += 1 + if torch.isinf(data[i]).any(): + result[1] += 1 + + if result[0] > 0: + result[0] += 10 + if result[1] > 0: + result[1] += 10 + + return result if sum(result) > 0 else None + + +class SequenceSideInfo(): + + def __init__(self, tokenizer=None): + if tokenizer is not None: + self.tokenizer = tokenizer + else: + from transformers import ElectraTokenizer + self.tokenizer = ElectraTokenizer.from_pretrained( + 'google/electra-small-generator') + + self.sen_tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer() + + tokens = [ + self.tokenizer.decode([i]) + for i in range(self.tokenizer.vocab_size) + ] + self.ind_subtokens = set( + [i for i in range(len(tokens)) if tokens[i][0:2] == '##']) + tmp = [ + 0 if t[0] == '[' and t[-1] == ']' else + (10 + min(5, + len(t) - 2) if t[0:2] == '##' else min(10, len(t))) + for t in tokens + ] + self.len_tokens = torch.tensor(tmp, dtype=torch.int8) + + def getSenTokIdx(self, sentence_position_embedding, inputs_str, + seq_len_total): + sentences = self.sen_tokenizer.tokenize(inputs_str) + sen_lengths = np.array([ + len(x) - 2 + for x in self.tokenizer.batch_encode_plus(sentences)['input_ids'] + ]) # -2: to drop the extra [CLS] and [SEP] added by sen_tokenizer + + sen_lengths[0] = seq_len_total - sen_lengths[1:].sum() + + idx_sen = np.concatenate([ + i * np.ones(sen_lengths[i], dtype=np.int8) + for i in range(len(sen_lengths)) + ]) + idx_tok = np.concatenate([ + np.arange(sen_lengths[i], dtype=np.int8) + for i in range(len(sen_lengths)) + ]) + + return np.concatenate((idx_sen, idx_tok)) + + def generate_seq_side_info(self, sentence_position_embedding, inputs_id): + is_np_array = False + if isinstance(inputs_id[0], (list, np.ndarray)): + is_np_array = True + inputs_id = torch.tensor(inputs_id) + + if hasattr(self.tokenizer, 'batch_decode'): + inputs_str = self.tokenizer.batch_decode(inputs_id) + sen_tok_idx = torch.tensor( + np.array([ + self.getSenTokIdx(sentence_position_embedding, input_str, + inputs_id.shape[1]) + for input_str in inputs_str + ]), + device=inputs_id.device) + else: + sen_tok_idx = torch.tensor( + np.array([ + self.getSenTokIdx(sentence_position_embedding, + self.tokenizer.decode(input_ori), + inputs_id.shape[1]) + for input_ori in inputs_id.numpy() + ]), + device=inputs_id.device) + + side_info_dict = dict() + seq_length = inputs_id.shape[1] + side_info_dict[ + 'ss_sentence_position_in_sequence'] = sen_tok_idx[:, 0:seq_length] + side_info_dict[ + 'ss_token_position_in_sentence'] = sen_tok_idx[:, 1 * seq_length:2 + * seq_length] + + if sentence_position_embedding >= 2: + # consider sub-word tokens + unique, _ = np.unique(inputs_id, return_inverse=True) + ind_subtokens = self.ind_subtokens.intersection(set(unique)) + + if len(ind_subtokens) > 0: + idx_tok_ww = torch.stack([ + inputs_id == st for st in ind_subtokens + ]).any(axis=0).char() + else: + idx_tok_ww = torch.zeros(inputs_id.shape, dtype=torch.int8) + + idx_tok_ww[:, 0] = 0 + idx_tok_ww_1 = idx_tok_ww[:, 1:] + for i in range(1, 11): + pos = torch.logical_and(idx_tok_ww_1 == i, + idx_tok_ww[:, 0:-1] == i) + if len(pos) == 0: + break + idx_tok_ww_1[pos] = i + 1 + side_info_dict['ss_token_position_in_whole_word'] = idx_tok_ww + + inputs_str_len = self.len_tokens[inputs_id.long()] + side_info_dict['ss_token_string_length'] = inputs_str_len + + if is_np_array: + for key in side_info_dict.keys(): + side_info_dict[key] = side_info_dict[key].numpy() + + return side_info_dict diff --git a/runtime/Lib/site_packages/models/nlp/peer/text_classification.py b/runtime/Lib/site_packages/models/nlp/peer/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..de55652c8935bcb01801610003da106977122e1c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/peer/text_classification.py @@ -0,0 +1,121 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from torch.nn import CrossEntropyLoss, MSELoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import (PeerClassificationHead, PeerModel, PeerPreTrainedModel, + PeerTopModel) + +logger = logging.get_logger() + + +@MODELS.register_module(Tasks.text_classification, module_name=Models.peer) +@MODELS.register_module(Tasks.nli, module_name=Models.peer) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.peer) +@MODELS.register_module(Tasks.sentence_similarity, module_name=Models.peer) +@MODELS.register_module( + Tasks.zero_shot_classification, module_name=Models.peer) +class PeerForSequenceClassification(PeerPreTrainedModel): + + def __init__(self, config, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + + config_discr_top = copy.deepcopy(config) + config_shared_bottom = copy.deepcopy(config) + + assert config.num_hidden_layers_shared > 0, 'config.num_hidden_layers_shared should be greater than 0!' + + config_shared_bottom.num_hidden_layers = config.num_hidden_layers_shared + config_discr_top.num_hidden_layers = config_discr_top.num_hidden_layers \ + - config_discr_top.num_hidden_layers_shared + + self.teams1_shared_bottom = PeerModel(config_shared_bottom) + self.teams1_discr_top = PeerTopModel(config_discr_top) + + self.classifier = PeerClassificationHead(config) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + side_info_sets=dict(), + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + hidden_states_discr_bottom = self.teams1_shared_bottom( + input_ids, attention_mask, token_type_ids, position_ids, head_mask, + inputs_embeds, output_attentions, output_hidden_states, + side_info_sets, return_dict) + + hidden_states_discr_top = self.teams1_discr_top( + hidden_states_discr_bottom[0], input_ids, attention_mask, + token_type_ids, position_ids, head_mask, inputs_embeds, + output_attentions, output_hidden_states, side_info_sets, + return_dict) + + discriminator_hidden_states = hidden_states_discr_top + + sequence_output = discriminator_hidden_states[0] + + logits = self.classifier(sequence_output) + loss = None + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + discriminator_hidden_states[1:] + return ((loss, ) + output) if loss is not None else output + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=discriminator_hidden_states.hidden_states, + attentions=discriminator_hidden_states.attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/plug/AnnealingLR.py b/runtime/Lib/site_packages/models/nlp/plug/AnnealingLR.py new file mode 100644 index 0000000000000000000000000000000000000000..3775d37553bc957ab9e521fb165fc4f109109b1a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/AnnealingLR.py @@ -0,0 +1,88 @@ +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch DataLoader for TFRecords""" + +import math + +import torch +from torch.optim.lr_scheduler import _LRScheduler + + +class AnnealingLR(_LRScheduler): + """Anneals the learning rate from start to zero along a cosine curve.""" + + DECAY_STYLES = ['linear', 'cosine', 'exponential', 'constant', 'None'] + + def __init__(self, + optimizer, + start_lr, + warmup_iter, + num_iters, + decay_style=None, + last_iter=-1): + self.optimizer = optimizer + self.start_lr = start_lr + self.warmup_iter = warmup_iter + self._step_count = last_iter + 1 + self.end_iter = num_iters + self.decay_style = decay_style.lower() if isinstance(decay_style, + str) else None + self.step(self._step_count) + if torch.distributed.get_rank() == 0: + print('learning rate decaying', decay_style) + + def get_lr(self): + # https://openreview.net/pdf?id=BJYwwY9ll pg. 4 + if self.warmup_iter > 0 and self._step_count <= self.warmup_iter: + return float(self.start_lr) * self._step_count / self.warmup_iter + else: + if self.decay_style == self.DECAY_STYLES[0]: + return self.start_lr * (( + self.end_iter - # noqa W504 + (self._step_count - self.warmup_iter)) / self.end_iter) + elif self.decay_style == self.DECAY_STYLES[1]: + return self.start_lr / 2.0 * ( + math.cos(math.pi * (self._step_count - self.warmup_iter) + / self.end_iter) + 1) + elif self.decay_style == self.DECAY_STYLES[2]: + # TODO: implement exponential decay + return self.start_lr + else: + return self.start_lr + + def step(self, step_num=None): + if step_num is None: + step_num = self._step_count + 1 + self._step_count = step_num + new_lr = self.get_lr() + for group in self.optimizer.param_groups: + group['lr'] = new_lr + + def state_dict(self): + sd = { + 'start_lr': self.start_lr, + 'warmup_iter': self.warmup_iter, + '_step_count': self._step_count, + 'decay_style': self.decay_style, + 'end_iter': self.end_iter + } + return sd + + def load_state_dict(self, sd): + self.start_lr = sd['start_lr'] + self.warmup_iter = sd['warmup_iter'] + self._step_count = sd['_step_count'] + self.end_iter = sd['end_iter'] + self.decay_style = sd['decay_style'] + self.step(self._step_count) diff --git a/runtime/Lib/site_packages/models/nlp/plug/__init__.py b/runtime/Lib/site_packages/models/nlp/plug/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..589a636a006e630724a1d864173f537fbb909f2b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import PlugNLGConfig + from .backbone import PlugModel + from .distributed_plug import DistributedPlug +else: + _import_structure = { + 'configuration': ['PlugNLGConfig'], + 'backbone': ['PlugModel'], + 'distributed_plug': ['DistributedPlug'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/plug/backbone.py b/runtime/Lib/site_packages/models/nlp/plug/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..37714ed7716ed56ce45021d645a8cfb6bbe5aa2a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/backbone.py @@ -0,0 +1,1009 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import logging +import math + +import torch +import torch.nn.functional as F +from megatron_util import mpu +from torch import nn + +from modelscope.utils.nlp.distributed import (normal_init_method, + scaled_init_method) +from .configuration import PlugNLGConfig, PlugNLUConfig + +logger = logging.getLogger(__name__) + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class BertLayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(BertLayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(BertEmbeddings, self).__init__() + self.word_embeddings = mpu.VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + init_method=normal_init_method( + mean=0.0, std=config.initializer_range)) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.fp32_layernorm = config.fp32_layernorm + self.fp32_embedding = config.fp32_embedding + self.fp32_tokentypes = config.fp32_tokentypes + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + if not self.fp32_tokentypes: + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + if self.fp32_embedding and not self.fp32_layernorm: + embeddings = embeddings.half() + previous_type = embeddings.type() + if self.fp32_layernorm: + embeddings = embeddings.float() + embeddings = self.LayerNorm(embeddings) + if self.fp32_layernorm: + if self.fp32_embedding: + embeddings = embeddings.half() + else: + embeddings = embeddings.type(previous_type) + else: + embeddings = words_embeddings.float() + position_embeddings.float( + ) + token_type_embeddings.float() + if self.fp32_tokentypes and not self.fp32_layernorm: + embeddings = embeddings.half() + previous_type = embeddings.type() + if self.fp32_layernorm: + embeddings = embeddings.float() + embeddings = self.LayerNorm(embeddings) + if self.fp32_layernorm: + if self.fp32_tokentypes: + embeddings = embeddings.half() + else: + embeddings = embeddings.type(previous_type) + embeddings = self.dropout(embeddings) + return embeddings + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super(BertSelfOutput, self).__init__() + if hasattr(config, 'deep_init') and config.deep_init: + init_method = scaled_init_method( + mean=0.0, + std=config.initializer_range, + num_layers=config.num_hidden_layers) + else: + init_method = normal_init_method( + mean=0.0, std=config.initializer_range) + self.dense = mpu.RowParallelLinear( + input_size=config.hidden_size, + output_size=config.hidden_size, + bias=True, + input_is_parallel=True, + stride=1, + init_method=init_method) + self.fp32_layernorm = config.fp32_layernorm + if not config.pre_ln: + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + else: + self.LayerNorm = None + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward( + self, + hidden_states, + input_tensor, + ): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + ln_input = hidden_states + input_tensor + if self.LayerNorm is not None: + previous_type = ln_input.type() + if self.fp32_layernorm: + ln_input = ln_input.float() + hidden_states = self.LayerNorm(ln_input) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + else: + hidden_states = ln_input + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config): + super(BertAttention, self).__init__() + self.fp32_layernorm = config.fp32_layernorm + if config.pre_ln: + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + else: + self.LayerNorm = None + self.self = mpu.BertParallelSelfAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + dropout_prob=config.attention_probs_dropout_prob, + output_parallel=True, + init_method=normal_init_method( + mean=0.0, std=config.initializer_range), + separate=config.attn_separate) + self.output = BertSelfOutput(config) + + def forward( + self, + input_tensor, + attention_mask, + ): + if self.LayerNorm is not None: + ln_input = input_tensor + previous_type = input_tensor.type() + if self.fp32_layernorm: + ln_input = input_tensor.float() + ln_output = self.LayerNorm(ln_input) + if self.fp32_layernorm: + ln_output = ln_output.type(previous_type) + self_output = self.self( + ln_output, + attention_mask, + ) + else: + self_output = self.self( + input_tensor, + attention_mask, + ) + + attention_output = self.output( + self_output, + input_tensor, + ) + return attention_output + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super(BertIntermediate, self).__init__() + self.dense = mpu.ColumnParallelLinear( + input_size=config.hidden_size, + output_size=config.intermediate_size, + bias=True, + gather_output=False, + stride=1, + init_method=normal_init_method( + mean=0.0, std=config.initializer_range)) + self.intermediate_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + + def forward( + self, + hidden_states, + ): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super(BertOutput, self).__init__() + if hasattr(config, 'deep_init') and config.deep_init: + init_method = scaled_init_method( + mean=0.0, + std=config.initializer_range, + num_layers=config.num_hidden_layers) + else: + init_method = normal_init_method( + mean=0.0, std=config.initializer_range) + self.dense = mpu.RowParallelLinear( + input_size=config.intermediate_size, + output_size=config.hidden_size, + bias=True, + input_is_parallel=True, + stride=1, + init_method=init_method) + self.fp32_layernorm = config.fp32_layernorm + if not config.pre_ln: + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + else: + self.LayerNorm = None + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward( + self, + hidden_states, + input_tensor, + ): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + ln_input = hidden_states + input_tensor + if self.LayerNorm is not None: + previous_type = ln_input.type() + if self.fp32_layernorm: + ln_input = ln_input.float() + hidden_states = self.LayerNorm(ln_input) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + else: + hidden_states = ln_input + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config): + super(BertLayer, self).__init__() + self.attention = BertAttention(config) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + self.fp32_layernorm = config.fp32_layernorm + if config.pre_ln: + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + else: + self.LayerNorm = None + + def forward(self, hidden_states, attention_mask): + attention_output = self.attention(hidden_states, attention_mask) + if self.LayerNorm is not None: + ln_input = attention_output + previous_type = attention_output.type() + if self.fp32_layernorm: + ln_input = attention_output.float() + ln_output = self.LayerNorm(ln_input) + if self.fp32_layernorm: + ln_output = ln_output.type(previous_type) + intermediate_output = self.intermediate(ln_output) + else: + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BertEncoder(nn.Module): + + def __init__(self, config): + super(BertEncoder, self).__init__() + self.layer = nn.ModuleList( + [BertLayer(config) for _ in range(config.num_hidden_layers)]) + self.fp32_layernorm = config.fp32_layernorm + if config.pre_ln: + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + else: + self.LayerNorm = None + + def forward( + self, + hidden_states, + attention_mask, + output_all_encoded_layers=True, + checkpoint_activations=False, + detach_index=-1, + ): + all_encoder_layers = [] + + def custom(start, end): + + def custom_forward(*inputs): + layers = self.layer[start:end] + x_ = inputs[0] + for layer in layers: + x_ = layer(x_, inputs[1]) + return x_ + + return custom_forward + + if checkpoint_activations: + layer_idx = 0 + num_layers = len(self.layer) + chunk_length = 1 + while layer_idx < num_layers: + hidden_states = mpu.checkpoint( + custom(layer_idx, layer_idx + chunk_length), hidden_states, + attention_mask * 1) + if detach_index == layer_idx: + hidden_states.detach_() + layer_idx += chunk_length + # decoder layers + else: + for i, layer_module in enumerate(self.layer): + hidden_states = layer_module(hidden_states, attention_mask) + if detach_index == i: + hidden_states.detach_() + if i == len(self.layer) - 1 and self.LayerNorm is not None: + previous_type = hidden_states.type() + if self.fp32_layernorm: + hidden_states = hidden_states.float() + hidden_states = self.LayerNorm(hidden_states) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + if output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + + if not output_all_encoded_layers or checkpoint_activations: + if self.LayerNorm is not None: + previous_type = hidden_states.type() + if self.fp32_layernorm: + hidden_states = hidden_states.float() + hidden_states = self.LayerNorm(hidden_states) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BertPooler(nn.Module): + + def __init__(self, config): + super(BertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super(BertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.transform_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + self.LayerNorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.fp32_layernorm = config.fp32_layernorm + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + previous_type = hidden_states.type() + if self.fp32_layernorm: + hidden_states = hidden_states.float() + hidden_states = self.LayerNorm(hidden_states) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder_weight = bert_model_embedding_weights + self.bias = nn.Parameter( + torch.zeros(bert_model_embedding_weights.size(0))) + self.bias.model_parallel = True + self.fp32_embedding = config.fp32_embedding + self.fp32_layernorm = config.fp32_layernorm + + def convert_to_type(tensor): + if self.fp32_embedding: + return tensor.half() + else: + return tensor + + self.type_converter = convert_to_type + self.converted = False + + def forward(self, hidden_states): + if not self.converted: + self.converted = True + if self.fp32_embedding: + self.transform.half() + if self.fp32_layernorm: + self.transform.LayerNorm.float() + hidden_states = self.transform(self.type_converter(hidden_states)) + hidden_states = mpu.copy_to_model_parallel_region(hidden_states) + hidden_states = F.linear( + self.type_converter(hidden_states), + self.type_converter(self.decoder_weight), + self.type_converter(self.bias)) + return hidden_states + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertPreTrainingHeads, self).__init__() + self.predictions = BertLMPredictionHead(config, + bert_model_embedding_weights) + self.seq_relationship = nn.Linear(config.hidden_size, 3) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + for p in self.seq_relationship.parameters(): + if p is None: + continue + pooled_output = pooled_output.type_as(p) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class PreTrainedBertModel(nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedBertModel, self).__init__() + if not isinstance(config, PlugNLUConfig) and not isinstance( + config, PlugNLGConfig): + raise ValueError( + 'Parameter config in `{}(config)` should be an instance of class `BertConfig`. ' + 'To create a model from a Google pretrained model use ' + '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format( + self.__class__.__name__, self.__class__.__name__)) + self.config = config + + def init_bert_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +class BertModel(PreTrainedBertModel): + """BERT model ("Bidirectional Embedding Representations from a Transformer"). + + Params: + config: a BertConfig class instance with the configuration to build a new model + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `output_all_encoded_layers`: boolean which controls the content of the `encoded_layers` output as + described below. Default: `True`. + + Outputs: Tuple of (encoded_layers, pooled_output) + `encoded_layers`: controled by `output_all_encoded_layers` argument: + - `output_all_encoded_layers=True`: outputs a list of the full sequences of encoded-hidden-states at the end + of each attention block (i.e. 12 full sequences for BERT-base, 24 for BERT-large), each + encoded-hidden-state is a torch.FloatTensor of size [batch_size, sequence_length, hidden_size], + - `output_all_encoded_layers=False`: outputs only the full sequence of hidden-states corresponding + to the last attention block of shape [batch_size, sequence_length, hidden_size], + `pooled_output`: a torch.FloatTensor of size [batch_size, hidden_size] which is the output of a + classifier pretrained on top of the hidden state associated to the first character of the + input (`CLF`) to train on the Next-Sentence task (see BERT's paper). + + Examples: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = modeling.BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = modeling.BertModel(config=config) + >>> all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config): + super(BertModel, self).__init__(config) + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder(config) + self.pooler = BertPooler(config) + self.apply(self.init_bert_weights) + + def forward( + self, + input_ids, + token_type_ids=None, + attention_mask=None, + output_all_encoded_layers=True, + checkpoint_activations=False, + detach_index=-1, + ): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.encoder.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output = self.embeddings(input_ids, token_type_ids) + encoded_layers = self.encoder( + embedding_output, + extended_attention_mask, + output_all_encoded_layers=output_all_encoded_layers, + checkpoint_activations=checkpoint_activations, + detach_index=detach_index) + sequence_output = encoded_layers[-1] + for p in self.pooler.parameters(): + if p is None: + continue + sequence_output = sequence_output.type_as(p) + break + + pooled_output = sequence_output[:, 0] + if not output_all_encoded_layers or checkpoint_activations: + encoded_layers = encoded_layers[-1] + return encoded_layers, pooled_output + + +class DecodeLayer(nn.Module): + + def __init__(self, config): + super(DecodeLayer, self).__init__() + init_method = normal_init_method( + mean=0.0, std=config.initializer_range) + output_layer_init_method = scaled_init_method( + mean=0.0, + std=config.initializer_range, + num_layers=config.num_hidden_layers) + + self.attention = mpu.GPT2ParallelSelfAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + attention_dropout_prob=config.attention_probs_dropout_prob, + output_dropout_prob=config.hidden_dropout_prob, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + ) + + self.cross_attention = mpu.PalmParallelCrossAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + attention_dropout_prob=config.attention_probs_dropout_prob, + output_dropout_prob=config.hidden_dropout_prob, + init_method=init_method, + attn_separate=False, + output_layer_init_method=output_layer_init_method, + ) + + self.input_layernorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.post_attention_layernorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.post_cross_attention_layernorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + + self.intermediate = mpu.ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + gather_output=False, + init_method=init_method, + ) + self.intermediate_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + self.output = mpu.RowParallelLinear( + config.intermediate_size, + config.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + ) + + self.dropout = torch.nn.Dropout(config.hidden_dropout_prob) + self.fp32_layernorm = config.fp32_layernorm + + def convert_to_type(tensor): + if self.fp32_layernorm: + return tensor.float() + else: + return tensor + + self.type_converter = convert_to_type + + # def forward(self, hidden_states, enc_attn_mask, dec_attn_mask): + def forward(self, + hidden_states, + enc_hidden_states, + enc_attn_mask, + dec_attn_mask, + is_infer=False): + residual = hidden_states + previous_type = hidden_states.type() + hidden_states = self.input_layernorm( + self.type_converter(hidden_states)) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + hidden_states = self.attention( + hidden_states, dec_attn_mask, is_infer=is_infer) + + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm( + self.type_converter(hidden_states)) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + hidden_states = self.cross_attention(hidden_states, enc_hidden_states, + enc_attn_mask) + hidden_states = residual + hidden_states + residual = hidden_states + hidden_states = self.post_cross_attention_layernorm( + self.type_converter(hidden_states)) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + hidden_states = self.intermediate(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + + hidden_states = self.output(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class BertDecoder(nn.Module): + + def __init__(self, config): + super(BertDecoder, self).__init__() + self.layer = nn.ModuleList( + [DecodeLayer(config) for _ in range(config.dec_hidden_layers)]) + + self.final_layernorm = BertLayerNorm( + config.hidden_size, eps=config.layernorm_epsilon) + self.fp32_layernorm = config.fp32_layernorm + + def forward(self, + hidden_states, + enc_hidden_states, + enc_attn_mask, + dec_attn_mask, + checkpoint_activations=False, + output_all_encoded_layers=False, + is_infer=False): + + def custom(start, end): + + def custom_forward(*inputs): + layers = self.layer[start:end] + x_ = inputs[0] + for layer in layers: + x_ = layer( + x_, + inputs[1], + inputs[2], + dec_attn_mask * 1, + is_infer=is_infer) + return x_ + + return custom_forward + + pre_enc_hidden = enc_hidden_states.data + if checkpoint_activations: + layer_idx = 0 + num_layers = len(self.layer) + chunk_length = 1 + while layer_idx < num_layers: + hidden_states = mpu.checkpoint( + custom(layer_idx, layer_idx + chunk_length), hidden_states, + enc_hidden_states, enc_attn_mask * 1) + enc_hidden_states.data = pre_enc_hidden + layer_idx += chunk_length + else: + for i, layer_module in enumerate(self.layer): + hidden_states = layer_module( + hidden_states, + enc_hidden_states, + enc_attn_mask, + dec_attn_mask, + is_infer=is_infer) + + previous_type = hidden_states.type() + if self.fp32_layernorm: + hidden_states = hidden_states.float() + hidden_states = self.final_layernorm(hidden_states) + if self.fp32_layernorm: + hidden_states = hidden_states.type(previous_type) + + return [hidden_states] + + +class DecodeModel(PreTrainedBertModel): + + def __init__(self, config): + super(DecodeModel, self).__init__(config) + self.decoder = BertDecoder(config) + self.apply(self.init_bert_weights) + + def forward(self, + embeddings, + sequence_output, + decode_input_ids, + position_ids=None, + enc_attn_mask=None, + dec_attn_mask=None, + checkpoint_activations=False, + is_infer=False): + extended_attention_mask = enc_attn_mask.unsqueeze(1).unsqueeze(2) + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.decoder.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output = embeddings(decode_input_ids) + sequence_output = self.decoder( + embedding_output, + sequence_output, + extended_attention_mask, + dec_attn_mask, + checkpoint_activations=False, + is_infer=is_infer) + return sequence_output[-1] + + +class PalmForPreTraining(PreTrainedBertModel): + + def __init__(self, config): + super(PalmForPreTraining, self).__init__(config) + self.bert = BertModel(config) + self.cls = BertPreTrainingHeads( + config, self.bert.embeddings.word_embeddings.weight) + self.decoder = DecodeModel(config) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + token_type_ids=None, + attention_mask=None, + decode_input_ids=None, + position_ids=None, + decode_attention_mask=None, + lm_labels=None, + checkpoint_activations=False, + is_infer=False, + sequence_output=None, + parallel_output=True): + if sequence_output is None: + sequence_output, pooled_output = self.bert( + input_ids, + token_type_ids, + attention_mask, + output_all_encoded_layers=False, + checkpoint_activations=checkpoint_activations) + prediction_scores, seq_relationship_score = self.cls( + sequence_output, pooled_output) + else: + prediction_scores = None + sequence_output = sequence_output.to( + dtype=next(self.decoder.parameters()).dtype) + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + decode_output = self.decoder( + self.bert.embeddings, + sequence_output, + decode_input_ids, + position_ids, + attention_mask, + decode_attention_mask, + checkpoint_activations=checkpoint_activations, + is_infer=is_infer) + + transformer_output_parallel = mpu.copy_to_model_parallel_region( + decode_output) + + logits_parallel = F.linear(transformer_output_parallel, + self.bert.embeddings.word_embeddings.weight) + + if parallel_output: + return prediction_scores, logits_parallel + if is_infer: + return prediction_scores, mpu.gather_from_model_parallel_region( + logits_parallel), sequence_output + return prediction_scores, mpu.gather_from_model_parallel_region( + logits_parallel) + + +class PlugModel(torch.nn.Module): + """ + The bare Plug Model transformer outputting raw hidden-states without any specific head on top. + This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + Parameters: + config ([`PlugNLGConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~DistributedPlug.initialize_model`] method to load the model weights. + Examples: + + >>> # The PLUG model has 27B parameters and usually need to run on multiple GPUs. The example given + >>> # here only initializes a slice of the model on a single GPU. + >>> # Check out the [`~DistributedPipeline.__init__`] method to initialize entire PLUG model. + >>> from modelscope.models.nlp.plug import PlugNLGConfig, PlugModel + + >>> # Initializing a Plug configuration + >>> configuration = PlugNLGConfig() + + >>> # Initializing a model from the configuration + >>> model = PlugModel(configuration) + """ + + def __init__(self, config): + super(PlugModel, self).__init__() + self.config = config + self.model = PalmForPreTraining(self.config) + + def forward(self, + input_tokens, + token_type_ids=None, + attention_mask=None, + target_tokens=None, + position_ids=None, + decode_attention_mask=None, + checkpoint_activations=False, + is_infer=False, + sequence_output=None, + parallel_output=True): + """ + Parameters: + input_tokens (`torch.LongTensor` of shape `(batch_size, input_tokens_length)`): + `input_tokens_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary. + Indices can be obtained using transformers [`BertTokenizer`]. See + [`TextGenerationPreprocessor.__call__`] for details. + token_type_ids (`torch.LongTensor` of shape `(batch_size, input_tokens_length)`, *optional*, defaults to + None): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + target_tokens (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): + Target token ids(labels) for language modeling. Note that the labels **are shifted** inside the model, + i.e. you can set `target_tokens = input_tokens` Indices are selected in + `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only + computed for labels in `[0, ..., config.vocab_size]` + + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults to None): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + `[0, config.max_position_embeddings - 1]`. + + decode_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*, defaults + to None): + Mask to avoid performing attention on padding token indices of target tokens. Mask values selected in + `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + checkpoint_activations (`boolean`, *optional*, defaults to `False`): + Whether gradient checkpointing is activated for this model or not. + is_infer (`boolean`, *optional*, defaults to `False`): + Whether or not to perform single inference. + sequence_output (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, + defaults to None): + Also known as last_hidden_state. Sequence of hidden-states at the output of the last layer of the + model. A single forward() call can produce one single token. To generate the current token, the + sequence_output generated by the `forward()` of the previous token is required. + parallel_output (`boolean`, *optional*, defaults to `True`): + To parallel return output, or gather it before return. + + + """ + return self.model( + input_tokens, + token_type_ids, + attention_mask, + target_tokens, + position_ids, + decode_attention_mask, + checkpoint_activations=checkpoint_activations, + is_infer=is_infer, + sequence_output=sequence_output, + parallel_output=parallel_output) + + def state_dict(self, destination=None, prefix='', keep_vars=False): + return self.model.state_dict( + destination=destination, prefix=prefix, keep_vars=keep_vars) + + def load_state_dict(self, state_dict, strict=True): + return self.model.load_state_dict(state_dict, strict=strict) diff --git a/runtime/Lib/site_packages/models/nlp/plug/configuration.py b/runtime/Lib/site_packages/models/nlp/plug/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..fedc66c0db50bcc61eece14161e70ac7f95abf9e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/configuration.py @@ -0,0 +1,253 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import json +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class PlugNLUConfig(PretrainedConfig): + model_type = 'plugNLU' + + def __init__(self, + vocab_size=21504, + original_vocab_size=21128, + hidden_size=8192, + num_hidden_layers=24, + num_attention_heads=128, + intermediate_size=32768, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=2048, + type_vocab_size=3, + initializer_range=0.00707, + lr_decay_style='linear', + weight_decay=1e-2, + clip_grad=1.0, + warmup=0.0333, + pre_ln=True, + fp16=True, + fp32_layernorm=True, + fp32_embedding=False, + fp32_tokentypes=False, + layernorm_epsilon=1e-5, + dec_hidden_layers=6, + attn_separate=False, + **kwargs): + super().__init__(layer_norm_eps=layernorm_epsilon, **kwargs) + + self.vocab_size = vocab_size + self.original_vocab_size = original_vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.lr_decay_style = lr_decay_style + self.weight_decay = weight_decay + self.clip_grad = clip_grad + self.warmup = warmup + self.pre_ln = pre_ln + self.fp16 = fp16 + self.fp32_layernorm = fp32_layernorm + self.fp32_embedding = fp32_embedding + self.layernorm_epsilon = layernorm_epsilon + self.fp32_tokentypes = fp32_tokentypes + self.dec_hidden_layers = dec_hidden_layers + self.attn_separate = attn_separate + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = PlugNLUConfig() + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def merge_args(self, args): + """merge values a `BertConfig` from a json file of parameters.""" + local_keys = self.__dict__.keys() + for key, value in args.__dict__.items(): + if key in local_keys: + continue + self.__dict__[key] = value + return self + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' + + +class PlugNLGConfig(PlugNLUConfig): + """ + This is the configuration class to store the configuration of a [`PlugModel`]. It is used to instantiate a + PLUG understanding model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the PLUG + [PLUG](https://modelscope.cn/models/damo/nlp_plug_text-generation_27B/summary) architecture. + + Configuration objects inherit from [`PlugNLUConfig`] and can be used to control the model outputs. Read the + documentation from [`PlugNLUConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 21504): + Padded vocabulary size of the PLUG model for vocab tensor parallel. Defines the number of different tokens + that can be represented by the `inputs_ids` passed when calling [`PlugModel`]. + original_vocab_size (`int`, *optional*, defaults to 21128): + True vocabulary size of the PLUG model. Defines the number of different tokens that can be represented. + hidden_size (`int`, *optional*, defaults to 8192): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 24): + Number of hidden layers in the Transformer encoder. + dec_hidden_layers (`int`, *optional*, defaults to 6): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 128): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 32768): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + hidden_act (`str`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the Transformer Attention. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 3): + The vocabulary size of the `token_type_ids` passed when calling [`PlugModel`]. + initializer_range (`float`, *optional*, defaults to 0.00707): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + lr_decay_style (`str`, *optional*, defaults to 'linear'): + The decay style of learning rate during fine-tunining. If string, `"linear"`, `"cosine"`, `"exponential"`, + `"constant"`, `"None"` are supported. + weight_decay (`float`, *optional*, defaults to 1e-2): + Decoupled weight decay to apply. + clip_grad (`float`, *optional*, defaults to 1.0): + Maximum gradient norm for gradient clipping. + warmup (`float`, *optional*, defaults to 0.01): + Ratio of total training steps used for a linear warmup from 0 to `learning_rate`. + pre_ln (`boolean`, *optional*, defaults to `True`): + Whether or not to apply LayerNorm to the input instead of the output in the blocks. + fp16 (`boolean`, *optional*, defaults to `True`): + Whether to use fp16 16-bit (mixed) precision training instead of 32-bit training. + fp32_layernorm (`boolean`, *optional*, defaults to `True`): + Whether to use fp32 32-bit precision LayerNorm training while the argument `fp16` set to `True`. + fp32_embedding (`boolean`, *optional*, defaults to `False`): + Whether to use fp32 32-bit precision Embedding training while the argument `fp16` set to `True`. + fp32_tokentypes (`boolean`, *optional*, defaults to `False`): + Whether to use fp32 32-bit precision token types training while the argument `fp16` set to `True`. + layernorm_epsilon (`float`, *optional*, defaults to 1e-5): + The epsilon to use in the layer normalization layers. + attn_separate (`boolean`, *optional*, defaults to `False`): + Whether or not to separate query-key-value to query, key, value in the Attention. + + Example: + + >>> # The PLUG model has 27B parameters and usually need to run on multiple GPUs. The example given + >>> # here only initializes a slice of the model on a single GPU. + >>> # Check out the [`~DistributedPipeline.__init__`] method to initialize entire PLUG model. + >>> from modelscope.models.nlp.plug import PlugNLGConfig, PlugModel + + >>> # Initializing a Plug configuration + >>> configuration = PlugNLGConfig() + + >>> # Initializing a model from the configuration + >>> model = PlugModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + """ + + model_type = 'plugNLG' + + def __init__(self, + vocab_size=21504, + original_vocab_size=21128, + hidden_size=8192, + num_hidden_layers=24, + dec_hidden_layers=6, + num_attention_heads=128, + intermediate_size=32768, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=2048, + type_vocab_size=3, + initializer_range=0.00707, + lr_decay_style='linear', + weight_decay=1e-2, + clip_grad=1.0, + warmup=0.01, + pre_ln=True, + fp16=True, + fp32_layernorm=True, + fp32_embedding=False, + fp32_tokentypes=False, + layernorm_epsilon=1e-12, + attn_separate=False, + **kwargs): + super().__init__(layer_norm_eps=layernorm_epsilon, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.lr_decay_style = lr_decay_style + self.weight_decay = weight_decay + self.clip_grad = clip_grad + self.warmup = warmup + self.pre_ln = pre_ln + self.fp16 = fp16 + self.fp32_layernorm = fp32_layernorm + self.fp32_embedding = fp32_embedding + self.layernorm_epsilon = layernorm_epsilon + self.fp32_tokentypes = fp32_tokentypes + self.dec_hidden_layers = dec_hidden_layers + self.attn_separate = attn_separate diff --git a/runtime/Lib/site_packages/models/nlp/plug/distributed_plug.py b/runtime/Lib/site_packages/models/nlp/plug/distributed_plug.py new file mode 100644 index 0000000000000000000000000000000000000000..6d1af8f49f297f9b617bad12a2a76e2521200d23 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/distributed_plug.py @@ -0,0 +1,255 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Dict + +import torch +from megatron_util import mpu, print_rank_0 +from megatron_util.fp16 import FP16_Module +from torch.nn import functional as F + +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.utils.logger import get_logger +from modelscope.utils.megatron_utils import init_megatron_util +from modelscope.utils.nlp.load_checkpoint import pre_load +from . import PlugModel +from .configuration import PlugNLGConfig + +logger = get_logger() + + +class DistributedPlug(TorchModel): + """ + The wapper class of PLUG Model to initialize parallel environment, load model weights, generate sentences. + Parameters: + model_dir (`str`, *required*): + Path to model damo/nlp_plug_text-generation_27B. + The model structure in model_dir should be like this: + model_dir + |_ config.json + |_ configuration.json + |_ ds_zero-offload_10B_config.json + |_ vocab.txt + |_ model <-- an empty directory + + Model binaries shall be downloaded separately to populate the model directory, so that + the model directory would contain the following binaries: + |_ model + |_ mp_rank_00_model_states.pt + |_ mp_rank_01_model_states.pt + |_ mp_rank_02_model_states.pt + |_ mp_rank_03_model_states.pt + |_ mp_rank_04_model_states.pt + |_ mp_rank_05_model_states.pt + |_ mp_rank_06_model_states.pt + |_ mp_rank_07_model_states.pt + rank (`int`, *required*): + Used to identify different GPUs in a tensor parallel environment. eg. The rank of GPU #0 is 0, and the + model file `mp_rank_00_model_states.pt` will be loaded on this GPU. + world_size (`int`, *required*, defaults to 8): + The parallel size in total. + model_parallel_size (`int`, *required*, defaults to 8): + The parallel size of model(tensor parallel). + master_ip (`str`, *required*): + The master IP, can usually be set to `"127.0.0.1"`, used as part of + [`~torch.distributed.init_process_group`] method parameter `init_method`. + `init_method` = `"tcp://{master_ip}:{master_port}"` + master_port (`str`, *required*): + The master port, can usually be set to `"29500"`, used as part of + [`~torch.distributed.init_process_group`] method parameter `init_method`. + `init_method` = `"tcp://{master_ip}:{master_port}"` + seed (`int`, *optional*, defaults to 42): + Random seed to control sampling. + """ + + def __init__(self, model_dir, rank, **kwargs): + super().__init__(model_dir, **kwargs) + self.rank = rank + self.model_cfg = kwargs + self.config = PlugNLGConfig.from_pretrained(model_dir) + + init_megatron_util(model_dir=model_dir, rank=rank) + + self.iteration = 0 + self.model = self.initialize_model(path_load_tag='model') + + def initialize_model(self, path_load_tag='model'): + """Build the model.""" + print_rank_0('Building Plug model. It will take a few minutes ...') + model = PlugModel(self.config) + + if mpu.get_data_parallel_rank() == 0: + logger.info( + ' > number of parameters on model parallel rank {}: {}'.format( + mpu.get_tensor_model_parallel_rank(), + sum([p.nelement() for p in model.parameters()]))) + + if self.config.deepspeed and self.config.fp16: + model.half() + + # GPU allocation. + model.cuda(torch.cuda.current_device()) + + # Fp16 conversion. + if self.config.fp16: + model = FP16_Module(model) + if self.config.fp32_embedding: + model.module.model.bert.embeddings.word_embeddings.float() + model.module.model.bert.embeddings.position_embeddings.float() + model.module.model.bert.embeddings.token_type_embeddings.float( + ) + if self.config.fp32_tokentypes: + model.module.model.bert.embeddings.token_type_embeddings.float( + ) + if self.config.fp32_layernorm: + for name, _module in model.named_modules(): + if 'LayerNorm' in name: + _module.float() + + load_model = pre_load( + mpu.get_tensor_model_parallel_rank(), + self.model_dir, + tag=path_load_tag) + model_dict = model.module.model.state_dict() + for key in load_model: + if key not in model_dict.keys(): + print_rank_0('Skip key: ' + key) + else: + print_rank_0('Loading key: ' + key) + model.module.model.load_state_dict(load_model, strict=False) + return model + + @staticmethod + def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')): + # This function has been mostly taken from huggingface conversational ai code at + # https://medium.com/huggingface/how-to-build-a-state-of-the-art- + # conversational-ai-with-transfer-learning-2d818ac26313 + + if top_k > 0: + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p > 0.0: + # convert to 1D + logits = logits.view(logits.size()[1]).contiguous() + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + indices_to_remove = sorted_indices[sorted_indices_to_remove] + logits[indices_to_remove] = filter_value + # going back to 2D + logits = logits.view(1, -1).contiguous() + return logits + + def forward(self, + input_tokens, + token_type_ids=None, + attention_mask=None, + target_tokens=None, + position_ids=None, + decode_attention_mask=None, + checkpoint_activations=False, + is_infer=False, + sequence_output=None, + parallel_output=True): + return self.model( + input_tokens, + token_type_ids, + attention_mask, + target_tokens, + position_ids, + decode_attention_mask, + checkpoint_activations=checkpoint_activations, + is_infer=is_infer, + sequence_output=sequence_output, + parallel_output=parallel_output) + + def generate(self, input: Dict[str, Tensor], out_length=128, *kwargs): + device = torch.cuda.current_device() + batch_size = input['input_ids'].shape[0] + tokens = input['input_ids'].view(1, -1).contiguous().to(device) + dec_input_ids = input['dec_input_ids'].to(device) + attention_mask = input['attention_mask'].to(device) + self.model.eval() + with torch.no_grad(): + # Only supports batch_size=1 + all_generate_tokens = [] + generate_tokens = [] + counter = 0 + sequence_output = None + vocab_size = self.config.original_vocab_size + sep_token_idx = 102 # index of [SEP] token in BertTokenizer + while counter < out_length: + if counter % 128 == 0 and counter != 0: + # Sliding window + generate_tokens.append(sep_token_idx) + start = (tokens == sep_token_idx).nonzero( + as_tuple=True)[-1] + if start + len(generate_tokens) >= 512: + tokens = torch.cat([ + tokens[:start], + torch.cuda.LongTensor(generate_tokens) + ], -1)[-512:] + else: + tokens[0][start:start + len(generate_tokens + )] = torch.cuda.LongTensor( + generate_tokens) + + attention_mask = (tokens != 0) + dec_input_ids = input['dec_input_ids'].to(device) + generate_tokens = [] + sequence_output = None + + position_ids = torch.full([batch_size, 1], + len(generate_tokens), + dtype=torch.long, + device=device) + _, logits, sequence_output = self.model( + tokens, + None, + attention_mask, + dec_input_ids, + attention_mask, + position_ids, + is_infer=True, + sequence_output=sequence_output, + parallel_output=False) + logits = logits[:, -1, :] + logits = logits / self.model_cfg['temperature'] + logits = self.top_k_logits( + logits, + top_k=self.model_cfg['top_k'], + top_p=self.model_cfg['top_p']) + log_probs = F.softmax(logits, dim=-1) + prev = torch.multinomial(log_probs, num_samples=1) + prev_token = prev[0].item() + if prev_token >= vocab_size: + prev_token = 100 + prev[0] = 100 + if prev_token == 102 and len(all_generate_tokens) > int( + max(1, out_length) * 0.8): + break + if prev_token == 102: + counter += 1 + continue + dec_input_ids = torch.cat([dec_input_ids, prev], dim=1) + generate_tokens.append(prev_token) + all_generate_tokens.append(prev_token) + counter += 1 + + generate_context = [] + for token in all_generate_tokens: + if generate_context and generate_context[ + -1] == 100 and token == 100: + continue + else: + generate_context.append(token) + return {'generate_context': generate_context} diff --git a/runtime/Lib/site_packages/models/nlp/plug/generator.py b/runtime/Lib/site_packages/models/nlp/plug/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..f4340fa3a9878729ad0a1987b6b23fafe8a4c95f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug/generator.py @@ -0,0 +1,225 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import torch + + +class TextGenerator(object): + + def __init__(self, + model, + vocab, + symbols, + global_scorer=None, + logger=None, + dump_beam=''): + self.alpha = 0.6 + + self.logger = logger + self.cuda = (torch.cuda.device_count() > 0) + + self.model = model + # TODO generator + self.vocab = vocab + self.symbols = symbols + self.start_token = 101 # ['[PAD]'] + self.end_token = 102 # '[PAD]'] + + self.global_scorer = global_scorer + self.beam_size = 5 + self.min_length = 5 + self.max_length = 384 + + self.dump_beam = dump_beam + + # for debugging + self.beam_trace = self.dump_beam != '' + self.beam_accum = None + + if self.beam_trace: + self.beam_accum = { + 'predicted_ids': [], + 'beam_parent_ids': [], + 'scores': [], + 'log_probs': [] + } + + def _build_target_tokens(self, pred): + tokens = [] + for tok in pred: + tok = int(tok) + tokens.append(tok) + if tokens[-1] == self.end_token: + tokens = tokens[:-1] + break + tokens = [t for t in tokens if t < len(self.vocab)] + tokens = self.vocab.DecodeIds(tokens).split(' ') + return tokens + + def tile(self, x, count, dim=0): + """ + Tiles x on dimension dim count times. + """ + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = x.view(batch, -1) \ + .transpose(0, 1) \ + .repeat(count, 1) \ + .transpose(0, 1) \ + .contiguous() \ + .view(*out_size) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + def translate_batch(self, encoder_inputs, fast=False): + with torch.no_grad(): + return self._fast_translate_batch( + encoder_inputs, self.max_length, min_length=self.min_length) + + def _fast_translate_batch(self, encoder_inputs, max_length, min_length=0): + + assert not self.dump_beam + + beam_size = self.beam_size + tokens, types, padding_mask = encoder_inputs + batch_size = tokens.size(0) + device = tokens.device + tmp_alive_seq = torch.full([batch_size, 1], + self.start_token, + dtype=torch.long, + device=device) + prediction_scores, dec_feat_seq, sequence_output = self.model( + tokens, + types, + padding_mask, + tmp_alive_seq, + None, + None, + checkpoint_activations=False, + is_infer=True, + parallel_output=False, + sequence_output=None) + src_features = sequence_output + + src_features = self.tile(src_features, beam_size, dim=0) + attention_mask = self.tile(padding_mask, beam_size, dim=0) + batch_offset = torch.arange( + batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, + batch_size * beam_size, + step=beam_size, + dtype=torch.long, + device=device) + alive_seq = torch.full([batch_size * beam_size, 1], + self.start_token, + dtype=torch.long, + device=device) + + # Give full probability to the first beam on the first step. + topk_log_probs = ( + torch.tensor( + [0.0] + [float('-inf')] * (beam_size - 1), + device=device).repeat(batch_size)) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results['predictions'] = [[] for _ in range(batch_size)] # noqa: F812 + results['scores'] = [[] for _ in range(batch_size)] # noqa: F812 + results['gold_score'] = [0] * batch_size + results['batch'] = [] + dec_attn_mask = None + dec_position_ids = None + + for step in range(max_length): + prediction_scores, dec_feat_seq, _ = self.model( + tokens, + types, + attention_mask, + alive_seq, + dec_position_ids, + dec_attn_mask, + checkpoint_activations=False, + is_infer=True, + parallel_output=False, + sequence_output=src_features) + + dec_feat_seq = dec_feat_seq[:, -1, :] + vocab_size = dec_feat_seq.size(-1) + log_probs = torch.log( + torch.softmax(dec_feat_seq.view(-1, vocab_size), dim=-1)) + + if step < min_length: + log_probs[:, self.end_token] = -1e20 + log_probs += topk_log_probs.view(-1).unsqueeze(1) + + alpha = self.alpha # global_scorer.alpha + length_penalty = ((5.0 + (step + 1)) / 6.0)**alpha + curr_scores = log_probs / length_penalty + + curr_scores = curr_scores.reshape(-1, beam_size * vocab_size) + topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1) + topk_log_probs = topk_scores * length_penalty + + # Resolve beam origin and true word ids. + topk_beam_index = topk_ids.div(vocab_size, rounding_mode='trunc') + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = ( + topk_beam_index + + beam_offset[:topk_beam_index.size(0)].unsqueeze(1)) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat([ + alive_seq.index_select(0, select_indices), + topk_ids.view(-1, 1) + ], -1) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(1) # self.end_token) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) # self.end_token) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(1) # self.end_token) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append( + (topk_scores[i, j], predictions[i, j, 1:])) + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted( + hypotheses[b], key=lambda x: x[0], reverse=True) + score, pred = best_hyp[0] + results['scores'][b].append(score) + results['predictions'][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished) \ + .view(-1, alive_seq.size(-1)) + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + attention_mask = attention_mask.index_select(0, select_indices) + + return results diff --git a/runtime/Lib/site_packages/models/nlp/plug_mental/__init__.py b/runtime/Lib/site_packages/models/nlp/plug_mental/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eb7a1b792526e6770d0c148b4bcee600205aea95 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug_mental/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import (PlugMentalModel, PlugMentalPreTrainedModel) + from .configuration import PlugMentalConfig + from .text_classification import PlugMentalForSequenceClassification +else: + _import_structure = { + 'backbone': ['PlugMentalModel', 'PlugMentalPreTrainedModel'], + 'configuration': ['PlugMentalConfig'], + 'text_classification': ['PlugMentalForSequenceClassification'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/plug_mental/adv_utils.py b/runtime/Lib/site_packages/models/nlp/plug_mental/adv_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eee441993e305d54ce720dc694ff610ac146102e --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug_mental/adv_utils.py @@ -0,0 +1,168 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def _symmetric_kl_div(logits1, logits2, attention_mask=None): + """ + Calclate two logits' the KL div value symmetrically. + :param logits1: The first logit. + :param logits2: The second logit. + :param attention_mask: An optional attention_mask which is used to mask some element out. + This is usually useful in token_classification tasks. + If the shape of logits is [N1, N2, ... Nn, D], the shape of attention_mask should be [N1, N2, ... Nn] + :return: The mean loss. + """ + labels_num = logits1.shape[-1] + KLDiv = nn.KLDivLoss(reduction='none') + loss = torch.sum( + KLDiv(nn.LogSoftmax(dim=-1)(logits1), + nn.Softmax(dim=-1)(logits2)), + dim=-1) + torch.sum( + KLDiv(nn.LogSoftmax(dim=-1)(logits2), + nn.Softmax(dim=-1)(logits1)), + dim=-1) + if attention_mask is not None: + loss = torch.sum( + loss * attention_mask) / torch.sum(attention_mask) / labels_num + else: + loss = torch.mean(loss) / labels_num + return loss + + +def compute_adv_loss(embedding, + model, + ori_logits, + ori_loss, + adv_grad_factor, + adv_bound=None, + sigma=5e-6, + **kwargs): + """ + Calculate the adv loss of the model. + :param embedding: Original sentense embedding + :param model: The model, or the forward function(including decoder/classifier), + accept kwargs as input, output logits + :param ori_logits: The original logits outputed from the model function + :param ori_loss: The original loss + :param adv_grad_factor: This factor will be multipled by the KL loss grad and then the result will be added to + the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value always be 1e-3~1e-7 + :param adv_bound: adv_bound is used to cut the top and the bottom bound of the produced embedding. + If not proveded, 2 * sigma will be used as the adv_bound factor + :param sigma: The std factor used to produce a 0 mean normal distribution. + If adv_bound not proveded, 2 * sigma will be used as the adv_bound factor + :param kwargs: the input param used in model function + :return: The original loss adds the adv loss + """ + adv_bound = adv_bound if adv_bound is not None else 2 * sigma + embedding_1 = embedding + embedding.data.new(embedding.size()).normal_( + 0, sigma) # 95% in +- 1e-5 + kwargs.pop('input_ids') + if 'inputs_embeds' in kwargs: + kwargs.pop('inputs_embeds') + with_attention_mask = False if 'with_attention_mask' not in kwargs else kwargs[ + 'with_attention_mask'] + attention_mask = kwargs['attention_mask'] + if not with_attention_mask: + attention_mask = None + if 'with_attention_mask' in kwargs: + kwargs.pop('with_attention_mask') + outputs = model(**kwargs, inputs_embeds=embedding_1) + v1_logits = outputs.logits + loss = _symmetric_kl_div(ori_logits, v1_logits, attention_mask) + emb_grad = torch.autograd.grad(loss, embedding_1)[0].data + emb_grad_norm = emb_grad.norm( + dim=2, keepdim=True, p=float('inf')).max( + 1, keepdim=True)[0] + is_nan = torch.any(torch.isnan(emb_grad_norm)) + if is_nan: + logger.warning('Nan occured when calculating adv loss.') + return ori_loss + emb_grad = emb_grad / (emb_grad_norm + 1e-6) + embedding_2 = embedding_1 + adv_grad_factor * emb_grad + embedding_2 = torch.max(embedding_1 - adv_bound, embedding_2) + embedding_2 = torch.min(embedding_1 + adv_bound, embedding_2) + outputs = model(**kwargs, inputs_embeds=embedding_2) + adv_logits = outputs.logits + adv_loss = _symmetric_kl_div(ori_logits, adv_logits, attention_mask) + return ori_loss + adv_loss + + +def compute_adv_loss_pair(embedding, + model, + start_logits, + end_logits, + ori_loss, + adv_grad_factor, + adv_bound=None, + sigma=5e-6, + **kwargs): + """ + Calculate the adv loss of the model. This function is used in the pair logits scenerio. + :param embedding: Original sentense embedding + :param model: The model, or the forward function(including decoder/classifier), + accept kwargs as input, output logits + :param start_logits: The original start logits outputed from the model function + :param end_logits: The original end logits outputed from the model function + :param ori_loss: The original loss + :param adv_grad_factor: This factor will be multipled by the KL loss grad and then the result will be added to + the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value always be 1e-3~1e-7 + :param adv_bound: adv_bound is used to cut the top and the bottom bound of the produced embedding. + If not proveded, 2 * sigma will be used as the adv_bound factor + :param sigma: The std factor used to produce a 0 mean normal distribution. + If adv_bound not proveded, 2 * sigma will be used as the adv_bound factor + :param kwargs: the input param used in model function + :return: The original loss adds the adv loss + """ + adv_bound = adv_bound if adv_bound is not None else 2 * sigma + embedding_1 = embedding + embedding.data.new(embedding.size()).normal_( + 0, sigma) # 95% in +- 1e-5 + kwargs.pop('input_ids') + if 'inputs_embeds' in kwargs: + kwargs.pop('inputs_embeds') + outputs = model(**kwargs, inputs_embeds=embedding_1) + v1_logits_start, v1_logits_end = outputs.logits + loss = _symmetric_kl_div(start_logits, + v1_logits_start) + _symmetric_kl_div( + end_logits, v1_logits_end) + loss = loss / 2 + emb_grad = torch.autograd.grad(loss, embedding_1)[0].data + emb_grad_norm = emb_grad.norm( + dim=2, keepdim=True, p=float('inf')).max( + 1, keepdim=True)[0] + is_nan = torch.any(torch.isnan(emb_grad_norm)) + if is_nan: + logger.warning('Nan occured when calculating pair adv loss.') + return ori_loss + emb_grad = emb_grad / emb_grad_norm + embedding_2 = embedding_1 + adv_grad_factor * emb_grad + embedding_2 = torch.max(embedding_1 - adv_bound, embedding_2) + embedding_2 = torch.min(embedding_1 + adv_bound, embedding_2) + outputs = model(**kwargs, inputs_embeds=embedding_2) + adv_logits_start, adv_logits_end = outputs.logits + adv_loss = _symmetric_kl_div(start_logits, + adv_logits_start) + _symmetric_kl_div( + end_logits, adv_logits_end) + return ori_loss + adv_loss diff --git a/runtime/Lib/site_packages/models/nlp/plug_mental/backbone.py b/runtime/Lib/site_packages/models/nlp/plug_mental/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..e8531f52968ac5476b33c543490256f6f3d08c22 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug_mental/backbone.py @@ -0,0 +1,1077 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Plug Mental model. mainly copied from :module:`~transformers.modeling_bert`""" + +import math +from dataclasses import dataclass +from typing import Optional, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from packaging import version +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import PlugMentalConfig + +logger = get_logger() + + +class SbertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + if version.parse(torch.__version__) > version.parse('1.6.0'): + self.register_buffer( + 'token_type_ids', + torch.zeros( + self.position_ids.size(), + dtype=torch.long, + device=self.position_ids.device), + persistent=False, + ) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + return_inputs_embeds=False): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs + # when its auto-generated, registered buffer helps users + # when tracing the model without passing token_type_ids, solves + # issue #5664 + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + if not return_inputs_embeds: + return embeddings + else: + return embeddings, inputs_embeds + + +class SbertSelfAttention(nn.Module): + + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in SbertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +class SbertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class SbertAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.self = SbertSelfAttention(config) + self.output = SbertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class SbertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class SbertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class SbertLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = SbertAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = SbertAttention(config) + self.intermediate = SbertIntermediate(config) + self.output = SbertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise ValueError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention ' + f'layers by setting `config.add_cross_attention=True`') + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class SbertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [SbertLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class SbertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class Adapter(nn.Module): + + def __init__(self, args, adapter_config): + super(Adapter, self).__init__() + self.adapter_config = adapter_config + self.args = args + self.down_project = nn.Linear( + self.adapter_config.project_hidden_size, + self.adapter_config.adapter_size, + ) + self.encoder = SbertEncoder(self.adapter_config) + self.up_project = nn.Linear(self.adapter_config.adapter_size, + adapter_config.project_hidden_size) + self.init_weights() + + def forward(self, hidden_states): + down_projected = self.down_project(hidden_states) + + input_shape = down_projected.size()[:-1] + attention_mask = torch.ones(input_shape, device=hidden_states.device) + # encoder_attention_mask = torch.ones( + # input_shape, device=hidden_states.device) + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + + if attention_mask.dim() == 2: + extended_attention_mask = attention_mask[:, None, None, :] + extended_attention_mask = extended_attention_mask.to( + dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + # if encoder_attention_mask.dim() == 3: + # encoder_extended_attention_mask = encoder_attention_mask[:, + # None, :, :] + # if encoder_attention_mask.dim() == 2: + # encoder_extended_attention_mask = encoder_attention_mask[:, None, + # None, :] + + head_mask = [None] * self.adapter_config.num_hidden_layers + encoder_outputs = self.encoder( + down_projected, + attention_mask=extended_attention_mask, + head_mask=head_mask) + + up_projected = self.up_project(encoder_outputs[0]) + return hidden_states + up_projected + + def init_weights(self): + self.down_project.weight.data.normal_( + mean=0.0, std=self.adapter_config.adapter_initializer_range) + self.down_project.bias.data.zero_() + self.up_project.weight.data.normal_( + mean=0.0, std=self.adapter_config.adapter_initializer_range) + self.up_project.bias.data.zero_() + + +class AdapterModel(nn.Module): + + def __init__( + self, + args, + pretrained_model_config, + ): + super(AdapterModel, self).__init__() + self.config = pretrained_model_config + self.args = args + self.adapter_size = self.args.adapter_size + + class AdapterConfig: + project_hidden_size: int = self.config.hidden_size + hidden_act: str = 'gelu' + adapter_size: int = self.adapter_size # 64 + adapter_initializer_range: float = 0.0002 + is_decoder: bool = False + attention_probs_dropout_prob: float = 0.1 + hidden_dropout_prob: float = 0.1 + hidden_size: int = 768 + initializer_range: float = 0.02 + intermediate_size: int = 3072 + layer_norm_eps: float = 1e-05 # 和Roberta large不一致,1e-12 + max_position_embeddings: int = 512 + num_attention_heads: int = 12 + num_hidden_layers: int = self.args.adapter_transformer_layers + output_attentions: bool = False + output_hidden_states: bool = False + torchscript: bool = False + type_vocab_size: int = 2 + vocab_size: int = 21128 + output_value_attentions: bool = False + chunk_size_feed_forward: int = 0 + add_cross_attention: bool = False + + self.adapter_skip_layers = self.args.adapter_skip_layers + # self.config.output_hidden_states=True + self.adapter_list = args.adapter_list + # self.adapter_list =[int(i) for i in self.adapter_list] + self.adapter_num = len(self.adapter_list) + # self.adapter = Adapter(args, AdapterConfig) + + self.adapter = nn.ModuleList( + [Adapter(args, AdapterConfig) for _ in range(self.adapter_num)]) + + # self.dense = nn.Linear(self.config.hidden_size * 2, self.config.hidden_size) + # self.dropout = nn.Dropout(self.config.hidden_dropout_prob) + + def forward( + self, + pretrained_model_outputs, + ): + # for i in pretrained_model_outputs: + # print(i.shape) + + outputs = pretrained_model_outputs + sequence_output = outputs.last_hidden_state + # pooler_output = outputs[1] + hidden_states = outputs.hidden_states + # num = len(hidden_states) + hidden_states_last = torch.zeros(sequence_output.size()).to( + sequence_output.device) + + adapter_hidden_states = [] + adapter_hidden_states_count = 0 + for i, adapter_module in enumerate(self.adapter): + fusion_state = hidden_states[ + self.adapter_list[i]] + hidden_states_last + hidden_states_last = adapter_module(fusion_state) + adapter_hidden_states.append(hidden_states_last) + adapter_hidden_states_count += 1 + if self.adapter_skip_layers >= 1: # if adapter_skip_layers>=1, skip connection + if adapter_hidden_states_count % self.adapter_skip_layers == 0: + hidden_states_last = hidden_states_last + adapter_hidden_states[ + int(adapter_hidden_states_count + / self.adapter_skip_layers)] + return hidden_states_last + + +class PlugMentalPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = PlugMentalConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, SbertEncoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + + if model_dir is None: + config = SbertConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + return model + + +@dataclass +class AttentionBackboneModelOutputWithEmbedding(AttentionBackboneModelOutput): + embedding_output: torch.FloatTensor = None + logits: Optional[Union[tuple, torch.FloatTensor]] = None + kwargs: dict = None + + +@MODELS.register_module(Tasks.backbone, module_name=Models.plug_mental) +class PlugMentalModel(PlugMentalPreTrainedModel): + """The PlugMental Model transformer outputting raw hidden-states without any specific head on top. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config (:class:`~modelscope.models.nlp.plug_mental.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the :obj:`is_decoder` argument of the configuration + set to :obj:`True`. To be used in a Seq2Seq model, the model needs to initialized with both :obj:`is_decoder` + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, + config: PlugMentalConfig, + add_pooling_layer=True, + **kwargs): + super().__init__(config) + self.config = config + + self.embeddings = SbertEmbeddings(config) + self.encoder = SbertEncoder(config) + + self.pooler = SbertPooler(config) if add_pooling_layer else None + + class AdapterModelConfig: + adapter_skip_layers = 0 + adapter_size = config.adapter_size + adapter_transformer_layers = config.adapter_transformer_layers + adapter_list = config.adapter_list + + self.adapter = AdapterModel(AdapterModelConfig, config) + self.adapter_dense = nn.Linear(self.config.hidden_size, + self.config.hidden_size) + self.com_dense = nn.Linear(self.config.hidden_size * 2, + self.config.hidden_size) + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` + indices into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` + under returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned + tensors for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used + in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple + having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up + decoding. + + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + + Returns: + Returns `modelscope.outputs.AttentionBackboneModelOutputWithEmbedding` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_plug-mental_backbone_base_std', task='backbone') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_plug-mental_backbone_base_std') + >>> print(model(**preprocessor('这是个测试'))) + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + + if token_type_ids is None: + if hasattr(self.embeddings, 'token_type_ids'): + buffered_token_type_ids = self.embeddings.token_type_ids[:, : + seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + batch_size, seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output, orignal_embeds = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + return_inputs_embeds=True, + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + adapter_model_output = self.adapter(encoder_outputs) + encoder_pooled_output = self.pooler(encoder_outputs.last_hidden_state) + # sequence_output = encoder_outputs[0] + # pooled_output = self.pooler( + # sequence_output) if self.pooler is not None else None + pooled_output = self.adapter_dense(encoder_pooled_output + + adapter_model_output[:, 0]) + sequence_output = self.com_dense( + torch.cat( + [encoder_outputs.last_hidden_state, adapter_model_output], + dim=2)) + + if not return_dict: + return (sequence_output, + pooled_output) + encoder_outputs[1:] + (orignal_embeds, ) + + return AttentionBackboneModelOutputWithEmbedding( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + embedding_output=orignal_embeds) diff --git a/runtime/Lib/site_packages/models/nlp/plug_mental/configuration.py b/runtime/Lib/site_packages/models/nlp/plug_mental/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..1157eeb9300213346f3a24a2e398f7f9a29eb9fc --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug_mental/configuration.py @@ -0,0 +1,142 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PLUG mental model configuration, mainly copied from :class:`~transformers.BertConfig` """ +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class PlugMentalConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration + of a :class:`~modelscope.models.nlp.plug_mental.PLugMentalModel`. + It is used to instantiate a PlugMental model according to the specified arguments. + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `optional`, defaults to 2): + The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (:obj:`str`, `optional`, defaults to :obj:`"absolute"`): + Type of position embedding. Choose one of :obj:`"absolute"`, :obj:`"relative_key"`, + :obj:`"relative_key_query"`. For positional embeddings use :obj:`"absolute"`. For more information on + :obj:`"relative_key"`, please refer to `Self-Attention with Relative Position Representations (Shaw et al.) + `__. For more information on :obj:`"relative_key_query"`, please refer to + `Method 4` in `Improve Transformer Models with Better Relative Position Embeddings (Huang et al.) + `__. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``config.is_decoder=True``. + classifier_dropout (:obj:`float`, `optional`): + The dropout ratio for the classification head. + adv_grad_factor (:obj:`float`, `optional`): This factor will be multiplied by the KL loss grad and then + the result will be added to the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value should between 1e-3~1e-7 + adv_bound (:obj:`float`, `optional`): adv_bound is used to cut the top and the bottom bound of + the produced embedding. + If not provided, 2 * sigma will be used as the adv_bound factor + sigma (:obj:`float`, `optional`): The std factor used to produce a 0 mean normal distribution. + If adv_bound not provided, 2 * sigma will be used as the adv_bound factor + """ + + model_type = 'plug-mental' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + adapter_size=768, + adapter_transformer_layers=1, + adapter_list=[11], + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + self.output_hidden_states = True + # adv_grad_factor, used in adv loss. + # Users can check adv_utils.py for details. + # if adv_grad_factor set to None, no adv loss will not applied to the model. + self.adv_grad_factor = None + # sigma value, used in adv loss. + self.sigma = 5e-6 if 'sigma' not in kwargs else kwargs['sigma'] + # adv_bound value, used in adv loss. + self.adv_bound = 2 * self.sigma if 'adv_bound' not in kwargs else kwargs[ + 'adv_bound'] + + # adapter config + self.adapter_size = adapter_size + self.adapter_transformer_layers = adapter_transformer_layers + self.adapter_list = adapter_list diff --git a/runtime/Lib/site_packages/models/nlp/plug_mental/text_classification.py b/runtime/Lib/site_packages/models/nlp/plug_mental/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..e09a912d7518327ba53947f887c92e3bf0207e90 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/plug_mental/text_classification.py @@ -0,0 +1,223 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .adv_utils import compute_adv_loss +from .backbone import PlugMentalModel, PlugMentalPreTrainedModel +from .configuration import PlugMentalConfig + +logger = logging.get_logger() + + +@MODELS.register_module( + Tasks.text_classification, module_name=Models.plug_mental) +@MODELS.register_module(Tasks.nli, module_name=Models.plug_mental) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.plug_mental) +@MODELS.register_module( + Tasks.sentence_similarity, module_name=Models.plug_mental) +@MODELS.register_module( + Tasks.zero_shot_classification, module_name=Models.plug_mental) +class PlugMentalForSequenceClassification(PlugMentalPreTrainedModel): + r"""PlugMental Model transformer with a sequence classification/regression head on top + (a linear layer on top of the pooled output) e.g. for GLUE tasks. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the text classification model of PlugMental, the preprocessor of this model + is `modelscope.preprocessors.TextClassificationTransformersPreprocessor`. + + Trainer: + This model is a normal PyTorch model, and can be trained by variable trainers, like EpochBasedTrainer, + NlpEpochBasedTrainer, or trainers from other frameworks. + The preferred trainer in ModelScope is NlpEpochBasedTrainer. + + Parameters: + config (:class:`~modelscope.models.nlp.plug_mental.PlugMentalConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + + def __init__(self, config: PlugMentalConfig, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + if self.config.adv_grad_factor is None: + logger.warning( + 'Adv parameters not set, skipping compute_adv_loss.') + + PlugMentalForSequenceClassification.base_model_prefix = getattr( + config, 'base_model_prefix', + PlugMentalForSequenceClassification.base_model_prefix) + setattr(self, self.base_model_prefix, PlugMentalModel(config)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + self.init_weights() + + def _forward_call(self, **kwargs): + outputs = self.base_model(**kwargs) + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + outputs['logits'] = logits + outputs.kwargs = kwargs + return outputs + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + *args, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` + indices into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See + ``attentions`` under returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under + returned tensors for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed + (Mean-Square loss), If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Returns: + Returns `modelscope.outputs.AttentionTextClassificationModelOutput` + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if not return_dict: + logger.error('Return tuple in sbert is not supported now.') + outputs = self._forward_call( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + return self.compute_loss(outputs, labels, **outputs.kwargs) + + def compute_loss(self, outputs, labels, **kwargs): + logits = outputs.logits + embedding_output = outputs.embedding_output + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = 'regression' + elif self.num_labels > 1 and (labels.dtype == torch.long + or labels.dtype == torch.int): + self.config.problem_type = 'single_label_classification' + else: + self.config.problem_type = 'multi_label_classification' + + if self.config.problem_type == 'regression': + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == 'single_label_classification': + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + if self.config.adv_grad_factor is not None and self.training: + loss = compute_adv_loss( + embedding=embedding_output, + model=self._forward_call, + ori_logits=logits, + ori_loss=loss, + adv_bound=self.config.adv_bound, + adv_grad_factor=self.config.adv_grad_factor, + sigma=self.config.sigma, + **kwargs) + elif self.config.problem_type == 'multi_label_classification': + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/polylm/__init__.py b/runtime/Lib/site_packages/models/nlp/polylm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..42c2f95039c85d6dd68fdf6392abedc2e9d8bac2 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/polylm/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .text_generation import PolyLMForTextGeneration +else: + _import_structure = {'text_generation': ['PolyLMForTextGeneration']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/polylm/text_generation.py b/runtime/Lib/site_packages/models/nlp/polylm/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..1881cf2bcdcad06f4e7098ed5045566817385eff --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/polylm/text_generation.py @@ -0,0 +1,55 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from collections import OrderedDict +from typing import Dict, Generator + +import torch +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer + +from modelscope.metainfo import Models +from modelscope.models.base import Tensor, TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import read_config +from modelscope.utils.streaming_output import StreamingOutputMixin + +__all__ = ['PolyLMForTextGeneration'] + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.polylm) +class PolyLMForTextGeneration(TorchModel, StreamingOutputMixin): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the text generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained( + model_dir, legacy=False, use_fast=False) + self.model = AutoModelForCausalLM.from_pretrained( + model_dir, device_map='auto', trust_remote_code=True) + self.model.eval() + + def forward(self, input: Dict[str, Tensor], **kwargs) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + """ + res = self.generate(input, **kwargs) + return res + + def generate(self, input: Dict[str, Tensor], + **kwargs) -> Dict[str, Tensor]: + device = self.model.device + inputs = self.tokenizer(input, return_tensors='pt') + outputs = self.model.generate( + inputs.input_ids.to(device), + attention_mask=inputs.attention_mask.to(device), + **kwargs) + pred = self.tokenizer.decode(outputs[0], skip_special_tokens=True) + return pred diff --git a/runtime/Lib/site_packages/models/nlp/ponet/__init__.py b/runtime/Lib/site_packages/models/nlp/ponet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..065cc260d0bbfaaa417e8c5dab9e3a41c29aa491 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/__init__.py @@ -0,0 +1,43 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import PoNetConfig + from .backbone import (PoNetModel, PoNetPreTrainedModel) + from .tokenization import PoNetTokenizer + from .fill_mask import PoNetForMaskedLM + from .document_segmentation import PoNetForDocumentSegmentation +else: + _import_structure = { + 'configuration': ['PoNetConfig'], + 'backbone': ['PoNetModel', 'PoNetPreTrainedModel'], + 'fill_mask': ['PoNetForMaskedLM'], + 'tokenization': ['PoNetTokenizer'], + 'document_segmentation': ['PoNetForDocumentSegmentation'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/ponet/backbone.py b/runtime/Lib/site_packages/models/nlp/ponet/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..aeb58bec09e9b501df9b8356a10046ab1f6a3838 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/backbone.py @@ -0,0 +1,900 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch PoNet model. """ + +import math +from distutils.version import LooseVersion + +import torch +import torch.utils.checkpoint +from packaging import version +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .configuration import PoNetConfig + +logger = get_logger() + +is_pytorch_12plus = LooseVersion(torch.__version__) >= LooseVersion('1.12.0') + +CLS_ID = 101 +EOS_ID = 102 + + +def segment_max(src, index, dim=1): + if is_pytorch_12plus: + out = torch.zeros_like(src).scatter_reduce( + dim, + index[:, :, None].expand_as(src), + src, + reduce='amax', + include_self=False) + else: + dummy_scatter_index = index[:, :, None].expand_as(src) + min_value = src.min() - 1 + dummpy_scatter_shape = (*src.shape[:-1], index.max() + 1, + src.shape[-1]) + dummy_scatter_index_expand = dummy_scatter_index.unsqueeze(-2).expand( + *dummpy_scatter_shape) + index_reconstruct_expand = torch.arange( + index.max() + 1, + device=src.device)[None, None, :, + None].expand(*dummpy_scatter_shape) + src_expand = src.unsqueeze(-2).expand(*dummpy_scatter_shape) + out, _ = src_expand.masked_scatter( + dummy_scatter_index_expand != index_reconstruct_expand, + torch.full_like(src_expand, min_value.item())).max(dim=1) + + dummy = index.unsqueeze(-1).expand(*index.shape[:2], out.size(-1)) + return torch.gather(out, dim, dummy).to(dtype=src.dtype) + + +def get_segment_index(input_ids, cls_id=CLS_ID, eos_id=EOS_ID): + mask = (input_ids == cls_id).to( + dtype=torch.long) + (input_ids == eos_id).to(dtype=torch.long) + mask = mask + torch.cat([torch.zeros_like(mask[:, 0:1]), mask[:, :-1]], + dim=1) + return mask.cumsum(dim=1) - 1 + + +def get_token_type_mask(input_ids, cls_id=CLS_ID, eos_id=EOS_ID): + mask = (input_ids == cls_id) | (input_ids == eos_id) + return mask + + +def get_win_max(hidden_states, kernel_size=3): + m = nn.MaxPool1d(kernel_size, stride=1, padding=kernel_size // 2) + out = m(hidden_states.permute(0, 2, 1)).permute(0, 2, 1) + return out + + +class PoNetEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + if version.parse(torch.__version__) > version.parse('1.6.0'): + self.register_buffer( + 'token_type_ids', + torch.zeros( + self.position_ids.size(), + dtype=torch.long, + device=self.position_ids.device), + persistent=False, + ) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class PoNetSelfAttention(nn.Module): + + def __init__(self, config): + super().__init__() + + self.dense_local = nn.Linear(config.hidden_size, config.hidden_size) + self.dense_segment = nn.Linear(config.hidden_size, config.hidden_size) + + self.num_attention_heads = config.num_attention_heads + self.clsgsepg = getattr(config, 'clsgsepg', True) + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.dense_q = nn.Linear(config.hidden_size, self.all_head_size) + self.dense_k = nn.Linear(config.hidden_size, self.all_head_size) + self.dense_o = nn.Linear(config.hidden_size, self.all_head_size) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) # bz, head, len, head_size + + def forward( + self, + hidden_states, + segment_index, + token_type_mask, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + + context_layer_q = self.transpose_for_scores( + self.dense_q(hidden_states)) + context_layer_k = self.transpose_for_scores( + self.dense_k(hidden_states)) + context_layer_v = context_layer_k + context_layer_o = self.transpose_for_scores( + self.dense_o(hidden_states)) + + if attention_mask is not None: + _attention_mask = (attention_mask.squeeze(1).unsqueeze(-1) < -1) + + if attention_mask is not None: + context_layer_q.masked_fill_(_attention_mask, 0.0) + q = context_layer_q.sum(dim=-2) / torch.ones_like( + _attention_mask).to(dtype=context_layer_q.dtype).masked_fill( + _attention_mask, 0.0).sum(dim=-2) + else: + q = context_layer_q.mean(dim=-2) + att = torch.einsum('bdh,bdlh -> bdl', q, context_layer_k) / math.sqrt( + context_layer_q.shape[-1]) + if attention_mask is not None: + att = att + attention_mask.squeeze(1) + att_prob = att.softmax(dim=-1) + v = torch.einsum('bdlh,bdl->bdh', context_layer_v, att_prob) + + context_layer_segment = self.dense_segment(hidden_states) + context_layer_local = self.dense_local(hidden_states) + if attention_mask is not None: + context_layer_local.masked_fill_( + _attention_mask.squeeze(1), -10000) + context_layer_segment.masked_fill_( + _attention_mask.squeeze(1), -10000) + + if self.clsgsepg: + # XXX: a trick to make sure the segment and local information will not leak + context_layer_local = get_win_max( + context_layer_local.masked_fill( + token_type_mask.unsqueeze(dim=-1), -10000)) + context_layer_segment = segment_max( + context_layer_segment, index=segment_index) + + context_layer_segment.masked_fill_( + token_type_mask.unsqueeze(dim=-1), 0.0) + context_layer_local.masked_fill_( + token_type_mask.unsqueeze(dim=-1), 0.0) + else: + context_layer_local = get_win_max(context_layer_local) + context_layer_segment = segment_max( + context_layer_segment, index=segment_index) + + context_layer_local = self.transpose_for_scores(context_layer_local) + context_layer_segment = self.transpose_for_scores( + context_layer_segment) + + context_layer = (v.unsqueeze(dim=-2) + context_layer_segment + ) * context_layer_o + context_layer_local + context_layer = context_layer.permute(0, 2, 1, 3).reshape( + *hidden_states.shape[:2], -1) + + if attention_mask is not None: + context_layer.masked_fill_(_attention_mask.squeeze(1), 0.0) + + outputs = (context_layer, + att_prob) if output_attentions else (context_layer, ) + return outputs + + +class PoNetSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class PoNetIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class PoNetOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class PoNetAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.self = PoNetSelfAttention(config) + self.output = PoNetSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + segment_index, + token_type_mask, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + segment_index, + token_type_mask, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class PoNetLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = PoNetAttention(config) + + config.is_decoder = False # XXX: Decoder is not yet impletemented. + self.is_decoder = config.is_decoder + + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + assert self.is_decoder, f'{self} should be used as a decoder model if cross attention is added' + self.crossattention = PoNetAttention(config) + self.intermediate = PoNetIntermediate(config) + self.output = PoNetOutput(config) + + def forward( + self, + hidden_states, + segment_index, + token_type_mask, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + segment_index, + token_type_mask, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + assert hasattr( + self, 'crossattention' + ), f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`' # noqa * + + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class PoNetEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [PoNetLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states, + segment_index, + token_type_mask, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if getattr(self.config, 'gradient_checkpointing', + False) and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting ' + '`use_cache=False`...') + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + segment_index, + token_type_mask, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + segment_index, + token_type_mask, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class PoNetPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class PoNetPreTrainedModel(TorchModel, PreTrainedModel): + """ + A base class to handle weights initialization and a simple interface for loading pretrained models. + """ + + config_class = PoNetConfig + base_model_prefix = 'ponet' + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + ponet_config = PoNetConfig(**kwargs) + model = cls(ponet_config) + else: + model = super( + Model, + cls).from_pretrained(pretrained_model_name_or_path=model_dir) + return model + + +@MODELS.register_module(Tasks.backbone, module_name=Models.ponet) +class PoNetModel(PoNetPreTrainedModel): + """The bare PoNet Model transformer outputting raw hidden-states without any specific head on top. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config (:class:`~modelscope.models.nlp.ponet.PoNetConfig`): + Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the :obj:`is_decoder` argument of the configuration + set to :obj:`True`. To be used in a Seq2Seq model, the model needs to initialized with both :obj:`is_decoder` + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True, **kwargs): + super().__init__(config, **kwargs) + self.config = config + + self.embeddings = PoNetEmbeddings(config) + self.encoder = PoNetEncoder(config) + + self.pooler = PoNetPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + segment_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.ponet.PoNetTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in + ``[0,1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0,config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` + indices into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. + encoder_hidden_states + (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used + in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` + with each tuple having 4 tensors of shape :obj: + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up + decoding. + + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + + Returns: + Returns `modelscope.outputs.AttentionBackboneModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_ponet_fill-mask_chinese-base', task='backbone') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_ponet_fill-mask_chinese-base') + >>> print(model(**preprocessor('这是个测试'))) + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + + segment_index = get_segment_index( + input_ids) if segment_ids is None else segment_ids + token_type_mask = get_token_type_mask(input_ids) + encoder_outputs = self.encoder( + embedding_output, + segment_index, + token_type_mask, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return AttentionBackboneModelOutput( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/ponet/configuration.py b/runtime/Lib/site_packages/models/nlp/ponet/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..b1ac0459ba174804d487bd6166e616e08a956be9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/configuration.py @@ -0,0 +1,115 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PoNet model configuration, mainly copied from :class:`~transformers.BertConfig` """ +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class PoNetConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration + of a :class:`~modelscope.models.nlp.ponet.PoNetModel`. + It is used to instantiate a PoNet model according to the specified arguments. + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `optional`, defaults to 2): + The vocabulary size of the :obj:`token_type_ids` passed. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (:obj:`str`, `optional`, defaults to :obj:`"absolute"`): + Type of position embedding. Choose one of :obj:`"absolute"`, :obj:`"relative_key"`, + :obj:`"relative_key_query"`. For positional embeddings use :obj:`"absolute"`. For more information on + :obj:`"relative_key"`, please refer to `Self-Attention with Relative Position Representations (Shaw et al.) + `__. For more information on :obj:`"relative_key_query"`, please refer to + `Method 4` in `Improve Transformer Models with Better Relative Position Embeddings (Huang et al.) + `__. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``config.is_decoder=True``. + classifier_dropout (:obj:`float`, `optional`): + The dropout ratio for the classification head. + clsgsepg (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not use a trick to make sure the segment and local information will not leak. + """ + model_type = 'ponet' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + clsgsepg=True, + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + self.clsgsepg = clsgsepg diff --git a/runtime/Lib/site_packages/models/nlp/ponet/document_segmentation.py b/runtime/Lib/site_packages/models/nlp/ponet/document_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..e2cb0812488991435c02d38788a256aad9ba4b59 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/document_segmentation.py @@ -0,0 +1,116 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Models +from modelscope.models.base import Model +from modelscope.models.builder import MODELS +from modelscope.models.nlp.bert import BertConfig +from modelscope.outputs import AttentionTokenClassificationModelOutput +from modelscope.utils.constant import Tasks +from .backbone import PoNetModel, PoNetPreTrainedModel +from .configuration import PoNetConfig + +__all__ = ['PoNetForDocumentSegmentation'] + + +@MODELS.register_module( + Tasks.document_segmentation, module_name=Models.ponet_for_ds) +@MODELS.register_module( + Tasks.extractive_summarization, module_name=Models.ponet_for_ds) +class PoNetForDocumentSegmentation(PoNetPreTrainedModel): + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + + self.ponet = PoNetModel(config, add_pooling_layer=False) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + segment_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the token classification loss. Indices should be in ``[0, ..., config.num_labels - + 1]``. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.ponet( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + segment_ids=segment_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits, ) + outputs[2:] + return ((loss, ) + output) if loss is not None else output + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + @classmethod + def _instantiate(cls, model_dir, model_config: Dict[str, Any], **kwargs): + if model_config['type'] == 'bert': + config = BertConfig.from_pretrained(model_dir, num_labels=2) + elif model_config['type'] == 'ponet': + config = PoNetConfig.from_pretrained(model_dir, num_labels=2) + else: + raise ValueError( + f'Expected config type bert and ponet, which is : {model_config["type"]}' + ) + model = super(Model, cls).from_pretrained(model_dir, config=config) + model.model_dir = model_dir + model.model_cfg = model_config + return model diff --git a/runtime/Lib/site_packages/models/nlp/ponet/fill_mask.py b/runtime/Lib/site_packages/models/nlp/ponet/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..053959b7465b355760f1b8bdb7e5423944d00ba4 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/fill_mask.py @@ -0,0 +1,253 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionFillMaskModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from .backbone import PoNetModel, PoNetPreTrainedModel + +logger = get_logger() + + +class PoNetPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class PoNetLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = PoNetPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class PoNetOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = PoNetLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +@MODELS.register_module(Tasks.fill_mask, module_name=Models.ponet) +class PoNetForMaskedLM(PoNetPreTrainedModel): + r"""PoNet Model with a `language modeling` head on top. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the fill_mask model of PoNet, the preprocessor of this model + is `modelscope.preprocessors.FillMaskPoNetPreprocessor`. + + Parameters: + config (:class:`~modelscope.models.nlp.ponet.PoNetConfig`): + Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config, **kwargs): + super().__init__(config) + + if config.is_decoder: + logger.warning( + 'If you want to use `PoNetForMaskedLM` make sure `config.is_decoder=False` for ' + 'bi-directional self-attention.') + + self.ponet = PoNetModel(config, add_pooling_layer=False) + self.cls = PoNetOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + segment_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`('batch_size, sequence_length')`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.ponet.PoNetTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`('batch_size, sequence_length')`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`('batch_size, sequence_length')`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`('batch_size, sequence_length')`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`('batch_size, sequence_length', hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` + indices into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + + Returns: + Returns `modelscope.outputs.AttentionFillMaskModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_ponet_fill-mask_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_ponet_fill-mask_chinese-base') + >>> # Call the model, return some tensors + >>> print(model(**preprocessor('你师父差得动你,你师父可[MASK]不动我。'))) + >>> # Call the pipeline + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('fill-mask', model=model, preprocessor=preprocessor) + >>> print(pipeline_ins('你师父差得动你,你师父可[MASK]不动我。')) + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.ponet( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + segment_ids=segment_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + if not return_dict: + output = (prediction_scores, ) + outputs[2:] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return AttentionFillMaskModelOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + input_ids=input_ids, + ) diff --git a/runtime/Lib/site_packages/models/nlp/ponet/tokenization.py b/runtime/Lib/site_packages/models/nlp/ponet/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..8cf9a035c75559bf7920c213124b97e1c9f955c6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/ponet/tokenization.py @@ -0,0 +1,156 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for PoNet """ + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from transformers.file_utils import PaddingStrategy +from transformers.models.bert.tokenization_bert import BertTokenizer +from transformers.tokenization_utils import BatchEncoding, EncodedInput + +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger + +logger = get_logger() + +VOCAB_FILES_NAMES = {'vocab_file': ModelFile.VOCAB_FILE} + +PRETRAINED_VOCAB_FILES_MAP = {'vocab_file': {}} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'nlp_ponet_fill-mask_chinese-base': 512, + 'nlp_ponet_fill-mask_english-base': 512, +} + +PRETRAINED_INIT_CONFIGURATION = { + 'nlp_ponet_fill-mask_chinese-base': { + 'do_lower_case': True + }, + 'nlp_ponet_fill-mask_english-base': { + 'do_lower_case': True + }, +} + + +class PoNetTokenizer(BertTokenizer): + r""" + Construct an PoNet tokenizer. Based on BertTokenizer. + + This tokenizer inherits from :class:`~transformers.BertTokenizer` which contains most of the main methods. + Users should refer to this superclass for more information regarding those methods. + + Refer to superclass :class:`~transformers.BertTokenizer` for usage examples and documentation concerning + parameters. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or + batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + >= 7.5 (Volta). + return_attention_mask: (optional) Set to False to avoid returning + attention mask (default: set to model specifics) + """ + # Load from model defaults + if return_attention_mask is None: + return_attention_mask = 'attention_mask' in self.model_input_names + + required_input = encoded_inputs[self.model_input_names[0]] + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + if needs_to_be_padded: + difference = max_length - len(required_input) + if self.padding_side == 'right': + if return_attention_mask: + encoded_inputs['attention_mask'] = [1] * len( + required_input) + [0] * difference + if 'token_type_ids' in encoded_inputs: + encoded_inputs['token_type_ids'] = ( + encoded_inputs['token_type_ids'] + + [self.pad_token_type_id] * difference) + if 'special_tokens_mask' in encoded_inputs: + encoded_inputs['special_tokens_mask'] = encoded_inputs[ + 'special_tokens_mask'] + [1] * difference + if 'segment_ids' in encoded_inputs: + encoded_inputs[ + 'segment_ids'] = encoded_inputs['segment_ids'] + [ + encoded_inputs['segment_ids'][-1] + 1 + ] * difference # noqa * + encoded_inputs[self.model_input_names[ + 0]] = required_input + [self.pad_token_id] * difference + elif self.padding_side == 'left': + if return_attention_mask: + encoded_inputs['attention_mask'] = [0] * difference + [ + 1 + ] * len(required_input) + if 'token_type_ids' in encoded_inputs: + encoded_inputs['token_type_ids'] = [ + self.pad_token_type_id + ] * difference + encoded_inputs['token_type_ids'] + if 'segment_ids' in encoded_inputs: + encoded_inputs['segment_ids'] = [encoded_inputs['segment_ids'][-1] + 1] * difference + \ + encoded_inputs['segment_ids'] # noqa * + if 'special_tokens_mask' in encoded_inputs: + encoded_inputs['special_tokens_mask'] = [ + 1 + ] * difference + encoded_inputs['special_tokens_mask'] + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + else: + raise ValueError('Invalid padding strategy:' + + str(self.padding_side)) + elif return_attention_mask and 'attention_mask' not in encoded_inputs: + encoded_inputs['attention_mask'] = [1] * len(required_input) + + return encoded_inputs diff --git a/runtime/Lib/site_packages/models/nlp/qwen/__init__.py b/runtime/Lib/site_packages/models/nlp/qwen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..55facff8416b314a77d3c4cecab9394254ba875d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import QWenConfig + from .text_generation import QWenForTextGeneration + from .backbone import QWenModel + from .tokenization import QWenTokenizer +else: + _import_structure = { + 'configuration': ['QWenConfig'], + 'backbone': ['QWenModel'], + 'tokenization': ['QWenTokenizer'], + 'text_generation': ['QWenForTextGeneration'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/qwen/backbone.py b/runtime/Lib/site_packages/models/nlp/qwen/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..ea03bf497bada098c89d104641d1615f50833aa0 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/backbone.py @@ -0,0 +1,846 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import importlib +import math +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.cuda.amp import autocast +from torch.nn import CrossEntropyLoss +from transformers import (GenerationConfig, PreTrainedTokenizer, + StoppingCriteriaList) +from transformers.generation.logits_process import LogitsProcessorList +from transformers.generation.utils import GenerateOutput +from transformers.modeling_outputs import (BaseModelOutputWithPast, + CausalLMOutputWithPast) +from transformers.modeling_utils import PreTrainedModel +from transformers.trainer_utils import set_seed +from transformers.utils import (ModelOutput, add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, logging) +from transformers.utils.model_parallel_utils import (assert_device_map, + get_device_map) + +from modelscope import Model, TorchModel +from modelscope.metainfo import Models +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from ... import MODELS +from .configuration import QWenConfig +from .qwen_generation_utils import (HistoryType, StopWordsLogitsProcessor, + decode_tokens, get_stop_words_ids, + make_context) + +if TYPE_CHECKING: + from transformers.generation.streamers import BaseStreamer + +try: + from einops import rearrange +except ImportError: + rearrange = None + +try: + from flash_attn.layers.rotary import apply_rotary_emb_func + from einops import rearrange + + use_flash_rotary = True +except ImportError: + use_flash_rotary = False + print( + 'Warning: import flash_attn rotary fail, please install FlashAttention rotary to get better performance ' + 'https://github.com/Dao-AILab/flash-attention/tree/main/csrc/rotary') + +try: + from flash_attn.ops.rms_norm import rms_norm +except ImportError: + rms_norm = None + print( + 'Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get better performance ' + 'https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm' + ) + +logger = get_logger() + +_CHECKPOINT_FOR_DOC = 'qwen-7b' +_CONFIG_FOR_DOC = 'QWenConfig' + +QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ['qwen-7b'] + +try: + from flash_attn.flash_attn_interface import flash_attn_unpadded_func +except ImportError: + flash_attn_unpadded_func = None + print('Warning: import flash_attn fail, please install FlashAttention ' + 'https://github.com/Dao-AILab/flash-attention') + + +class FlashSelfAttention(torch.nn.Module): + + def __init__( + self, + causal=False, + softmax_scale=None, + attention_dropout=0.0, + ): + super().__init__() + assert flash_attn_unpadded_func is not None, ( + 'Please install FlashAttention first, ' + 'e.g., with pip install flash-attn') + assert (rearrange is not None + ), 'Please install einops first, e.g., with pip install einops' + self.causal = causal + self.softmax_scale = softmax_scale + self.dropout_p = attention_dropout + + def forward(self, q, k, v): + assert all( + (i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v))) + assert all((i.is_cuda for i in (q, k, v))) + batch_size, seqlen_q = q.shape[0], q.shape[1] + seqlen_k = k.shape[1] + q, k, v = [rearrange(x, 'b s ... -> (b s) ...') for x in [q, k, v]] + cu_seqlens_q = torch.arange( + 0, + (batch_size + 1) * seqlen_q, + step=seqlen_q, + dtype=torch.int32, + device=q.device, + ) + + if self.training: + assert seqlen_k == seqlen_q + + is_causal = self.causal + cu_seqlens_k = cu_seqlens_q + else: + is_causal = seqlen_q == seqlen_k + cu_seqlens_k = torch.arange( + 0, + (batch_size + 1) * seqlen_k, + step=seqlen_k, + dtype=torch.int32, + device=q.device, + ) + self.dropout_p = 0 + output = flash_attn_unpadded_func( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + seqlen_q, + seqlen_k, + self.dropout_p, + softmax_scale=self.softmax_scale, + causal=is_causal, + ) + + output = rearrange(output, '(b s) ... -> b s ...', b=batch_size) + return output + + +class QWenAttention(nn.Module): + + def __init__(self, config, layer_number=None): + super().__init__() + + max_positions = config.max_position_embeddings + self.register_buffer( + 'bias', + torch.tril( + torch.ones((max_positions, max_positions), + dtype=torch.bool)).view(1, 1, max_positions, + max_positions), + persistent=False, + ) + self.register_buffer( + 'masked_bias', torch.tensor(-1e4), persistent=False) + self.layer_number = max(1, layer_number) + self.params_dtype = config.params_dtype + self.seq_length = config.seq_length + + self.hidden_size = config.hidden_size + self.split_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + + self.use_flash_attn = config.use_flash_attn + self.scale_attn_weights = True + + self.layer_idx = None + + self.projection_size = config.kv_channels * config.num_attention_heads + + assert self.projection_size % config.num_attention_heads == 0 + self.hidden_size_per_attention_head = ( + self.projection_size // config.num_attention_heads) + + self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size) + + self.c_proj = nn.Linear( + config.hidden_size, self.projection_size, bias=not config.no_bias) + + self.is_fp32 = not (config.bf16 or config.fp16) + if self.use_flash_attn and flash_attn_unpadded_func is not None and not self.is_fp32: + self.core_attention_flash = FlashSelfAttention( + causal=True, attention_dropout=config.attn_pdrop) + + self.bf16 = config.bf16 + + if config.rotary_pct == 1.0: + self.rotary_ndims = None + else: + assert config.rotary_pct < 1 + self.rotary_ndims = int(self.hidden_size_per_attention_head + * config.rotary_pct) + dim = ( + self.rotary_ndims if self.rotary_ndims is not None else + self.hidden_size_per_attention_head) + self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base) + + self.use_dynamic_ntk = config.use_dynamic_ntk + self.use_logn_attn = config.use_logn_attn + + logn_list = [ + math.log(i, self.seq_length) if i > self.seq_length else 1 + for i in range(1, 32768) + ] + self.logn_tensor = torch.Tensor(logn_list)[None, :, None, None] + self._ntk_cached = 1.0 + + self.attn_dropout = nn.Dropout(config.attn_pdrop) + + def _attn(self, query, key, value, attention_mask=None, head_mask=None): + attn_weights = torch.matmul(query, key.transpose(-1, -2)) + + if self.scale_attn_weights: + attn_weights = attn_weights / torch.full( + [], + value.size(-1)**0.5, + dtype=attn_weights.dtype, + device=attn_weights.device, + ) + + query_length, key_length = query.size(-2), key.size(-2) + causal_mask = self.bias[:, :, key_length + - query_length:key_length, :key_length] + mask_value = torch.finfo(attn_weights.dtype).min + mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to( + attn_weights.device) + attn_weights = torch.where(causal_mask, + attn_weights.to(attn_weights.dtype), + mask_value) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + attn_weights = attn_weights.type(value.dtype) + attn_weights = self.attn_dropout(attn_weights) + + if head_mask is not None: + attn_weights = attn_weights * head_mask + + attn_output = torch.matmul(attn_weights, value) + attn_output = attn_output.transpose(1, 2) + + return attn_output, attn_weights + + def _upcast_and_reordered_attn(self, + query, + key, + value, + attention_mask=None, + head_mask=None): + bsz, num_heads, q_seq_len, dk = query.size() + _, _, k_seq_len, _ = key.size() + + attn_weights = torch.empty( + bsz * num_heads, + q_seq_len, + k_seq_len, + dtype=torch.float32, + device=query.device, + ) + + scale_factor = 1.0 + if self.scale_attn_weights: + scale_factor /= float(value.size(-1))**0.5 + + with autocast(enabled=False): + q, k = query.reshape(-1, q_seq_len, + dk), key.transpose(-1, -2).reshape( + -1, dk, k_seq_len) + attn_weights = torch.baddbmm( + attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor) + attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, + k_seq_len) + + query_length, key_length = query.size(-2), key.size(-2) + causal_mask = self.bias[:, :, key_length + - query_length:key_length, :key_length] + mask_value = torch.finfo(attn_weights.dtype).min + mask_value = torch.tensor( + mask_value, dtype=attn_weights.dtype).to(attn_weights.device) + attn_weights = torch.where(causal_mask, attn_weights, mask_value) + + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if attn_weights.dtype != torch.float32: + raise RuntimeError( + 'Error with upcasting, attn_weights does not have dtype torch.float32' + ) + attn_weights = attn_weights.type(value.dtype) + attn_weights = self.attn_dropout(attn_weights) + + if head_mask is not None: + attn_weights = attn_weights * head_mask + + attn_output = torch.matmul(attn_weights, value) + + return attn_output, attn_weights + + def _split_heads(self, tensor, num_heads, attn_head_size): + new_shape = tensor.size()[:-1] + (num_heads, attn_head_size) + tensor = tensor.view(new_shape) + return tensor + + def _merge_heads(self, tensor, num_heads, attn_head_size): + tensor = tensor.contiguous() + new_shape = tensor.size()[:-2] + (num_heads * attn_head_size, ) + return tensor.view(new_shape) + + def forward( + self, + hidden_states: Optional[Tuple[torch.FloatTensor]], + layer_past: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + ): + + mixed_x_layer = self.c_attn(hidden_states) + query, key, value = mixed_x_layer.split(self.split_size, dim=2) + + query = self._split_heads(query, self.num_heads, self.head_dim) + key = self._split_heads(key, self.num_heads, self.head_dim) + value = self._split_heads(value, self.num_heads, self.head_dim) + + kv_seq_len = hidden_states.size()[1] + if layer_past: + kv_seq_len += layer_past[0].shape[1] + if (self.use_dynamic_ntk and kv_seq_len == hidden_states.size()[1] + and not self.training): + context_value = math.log(kv_seq_len / self.seq_length, 2) + 1 + ntk_alpha = 2**math.ceil(context_value) - 1 + ntk_alpha = max(ntk_alpha, 1) + self._ntk_cached = ntk_alpha + else: + ntk_alpha = self._ntk_cached + rotary_pos_emb = self.rotary_emb( + kv_seq_len, ntk_alpha=ntk_alpha).to(hidden_states.device) + + if rotary_pos_emb is not None: + if isinstance(rotary_pos_emb, tuple): + rotary_pos_emb = rotary_pos_emb + else: + rotary_pos_emb = (rotary_pos_emb, ) * 2 + + if rotary_pos_emb is not None: + q_pos_emb, k_pos_emb = rotary_pos_emb + cur_len = query.shape[1] + q_pos_emb = q_pos_emb[:, -cur_len:, :, :] + k_pos_emb = k_pos_emb[:, -cur_len:, :, :] + query = apply_rotary_pos_emb(query, q_pos_emb) + key = apply_rotary_pos_emb(key, k_pos_emb) + + if layer_past is not None: + past_key, past_value = layer_past[0], layer_past[1] + key = torch.cat((past_key, key), dim=1) + value = torch.cat((past_value, value), dim=1) + + if use_cache: + present = (key, value) + else: + present = None + + if self.use_logn_attn and not self.training: + if self.logn_tensor.device != query.device: + self.logn_tensor = self.logn_tensor.to( + query.device).type_as(query) + seq_start = key.size(1) - query.size(1) + seq_end = key.size(1) + logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :] + query = query * logn_tensor.expand_as(query) + + if self.use_flash_attn and flash_attn_unpadded_func is not None and not self.is_fp32 and query.is_cuda: + q, k, v = query, key, value + context_layer = self.core_attention_flash(q, k, v) + + context_layer = rearrange(context_layer, + 'b s h d -> b s (h d)').contiguous() + else: + query = query.permute(0, 2, 1, 3) + key = key.permute(0, 2, 1, 3) + value = value.permute(0, 2, 1, 3) + attn_output, attn_weight = self._attn(query, key, value, + attention_mask, head_mask) + context_layer = self._merge_heads(attn_output, self.num_heads, + self.head_dim) + + attn_output = self.c_proj(context_layer) + outputs = (attn_output, present) + if output_attentions: + if self.use_flash_attn and flash_attn_unpadded_func is not None and not self.is_fp32: + raise ValueError( + 'Cannot output attentions while using flash-attn') + else: + outputs += (attn_weight, ) + + return outputs + + +class QWenMLP(nn.Module): + + def __init__(self, config): + super().__init__() + self.w1 = nn.Linear( + config.hidden_size, + config.ffn_hidden_size // 2, + bias=not config.no_bias) + self.w2 = nn.Linear( + config.hidden_size, + config.ffn_hidden_size // 2, + bias=not config.no_bias) + ff_dim_in = config.ffn_hidden_size // 2 + self.c_proj = nn.Linear( + ff_dim_in, config.hidden_size, bias=not config.no_bias) + + def forward(self, hidden_states): + a1 = self.w1(hidden_states) + a2 = self.w2(hidden_states) + intermediate_parallel = a1 * F.silu(a2) + output = self.c_proj(intermediate_parallel) + return output + + +class QWenBlock(nn.Module): + + def __init__(self, config, layer_idx=None, num_expert=1): + super().__init__() + self.num_expert = num_expert + self.layer_number = layer_idx + self.apply_residual_connection_post_layernorm = ( + config.apply_residual_connection_post_layernorm) + hidden_size = config.hidden_size + self.apply_residual_connection_post_layernorm = ( + config.apply_residual_connection_post_layernorm) + self.bf16 = config.bf16 + + self.ln_1 = RMSNorm( + hidden_size, + eps=config.layer_norm_epsilon, + ) + self.attn = QWenAttention(config, layer_number=layer_idx) + self.ln_2 = RMSNorm( + hidden_size, + eps=config.layer_norm_epsilon, + ) + + self.mlp = QWenMLP(config) + + def forward( + self, + hidden_states: Optional[Tuple[torch.FloatTensor]], + layer_past: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = False, + ): + layernorm_output = self.ln_1(hidden_states) + + attn_outputs = self.attn( + layernorm_output, + layer_past=layer_past, + attention_mask=attention_mask, + head_mask=head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + attn_output = attn_outputs[0] + + outputs = attn_outputs[1:] + + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + layernorm_input = attn_output + residual + + layernorm_output = self.ln_2(layernorm_input) + + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + mlp_output = self.mlp(layernorm_output) + hidden_states = residual + mlp_output + + if use_cache: + outputs = (hidden_states, ) + outputs + else: + outputs = (hidden_states, ) + outputs[1:] + + return outputs + + +class QWenPreTrainedModel(TorchModel, PreTrainedModel): + config_class = QWenConfig + base_model_prefix = 'transformer' + is_parallelizable = False + supports_gradient_checkpointing = True + _no_split_modules = ['QWenBlock'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights.""" + if isinstance(module, nn.Linear): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, RMSNorm): + module.weight.data.fill_(1.0) + + for name, p in module.named_parameters(): + if name == 'c_proj.weight': + p.data.normal_( + mean=0.0, + std=(self.config.initializer_range + / math.sqrt(2 * self.config.n_layer)), + ) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, QWenModel): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + config = QWenConfig(**kwargs) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **kwargs) + model.model_dir = model_dir + return model + + +@MODELS.register_module(Tasks.backbone, module_name=Models.qwen_7b) +class QWenModel(QWenPreTrainedModel): + _keys_to_ignore_on_load_missing = ['attn.masked_bias'] + + def __init__(self, config): + super().__init__(config) + self.vocab_size = config.padded_vocab_size + self.num_hidden_layers = config.num_hidden_layers + self.embed_dim = config.hidden_size + + max_sequence_length = config.max_position_embeddings + self.position_embedding_type = config.pos_emb + self.gradient_checkpointing = False + + if self.position_embedding_type == 'learned': + self.wpe = nn.Embedding(max_sequence_length, self.embed_dim) + self.init_method(self.position_embeddings.weight) + self._position_embeddings_key = 'position_embeddings' + self.init_method(self.position_embeddings.weight) + else: + self.wpe = None + self._position_embeddings_key = '' + + self.wte = nn.Embedding(self.vocab_size, self.embed_dim) + + self.drop = nn.Dropout(config.embd_pdrop) + self.h = nn.ModuleList([ + QWenBlock( + config, + layer_idx=i, + ) for i in range(config.num_hidden_layers) + ]) + self.ln_f = RMSNorm( + self.embed_dim, + eps=config.layer_norm_epsilon, + ) + + self.post_init() + + def get_input_embeddings(self): + return self.wte + + def set_input_embeddings(self, new_embeddings): + self.wte = new_embeddings + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + output_attentions = ( + output_attentions if output_attentions is not None else + self.config.output_attentions) + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = ( + return_dict + if return_dict is not None else self.config.use_return_dict) + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + batch_size = input_ids.shape[0] + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size = inputs_embeds.shape[0] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if token_type_ids is not None: + token_type_ids = token_type_ids.view(-1, input_shape[-1]) + if position_ids is not None: + position_ids = position_ids.view(-1, input_shape[-1]) + + if past_key_values is None: + past_length = 0 + past_key_values = tuple([None] * len(self.h)) + else: + past_length = past_key_values[0][0].size(-2) + + if position_ids is None: + position_ids = torch.arange( + past_length, + input_shape[-1] + past_length, + dtype=torch.long, + device=device, + ) + position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) + + if attention_mask is not None: + if batch_size <= 0: + raise ValueError('batch_size has to be defined and > 0') + attention_mask = attention_mask.view(batch_size, -1) + attention_mask = attention_mask[:, None, None, :] + attention_mask = attention_mask.to(dtype=self.dtype) + attention_mask = (1.0 - attention_mask) * torch.finfo( + self.dtype).min + + encoder_attention_mask = None + head_mask = self.get_head_mask(head_mask, self.config.n_layer) + + if inputs_embeds is None: + inputs_embeds = self.wte(input_ids) + hidden_states = inputs_embeds + if self.wpe is not None: + position_embeds = self.wpe(position_ids) + hidden_states = hidden_states + position_embeds + + hidden_states = self.drop(hidden_states) + output_shape = input_shape + (hidden_states.size(-1), ) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + presents = () if use_cache else None + all_self_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, use_cache, output_attentions) + + return custom_forward + + outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + None, + attention_mask, + head_mask[i], + encoder_hidden_states, + encoder_attention_mask, + ) + else: + outputs = block( + hidden_states, + layer_past=layer_past, + attention_mask=attention_mask, + head_mask=head_mask[i], + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + hidden_states = outputs[0] + if use_cache is True: + presents = presents + ( + outputs[2 if output_attentions else 1], ) + + if output_attentions: + all_self_attentions = all_self_attentions + (outputs[1], ) + + hidden_states = self.ln_f(hidden_states) + hidden_states = hidden_states.view(output_shape) + + if not return_dict: + return tuple(v + for v in [hidden_states, presents, all_hidden_states] + if v is not None) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +class RotaryEmbedding(torch.nn.Module): + + def __init__(self, dim, base=10000): + super().__init__() + self.dim = dim + self.base = base + self.inv_freq = 1.0 / (base**(torch.arange(0, dim, 2).float() / dim)) + if importlib.util.find_spec('einops') is None: + raise RuntimeError('einops is required for Rotary Embedding') + + self._rotary_pos_emb_cache = None + self._seq_len_cached = 0 + self._ntk_alpha_cached = 1.0 + + def update_rotary_pos_emb_cache(self, + max_seq_len, + offset=0, + ntk_alpha=1.0): + seqlen = max_seq_len + offset + if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached: + base = self.base * ntk_alpha**(self.dim / (self.dim - 2)) + ''' + self.inv_freq = 1.0 / ( + base**(torch.arange( + 0, self.dim, 2, device=self.inv_freq.device).float() + / self.dim)) + ''' + self.inv_freq = torch.arange( + 0, self.dim, 2, device=self.inv_freq.device).float() / self.dim + self.inv_freq = 1.0 / (base**self.inv_freq) + self._seq_len_cached = seqlen + self._ntk_alpha_cached = ntk_alpha + seq = torch.arange(seqlen, device=self.inv_freq.device) + freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq) + emb = torch.cat((freqs, freqs), dim=-1) + from einops import rearrange + + self._rotary_pos_emb_cache = rearrange(emb, 'n d -> 1 n 1 d') + + def forward(self, max_seq_len, offset=0, ntk_alpha=1.0): + self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha) + return self._rotary_pos_emb_cache[:, offset:offset + max_seq_len] + + +def _rotate_half(x): + from einops import rearrange + + x = rearrange(x, '... (j d) -> ... j d', j=2) + x1, x2 = x.unbind(dim=-2) + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(t, freqs, use_flash_rotary=False): + if use_flash_rotary: + t_ = t.float() + freqs = freqs.squeeze(0).squeeze(1) + cos = freqs[:, :freqs.shape[-1] // 2].cos() + sin = freqs[:, :freqs.shape[-1] // 2].sin() + output = apply_rotary_emb_func(t_, cos, sin).type_as(t) + return output + else: + rot_dim = freqs.shape[-1] + t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:] + t_ = t_.float() + t_pass_ = t_pass_.float() + t_ = (t_ * freqs.cos()) + (_rotate_half(t_) * freqs.sin()) + return torch.cat((t_, t_pass_), dim=-1).type_as(t) + + +class RMSNorm(torch.nn.Module): + + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + if rms_norm is not None and x.is_cuda: + return rms_norm(x, self.weight, self.eps) + else: + output = self._norm(x.float()).type_as(x) + return output * self.weight diff --git a/runtime/Lib/site_packages/models/nlp/qwen/configuration.py b/runtime/Lib/site_packages/models/nlp/qwen/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..fa1bbcbcf464caa4663399b7b0c0ad2abdfc2e7f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/configuration.py @@ -0,0 +1,79 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from transformers import PretrainedConfig + +from modelscope.utils.logger import get_logger + +logger = get_logger() + +QWEN_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + + +class QWenConfig(PretrainedConfig): + model_type = 'qwen' + keys_to_ignore_at_inference = ['past_key_values'] + attribute_map = { + 'hidden_size': 'n_embd', + 'num_attention_heads': 'n_head', + 'max_position_embeddings': 'n_positions', + 'num_hidden_layers': 'n_layer', + } + + def __init__( + self, + vocab_size=151851, + n_embd=4096, + n_layer=32, + n_head=32, + n_inner=None, + embd_pdrop=0.0, + attn_pdrop=0.0, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + scale_attn_weights=True, + use_cache=True, + eos_token_id=151643, + apply_residual_connection_post_layernorm=False, + bf16=True, + kv_channels=128, + rotary_pct=1.0, + rotary_emb_base=10000, + use_dynamic_ntk=False, + use_logn_attn=False, + use_flash_attn=True, + ffn_hidden_size=22016, + no_bias=True, + tie_word_embeddings=False, + **kwargs, + ): + self.eos_token_id = eos_token_id + super().__init__( + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs) + + self.vocab_size = vocab_size + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.n_inner = n_inner + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.scale_attn_weights = scale_attn_weights + self.use_cache = use_cache + self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm + self.bf16 = bf16 + self.kv_channels = kv_channels + self.rotary_pct = rotary_pct + self.rotary_emb_base = rotary_emb_base + self.use_dynamic_ntk = use_dynamic_ntk + self.use_logn_attn = use_logn_attn + self.use_flash_attn = use_flash_attn + self.ffn_hidden_size = ffn_hidden_size + self.no_bias = no_bias + self.tie_word_embeddings = tie_word_embeddings diff --git a/runtime/Lib/site_packages/models/nlp/qwen/qwen_generation_utils.py b/runtime/Lib/site_packages/models/nlp/qwen/qwen_generation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc9df5dfd3c808a2958372600699a746a44286f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/qwen_generation_utils.py @@ -0,0 +1,409 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +"""Generation support.""" +from typing import Iterable, List, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import PreTrainedTokenizer +from transformers.generation import LogitsProcessor + +from modelscope.utils.logger import get_logger + +logger = get_logger() + +# Types. +HistoryType = List[Tuple[str, str]] +TokensType = List[int] +BatchTokensType = List[List[int]] + + +def pad_batch(batch: BatchTokensType, pad_id: int, + seq_length: int) -> BatchTokensType: + for tokens in batch: + context_length = len(tokens) + if context_length < seq_length: + tokens.extend([pad_id] * (seq_length - context_length)) + return batch + + +def get_ltor_masks_and_position_ids( + data, + eod_token, + reset_position_ids, + reset_attention_mask, + eod_mask_loss, +): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size() + + # Attention mask (lower triangular). + if reset_attention_mask: + att_mask_batch = micro_batch_size + else: + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), + device=data.device)).view(att_mask_batch, 1, seq_length, + seq_length) + + # Loss mask. + loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device) + if eod_mask_loss: + loss_mask[data == eod_token] = 0.0 + + # Position ids. + position_ids = torch.arange( + seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data) + # We need to clone as the ids will be modifed based on batch index. + if reset_position_ids: + position_ids = position_ids.clone() + + if reset_position_ids or reset_attention_mask: + # Loop through the batches: + for b in range(micro_batch_size): + + # Find indecies where EOD token is. + eod_index = position_ids[b, data[b] == eod_token] + # Detach indecies from positions if going to modify positions. + if reset_position_ids: + eod_index = eod_index.clone() + + # Loop through EOD indecies: + prev_index = 0 + for j in range(eod_index.size()[0]): + i = eod_index[j] + # Mask attention loss. + if reset_attention_mask: + attention_mask[b, 0, (i + 1):, :(i + 1)] = 0 + # Reset positions. + if reset_position_ids: + position_ids[b, (i + 1):] -= i + 1 - prev_index + prev_index = i + 1 + + # Convert attention mask to binary: + attention_mask = attention_mask < 0.5 + + return attention_mask, loss_mask, position_ids + + +def get_batch(context_tokens: torch.LongTensor, eod_id: int): + """Generate batch from context tokens.""" + # Move to GPU. + tokens = context_tokens.contiguous().to(context_tokens.device) + # Get the attention mask and postition ids. + attention_mask, _, position_ids = get_ltor_masks_and_position_ids( + tokens, + eod_id, + reset_position_ids=False, + reset_attention_mask=False, + eod_mask_loss=False, + ) + return tokens, attention_mask, position_ids + + +def get_stop_words_ids(chat_format, tokenizer): + if chat_format == 'raw': + stop_words_ids = [tokenizer.encode('Human:'), [tokenizer.eod_id]] + elif chat_format == 'chatml': + stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]] + else: + raise NotImplementedError(f'Unknown chat format {chat_format!r}') + return stop_words_ids + + +def make_context( + tokenizer: PreTrainedTokenizer, + query: str, + history: List[Tuple[str, str]] = None, + system: str = '', + max_window_size: int = 6144, + chat_format: str = 'chatml', +): + if history is None: + history = [] + + if chat_format == 'chatml': + im_start, im_end = '<|im_start|>', '<|im_end|>' + im_start_tokens = [tokenizer.im_start_id] + im_end_tokens = [tokenizer.im_end_id] + nl_tokens = tokenizer.encode('\n') + + def _tokenize_str(role, content): + return f'{role}\n{content}', tokenizer.encode( + role) + nl_tokens + tokenizer.encode(content) + + system_text, system_tokens_part = _tokenize_str('system', system) + system_tokens = im_start_tokens + system_tokens_part + im_end_tokens + + raw_text = '' + context_tokens = [] + + for turn_query, turn_response in reversed(history): + query_text, query_tokens_part = _tokenize_str('user', turn_query) + query_tokens = im_start_tokens + query_tokens_part + im_end_tokens + response_text, response_tokens_part = _tokenize_str( + 'assistant', turn_response) + response_tokens = im_start_tokens + response_tokens_part + im_end_tokens + + next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens + prev_chat = ( + f'\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}' + ) + + current_context_size = ( + len(system_tokens) + len(next_context_tokens) + + len(context_tokens)) + if current_context_size < max_window_size: + context_tokens = next_context_tokens + context_tokens + raw_text = prev_chat + raw_text + else: + break + + context_tokens = system_tokens + context_tokens + raw_text = f'{im_start}{system_text}{im_end}' + raw_text + context_tokens += ( + nl_tokens + im_start_tokens + _tokenize_str('user', query)[1] + + im_end_tokens + nl_tokens + im_start_tokens + + tokenizer.encode('assistant') + nl_tokens) + raw_text += f'\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n' + + elif chat_format == 'raw': + raw_text = query + context_tokens = tokenizer.encode(raw_text) + else: + raise NotImplementedError(f'Unknown chat format {chat_format!r}') + + return raw_text, context_tokens + + +def _decode_default( + tokens: List[int], + *, + stop_words: List[str], + eod_words: List[str], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + verbose: bool = False, + return_end_reason: bool = False, +): + trim_decode_tokens = tokenizer.decode(tokens)[raw_text_len:] + if verbose: + print('\nRaw Generate: ', trim_decode_tokens) + + end_reason = f'Gen length {len(tokens)}' + for stop_word in stop_words: + trim_decode_tokens = trim_decode_tokens.replace(stop_word, '').strip() + for eod_word in eod_words: + if eod_word in trim_decode_tokens: + end_reason = f'Gen {eod_word!r}' + trim_decode_tokens = trim_decode_tokens.split(eod_word)[0] + trim_decode_tokens = trim_decode_tokens.strip() + if verbose: + print('\nEnd Reason:', end_reason) + print('\nGenerate: ', trim_decode_tokens) + + if return_end_reason: + return trim_decode_tokens, end_reason + else: + return trim_decode_tokens + + +def _decode_chatml( + tokens: List[int], + *, + stop_words: List[str], + eod_token_ids: List[int], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + context_length: int, + verbose: bool = False, + return_end_reason: bool = False, +): + end_reason = f'Gen length {len(tokens)}' + eod_token_idx = context_length + for eod_token_idx in range(context_length, len(tokens)): + if tokens[eod_token_idx] in eod_token_ids: + end_reason = f'Gen {tokenizer.decode([tokens[eod_token_idx]])!r}' + break + + trim_decode_tokens = tokenizer.decode( + tokens[:eod_token_idx])[raw_text_len:] + if verbose: + print('\nRaw Generate w/o EOD:', + tokenizer.decode(tokens)[raw_text_len:]) + print('\nRaw Generate:', trim_decode_tokens) + print('\nEnd Reason:', end_reason) + for stop_word in stop_words: + trim_decode_tokens = trim_decode_tokens.replace(stop_word, '').strip() + trim_decode_tokens = trim_decode_tokens.strip() + if verbose: + print('\nGenerate:', trim_decode_tokens) + + if return_end_reason: + return trim_decode_tokens, end_reason + else: + return trim_decode_tokens + + +def decode_tokens( + tokens: Union[torch.LongTensor, TokensType], + tokenizer: PreTrainedTokenizer, + raw_text_len: int, + context_length: int, + chat_format: str, + verbose: bool = False, + return_end_reason: bool = False, +) -> str: + if torch.is_tensor(tokens): + tokens = tokens.cpu().numpy().tolist() + + if chat_format == 'chatml': + return _decode_chatml( + tokens, + stop_words=[], + eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id], + tokenizer=tokenizer, + raw_text_len=raw_text_len, + context_length=context_length, + verbose=verbose, + return_end_reason=return_end_reason, + ) + elif chat_format == 'raw': + return _decode_default( + tokens, + stop_words=['<|endoftext|>'], + eod_words=['<|endoftext|>'], + tokenizer=tokenizer, + raw_text_len=raw_text_len, + verbose=verbose, + return_end_reason=return_end_reason, + ) + else: + raise NotImplementedError(f'Unknown chat format {chat_format!r}') + + +class StopWordsLogitsProcessor(LogitsProcessor): + """ + :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration. + + Args: + stop_words_ids (:obj:`List[List[int]]`): + List of list of token ids of stop ids. In order to get the tokens of the words + that should not appear in the generated text, use :obj:`tokenizer(bad_word, + add_prefix_space=True).input_ids`. + eos_token_id (:obj:`int`): + The id of the `end-of-sequence` token. + """ + + def __init__(self, stop_words_ids: Iterable[Iterable[int]], + eos_token_id: int): + + if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0: + raise ValueError( + f'`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}.' + ) + if any(not isinstance(bad_word_ids, list) + for bad_word_ids in stop_words_ids): + raise ValueError( + f'`stop_words_ids` has to be a list of lists, but is {stop_words_ids}.' + ) + if any( + any((not isinstance(token_id, (int, + np.integer)) or token_id < 0) + for token_id in stop_word_ids) + for stop_word_ids in stop_words_ids): + raise ValueError( + f'Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}.' + ) + + self.stop_words_ids = list( + filter(lambda bad_token_seq: bad_token_seq != [eos_token_id], + stop_words_ids)) + self.eos_token_id = eos_token_id + for stop_token_seq in self.stop_words_ids: + assert ( + len(stop_token_seq) > 0 + ), 'Stop words token sequences {} cannot have an empty list'.format( + stop_words_ids) + + def __call__(self, input_ids: torch.LongTensor, + scores: torch.FloatTensor) -> torch.FloatTensor: + stopped_samples = self._calc_stopped_samples(input_ids) + for i, should_stop in enumerate(stopped_samples): + if should_stop: + scores[i, self.eos_token_id] = float(2**15) + return scores + + def _tokens_match(self, prev_tokens: torch.LongTensor, + tokens: List[int]) -> bool: + if len(tokens) == 0: + # if bad word tokens is just one token always ban it + return True + elif len(tokens) > len(prev_tokens): + # if bad word tokens are longer then prev input_ids they can't be equal + return False + elif prev_tokens[-len(tokens):].tolist() == tokens: + # if tokens match + return True + else: + return False + + def _calc_stopped_samples(self, + prev_input_ids: Iterable[int]) -> Iterable[int]: + stopped_samples = [] + for prev_input_ids_slice in prev_input_ids: + match = False + for stop_token_seq in self.stop_words_ids: + if self._tokens_match(prev_input_ids_slice, stop_token_seq): + # if tokens do not match continue + match = True + break + stopped_samples.append(match) + + return stopped_samples + + +def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')): + """This function has been mostly taken from huggingface conversational + ai code at + https://medium.com/huggingface/how-to-build-a-state-of-the-art- + conversational-ai-with-transfer-learning-2d818ac26313""" + + if top_k > 0: + # Remove all tokens with a probability less than the + # last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, + None] + logits[indices_to_remove] = filter_value + + if top_p > 0.0: + # Cconvert to 1D + sorted_logits, sorted_indices = torch.sort( + logits, descending=True, dim=-1) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1) + + # Remove tokens with cumulative probability above the threshold + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep also the first token + # above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + for i in range(sorted_indices.size(0)): + indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]] + logits[i][indices_to_remove] = filter_value + + return logits + + +def switch(val1, val2, boolean): + boolean = boolean.type_as(val1) + return (1 - boolean) * val1 + boolean * val2 diff --git a/runtime/Lib/site_packages/models/nlp/qwen/text_generation.py b/runtime/Lib/site_packages/models/nlp/qwen/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..a91819b07231515af87a96030728db311c085cc7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/text_generation.py @@ -0,0 +1,254 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import warnings +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss +from transformers import (GenerationConfig, PreTrainedTokenizer, + StoppingCriteriaList) +from transformers.generation.logits_process import LogitsProcessorList +from transformers.generation.utils import GenerateOutput +from transformers.modeling_outputs import CausalLMOutputWithPast + +from modelscope.metainfo import Models +from modelscope.outputs import OutputKeys +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from ... import MODELS +from .backbone import QWenModel, QWenPreTrainedModel +from .qwen_generation_utils import (BatchTokensType, HistoryType, + StopWordsLogitsProcessor, decode_tokens, + get_batch, get_stop_words_ids, + make_context, pad_batch, switch, + top_k_logits) + +if TYPE_CHECKING: + from transformers.generation.streamers import BaseStreamer + +logger = get_logger() + + +@MODELS.register_module(Tasks.text_generation, module_name=Models.qwen_7b) +@MODELS.register_module(Tasks.chat, module_name=Models.qwen_7b) +class QWenForTextGeneration(QWenPreTrainedModel): + _keys_to_ignore_on_load_missing = [r'h\.\d+\.attn\.rotary_emb\.inv_freq'] + _keys_to_ignore_on_load_unexpected = [r'h\.\d+\.attn\.masked_bias'] + + def __init__(self, config): + super().__init__(config) + self.transformer = QWenModel(config) + self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + assert not (config.bf16 and config.fp16 + ), 'In config, bf16 and fp16 cannot both be true' + if config.bf16: + self.transformer.bfloat16() + self.lm_head.bfloat16() + if config.fp16: + self.transformer.half() + self.lm_head.half() + self.post_init() + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def prepare_inputs_for_generation(self, + input_ids, + past_key_values=None, + inputs_embeds=None, + **kwargs): + token_type_ids = kwargs.get('token_type_ids', None) + if past_key_values: + input_ids = input_ids[:, -1].unsqueeze(-1) + if token_type_ids is not None: + token_type_ids = token_type_ids[:, -1].unsqueeze(-1) + + attention_mask = kwargs.get('attention_mask', None) + position_ids = kwargs.get('position_ids', None) + + if attention_mask is not None and position_ids is None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + else: + position_ids = None + + if inputs_embeds is not None and past_key_values is None: + model_inputs = {'inputs_embeds': inputs_embeds} + else: + model_inputs = {'input_ids': input_ids} + + model_inputs.update({ + 'past_key_values': past_key_values, + 'use_cache': kwargs.get('use_cache'), + 'position_ids': position_ids, + 'attention_mask': attention_mask, + 'token_type_ids': token_type_ids, + }) + return model_inputs + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.transformer( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + labels = labels.to(lm_logits.device) + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + loss_fct = CrossEntropyLoss() + loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + + if not return_dict: + output = (lm_logits, ) + transformer_outputs[1:] + return ((loss, ) + output) if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + @staticmethod + def _reorder_cache(past_key_values: Tuple[Tuple[torch.Tensor]], + beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]: + + return tuple( + tuple( + past_state.index_select(0, beam_idx.to(past_state.device)) + for past_state in layer_past) + for layer_past in past_key_values) + + def chat( + self, + tokenizer: PreTrainedTokenizer, + query: str, + history: Optional[HistoryType], + system: str = 'You are a helpful assistant.', + append_history: bool = True, + ) -> Tuple[str, HistoryType]: + + if history is None: + history = [] + + raw_text, context_tokens = make_context( + tokenizer, + query, + history=history, + system=system, + max_window_size=6144, + chat_format=self.generation_config.chat_format) + + stop_words_ids = get_stop_words_ids(self.generation_config.chat_format, + tokenizer) + input_ids = torch.tensor([context_tokens]).to(self.device) + + outputs = self.generate( + input_ids, + stop_words_ids=stop_words_ids, + return_dict_in_generate=False, + ) + + response = decode_tokens( + outputs[0], + tokenizer, + raw_text_len=len(raw_text), + context_length=len(context_tokens), + chat_format=self.generation_config.chat_format, + verbose=False, + ) + + if append_history: + history.append((query, response)) + + return {OutputKeys.RESPONSE: response, OutputKeys.HISTORY: history} + + def generate( + self, + inputs: Optional[torch.Tensor] = None, + generation_config: Optional[GenerationConfig] = None, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], + List[int]]] = None, + synced_gpus: Optional[bool] = None, + streamer: Optional['BaseStreamer'] = None, + **kwargs, + ) -> Union[GenerateOutput, torch.LongTensor]: + # Process stop_words_ids + stop_words_ids = kwargs.pop('stop_words_ids', None) + if stop_words_ids is None and generation_config is not None: + stop_words_ids = getattr(generation_config, 'stop_words_ids', None) + if stop_words_ids is None: + stop_words_ids = getattr(self.generation_config, 'stop_words_ids', + None) + + if stop_words_ids is not None: + stop_words_logits_processor = StopWordsLogitsProcessor( + stop_words_ids=stop_words_ids, + eos_token_id=self.generation_config.eos_token_id) + if logits_processor is None: + logits_processor = LogitsProcessorList( + [stop_words_logits_processor]) + else: + logits_processor.append(stop_words_logits_processor) + + return super().generate( + inputs, + generation_config, + logits_processor, + stopping_criteria, + prefix_allowed_tokens_fn, + synced_gpus, + streamer=streamer, + **kwargs, + ) diff --git a/runtime/Lib/site_packages/models/nlp/qwen/tokenization.py b/runtime/Lib/site_packages/models/nlp/qwen/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..0446854ceddbfd44043c20720b27f5f07c2f1372 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/qwen/tokenization.py @@ -0,0 +1,252 @@ +# Copyright (c) Alibaba Cloud. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +"""Tokenization classes for QWen.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) +import base64 +import logging +import os +import unicodedata +from io import open +from typing import List, Optional, Tuple, Union + +import json +import tiktoken +from transformers import AddedToken, PreTrainedTokenizer + +from modelscope.utils.logger import get_logger + +logger = get_logger() + +VOCAB_FILES_NAMES = {'vocab_file': 'qwen.tiktoken'} + + +class QWenTokenizer(PreTrainedTokenizer): + """QWen tokenizer.""" + """NOTE: This tokenizer will not handle special tokens to avoid injection attacks""" + + vocab_files_names = VOCAB_FILES_NAMES + + def __init__( + self, + vocab_file, + errors='replace', + max_len=None, + unk_token='<|endoftext|>', + bos_token='<|endoftext|>', + eos_token='<|endoftext|>', + pad_token=None, + add_prefix_space=False, + add_bos_token=False, + add_more_sp_tokens=True, + **kwargs, + ): + bos_token = ( + AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance( + bos_token, str) else bos_token) + eos_token = ( + AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance( + eos_token, str) else eos_token) + unk_token = ( + AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance( + unk_token, str) else unk_token) + pad_token = ( + AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance( + pad_token, str) else pad_token) + super().__init__( + errors=errors, + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + add_prefix_space=add_prefix_space, + add_bos_token=add_bos_token, + ) + self.add_bos_token = add_bos_token + self.max_len = max_len if max_len is not None else int(1e12) + + self.errors = errors # how to handle errors in decoding + + name = 'Qwen' + ENDOFTEXT = '<|endoftext|>' + IMSTART = '<|im_start|>' + IMEND = '<|im_end|>' + if add_more_sp_tokens: + special_tokens = ( + ENDOFTEXT, + IMSTART, + IMEND, + '', + '', + '', + '', + '', + ) + tuple([f'' for i in range(200)]) + else: + special_tokens = (ENDOFTEXT, IMSTART, IMEND) + + PAT_STR = ( + r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}|""" + r""" ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""") + + def load_tiktoken_bpe(tiktoken_bpe_file: str) -> 'dict[bytes, int]': + contents = open(tiktoken_bpe_file, 'rb').read() + return { + base64.b64decode(token): int(rank) + for token, rank in (line.split() + for line in contents.splitlines() if line) + } + + mergeable_ranks = load_tiktoken_bpe(vocab_file) + special_tokens = { + token: index + for index, token in enumerate( + special_tokens, start=len(mergeable_ranks)) + } + self.special_tokens = special_tokens + enc = tiktoken.Encoding( + name, + pat_str=PAT_STR, + mergeable_ranks=mergeable_ranks, + special_tokens=special_tokens, + ) + assert ( + len(mergeable_ranks) + len(special_tokens) == enc.n_vocab + ), f'{len(mergeable_ranks) + len(special_tokens)} != {enc.n_vocab} in encoding' + + self.mergeable_ranks = mergeable_ranks + self.encoder = self.mergeable_ranks + self.decoder = {v: k for k, v in self.encoder.items()} + self.tokenizer = enc # type: tiktoken.Encoding + self.eod_id = self.tokenizer.eot_token + self.im_start_id = special_tokens[IMSTART] + self.im_end_id = special_tokens[IMEND] + + def __len__(self): + return self.tokenizer.n_vocab + + def get_vocab(self): + return self.mergeable_ranks + + def convert_tokens_to_ids(self, tokens): + ids = [] + # Remove support for py2 + if isinstance(tokens, str): + if tokens in self.special_tokens: + return self.special_tokens[tokens] + else: + return self.encoder.get(tokens) + for token in tokens: + if token in self.special_tokens: + ids.append(self.special_tokens[token]) + else: + ids.append(self.encoder.get(token)) + if len(ids) > self.max_len: + logger.warning( + 'Token indices sequence length is longer than the specified maximum ' + ' sequence length for this model ({} > {}). Running this' + ' sequence through the model will result in indexing errors'. + format(len(ids), self.max_len)) + return ids + + def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]: + """ + Save only the vocabulary of the tokenizer (vocabulary + added tokens). + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + file_path = os.path.join(save_directory, 'qwen.tiktoken') + with open(file_path, 'w', encoding='utf8') as w: + for k, v in self.mergeable_ranks.items(): + line = base64.b64encode(k).decode('utf8') + ' ' + str(v) + '\n' + w.write(line) + return (file_path, ) + + def tokenize(self, text: str, **kwargs) -> List[str]: + """ + Converts a string in a sequence of tokens, replacing unknown tokens with the `unk_token`. + + Args: + text (`str`): + The sequence to be encoded. + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific encode method. See details in + [`~PreTrainedTokenizerBase.__call__`] + + Returns: + `List[str]`: The list of tokens. + """ + tokens = [] + text = unicodedata.normalize('NFC', text) + for t in self.tokenizer.encode_ordinary(text): + tokens.append(self.decoder[t]) + return tokens + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + """ + Converts a sequence of tokens in a single string. The most simple way to do it is `" ".join(tokens)` but we + often want to remove sub-word tokenization artifacts at the same time. + """ + text = ''.join(tokens) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors=self.errors) + return text + + @property + def vocab_size(self): + return self.tokenizer.n_vocab + + def _convert_id_to_token(self, index: int) -> str: + if index >= self.tokenizer.n_vocab: + return self.unk_token + return self.tokenizer.decode([index]) + + def _convert_token_to_id(self, token: str) -> int: + """Converts a token to an id using the vocab.""" + return self.encoder.get( + token.encode('UTF-8'), + self.tokenizer.encode(self.unk_token, allowed_special='all')[0], + ) + + @property + def all_special_tokens(self) -> List[str]: + """ + `List[str]`: All the special tokens (`''`, `''`, etc.) mapped to class attributes. + + Convert tokens of `tokenizers.AddedToken` type to string. + """ + all_toks = [str(s) for s in self.special_tokens.keys()] + return all_toks + + @property + def all_special_ids(self) -> List[int]: + """ + `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes. + """ + all_ids = [v for v in self.special_tokens.values()] + return all_ids + + def _tokenize(self, text, **kwargs): + """ + Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based + vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). + + Do NOT take care of added tokens. + """ + raise NotImplementedError + + def _decode( + self, + token_ids: Union[int, List[int]], + skip_special_tokens: bool = False, + **kwargs, + ) -> str: + if isinstance(token_ids, int): + token_ids = [token_ids] + if skip_special_tokens: + token_ids = [i for i in token_ids if i not in self.all_special_ids] + return self.tokenizer.decode(token_ids) diff --git a/runtime/Lib/site_packages/models/nlp/space/__init__.py b/runtime/Lib/site_packages/models/nlp/space/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..32713c34ec664c497df57a25b44753d52b6a8c0f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .model import SpaceGenerator + from .model import SpaceModelBase, SpaceTokenizer + from .dialog_intent_prediction import SpaceForDialogIntent + from .dialog_modeling import SpaceForDialogModeling + from .dialog_state_tracking import SpaceForDST + from .configuration import SpaceConfig +else: + _import_structure = { + 'model': ['SpaceGenerator', 'SpaceModelBase', 'SpaceTokenizer'], + 'dialog_intent_prediction': ['SpaceForDialogIntent'], + 'dialog_modeling': ['SpaceForDialogModeling'], + 'dialog_state_tracking': ['SpaceForDST'], + 'configuration': ['SpaceConfig'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/space/configuration.py b/runtime/Lib/site_packages/models/nlp/space/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..8f125b03e1e4404e3d70e8d6ead1e79785df1648 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/configuration.py @@ -0,0 +1,32 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors. +# Copyright 2020 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Space configuration, mainly copied from :class:`~transformers.configuration_xlm_roberta` """ + +from modelscope.models.nlp.structbert import SbertConfig +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class SpaceConfig(SbertConfig): + """ + This class overrides [`SbertConfig`]. Please check the superclass for the appropriate + documentation alongside usage examples. + """ + + model_type = 'space' diff --git a/runtime/Lib/site_packages/models/nlp/space/dialog_intent_prediction.py b/runtime/Lib/site_packages/models/nlp/space/dialog_intent_prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..79ff01cd5709c783c67af70a7bee15bdb862a4c6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/dialog_intent_prediction.py @@ -0,0 +1,99 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Dict + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.models.nlp.space import SpaceGenerator, SpaceModelBase +from modelscope.preprocessors.nlp import IntentBPETextField +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['SpaceForDialogIntent'] + + +@MODELS.register_module( + Tasks.task_oriented_conversation, module_name=Models.space_intent) +class SpaceForDialogIntent(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the test generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + text_field (`BPETextField`, *optional*, defaults to `IntentBPETextField`): + The text field. + config (`Config`, *optional*, defaults to config in model hub): + The config. + """ + + super().__init__(model_dir, *args, **kwargs) + from modelscope.trainers.nlp.space.trainer.intent_trainer import \ + IntentTrainer + self.model_dir = model_dir + self.config = kwargs.pop( + 'config', + Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION))) + self.text_field = kwargs.pop( + 'text_field', + IntentBPETextField(self.model_dir, config=self.config)) + + self.generator = SpaceGenerator.create( + self.config, reader=self.text_field) + self.model = SpaceModelBase.create( + model_dir=model_dir, + config=self.config, + reader=self.text_field, + generator=self.generator) + + def to_tensor(array): + """ + numpy array -> tensor + """ + import torch + array = torch.tensor(array) + return array.cuda() if self.config.use_gpu else array + + self.trainer = IntentTrainer( + model=self.model, + to_tensor=to_tensor, + config=self.config, + reader=self.text_field) + self.trainer.load() + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'pred': array([2.62349960e-03 4.12110658e-03 4.12748595e-05 3.77560973e-05 + 1.08599677e-04 1.72710388e-05 2.95618793e-05 1.93638436e-04 + 6.45841064e-05 1.15997791e-04 5.11605394e-05 9.87020373e-01 + 2.66957268e-05 4.72324500e-05 9.74208378e-05], dtype=float32), + } + Example: + >>> from modelscope.hub.snapshot_download import snapshot_download + >>> from modelscope.models.nlp import SpaceForDialogIntent + >>> from modelscope.preprocessors import DialogIntentPredictionPreprocessor + >>> cache_path = snapshot_download('damo/nlp_space_dialog-intent-prediction') + >>> preprocessor = DialogIntentPredictionPreprocessor(model_dir=cache_path) + >>> model = SpaceForDialogIntent( + model_dir=cache_path, + text_field=preprocessor.text_field, + config=preprocessor.config) + >>> print(model(preprocessor("What do I need to do for the card activation?"))) + """ + import numpy as np + pred = self.trainer.forward(input) + pred = np.squeeze(pred[0], 0) + + return {'pred': pred} diff --git a/runtime/Lib/site_packages/models/nlp/space/dialog_modeling.py b/runtime/Lib/site_packages/models/nlp/space/dialog_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..faecfa54f8dcd21724005ad3acb22a9205071bed --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/dialog_modeling.py @@ -0,0 +1,119 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Dict + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.models.nlp.space import SpaceGenerator, SpaceModelBase +from modelscope.preprocessors.nlp import MultiWOZBPETextField +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['SpaceForDialogModeling'] + + +@MODELS.register_module( + Tasks.task_oriented_conversation, module_name=Models.space_modeling) +class SpaceForDialogModeling(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the test generation model from the `model_dir` path. + + Args: + model_dir (`str`): + The model path. + text_field (`BPETextField`, *optional*, defaults to `MultiWOZBPETextField`): + The text field. + config (`Config`, *optional*, defaults to config in model hub): + The config. + """ + + super().__init__(model_dir, *args, **kwargs) + from modelscope.trainers.nlp.space.trainer.gen_trainer import MultiWOZTrainer + self.model_dir = model_dir + self.config = kwargs.pop( + 'config', + Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION))) + + import torch + self.config.use_gpu = True if ( + 'device' not in kwargs or kwargs['device'] + == 'gpu') and torch.cuda.is_available() else False + + self.text_field = kwargs.pop( + 'text_field', + MultiWOZBPETextField(config=self.config, model_dir=self.model_dir)) + self.generator = SpaceGenerator.create( + self.config, reader=self.text_field) + self.model = SpaceModelBase.create( + model_dir=model_dir, + config=self.config, + reader=self.text_field, + generator=self.generator) + + def to_tensor(array): + """ + numpy array -> tensor + """ + import torch + array = torch.tensor(array) + return array.cuda() if self.config.use_gpu else array + + self.trainer = MultiWOZTrainer( + model=self.model, + to_tensor=to_tensor, + config=self.config, + reader=self.text_field, + evaluator=None) + self.trainer.load() + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'labels': array([1,192,321,12]), # lable + 'resp': array([293,1023,123,1123]), #vocab label for response + 'bspn': array([123,321,2,24,1 ]), + 'aspn': array([47,8345,32,29,1983]), + 'db': array([19, 24, 20]), + } + + Examples: + >>> from modelscope.hub.snapshot_download import snapshot_download + >>> from modelscope.models.nlp import SpaceForDialogModeling + >>> from modelscope.preprocessors import DialogModelingPreprocessor + >>> cache_path = snapshot_download('damo/nlp_space_dialog-modeling') + >>> preprocessor = DialogModelingPreprocessor(model_dir=cache_path) + >>> model = SpaceForDialogModeling(model_dir=cache_path, + text_field=preprocessor.text_field, + config=preprocessor.config) + >>> print(model(preprocessor({ + 'user_input': 'i would like a taxi from saint john \'s college to pizza hut fen ditton .', + 'history': {} + }))) + """ + + first_turn = input['first_turn'] + batch = input['batch'] + prompt_id = input['prompt_id'] + labels = input['labels'] + old_pv_turn = input['history'] + + pv_turn = self.trainer.forward( + first_turn=first_turn, + batch=batch, + prompt_id=prompt_id, + labels=labels, + old_pv_turn=old_pv_turn) + + return pv_turn diff --git a/runtime/Lib/site_packages/models/nlp/space/dialog_state_tracking.py b/runtime/Lib/site_packages/models/nlp/space/dialog_state_tracking.py new file mode 100644 index 0000000000000000000000000000000000000000..9a713a592f7adb1d88e7b9bc46c35faa715dedea --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/dialog_state_tracking.py @@ -0,0 +1,392 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Space model. mainly copied from :module:`~transformers.modeling_xlm_roberta`""" + +from typing import Dict + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss +from transformers.file_utils import add_start_docstrings +from transformers.modeling_utils import PreTrainedModel + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.models.nlp.structbert import (SbertForMaskedLM, SbertModel, + SbertPreTrainedModel) +from modelscope.utils.constant import Tasks +from .configuration import SpaceConfig + +SPACE_START_DOCSTRING = r""" + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config ([`SpaceConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model + weights. +""" + + +@add_start_docstrings( + 'The bare Space Model transformer outputting raw hidden-states without any specific head on top. ' + 'It is identical with the Bert Model from Transformers', + SPACE_START_DOCSTRING, +) +class SpaceModel(SbertModel): + """ + This class overrides [`SbertModel`]. Please check the superclass for the appropriate + documentation alongside usage examples. + """ + + config_class = SpaceConfig + + +class SpacePreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = SpaceConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + @param kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + @return: The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + config = SpaceConfig(**kwargs) + model = cls(config) + else: + model_kwargs = {} + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_kwargs) + return model + + +@add_start_docstrings( + """ + Space Model transformer with Dialog state tracking heads on top (a inform projection + layer with a dialog state layer and a set of slots including history infromation from + previous dialog) e.g. for multiwoz2.2 tasks. + """, + SPACE_START_DOCSTRING, +) +@MODELS.register_module( + Tasks.task_oriented_conversation, module_name=Models.space_dst) +class SpaceForDST(SpacePreTrainedModel): + + def __init__(self, config): + super(SpaceForDST, self).__init__(config) + self.slot_list = config.dst_slot_list + self.class_types = config.dst_class_types + self.class_labels = config.dst_class_labels + self.token_loss_for_nonpointable = config.dst_token_loss_for_nonpointable + self.refer_loss_for_nonpointable = config.dst_refer_loss_for_nonpointable + self.class_aux_feats_inform = config.dst_class_aux_feats_inform + self.class_aux_feats_ds = config.dst_class_aux_feats_ds + self.class_loss_ratio = config.dst_class_loss_ratio + + # Only use refer loss if refer class is present in dataset. + if 'refer' in self.class_types: + self.refer_index = self.class_types.index('refer') + else: + self.refer_index = -1 + + self.bert = SpaceModel(config) + self.dropout = nn.Dropout(config.dst_dropout_rate) + self.dropout_heads = nn.Dropout(config.dst_heads_dropout_rate) + + if self.class_aux_feats_inform: + self.add_module( + 'inform_projection', + nn.Linear(len(self.slot_list), len(self.slot_list))) + if self.class_aux_feats_ds: + self.add_module( + 'ds_projection', + nn.Linear(len(self.slot_list), len(self.slot_list))) + + aux_dims = len(self.slot_list) * ( + self.class_aux_feats_inform + self.class_aux_feats_ds + ) # second term is 0, 1 or 2 + + for slot in self.slot_list: + self.add_module( + 'class_' + slot, + nn.Linear(config.hidden_size + aux_dims, self.class_labels)) + self.add_module('token_' + slot, nn.Linear(config.hidden_size, 2)) + self.add_module( + 'refer_' + slot, + nn.Linear(config.hidden_size + aux_dims, + len(self.slot_list) + 1)) + + self.init_weights() + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + { + 'inputs': dict(input_ids, input_masks,start_pos), # tracking states + 'outputs': dict(slots_logits), + 'unique_ids': str(test-example.json-0), # default value + 'input_ids_unmasked': array([101, 7632, 1010,0,0,0]) + 'values': array([{'taxi-leaveAt': 'none', 'taxi-destination': 'none'}]), + 'inform': array([{'taxi-leaveAt': 'none', 'taxi-destination': 'none'}]), + 'prefix': str('final'), #default value + 'ds': array([{'taxi-leaveAt': 'none', 'taxi-destination': 'none'}]) + } + + Example: + >>> from modelscope.hub.snapshot_download import snapshot_download + >>> from modelscope.models.nlp import SpaceForDST + >>> from modelscope.preprocessors import DialogStateTrackingPreprocessor + >>> cache_path = snapshot_download('damo/nlp_space_dialog-state-tracking') + >>> model = SpaceForDST.from_pretrained(cache_path) + >>> preprocessor = DialogStateTrackingPreprocessor(model_dir=cache_path) + >>> print(model(preprocessor({ + 'utter': { + 'User-1': "Hi, I'm looking for a train that is going" + "to cambridge and arriving there by 20:45, is there anything like that?" + }, + 'history_states': [{}] + }))) + """ + import numpy as np + import torch + + # self.model.eval() ???? + batch = input['batch'] + + features = input['features'] + diag_state = input['diag_state'] + turn_itrs = [features[i.item()].guid.split('-')[2] for i in batch[9]] + reset_diag_state = np.where(np.array(turn_itrs) == '0')[0] + for slot in self.config.dst_slot_list: + for i in reset_diag_state: + diag_state[slot][i] = 0 + + with torch.no_grad(): + inputs = { + 'input_ids': batch[0], + 'input_mask': batch[1], + 'segment_ids': batch[2], + 'start_pos': batch[3], + 'end_pos': batch[4], + 'inform_slot_id': batch[5], + 'refer_id': batch[6], + 'diag_state': diag_state, + 'class_label_id': batch[8] + } + unique_ids = [features[i.item()].guid for i in batch[9]] + values = [features[i.item()].values for i in batch[9]] + input_ids_unmasked = [ + features[i.item()].input_ids_unmasked for i in batch[9] + ] + inform = [features[i.item()].inform for i in batch[9]] + outputs = self._forward(**inputs) + + # Update dialog state for next turn. + for slot in self.config.dst_slot_list: + updates = outputs[2][slot].max(1)[1] + for i, u in enumerate(updates): + if u != 0: + diag_state[slot][i] = u + + return { + 'inputs': inputs, + 'outputs': outputs, + 'unique_ids': unique_ids, + 'input_ids_unmasked': input_ids_unmasked, + 'values': values, + 'inform': inform, + 'prefix': 'final', + 'ds': input['ds'] + } + + def _forward(self, + input_ids, + input_mask=None, + segment_ids=None, + position_ids=None, + head_mask=None, + start_pos=None, + end_pos=None, + inform_slot_id=None, + refer_id=None, + class_label_id=None, + diag_state=None): + outputs = self.bert( + input_ids, + attention_mask=input_mask, + token_type_ids=segment_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs.last_hidden_state + pooled_output = outputs.pooler_output + + sequence_output = self.dropout(sequence_output) + pooled_output = self.dropout(pooled_output) + + # TODO: establish proper format in labels already? + if inform_slot_id is not None: + inform_labels = torch.stack(list(inform_slot_id.values()), + 1).float() + if diag_state is not None: + diag_state_labels = torch.clamp( + torch.stack(list(diag_state.values()), 1).float(), 0.0, 1.0) + + total_loss = 0 + per_slot_per_example_loss = {} + per_slot_class_logits = {} + per_slot_start_logits = {} + per_slot_end_logits = {} + per_slot_refer_logits = {} + for slot in self.slot_list: + if self.class_aux_feats_inform and self.class_aux_feats_ds: + pooled_output_aux = torch.cat( + (pooled_output, self.inform_projection(inform_labels), + self.ds_projection(diag_state_labels)), 1) + elif self.class_aux_feats_inform: + pooled_output_aux = torch.cat( + (pooled_output, self.inform_projection(inform_labels)), 1) + elif self.class_aux_feats_ds: + pooled_output_aux = torch.cat( + (pooled_output, self.ds_projection(diag_state_labels)), 1) + else: + pooled_output_aux = pooled_output + class_logits = self.dropout_heads( + getattr(self, 'class_' + slot)(pooled_output_aux)) + + token_logits = self.dropout_heads( + getattr(self, 'token_' + slot)(sequence_output)) + start_logits, end_logits = token_logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + refer_logits = self.dropout_heads( + getattr(self, 'refer_' + slot)(pooled_output_aux)) + + per_slot_class_logits[slot] = class_logits + per_slot_start_logits[slot] = start_logits + per_slot_end_logits[slot] = end_logits + per_slot_refer_logits[slot] = refer_logits + + # If there are no labels, don't compute loss + if class_label_id is not None and start_pos is not None and end_pos is not None and refer_id is not None: + # If we are on multi-GPU, split add a dimension + if len(start_pos[slot].size()) > 1: + start_pos[slot] = start_pos[slot].squeeze(-1) + if len(end_pos[slot].size()) > 1: + end_pos[slot] = end_pos[slot].squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) # This is a single index + start_pos[slot].clamp_(0, ignored_index) + end_pos[slot].clamp_(0, ignored_index) + + class_loss_fct = CrossEntropyLoss(reduction='none') + token_loss_fct = CrossEntropyLoss( + reduction='none', ignore_index=ignored_index) + refer_loss_fct = CrossEntropyLoss(reduction='none') + + start_loss = token_loss_fct(start_logits, start_pos[slot]) + end_loss = token_loss_fct(end_logits, end_pos[slot]) + token_loss = (start_loss + end_loss) / 2.0 + + token_is_pointable = (start_pos[slot] > 0).float() + if not self.token_loss_for_nonpointable: + token_loss *= token_is_pointable + + refer_loss = refer_loss_fct(refer_logits, refer_id[slot]) + token_is_referrable = torch.eq(class_label_id[slot], + self.refer_index).float() + if not self.refer_loss_for_nonpointable: + refer_loss *= token_is_referrable + + class_loss = class_loss_fct(class_logits, class_label_id[slot]) + + if self.refer_index > -1: + per_example_loss = (self.class_loss_ratio) * class_loss + ( + (1 - self.class_loss_ratio) / 2) * token_loss + ( + (1 - self.class_loss_ratio) / 2) * refer_loss + else: + per_example_loss = self.class_loss_ratio * class_loss + ( + 1 - self.class_loss_ratio) * token_loss + + total_loss += per_example_loss.sum() + per_slot_per_example_loss[slot] = per_example_loss + + # add hidden states and attention if they are here + outputs = (total_loss, ) + ( + per_slot_per_example_loss, + per_slot_class_logits, + per_slot_start_logits, + per_slot_end_logits, + per_slot_refer_logits, + ) + (outputs.embedding_output, ) + + return outputs diff --git a/runtime/Lib/site_packages/models/nlp/space/model/__init__.py b/runtime/Lib/site_packages/models/nlp/space/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cfff335d9b5b1b8ab0917f6e65f70eae8664b928 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from .gen_unified_transformer import GenUnifiedTransformer +from .generator import SpaceGenerator +from .intent_unified_transformer import IntentUnifiedTransformer +from .model_base import SpaceModelBase +from .tokenization_space import (BasicTokenizer, SpaceTokenizer, + WordpieceTokenizer) +from .unified_transformer import UnifiedTransformer diff --git a/runtime/Lib/site_packages/models/nlp/space/model/gen_unified_transformer.py b/runtime/Lib/site_packages/models/nlp/space/model/gen_unified_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..07cc8d7f8cf914f4a540ba1d65cfa6c2a6bc546b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/gen_unified_transformer.py @@ -0,0 +1,290 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch + +from .unified_transformer import UnifiedTransformer + + +class GenUnifiedTransformer(UnifiedTransformer): + """ + Implement generation unified transformer. + """ + + def __init__(self, model_dir, config, reader, generator): + super(GenUnifiedTransformer, self).__init__(model_dir, config, reader, + generator) + self.understand = config.BPETextField.understand + if torch.cuda.is_available(): + self.use_gpu = True + if self.use_gpu: + self.cuda() + return + + def _forward(self, inputs, is_training, with_label): + """ Real forward process of model in different mode(train/test). """ + + def cat(x, y, dim=1): + return torch.cat([x, y], dim=dim) + + outputs = {} + + if self.understand or self.policy: + if self.understand: + prompt_token = inputs['understand_token'] + prompt_mask = inputs['understand_mask'] + if self.policy: + prompt_token = cat(prompt_token, inputs['policy_token']) + prompt_mask = cat(prompt_mask, inputs['policy_mask']) + else: + prompt_token = inputs['policy_token'] + prompt_mask = inputs['policy_mask'] + + enc_embed, dec_embed, prompt_embed = self._encoder_prompt_decoder_network( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + tgt_token=inputs['tgt_token'][:, :-1], + tgt_mask=inputs['tgt_mask'][:, :-1], + prompt_token=prompt_token, + prompt_mask=prompt_mask, + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn'], + tgt_pos=inputs['tgt_pos'][:, :-1], + tgt_type=inputs['tgt_type'][:, :-1], + tgt_turn=inputs['tgt_turn'][:, :-1]) + else: + enc_embed, dec_embed = self._encoder_decoder_network( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + tgt_token=inputs['tgt_token'][:, :-1], + tgt_mask=inputs['tgt_mask'][:, :-1], + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn'], + tgt_pos=inputs['tgt_pos'][:, :-1], + tgt_type=inputs['tgt_type'][:, :-1], + tgt_turn=inputs['tgt_turn'][:, :-1]) + + outputs['dec_probs'] = self._dec_head(dec_embed=dec_embed) + return outputs + + def _collect_metrics(self, inputs, outputs, with_label, data_file): + + metrics = {} + loss = 0. + + label = inputs['tgt_token'][:, 1:] + token_num = torch.sum(torch.sum(inputs['tgt_mask'], dim=1) - 1) + nll = self.nll_loss( + torch.log(outputs['dec_probs'] + 1e-12).permute(0, 2, 1), label) + nll = torch.sum(nll, dim=1) + token_nll = torch.sum(nll) / token_num + nll = torch.mean(nll) + metrics['nll'] = nll + metrics['token_nll'] = token_nll + metrics['token_num'] = token_num + loss = loss + (token_nll if self.token_loss else nll) + + metrics['loss'] = loss + if self.gpu > 1: + return nll, token_nll, token_num + else: + return metrics + + def _optimize(self, loss, do_update=False, optimizer=None): + """ Optimize loss function and update model. """ + assert optimizer is not None + + if self.gradient_accumulation_steps > 1: + loss = loss / self.gradient_accumulation_steps + + loss.backward() + + if self.grad_clip is not None and self.grad_clip > 0: + torch.nn.utils.clip_grad_norm_( + parameters=self.parameters(), max_norm=self.grad_clip) + + if do_update: + optimizer.step() + optimizer.zero_grad() + + return + + def _init_state(self, + src_token, + src_mask, + src_pos=None, + src_type=None, + src_turn=None): + """ Initialize decode state. """ + state = {} + batch_size = src_token.shape[0] + + src_embed = self.embedder(src_token, src_pos, src_type, src_turn) + src_embed = self.embed_layer_norm(src_embed) + + mask = self._create_mask(src_mask, append_head=False) + + enc_out = src_embed + + cache = {} + for _l, layer in enumerate(self.layers): + cache[f'layer_{_l}'] = {} + enc_out = layer(enc_out, mask, cache[f'layer_{_l}']) + + state['cache'] = cache + state['mask'] = mask[:, :1] + state['batch_size'] = batch_size + shape = [batch_size, 1, 1] + state['pred_mask'] = torch.ones(shape, dtype=torch.float32) + state['pred_pos'] = torch.zeros(shape, dtype=torch.int64) + state['pred_type'] = torch.zeros(shape, dtype=torch.int64) + state['pred_turn'] = torch.zeros(shape, dtype=torch.int64) + if self.use_gpu: + state['pred_mask'] = state['pred_mask'].cuda() + state['pred_pos'] = state['pred_pos'].cuda() + state['pred_type'] = state['pred_type'].cuda() + state['pred_turn'] = state['pred_turn'].cuda() + + return state + + def _init_prompt_state(self, + src_token, + src_mask, + prompt_token, + prompt_mask, + src_pos=None, + src_type=None, + src_turn=None, + prompt_pos=None, + prompt_type=None, + prompt_turn=None): + """ Initialize decode state. """ + state = {} + batch_size = src_token.shape[0] + + src_embed = self.embedder(src_token, src_pos, src_type, src_turn) + prompt_embed = self.embedder(prompt_token, prompt_pos, prompt_type, + prompt_turn) + embed = torch.cat([src_embed, prompt_embed], dim=1) + embed = self.embed_layer_norm(embed) + enc_out = embed + + enc_mask = self._create_mask(src_mask, auto_regressive=False) + dec_mask = self._create_mask(prompt_mask, auto_regressive=True) + mask = self._join_mask(enc_mask, dec_mask) + + cache = {} + for _l, layer in enumerate(self.layers): + cache[f'layer_{_l}'] = {} + enc_out = layer(enc_out, mask, cache[f'layer_{_l}']) + + state['cache'] = cache + state['mask'] = mask[:, -1:] # state["mask"] = mask[:, :1] + state['batch_size'] = batch_size + shape = [batch_size, 1, 1] + state['pred_mask'] = torch.ones(shape, dtype=torch.float32) + state['pred_pos'] = torch.zeros(shape, dtype=torch.int64) + state['pred_type'] = torch.zeros(shape, dtype=torch.int64) + state['pred_turn'] = torch.zeros(shape, dtype=torch.int64) + if self.use_gpu: + state['pred_mask'] = state['pred_mask'].cuda() + state['pred_pos'] = state['pred_pos'].cuda() + state['pred_type'] = state['pred_type'].cuda() + state['pred_turn'] = state['pred_turn'].cuda() + + return state + + def _decode(self, state): + """ Decoding one time stamp. """ + + # shape: [batch_size, 1, seq_len] + mask = state['mask'] + + # shape: [batch_size, 1, 1] + if self.use_gpu: + pred_token = state['pred_token'].cuda() + pred_mask = state['pred_mask'].cuda() + pred_pos = state['pred_pos'].cuda() + pred_type = state['pred_type'].cuda() + pred_turn = state['pred_turn'].cuda() + else: + pred_token = state['pred_token'] + pred_mask = state['pred_mask'] + pred_pos = state['pred_pos'] + pred_type = state['pred_type'] + pred_turn = state['pred_turn'] + + # list of shape(len: num_layers): [batch_size, seq_len, hidden_dim] + cache = state['cache'] + pred_embed = self.embedder(pred_token, pred_pos, pred_type, + pred_turn).squeeze(-2) + pred_embed = self.embed_layer_norm(pred_embed) + + # shape: [batch_size, 1, seq_len + 1] + mask = torch.cat([mask, 1 - pred_mask], dim=2) + + # shape: [batch_size, 1, hidden_dim] + for _l, layer in enumerate(self.layers): + pred_embed = layer(pred_embed, mask, cache[f'layer_{_l}']) + + # shape: [batch_size, vocab_size] + pred_probs = self._dec_head(dec_embed=pred_embed[:, 0]) + pred_logits = torch.log(pred_probs) + + state['mask'] = mask + return pred_logits, state + + def _infer(self, + inputs, + start_id=None, + eos_id=None, + max_gen_len=None, + prev_input=None): + """ Real inference process of model. """ + + def cat(x, y, dim=1): + return torch.cat([x, y], dim=dim) + + # Initial decode state. + if self.understand or self.policy: + if self.understand: + prompt_token = inputs['understand_token'] + prompt_mask = inputs['understand_mask'] + if self.policy: + prompt_token = cat(prompt_token, inputs['policy_token']) + prompt_mask = cat(prompt_mask, inputs['policy_mask']) + else: + prompt_token = inputs['policy_token'] + prompt_mask = inputs['policy_mask'] + + state = self._init_prompt_state( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + prompt_token=prompt_token, + prompt_mask=prompt_mask, + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn']) + else: + state = self._init_state( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn']) + + # Generation process. + gen_results = self.generator( + step_fn=self._decode, + state=state, + start_id=start_id, + eos_id=eos_id, + max_gen_len=max_gen_len, + prev_input=prev_input) + + outputs = gen_results['preds'] + return outputs + + +GenUnifiedTransformer.register('GenUnifiedTransformer') diff --git a/runtime/Lib/site_packages/models/nlp/space/model/generator.py b/runtime/Lib/site_packages/models/nlp/space/model/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..e19fd29b71c4b9faf362af5c56eac43a4a7df620 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/generator.py @@ -0,0 +1,283 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import numpy as np +import torch + + +def repeat(var, times): + if isinstance(var, list): + return [repeat(x, times) for x in var] + elif isinstance(var, dict): + return {k: repeat(v, times) for k, v in var.items()} + elif isinstance(var, torch.Tensor): + var = var.unsqueeze(1) + expand_times = [1] * len(var.shape) + expand_times[1] = times + dtype = var.dtype + var = var.float() + var = var.repeat(*expand_times) + shape = [var.shape[0] * var.shape[1]] + list(var.shape[2:]) + var = var.reshape(*shape) + var = torch.tensor(var, dtype=dtype) + return var + else: + return var + + +def gather(var, idx): + if isinstance(var, list): + return [gather(x, idx) for x in var] + elif isinstance(var, dict): + return {k: gather(v, idx) for k, v in var.items()} + elif isinstance(var, torch.Tensor): + out = var.index_select(dim=0, index=idx) + return out + else: + return var + + +class SpaceGenerator(object): + """ Genrator class. """ + + _registry = dict() + + @classmethod + def register(cls, name): + SpaceGenerator._registry[name] = cls + return + + @staticmethod + def by_name(name): + return SpaceGenerator._registry[name] + + @staticmethod + def create(config, *args, **kwargs): + """ Create generator. """ + generator_cls = SpaceGenerator.by_name(config.Generator.generator) + return generator_cls(config, *args, **kwargs) + + def __init__(self, config, reader): + self.vocab_size = reader.vocab_size + self.bos_id = reader.bos_id + self.eos_id = reader.eos_id + self.unk_id = reader.unk_id + self.pad_id = reader.pad_id + self.min_gen_len = config.Generator.min_gen_len + self.max_gen_len = config.Generator.max_gen_len + self.use_gpu = config.use_gpu + if torch.cuda.is_available(): + self.use_gpu = True + assert 1 <= self.min_gen_len <= self.max_gen_len + return + + def __call__(self, step_fn, state): + """Running generation. + + Args: + step_fn (`function`) : decoding one step + state(`dict`) : initial state + """ + raise NotImplementedError + + +class BeamSearch(SpaceGenerator): + """ BeamSearch generator. """ + + def __init__(self, config, reader): + super().__init__(config, reader) + self.beam_size = config.Generator.beam_size + self.length_average = config.Generator.length_average + self.length_penalty = config.Generator.length_penalty + self.ignore_unk = config.Generator.ignore_unk + return + + def __call__(self, + step_fn, + state, + start_id=None, + eos_id=None, + max_gen_len=None, + prev_input=None): + """ + Running beam search. + + Args: + step_fn(`function`) : decoding one step + state(`dict`) : initial state + """ + if prev_input is not None: + + if isinstance(prev_input, list): + length = max(list(map(lambda x: len(x), prev_input))) + prev_input_numpy = np.full((len(prev_input), length), + self.pad_id) + for i, x in enumerate(prev_input): + prev_input_numpy[i, :len(x)] = x + prev_input_tensor = torch.from_numpy(prev_input_numpy) + if self.use_gpu: + prev_input_tensor = prev_input_tensor.cuda() + + for i in range(length): + state['pred_token'] = prev_input_tensor[:, i].unsqueeze( + -1).unsqueeze(-1) + if i != 0: + state['pred_mask'] = torch.not_equal( + state['pred_token'], self.pad_id).float() + state['pred_pos'] = state['pred_pos'] + state[ + 'pred_mask'].int() + _, state = step_fn(state) + else: + assert isinstance(prev_input, torch.Tensor) + for i, input in enumerate(prev_input): + state['pred_token'] = input.expand(1, 1, 1) + if i != 0: + state['pred_mask'] = torch.not_equal( + state['pred_token'], self.pad_id).float() + state['pred_pos'] = state['pred_pos'] + 1 + _, state = step_fn(state) + + batch_size = state['batch_size'] + beam_size = self.beam_size + + # shape: [batch_size, 1] + pos_index = torch.arange( + 0, batch_size, 1, dtype=torch.int64) * beam_size + pos_index = pos_index.unsqueeze(1) + + # shape: [batch_size, beam_size, 1] + if start_id is None: + start_id = self.bos_id + if eos_id is None: + eos_id = self.eos_id + predictions = torch.ones([batch_size, beam_size, 1], + dtype=torch.int64) * start_id + + if self.use_gpu: + pos_index = pos_index.cuda() + predictions = predictions.cuda() + + # initial input (start_id) + state['pred_token'] = predictions[:, :1] + if prev_input is not None: + state['pred_mask'] = torch.not_equal(state['pred_token'], + self.pad_id).float() + state['pred_pos'] = state['pred_pos'] + 1 + + # shape: [batch_size, vocab_size] + scores, state = step_fn(state) + + unk_penalty = np.zeros(self.vocab_size, dtype='float32') + unk_penalty[self.unk_id] = -1e10 + unk_penalty = torch.from_numpy(unk_penalty) + + eos_penalty = np.zeros(self.vocab_size, dtype='float32') + eos_penalty[eos_id] = -1e10 + eos_penalty = torch.from_numpy(eos_penalty) + + scores_after_end = np.full(self.vocab_size, -1e10, dtype='float32') + scores_after_end[ + self. + pad_id] = 0 # we want is generated after ,so maximum log(p()) is (0) + scores_after_end = torch.from_numpy(scores_after_end) + + if self.use_gpu: + unk_penalty = unk_penalty.cuda() + eos_penalty = eos_penalty.cuda() + scores_after_end = scores_after_end.cuda() + if self.ignore_unk: + scores = scores + unk_penalty + scores = scores + eos_penalty + + # shape: [batch_size, beam_size] + sequence_scores, preds = torch.topk(scores, self.beam_size) + + predictions = torch.cat([predictions, preds.unsqueeze(2)], dim=2) + state = repeat(state, beam_size) + + if max_gen_len is None: + max_gen_len = self.max_gen_len + for step in range(2, max_gen_len + 1): + pre_ids = predictions[:, :, -1:] + state['pred_token'] = pre_ids.reshape(batch_size * beam_size, 1, 1) + state['pred_mask'] = torch.not_equal(state['pred_token'], + self.pad_id).float() + state['pred_pos'] = state['pred_pos'] + 1 + scores, state = step_fn(state) + + # Generate next + # scores shape: [batch_size * beam_size, vocab_size] + if self.ignore_unk: + scores = scores + unk_penalty + + if step <= self.min_gen_len: + scores = scores + eos_penalty + + # scores shape: [batch_size, beam_size, vocab_size] + scores = scores.reshape(batch_size, beam_size, self.vocab_size) + + # previous token is [PAD] or [EOS] + pre_eos_mask = (1 - torch.not_equal(pre_ids, eos_id).float()) + \ + (1 - torch.not_equal(pre_ids, self.pad_id).float()) + + scores = scores * (1 - pre_eos_mask) + pre_eos_mask.repeat( + 1, 1, self.vocab_size) * scores_after_end + if self.length_average: + scaled_value = \ + pre_eos_mask + (1 - pre_eos_mask) * (1 - 1 / step) + sequence_scores = sequence_scores.unsqueeze(2) * scaled_value + scaled_value = pre_eos_mask + (1 - pre_eos_mask) * (1 / step) + scores = scores * scaled_value + elif self.length_penalty >= 0.0: + scaled_value = pre_eos_mask + (1 - pre_eos_mask) * \ + (math.pow((4 + step) / (5 + step), self.length_penalty)) + sequence_scores = scaled_value * sequence_scores + scaled_value = pre_eos_mask + (1 - pre_eos_mask) * \ + (math.pow(1 / (5 + step), self.length_penalty)) + scores = scores * scaled_value + scores = scores + sequence_scores.unsqueeze(-1) + scores = scores.reshape(batch_size, beam_size * self.vocab_size) + + topk_scores, topk_indices = torch.topk(scores, beam_size) + # topk_indices: [batch_size, beam_size * self.vocab_size] (already reshaped) + parent_idx = topk_indices.floor_divide(self.vocab_size) + preds = topk_indices % self.vocab_size + + # Gather state / sequence_scores + parent_idx = parent_idx + pos_index + parent_idx = parent_idx.reshape(batch_size * beam_size) + state = gather(state, parent_idx) + sequence_scores = topk_scores + + predictions = predictions.reshape(batch_size * beam_size, step) + predictions = gather(predictions, parent_idx) + predictions = predictions.reshape(batch_size, beam_size, step) + predictions = torch.cat([predictions, preds.unsqueeze(2)], dim=2) + + # The last token should be or + pre_ids = predictions[:, :, -1] + pre_eos_mask = (1 - torch.not_equal(pre_ids, eos_id).float()) + \ + (1 - torch.not_equal(pre_ids, self.pad_id).float()) + sequence_scores = sequence_scores * pre_eos_mask + ( + 1 - pre_eos_mask) * (-1e10) + + # first get ascending ordered index,then sort "predictions" and "sequence_scores" + indices = torch.argsort(sequence_scores, dim=1) + indices = indices + pos_index + indices = indices.reshape(-1) + sequence_scores = sequence_scores.reshape(batch_size * beam_size) + predictions = predictions.reshape(batch_size * beam_size, -1) + sequence_scores = gather(sequence_scores, indices) + predictions = gather(predictions, indices) + sequence_scores = sequence_scores.reshape(batch_size, beam_size) + predictions = predictions.reshape(batch_size, beam_size, -1) + + results = { + 'preds': predictions[:, -1], + 'scores': sequence_scores[:, -1] + } + return results + + +BeamSearch.register('BeamSearch') diff --git a/runtime/Lib/site_packages/models/nlp/space/model/intent_unified_transformer.py b/runtime/Lib/site_packages/models/nlp/space/model/intent_unified_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..11385a6f291204ef4e5ba193d4f1d6a79d68cf8d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/intent_unified_transformer.py @@ -0,0 +1,197 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.utils.nlp.space.criterions import compute_kl_loss +from .unified_transformer import UnifiedTransformer + + +class IntentUnifiedTransformer(UnifiedTransformer): + """ + Implement intent unified transformer. + """ + + def __init__(self, model_dir, config, reader, generator): + super(IntentUnifiedTransformer, self).__init__(model_dir, config, + reader, generator) + self.example = config.Model.example + self.num_intent = config.Model.num_intent + self.with_rdrop = config.Model.with_rdrop + self.kl_ratio = config.Model.kl_ratio + self.loss_fct = nn.CrossEntropyLoss() + if self.example: + self.loss_fct = nn.NLLLoss() + else: + self.intent_classifier = nn.Linear(self.hidden_dim, + self.num_intent) + self.loss_fct = nn.CrossEntropyLoss() + + if self.use_gpu: + self.cuda() + return + + def _forward(self, inputs, is_training, with_label): + """ Real forward process of model in different mode(train/test). """ + + def aug(v): + assert isinstance(v, torch.Tensor) + return torch.cat([v, v], dim=0) + + outputs = {} + + if self.with_mlm: + mlm_embed = self._encoder_network( + input_token=inputs['mlm_token'], + input_mask=inputs['src_mask'], + input_pos=inputs['src_pos'], + input_type=inputs['src_type'], + input_turn=inputs['src_turn']) + outputs['mlm_probs'] = self._mlm_head(mlm_embed=mlm_embed) + + if self.with_rdrop or self.with_contrastive: + enc_embed, dec_embed = self._encoder_decoder_network( + src_token=aug(inputs['src_token']), + src_mask=aug(inputs['src_mask']), + tgt_token=aug(inputs['tgt_token']), + tgt_mask=aug(inputs['tgt_mask']), + src_pos=aug(inputs['src_pos']), + src_type=aug(inputs['src_type']), + src_turn=aug(inputs['src_turn'])) + else: + enc_embed, dec_embed = self._encoder_decoder_network( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + tgt_token=inputs['tgt_token'], + tgt_mask=inputs['tgt_mask'], + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn']) + features = dec_embed[:, -1] + features = self.pooler(features) if self.with_pool else features + + if self.example: + assert not self.with_rdrop + ex_enc_embed, ex_dec_embed = self._encoder_decoder_network( + src_token=inputs['example_src_token'], + src_mask=inputs['example_src_mask'], + tgt_token=inputs['example_tgt_token'], + tgt_mask=inputs['example_tgt_mask'], + src_pos=inputs['example_src_pos'], + src_type=inputs['example_src_type'], + src_turn=inputs['example_src_turn']) + ex_features = ex_dec_embed[:, -1] + ex_features = self.pooler( + ex_features) if self.with_pool else ex_features + + probs = self.softmax(features.mm(ex_features.t())) + example_intent = inputs['example_intent'].unsqueeze(0) + intent_probs = torch.zeros(probs.size(0), self.num_intent) + intent_probs = intent_probs.cuda( + ) if self.use_gpu else intent_probs + intent_probs = intent_probs.scatter_add( + -1, example_intent.repeat(probs.size(0), 1), probs) + outputs['intent_probs'] = intent_probs + else: + intent_logits = self.intent_classifier(features) + outputs['intent_logits'] = intent_logits + + if self.with_contrastive: + features = features if self.with_pool else self.pooler(features) + batch_size = features.size(0) // 2 + features = \ + torch.cat( + [features[:batch_size].unsqueeze(1), features[batch_size:].unsqueeze(1)], + dim=1 + ) + features = F.normalize(features, dim=-1, p=2) + outputs['features'] = features + + return outputs + + def _collect_metrics(self, inputs, outputs, with_label, data_file): + + metrics = {} + batch_size = inputs['src_token'].size(0) + + intent_label = torch.cat([inputs['intent_label'], inputs['intent_label']], dim=0) \ + if self.with_rdrop or self.with_contrastive else inputs['intent_label'] + + if self.example: + intent_loss = self.loss_fct( + torch.log(outputs['intent_probs'] + 1e-12).view( + -1, self.num_intent), intent_label.type(torch.long)) + else: + intent_loss = self.loss_fct( + outputs['intent_logits'].view(-1, self.num_intent), + intent_label.type(torch.long)) + metrics['intent_loss'] = intent_loss + loss = intent_loss + + if self.with_mlm: + mlm_num = torch.sum(torch.sum(inputs['mlm_mask'], dim=1)) + mlm = self.nll_loss( + torch.log(outputs['mlm_probs'] + 1e-12).permute(0, 2, 1), + inputs['mlm_label']) + mlm = torch.sum(mlm, dim=1) + token_mlm = torch.sum(mlm) / mlm_num + mlm = torch.mean(mlm) + metrics['mlm'] = mlm + metrics['token_mlm'] = token_mlm + metrics['mlm_num'] = mlm_num + loss = loss + (token_mlm + if self.token_loss else mlm) * self.mlm_ratio + else: + mlm, token_mlm, mlm_num = None, None, None + + if self.with_rdrop: + kl = compute_kl_loss( + p=outputs['intent_logits'][:batch_size], + q=outputs['intent_logits'][batch_size:]) + metrics['kl'] = kl + loss = loss + kl * self.kl_ratio + else: + kl = None + + if self.with_contrastive: + pass + con = None + else: + con = None + + metrics['loss'] = loss + + if self.gpu > 1: + return intent_loss, mlm, token_mlm, mlm_num, kl, con + else: + return metrics + + def _infer(self, + inputs, + start_id=None, + eos_id=None, + max_gen_len=None, + prev_input=None): + """ Real inference process of model. """ + results = {} + enc_embed, dec_embed = self._encoder_decoder_network( + src_token=inputs['src_token'], + src_mask=inputs['src_mask'], + tgt_token=inputs['tgt_token'], + tgt_mask=inputs['tgt_mask'], + src_pos=inputs['src_pos'], + src_type=inputs['src_type'], + src_turn=inputs['src_turn']) + features = dec_embed[:, -1] + features = self.pooler(features) if self.with_pool else features + if self.example: + results['features'] = features + else: + intent_logits = self.intent_classifier(features) + intent_probs = self.softmax(intent_logits) + results['intent_probs'] = intent_probs + return results + + +IntentUnifiedTransformer.register('IntentUnifiedTransformer') diff --git a/runtime/Lib/site_packages/models/nlp/space/model/model_base.py b/runtime/Lib/site_packages/models/nlp/space/model/model_base.py new file mode 100644 index 0000000000000000000000000000000000000000..4c5755e9c0362acedd0a8010385a9851261f4834 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/model_base.py @@ -0,0 +1,100 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +import torch.nn as nn + +from modelscope.utils.constant import ModelFile + + +class SpaceModelBase(nn.Module): + """ + Basic model wrapper for static graph and dygrpah. + """ + _registry = dict() + + @classmethod + def register(cls, name): + SpaceModelBase._registry[name] = cls + return + + @staticmethod + def by_name(name): + return SpaceModelBase._registry[name] + + @staticmethod + def create(model_dir, config, *args, **kwargs): + model_cls = SpaceModelBase.by_name(config.Model.model) + return model_cls(model_dir, config, *args, **kwargs) + + def __init__(self, model_dir, config): + super(SpaceModelBase, self).__init__() + self.init_checkpoint = os.path.join(model_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + self.abandon_label = config.Dataset.abandon_label + self.use_gpu = config.use_gpu + self.gpu = config.Trainer.gpu + return + + def _create_parameters(self): + """ Create model's parameters. """ + raise NotImplementedError + + def _forward(self, inputs, is_training, with_label): + """ NO LABEL: Real forward process of model in different mode(train/test). """ + raise NotImplementedError + + def _collect_metrics(self, inputs, outputs, with_label, data_file): + """ NO LABEL: Calculate loss function by using inputs and outputs. """ + raise NotImplementedError + + def _optimize(self, loss, optimizer, lr_scheduler): + """ Optimize loss function and update model. """ + raise NotImplementedError + + def _infer(self, inputs, start_id, eos_id, max_gen_len, prev_input): + """ Real inference process of model. """ + raise NotImplementedError + + def forward(self, + inputs, + is_training=False, + with_label=False, + data_file=None): + """ + Forward process, include real forward, collect metrices and optimize(optional) + + Args: + inputs(`dict` of numpy.ndarray/int/float/...) : input data + """ + if is_training: + self.train() + else: + self.eval() + + with_label = False if self.abandon_label else with_label + outputs = self._forward(inputs, is_training, with_label=with_label) + metrics = self._collect_metrics( + inputs, outputs, with_label=with_label, data_file=data_file) + + return metrics + + def infer(self, + inputs, + start_id=None, + eos_id=None, + max_gen_len=None, + prev_input=None): + """Inference process. + + Args: + inputs(`dict` of numpy.ndarray/int/float/...) : input data + """ + self.eval() + results = self._infer( + inputs, + start_id=start_id, + eos_id=eos_id, + max_gen_len=max_gen_len, + prev_input=prev_input) + return results diff --git a/runtime/Lib/site_packages/models/nlp/space/model/tokenization_space.py b/runtime/Lib/site_packages/models/nlp/space/model/tokenization_space.py new file mode 100644 index 0000000000000000000000000000000000000000..cc57eb033e38720fccc36e746b2c81ddd7d94a89 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/tokenization_space.py @@ -0,0 +1,29 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Tokenization classes for Space. mainly copied from :module:`~transformers.tokenization_xlm_roberta`""" + +from transformers import BasicTokenizer, BertTokenizer, WordpieceTokenizer + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class SpaceTokenizer(BertTokenizer): + """ + This class overrides [`SpaceTokenizer`]. Please check the superclass for the appropriate + documentation alongside usage examples. + """ diff --git a/runtime/Lib/site_packages/models/nlp/space/model/unified_transformer.py b/runtime/Lib/site_packages/models/nlp/space/model/unified_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..53bf3146ae039ede57e32d711ff46f2e86e9ed19 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/model/unified_transformer.py @@ -0,0 +1,308 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.models.nlp.space.model.model_base import SpaceModelBase +from modelscope.models.nlp.space.modules.embedder import Embedder +from modelscope.models.nlp.space.modules.transformer_block import \ + TransformerBlock + + +class UnifiedTransformer(SpaceModelBase): + """ + Implement unified transformer. + """ + + def __init__(self, model_dir, config, reader, generator, dtype='float32'): + super(UnifiedTransformer, self).__init__(model_dir, config) + self.reader = reader + self.generator = generator + self.policy = config.BPETextField.policy + self.generation = config.BPETextField.generation + self.num_token_embeddings = config.Model.num_token_embeddings + self.num_pos_embeddings = config.Model.num_pos_embeddings + self.num_type_embeddings = config.Model.num_type_embeddings + self.num_turn_embeddings = config.Model.num_turn_embeddings + self.temperature = config.Model.temperature + self.hidden_dim = config.Model.hidden_dim + self.num_heads = config.Model.num_heads + self.num_layers = config.Model.num_layers + self.padding_idx = config.Model.padding_idx + self.dropout = config.Model.dropout + self.embed_dropout = config.Model.embed_dropout + self.attn_dropout = config.Model.attn_dropout + self.ff_dropout = config.Model.ff_dropout + self.mlm_ratio = config.Model.mlm_ratio + self.mmd_ratio = config.Model.mmd_ratio + self.pos_trainable = config.Model.pos_trainable + self.label_smooth = config.Model.label_smooth + self.initializer_range = config.Model.initializer_range + self.gradient_accumulation_steps = config.Model.gradient_accumulation_steps + self.token_loss = config.Trainer.token_loss + self.learning_method = config.Dataset.learning_method + self.with_contrastive = config.Dataset.with_contrastive + self.with_query_bow = config.BPETextField.with_query_bow + self.with_resp_bow = config.BPETextField.with_resp_bow + self.with_pool = config.Model.with_pool + self.with_mlm = config.Dataset.with_mlm + self._dtype = dtype + + self.embedder = Embedder( + self.hidden_dim, + self.num_token_embeddings, + self.num_pos_embeddings, + self.num_type_embeddings, + self.num_turn_embeddings, + padding_idx=self.padding_idx, + dropout=self.embed_dropout, + pos_trainable=self.pos_trainable) + self.embed_layer_norm = nn.LayerNorm( + normalized_shape=self.hidden_dim, + eps=1e-12, + elementwise_affine=True) + + self.layers = nn.ModuleList([ + TransformerBlock(self.hidden_dim, self.num_heads, self.dropout, + self.attn_dropout, self.ff_dropout) + for _ in range(config.Model.num_layers) + ]) + + if self.with_mlm: + self.mlm_transform = nn.Sequential( + nn.Linear(self.hidden_dim, self.hidden_dim), nn.GELU(), + nn.LayerNorm( + normalized_shape=self.hidden_dim, + eps=1e-12, + elementwise_affine=True)) + self.mlm_bias = nn.Parameter( + torch.zeros(self.num_token_embeddings)) + + self.pooler = nn.Sequential( + nn.Linear(self.hidden_dim, self.hidden_dim), nn.Tanh()) + + if self.with_query_bow or self.with_resp_bow: + self.bow_predictor = nn.Linear( + self.hidden_dim, self.num_token_embeddings, bias=False) + + self.sigmoid = nn.Sigmoid() + self.softmax = nn.Softmax(dim=-1) + self.bce_loss = nn.BCELoss(reduction='none') + self.nll_loss = nn.NLLLoss( + ignore_index=self.padding_idx, reduction='none') + self._create_parameters() + + self.max_grad_norm = config.Model.max_grad_norm + if self.max_grad_norm is not None: + self.grad_clip = self.max_grad_norm + else: + self.grad_clip = None + self.weight_decay = config.Model.weight_decay + + if self.use_gpu: + self.cuda() + + return + + def _create_parameters(self): + """ Create model's parameters. """ + sequence_mask = np.tri( + self.num_pos_embeddings, + self.num_pos_embeddings, + dtype=self._dtype) + self.sequence_mask = torch.tensor(sequence_mask) + return + + def _create_mask(self, + input_mask, + append_head=False, + auto_regressive=False): + """Create attention mask. + from sequence to matrix:[batch_size, max_seq_len, 1] -> [batch_size, max_seq_len, max_seq_len] + + Args: + input_mask (Variable(shape: [batch_size, max_seq_len])) + auto_regressive(bool) + """ + seq_len = input_mask.shape[1] + + input_mask = input_mask.float() + mask1 = input_mask.unsqueeze(-1).repeat(1, 1, seq_len) + mask2 = mask1.permute(0, 2, 1) + mask = mask1 * mask2 + + if append_head: + mask = torch.cat([mask[:, :1, :], mask], dim=1) + mask = torch.cat([mask[:, :, :1], mask], dim=2) + seq_len += 1 + + if auto_regressive: + seq_mask = self.sequence_mask[:seq_len, :seq_len] + seq_mask = seq_mask.to(mask.device) + mask = mask * seq_mask + + mask = 1 - mask + return mask + + def _join_mask(self, mask1, mask2): + """Merge source attention mask and target attention mask. + There are four parts:left upper (lu) / right upper (ru) / left below (lb) / right below (rb) + + Args: + mask1(Variable(shape: [batch_size, max_src_len, max_src_len])) : source attention mask + mask2(Variable(shape: [batch_size, max_tgt_len, max_tgt_len])) : target attention mask + """ + batch_size = mask1.shape[0] + seq_len1 = mask1.shape[1] + seq_len2 = mask2.shape[1] + # seq_len = seq_len1 + seq_len2 + + mask_lu = mask1 + mask_ru = torch.ones(batch_size, seq_len1, seq_len2).to(mask_lu.device) + if self.use_gpu: + mask_ru = mask_ru.cuda() + mask3 = mask2[:, :, :1].repeat(1, 1, seq_len1) + mask4 = mask1[:, :1].repeat(1, seq_len2, 1) + mask_lb = mask3 + mask4 - mask3 * mask4 + mask_rb = mask2 + mask_u = torch.cat([mask_lu, mask_ru], dim=2) + mask_b = torch.cat([mask_lb, mask_rb], dim=2) + mask = torch.cat([mask_u, mask_b], dim=1) + return mask + + def _mlm_head(self, mlm_embed): + mlm_embed = self.mlm_transform(mlm_embed) + mlm_logits = torch.matmul( + mlm_embed, self.embedder.token_embedding.weight.T) + self.mlm_bias + mlm_probs = self.softmax(mlm_logits) + return mlm_probs + + def _dec_head(self, dec_embed): + dec_logits = torch.matmul(dec_embed, + self.embedder.token_embedding.weight.T) + dec_probs = self.softmax(dec_logits) + return dec_probs + + def _refactor_feature(self, features): + features = self.pooler(features) if self.with_pool else features + batch_size = features.size(0) // 2 + features = \ + torch.cat( + [features[:batch_size].unsqueeze(1), features[batch_size:].unsqueeze(1)], + dim=1 + ) + features = F.normalize(features, dim=-1, p=2) + return features + + def _encoder_network(self, + input_token, + input_mask, + input_pos=None, + input_type=None, + input_turn=None): + embed = self.embedder(input_token, input_pos, input_type, input_turn) + embed = self.embed_layer_norm(embed) + mask = self._create_mask(input_mask, auto_regressive=False) + + for layer in self.layers: + embed = layer(embed, mask, None) + + return embed + + def _encoder_decoder_network(self, + src_token, + src_mask, + tgt_token, + tgt_mask, + src_pos=None, + src_type=None, + src_turn=None, + tgt_pos=None, + tgt_type=None, + tgt_turn=None): + src_embed = self.embedder(src_token, src_pos, src_type, src_turn) + tgt_embed = self.embedder(tgt_token, tgt_pos, tgt_type, tgt_turn) + embed = torch.cat([src_embed, tgt_embed], dim=1) + embed = self.embed_layer_norm(embed) + + enc_mask = self._create_mask(src_mask, auto_regressive=False) + dec_mask = self._create_mask(tgt_mask, auto_regressive=True) + mask = self._join_mask(enc_mask, dec_mask) + + for layer in self.layers: + embed = layer(embed, mask, None) + + tgt_len = tgt_token.shape[1] + enc_embed = embed[:, :-tgt_len] + dec_embed = embed[:, -tgt_len:] + + return enc_embed, dec_embed + + def _encoder_prompt_decoder_network(self, + src_token, + src_mask, + tgt_token, + tgt_mask, + prompt_token, + prompt_mask, + src_pos=None, + src_type=None, + src_turn=None, + tgt_pos=None, + tgt_type=None, + tgt_turn=None, + prompt_pos=None, + prompt_type=None, + prompt_turn=None): + src_embed = self.embedder(src_token, src_pos, src_type, src_turn) + tgt_embed = self.embedder(tgt_token, tgt_pos, tgt_type, tgt_turn) + prompt_embed = self.embedder(prompt_token, prompt_pos, prompt_type, + prompt_turn) + + embed = torch.cat([src_embed, prompt_embed, tgt_embed], dim=1) + embed = self.embed_layer_norm(embed) + + enc_mask = self._create_mask(src_mask, auto_regressive=False) + dec_mask = self._create_mask( + torch.cat([prompt_mask, tgt_mask], dim=1), auto_regressive=True) + mask = self._join_mask(enc_mask, dec_mask) + + for layer in self.layers: + embed = layer(embed, mask, None) + + src_len = src_token.shape[1] + tgt_len = tgt_token.shape[1] + enc_embed = embed[:, :src_len] + dec_embed = embed[:, -tgt_len:] + prompt_embed = embed[:, src_len:-tgt_len] + + return enc_embed, dec_embed, prompt_embed + + def _optimize(self, loss, optimizer=None, lr_scheduler=None): + """ Optimize loss function and update model. """ + assert optimizer is not None + optimizer.zero_grad() + loss.backward() + + if self.grad_clip is not None and self.grad_clip > 0: + torch.nn.utils.clip_grad_norm_( + parameters=self.parameters(), max_norm=self.grad_clip) + optimizer.step() + if lr_scheduler is not None: + lr_scheduler.step() + return + + def _infer(self, + inputs, + start_id=None, + eos_id=None, + max_gen_len=None, + prev_input=None): + """ Real inference process of model. """ + results = {} + return results + + +UnifiedTransformer.register('UnifiedTransformer') diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/__init__.py b/runtime/Lib/site_packages/models/nlp/space/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/embedder.py b/runtime/Lib/site_packages/models/nlp/space/modules/embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..e68ac7d31e48b000a3b90b0b7610ff1c87e04bf2 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/modules/embedder.py @@ -0,0 +1,65 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + + +class Embedder(nn.Module): + """ + Composite embedding layer. + """ + + def __init__(self, + hidden_dim, + num_token_embeddings, + num_pos_embeddings, + num_type_embeddings, + num_turn_embeddings, + padding_idx=None, + dropout=0.1, + pos_trainable=False): + super(Embedder, self).__init__() + + self.token_embedding = nn.Embedding(num_token_embeddings, hidden_dim) + self.pos_embedding = nn.Embedding(num_pos_embeddings, hidden_dim) + self.pos_embedding.weight.requires_grad = pos_trainable + self.type_embedding = nn.Embedding(num_type_embeddings, hidden_dim) + self.turn_embedding = nn.Embedding(num_turn_embeddings, hidden_dim) + self.dropout_layer = nn.Dropout(p=dropout) + + # follow the default xavier_uniform initializer in paddle version + # otherwise, there are bugs for dec_probs computation in weight typing setting + # default norm initializer in nn.Embedding in pytorch, which samples larger values + nn.init.xavier_uniform_(self.token_embedding.weight) + nn.init.xavier_uniform_(self.pos_embedding.weight) + nn.init.xavier_uniform_(self.type_embedding.weight) + nn.init.xavier_uniform_(self.turn_embedding.weight) + return + + def forward(self, token_inp, pos_inp=None, type_inp=None, turn_inp=None): + embed = self.token_embedding(token_inp) + if pos_inp is not None: + embed += self.pos_embedding(pos_inp) + if type_inp is not None: + embed += self.type_embedding(type_inp) + if turn_inp is not None: + embed += self.turn_embedding(turn_inp) + embed = self.dropout_layer(embed) + return embed + + +def main(): + import numpy as np + + model = Embedder(10, 20, 20, 20, 20) + token_inp = torch.tensor( + np.random.randint(0, 19, [10, 10]).astype('int64')) + pos_inp = torch.tensor(np.random.randint(0, 19, [10, 10]).astype('int64')) + type_inp = torch.tensor(np.random.randint(0, 19, [10, 10]).astype('int64')) + turn_inp = torch.tensor(np.random.randint(0, 19, [10, 10]).astype('int64')) + out = model(token_inp, pos_inp, type_inp, turn_inp) + print(out) + + +if __name__ == '__main__': + main() diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/feedforward.py b/runtime/Lib/site_packages/models/nlp/space/modules/feedforward.py new file mode 100644 index 0000000000000000000000000000000000000000..43318eb63e3f60c723027002c5844de61d351a2f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/modules/feedforward.py @@ -0,0 +1,41 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + + +class FeedForward(nn.Module): + """ + Positional feed forward layer. + """ + + def __init__(self, hidden_dim, inner_dim, dropout): + super(FeedForward, self).__init__() + + self.hidden_dim = hidden_dim + self.inner_dim = inner_dim + self.linear_hidden = nn.Sequential( + nn.Linear(hidden_dim, inner_dim), nn.GELU()) + self.linear_out = nn.Linear(inner_dim, hidden_dim) + self.dropout_layer = nn.Dropout(p=dropout) + return + + def forward(self, x): + out = self.linear_hidden(x) + out = self.dropout_layer(out) + out = self.linear_out(out) + return out + + +def main(): + import numpy as np + + model = FeedForward(10, 20, 0.5) + inp = np.random.rand(2, 3, 10).astype('float32') + inp = torch.tensor(inp) + out = model(inp) + print(out) + + +if __name__ == '__main__': + main() diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/functions.py b/runtime/Lib/site_packages/models/nlp/space/modules/functions.py new file mode 100644 index 0000000000000000000000000000000000000000..daa62bb47726adc6ed32eae99771e3d125a4b4c9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/modules/functions.py @@ -0,0 +1,62 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import numpy as np +import torch +import torch.nn.functional as F + + +def unsqueeze(input, dims): + """ Implement multi-dimension unsqueeze function. """ + if isinstance(dims, (list, tuple)): + dims = [ + dim if dim >= 0 else dim + len(input.shape) + 1 for dim in dims + ] + dims = sorted(dims, reverse=True) + shape = list(input.shape) + for dim in dims: + shape.insert(dim, 1) + return torch.reshape(input, shape) + elif isinstance(dims, int): + return input.unsqueeze(dims) + else: + raise ValueError('Warning: type(dims) must in (list, tuple, int)!') + + +def gumbel_softmax(input, tau=1, eps=1e-10): + """ Basic implement of gumbel_softmax. """ + U = torch.tensor(np.random.rand(*input.shape)) + gumbel = 0.0 - torch.log(eps - torch.log(U + eps)) + y = input + gumbel + return F.softmax(y / tau) + + +def equal(x, y, dtype=None): + """ Implement equal in dygraph mode. (paddle) """ + if dtype is None: + dtype = 'float32' + if isinstance(x, torch.Tensor): + x = x.numpy() + if isinstance(y, torch.Tensor): + y = y.numpy() + out = np.equal(x, y).astype(dtype) + return torch.tensor(out) + + +def not_equal(x, y, dtype=None): + """ Implement not_equal in dygraph mode. (paddle) """ + return 1 - equal(x, y, dtype) + + +if __name__ == '__main__': + a = torch.tensor([[1, 1], [3, 4]]) + b = torch.tensor([[1, 1], [3, 4]]) + c = torch.equal(a, a) + c1 = equal(a, 3) + d = 1 - torch.not_equal(a, 3).float() + print(c) + print(c1) + print(d) + e = F.gumbel_softmax(a) + f = a.unsqueeze(a) + g = unsqueeze(a, dims=[0, 0, 1]) + print(g, g.shape) diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/multihead_attention.py b/runtime/Lib/site_packages/models/nlp/space/modules/multihead_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..d075e9c5ce6ae665b308ca73cdafa2b29b827b2b --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/modules/multihead_attention.py @@ -0,0 +1,105 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + + +class MultiheadAttention(nn.Module): + """ + Multi head attention layer. + """ + + def __init__(self, hidden_dim, num_heads, dropout): + assert hidden_dim % num_heads == 0 + super(MultiheadAttention, self).__init__() + + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.head_dim = hidden_dim // num_heads + self.scale = self.head_dim**-0.5 + self.linear_qkv = nn.Linear(hidden_dim, hidden_dim * 3) + self.linear_out = nn.Linear(hidden_dim, hidden_dim) + self.dropout_layer = nn.Dropout(p=dropout) + self.softmax = nn.Softmax(dim=-1) + return + + def _split_heads(self, x, is_key=False): + x = x.reshape(x.size(0), x.size(1), self.num_heads, self.head_dim) + x = x.permute(0, 2, 3, 1) if is_key else x.permute(0, 2, 1, 3) + return x + + def _merge_heads(self, x): + x = x.permute(0, 2, 1, 3) + x = x.reshape(x.size(0), x.size(1), self.hidden_dim) + return x + + def _attn(self, query, key, value, mask): + # shape: [batch_size, num_head, seq_len, seq_len] + scores = torch.matmul(query, key) + scores = scores * self.scale + + if mask is not None: + mask = mask.unsqueeze(1) + mask = mask.repeat(1, self.num_heads, 1, 1) + scores.masked_fill_( + mask.bool(), + float('-inf')) # scores = (1 - mask) * scores + mask * (-1e10) + + attn = self.softmax(scores) + attn = self.dropout_layer(attn) + + if mask is not None: + ''' + mask: [batch size, num_heads, seq_len, seq_len] + + >>> F.softmax([-1e10, -100, -100]) + >>> [0.00, 0.50, 0.50] + >>> F.softmax([-1e10, -1e10, -1e10]) + >>> [0.33, 0.33, 0.33] + ==> [0.00, 0.00, 0.00] + ''' + attn.masked_fill_(mask.bool(), 0.) # attn = (1 - mask) * attn + + out = torch.matmul(attn, value) + return out + + def forward(self, inp, mask=None, cache=None): + """ Forward process of self attention. """ + # shape: [batch_size, seq_len, 3 * hidden_dim] + qkv = self.linear_qkv(inp) + query, key, value = torch.split(qkv, self.hidden_dim, dim=2) + + # shape: [batch_size, num_head, seq_len, head_dim] + query = self._split_heads(query) + # shape: [batch_size, num_head, head_dim, seq_len] + key = self._split_heads(key, is_key=True) + # shape: [batch_size, num_head, seq_len, head_dim] + value = self._split_heads(value) + + if cache is not None: + if 'key' in cache and 'value' in cache: + key = torch.cat([cache['key'], key], dim=3) + value = torch.cat([cache['value'], value], dim=2) + cache['key'] = key + cache['value'] = value + + out = self._attn(query, key, value, mask) + out = self._merge_heads(out) + out = self.linear_out(out) + return out + + +def main(): + import numpy as np + + model = MultiheadAttention(10, 2, 0.5) + inp = np.random.rand(2, 3, 10).astype('float32') + inp = torch.tensor(inp) + mask = (np.random.rand(2, 3, 3) > 0.5).astype('float32') + mask = torch.tensor(mask) + out = model(inp, mask=mask, cache=None) + print(out) + + +if __name__ == '__main__': + main() diff --git a/runtime/Lib/site_packages/models/nlp/space/modules/transformer_block.py b/runtime/Lib/site_packages/models/nlp/space/modules/transformer_block.py new file mode 100644 index 0000000000000000000000000000000000000000..3044963ad1942e97f38e1957ce37c64dfe53176f --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space/modules/transformer_block.py @@ -0,0 +1,65 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import torch +import torch.nn as nn + +from .feedforward import FeedForward +from .multihead_attention import MultiheadAttention + + +class TransformerBlock(nn.Module): + """ + Transformer block module. + """ + + def __init__(self, hidden_dim, num_heads, dropout, attn_dropout, + ff_dropout): + super(TransformerBlock, self).__init__() + + self.attn = MultiheadAttention( + hidden_dim=hidden_dim, num_heads=num_heads, dropout=attn_dropout) + self.attn_norm = nn.LayerNorm( + normalized_shape=hidden_dim, eps=1e-12, elementwise_affine=True) + self.ff = FeedForward( + hidden_dim=hidden_dim, + inner_dim=4 * hidden_dim, + dropout=ff_dropout) + self.ff_norm = nn.LayerNorm( + normalized_shape=hidden_dim, eps=1e-12, elementwise_affine=True) + self.dropout_layer = nn.Dropout(p=dropout) + return + + def forward(self, inp, mask=None, cache=None): + """Forward process on one transformer layer. + + Args: + x(Variable(shape: [batch_size, seq_len, hidden_size])) + memory(Variable(shape: [batch_size, seq_len, hidden_size])) + mask + cache + """ + attn_out = self.attn(inp, mask, cache) + attn_out = self.dropout_layer(attn_out) + attn_out = self.attn_norm(attn_out + inp) + + ff_out = self.ff(attn_out) + ff_out = self.dropout_layer(ff_out) + ff_out = self.ff_norm(ff_out + attn_out) + + return ff_out + + +def main(): + import numpy as np + + model = TransformerBlock(10, 2, 0.5, 0.5, 0.5) + inp = np.random.rand(2, 3, 10).astype('float32') + inp = torch.tensor(inp) + mask = (np.random.rand(2, 3, 3) > 0.5).astype('float32') + mask = torch.tensor(mask) + out = model(inp, mask=mask, cache=None) + print(out) + + +if __name__ == '__main__': + main() diff --git a/runtime/Lib/site_packages/models/nlp/space_T_cn/__init__.py b/runtime/Lib/site_packages/models/nlp/space_T_cn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b9deb7003dc711c068cbe681538a7745f87201d1 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_cn/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .table_question_answering import TableQuestionAnswering +else: + _import_structure = { + 'table_question_answering': ['TableQuestionAnswering'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/space_T_cn/backbone.py b/runtime/Lib/site_packages/models/nlp/space_T_cn/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..b1df58bad773e376593c4055e3a1c9b3aa614589 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_cn/backbone.py @@ -0,0 +1,1002 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +from __future__ import absolute_import, division, print_function +import copy +import math +import os +import shutil +import tarfile +import tempfile + +import numpy as np +import torch +from torch import nn + +from modelscope.models.nlp.space_T_cn.configuration import SpaceTCnConfig +from modelscope.utils.constant import ModelFile +from modelscope.utils.logger import get_logger + +logger = get_logger() + +CONFIG_NAME = ModelFile.CONFIGURATION +WEIGHTS_NAME = ModelFile.TORCH_MODEL_BIN_FILE + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class BertLayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(BertLayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class BertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(BertEmbeddings, self).__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, + config.hidden_size) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + self.match_type_embeddings = nn.Embedding(11, config.hidden_size) + self.type_embeddings = nn.Embedding(6, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, + input_ids, + header_ids, + token_type_ids=None, + match_type_ids=None, + l_hs=None, + header_len=None, + type_idx=None, + col_dict_list=None, + ids=None, + header_flatten_tokens=None, + header_flatten_index=None, + header_flatten_output=None, + token_column_id=None, + token_column_mask=None, + column_start_index=None, + headers_length=None): + seq_length = input_ids.size(1) + position_ids = torch.arange( + seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + words_embeddings = self.word_embeddings(input_ids) + header_embeddings = self.word_embeddings(header_ids) + + if col_dict_list is not None and l_hs is not None: + col_dict_list = np.array(col_dict_list)[ids.cpu().numpy()].tolist() + header_len = np.array( + header_len, dtype=object)[ids.cpu().numpy()].tolist() + for bi, col_dict in enumerate(col_dict_list): + for ki, vi in col_dict.items(): + length = header_len[bi][vi] + if length == 0: + continue + words_embeddings[bi, ki, :] = torch.mean( + header_embeddings[bi, vi, :length, :], dim=0) + + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + embeddings = words_embeddings + position_embeddings + token_type_embeddings + + if match_type_ids is not None: + match_type_embeddings = self.match_type_embeddings(match_type_ids) + embeddings += match_type_embeddings + + if type_idx is not None: + type_embeddings = self.type_embeddings(type_idx) + embeddings += type_embeddings + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + + return embeddings + + +class BertSelfAttention(nn.Module): + + def __init__(self, config): + super(BertSelfAttention, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask, schema_link_matrix=None): + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +class BertSelfAttentionWithRelationsRAT(nn.Module): + ''' + Adapted from https://github.com/microsoft/rat-sql/blob/master/ratsql/models/transformer.py + ''' + + def __init__(self, config): + super(BertSelfAttentionWithRelationsRAT, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + self.relation_k_emb = nn.Embedding( + 7, config.hidden_size // config.num_attention_heads) + self.relation_v_emb = nn.Embedding( + 7, config.hidden_size // config.num_attention_heads) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask, relation): + ''' + relation is [batch, seq len, seq len] + ''' + mixed_query_layer = self.query( + hidden_states) # [batch, seq len, hidden dim] + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + relation_k = self.relation_k_emb( + relation) # [batch, seq len, seq len, head dim] + relation_v = self.relation_v_emb( + relation) # [batch, seq len, seq len, head dim] + + query_layer = self.transpose_for_scores( + mixed_query_layer) # [batch, num attn heads, seq len, head dim] + key_layer = self.transpose_for_scores( + mixed_key_layer) # [batch, num attn heads, seq len, head dim] + value_layer = self.transpose_for_scores( + mixed_value_layer) # [batch, num attn heads, seq len, head dim] + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose( + -1, -2)) # [batch, num attn heads, seq len, seq len] + + # relation_k_t is [batch, seq len, head dim, seq len] + relation_k_t = relation_k.transpose(-2, -1) + # query_layer_t is [batch, seq len, num attn heads, head dim] + query_layer_t = query_layer.permute(0, 2, 1, 3) + # relation_attention_scores is [batch, seq len, num attn heads, seq len] + relation_attention_scores = torch.matmul(query_layer_t, relation_k_t) + # relation_attention_scores_t is [batch, num attn heads, seq len, seq len] + relation_attention_scores_t = relation_attention_scores.permute( + 0, 2, 1, 3) + + merged_attention_scores = (attention_scores + + relation_attention_scores_t) / math.sqrt( + self.attention_head_size) + + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + merged_attention_scores = merged_attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(merged_attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + # attention_probs is [batch, num attn heads, seq len, seq len] + attention_probs = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs, value_layer) + + # attention_probs_t is [batch, seq len, num attn heads, seq len] + attention_probs_t = attention_probs.permute(0, 2, 1, 3) + + # [batch, seq len, num attn heads, seq len] + # * [batch, seq len, seq len, head dim] + # = [batch, seq len, num attn heads, head dim] + context_relation = torch.matmul(attention_probs_t, relation_v) + + # context_relation_t is [batch, num attn heads, seq len, head dim] + context_relation_t = context_relation.permute(0, 2, 1, 3) + + merged_context_layer = context_layer + context_relation_t + merged_context_layer = merged_context_layer.permute(0, 2, 1, + 3).contiguous() + new_context_layer_shape = merged_context_layer.size()[:-2] + ( + self.all_head_size, ) + merged_context_layer = merged_context_layer.view( + *new_context_layer_shape) + return merged_context_layer + + +class BertSelfAttentionWithRelationsTableformer(nn.Module): + + def __init__(self, config): + super(BertSelfAttentionWithRelationsTableformer, self).__init__() + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + 'The hidden size (%d) is not a multiple of the number of attention ' + 'heads (%d)' % + (config.hidden_size, config.num_attention_heads)) + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.schema_link_embeddings = nn.Embedding(7, self.num_attention_heads) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask, relation): + ''' + relation is [batch, seq len, seq len] + ''' + mixed_query_layer = self.query( + hidden_states) # [batch, seq len, hidden dim] + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + schema_link_embeddings = self.schema_link_embeddings( + relation) # [batch, seq len, seq len, 1] + schema_link_embeddings = schema_link_embeddings.permute(0, 3, 1, 2) + + query_layer = self.transpose_for_scores( + mixed_query_layer) # [batch, num attn heads, seq len, head dim] + key_layer = self.transpose_for_scores( + mixed_key_layer) # [batch, num attn heads, seq len, head dim] + value_layer = self.transpose_for_scores( + mixed_value_layer) # [batch, num attn heads, seq len, head dim] + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose( + -1, -2)) # [batch, num attn heads, seq len, seq len] + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + + merged_attention_scores = attention_scores + schema_link_embeddings + + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + merged_attention_scores = merged_attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(merged_attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + # attention_probs is [batch, num attn heads, seq len, seq len] + attention_probs = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + +class BertSelfOutput(nn.Module): + + def __init__(self, config): + super(BertSelfOutput, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + + def __init__(self, config, schema_link_module='none'): + super(BertAttention, self).__init__() + if schema_link_module == 'none': + self.self = BertSelfAttention(config) + if schema_link_module == 'rat': + self.self = BertSelfAttentionWithRelationsRAT(config) + if schema_link_module == 'add': + self.self = BertSelfAttentionWithRelationsTableformer(config) + self.output = BertSelfOutput(config) + + def forward(self, input_tensor, attention_mask, schema_link_matrix=None): + self_output = self.self(input_tensor, attention_mask, + schema_link_matrix) + attention_output = self.output(self_output, input_tensor) + return attention_output + + +class BertIntermediate(nn.Module): + + def __init__(self, config): + super(BertIntermediate, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + self.intermediate_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BertOutput(nn.Module): + + def __init__(self, config): + super(BertOutput, self).__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertLayer(nn.Module): + + def __init__(self, config, schema_link_module='none'): + super(BertLayer, self).__init__() + self.attention = BertAttention( + config, schema_link_module=schema_link_module) + self.intermediate = BertIntermediate(config) + self.output = BertOutput(config) + + def forward(self, hidden_states, attention_mask, schema_link_matrix=None): + attention_output = self.attention(hidden_states, attention_mask, + schema_link_matrix) + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class SqlBertEncoder(nn.Module): + + def __init__(self, layers, config): + super(SqlBertEncoder, self).__init__() + layer = BertLayer(config) + self.layer = nn.ModuleList( + [copy.deepcopy(layer) for _ in range(layers)]) + + def forward(self, + hidden_states, + attention_mask, + output_all_encoded_layers=True): + all_encoder_layers = [] + for layer_module in self.layer: + hidden_states = layer_module(hidden_states, attention_mask) + if output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + if not output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BertEncoder(nn.Module): + + def __init__(self, config, schema_link_module='none'): + super(BertEncoder, self).__init__() + layer = BertLayer(config, schema_link_module=schema_link_module) + self.layer = nn.ModuleList( + [copy.deepcopy(layer) for _ in range(config.num_hidden_layers)]) + + def forward(self, + hidden_states, + attention_mask, + all_schema_link_matrix=None, + all_schema_link_mask=None, + output_all_encoded_layers=True): + all_encoder_layers = [] + for layer_module in self.layer: + hidden_states = layer_module(hidden_states, attention_mask, + all_schema_link_matrix) + if output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + if not output_all_encoded_layers: + all_encoder_layers.append(hidden_states) + return all_encoder_layers + + +class BertPooler(nn.Module): + + def __init__(self, config): + super(BertPooler, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super(BertPredictionHeadTransform, self).__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.transform_act_fn = ACT2FN[config.hidden_act] \ + if isinstance(config.hidden_act, str) else config.hidden_act + self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BertLMPredictionHead(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertLMPredictionHead, self).__init__() + self.transform = BertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + bert_model_embedding_weights.size(1), + bert_model_embedding_weights.size(0), + bias=False) + self.decoder.weight = bert_model_embedding_weights + self.bias = nn.Parameter( + torch.zeros(bert_model_embedding_weights.size(0))) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class BertOnlyMLMHead(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertOnlyMLMHead, self).__init__() + self.predictions = BertLMPredictionHead(config, + bert_model_embedding_weights) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BertOnlyNSPHead(nn.Module): + + def __init__(self, config): + super(BertOnlyNSPHead, self).__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class BertPreTrainingHeads(nn.Module): + + def __init__(self, config, bert_model_embedding_weights): + super(BertPreTrainingHeads, self).__init__() + self.predictions = BertLMPredictionHead(config, + bert_model_embedding_weights) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +class PreTrainedBertModel(nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedBertModel, self).__init__() + if not isinstance(config, SpaceTCnConfig): + raise ValueError( + 'Parameter config in `{}(config)` should be an instance of class `SpaceTCnConfig`. ' + 'To create a model from a Google pretrained model use ' + '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format( + self.__class__.__name__, self.__class__.__name__)) + self.config = config + + def init_bert_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, BertLayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + @classmethod + def from_pretrained(cls, + pretrained_model_name, + state_dict=None, + cache_dir=None, + *inputs, + **kwargs): + """ + Instantiate a PreTrainedBertModel from a pre-trained model file or a pytorch state dict. + Download and cache the pre-trained model file if needed. + + Params: + pretrained_model_name: either: + - a str with the name of a pre-trained model to load selected in the list of: + . `bert-base-uncased` + . `bert-large-uncased` + . `bert-base-cased` + . `bert-large-cased` + . `bert-base-multilingual-uncased` + . `bert-base-multilingual-cased` + . `bert-base-chinese` + - a path or url to a pretrained model archive containing: + . `bert_config.json` a configuration file for the model + . `pytorch_model.bin` a PyTorch dump of a BertForPreTraining instance + cache_dir: an optional path to a folder in which the pre-trained models will be cached. + state_dict: an optional state dictionnary (collections.OrderedDict object) + to use instead of Google pre-trained models + *inputs, **kwargs: additional input for the specific Bert class + (ex: num_labels for BertForSequenceClassification) + """ + resolved_archive_file = pretrained_model_name + # redirect to the cache, if necessary + tempdir = None + if os.path.isdir(resolved_archive_file): + serialization_dir = resolved_archive_file + else: + # Extract archive to temp dir + tempdir = tempfile.mkdtemp() + logger.info('extracting archive file {} to temp dir {}'.format( + resolved_archive_file, tempdir)) + with tarfile.open(resolved_archive_file, 'r:gz') as archive: + archive.extractall(tempdir) + serialization_dir = tempdir + # Load config + config_file = os.path.join(serialization_dir, CONFIG_NAME) + config = SpaceTCnConfig.from_json_file(config_file) + logger.info('Model config {}'.format(config)) + # Instantiate model. + model = cls(config, *inputs, **kwargs) + if state_dict is None: + weights_path = os.path.join(serialization_dir, WEIGHTS_NAME) + state_dict = torch.load(weights_path) + + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, + True, missing_keys, unexpected_keys, + error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix='' if hasattr(model, 'bert') else 'bert.') + if len(missing_keys) > 0: + logger.info( + 'Weights of {} not initialized from pretrained model: {}'. + format(model.__class__.__name__, missing_keys)) + print() + print('*' * 10, 'WARNING missing weights', '*' * 10) + print('Weights of {} not initialized from pretrained model: {}'. + format(model.__class__.__name__, missing_keys)) + print() + if len(unexpected_keys) > 0: + logger.info( + 'Weights from pretrained model not used in {}: {}'.format( + model.__class__.__name__, unexpected_keys)) + print() + print('*' * 10, 'WARNING unexpected weights', '*' * 10) + print('Weights from pretrained model not used in {}: {}'.format( + model.__class__.__name__, unexpected_keys)) + print() + if tempdir: + # Clean up temp dir + shutil.rmtree(tempdir) + return model + + +class SpaceTCnModel(PreTrainedBertModel): + """SpaceTCnModel model ("Bidirectional Embedding Representations from a Transformer pretrained on STAR-T-CN"). + + Params: + config: a SpaceTCnConfig class instance with the configuration to build a new model + + Inputs: + `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] + with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts + `extract_features.py`, `run_classifier.py` and `run_squad.py`) + `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token + types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to + a `sentence B` token (see BERT paper for more details). + `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices + selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max + input sequence length in the current batch. It's the mask that we typically use for attention when + a batch has varying length sentences. + `output_all_encoded_layers`: boolean which controls the content of the `encoded_layers` output + as described below. Default: `True`. + + Outputs: Tuple of (encoded_layers, pooled_output) + `encoded_layers`: controled by `output_all_encoded_layers` argument: + - `output_all_encoded_layers=True`: outputs a list of the full sequences of encoded-hidden-states at the end + of each attention block (i.e. 12 full sequences for BERT-base, 24 for BERT-large), each + encoded-hidden-state is a torch.FloatTensor of size [batch_size, sequence_length, hidden_size], + - `output_all_encoded_layers=False`: outputs only the full sequence of hidden-states corresponding + to the last attention block of shape [batch_size, sequence_length, hidden_size], + `pooled_output`: a torch.FloatTensor of size [batch_size, hidden_size] which is the output of a + classifier pretrained on top of the hidden state associated to the first character of the + input (`CLF`) to train on the Next-Sentence task (see BERT's paper). + + Example: + >>> # Already been converted into WordPiece token ids + >>> input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) + >>> input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) + >>> token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) + + >>> config = modeling.SpaceTCnConfig(vocab_size_or_config_json_file=32000, hidden_size=768, + >>> num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) + + >>> model = modeling.SpaceTCnModel(config=config) + >>> all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) + """ + + def __init__(self, config, schema_link_module='none'): + super(SpaceTCnModel, self).__init__(config) + self.embeddings = BertEmbeddings(config) + self.encoder = BertEncoder( + config, schema_link_module=schema_link_module) + self.pooler = BertPooler(config) + self.apply(self.init_bert_weights) + + def forward(self, + input_ids, + header_ids, + token_order_ids=None, + token_type_ids=None, + attention_mask=None, + match_type_ids=None, + l_hs=None, + header_len=None, + type_ids=None, + col_dict_list=None, + ids=None, + header_flatten_tokens=None, + header_flatten_index=None, + header_flatten_output=None, + token_column_id=None, + token_column_mask=None, + column_start_index=None, + headers_length=None, + all_schema_link_matrix=None, + all_schema_link_mask=None, + output_all_encoded_layers=True): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + # Bowen: comment out the following line for Pytorch >= 1.5 + # https://github.com/huggingface/transformers/issues/3936#issuecomment-793764416 + # extended_attention_mask = extended_attention_mask.to(self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + embedding_output = self.embeddings( + input_ids, header_ids, token_type_ids, match_type_ids, l_hs, + header_len, type_ids, col_dict_list, ids, header_flatten_tokens, + header_flatten_index, header_flatten_output, token_column_id, + token_column_mask, column_start_index, headers_length) + encoded_layers = self.encoder( + embedding_output, + extended_attention_mask, + all_schema_link_matrix=all_schema_link_matrix, + all_schema_link_mask=all_schema_link_mask, + output_all_encoded_layers=output_all_encoded_layers) + sequence_output = encoded_layers[-1] + pooled_output = self.pooler(sequence_output) + if not output_all_encoded_layers: + encoded_layers = encoded_layers[-1] + return encoded_layers, pooled_output + + +class Seq2SQL(nn.Module): + + def __init__(self, iS, hS, lS, dr, n_cond_ops, n_agg_ops, n_action_ops, + max_select_num, max_where_num, device): + super(Seq2SQL, self).__init__() + self.iS = iS + self.hS = hS + self.ls = lS + self.dr = dr + self.device = device + + self.n_agg_ops = n_agg_ops + self.n_cond_ops = n_cond_ops + self.n_action_ops = n_action_ops + self.max_select_num = max_select_num + self.max_where_num = max_where_num + + self.w_sss_model = nn.Linear(iS, max_where_num) + self.w_sse_model = nn.Linear(iS, max_where_num) + self.s_ht_model = nn.Linear(iS, max_select_num) + self.wc_ht_model = nn.Linear(iS, max_where_num) + + self.select_agg_model = nn.Linear(iS * max_select_num, + n_agg_ops * max_select_num) + self.w_op_model = nn.Linear(iS * max_where_num, + n_cond_ops * max_where_num) + + self.conn_model = nn.Linear(iS, 3) + self.action_model = nn.Linear(iS, n_action_ops + 1) + self.slen_model = nn.Linear(iS, max_select_num + 1) + self.wlen_model = nn.Linear(iS, max_where_num + 1) + + def set_device(self, device): + self.device = device + + def forward(self, wemb_layer, l_n, l_hs, start_index, column_index, tokens, + ids): + # chunk input lists for multi-gpu + max_l_n = max(l_n) + max_l_hs = max(l_hs) + l_n = np.array(l_n)[ids.cpu().numpy()].tolist() + l_hs = np.array(l_hs)[ids.cpu().numpy()].tolist() + start_index = np.array(start_index)[ids.cpu().numpy()].tolist() + column_index = np.array(column_index)[ids.cpu().numpy()].tolist() + # tokens = np.array(tokens)[ids.cpu().numpy()].tolist() + + conn_index = [] + slen_index = [] + wlen_index = [] + action_index = [] + where_op_index = [] + select_agg_index = [] + header_pos_index = [] + query_index = [] + for ib, elem in enumerate(start_index): + # [SEP] conn [SEP] wlen [SEP] (wop [SEP])*wn slen [SEP] (agg [SEP])*sn + action_index.append(elem + 1) + conn_index.append(elem + 2) + wlen_index.append(elem + 3) + woi = [elem + 4 + i for i in range(self.max_where_num)] + + slen_index.append(elem + 4 + self.max_where_num) + sai = [ + elem + 5 + self.max_where_num + i + for i in range(self.max_select_num) + ] + where_op_index.append(woi) + select_agg_index.append(sai) + + qilist = [i for i in range(l_n[ib] + 2)] + [l_n[ib] + 1] * ( + max_l_n - l_n[ib]) + query_index.append(qilist) + + index = [column_index[ib] + i for i in range(0, l_hs[ib], 1)] + index += [index[0] for _ in range(max_l_hs - len(index))] + header_pos_index.append(index) + + # print("tokens: ", tokens) + # print("conn_index: ", conn_index, "start_index: ", start_index) + conn_index = torch.tensor(conn_index, dtype=torch.long).to(self.device) + slen_index = torch.tensor(slen_index, dtype=torch.long).to(self.device) + wlen_index = torch.tensor(wlen_index, dtype=torch.long).to(self.device) + action_index = torch.tensor( + action_index, dtype=torch.long).to(self.device) + where_op_index = torch.tensor( + where_op_index, dtype=torch.long).to(self.device) + select_agg_index = torch.tensor( + select_agg_index, dtype=torch.long).to(self.device) + query_index = torch.tensor( + query_index, dtype=torch.long).to(self.device) + header_index = torch.tensor( + header_pos_index, dtype=torch.long).to(self.device) + + bS = len(l_n) + conn_emb = torch.zeros([bS, self.iS]).to(self.device) + slen_emb = torch.zeros([bS, self.iS]).to(self.device) + wlen_emb = torch.zeros([bS, self.iS]).to(self.device) + action_emb = torch.zeros([bS, self.iS]).to(self.device) + wo_emb = torch.zeros([bS, self.max_where_num, self.iS]).to(self.device) + sa_emb = torch.zeros([bS, self.max_select_num, + self.iS]).to(self.device) + qv_emb = torch.zeros([bS, max_l_n + 2, self.iS]).to(self.device) + ht_emb = torch.zeros([bS, max_l_hs, self.iS]).to(self.device) + for i in range(bS): + conn_emb[i, :] = wemb_layer[i].index_select(0, conn_index[i]) + slen_emb[i, :] = wemb_layer[i].index_select(0, slen_index[i]) + wlen_emb[i, :] = wemb_layer[i].index_select(0, wlen_index[i]) + action_emb[i, :] = wemb_layer[i].index_select(0, action_index[i]) + + wo_emb[i, :, :] = wemb_layer[i].index_select( + 0, where_op_index[i, :]) + sa_emb[i, :, :] = wemb_layer[i].index_select( + 0, select_agg_index[i, :]) + qv_emb[i, :, :] = wemb_layer[i].index_select(0, query_index[i, :]) + ht_emb[i, :, :] = wemb_layer[i].index_select(0, header_index[i, :]) + + s_cco = self.conn_model(conn_emb.reshape(-1, self.iS)).reshape(bS, 3) + s_slen = self.slen_model(slen_emb.reshape(-1, self.iS)).reshape( + bS, self.max_select_num + 1) + s_wlen = self.wlen_model(wlen_emb.reshape(-1, self.iS)).reshape( + bS, self.max_where_num + 1) + s_action = self.action_model(action_emb.reshape(-1, self.iS)).reshape( + bS, self.n_action_ops + 1) + wo_output = self.w_op_model( + wo_emb.reshape(-1, self.iS * self.max_where_num)).reshape( + bS, -1, self.n_cond_ops) + + wc_output = self.wc_ht_model(ht_emb.reshape(-1, self.iS)).reshape( + bS, -1, self.max_where_num).transpose(1, 2) + + wv_ss = self.w_sss_model(qv_emb.reshape(-1, self.iS)).reshape( + bS, -1, self.max_where_num).transpose(1, 2) + wv_se = self.w_sse_model(qv_emb.reshape(-1, self.iS)).reshape( + bS, -1, self.max_where_num).transpose(1, 2) + + sc_output = self.s_ht_model(ht_emb.reshape(-1, self.iS)).reshape( + bS, -1, self.max_select_num).transpose(1, 2) + sa_output = self.select_agg_model( + sa_emb.reshape(-1, self.iS * self.max_select_num)).reshape( + bS, -1, self.n_agg_ops) + + return s_action, sc_output, sa_output, s_cco, wc_output, wo_output, ( + wv_ss, wv_se), (s_slen, s_wlen) diff --git a/runtime/Lib/site_packages/models/nlp/space_T_cn/configuration.py b/runtime/Lib/site_packages/models/nlp/space_T_cn/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..e698b310d4f2c772f53796d94391befa7dab1fe6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_cn/configuration.py @@ -0,0 +1,115 @@ +# Copyright 2021-2022 The Alibaba DAMO Team Authors. All rights reserved. +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT configuration.""" + +from __future__ import absolute_import, division, print_function +import copy +import logging + +import json + +logger = logging.getLogger(__name__) + + +class SpaceTCnConfig(object): + """Configuration class to store the configuration of a `SpaceTCnModel`. + """ + + def __init__(self, + vocab_size_or_config_json_file, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02): + """Constructs SpaceTCnConfig. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `SpaceTCnConfig`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into `SpaceTCnConfig`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + if isinstance(vocab_size_or_config_json_file, str): + with open( + vocab_size_or_config_json_file, 'r', + encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + else: + raise ValueError( + 'First argument must be either a vocabulary size (int)' + 'or the path to a pretrained model config file (str)') + + @classmethod + def from_dict(cls, json_object): + """Constructs a `SpaceTCnConfig` from a Python dictionary of parameters.""" + config = SpaceTCnConfig(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `SpaceTCnConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' diff --git a/runtime/Lib/site_packages/models/nlp/space_T_cn/table_question_answering.py b/runtime/Lib/site_packages/models/nlp/space_T_cn/table_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..7f8eda994f636b683a2e67bc143d76250de413d8 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_cn/table_question_answering.py @@ -0,0 +1,782 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Dict + +import numpy +import torch +import torch.nn.functional as F +from transformers import BertTokenizer + +from modelscope.metainfo import Models +from modelscope.models.base import Model, Tensor +from modelscope.models.builder import MODELS +from modelscope.preprocessors.nlp.space_T_cn.fields.struct import Constant +from modelscope.utils.constant import ModelFile, Tasks +from .backbone import Seq2SQL, SpaceTCnModel +from .configuration import SpaceTCnConfig + +__all__ = ['TableQuestionAnswering'] + + +@MODELS.register_module( + Tasks.table_question_answering, module_name=Models.space_T_cn) +class TableQuestionAnswering(Model): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the table-question-answering model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.tokenizer = BertTokenizer( + os.path.join(model_dir, ModelFile.VOCAB_FILE)) + + state_dict = torch.load( + os.path.join(self.model_dir, ModelFile.TORCH_MODEL_BIN_FILE), + map_location='cpu') + + self.backbone_config = SpaceTCnConfig.from_json_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION)) + self.backbone_model = SpaceTCnModel( + config=self.backbone_config, schema_link_module='rat') + self.backbone_model.load_state_dict(state_dict['backbone_model']) + + constant = Constant() + self.agg_ops = constant.agg_ops + self.cond_ops = constant.cond_ops + self.cond_conn_ops = constant.cond_conn_ops + self.action_ops = constant.action_ops + self.max_select_num = constant.max_select_num + self.max_where_num = constant.max_where_num + self.col_type_dict = constant.col_type_dict + self.schema_link_dict = constant.schema_link_dict + self.n_cond_ops = len(self.cond_ops) + self.n_agg_ops = len(self.agg_ops) + self.n_action_ops = len(self.action_ops) + iS = self.backbone_config.hidden_size + self.head_model = Seq2SQL( + iS, + 100, + 2, + 0.0, + self.n_cond_ops, + self.n_agg_ops, + self.n_action_ops, + self.max_select_num, + self.max_where_num, + device=self._device_name) + self.device = self._device_name + self.head_model.load_state_dict(state_dict['head_model'], strict=False) + + def to(self, device): + self.device = device + self.backbone_model.to(device) + self.head_model.to(device) + self.head_model.set_device(device) + + def convert_string(self, pr_wvi, nlu, nlu_tt): + convs = [] + for b, nlu1 in enumerate(nlu): + conv_dict = {} + nlu_tt1 = nlu_tt[b] + idx = 0 + convflag = True + for i, ntok in enumerate(nlu_tt1): + if idx >= len(nlu1): + convflag = False + break + + if ntok.startswith('##'): + ntok = ntok.replace('##', '') + tok = nlu1[idx:idx + 1].lower() + if ntok == tok: + conv_dict[i] = [idx, idx + 1] + idx += 1 + elif ntok == '#': + conv_dict[i] = [idx, idx] + elif ntok == '[UNK]': + conv_dict[i] = [idx, idx + 1] + j = i + 1 + idx += 1 + if idx < len(nlu1) and j < len( + nlu_tt1) and nlu_tt1[j] != '[UNK]': + while idx < len(nlu1): + val = nlu1[idx:idx + 1].lower() + if nlu_tt1[j].startswith(val): + break + idx += 1 + conv_dict[i][1] = idx + elif tok in ntok: + startid = idx + idx += 1 + while idx < len(nlu1): + tok += nlu1[idx:idx + 1].lower() + if ntok == tok: + conv_dict[i] = [startid, idx + 1] + break + idx += 1 + idx += 1 + else: + convflag = False + + conv = [] + if convflag: + for pr_wvi1 in pr_wvi[b]: + s1, e1 = conv_dict[pr_wvi1[0]] + s2, e2 = conv_dict[pr_wvi1[1]] + newidx = pr_wvi1[1] + while newidx + 1 < len( + nlu_tt1) and s2 == e2 and nlu_tt1[newidx] == '#': + newidx += 1 + s2, e2 = conv_dict[newidx] + if newidx + 1 < len(nlu_tt1) and nlu_tt1[ + newidx + 1].startswith('##'): + s2, e2 = conv_dict[newidx + 1] + phrase = nlu1[s1:e2] + conv.append(phrase) + else: + for pr_wvi1 in pr_wvi[b]: + phrase = ''.join(nlu_tt1[pr_wvi1[0]:pr_wvi1[1] + + 1]).replace('##', '') + conv.append(phrase) + convs.append(conv) + + return convs + + def get_fields_info(self, t1s, tables, train=True): + nlu, nlu_t, sql_i, q_know, t_know, action, hs_t, types, units, his_sql, schema_link = \ + [], [], [], [], [], [], [], [], [], [], [] + for t1 in t1s: + nlu.append(t1['question']) + nlu_t.append(t1['question_tok']) + hs_t.append(t1['header_tok']) + q_know.append(t1['bertindex_knowledge']) + t_know.append(t1['header_knowledge']) + types.append(t1['types']) + units.append(t1['units']) + his_sql.append(t1.get('history_sql', None)) + schema_link.append(t1.get('schema_link', [])) + if train: + action.append(t1.get('action', [0])) + sql_i.append(t1['sql']) + + return nlu, nlu_t, sql_i, q_know, t_know, action, hs_t, types, units, his_sql, schema_link + + def get_history_select_where(self, his_sql, header_len): + if his_sql is None: + return [0], [0] + + sel = [] + for seli in his_sql['sel']: + if seli + 1 < header_len and seli + 1 not in sel: + sel.append(seli + 1) + + whe = [] + for condi in his_sql['conds']: + if condi[0] + 1 < header_len and condi[0] + 1 not in whe: + whe.append(condi[0] + 1) + + if len(sel) == 0: + sel.append(0) + if len(whe) == 0: + whe.append(0) + + sel.sort() + whe.sort() + + return sel, whe + + def get_types_ids(self, col_type): + for key, type_ids in self.col_type_dict.items(): + if key in col_type.lower(): + return type_ids + return self.col_type_dict['null'] + + def generate_inputs(self, nlu1_tok, hs_t_1, type_t, unit_t, his_sql, + q_know, t_know, s_link): + tokens = [] + orders = [] + types = [] + segment_ids = [] + matchs = [] + col_dict = {} + schema_tok = [] + + tokens.append('[CLS]') + orders.append(0) + types.append(0) + i_st_nlu = len(tokens) + + matchs.append(0) + segment_ids.append(0) + for idx, token in enumerate(nlu1_tok): + if q_know[idx] == 100: + break + elif q_know[idx] >= 5: + matchs.append(1) + else: + matchs.append(q_know[idx] + 1) + tokens.append(token) + orders.append(0) + types.append(0) + segment_ids.append(0) + + i_ed_nlu = len(tokens) + + tokens.append('[SEP]') + orders.append(0) + types.append(0) + matchs.append(0) + segment_ids.append(0) + + sel, whe = self.get_history_select_where(his_sql, len(hs_t_1)) + + if len(sel) == 1 and sel[0] == 0 \ + and len(whe) == 1 and whe[0] == 0: + pass + else: + tokens.append('select') + orders.append(0) + types.append(0) + matchs.append(10) + segment_ids.append(0) + + for seli in sel: + tokens.append('[PAD]') + orders.append(0) + types.append(0) + matchs.append(10) + segment_ids.append(0) + col_dict[len(tokens) - 1] = seli + + tokens.append('where') + orders.append(0) + types.append(0) + matchs.append(10) + segment_ids.append(0) + + for whei in whe: + tokens.append('[PAD]') + orders.append(0) + types.append(0) + matchs.append(10) + segment_ids.append(0) + col_dict[len(tokens) - 1] = whei + + tokens.append('[SEP]') + orders.append(0) + types.append(0) + matchs.append(10) + segment_ids.append(0) + + column_start = len(tokens) + i_hds_f = [] + header_flatten_tokens, header_flatten_index = [], [] + for i, hds11 in enumerate(hs_t_1): + if len(unit_t[i]) == 1 and unit_t[i][0] == 'null': + temp_header_tokens = hds11 + else: + temp_header_tokens = hds11 + unit_t[i] + schema_tok.append(temp_header_tokens) + header_flatten_tokens.extend(temp_header_tokens) + header_flatten_index.extend([i + 1] * len(temp_header_tokens)) + i_st_hd_f = len(tokens) + tokens += ['[PAD]'] + orders.append(0) + types.append(self.get_types_ids(type_t[i])) + i_ed_hd_f = len(tokens) + col_dict[len(tokens) - 1] = i + i_hds_f.append((i_st_hd_f, i_ed_hd_f)) + if i == 0: + matchs.append(6) + else: + matchs.append(t_know[i - 1] + 6) + segment_ids.append(1) + + tokens.append('[SEP]') + orders.append(0) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + # position where + # [SEP] + start_ids = len(tokens) - 1 + + tokens.append('action') # action + orders.append(1) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + tokens.append('connect') # column + orders.append(1) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + tokens.append('allen') # select len + orders.append(1) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + for x in range(self.max_where_num): + tokens.append('act') # op + orders.append(2 + x) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + tokens.append('size') # where len + orders.append(1) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + for x in range(self.max_select_num): + tokens.append('focus') # agg + orders.append(2 + x) + types.append(0) + matchs.append(0) + segment_ids.append(1) + + i_nlu = (i_st_nlu, i_ed_nlu) + + schema_link_matrix = numpy.zeros((len(tokens), len(tokens)), + dtype='int32') + schema_link_mask = numpy.zeros((len(tokens), len(tokens)), + dtype='float32') + for relation in s_link: + if relation['label'] in ['col', 'val']: + [q_st, q_ed] = relation['question_index'] + cid = max(0, relation['column_index']) + schema_link_matrix[ + i_st_nlu + q_st: i_st_nlu + q_ed + 1, + column_start + cid + 1: column_start + cid + 1 + 1] = \ + self.schema_link_dict[relation['label'] + '_middle'] + schema_link_matrix[ + i_st_nlu + q_st, + column_start + cid + 1: column_start + cid + 1 + 1] = \ + self.schema_link_dict[relation['label'] + '_start'] + schema_link_matrix[ + i_st_nlu + q_ed, + column_start + cid + 1: column_start + cid + 1 + 1] = \ + self.schema_link_dict[relation['label'] + '_end'] + schema_link_mask[i_st_nlu + q_st:i_st_nlu + q_ed + 1, + column_start + cid + 1:column_start + cid + 1 + + 1] = 1.0 + + return tokens, orders, types, segment_ids, matchs, \ + i_nlu, i_hds_f, start_ids, column_start, col_dict, schema_tok, \ + header_flatten_tokens, header_flatten_index, schema_link_matrix, schema_link_mask + + def gen_l_hpu(self, i_hds): + """ + Treat columns as if it is a batch of natural language utterance + with batch-size = # of columns * # of batch_size + i_hds = [(17, 18), (19, 21), (22, 23), (24, 25), (26, 29), (30, 34)]) + """ + l_hpu = [] + for i_hds1 in i_hds: + for i_hds11 in i_hds1: + l_hpu.append(i_hds11[1] - i_hds11[0]) + + return l_hpu + + def get_bert_output(self, model_bert, tokenizer, nlu_t, hs_t, col_types, + units, his_sql, q_know, t_know, schema_link): + """ + Here, input is toknized further by WordPiece (WP) tokenizer and fed into BERT. + + INPUT + :param model_bert: + :param tokenizer: WordPiece toknizer + :param nlu: Question + :param nlu_t: CoreNLP tokenized nlu. + :param hds: Headers + :param hs_t: None or 1st-level tokenized headers + :param max_seq_length: max input token length + + OUTPUT + tokens: BERT input tokens + nlu_tt: WP-tokenized input natural language questions + orig_to_tok_index: map the index of 1st-level-token to the index of 2nd-level-token + tok_to_orig_index: inverse map. + + """ + + l_n = [] + l_hs = [] # The length of columns for each batch + + input_ids = [] + order_ids = [] + type_ids = [] + segment_ids = [] + match_ids = [] + input_mask = [] + + i_nlu = [ + ] # index to retreive the position of contextual vector later. + i_hds = [] + tokens = [] + orders = [] + types = [] + matchs = [] + segments = [] + schema_link_matrix_list = [] + schema_link_mask_list = [] + start_index = [] + column_index = [] + col_dict_list = [] + header_list = [] + header_flatten_token_list = [] + header_flatten_tokenid_list = [] + header_flatten_index_list = [] + + header_tok_max_len = 0 + cur_max_length = 0 + + for b, nlu_t1 in enumerate(nlu_t): + hs_t1 = [hs_t[b][-1]] + hs_t[b][:-1] + type_t1 = [col_types[b][-1]] + col_types[b][:-1] + unit_t1 = [units[b][-1]] + units[b][:-1] + l_hs.append(len(hs_t1)) + + # [CLS] nlu [SEP] col1 [SEP] col2 [SEP] ...col-n [SEP] + # 2. Generate BERT inputs & indices. + tokens1, orders1, types1, segment1, match1, i_nlu1, i_hds_1, \ + start_idx, column_start, col_dict, schema_tok, \ + header_flatten_tokens, header_flatten_index, schema_link_matrix, schema_link_mask = \ + self.generate_inputs( + nlu_t1, hs_t1, type_t1, unit_t1, his_sql[b], + q_know[b], t_know[b], schema_link[b]) + + l_n.append(i_nlu1[1] - i_nlu1[0]) + start_index.append(start_idx) + column_index.append(column_start) + col_dict_list.append(col_dict) + tokens.append(tokens1) + orders.append(orders1) + types.append(types1) + segments.append(segment1) + matchs.append(match1) + i_nlu.append(i_nlu1) + i_hds.append(i_hds_1) + schema_link_matrix_list.append(schema_link_matrix) + schema_link_mask_list.append(schema_link_mask) + header_flatten_token_list.append(header_flatten_tokens) + header_flatten_index_list.append(header_flatten_index) + header_list.append(schema_tok) + header_max = max([len(schema_tok1) for schema_tok1 in schema_tok]) + if header_max > header_tok_max_len: + header_tok_max_len = header_max + + if len(tokens1) > cur_max_length: + cur_max_length = len(tokens1) + + if len(tokens1) > 512: + print('input too long!!! total_num:%d\t question:%s' % + (len(tokens1), ''.join(nlu_t1))) + + assert cur_max_length <= 512 + + for i, tokens1 in enumerate(tokens): + segment_ids1 = segments[i] + order_ids1 = orders[i] + type_ids1 = types[i] + match_ids1 = matchs[i] + input_ids1 = tokenizer.convert_tokens_to_ids(tokens1) + input_mask1 = [1] * len(input_ids1) + + while len(input_ids1) < cur_max_length: + input_ids1.append(0) + input_mask1.append(0) + segment_ids1.append(0) + order_ids1.append(0) + type_ids1.append(0) + match_ids1.append(0) + + if len(input_ids1) != cur_max_length: + print('Error: ', nlu_t1, tokens1, len(input_ids1), + cur_max_length) + + assert len(input_ids1) == cur_max_length + assert len(input_mask1) == cur_max_length + assert len(order_ids1) == cur_max_length + assert len(segment_ids1) == cur_max_length + assert len(match_ids1) == cur_max_length + assert len(type_ids1) == cur_max_length + + input_ids.append(input_ids1) + order_ids.append(order_ids1) + type_ids.append(type_ids1) + segment_ids.append(segment_ids1) + input_mask.append(input_mask1) + match_ids.append(match_ids1) + + header_len = [] + header_ids = [] + header_max_len = max( + [len(header_list1) for header_list1 in header_list]) + for header1 in header_list: + header_len1 = [] + header_ids1 = [] + for header_tok in header1: + header_len1.append(len(header_tok)) + header_tok_ids1 = tokenizer.convert_tokens_to_ids(header_tok) + while len(header_tok_ids1) < header_tok_max_len: + header_tok_ids1.append(0) + header_ids1.append(header_tok_ids1) + while len(header_ids1) < header_max_len: + header_ids1.append([0] * header_tok_max_len) + header_len.append(header_len1) + header_ids.append(header_ids1) + + for i, header_flatten_token in enumerate(header_flatten_token_list): + header_flatten_tokenid = tokenizer.convert_tokens_to_ids( + header_flatten_token) + header_flatten_tokenid_list.append(header_flatten_tokenid) + + # Convert to tensor + all_input_ids = torch.tensor( + input_ids, dtype=torch.long).to(self.device) + all_order_ids = torch.tensor( + order_ids, dtype=torch.long).to(self.device) + all_type_ids = torch.tensor(type_ids, dtype=torch.long).to(self.device) + all_input_mask = torch.tensor( + input_mask, dtype=torch.long).to(self.device) + all_segment_ids = torch.tensor( + segment_ids, dtype=torch.long).to(self.device) + all_match_ids = torch.tensor( + match_ids, dtype=torch.long).to(self.device) + all_header_ids = torch.tensor( + header_ids, dtype=torch.long).to(self.device) + all_ids = torch.arange( + all_input_ids.shape[0], dtype=torch.long).to(self.device) + + bS = len(header_flatten_tokenid_list) + max_header_flatten_token_length = max( + [len(x) for x in header_flatten_tokenid_list]) + all_header_flatten_tokens = numpy.zeros( + (bS, max_header_flatten_token_length), dtype='int32') + all_header_flatten_index = numpy.zeros( + (bS, max_header_flatten_token_length), dtype='int32') + for i, header_flatten_tokenid in enumerate( + header_flatten_tokenid_list): + for j, tokenid in enumerate(header_flatten_tokenid): + all_header_flatten_tokens[i, j] = tokenid + for j, hdindex in enumerate(header_flatten_index_list[i]): + all_header_flatten_index[i, j] = hdindex + all_header_flatten_output = numpy.zeros((bS, header_max_len + 1), + dtype='int32') + all_header_flatten_tokens = torch.tensor( + all_header_flatten_tokens, dtype=torch.long).to(self.device) + all_header_flatten_index = torch.tensor( + all_header_flatten_index, dtype=torch.long).to(self.device) + all_header_flatten_output = torch.tensor( + all_header_flatten_output, dtype=torch.float32).to(self.device) + + all_token_column_id = numpy.zeros((bS, cur_max_length), dtype='int32') + all_token_column_mask = numpy.zeros((bS, cur_max_length), + dtype='float32') + for bi, col_dict in enumerate(col_dict_list): + for ki, vi in col_dict.items(): + all_token_column_id[bi, ki] = vi + 1 + all_token_column_mask[bi, ki] = 1.0 + all_token_column_id = torch.tensor( + all_token_column_id, dtype=torch.long).to(self.device) + all_token_column_mask = torch.tensor( + all_token_column_mask, dtype=torch.float32).to(self.device) + + all_schema_link_matrix = numpy.zeros( + (bS, cur_max_length, cur_max_length), dtype='int32') + all_schema_link_mask = numpy.zeros( + (bS, cur_max_length, cur_max_length), dtype='float32') + for i, schema_link_matrix in enumerate(schema_link_matrix_list): + temp_len = schema_link_matrix.shape[0] + all_schema_link_matrix[i, 0:temp_len, + 0:temp_len] = schema_link_matrix + all_schema_link_mask[i, 0:temp_len, + 0:temp_len] = schema_link_mask_list[i] + all_schema_link_matrix = torch.tensor( + all_schema_link_matrix, dtype=torch.long).to(self.device) + all_schema_link_mask = torch.tensor( + all_schema_link_mask, dtype=torch.long).to(self.device) + + # 5. generate l_hpu from i_hds + l_hpu = self.gen_l_hpu(i_hds) + + # 4. Generate BERT output. + all_encoder_layer, pooled_output = model_bert( + all_input_ids, + all_header_ids, + token_order_ids=all_order_ids, + token_type_ids=all_segment_ids, + attention_mask=all_input_mask, + match_type_ids=all_match_ids, + l_hs=l_hs, + header_len=header_len, + type_ids=all_type_ids, + col_dict_list=col_dict_list, + ids=all_ids, + header_flatten_tokens=all_header_flatten_tokens, + header_flatten_index=all_header_flatten_index, + header_flatten_output=all_header_flatten_output, + token_column_id=all_token_column_id, + token_column_mask=all_token_column_mask, + column_start_index=column_index, + headers_length=l_hs, + all_schema_link_matrix=all_schema_link_matrix, + all_schema_link_mask=all_schema_link_mask, + output_all_encoded_layers=False) + + return all_encoder_layer, pooled_output, tokens, i_nlu, i_hds, \ + l_n, l_hpu, l_hs, start_index, column_index, all_ids + + def predict(self, querys): + self.head_model.eval() + self.backbone_model.eval() + + nlu, nlu_t, sql_i, q_know, t_know, tb, hs_t, types, units, his_sql, schema_link = \ + self.get_fields_info(querys, None, train=False) + + with torch.no_grad(): + all_encoder_layer, _, tokens, i_nlu, i_hds, l_n, l_hpu, l_hs, start_index, column_index, ids = \ + self.get_bert_output( + self.backbone_model, self.tokenizer, + nlu_t, hs_t, types, units, his_sql, q_know, t_know, schema_link) + + s_action, s_sc, s_sa, s_cco, s_wc, s_wo, s_wvs, s_len = self.head_model( + all_encoder_layer, l_n, l_hs, start_index, column_index, + tokens, ids) + + action_batch = torch.argmax(F.softmax(s_action, -1), -1).cpu().tolist() + scco_batch = torch.argmax(F.softmax(s_cco, -1), -1).cpu().tolist() + sc_batch = torch.argmax(F.softmax(s_sc, -1), -1).cpu().tolist() + sa_batch = torch.argmax(F.softmax(s_sa, -1), -1).cpu().tolist() + wc_batch = torch.argmax(F.softmax(s_wc, -1), -1).cpu().tolist() + wo_batch = torch.argmax(F.softmax(s_wo, -1), -1).cpu().tolist() + s_wvs_s, s_wvs_e = s_wvs + wvss_batch = torch.argmax(F.softmax(s_wvs_s, -1), -1).cpu().tolist() + wvse_batch = torch.argmax(F.softmax(s_wvs_e, -1), -1).cpu().tolist() + s_slen, s_wlen = s_len + slen_batch = torch.argmax(F.softmax(s_slen, -1), -1).cpu().tolist() + wlen_batch = torch.argmax(F.softmax(s_wlen, -1), -1).cpu().tolist() + + pr_wvi = [] + for i in range(len(querys)): + wvi = [] + for j in range(wlen_batch[i]): + wvi.append([ + max(0, wvss_batch[i][j] - 1), + max(0, wvse_batch[i][j] - 1) + ]) + pr_wvi.append(wvi) + pr_wvi_str = self.convert_string(pr_wvi, nlu, nlu_t) + + pre_results = [] + for ib in range(len(querys)): + res_one = {} + sql = {} + sql['cond_conn_op'] = scco_batch[ib] + sl = slen_batch[ib] + sql['sel'] = list( + numpy.array(sc_batch[ib][:sl]).astype(numpy.int32) - 1) + sql['agg'] = list( + numpy.array(sa_batch[ib][:sl]).astype(numpy.int32)) + sels = [] + aggs = [] + for ia, sel in enumerate(sql['sel']): + if sel == -1: + if sql['agg'][ia] > 0: + sels.append(l_hs[ib] - 1) + aggs.append(sql['agg'][ia]) + continue + sels.append(int(sel)) + if sql['agg'][ia] == -1: + aggs.append(0) + else: + aggs.append(int(sql['agg'][ia])) + if len(sels) == 0: + sels.append(l_hs[ib] - 1) + aggs.append(0) + assert len(sels) == len(aggs) + sql['sel'] = sels + sql['agg'] = aggs + + conds = [] + wl = wlen_batch[ib] + wc_os = list( + numpy.array(wc_batch[ib][:wl]).astype(numpy.int32) - 1) + wo_os = list(numpy.array(wo_batch[ib][:wl]).astype(numpy.int32)) + res_one['question_tok'] = querys[ib]['question_tok'] + for i in range(wl): + if wc_os[i] == -1: + continue + conds.append([int(wc_os[i]), int(wo_os[i]), pr_wvi_str[ib][i]]) + if len(conds) == 0: + conds.append([l_hs[ib] - 1, 2, 'Nulll']) + sql['conds'] = conds + res_one['question'] = querys[ib]['question'] + res_one['table_id'] = querys[ib]['table_id'] + res_one['sql'] = sql + res_one['action'] = action_batch[ib] + res_one['model_out'] = [ + sc_batch[ib], sa_batch[ib], wc_batch[ib], wo_batch[ib], + wvss_batch[ib], wvse_batch[ib] + ] + pre_results.append(res_one) + + return pre_results + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + + Returns: + Dict[str, Tensor]: results dict as follows: + + >>> { + >>> 'result': + >>> { + >>> 'question_tok': ['有', '哪', '些', '风', '险', '类', '型', '?'], + >>> 'question': '有哪些风险类型?', + >>> 'table_id': 'fund', + >>> 'sql': { + >>> 'cond_conn_op': 0, + >>> 'sel': [5], + >>> 'agg': [0], + >>> 'conds': [[10, 2, 'Nulll']] + >>> }, + >>> 'action': 10, + >>> 'model_out': [ + >>> [6, 0, 0, 0], + >>> [0, 0, 0, 0], + >>> [0, 0, 0, 0, 0, 0], + >>> [2, 0, 0, 0, 0, 0], + >>> [0, 0, 0, 0, 0, 0], + >>> [0, 0, 0, 0, 0, 0] + >>> ] + >>> }, + >>> 'history_sql': None + >>> } + + Example: + >>> from modelscope.models.nlp import TableQuestionAnswering + >>> from modelscope.preprocessors import TableQuestionAnsweringPreprocessor + >>> model = TableQuestionAnswering.from_pretrained('damo/nlp_convai_text2sql_pretrain_cn') + >>> preprocessor = TableQuestionAnsweringPreprocessor(model_dir=model.model_dir) + >>> print(model(preprocessor({'question': '有哪些风险类型?'}))) + """ + result = self.predict(input['datas'])[0] + + return { + 'result': result, + 'history_sql': input['datas'][0]['history_sql'] + } diff --git a/runtime/Lib/site_packages/models/nlp/space_T_en/__init__.py b/runtime/Lib/site_packages/models/nlp/space_T_en/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..46c8b38cc4230e8a6e9ec8e08302f05dae2f76d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_en/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .text_to_sql import StarForTextToSql +else: + _import_structure = { + 'text_to_sql': ['StarForTextToSql'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/space_T_en/text_to_sql.py b/runtime/Lib/site_packages/models/nlp/space_T_en/text_to_sql.py new file mode 100644 index 0000000000000000000000000000000000000000..d53003bc353b4dbf54c92bff9080175cd8be342a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/space_T_en/text_to_sql.py @@ -0,0 +1,99 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Dict, Optional + +import torch +from text2sql_lgesql.asdl.asdl import ASDLGrammar +from text2sql_lgesql.asdl.transition_system import TransitionSystem +from text2sql_lgesql.model.model_constructor import Text2SQL + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile, Tasks + +__all__ = ['StarForTextToSql'] + + +@MODELS.register_module( + Tasks.table_question_answering, module_name=Models.space_T_en) +class StarForTextToSql(TorchModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the star model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + self.beam_size = 5 + self.config = kwargs.pop( + 'config', + Config.from_file( + os.path.join(self.model_dir, ModelFile.CONFIGURATION))) + self.config.model.model_dir = model_dir + self.grammar = ASDLGrammar.from_filepath( + os.path.join(model_dir, 'sql_asdl_v2.txt')) + self.trans = TransitionSystem.get_class_by_lang('sql')(self.grammar) + self.arg = self.config.model + self.device = 'cuda' if \ + ('device' not in kwargs or kwargs['device'] == 'gpu') \ + and torch.cuda.is_available() else 'cpu' + self.model = Text2SQL(self.arg, self.trans) + check_point = torch.load( + open( + os.path.join(model_dir, ModelFile.TORCH_MODEL_BIN_FILE), 'rb'), + map_location=self.device) + compatible_position_ids( + check_point['model'], + 'encoder.input_layer.plm_model.embeddings.position_ids') + self.model.load_state_dict(check_point['model']) + + def forward(self, input: Dict[str, Tensor]) -> Dict[str, Tensor]: + """return the result by the model + + Args: + input (Dict[str, Tensor]): the preprocessed data + + Returns: + Dict[str, Tensor]: results + Example: + + Example: + >>> from modelscope.hub.snapshot_download import snapshot_download + >>> from modelscope.models.nlp import StarForTextToSql + >>> from modelscope.preprocessors import ConversationalTextToSqlPreprocessor + >>> test_case = { + 'database_id': 'employee_hire_evaluation', + 'local_db_path': None, + 'utterance': [ + "I'd like to see Shop names.", 'Which of these are hiring?', + 'Which shop is hiring the highest number of employees?' + ' | do you want the name of the shop ? | Yes' + ] + } + >>> cache_path = snapshot_download('damo/nlp_star_conversational-text-to-sql') + >>> preprocessor = ConversationalTextToSqlPreprocessor( + model_dir=cache_path, + database_id=test_case['database_id'], + db_content=True) + >>> model = StarForTextToSql(cache_path, config=preprocessor.config) + >>> print(model(preprocessor({ + 'utterance': "I'd like to see Shop names.", + 'history': [], + 'last_sql': '', + 'database_id': 'employee_hire_evaluation', + 'local_db_path': None + }))) + """ + self.model.eval() + hyps = self.model.parse(input['batch'], self.beam_size) # + db = input['batch'].examples[0].db + + predict = {'predict': hyps, 'db': db} + return predict diff --git a/runtime/Lib/site_packages/models/nlp/structbert/__init__.py b/runtime/Lib/site_packages/models/nlp/structbert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1d81116edabe1f94f4ce3c4386efa98fd85195f6 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .backbone import (SbertModel, SbertPreTrainedModel) + from .configuration import SbertConfig + from .faq_question_answering import SbertForFaqQuestionAnswering + from .fill_mask import SbertForMaskedLM + from .text_classification import SbertForSequenceClassification + from .token_classification import SbertForTokenClassification +else: + _import_structure = { + 'backbone': ['SbertModel', 'SbertPreTrainedModel'], + 'configuration': ['SbertConfig'], + 'fill_mask': ['SbertForMaskedLM'], + 'faq_question_answering': ['SbertForFaqQuestionAnswering'], + 'text_classification': ['SbertForSequenceClassification'], + 'token_classification': ['SbertForTokenClassification'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/structbert/adv_utils.py b/runtime/Lib/site_packages/models/nlp/structbert/adv_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4b4c8a7da13dd7e6ee48b1abd869229a7f6d55f3 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/adv_utils.py @@ -0,0 +1,168 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +def _symmetric_kl_div(logits1, logits2, attention_mask=None): + """ + Calclate two logits' the KL div value symmetrically. + :param logits1: The first logit. + :param logits2: The second logit. + :param attention_mask: An optional attention_mask which is used to mask some element out. + This is usually useful in token_classification tasks. + If the shape of logits is [N1, N2, ... Nn, D], the shape of attention_mask should be [N1, N2, ... Nn] + :return: The mean loss. + """ + labels_num = logits1.shape[-1] + KLDiv = nn.KLDivLoss(reduction='none') + loss = torch.sum( + KLDiv(nn.LogSoftmax(dim=-1)(logits1), + nn.Softmax(dim=-1)(logits2)), + dim=-1) + torch.sum( + KLDiv(nn.LogSoftmax(dim=-1)(logits2), + nn.Softmax(dim=-1)(logits1)), + dim=-1) + if attention_mask is not None: + loss = torch.sum( + loss * attention_mask) / torch.sum(attention_mask) / labels_num + else: + loss = torch.mean(loss) / labels_num + return loss + + +def compute_adv_loss(embedding, + model, + ori_logits, + ori_loss, + adv_grad_factor, + adv_bound=None, + sigma=5e-6, + **kwargs): + """ + Calculate the adv loss of the model. + :param embedding: Original sentense embedding + :param model: The model, or the forward function(including decoder/classifier), + accept kwargs as input, output logits + :param ori_logits: The original logits outputed from the model function + :param ori_loss: The original loss + :param adv_grad_factor: This factor will be multipled by the KL loss grad and then the result will be added to + the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value always be 1e-3~1e-7 + :param adv_bound: adv_bound is used to cut the top and the bottom bound of the produced embedding. + If not proveded, 2 * sigma will be used as the adv_bound factor + :param sigma: The std factor used to produce a 0 mean normal distribution. + If adv_bound not proveded, 2 * sigma will be used as the adv_bound factor + :param kwargs: the input param used in model function + :return: The original loss adds the adv loss + """ + adv_bound = adv_bound if adv_bound is not None else 2 * sigma + embedding_1 = embedding + embedding.data.new(embedding.size()).normal_( + 0, sigma) # 95% in +- 1e-5 + kwargs.pop('input_ids') + if 'inputs_embeds' in kwargs: + kwargs.pop('inputs_embeds') + with_attention_mask = False if 'with_attention_mask' not in kwargs else kwargs[ + 'with_attention_mask'] + attention_mask = kwargs['attention_mask'] + if not with_attention_mask: + attention_mask = None + if 'with_attention_mask' in kwargs: + kwargs.pop('with_attention_mask') + outputs = model(**kwargs, inputs_embeds=embedding_1) + v1_logits = outputs.logits + loss = _symmetric_kl_div(ori_logits, v1_logits, attention_mask) + emb_grad = torch.autograd.grad(loss, embedding_1)[0].data + emb_grad_norm = emb_grad.norm( + dim=2, keepdim=True, p=float('inf')).max( + 1, keepdim=True)[0] + is_nan = torch.any(torch.isnan(emb_grad_norm)) + if is_nan: + logger.warning('Nan occurred when calculating adv loss.') + return ori_loss + emb_grad = emb_grad / (emb_grad_norm + 1e-6) + embedding_2 = embedding_1 + adv_grad_factor * emb_grad + embedding_2 = torch.max(embedding_1 - adv_bound, embedding_2) + embedding_2 = torch.min(embedding_1 + adv_bound, embedding_2) + outputs = model(**kwargs, inputs_embeds=embedding_2) + adv_logits = outputs.logits + adv_loss = _symmetric_kl_div(ori_logits, adv_logits, attention_mask) + return ori_loss + adv_loss + + +def compute_adv_loss_pair(embedding, + model, + start_logits, + end_logits, + ori_loss, + adv_grad_factor, + adv_bound=None, + sigma=5e-6, + **kwargs): + """ + Calculate the adv loss of the model. This function is used in the pair logits scenerio. + :param embedding: Original sentense embedding + :param model: The model, or the forward function(including decoder/classifier), + accept kwargs as input, output logits + :param start_logits: The original start logits outputed from the model function + :param end_logits: The original end logits outputed from the model function + :param ori_loss: The original loss + :param adv_grad_factor: This factor will be multipled by the KL loss grad and then the result will be added to + the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value always be 1e-3~1e-7 + :param adv_bound: adv_bound is used to cut the top and the bottom bound of the produced embedding. + If not proveded, 2 * sigma will be used as the adv_bound factor + :param sigma: The std factor used to produce a 0 mean normal distribution. + If adv_bound not proveded, 2 * sigma will be used as the adv_bound factor + :param kwargs: the input param used in model function + :return: The original loss adds the adv loss + """ + adv_bound = adv_bound if adv_bound is not None else 2 * sigma + embedding_1 = embedding + embedding.data.new(embedding.size()).normal_( + 0, sigma) # 95% in +- 1e-5 + kwargs.pop('input_ids') + if 'inputs_embeds' in kwargs: + kwargs.pop('inputs_embeds') + outputs = model(**kwargs, inputs_embeds=embedding_1) + v1_logits_start, v1_logits_end = outputs.logits + loss = _symmetric_kl_div(start_logits, + v1_logits_start) + _symmetric_kl_div( + end_logits, v1_logits_end) + loss = loss / 2 + emb_grad = torch.autograd.grad(loss, embedding_1)[0].data + emb_grad_norm = emb_grad.norm( + dim=2, keepdim=True, p=float('inf')).max( + 1, keepdim=True)[0] + is_nan = torch.any(torch.isnan(emb_grad_norm)) + if is_nan: + logger.warning('Nan occurred when calculating pair adv loss.') + return ori_loss + emb_grad = emb_grad / emb_grad_norm + embedding_2 = embedding_1 + adv_grad_factor * emb_grad + embedding_2 = torch.max(embedding_1 - adv_bound, embedding_2) + embedding_2 = torch.min(embedding_1 + adv_bound, embedding_2) + outputs = model(**kwargs, inputs_embeds=embedding_2) + adv_logits_start, adv_logits_end = outputs.logits + adv_loss = _symmetric_kl_div(start_logits, + adv_logits_start) + _symmetric_kl_div( + end_logits, adv_logits_end) + return ori_loss + adv_loss diff --git a/runtime/Lib/site_packages/models/nlp/structbert/backbone.py b/runtime/Lib/site_packages/models/nlp/structbert/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..58d324a8d0754b6fce6282505293356c5977cc80 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/backbone.py @@ -0,0 +1,919 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch StructBERT model. mainly copied from :module:`~transformers.modeling_bert`""" + +import math +from dataclasses import dataclass +from typing import Optional, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from packaging import version +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import SbertConfig + +logger = get_logger() + + +class SbertEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + if version.parse(torch.__version__) > version.parse('1.6.0'): + self.register_buffer( + 'token_type_ids', + torch.zeros( + self.position_ids.size(), + dtype=torch.long, + device=self.position_ids.device), + persistent=False, + ) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0, + return_inputs_embeds=False): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, + past_key_values_length:seq_length + + past_key_values_length] + + # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs + # when its auto-generated, registered buffer helps users + # when tracing the model without passing token_type_ids, solves + # issue #5664 + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + if not return_inputs_embeds: + return embeddings + else: + return embeddings, inputs_embeds + + +class SbertSelfAttention(nn.Module): + + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in SbertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +class SbertSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class SbertAttention(nn.Module): + + def __init__(self, config): + super().__init__() + self.self = SbertSelfAttention(config) + self.output = SbertSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +class SbertIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class SbertOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class SbertLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = SbertAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = SbertAttention(config) + self.intermediate = SbertIntermediate(config) + self.output = SbertOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise ValueError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention ' + f'layers by setting `config.add_cross_attention=True`') + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class SbertEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [SbertLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class SbertPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class SbertPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = SbertConfig + base_model_prefix = 'bert' + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, SbertEncoder): + module.gradient_checkpointing = value + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + + if model_dir is None: + config = SbertConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + return model + + +@dataclass +class AttentionBackboneModelOutputWithEmbedding(AttentionBackboneModelOutput): + embedding_output: torch.FloatTensor = None + logits: Optional[Union[tuple, torch.FloatTensor]] = None + kwargs: dict = None + + +@MODELS.register_module(Tasks.backbone, module_name=Models.structbert) +class SbertModel(SbertPreTrainedModel): + """The StructBERT Model transformer outputting raw hidden-states without any specific head on top. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the :obj:`is_decoder` argument of the configuration + set to :obj:`True`. To be used in a Seq2Seq model, the model needs to initialized with both :obj:`is_decoder` + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config: SbertConfig, add_pooling_layer=True, **kwargs): + super().__init__(config) + self.config = config + + self.embeddings = SbertEmbeddings(config) + self.encoder = SbertEncoder(config) + + self.pooler = SbertPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in + ``[0, 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used + in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple + having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up + decoding. + + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + + Returns: + Returns `modelscope.outputs.AttentionBackboneModelOutputWithEmbedding` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_backbone_base_std', task='backbone') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_backbone_base_std') + >>> print(model(**preprocessor('这是个测试'))) + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + + if token_type_ids is None: + if hasattr(self.embeddings, 'token_type_ids'): + buffered_token_type_ids = self.embeddings.token_type_ids[:, : + seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + batch_size, seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output, orignal_embeds = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + return_inputs_embeds=True, + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, + pooled_output) + encoder_outputs[1:] + (orignal_embeds, ) + + return AttentionBackboneModelOutputWithEmbedding( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + embedding_output=orignal_embeds) diff --git a/runtime/Lib/site_packages/models/nlp/structbert/configuration.py b/runtime/Lib/site_packages/models/nlp/structbert/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..fcd4c653c25d8dd05e6df6ecfdc2c06d27522716 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/configuration.py @@ -0,0 +1,134 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" StructBERT model configuration, mainly copied from :class:`~transformers.BertConfig` """ +from transformers import PretrainedConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class SbertConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration + of a :class:`~modelscope.models.nlp.structbert.SbertModel`. + It is used to instantiate a StructBERT model according to the specified arguments. + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (:obj:`int`, `optional`, defaults to 2): + The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.BertModel` or + :class:`~transformers.TFBertModel`. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (:obj:`str`, `optional`, defaults to :obj:`"absolute"`): + Type of position embedding. Choose one of :obj:`"absolute"`, :obj:`"relative_key"`, + :obj:`"relative_key_query"`. For positional embeddings use :obj:`"absolute"`. For more information on + :obj:`"relative_key"`, please refer to `Self-Attention with Relative Position Representations (Shaw et al.) + `__. For more information on :obj:`"relative_key_query"`, please refer to + `Method 4` in `Improve Transformer Models with Better Relative Position Embeddings (Huang et al.) + `__. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``config.is_decoder=True``. + classifier_dropout (:obj:`float`, `optional`): + The dropout ratio for the classification head. + adv_grad_factor (:obj:`float`, `optional`): This factor will be multiplied by the KL loss grad and then + the result will be added to the original embedding. + More details please check:https://arxiv.org/abs/1908.04577 + The range of this value should between 1e-3~1e-7 + adv_bound (:obj:`float`, `optional`): adv_bound is used to cut the top and the bottom bound of + the produced embedding. + If not provided, 2 * sigma will be used as the adv_bound factor + sigma (:obj:`float`, `optional`): The std factor used to produce a 0 mean normal distribution. + If adv_bound not provided, 2 * sigma will be used as the adv_bound factor + """ + + model_type = 'structbert' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + **kwargs): + super().__init__(pad_token_id=pad_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + # adv_grad_factor, used in adv loss. + # Users can check adv_utils.py for details. + # if adv_grad_factor set to None, no adv loss will not applied to the model. + self.adv_grad_factor = 5e-5 if 'adv_grad_factor' not in kwargs else kwargs[ + 'adv_grad_factor'] + # sigma value, used in adv loss. + self.sigma = 5e-6 if 'sigma' not in kwargs else kwargs['sigma'] + # adv_bound value, used in adv loss. + self.adv_bound = 2 * self.sigma if 'adv_bound' not in kwargs else kwargs[ + 'adv_bound'] diff --git a/runtime/Lib/site_packages/models/nlp/structbert/faq_question_answering.py b/runtime/Lib/site_packages/models/nlp/structbert/faq_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..bc22ab617cb683792c503710966f6e4bef2e24f9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/faq_question_answering.py @@ -0,0 +1,653 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import os +from collections import namedtuple +from typing import Dict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import BCEWithLogitsLoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.models.nlp.structbert import SbertConfig, SbertModel +from modelscope.models.nlp.task_models.task_model import BaseTaskModel +from modelscope.outputs import FaqQuestionAnsweringOutput +from modelscope.utils.config import Config, ConfigFields +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +activations = { + 'relu': F.relu, + 'tanh': torch.tanh, + 'linear': lambda x: x, +} + +activation_coeffs = { + 'relu': math.sqrt(2), + 'tanh': 5 / 3, + 'linear': 1., +} + + +class LinearProjection(nn.Module): + + def __init__(self, + in_features, + out_features, + activation='linear', + bias=True): + super().__init__() + self.activation = activations[activation] + activation_coeff = activation_coeffs[activation] + linear = nn.Linear(in_features, out_features, bias=bias) + nn.init.normal_( + linear.weight, std=math.sqrt(1. / in_features) * activation_coeff) + if bias: + nn.init.zeros_(linear.bias) + self.model = nn.utils.weight_norm(linear) + + def forward(self, x): + return self.activation(self.model(x)) + + +class RelationModule(nn.Module): + + def __init__(self, args): + super(RelationModule, self).__init__() + input_size = args.proj_hidden_size * 4 + self.prediction = torch.nn.Sequential( + LinearProjection( + input_size, args.proj_hidden_size * 4, activation='relu'), + nn.Dropout(args.dropout), + LinearProjection(args.proj_hidden_size * 4, 1)) + + def forward(self, query, protos): + n_cls = protos.shape[0] + n_query = query.shape[0] + protos = protos.unsqueeze(0).repeat(n_query, 1, 1) + query = query.unsqueeze(1).repeat(1, n_cls, 1) + input_feat = torch.cat( + [query, protos, (protos - query).abs(), query * protos], dim=-1) + dists = self.prediction(input_feat) # [bsz,n_query,n_cls,1] + return dists.squeeze(-1) + + +class MetricsLayer(nn.Module): + + def __init__(self, args): + super(MetricsLayer, self).__init__() + self.args = args + assert args.metrics in ('relation', 'cosine') + if args.metrics == 'relation': + self.relation_net = RelationModule(args) + + @property + def name(self): + return self.args.metrics + + def forward(self, query, protos): + if self.args.metrics == 'cosine': + supervised_dists = self.cosine_similarity(query, protos) + if self.training: + supervised_dists *= 5 + elif self.args.metrics in ('relation', ): + supervised_dists = self.relation_net(query, protos) + else: + raise NotImplementedError + return supervised_dists + + def cosine_similarity(self, x, y): + n_query = x.shape[0] + n_cls = y.shape[0] + dim = x.shape[-1] + x = x.unsqueeze(1).expand([n_query, n_cls, dim]) + y = y.unsqueeze(0).expand([n_query, n_cls, dim]) + return F.cosine_similarity(x, y, -1) + + +class AveragePooling(nn.Module): + + def forward(self, x, mask, dim=1): + return torch.sum( + x * mask.float(), dim=dim) / torch.sum( + mask.float(), dim=dim) + + +class AttnPooling(nn.Module): + + def __init__(self, input_size, hidden_size=None, output_size=None): + super().__init__() + self.input_proj = nn.Sequential( + LinearProjection(input_size, hidden_size), nn.Tanh(), + LinearProjection(hidden_size, 1, bias=False)) + self.output_proj = LinearProjection( + input_size, output_size) if output_size else lambda x: x + + def forward(self, x, mask): + score = self.input_proj(x) + score = score * mask.float() + -1e4 * (1. - mask.float()) + score = F.softmax(score, dim=1) + features = self.output_proj(x) + return torch.matmul(score.transpose(1, 2), features).squeeze(1) + + +class PoolingLayer(nn.Module): + + def __init__(self, args): + super(PoolingLayer, self).__init__() + if args.pooling == 'attn': + self.pooling = AttnPooling(args.proj_hidden_size, + args.proj_hidden_size, + args.proj_hidden_size) + elif args.pooling == 'avg': + self.pooling = AveragePooling() + else: + raise NotImplementedError(args.pooling) + + def forward(self, x, mask): + return self.pooling(x, mask) + + +class Alignment(nn.Module): + + def __init__(self): + super().__init__() + + def _attention(self, a, b): + return torch.matmul(a, b.transpose(1, 2)) + + def forward(self, a, b, mask_a, mask_b): + attn = self._attention(a, b) + mask = torch.matmul(mask_a.float(), mask_b.transpose(1, 2).float()) + mask = mask.bool() + attn.masked_fill_(~mask, -1e4) + return attn + + +def _create_args(model_config, hidden_size): + metric = model_config.get('metric', 'cosine') + pooling_method = model_config.get('pooling', 'avg') + Arg = namedtuple( + 'args', + ['metrics', 'proj_hidden_size', 'hidden_size', 'dropout', 'pooling']) + args = Arg( + metrics=metric, + proj_hidden_size=hidden_size, + hidden_size=hidden_size, + dropout=0.0, + pooling=pooling_method) + return args + + +@MODELS.register_module( + Tasks.faq_question_answering, module_name=Models.structbert) +class SbertForFaqQuestionAnswering(BaseTaskModel): + _backbone_prefix = '' + PROTO_NET = 'protonet' + MGIMN_NET = 'mgimnnet' + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir') + model = cls(model_dir, **kwargs) + model.load_checkpoint(model_dir) + return model + + def __init__(self, model_dir, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + backbone_cfg = SbertConfig.from_pretrained(model_dir) + model_config = Config.from_file( + os.path.join(model_dir, + ModelFile.CONFIGURATION)).get(ConfigFields.model, {}) + model_config.update(kwargs) + + network_name = model_config.get('network', self.PROTO_NET) + if network_name == self.PROTO_NET: + network = ProtoNet(backbone_cfg, model_config) + elif network_name == self.MGIMN_NET: + network = MGIMNNet(backbone_cfg, model_config) + else: + raise NotImplementedError(network_name) + logger.info(f'faq task build {network_name} network') + self.network = network + + def forward(self, input: Dict[str, Tensor]) -> FaqQuestionAnsweringOutput: + """ + Args: + input (Dict[str, Tensor]): the preprocessed data, it contains the following keys: + + - query(:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + The query to be predicted. + - support(:obj:`torch.LongTensor` of shape :obj:`(support_size, sequence_length)`): + The support set. + - support_label(:obj:`torch.LongTensor` of shape :obj:`(support_size, )`): + The labels of support set. + + Returns: + Dict[str, Tensor]: result, it contains the following key: + + - scores(:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_cls)`): + Predicted scores of all classes for each query. + + Examples: + >>> from modelscope.hub.snapshot_download import snapshot_download + >>> from modelscope.preprocessors import FaqQuestionAnsweringTransformersPreprocessor + >>> from modelscope.models.nlp import SbertForFaqQuestionAnswering + >>> cache_path = snapshot_download('damo/nlp_structbert_faq-question-answering_chinese-base') + >>> preprocessor = FaqQuestionAnsweringTransformersPreprocessor.from_pretrained(cache_path) + >>> model = SbertForFaqQuestionAnswering.from_pretrained(cache_path) + >>> param = { + >>> 'query_set': ['如何使用优惠券', '在哪里领券', '在哪里领券'], + >>> 'support_set': [{ + >>> 'text': '卖品代金券怎么用', + >>> 'label': '6527856' + >>> }, { + >>> 'text': '怎么使用优惠券', + >>> 'label': '6527856' + >>> }, { + >>> 'text': '这个可以一起领吗', + >>> 'label': '1000012000' + >>> }, { + >>> 'text': '付款时送的优惠券哪里领', + >>> 'label': '1000012000' + >>> }, { + >>> 'text': '购物等级怎么长', + >>> 'label': '13421097' + >>> }, { + >>> 'text': '购物等级二心', + >>> 'label': '13421097' + >>> }] + >>> } + >>> result = model(preprocessor(param)) + """ + query = input['query'] + support = input['support'] + query_mask = input['query_attention_mask'] + support_mask = input['support_attention_mask'] + support_labels = input['support_labels'] + logits, scores = self.network(query, support, query_mask, support_mask, + support_labels) + + if 'labels' in input: + query_labels = input['labels'] + num_cls = torch.max(support_labels) + 1 + loss = self._compute_loss(logits, query_labels, num_cls) + pred_labels = torch.argmax(scores, dim=1) + return FaqQuestionAnsweringOutput( + loss=loss, logits=scores, labels=pred_labels).to_dict() + else: + return FaqQuestionAnsweringOutput(scores=scores) + + def _compute_loss(self, logits, target, num_cls): + onehot_labels = get_onehot_labels(target, num_cls) + loss = BCEWithLogitsLoss(reduction='mean')(logits, onehot_labels) + return loss + + def forward_sentence_embedding(self, inputs): + return self.network.sentence_embedding(inputs) + + def load_checkpoint(self, + model_local_dir, + default_dtype=None, + load_state_fn=None, + **kwargs): + ckpt_file = os.path.join(model_local_dir, 'pytorch_model.bin') + state_dict = torch.load(ckpt_file, map_location='cpu') + # compatible with the old checkpoints + new_state_dict = {} + for var_name, var_value in state_dict.items(): + new_var_name = var_name + if not str(var_name).startswith('network'): + new_var_name = f'network.{var_name}' + new_state_dict[new_var_name] = var_value + if default_dtype is not None: + torch.set_default_dtype(default_dtype) + + missing_keys, unexpected_keys, mismatched_keys, error_msgs = self._load_checkpoint( + new_state_dict, + load_state_fn=load_state_fn, + ignore_mismatched_sizes=True, + _fast_init=True, + ) + + return { + 'missing_keys': missing_keys, + 'unexpected_keys': unexpected_keys, + 'mismatched_keys': mismatched_keys, + 'error_msgs': error_msgs, + } + + +def get_onehot_labels(target, num_cls): + target = target.view(-1, 1) + size = target.shape[0] + target_oh = torch.zeros(size, num_cls).to(target) + target_oh.scatter_(dim=1, index=target, value=1) + return target_oh.view(size, num_cls).float() + + +class ProtoNet(nn.Module): + + def __init__(self, backbone_config, model_config): + super(ProtoNet, self).__init__() + self.bert = SbertModel(backbone_config) + args = _create_args(model_config, self.bert.config.hidden_size) + self.metrics_layer = MetricsLayer(args) + self.pooling = PoolingLayer(args) + + def __call__(self, query, support, query_mask, support_mask, + support_labels): + n_query = query.shape[0] + + num_cls = torch.max(support_labels) + 1 + onehot_labels = get_onehot_labels(support_labels, num_cls) + + input_ids = torch.cat([query, support]) + input_mask = torch.cat([query_mask, support_mask], dim=0) + pooled_representation = self.sentence_embedding({ + 'input_ids': + input_ids, + 'attention_mask': + input_mask + }) + z_query = pooled_representation[:n_query] + z_support = pooled_representation[n_query:] + cls_n_support = torch.sum(onehot_labels, dim=-2) + 1e-5 + protos = torch.matmul(onehot_labels.transpose(0, 1), + z_support) / cls_n_support.unsqueeze(-1) + logits = self.metrics_layer(z_query, protos).view([n_query, num_cls]) + if self.metrics_layer.name == 'relation': + scores = torch.sigmoid(logits) + else: + scores = logits + return logits, scores + + def sentence_embedding(self, inputs: Dict[str, Tensor]): + input_ids = inputs['input_ids'] + input_mask = inputs['attention_mask'] + if not isinstance(input_ids, Tensor): + input_ids = torch.IntTensor(input_ids) + if not isinstance(input_mask, Tensor): + input_mask = torch.IntTensor(input_mask) + rst = self.bert(input_ids, input_mask) + last_hidden_states = rst.last_hidden_state + if len(input_mask.shape) == 2: + input_mask = input_mask.unsqueeze(-1) + pooled_representation = self.pooling(last_hidden_states, input_mask) + return pooled_representation + + +class MGIMNNet(nn.Module): + # default use class_level_interaction only + INSTANCE_LEVEL_INTERACTION = 'instance_level_interaction' + EPISODE_LEVEL_INTERACTION = 'episode_level_interaction' + + def __init__(self, backbone_config, model_config): + super(MGIMNNet, self).__init__() + self.bert = SbertModel(backbone_config) + self.model_config = model_config + self.alignment = Alignment() + hidden_size = self.bert.config.hidden_size + use_instance_level_interaction = self.safe_get( + self.INSTANCE_LEVEL_INTERACTION, True) + use_episode_level_interaction = self.safe_get( + self.EPISODE_LEVEL_INTERACTION, True) + output_size = 1 + int(use_instance_level_interaction) + int( + use_episode_level_interaction) + logger.info( + f'faq MGIMN model class-level-interaction:true, instance-level-interaction:{use_instance_level_interaction}, \ + episode-level-interaction:{use_episode_level_interaction}') + self.fuse_proj = LinearProjection( + hidden_size + hidden_size * 3 * output_size, + hidden_size, + activation='relu') + args = _create_args(model_config, hidden_size) + self.pooling = PoolingLayer(args) + new_args = args._replace(pooling='avg') + self.avg_pooling = PoolingLayer(new_args) + + self.instance_compare_layer = torch.nn.Sequential( + LinearProjection(hidden_size * 4, hidden_size, activation='relu')) + + self.prediction = torch.nn.Sequential( + LinearProjection(hidden_size * 2, hidden_size, activation='relu'), + nn.Dropout(0), LinearProjection(hidden_size, 1)) + + def __call__(self, query, support, query_mask, support_mask, + support_labels): + z_query, z_support = self.context_embedding(query, support, query_mask, + support_mask) + n_cls = int(torch.max(support_labels)) + 1 + n_query, sent_len = query.shape + n_support = support.shape[0] + k_shot = n_support // n_cls + + q_params, s_params = { + 'n_cls': n_cls, + 'k_shot': k_shot + }, { + 'n_query': n_query + } + if self.safe_get(self.INSTANCE_LEVEL_INTERACTION, True): + ins_z_query, ins_z_support = self._instance_level_interaction( + z_query, query_mask, z_support, support_mask) + q_params['ins_z_query'] = ins_z_query + s_params['ins_z_support'] = ins_z_support + + cls_z_query, cls_z_support = self._class_level_interaction( + z_query, query_mask, z_support, support_mask, n_cls) + q_params['cls_z_query'] = cls_z_query + s_params['cls_z_support'] = cls_z_support + if self.safe_get(self.EPISODE_LEVEL_INTERACTION, True): + eps_z_query, eps_z_support = self._episode_level_interaction( + z_query, query_mask, z_support, support_mask) + q_params['eps_z_query'] = eps_z_query + s_params['eps_z_support'] = eps_z_support + fused_z_query = self._fuse_query(z_query, **q_params) + fused_z_support = self._fuse_support(z_support, **s_params) + query_mask_expanded = query_mask.unsqueeze(1).repeat( + 1, n_support, 1).view(n_query * n_support, sent_len, 1) + support_mask_expanded = support_mask.unsqueeze(0).repeat( + n_query, 1, 1).view(n_query * n_support, sent_len, 1) + Q = self.pooling(fused_z_query, query_mask_expanded) + S = self.pooling(fused_z_support, support_mask_expanded) + matching_feature = self._instance_compare(Q, S, n_query, n_cls, k_shot) + logits = self.prediction(matching_feature) + logits = logits.view(n_query, n_cls) + return logits, torch.sigmoid(logits) + + def _instance_compare(self, Q, S, n_query, n_cls, k_shot): + z_dim = Q.shape[-1] + S = S.view(n_query, n_cls * k_shot, z_dim) + Q = Q.view(n_query, k_shot * n_cls, z_dim) + cat_features = torch.cat([Q, S, Q * S, (Q - S).abs()], dim=-1) + instance_matching_feature = self.instance_compare_layer(cat_features) + instance_matching_feature = instance_matching_feature.view( + n_query, n_cls, k_shot, z_dim) + cls_matching_feature_mean = instance_matching_feature.mean(2) + cls_matching_feature_max, _ = instance_matching_feature.max(2) + cls_matching_feature = torch.cat( + [cls_matching_feature_mean, cls_matching_feature_max], dim=-1) + return cls_matching_feature + + def _instance_level_interaction(self, z_query, query_mask, z_support, + support_mask): + n_query, sent_len, z_dim = z_query.shape + n_support = z_support.shape[0] + z_query = z_query.unsqueeze(1).repeat(1, n_support, 1, + 1).view(n_query * n_support, + sent_len, z_dim) + query_mask = query_mask.unsqueeze(1).repeat(1, n_support, 1).view( + n_query * n_support, sent_len, 1) + z_support = z_support.unsqueeze(0).repeat(n_query, 1, 1, 1).view( + n_query * n_support, sent_len, z_dim) + support_mask = support_mask.unsqueeze(0).repeat(n_query, 1, 1, 1).view( + n_query * n_support, sent_len, 1) + attn = self.alignment(z_query, z_support, query_mask, support_mask) + attn_a = F.softmax(attn, dim=1) + attn_b = F.softmax(attn, dim=2) + ins_support = torch.matmul(attn_a.transpose(1, 2), z_query) + ins_query = torch.matmul(attn_b, z_support) + return ins_query, ins_support + + def _class_level_interaction(self, z_query, query_mask, z_support, + support_mask, n_cls): + z_support_ori = z_support + support_mask_ori = support_mask + + n_query, sent_len, z_dim = z_query.shape + n_support = z_support.shape[0] + k_shot = n_support // n_cls + + # class-based query encoding + z_query = z_query.unsqueeze(1).repeat(1, n_cls, 1, + 1).view(n_query * n_cls, + sent_len, z_dim) + query_mask = query_mask.unsqueeze(1).unsqueeze(-1).repeat( + 1, n_cls, 1, 1).view(n_query * n_cls, sent_len, 1) + z_support = z_support.unsqueeze(0).repeat(n_query, 1, 1, 1).view( + n_query * n_cls, k_shot * sent_len, z_dim) + support_mask = support_mask.unsqueeze(0).unsqueeze(-1).repeat( + n_query, 1, 1, 1).view(n_query * n_cls, k_shot * sent_len, 1) + attn = self.alignment(z_query, z_support, query_mask, support_mask) + attn_b = F.softmax(attn, dim=2) + cls_query = torch.matmul(attn_b, z_support) + cls_query = cls_query.view(n_query, n_cls, sent_len, z_dim) + + # class-based support encoding + z_support = z_support_ori.view(n_cls, k_shot * sent_len, z_dim) + support_mask = support_mask_ori.view(n_cls, k_shot * sent_len, 1) + attn = self.alignment(z_support, z_support, support_mask, support_mask) + attn_b = F.softmax(attn, dim=2) + cls_support = torch.matmul(attn_b, z_support) + cls_support = cls_support.view(n_cls * k_shot, sent_len, z_dim) + return cls_query, cls_support + + def _episode_level_interaction(self, z_query, query_mask, z_support, + support_mask): + z_support_ori = z_support + support_mask_ori = support_mask + + n_query, sent_len, z_dim = z_query.shape + n_support = z_support.shape[0] + + # episode-based query encoding + query_mask = query_mask.view(n_query, sent_len, 1) + z_support = z_support.unsqueeze(0).repeat(n_query, 1, 1, 1).view( + n_query, n_support * sent_len, z_dim) + support_mask = support_mask.unsqueeze(0).unsqueeze(-1).repeat( + n_query, 1, 1, 1).view(n_query, n_support * sent_len, 1) + attn = self.alignment(z_query, z_support, query_mask, support_mask) + attn_b = F.softmax(attn, dim=2) + eps_query = torch.matmul(attn_b, z_support) + + # episode-based support encoding + z_support2 = z_support_ori.view(1, n_support * sent_len, + z_dim).repeat(n_support, 1, 1) + support_mask = support_mask_ori.view(1, n_support * sent_len, + 1).repeat(n_support, 1, 1) + attn = self.alignment(z_support_ori, z_support2, + support_mask_ori.unsqueeze(-1), support_mask) + attn_b = F.softmax(attn, dim=2) + eps_support = torch.matmul(attn_b, z_support2) + eps_support = eps_support.view(n_support, sent_len, z_dim) + return eps_query, eps_support + + def _fuse_query(self, + x, + n_cls, + k_shot, + ins_z_query=None, + cls_z_query=None, + eps_z_query=None): + n_query, sent_len, z_dim = x.shape + assert cls_z_query is not None + cls_features = cls_z_query.unsqueeze(2).repeat( + 1, 1, k_shot, 1, 1).view(n_cls * k_shot * n_query, sent_len, z_dim) + x = x.unsqueeze(1).repeat(1, n_cls * k_shot, 1, + 1).view(n_cls * k_shot * n_query, sent_len, + z_dim) + features = [ + x, cls_features, x * cls_features, (x - cls_features).abs() + ] + if ins_z_query is not None: + features.extend( + [ins_z_query, ins_z_query * x, (ins_z_query - x).abs()]) + if eps_z_query is not None: + eps_z_query = eps_z_query.unsqueeze(1).repeat( + 1, n_cls * k_shot, 1, 1).view(n_cls * k_shot * n_query, + sent_len, z_dim) + features.extend( + [eps_z_query, eps_z_query * x, (eps_z_query - x).abs()]) + features = torch.cat(features, dim=-1) + fusion_feat = self.fuse_proj(features) + return fusion_feat + + def _fuse_support(self, + x, + n_query, + ins_z_support=None, + cls_z_support=None, + eps_z_support=None): + assert cls_z_support is not None + n_support, sent_len, z_dim = x.shape + x = x.unsqueeze(0).repeat(n_query, 1, 1, + 1).view(n_support * n_query, sent_len, z_dim) + cls_features = cls_z_support.unsqueeze(0).repeat( + n_query, 1, 1, 1).view(n_support * n_query, sent_len, z_dim) + features = [ + x, cls_features, x * cls_features, (x - cls_features).abs() + ] + if ins_z_support is not None: + features.extend( + [ins_z_support, ins_z_support * x, (ins_z_support - x).abs()]) + if eps_z_support is not None: + eps_z_support = eps_z_support.unsqueeze(0).repeat( + n_query, 1, 1, 1).view(n_query * n_support, sent_len, z_dim) + features.extend( + [eps_z_support, eps_z_support * x, (eps_z_support - x).abs()]) + features = torch.cat(features, dim=-1) + fusion_feat = self.fuse_proj(features) + return fusion_feat + + def context_embedding(self, query, support, query_mask, support_mask): + n_query = query.shape[0] + n_support = support.shape[0] + x = torch.cat([query, support], dim=0) + x_mask = torch.cat([query_mask, support_mask], dim=0) + last_hidden_state = self.bert(x, x_mask).last_hidden_state + z_dim = last_hidden_state.shape[-1] + sent_len = last_hidden_state.shape[-2] + z_query = last_hidden_state[:n_query].view([n_query, sent_len, z_dim]) + z_support = last_hidden_state[n_query:].view( + [n_support, sent_len, z_dim]) + return z_query, z_support + + def sentence_embedding(self, inputs: Dict[str, Tensor]): + input_ids = inputs['input_ids'] + input_mask = inputs['attention_mask'] + if not isinstance(input_ids, Tensor): + input_ids = torch.IntTensor(input_ids) + if not isinstance(input_mask, Tensor): + input_mask = torch.IntTensor(input_mask) + rst = self.bert(input_ids, input_mask) + last_hidden_states = rst.last_hidden_state + if len(input_mask.shape) == 2: + input_mask = input_mask.unsqueeze(-1) + pooled_representation = self.avg_pooling(last_hidden_states, + input_mask) + return pooled_representation + + def safe_get(self, k, default=None): + try: + return self.model_config.get(k, default) + except Exception as e: + logger.debug(f'{k} not in model_config, use default:{default}') + logger.debug(e) + return default diff --git a/runtime/Lib/site_packages/models/nlp/structbert/fill_mask.py b/runtime/Lib/site_packages/models/nlp/structbert/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..4ded0d6ca92a314db0788e7a9a16a06f1fffcdcb --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/fill_mask.py @@ -0,0 +1,286 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss +from transformers.activations import ACT2FN + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionFillMaskModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .backbone import SbertModel, SbertPreTrainedModel +from .configuration import SbertConfig + +logger = logging.get_logger() + + +class SbertPredictionHeadTransform(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class SbertLMPredictionHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.transform = SbertPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.hidden_size, config.vocab_size) + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class SbertOnlyMLMHead(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = SbertLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class SbertPreTrainingHeads(nn.Module): + + def __init__(self, config): + super().__init__() + self.predictions = SbertLMPredictionHead(config) + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, sequence_output, pooled_output): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + return prediction_scores, seq_relationship_score + + +@MODELS.register_module(Tasks.fill_mask, module_name=Models.structbert) +class SbertForMaskedLM(SbertPreTrainedModel): + r"""StructBERT Model with a `language modeling` head on top. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the fill_mask model of StructBERT, the preprocessor of this model + is `modelscope.preprocessors.FillMaskTransformersPreprocessor`. + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def __init__(self, config: SbertConfig, **kwargs): + super().__init__(config) + + if config.is_decoder: + logger.warning( + 'If you want to use `SbertForMaskedLM` make sure `config.is_decoder=False` for ' + 'bi-directional self-attention.') + + self.bert = SbertModel(config) + self.cls = SbertOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + `What are input IDs? <../glossary.html#input-ids>`__ + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + `What are attention masks? <../glossary.html#attention-mask>`__ + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + + Returns: + Returns `modelscope.outputs.AttentionFillMaskModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor, FillMaskTransformersPreprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_fill-mask_chinese-large') + >>> preprocessor = FillMaskTransformersPreprocessor('damo/nlp_structbert_fill-mask_chinese-large') + >>> # Call the model, return some tensors + >>> print(model(**preprocessor('你师父差得动你,你师父可[MASK]不动我。'))) + >>> # Call the pipeline + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('fill-mask', model=model, preprocessor=preprocessor) + >>> print(pipeline_ins('你师父差得动你,你师父可[MASK]不动我。')) + """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct( + prediction_scores.view(-1, self.config.vocab_size), + labels.view(-1)) + + if not return_dict: + output = (prediction_scores, ) + outputs[2:-1] + return ((masked_lm_loss, ) + + output) if masked_lm_loss is not None else output + + return AttentionFillMaskModelOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + input_ids=input_ids, + ) + + def prepare_inputs_for_generation(self, + input_ids, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + assert self.config.pad_token_id is not None, 'The PAD token should be defined for generation' + attention_mask_zero = attention_mask.new_zeros( + (attention_mask.shape[0], 1)) + attention_mask = torch.cat([attention_mask, attention_mask_zero], + dim=-1) + dummy_token = torch.full((effective_batch_size, 1), + self.config.pad_token_id, + dtype=torch.long, + device=input_ids.device) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {'input_ids': input_ids, 'attention_mask': attention_mask} diff --git a/runtime/Lib/site_packages/models/nlp/structbert/text_classification.py b/runtime/Lib/site_packages/models/nlp/structbert/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..d1cdb01d0b670e3318801de76455b776f505ec60 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/text_classification.py @@ -0,0 +1,238 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .adv_utils import compute_adv_loss +from .backbone import SbertModel, SbertPreTrainedModel +from .configuration import SbertConfig + +logger = logging.get_logger() + + +@MODELS.register_module( + Tasks.text_classification, module_name=Models.structbert) +@MODELS.register_module(Tasks.nli, module_name=Models.structbert) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.structbert) +@MODELS.register_module( + Tasks.sentence_similarity, module_name=Models.structbert) +@MODELS.register_module( + Tasks.zero_shot_classification, module_name=Models.structbert) +class SbertForSequenceClassification(SbertPreTrainedModel): + r"""StructBERT Model transformer with a sequence classification/regression head on top + (a linear layer on top of the pooled output) e.g. for GLUE tasks. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the text classification model of StructBERT, the preprocessor of this model + is `modelscope.preprocessors.TextClassificationTransformersPreprocessor`. + + Trainer: + This model is a normal PyTorch model, and can be trained by variable trainers, like EpochBasedTrainer, + NlpEpochBasedTrainer, or trainers from other frameworks. + The preferred trainer in ModelScope is NlpEpochBasedTrainer. + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + + def __init__(self, config: SbertConfig, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + if self.config.adv_grad_factor is None: + logger.warning( + 'Adv parameters not set, skipping compute_adv_loss.') + + SbertForSequenceClassification.base_model_prefix = getattr( + config, 'base_model_prefix', + SbertForSequenceClassification.base_model_prefix) + setattr(self, self.base_model_prefix, SbertModel(config)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + self.init_weights() + + def _forward_call(self, **kwargs): + outputs = self.base_model(**kwargs) + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + outputs['logits'] = logits + outputs.kwargs = kwargs + return outputs + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + *args, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed + (Mean-Square loss), If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Returns: + Returns `modelscope.outputs.AttentionTextClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> # Call the model, return some tensors + >>> print(model(**preprocessor(('这是个测试', '这也是个测试')))) + >>> # Call the pipeline + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('text-classification', model=model, preprocessor=preprocessor) + >>> print(pipeline_ins(('这是个测试', '这也是个测试'))) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if not return_dict: + logger.error('Return tuple in sbert is not supported now.') + outputs = self._forward_call( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + return self.compute_loss(outputs, labels, **outputs.kwargs) + + def compute_loss(self, outputs, labels, **kwargs): + logits = outputs.logits + embedding_output = outputs.embedding_output + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = 'regression' + elif self.num_labels > 1 and (labels.dtype == torch.long + or labels.dtype == torch.int): + self.config.problem_type = 'single_label_classification' + else: + self.config.problem_type = 'multi_label_classification' + + if self.config.problem_type == 'regression': + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == 'single_label_classification': + loss_fct = CrossEntropyLoss() + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + if self.config.adv_grad_factor is not None and self.training: + loss = compute_adv_loss( + embedding=embedding_output, + model=self._forward_call, + ori_logits=logits, + ori_loss=loss, + adv_bound=self.config.adv_bound, + adv_grad_factor=self.config.adv_grad_factor, + sigma=self.config.sigma, + **kwargs) + elif self.config.problem_type == 'multi_label_classification': + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + + return AttentionTextClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/structbert/token_classification.py b/runtime/Lib/site_packages/models/nlp/structbert/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..4ee2dda718f3ab2e18383ff6eaec03187ccff8d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/structbert/token_classification.py @@ -0,0 +1,230 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss + +from modelscope.metainfo import Models +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTokenClassificationModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .adv_utils import compute_adv_loss +from .backbone import SbertModel, SbertPreTrainedModel +from .configuration import SbertConfig + +logger = logging.get_logger() + + +@MODELS.register_module( + Tasks.token_classification, module_name=Models.structbert) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.structbert) +@MODELS.register_module(Tasks.part_of_speech, module_name=Models.structbert) +class SbertForTokenClassification(SbertPreTrainedModel): + r"""StructBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) + e.g. for Named-Entity-Recognition (NER) tasks. + + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the token-classification model of StructBERT, the preprocessor of this model + is `modelscope.preprocessors.TokenClassificationTransformersPreprocessor`. + + Trainer: + This model is a normal PyTorch model, and can be trained by variable trainers, like EpochBasedTrainer, + NlpEpochBasedTrainer, or trainers from other frameworks. + The preferred trainer in modelscope is NlpEpochBasedTrainer. + + Parameters: + config (:class:`~modelscope.models.nlp.structbert.SbertConfig`): Model configuration class with + all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. + """ + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + + def __init__(self, config: SbertConfig, **kwargs): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + if self.config.adv_grad_factor is None: + logger.warning( + 'Adv parameters not set, skipping compute_adv_loss.') + setattr(self, self.base_model_prefix, + SbertModel(config, add_pooling_layer=False)) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None + else config.hidden_dropout_prob) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def _forward_call(self, **kwargs): + outputs = self.bert(**kwargs) + sequence_output = outputs[0] + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + outputs['logits'] = logits + outputs.kwargs = kwargs + return outputs + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + offset_mapping=None, + label_mask=None, + ): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` + for details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range + ``[0, config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, + `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, + `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded + representation. This is useful if you want more control over how to convert :obj:`input_ids` indices + into associated vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under + returned tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors + for more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the token classification loss. Indices should be in + ``[0, ..., config.num_labels - 1]``. + offset_mapping (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the sentence. + Selected in the range ``[0, sequence_length - 1]``. + label_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask + values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + Returns `modelscope.outputs.AttentionTokenClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_word-segmentation_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_word-segmentation_chinese-base') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if not return_dict: + logger.error('Return tuple in sbert is not supported now.') + + outputs = self._forward_call( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict) + + logits = outputs.logits + embedding_output = outputs.embedding_output + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), + torch.tensor(loss_fct.ignore_index).type_as(labels)) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct( + logits.view(-1, self.num_labels), labels.view(-1)) + if self.config.adv_grad_factor is not None and self.training: + loss = compute_adv_loss( + embedding=embedding_output, + model=self._forward_call, + ori_logits=logits, + ori_loss=loss, + adv_bound=self.config.adv_bound, + adv_grad_factor=self.config.adv_grad_factor, + sigma=self.config.sigma, + with_attention_mask=attention_mask is not None, + **outputs.kwargs) + + return AttentionTokenClassificationModelOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + offset_mapping=offset_mapping, + label_mask=label_mask, + ) diff --git a/runtime/Lib/site_packages/models/nlp/task_models/__init__.py b/runtime/Lib/site_packages/models/nlp/task_models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6900d5cace38e9a349360762cb4797e52539d83c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .information_extraction import ModelForInformationExtraction + from .feature_extraction import ModelForFeatureExtraction + from .fill_mask import ModelForFillMask + from .text_classification import ModelForTextClassification + from .task_model import SingleBackboneTaskModelBase + from .token_classification import (ModelForTokenClassification, + ModelForTokenClassificationWithCRF) + from .text_generation import ModelForTextGeneration + from .text_ranking import ModelForTextRanking + from .machine_reading_comprehension import ModelForMachineReadingComprehension + +else: + _import_structure = { + 'information_extraction': ['ModelForInformationExtraction'], + 'feature_extraction': ['ModelForFeatureExtraction'], + 'fill_mask': ['ModelForFillMask'], + 'text_classification': ['ModelForTextClassification'], + 'task_model': ['SingleBackboneTaskModelBase'], + 'token_classification': + ['ModelForTokenClassification', 'ModelForTokenClassificationWithCRF'], + 'text_generation': ['ModelForTextGeneration'], + 'text_ranking': ['ModelForTextRanking'], + 'machine_reading_comprehension': + ['ModelForMachineReadingComprehension'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/task_models/feature_extraction.py b/runtime/Lib/site_packages/models/nlp/task_models/feature_extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..e39f5d8d149760c29d212927d20824f53945ed57 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/feature_extraction.py @@ -0,0 +1,109 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np + +from modelscope.metainfo import TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.outputs import FeatureExtractionOutput, OutputKeys +from modelscope.utils.constant import Tasks + +__all__ = ['ModelForFeatureExtraction'] + + +@MODELS.register_module( + Tasks.feature_extraction, module_name=TaskModels.feature_extraction) +class ModelForFeatureExtraction(EncoderModel): + task = Tasks.feature_extraction + + # There is no head for feature extraction method + head_type = None + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + + Returns: + Returns `modelscope.outputs.AttentionTextClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + + # backbone do not need labels, only head need for loss compute + feature = self.extract_feature( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + return FeatureExtractionOutput( + text_embedding=feature.last_hidden_state) diff --git a/runtime/Lib/site_packages/models/nlp/task_models/fill_mask.py b/runtime/Lib/site_packages/models/nlp/task_models/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..b96e86c5f9604cc32abfd35bf1b48e0c1e5a5656 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/fill_mask.py @@ -0,0 +1,74 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np +import torch + +from modelscope.metainfo import Heads, TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.utils.constant import Tasks + +__all__ = ['ModelForFillMask'] + + +@MODELS.register_module(Tasks.fill_mask, module_name=TaskModels.fill_mask) +class ModelForFillMask(EncoderModel): + task = Tasks.fill_mask + + # The default base head type is fill-mask for this head + head_type = Heads.fill_mask + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [ + r'position_ids', r'predictions.decoder.bias' + ] + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs): + outputs = super().forward(input_ids, attention_mask, token_type_ids, + position_ids, head_mask, inputs_embeds, + labels, output_attentions, + output_hidden_states, **kwargs) + + outputs.input_ids = input_ids + return outputs + + def get_output_embeddings(self): + return self.head.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.head.cls.predictions.decoder = new_embeddings + + def prepare_inputs_for_generation(self, + input_ids, + attention_mask=None, + **model_kwargs): + input_shape = input_ids.shape + effective_batch_size = input_shape[0] + + # add a dummy token + if self.config.pad_token_id is None: + raise ValueError('The PAD token should be defined for generation') + attention_shape0 = attention_mask.shape[0] + attention_mask = torch.cat( + [attention_mask, + attention_mask.new_zeros((attention_shape0, 1))], + dim=-1) + dummy_token = torch.full((effective_batch_size, 1), + self.config.pad_token_id, + dtype=torch.long, + device=input_ids.device) + input_ids = torch.cat([input_ids, dummy_token], dim=1) + + return {'input_ids': input_ids, 'attention_mask': attention_mask} diff --git a/runtime/Lib/site_packages/models/nlp/task_models/information_extraction.py b/runtime/Lib/site_packages/models/nlp/task_models/information_extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..0eb142772b1eb973cb40f5fbe67e101cc0d32822 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/information_extraction.py @@ -0,0 +1,23 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np + +from modelscope.metainfo import Heads, TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.utils.constant import Tasks + +__all__ = ['ModelForInformationExtraction'] + + +@MODELS.register_module( + Tasks.information_extraction, + module_name=TaskModels.information_extraction) +@MODELS.register_module( + Tasks.relation_extraction, module_name=TaskModels.information_extraction) +class ModelForInformationExtraction(EncoderModel): + task = Tasks.information_extraction + + # The default base head type is fill-mask for this head + head_type = Heads.information_extraction diff --git a/runtime/Lib/site_packages/models/nlp/task_models/machine_reading_comprehension.py b/runtime/Lib/site_packages/models/nlp/task_models/machine_reading_comprehension.py new file mode 100644 index 0000000000000000000000000000000000000000..add62ce8a72410cb9b2423f6f85812008f9cc3ee --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/machine_reading_comprehension.py @@ -0,0 +1,142 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from dataclasses import dataclass +from typing import Any, Dict, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn +from transformers import AutoConfig +from transformers.modeling_outputs import ModelOutput +from transformers.models.roberta.modeling_roberta import ( + RobertaModel, RobertaPreTrainedModel) + +from modelscope.metainfo import Heads, Models, TaskModels +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.outputs import MachineReadingComprehensionOutput, OutputKeys +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.hub import parse_label_mapping + +__all__ = ['ModelForMachineReadingComprehension'] + + +@MODELS.register_module( + Tasks.machine_reading_comprehension, + module_name=TaskModels.machine_reading_comprehension) +class ModelForMachineReadingComprehension(TorchModel): + ''' + Pretrained Machine Reader (PMR) model (https://arxiv.org/pdf/2212.04755.pdf) + + ''' + + _keys_to_ignore_on_load_unexpected = [r'pooler'] + _keys_to_ignore_on_load_missing = [r'position_ids'] + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = AutoConfig.from_pretrained(model_dir) + self.num_labels = self.config.num_labels + self.roberta = RobertaModel(self.config, add_pooling_layer=False) + self.span_transfer = MultiNonLinearProjection( + self.config.hidden_size, + self.config.hidden_size, + self.config.hidden_dropout_prob, + intermediate_hidden_size=self.config. + projection_intermediate_hidden_size) + state_dict = torch.load( + os.path.join(model_dir, ModelFile.TORCH_MODEL_BIN_FILE)) + compatible_position_ids(state_dict, 'roberta.embeddings.position_ids') + self.load_state_dict(state_dict) + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + label_mask=None, + match_labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.roberta( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + # adapted from https://github.com/ShannonAI/mrc-for-flat-nested-ner + # for every position $i$ in sequence, should concate $j$ to + # predict if $i$ and $j$ are start_pos and end_pos for an entity. + # [batch, seq_len, hidden] + span_intermediate = self.span_transfer(sequence_output) + # [batch, seq_len, seq_len] + span_logits = torch.matmul(span_intermediate, + sequence_output.transpose(-1, -2)) + + total_loss = None + if match_labels is not None: + match_loss = self.compute_loss(span_logits, match_labels, + label_mask) + total_loss = match_loss + if not return_dict: + output = (span_logits) + outputs[2:] + return ((total_loss, ) + + output) if total_loss is not None else output + + return MachineReadingComprehensionOutput( + loss=total_loss, + span_logits=span_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class MultiNonLinearProjection(nn.Module): + + def __init__(self, + hidden_size, + num_label, + dropout_rate, + act_func='gelu', + intermediate_hidden_size=None): + super(MultiNonLinearProjection, self).__init__() + self.num_label = num_label + self.intermediate_hidden_size = hidden_size if intermediate_hidden_size is None else intermediate_hidden_size + self.classifier1 = nn.Linear(hidden_size, + self.intermediate_hidden_size) + self.classifier2 = nn.Linear(self.intermediate_hidden_size, + self.num_label) + self.dropout = nn.Dropout(dropout_rate) + self.act_func = act_func + + def forward(self, input_features): + features_output1 = self.classifier1(input_features) + if self.act_func == 'gelu': + features_output1 = F.gelu(features_output1) + elif self.act_func == 'relu': + features_output1 = F.relu(features_output1) + elif self.act_func == 'tanh': + features_output1 = F.tanh(features_output1) + else: + raise ValueError + features_output1 = self.dropout(features_output1) + features_output2 = self.classifier2(features_output1) + return features_output2 diff --git a/runtime/Lib/site_packages/models/nlp/task_models/task_model.py b/runtime/Lib/site_packages/models/nlp/task_models/task_model.py new file mode 100644 index 0000000000000000000000000000000000000000..70f11de95bcf296f484a714d7a4b599b4900d8be --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/task_model.py @@ -0,0 +1,705 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os.path +import re +from abc import ABC +from collections import OrderedDict +from typing import Any, Dict, Optional + +import torch +from torch import nn + +from modelscope.models.base import TorchModel +from modelscope.models.builder import build_backbone, build_head +from modelscope.outputs import OutputKeys +from modelscope.utils.checkpoint import load_task_model_checkpoint +from modelscope.utils.config import Config, ConfigDict +from modelscope.utils.constant import DEFAULT_MODEL_REVISION, ModelFile +from modelscope.utils.file_utils import func_receive_dict_inputs +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['EncoderModel', 'SingleBackboneTaskModelBase'] + + +def _repr(modules, depth=1): + # model name log level control + if depth == 0: + return modules._get_name() + # We treat the extra repr like the sub-module, one item per line + extra_lines = [] + extra_repr = modules.extra_repr() + # empty string will be split into list [''] + if extra_repr: + extra_lines = extra_repr.split('\n') + child_lines = [] + + def _addindent(s_, numSpaces): + s = s_.split('\n') + # don't do anything for single-line stuff + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(numSpaces * ' ') + line for line in s] + s = '\n'.join(s) + s = first + '\n' + s + return s + + for key, module in modules._modules.items(): + mod_str = _repr(module, depth - 1) + mod_str = _addindent(mod_str, 2) + child_lines.append('(' + key + '): ' + mod_str) + lines = extra_lines + child_lines + + main_str = modules._get_name() + '(' + if lines: + # simple one-liner info, which most builtin Modules will use + if len(extra_lines) == 1 and not child_lines: + main_str += extra_lines[0] + else: + main_str += '\n ' + '\n '.join(lines) + '\n' + + main_str += ')' + return main_str + + +class BaseTaskModel(TorchModel, ABC): + """ Base task model interface for nlp + + """ + # keys to ignore when load missing + _keys_to_ignore_on_load_missing = None + # keys to ignore when load unexpected + _keys_to_ignore_on_load_unexpected = None + # backbone prefix, default None + _backbone_prefix = None + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = ConfigDict(kwargs) + + def __repr__(self): + # only log backbone and head name + depth = 1 + return _repr(self, depth) + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.get('model_dir') + model = cls(**kwargs) + model.load_checkpoint(model_local_dir=model_dir, **kwargs) + return model + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + pass + + def load_checkpoint(self, + model_local_dir, + default_dtype=None, + load_state_fn=None, + **kwargs): + """ + Load model checkpoint file and feed the parameters into the model. + Args: + model_local_dir: The actual checkpoint dir on local disk. + default_dtype: Set the default float type by 'torch.set_default_dtype' + load_state_fn: An optional load_state_fn used to load state_dict into the model. + + Returns: + + """ + # TODO Sharded ckpt + ckpt_file = os.path.join(model_local_dir, + ModelFile.TORCH_MODEL_BIN_FILE) + state_dict = torch.load(ckpt_file, map_location='cpu') + if default_dtype is not None: + torch.set_default_dtype(default_dtype) + + missing_keys, unexpected_keys, mismatched_keys, error_msgs = self._load_checkpoint( + state_dict, + load_state_fn=load_state_fn, + ignore_mismatched_sizes=True, + _fast_init=True, + ) + + return { + 'missing_keys': missing_keys, + 'unexpected_keys': unexpected_keys, + 'mismatched_keys': mismatched_keys, + 'error_msgs': error_msgs, + } + + def _load_checkpoint( + self, + state_dict, + load_state_fn, + ignore_mismatched_sizes, + _fast_init, + ): + # Retrieve missing & unexpected_keys + model_state_dict = self.state_dict() + prefix = self._backbone_prefix + + # add head prefix + new_state_dict = OrderedDict() + for name, module in state_dict.items(): + if not name.startswith(prefix) and not name.startswith('head'): + new_state_dict['.'.join(['head', name])] = module + else: + new_state_dict[name] = module + state_dict = new_state_dict + + loaded_keys = [k for k in state_dict.keys()] + expected_keys = list(model_state_dict.keys()) + + def _fix_key(key): + if 'beta' in key: + return key.replace('beta', 'bias') + if 'gamma' in key: + return key.replace('gamma', 'weight') + return key + + original_loaded_keys = loaded_keys + loaded_keys = [_fix_key(key) for key in loaded_keys] + + if len(prefix) > 0: + has_prefix_module = any(s.startswith(prefix) for s in loaded_keys) + expects_prefix_module = any( + s.startswith(prefix) for s in expected_keys) + else: + has_prefix_module = False + expects_prefix_module = False + + # key re-naming operations are never done on the keys + # that are loaded, but always on the keys of the newly initialized model + remove_prefix_from_model = not has_prefix_module and expects_prefix_module + add_prefix_to_model = has_prefix_module and not expects_prefix_module + + if remove_prefix_from_model: + expected_keys_not_prefixed = [ + s for s in expected_keys if not s.startswith(prefix) + ] + expected_keys = [ + '.'.join(s.split('.')[1:]) if s.startswith(prefix) else s + for s in expected_keys + ] + elif add_prefix_to_model: + expected_keys = ['.'.join([prefix, s]) for s in expected_keys] + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + if self._keys_to_ignore_on_load_missing is not None: + for pat in self._keys_to_ignore_on_load_missing: + missing_keys = [ + k for k in missing_keys if re.search(pat, k) is None + ] + + if self._keys_to_ignore_on_load_unexpected is not None: + for pat in self._keys_to_ignore_on_load_unexpected: + unexpected_keys = [ + k for k in unexpected_keys if re.search(pat, k) is None + ] + + if _fast_init: + # retrieve uninitialized modules and initialize + uninitialized_modules = self.retrieve_modules_from_names( + missing_keys, + prefix=prefix, + add_prefix=add_prefix_to_model, + remove_prefix=remove_prefix_from_model) + for module in uninitialized_modules: + self._init_weights(module) + + # Make sure we are able to load base models as well as derived models (with heads) + start_prefix = '' + model_to_load = self + if len(prefix) > 0 and not hasattr(self, prefix) and has_prefix_module: + start_prefix = prefix + '.' + if len(prefix) > 0 and hasattr(self, prefix) and not has_prefix_module: + model_to_load = getattr(self, prefix) + if any(key in expected_keys_not_prefixed for key in loaded_keys): + raise ValueError( + 'The state dictionary of the model you are trying to load is corrupted. Are you sure it was ' + 'properly saved?') + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + add_prefix_to_model, + remove_prefix_from_model, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + if remove_prefix_from_model: + # The model key starts with `prefix` but `checkpoint_key` doesn't so we add it. + model_key = f'{prefix}.{checkpoint_key}' + elif add_prefix_to_model: + # The model key doesn't start with `prefix` but `checkpoint_key` does so we remove it. + model_key = '.'.join(checkpoint_key.split('.')[1:]) + + if (model_key in model_state_dict): + model_shape = model_state_dict[model_key].shape + checkpoint_shape = state_dict[checkpoint_key].shape + if (checkpoint_shape != model_shape): + mismatched_keys.append( + (checkpoint_key, + state_dict[checkpoint_key].shape, + model_state_dict[model_key].shape)) + del state_dict[checkpoint_key] + return mismatched_keys + + def _load_state_dict_into_model(model_to_load, state_dict, + start_prefix): + # Convert old format to new format if needed from a PyTorch state_dict + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + error_msgs = [] + + if load_state_fn is not None: + load_state_fn( + model_to_load, + state_dict, + prefix=start_prefix, + local_metadata=None, + error_msgs=error_msgs) + else: + + def load(module: nn.Module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + args = (state_dict, prefix, local_metadata, True, [], [], + error_msgs) + module._load_from_state_dict(*args) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model_to_load, prefix=start_prefix) + + return error_msgs + + # Whole checkpoint + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + original_loaded_keys, + add_prefix_to_model, + remove_prefix_from_model, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict, + start_prefix) + + if len(error_msgs) > 0: + error_msg = '\n\t'.join(error_msgs) + raise RuntimeError( + f'Error(s) in loading state_dict for {self.__class__.__name__}:\n\t{error_msg}' + ) + + if len(unexpected_keys) > 0: + logger.warning( + f'Some weights of the model checkpoint were not used when' + f' initializing {self.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are' + f' initializing {self.__class__.__name__} from the checkpoint of a model trained on another task or' + ' with another architecture (e.g. initializing a BertForTokenClassification model from a' + ' BertForPreTraining model).\n- This IS NOT expected if you are initializing' + f' {self.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical' + ' (initializing a BertForTokenClassification model from a BertForTokenClassification model).' + ) + else: + logger.info( + f'All model checkpoint weights were used when initializing {self.__class__.__name__}.\n' + ) + if len(missing_keys) > 0: + logger.warning( + f'Some weights of {self.__class__.__name__} were not initialized from the model checkpoint' + f' and are newly initialized: {missing_keys}\nYou should probably' + ' TRAIN this model on a down-stream task to be able to use it for predictions and inference.' + ) + elif len(mismatched_keys) == 0: + logger.info( + f'All the weights of {self.__class__.__name__} were initialized from the model checkpoint ' + f'If your task is similar to the task the model of the checkpoint' + f' was trained on, you can already use {self.__class__.__name__} for predictions without further' + ' training.') + if len(mismatched_keys) > 0: + mismatched_warning = '\n'.join([ + f'- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated' + for key, shape1, shape2 in mismatched_keys + ]) + logger.warning( + f'Some weights of {self.__class__.__name__} were not initialized from the model checkpoint' + f' and are newly initialized because the shapes did not' + f' match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be able' + ' to use it for predictions and inference.') + + return missing_keys, unexpected_keys, mismatched_keys, error_msgs + + def retrieve_modules_from_names(self, + names, + prefix=None, + add_prefix=False, + remove_prefix=False): + module_keys = set(['.'.join(key.split('.')[:-1]) for key in names]) + + # torch.nn.ParameterList is a special case where two parameter keywords + # are appended to the module name, *e.g.* bert.special_embeddings.0 + module_keys = module_keys.union( + set([ + '.'.join(key.split('.')[:-2]) for key in names + if key[-1].isdigit() + ])) + + retrieved_modules = [] + # retrieve all modules that has at least one missing weight name + for name, module in self.named_modules(): + if remove_prefix: + name = '.'.join( + name.split('.')[1:]) if name.startswith(prefix) else name + elif add_prefix: + name = '.'.join([prefix, name]) if len(name) > 0 else prefix + + if name in module_keys: + retrieved_modules.append(module) + + return retrieved_modules + + +class SingleBackboneTaskModelBase(BaseTaskModel): + """ + This is the base class of any single backbone nlp task classes. + """ + # The backbone prefix defaults to "bert" + _backbone_prefix = 'bert' + + # The head prefix defaults to "head" + _head_prefix = 'head' + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.backbone_cfg = self.config.get('backbone', None) + assert self.backbone_cfg is not None + self.head_cfg = self.config.get('head', None) + + def build_backbone(self, cfg): + if 'prefix' in cfg: + self._backbone_prefix = cfg['prefix'] + backbone = build_backbone(cfg) + setattr(self, cfg['prefix'], backbone) + + def build_head(self, cfg): + if cfg is None: + raise ValueError( + 'Head config is missing, check if this was a backbone-only model' + ) + if 'prefix' in cfg: + self._head_prefix = cfg['prefix'] + head = build_head(cfg, task_name=self.group_key) + setattr(self, self._head_prefix, head) + return head + + @property + def backbone(self): + if 'backbone' != self._backbone_prefix: + return getattr(self, self._backbone_prefix) + return super().__getattr__('backbone') + + @property + def head(self): + if 'head' != self._head_prefix: + return getattr(self, self._head_prefix) + return super().__getattr__('head') + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + """default forward method is the backbone-only forward""" + if func_receive_dict_inputs(self.backbone.forward): + outputs = self.backbone.forward(input) + else: + outputs = self.backbone.forward(**input) + return outputs + + def compute_loss(self, outputs, labels): + loss = self.head.compute_loss(outputs, labels) + return loss + + def extract_backbone_outputs(self, outputs): + sequence_output = None + pooled_output = None + if hasattr(self.backbone, 'extract_sequence_outputs'): + sequence_output = self.backbone.extract_sequence_outputs(outputs) + if hasattr(self.backbone, 'extract_pooled_outputs'): + pooled_output = self.backbone.extract_pooled_outputs(outputs) + return sequence_output, pooled_output + + +class EncoderModel(TorchModel): + """ + This is the base class of any encoder nlp task classes. + """ + # keys to ignore when load missing + _keys_to_ignore_on_load_missing = None + # keys to ignore when load unexpected + _keys_to_ignore_on_load_unexpected = None + # The encoder prefix defaults to "encoder" + base_model_prefix = 'encoder' + # The default backbone model type is None, should be bert/T5 + base_model_type = None + # The head prefix defaults to "head" + head_prefix = 'head' + # The head type defaults as None + head_type = None + # override base model prefix by task model prefix + override_base_model_prefix = False + # override base model type by task model type + override_base_model_type = False + + def __init__(self, model_dir: str, *args, **kwargs): + super().__init__(model_dir, *args, **kwargs) + self.config = ConfigDict(kwargs) + backbone_cfg = self.parse_encoder_cfg() + head_cfg = self.parse_head_cfg() + self.build_encoder(backbone_cfg) + if head_cfg.type is not None: + self.build_head(head_cfg) + self.post_init() + + def post_init(self): + try: + head_keys_to_ignore_on_load_missing = getattr( + self.head, '_keys_to_ignore_on_load_missing') + for i in head_keys_to_ignore_on_load_missing: + self._keys_to_ignore_on_load_missing.append('head.' + i) + except Exception: + logger.info('head has no _keys_to_ignore_on_load_missing') + + def __repr__(self): + # only log backbone and head name + depth = 1 + return _repr(self, depth) + + def _get_transformer_config(self): + transformer_config_file = os.path.join(self.model_dir, + ModelFile.CONFIG) + transformer_config = None + if os.path.exists(transformer_config_file): + transformer_config = Config.from_file(transformer_config_file) + return transformer_config.copy() + + def _use_transformer_config(self, cfg): + if 'model_type' not in cfg and 'type' not in cfg: + return True + else: + return False + + def parse_encoder_cfg(self): + # get encoder from backbone-head configuration format + encoder_cfg = self.config.get('backbone', None) + if encoder_cfg is None: + encoder_cfg = self.config.copy() + if 'model_type' in encoder_cfg and 'type' not in encoder_cfg: + encoder_cfg.type = encoder_cfg.model_type + elif self._use_transformer_config(encoder_cfg): + encoder_cfg = self._get_transformer_config() + encoder_cfg.type = encoder_cfg.model_type + + if 'type' not in encoder_cfg or self.override_base_model_type: + encoder_cfg.type = self.base_model_type + if encoder_cfg.type is None: + raise KeyError( + 'Missing encoder type, please explicit define encoder type in configuration.json' + ) + encoder_cfg.model_dir = self.model_dir + return encoder_cfg + + def parse_head_cfg(self): + head_cfg = self.config.get('head', None) + if head_cfg is None: + head_cfg = self.config.copy() + if 'head_type' in head_cfg and 'type' not in head_cfg: + head_cfg.type = head_cfg.head_type + elif self._use_transformer_config(head_cfg): + head_cfg = self._get_transformer_config() + head_cfg.type = self.head_type + if 'type' not in head_cfg: + head_cfg.type = self.head_type + return head_cfg + + def build_encoder(self, cfg): + backbone = build_backbone(cfg) + if 'prefix' in cfg: + self.base_model_prefix = cfg['prefix'] + elif 'base_model_prefix' in cfg: + self.base_model_prefix = cfg['base_model_prefix'] + elif hasattr(backbone, 'base_model_prefix') \ + and not self.override_base_model_prefix: + self.base_model_prefix = backbone.base_model_prefix + setattr(self, self.base_model_prefix, backbone) + + def build_head(self, cfg): + if cfg is None: + raise ValueError( + 'Head config is missing, check if this was a backbone-only model' + ) + head = build_head(cfg, task_name=self.group_key) + setattr(self, self.head_prefix, head) + + @property + def encoder(self): + if 'encoder' != self.base_model_prefix: + return getattr(self, self.base_model_prefix) + return super().__getattr__('encoder') + + @property + def head(self): + if 'head' != self.head_prefix: + return getattr(self, self.head_prefix) + return super().__getattr__('head') + + def extract_feature(self, **input: Dict[str, Any]) -> Dict[str, Any]: + """default forward method is the backbone-only forward""" + if func_receive_dict_inputs(self.encoder.forward): + outputs = self.encoder.forward(input) + else: + outputs = self.encoder.forward(**input) + return outputs + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + *args, + **kwargs): + r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See + :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for + details. + + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, + 1]``: + + - 0 corresponds to a `sentence A` token, + - 1 corresponds to a `sentence B` token. + + position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, + config.max_position_embeddings - 1]``. + + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple. + *args: + In Torch 1.11 onnx has a bug in the _slow_forward method, could only keep *args solving the problem + **kwargs: + Accept additional kwargs in the children class + + Returns: + Returns `modelscope.outputs.ModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') + >>> print(model(**preprocessor(('This is a test', 'This is also a test')))) + """ + + if OutputKeys.LABEL in kwargs and labels is None: + labels = kwargs.pop(OutputKeys.LABEL, None) + feature = self.extract_feature( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + outputs = self.head.forward(feature, attention_mask, labels, **kwargs) + return outputs + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.get('model_dir') + model = cls(**kwargs) + model_load_handler = load_task_model_checkpoint( + model_to_load=model, model_local_dir=model_dir, **kwargs) + return model_load_handler['model'] + + @classmethod + def from_pretrained(cls, + model_name_or_path: str, + revision: Optional[str] = DEFAULT_MODEL_REVISION, + cfg_dict: Config = None, + device: str = None, + **kwargs): + task = kwargs.pop('task', None) + return super(TorchModel, cls).from_pretrained( + model_name_or_path=model_name_or_path, + revision=revision, + cfg_dict=cfg_dict, + devic=device, + task=task if task is not None else cls.task, + **kwargs) diff --git a/runtime/Lib/site_packages/models/nlp/task_models/text_classification.py b/runtime/Lib/site_packages/models/nlp/task_models/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..e7f311a61e9ca46da794059637787ce747277da9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/text_classification.py @@ -0,0 +1,51 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np + +from modelscope.metainfo import Heads, TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import parse_label_mapping + +__all__ = ['ModelForTextClassification'] + + +@MODELS.register_module( + Tasks.text_classification, module_name=TaskModels.text_classification) +class ModelForTextClassification(EncoderModel): + task = Tasks.text_classification + + # The default base head type is text-classification for this head + head_type = Heads.text_classification + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the sequence classification model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + # get the num_labels from label_mapping.json + self.id2label = {} + + # get the num_labels + num_labels = kwargs.get('num_labels') + if num_labels is None: + label2id = parse_label_mapping(model_dir) + if label2id is not None and len(label2id) > 0: + num_labels = len(label2id) + self.id2label = {id: label for label, id in label2id.items()} + kwargs['num_labels'] = num_labels + super().__init__(model_dir, *args, **kwargs) + + def parse_head_cfg(self): + head_cfg = super().parse_head_cfg() + if hasattr(head_cfg, 'classifier_dropout'): + head_cfg['classifier_dropout'] = ( + head_cfg.classifier_dropout if head_cfg['classifier_dropout'] + is not None else head_cfg.hidden_dropout_prob) + else: + head_cfg['classifier_dropout'] = head_cfg.hidden_dropout_prob + head_cfg['num_labels'] = self.config.num_labels + return head_cfg diff --git a/runtime/Lib/site_packages/models/nlp/task_models/text_generation.py b/runtime/Lib/site_packages/models/nlp/task_models/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..85d9066a7fd02c0c0a9703bed475531ef7533bb5 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/text_generation.py @@ -0,0 +1,224 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np +import torch +from transformers.modeling_utils import PreTrainedModel + +from modelscope.metainfo import TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import \ + SingleBackboneTaskModelBase +from modelscope.outputs import (OutputKeys, TextGenerationModelOutput, + TokenGeneratorOutput) +from modelscope.utils.constant import Tasks + +__all__ = ['ModelForTextGeneration'] + + +@MODELS.register_module( + Tasks.text_generation, module_name=TaskModels.text_generation) +class ModelForTextGeneration(SingleBackboneTaskModelBase, PreTrainedModel): + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the text generation model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + super().__init__(model_dir, *args, **kwargs) + if 'base_model_prefix' in kwargs: + self._base_model_prefix = kwargs['base_model_prefix'] + + self.build_backbone(self.backbone_cfg) + self.build_head(self.head_cfg) + if self.config.get('shared_embedding', False): + input_embeddings = self.backbone.get_input_embeddings() + output_embeddings = self.head.get_output_embeddings() + output_embeddings.weight = input_embeddings.weight + + def forward(self, **inputs) -> Dict[str, np.ndarray]: + # backbone do not need labels, only head need for loss compute + labels = inputs.pop(OutputKeys.LABELS, None) + + backbone_outputs = super().forward(inputs) + hidden_states = backbone_outputs[0] + + logits = self.head.forward(hidden_states) + loss = None + if labels is not None: + inputs[OutputKeys.LABELS] = labels + loss = self.compute_loss(logits, labels) + return TextGenerationModelOutput(logits=logits, loss=loss) + + def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs): + # only last token for inputs_ids if past is defined in kwargs + if past: + input_ids = input_ids[:, -1].unsqueeze(-1) + + attention_mask = kwargs.get('attention_mask', None) + position_ids = kwargs.get('position_ids', None) + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past: + position_ids = position_ids[:, -1].unsqueeze(-1) + else: + position_ids = None + return { + 'input_ids': input_ids, + 'past_key_values': past, + 'use_cache': kwargs.get('use_cache'), + 'position_ids': position_ids, + 'attention_mask': attention_mask, + } + + def generate(self, inputs, temperature=1.0, **kwargs): + tokens = inputs['input_ids'] if isinstance(inputs, Dict) else inputs + top_k = kwargs.pop('top_k', + self.config.top_k if 'top_k' in self.config else 1) + top_p = kwargs.pop('top_p', + self.config.top_p if 'top_p' in self.config else 0.) + max_length = kwargs.pop('max_length', self.config.max_length) + + batch_size = tokens.size(0) + lengths = kwargs.pop( + 'prompt_length', + torch.tensor([tokens.size(1)], device=tokens.device)) + + min_prompt_length = lengths.min().item() + max_sequence_length = max_length + + # If the context is too big, this happens + if min_prompt_length >= max_sequence_length: + raise ValueError('context length too large') + + pad_length = max_sequence_length - tokens.size(1) + if pad_length > 0: + pads = torch.zeros( + batch_size, pad_length, device=tokens.device).long() + tokens = torch.cat((tokens, pads), dim=-1) + + # Added termination_id to support the case that we want to terminate the + # generation once that id is generated. + termination_id = self.config.eos_token_id + + # Whether we have reached a termination id. + is_generation_done = torch.zeros( + batch_size, dtype=torch.uint8, device=tokens.device) + + with torch.no_grad(): + for context_length in range(min_prompt_length, + max_sequence_length): + + # Pick the slice that we need to pass through the network. + tokens2use = tokens[:, :context_length] + + # logits will be meanigful only in the last pipeline stage. + logits = self(input_ids=tokens2use).logits + + # Sample. + last_token_logits = logits[:, -1, :] + new_sample = sample( + last_token_logits, + top_k=top_k, + top_p=top_p, + temperature=temperature, + vocab_size=self.head_cfg.vocab_size) + + # If a prompt length is smaller or equal th current context + # length, it means we have started generating tokens + started = lengths <= context_length + # Update the tokens. + tokens[started, context_length] = new_sample[started] + + done_token = (new_sample == termination_id).byte() & \ + started.byte() + + is_generation_done = is_generation_done | done_token + done = torch.all(is_generation_done) + + if done: + break + + tokens = tokens[:, :(context_length + 1)] + return TokenGeneratorOutput(sequences=tokens) + + +def sample(logits, top_k=0, top_p=0.0, temperature=1.0, vocab_size=None): + """ Sample and generate a token. + Note: logits has the dimension [b, v] where b is the batch size + and v is the vocabulary size. + If vocab_size is provided, we will make sure the sample that is + generated is in [0, vocab-size). This will avoid out of vocabulary + generations due to padding. + """ + + # Check logits for consistency. + assert logits.ndim == 2, 'expected the logits to be of [b, v] shape.' + + # Greedy is just simple argmax. + if top_k == 1: + assert top_p == 0.0, 'cannot set both greedy and top-p samplings.' + samples = torch.argmax(logits, dim=-1) + + # Top-k or top-p sampling. + else: + # Clone so we do not modify the inputs, + logits = logits.clone() + # Apply temperature in place. + if temperature != 1.0: + logits.div_(temperature) + + if top_k > 1: + top_p == 0.0 + assert top_k <= logits.size(1), 'top-k is larger than logit size.' + if vocab_size: + assert top_k < vocab_size, 'top-k is larger than vocab size.' + modify_logits_for_top_k_filtering(logits, top_k) + + elif top_p > 0.0: + assert top_p <= 1.0, 'top-p should be in (0, 1].' + modify_logits_for_top_p_filtering(logits, top_p) + + # After filtering, we need to recalculate the distribution. + probs = logits.softmax(dim=-1) + samples = torch.multinomial(probs, num_samples=1).view(-1) + + # If vocab size is provided, make sure the samples are in + # in the range [0, vocab-size). + if vocab_size: + samples = torch.clamp(samples, min=0, max=(vocab_size - 1)) + + return samples + + +def modify_logits_for_top_k_filtering(logits, top_k): + """Set the logits for none top-k values to -inf.""" + + filter_ = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits.masked_fill_(filter_, float('-Inf')) + + +def modify_logits_for_top_p_filtering(logits, top_p): + """Set the logits for none top-p values to -inf.""" + + # First sort and calculate cumulative sum of probabilities. + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Filteration based on the cumulative sum. + filter_ = cumulative_probs > top_p + # This shift by 1 is weird and I cannot justify it. This existed + # in the original implementation: + # https://github.com/ari-holtzman/degen/blob/master/gen.py + # and I guess it is needed so keeping it for now. + filter_[:, 1:] = filter_[:, :-1].clone() + # Make sure we at least have one token to select from. + filter_[..., 0] = 0 + + # Fill in the filtered part + filter_ = filter_.scatter(1, sorted_indices, filter_) + logits.masked_fill_(filter_, float('-Inf')) diff --git a/runtime/Lib/site_packages/models/nlp/task_models/text_ranking.py b/runtime/Lib/site_packages/models/nlp/task_models/text_ranking.py new file mode 100644 index 0000000000000000000000000000000000000000..e883d76cdf7f38c0a11d3d8767c417f81815f011 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/text_ranking.py @@ -0,0 +1,58 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import numpy as np + +from modelscope.metainfo import Heads, TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import parse_label_mapping + +__all__ = ['ModelForTextRanking'] + + +@MODELS.register_module( + Tasks.text_ranking, module_name=TaskModels.text_ranking) +class ModelForTextRanking(EncoderModel): + task = Tasks.text_ranking + + # The default base head type is text-ranking for this head + head_type = Heads.text_ranking + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the sequence classification model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + # get the num_labels from label_mapping.json + self.id2label = {} + + # get the num_labels + num_labels = kwargs.get('num_labels') + if num_labels is None: + label2id = parse_label_mapping(model_dir) + if label2id is not None and len(label2id) > 0: + num_labels = len(label2id) + self.id2label = {id: label for label, id in label2id.items()} + elif label2id is None: + num_labels = 1 + kwargs['num_labels'] = num_labels + super().__init__(model_dir, *args, **kwargs) + + def parse_encoder_cfg(self): + encoder_cfg = super().parse_encoder_cfg() + encoder_cfg['add_pooling_layer'] = True + return encoder_cfg + + def parse_head_cfg(self): + head_cfg = super().parse_head_cfg() + if hasattr(head_cfg, 'classifier_dropout'): + head_cfg['classifier_dropout'] = ( + head_cfg.classifier_dropout if head_cfg['classifier_dropout'] + is not None else head_cfg.hidden_dropout_prob) + else: + head_cfg['classifier_dropout'] = head_cfg.hidden_dropout_prob + head_cfg['num_labels'] = self.config.num_labels + return head_cfg diff --git a/runtime/Lib/site_packages/models/nlp/task_models/token_classification.py b/runtime/Lib/site_packages/models/nlp/task_models/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5142b9681d0d0fb85d89435dab51d0c3c442d5 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/task_models/token_classification.py @@ -0,0 +1,129 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import torch + +from modelscope.metainfo import Heads, Models, TaskModels +from modelscope.models.builder import MODELS +from modelscope.models.nlp.task_models.task_model import EncoderModel +from modelscope.outputs import (AttentionTokenClassificationModelOutput, + OutputKeys) +from modelscope.utils.constant import Tasks +from modelscope.utils.hub import parse_label_mapping + +__all__ = ['ModelForTokenClassification', 'ModelForTokenClassificationWithCRF'] + + +@MODELS.register_module( + Tasks.token_classification, module_name=TaskModels.token_classification) +@MODELS.register_module( + Tasks.part_of_speech, module_name=TaskModels.token_classification) +@MODELS.register_module( + Tasks.named_entity_recognition, + module_name=Models.token_classification_for_ner) +class ModelForTokenClassification(EncoderModel): + task = Tasks.token_classification + + # The default base head type is token-classification for this head + head_type = Heads.token_classification + + # The default base model prefix for this task is encoder and ignore the base model prefix + base_model_prefix = 'encoder' + override_base_model_prefix = True + + def __init__(self, model_dir: str, *args, **kwargs): + """initialize the sequence classification model from the `model_dir` path. + + Args: + model_dir (str): the model path. + """ + # get the num_labels from label_mapping.json + self.id2label = {} + + # get the num_labels + num_labels = kwargs.get('num_labels') + if num_labels is None: + label2id = parse_label_mapping(model_dir) + if label2id is not None and len(label2id) > 0: + num_labels = len(label2id) + self.id2label = {id: label for label, id in label2id.items()} + kwargs['num_labels'] = num_labels + super().__init__(model_dir, *args, **kwargs) + + def parse_head_cfg(self): + head_cfg = super().parse_head_cfg() + if hasattr(head_cfg, 'classifier_dropout'): + head_cfg['classifier_dropout'] = ( + head_cfg.classifier_dropout if head_cfg['classifier_dropout'] + is not None else head_cfg.hidden_dropout_prob) + else: + head_cfg['classifier_dropout'] = head_cfg.hidden_dropout_prob + head_cfg['num_labels'] = self.config.num_labels + return head_cfg + + def forward(self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + offset_mapping=None, + label_mask=None, + **kwargs): + kwargs['offset_mapping'] = offset_mapping + kwargs['label_mask'] = label_mask + outputs = super().forward(input_ids, attention_mask, token_type_ids, + position_ids, head_mask, inputs_embeds, + labels, output_attentions, + output_hidden_states, **kwargs) + + outputs.offset_mapping = offset_mapping + outputs.label_mask = label_mask + + return outputs + + +@MODELS.register_module(Tasks.transformer_crf, module_name=Models.tcrf) +@MODELS.register_module(Tasks.token_classification, module_name=Models.tcrf) +@MODELS.register_module( + Tasks.token_classification, module_name=Models.tcrf_wseg) +@MODELS.register_module( + Tasks.named_entity_recognition, module_name=Models.tcrf) +@MODELS.register_module(Tasks.part_of_speech, module_name=Models.tcrf) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.tcrf) +@MODELS.register_module(Tasks.word_segmentation, module_name=Models.tcrf_wseg) +class ModelForTokenClassificationWithCRF(ModelForTokenClassification): + head_type = Heads.transformer_crf + base_model_prefix = 'encoder' + + def postprocess(self, inputs, **kwargs): + logits = inputs['logits'] + predicts = self.head.decode(inputs['logits'], inputs['label_mask']) + offset_mapping = inputs['offset_mapping'] + mask = inputs['label_mask'] + + # revert predicts to original position with respect of label mask + masked_predict = torch.zeros_like(predicts) + for i in range(len(mask)): + masked_lengths = mask[i].sum(-1).long().cpu().item() + selected_predicts = torch.narrow( + predicts[i], 0, 0, + masked_lengths) # index_select only move loc, not resize + mask_position = mask[i].bool() + masked_predict[i][mask_position] = selected_predicts + predicts = masked_predict + + return AttentionTokenClassificationModelOutput( + loss=None, + logits=logits, + hidden_states=None, + attentions=None, + label_mask=mask, + offset_mapping=offset_mapping, + predictions=predicts, + ) diff --git a/runtime/Lib/site_packages/models/nlp/unite/__init__.py b/runtime/Lib/site_packages/models/nlp/unite/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..939f0ab739668cde6304101d34742556db11f64c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/unite/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import UniTEConfig + from .translation_evaluation import UniTEForTranslationEvaluation +else: + _import_structure = { + 'configuration': ['UniTEConfig'], + 'translation_evaluation': ['UniTEForTranslationEvaluation'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/unite/configuration.py b/runtime/Lib/site_packages/models/nlp/unite/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..402538f7a36bcd35d82504a0a3cd9c5582019aa7 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/unite/configuration.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""UniTE model configuration""" + +from enum import Enum + +from modelscope.utils import logger as logging +from modelscope.utils.config import Config + +logger = logging.get_logger() + + +class InputFormat(Enum): + SRC = 'src' + REF = 'ref' + SRC_REF = 'src-ref' + + +class UniTEConfig(Config): + + def __init__(self, **kwargs): + super().__init__(**kwargs) diff --git a/runtime/Lib/site_packages/models/nlp/unite/translation_evaluation.py b/runtime/Lib/site_packages/models/nlp/unite/translation_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..6fa3264ce2423c3027e50fb3f9a74288492da3d9 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/unite/translation_evaluation.py @@ -0,0 +1,430 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""PyTorch UniTE model.""" + +import warnings +from dataclasses import dataclass +from math import ceil +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.utils.checkpoint +from packaging import version +from torch.nn import (Dropout, Linear, Module, Parameter, ParameterList, + Sequential) +from torch.nn.functional import softmax +from torch.nn.utils.rnn import pad_sequence +from transformers import XLMRobertaConfig, XLMRobertaModel +from transformers.activations import ACT2FN + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.nlp.unite.configuration import InputFormat +from modelscope.outputs.nlp_outputs import TranslationEvaluationOutput +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['UniTEForTranslationEvaluation'] + + +def _layer_norm_all(tensor, mask_float): + broadcast_mask = mask_float.unsqueeze(dim=-1) + num_elements_not_masked = broadcast_mask.sum() * tensor.size(-1) + tensor_masked = tensor * broadcast_mask + + mean = tensor_masked.sum([-1, -2, -3], + keepdim=True) / num_elements_not_masked + variance = (((tensor_masked - mean) * broadcast_mask)**2).sum( + [-1, -2, -3], keepdim=True) / num_elements_not_masked + + return (tensor - mean) / torch.sqrt(variance + 1e-12) + + +class LayerwiseAttention(Module): + + def __init__( + self, + num_layers: int, + model_dim: int, + dropout: float = None, + ) -> None: + super(LayerwiseAttention, self).__init__() + self.num_layers = num_layers + self.model_dim = model_dim + self.dropout = dropout + + self.scalar_parameters = Parameter( + torch.zeros((num_layers, ), requires_grad=True)) + self.gamma = Parameter(torch.FloatTensor([1.0]), requires_grad=True) + + if self.dropout: + dropout_mask = torch.zeros(len(self.scalar_parameters)) + dropout_fill = torch.empty(len( + self.scalar_parameters)).fill_(-1e20) + self.register_buffer('dropout_mask', dropout_mask) + self.register_buffer('dropout_fill', dropout_fill) + + def forward( + self, + tensors: List[torch.Tensor], # pylint: disable=arguments-differ + mask: torch.Tensor = None, + ) -> torch.Tensor: + tensors = torch.cat(list(x.unsqueeze(dim=0) for x in tensors), dim=0) + + if self.training and self.dropout: + normed_weights = softmax( + torch.where(self.dropout_mask.uniform_() > self.dropout, + self.scalar_parameters, self.dropout_fill), + dim=-1) + else: + normed_weights = softmax(self.scalar_parameters, dim=-1) + + normed_weights = normed_weights.view(-1, 1, 1, 1) + + mask_float = mask.float() + weighted_sum = (normed_weights + * _layer_norm_all(tensors, mask_float)).sum(dim=0) + weighted_sum = weighted_sum[:, 0, :] + + return self.gamma * weighted_sum + + +class FeedForward(Module): + + def __init__( + self, + in_dim: int, + out_dim: int = 1, + hidden_sizes: List[int] = [3072, 768], + activations: str = 'Sigmoid', + final_activation: Optional[str] = None, + dropout: float = 0.1, + ) -> None: + """ + Feed Forward Neural Network. + + Args: + in_dim (:obj:`int`): + Number of input features. + out_dim (:obj:`int`, defaults to 1): + Number of output features. Default is 1 -- a single scalar. + hidden_sizes (:obj:`List[int]`, defaults to `[3072, 768]`): + List with hidden layer sizes. + activations (:obj:`str`, defaults to `Sigmoid`): + Name of the activation function to be used in the hidden layers. + final_activation (:obj:`str`, Optional, defaults to `None`): + Name of the final activation function if any. + dropout (:obj:`float`, defaults to 0.1): + Dropout ratio to be used in the hidden layers. + """ + super().__init__() + modules = [] + modules.append(Linear(in_dim, hidden_sizes[0])) + modules.append(self.build_activation(activations)) + modules.append(Dropout(dropout)) + + for i in range(1, len(hidden_sizes)): + modules.append(Linear(hidden_sizes[i - 1], hidden_sizes[i])) + modules.append(self.build_activation(activations)) + modules.append(Dropout(dropout)) + + modules.append(Linear(hidden_sizes[-1], int(out_dim))) + if final_activation is not None: + modules.append(self.build_activation(final_activation)) + + self.ff = Sequential(*modules) + + def build_activation(self, activation: str) -> Module: + return ACT2FN[activation] + + def forward(self, in_features: torch.Tensor) -> torch.Tensor: + return self.ff(in_features) + + +@MODELS.register_module(Tasks.translation_evaluation, module_name=Models.unite) +class UniTEForTranslationEvaluation(TorchModel): + + def __init__(self, + attention_probs_dropout_prob: float = 0.1, + bos_token_id: int = 0, + eos_token_id: int = 2, + pad_token_id: int = 1, + hidden_act: str = 'gelu', + hidden_dropout_prob: float = 0.1, + hidden_size: int = 1024, + initializer_range: float = 0.02, + intermediate_size: int = 4096, + layer_norm_eps: float = 1e-05, + max_position_embeddings: int = 512, + num_attention_heads: int = 16, + num_hidden_layers: int = 24, + type_vocab_size: int = 1, + use_cache: bool = True, + vocab_size: int = 250002, + mlp_hidden_sizes: List[int] = [3072, 1024], + mlp_act: str = 'tanh', + mlp_final_act: Optional[str] = None, + mlp_dropout: float = 0.1, + **kwargs): + r"""The UniTE Model which outputs the scalar to describe the corresponding + translation quality of hypothesis. The model architecture includes two + modules: a pre-trained language model (PLM) to derive representations, + and a multi-layer perceptron (MLP) to give predicted score. + + Args: + attention_probs_dropout_prob (:obj:`float`, defaults to 0.1): + The dropout ratio for attention weights inside PLM. + bos_token_id (:obj:`int`, defaults to 0): + The numeric id representing beginning-of-sentence symbol. + eos_token_id (:obj:`int`, defaults to 2): + The numeric id representing ending-of-sentence symbol. + pad_token_id (:obj:`int`, defaults to 1): + The numeric id representing padding symbol. + hidden_act (:obj:`str`, defaults to :obj:`"gelu"`): + Activation inside PLM. + hidden_dropout_prob (:obj:`float`, defaults to 0.1): + The dropout ratio for activation states inside PLM. + hidden_size (:obj:`int`, defaults to 1024): + The dimensionality of PLM. + initializer_range (:obj:`float`, defaults to 0.02): + The hyper-parameter for initializing PLM. + intermediate_size (:obj:`int`, defaults to 4096): + The dimensionality of PLM inside feed-forward block. + layer_norm_eps (:obj:`float`, defaults to 1e-5): + The value for setting epsilon to avoid zero-division inside + layer normalization. + max_position_embeddings: (:obj:`int`, defaults to 512): + The maximum value for identifying the length of input sequence. + num_attention_heads (:obj:`int`, defaults to 16): + The number of attention heads inside multi-head attention layer. + num_hidden_layers (:obj:`int`, defaults to 24): + The number of layers inside PLM. + type_vocab_size (:obj:`int`, defaults to 1): + The number of type embeddings. + use_cache (:obj:`bool`, defaults to :obj:`True`): + Whether to use cached buffer to initialize PLM. + vocab_size (:obj:`int`, defaults to 250002): + The size of vocabulary. + mlp_hidden_sizes (:obj:`List[int]`, defaults to `[3072, 1024]`): + The size of hidden states inside MLP. + mlp_act (:obj:`str`, defaults to :obj:`"tanh"`): + Activation inside MLP. + mlp_final_act (:obj:`str`, `optional`, defaults to :obj:`None`): + Activation at the end of MLP. + mlp_dropout (:obj:`float`, defaults to 0.1): + The dropout ratio for MLP. + """ + super().__init__(**kwargs) + + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.hidden_size = hidden_size + self.initializer_range = initializer_range + self.intermediate_size = intermediate_size + self.layer_norm_eps = layer_norm_eps + self.max_position_embeddings = max_position_embeddings + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.type_vocab_size = type_vocab_size + self.use_cache = use_cache + self.vocab_size = vocab_size + self.mlp_hidden_sizes = mlp_hidden_sizes + self.mlp_act = mlp_act + self.mlp_final_act = mlp_final_act + self.mlp_dropout = mlp_dropout + + self.encoder_config = XLMRobertaConfig( + bos_token_id=self.bos_token_id, + eos_token_id=self.eos_token_id, + pad_token_id=self.pad_token_id, + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range, + layer_norm_eps=self.layer_norm_eps, + use_cache=self.use_cache) + + self.encoder = XLMRobertaModel( + self.encoder_config, add_pooling_layer=False) + + self.layerwise_attention = LayerwiseAttention( + num_layers=self.num_hidden_layers + 1, + model_dim=self.hidden_size, + dropout=self.mlp_dropout) + + self.estimator = FeedForward( + in_dim=self.hidden_size, + out_dim=1, + hidden_sizes=self.mlp_hidden_sizes, + activations=self.mlp_act, + final_activation=self.mlp_final_act, + dropout=self.mlp_dropout) + + return + + def forward(self, + input_ids: torch.Tensor, + input_format: Optional[List[InputFormat]] = None, + score: Optional[torch.Tensor] = None, + **kwargs) -> TranslationEvaluationOutput: + attention_mask = input_ids.ne(self.pad_token_id).long() + outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + return_dict=True) + mix_states = self.layerwise_attention(outputs['hidden_states'], + attention_mask) + pred = self.estimator(mix_states).squeeze(dim=-1) + output = TranslationEvaluationOutput( + score=pred.cpu().tolist(), input_format=input_format) + + if score is not None: + loss = (pred - score).pow(2).mean() + output['loss'] = loss + + return output + + def load_checkpoint(self, path: str, device: torch.device, plm_only: bool): + if plm_only: + self.encoder = self.encoder.from_pretrained(path).to(device) + self.encoder.pooler = None + else: + state_dict = torch.load(path, map_location=device) + compatible_position_ids(state_dict, + 'encoder.embeddings.position_ids') + self.load_state_dict(state_dict) + logger.info('Loading checkpoint parameters from %s' % path) + return + + +def combine_input_sentences(all_input_concat: List[List[torch.Tensor]], + maximum_length: int = 512, + pad_idx: int = 1, + eos_idx: int = 2): + for group in all_input_concat[1:]: + group[:, 0] = eos_idx + + if len(all_input_concat) == 3: + return cut_long_sequences3(all_input_concat, maximum_length, pad_idx) + else: + return cut_long_sequences2(all_input_concat, maximum_length, pad_idx) + + +def cut_long_sequences2(all_input_concat: List[List[torch.Tensor]], + maximum_length: int = 512, + pad_idx: int = 1): + all_input_concat = list(zip(*all_input_concat)) + collected_tuples = list() + for tensor_tuple in all_input_concat: + tensor_tuple = tuple( + x.masked_select(x.ne(pad_idx)) for x in tensor_tuple) + all_lens = tuple(len(x) for x in tensor_tuple) + + if sum(all_lens) > maximum_length: + lengths = dict(enumerate(all_lens)) + lengths_sorted_idxes = list(x[0] for x in sorted( + lengths.items(), key=lambda d: d[1], reverse=True)) + + offset = ceil((sum(lengths.values()) - maximum_length) / 2) + + if min(all_lens) > (maximum_length + // 2) and min(all_lens) > offset: + lengths = dict((k, v - offset) for k, v in lengths.items()) + else: + lengths[lengths_sorted_idxes[0]] = maximum_length - lengths[ + lengths_sorted_idxes[1]] + + new_lens = list(lengths[k] for k in range(0, len(tensor_tuple))) + new_tensor_tuple = tuple(x[:y] + for x, y in zip(tensor_tuple, new_lens)) + for x, y in zip(new_tensor_tuple, tensor_tuple): + x[-1] = y[-1] + collected_tuples.append(new_tensor_tuple) + else: + collected_tuples.append(tensor_tuple) + + concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) + all_input_concat_padded = pad_sequence( + concat_tensor, batch_first=True, padding_value=pad_idx) + return all_input_concat_padded + + +def cut_long_sequences3(all_input_concat: List[List[torch.Tensor]], + maximum_length: int = 512, + pad_idx: int = 1): + all_input_concat = list(zip(*all_input_concat)) + collected_tuples = list() + for tensor_tuple in all_input_concat: + tensor_tuple = tuple( + x.masked_select(x.ne(pad_idx)) for x in tensor_tuple) + all_lens = tuple(len(x) for x in tensor_tuple) + + if sum(all_lens) > maximum_length: + lengths = dict(enumerate(all_lens)) + lengths_sorted_idxes = list(x[0] for x in sorted( + lengths.items(), key=lambda d: d[1], reverse=True)) + + offset = ceil((sum(lengths.values()) - maximum_length) / 3) + + if min(all_lens) > (maximum_length + // 3) and min(all_lens) > offset: + lengths = dict((k, v - offset) for k, v in lengths.items()) + else: + while sum(lengths.values()) > maximum_length: + if lengths[lengths_sorted_idxes[0]] > lengths[ + lengths_sorted_idxes[1]]: + offset = maximum_length - lengths[lengths_sorted_idxes[ + 1]] - lengths[lengths_sorted_idxes[2]] + if offset > lengths[lengths_sorted_idxes[1]]: + lengths[lengths_sorted_idxes[0]] = offset + else: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] + elif lengths[lengths_sorted_idxes[0]] == lengths[ + lengths_sorted_idxes[1]] > lengths[ + lengths_sorted_idxes[2]]: + offset = (maximum_length + - lengths[lengths_sorted_idxes[2]]) // 2 + if offset > lengths[lengths_sorted_idxes[2]]: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] = offset + else: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] = lengths[ + lengths_sorted_idxes[2]] + else: + lengths[lengths_sorted_idxes[0]] = lengths[ + lengths_sorted_idxes[1]] = lengths[ + lengths_sorted_idxes[2]] = maximum_length // 3 + + new_lens = list(lengths[k] for k in range(0, len(lengths))) + new_tensor_tuple = tuple(x[:y] + for x, y in zip(tensor_tuple, new_lens)) + + for x, y in zip(new_tensor_tuple, tensor_tuple): + x[-1] = y[-1] + collected_tuples.append(new_tensor_tuple) + else: + collected_tuples.append(tensor_tuple) + + concat_tensor = list(torch.cat(x, dim=0) for x in collected_tuples) + all_input_concat_padded = pad_sequence( + concat_tensor, batch_first=True, padding_value=pad_idx) + return all_input_concat_padded diff --git a/runtime/Lib/site_packages/models/nlp/use/__init__.py b/runtime/Lib/site_packages/models/nlp/use/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e4ea59982b03e6042478b3cb169bcc1c5840854 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/use/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .user_satisfaction_estimation import UserSatisfactionEstimation +else: + _import_structure = { + 'user_satisfaction_estimation': ['UserSatisfactionEstimation'] + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/use/transformer.py b/runtime/Lib/site_packages/models/nlp/use/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..d3918903031baa9fcc381c4471554faccd679b4a --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/use/transformer.py @@ -0,0 +1,148 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math + +import torch +import torch.nn as nn + + +def gelu(x): + return 0.5 * x * (1 + torch.tanh( + math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, d_model, d_ff, dropout=0.1): + super(PositionwiseFeedForward, self).__init__() + self.w_1 = nn.Linear(d_model, d_ff) + self.w_2 = nn.Linear(d_ff, d_model) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.actv = gelu + self.dropout_1 = nn.Dropout(dropout) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x): + inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x)))) + output = self.dropout_2(self.w_2(inter)) + return output + x + + +class MultiHeadedAttention(nn.Module): + + def __init__(self, head_count, model_dim, dropout=0.1): + assert model_dim % head_count == 0 + self.dim_per_head = model_dim // head_count + self.model_dim = model_dim + + super(MultiHeadedAttention, self).__init__() + self.head_count = head_count + + self.linear_k = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_v = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_q = nn.Linear(model_dim, head_count * self.dim_per_head) + self.softmax = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + self.linear = nn.Linear(model_dim, model_dim) + + def forward(self, key, value, query, mask=None): + batch_size = key.size(0) + dim_per_head = self.dim_per_head + head_count = self.head_count + + def shape(x): + """ projection """ + return x.view(batch_size, -1, head_count, dim_per_head) \ + .transpose(1, 2) + + def unshape(x): + """ compute context """ + return x.transpose(1, 2).contiguous() \ + .view(batch_size, -1, head_count * dim_per_head) + + key = self.linear_k(key).view(batch_size, -1, head_count, + dim_per_head).transpose(1, 2) + value = self.linear_v(value).view(batch_size, -1, head_count, + dim_per_head).transpose(1, 2) + query = self.linear_q(query).view(batch_size, -1, head_count, + dim_per_head).transpose(1, 2) + + query = query / math.sqrt(dim_per_head) + scores = torch.matmul(query, key.transpose(2, 3)) + + if mask is not None: + mask = mask.unsqueeze(1).expand_as(scores) + scores = scores.masked_fill(mask, -1e10) + + attn = self.softmax(scores) + + drop_attn = self.dropout(attn) + context = torch.matmul(drop_attn, + value).transpose(1, 2).contiguous().view( + batch_size, -1, head_count * dim_per_head) + output = self.linear(context) + return output + + +class PositionalEncoding(nn.Module): + + def __init__(self, dim, max_len=512): + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) + * -(math.log(10000.0) / dim))) + pe[:, 0::2] = torch.sin(position.float() * div_term) + pe[:, 1::2] = torch.cos(position.float() * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + + def forward(self, x): + L = x.size(1) + pos_emb = self.pe[:, :L] + x = x + pos_emb + return x + + +class TransformerEncoderLayer(nn.Module): + + def __init__(self, d_model, heads, d_ff, dropout): + super(TransformerEncoderLayer, self).__init__() + + self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.dropout = nn.Dropout(dropout) + + def forward(self, iter, query, inputs, mask): + if iter != 0: + input_norm = self.layer_norm(inputs) + else: + input_norm = inputs + + mask = mask.unsqueeze(1) + context = self.self_attn(input_norm, input_norm, input_norm, mask=mask) + out = self.dropout(context) + inputs + return self.feed_forward(out) + + +class TransformerEncoder(nn.Module): + + def __init__(self, d_model, d_ff, heads, layers, dropout=0.1): + super(TransformerEncoder, self).__init__() + + self.d_model = d_model + self.layers = layers + self.pos_emb = PositionalEncoding(d_model) + self.transformer_inter = nn.ModuleList([ + TransformerEncoderLayer(d_model, heads, d_ff, dropout) + for _ in range(layers) + ]) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, mask): + x = self.pos_emb(x) + x = self.dropout(x) + for i in range(self.layers): + x = self.transformer_inter[i](i, x, x, mask.eq(0)) + return x diff --git a/runtime/Lib/site_packages/models/nlp/use/user_satisfaction_estimation.py b/runtime/Lib/site_packages/models/nlp/use/user_satisfaction_estimation.py new file mode 100644 index 0000000000000000000000000000000000000000..cb136b8145d144cd65b487b9d4a7bd8ffe9bf72c --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/use/user_satisfaction_estimation.py @@ -0,0 +1,175 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from typing import Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_ +from transformers import BertModel + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.base import Tensor +from modelscope.models.builder import MODELS +from modelscope.outputs import DialogueUserSatisfactionEstimationModelOutput +from modelscope.utils.compatible_with_transformers import \ + compatible_position_ids +from modelscope.utils.constant import ModelFile, Tasks +from .transformer import TransformerEncoder + +__all__ = ['UserSatisfactionEstimation'] + + +@MODELS.register_module(Tasks.text_classification, module_name=Models.use) +class UserSatisfactionEstimation(TorchModel): + + def __init__(self, + model_dir: str, + bert_name: str = None, + device: str = None, + **kwargs): + """initialize the user satisfaction estimation model from the `model_dir` path. The default preprocessor + for this task is DialogueClassificationUsePreprocessor. + + Args: + model_dir: The model dir containing the model. + bert_name: The pretrained model, default bert-base-chinese + device: The device of running model, default cpu + """ + super().__init__(model_dir, **kwargs) + self.model_dir = model_dir + self.bert_name = bert_name if bert_name is not None else 'bert-base-chinese' + self.device = 'cpu' + if device is not None and torch.cuda.is_available(): + self.device = device + self.model = self.init_model() + model_ckpt = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + stats_dict = torch.load(model_ckpt, map_location=torch.device('cpu')) + compatible_position_ids(stats_dict, + 'private.bert.embeddings.position_ids') + self.model.load_state_dict(stats_dict) + + def init_model(self): + configs = { + 'bert_name': self.bert_name, + 'cache_dir': self.model_dir, + 'dropout': 0.1 + } + model = USE(configs) + return model + + def forward( + self, input_ids: Tensor + ) -> Union[DialogueUserSatisfactionEstimationModelOutput, Dict[str, + Tensor]]: + """Compute the logits of satisfaction polarities for a dialogue. + + Args: + input_ids (Tensor): the preprocessed dialogue input + Returns: + output (Dict[str, Any] or DialogueUserSatisfactionEstimationModelOutput): The results of user satisfaction. + + Example: + >>> {'logits': tensor([[-2.1795, 1.1323, 1.8605]])} + """ + logits = self.model(input_ids) + return DialogueUserSatisfactionEstimationModelOutput(logits=logits) + + +def init_params(model): + for name, param in model.named_parameters(): + if param.data.dim() > 1: + xavier_uniform_(param.data) + else: + pass + + +def universal_sentence_embedding(sentences, mask, sqrt=True): + sentence_sums = torch.bmm( + sentences.permute(0, 2, 1), + mask.float().unsqueeze(-1)).squeeze(-1) + divisor = (mask.sum(dim=1).view(-1, 1).float()) + if sqrt: + divisor = divisor.sqrt() + sentence_sums /= divisor + return sentence_sums + + +class BERTBackbone(nn.Module): + + def __init__(self, **config): + super().__init__() + bert_name = config.get('bert_name', 'bert-base-chinese') + cache_dir = config.get('cache_dir') + self.bert = BertModel.from_pretrained(bert_name, cache_dir=cache_dir) + self.d_model = 768 * 2 + + def forward(self, input_ids): + attention_mask = input_ids.ne(0).detach() + outputs = self.bert(input_ids, attention_mask) + h = universal_sentence_embedding(outputs[0], attention_mask) + cls = outputs[1] + out = torch.cat([cls, h], dim=-1) + return out + + +class MLP(nn.Module): + + def __init__(self, input_size, output_size, hidden_size): + super().__init__() + self.fc1 = nn.Linear(input_size, hidden_size) + self.fc2 = nn.Linear(hidden_size, output_size) + + def forward(self, din): + dout = F.relu(self.fc1(din)) + dout = F.relu(self.fc2(dout)) + return dout + + +class USE(nn.Module): + + def __init__(self, args): + super().__init__() + self.drop_out = nn.Dropout(args['dropout']) + self.private = BERTBackbone( + bert_name=args['bert_name'], cache_dir=args['cache_dir']) + d_model = self.private.d_model + self.encoder = TransformerEncoder(d_model, d_model * 2, 8, 2, 0.1) + self.content_gru = nn.GRU( + d_model, + d_model, + num_layers=1, + bidirectional=False, + batch_first=True) + self.sat_classifier = nn.Linear(d_model, 3) + + self.U_c = nn.Linear(d_model, d_model) + self.w_c = nn.Linear(d_model, 1, bias=False) + + init_params(self.encoder) + init_params(self.sat_classifier) + init_params(self.U_c) + init_params(self.w_c) + + def forward(self, input_ids): + self.content_gru.flatten_parameters() + batch_size, dialog_len, utt_len = input_ids.size() + attention_mask = input_ids[:, :, 0].squeeze(-1).ne(0).detach() + input_ids = input_ids.view(-1, utt_len) + + private_out = self.private(input_ids=input_ids) + private_out = private_out.view(batch_size, dialog_len, -1) + H = self.encoder(private_out, attention_mask) + H = self.drop_out(H) + + H, _ = self.content_gru(H) + att_c = self.w_c(torch.tanh(self.U_c(H))).squeeze(-1) + att_c = F.softmax( + att_c.masked_fill(mask=~attention_mask, value=-np.inf), dim=1) + hidden = torch.bmm(H.permute(0, 2, 1), att_c.unsqueeze(-1)).squeeze(-1) + + sat_res = self.sat_classifier(hidden) + return sat_res diff --git a/runtime/Lib/site_packages/models/nlp/veco/__init__.py b/runtime/Lib/site_packages/models/nlp/veco/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f70f3f66256ab66fe69a2996f64d8f796c99ace --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/__init__.py @@ -0,0 +1,43 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import VecoConfig + from .backbone import VecoModel + from .text_classification import VecoForSequenceClassification + from .token_classification import VecoForTokenClassification + from .fill_mask import VecoForMaskedLM +else: + _import_structure = { + 'configuration': ['VecoConfig'], + 'backbone': ['VecoModel'], + 'text_classification': ['VecoForSequenceClassification'], + 'fill_mask': ['VecoForMaskedLM'], + 'token_classification': ['VecoForTokenClassification'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/veco/backbone.py b/runtime/Lib/site_packages/models/nlp/veco/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..93ccead6a3f120326d2540d7e26db5f660021fb4 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/backbone.py @@ -0,0 +1,96 @@ +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Veco model. mainly copied from :module:`~transformers.modeling_xlm_roberta`""" + +from transformers import RobertaModel + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils import logger as logging +from modelscope.utils.constant import Tasks +from .configuration import VecoConfig + +logger = logging.get_logger() + +VECO_PRETRAINED_MODEL_ARCHIVE_LIST = [] + + +@MODELS.register_module(Tasks.backbone, module_name=Models.veco) +class VecoModel(TorchModel, RobertaModel): + """The bare Veco Model transformer outputting raw hidden-states without any specific head on top. + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config ([`VecoConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + This class overrides [`RobertaModel`]. Please check the superclass for the appropriate + documentation alongside usage examples. + """ + + config_class = VecoConfig + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def forward(self, *args, **kwargs): + """ + Returns: + Returns `modelscope.outputs.AttentionBackboneModelOutputWithEmbedding` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_veco_fill-mask-large', task='backbone') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_veco_fill-mask-large') + >>> print(model(**preprocessor('这是个测试'))) + + """ + kwargs['return_dict'] = True + outputs = super(Model, self).forward(*args, **kwargs) + return AttentionBackboneModelOutput( + last_hidden_state=outputs.last_hidden_state, + pooler_output=outputs.pooler_output, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + ponet_config = VecoConfig(**kwargs) + model = cls(ponet_config) + else: + model = super( + Model, + cls).from_pretrained(pretrained_model_name_or_path=model_dir) + return model diff --git a/runtime/Lib/site_packages/models/nlp/veco/configuration.py b/runtime/Lib/site_packages/models/nlp/veco/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..844314a95414f6c403d0bf9bcf783ace49bf004d --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/configuration.py @@ -0,0 +1,33 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# Copyright 2018 The Google AI Language Team Authors. +# Copyright 2020 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Veco configuration, mainly copied from :class:`~transformers.configuration_xlm_roberta` """ + +from transformers import RobertaConfig + +from modelscope.utils import logger as logging + +logger = logging.get_logger() + + +class VecoConfig(RobertaConfig): + """ + This class overrides [`RobertaConfig`]. Please check the superclass for the appropriate + documentation alongside usage examples. + """ + + model_type = 'veco' diff --git a/runtime/Lib/site_packages/models/nlp/veco/fill_mask.py b/runtime/Lib/site_packages/models/nlp/veco/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..fc37f920207f68291bfde895589bf9502182a416 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/fill_mask.py @@ -0,0 +1,99 @@ +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import RobertaForMaskedLM + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionFillMaskModelOutput +from modelscope.utils.constant import Tasks +from .configuration import VecoConfig + + +@MODELS.register_module(Tasks.fill_mask, module_name=Models.veco) +class VecoForMaskedLM(TorchModel, RobertaForMaskedLM): + """Veco Model transformer with a masked language model head on top (a linear layer on top of the + pooled output). + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the fill_mask model of StructBERT, the preprocessor of this model + is `modelscope.preprocessors.FillMaskTransformersPreprocessor`. + + Parameters: + config ([`VecoConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + This class overrides [`RobertaForMaskedLM`]. Please check the superclass for the + appropriate documentation alongside usage examples. + """ + + config_class = VecoConfig + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def forward(self, *args, **kwargs): + """ + Returns: + Returns `modelscope.outputs.AttentionFillMaskModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_veco_fill-mask-large') + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_veco_fill-mask-large') + >>> # Call the model, return some tensors + >>> print(model(**preprocessor('你师父差得动你,你师父可不动我。'))) + >>> # Call the pipeline + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('fill-mask', model=model, preprocessor=preprocessor) + >>> print(pipeline_ins('你师父差得动你,你师父可不动我。')) + """ + + kwargs['return_dict'] = True + outputs = super(Model, self).forward(*args, **kwargs) + return AttentionFillMaskModelOutput( + loss=outputs.loss, + logits=outputs.logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + input_ids=kwargs['input_ids'], + ) + + @classmethod + def _instantiate(cls, **kwargs): + model_dir = kwargs.pop('model_dir', None) + if model_dir is None: + ponet_config = VecoConfig(**kwargs) + model = cls(ponet_config) + else: + model = super( + Model, + cls).from_pretrained(pretrained_model_name_or_path=model_dir) + return model diff --git a/runtime/Lib/site_packages/models/nlp/veco/text_classification.py b/runtime/Lib/site_packages/models/nlp/veco/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..64f3aadda581aac58f8ef8075c14b2c5599b0f63 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/text_classification.py @@ -0,0 +1,136 @@ +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from transformers import RobertaForSequenceClassification + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTextClassificationModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import VecoConfig + + +@MODELS.register_module(Tasks.nli, module_name=Models.veco) +@MODELS.register_module( + Tasks.sentiment_classification, module_name=Models.veco) +@MODELS.register_module(Tasks.sentence_similarity, module_name=Models.veco) +@MODELS.register_module(Tasks.text_classification, module_name=Models.veco) +class VecoForSequenceClassification(TorchModel, + RobertaForSequenceClassification): + """Veco Model transformer with a sequence classification/regression head on top (a linear layer on top of the + pooled output) e.g. for GLUE tasks. + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Preprocessor: + This is the text classification model of Veco, the preprocessor of this model + is `modelscope.preprocessors.TextClassificationTransformersPreprocessor`. + + Trainer: + This model should be trained by dataset which has mixed languages, + and evaluated by datasets of languages one by one. + For example, if the training dataset is xnli (which has sub datasets of multiple languages), then you + should mix the sub-datasets with the languages you want to train to one training dataset, and evaluate + the model one sub-dataset by one sub-dataset of different languages. + This procedure can be done by custom code. If you are using trainer of ModelScope, + the `VecoTrainer` is suggested to use to train this model. This trainer overrides the basic evaluation + loop, and will call the evaluation dataset one by one. Besides, this trainer will use the `VecoTaskDataset` + to mix the input datasets to one, you can check the API Doc for the details. + + To check the complete example please + view the unittest `test_veco_xnli` in `tests.trainers.test_finetune_sequence_classification.py` + + Parameters: + config ([`VecoConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + This class overrides [`RobertaForSequenceClassification`]. Please check the superclass for the + appropriate documentation alongside usage examples. + """ + + config_class = VecoConfig + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def forward(self, *args, **kwargs): + """ + Returns: + Returns `modelscope.outputs.AttentionTextClassificationModelOutput` + + Examples: + >>> from modelscope.models import Model + >>> from modelscope.preprocessors import Preprocessor + >>> model = Model.from_pretrained('damo/nlp_veco_fill-mask-large', + >>> task='text-classification', num_labels=2) + >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_veco_fill-mask-large', + >>> label2id={'0': 0, '1': 1}) + >>> # Call the model, return some tensors + >>> print(model(**preprocessor('这是个测试'))) + >>> # Call the pipeline, the result may be incorrect + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('text-classification', pipeline_name='text-classification', + >>> model=model, preprocessor=preprocessor) + >>> print(pipeline_ins('这是个测试')) + """ + + kwargs['return_dict'] = True + outputs = super(Model, self).forward(*args, **kwargs) + return AttentionTextClassificationModelOutput( + loss=outputs.loss, + logits=outputs.logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + + if model_dir is None: + config = VecoConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + return model diff --git a/runtime/Lib/site_packages/models/nlp/veco/token_classification.py b/runtime/Lib/site_packages/models/nlp/veco/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..4fc96c714b47a5977b73c046e26e0b2819331ff5 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/veco/token_classification.py @@ -0,0 +1,95 @@ +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from transformers import RobertaForTokenClassification + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionTokenClassificationModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import VecoConfig + + +@MODELS.register_module(Tasks.token_classification, module_name=Models.veco) +class VecoForTokenClassification(TorchModel, RobertaForTokenClassification): + """Veco Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. + for Named-Entity-Recognition (NER) tasks. + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic + methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config ([`VecoConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model + weights. + + This class overrides [`RobertaForTokenClassification`]. Please check the superclass for the + appropriate documentation alongside usage examples. + """ + + config_class = VecoConfig + + def __init__(self, config, **kwargs): + super().__init__(config.name_or_path, **kwargs) + super(Model, self).__init__(config) + + def forward(self, *args, **kwargs): + kwargs['return_dict'] = True + outputs = super(Model, self).forward(*args, **kwargs) + + return AttentionTokenClassificationModelOutput( + loss=outputs.loss, + logits=outputs.logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels is not input. + label2id: An optional label2id mapping, which will cover the label2id in configuration (if exists). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + + if model_dir is None: + config = VecoConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + return model diff --git a/runtime/Lib/site_packages/models/nlp/xlm_roberta/__init__.py b/runtime/Lib/site_packages/models/nlp/xlm_roberta/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c012da59d1ff5bcac2a16e2a5e19b2541cd65854 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/xlm_roberta/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2021-2022 The Alibaba DAMO NLP Team Authors. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + from .configuration import XLMRobertaConfig + from .backbone import XLMRobertaModel +else: + _import_structure = { + 'configuration': ['XLMRobertaConfig'], + 'backbone': ['XLMRobertaModel'], + } + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/nlp/xlm_roberta/backbone.py b/runtime/Lib/site_packages/models/nlp/xlm_roberta/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..d1a7cf16184fcc0969522d9407a1b98cc6312007 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/xlm_roberta/backbone.py @@ -0,0 +1,976 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch XLM-RoBERTa model.""" + +import math + +import torch +import torch.utils.checkpoint +from packaging import version +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_utils import (PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer) + +from modelscope.metainfo import Models +from modelscope.models import Model, TorchModel +from modelscope.models.builder import MODELS +from modelscope.outputs import AttentionBackboneModelOutput +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.nlp.utils import parse_labels_in_order +from .configuration import XLMRobertaConfig + +logger = get_logger() + +_CONFIG_FOR_DOC = 'XLMRobertaConfig' + + +# Copied from transformers.models.roberta.modeling_roberta.create_position_ids_from_input_ids +def create_position_ids_from_input_ids(input_ids, + padding_idx, + past_key_values_length=0): + """ + Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols + are ignored. This is modified from fairseq's `utils.make_positions`. + + Args: + x: torch.Tensor x: + + Returns: torch.Tensor + """ + # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA. + mask = input_ids.ne(padding_idx).int() + incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + + past_key_values_length) * mask + return incremental_indices.long() + padding_idx + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaEmbeddings with Roberta->XLMRoberta +class XLMRobertaEmbeddings(nn.Module): + """ + Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. + """ + + # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__ + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, + config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.position_embedding_type = getattr(config, + 'position_embedding_type', + 'absolute') + self.register_buffer( + 'position_ids', + torch.arange(config.max_position_embeddings).expand((1, -1))) + self.register_buffer( + 'token_type_ids', + torch.zeros(self.position_ids.size(), dtype=torch.long), + persistent=False) + + # End copy + self.padding_idx = config.pad_token_id + self.position_embeddings = nn.Embedding( + config.max_position_embeddings, + config.hidden_size, + padding_idx=self.padding_idx) + + def forward(self, + input_ids=None, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + past_key_values_length=0): + if position_ids is None: + if input_ids is not None: + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = create_position_ids_from_input_ids( + input_ids, self.padding_idx, past_key_values_length) + else: + position_ids = self.create_position_ids_from_inputs_embeds( + inputs_embeds) + + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs + # when its auto-generated, registered buffer helps users when tracing the model without + # passing token_type_ids, solves + # issue #5664 + if token_type_ids is None: + if hasattr(self, 'token_type_ids'): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + token_type_embeddings + if self.position_embedding_type == 'absolute': + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + def create_position_ids_from_inputs_embeds(self, inputs_embeds): + """ + We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. + + Args: + inputs_embeds: torch.Tensor + + Returns: torch.Tensor + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange( + self.padding_idx + 1, + sequence_length + self.padding_idx + 1, + dtype=torch.long, + device=inputs_embeds.device) + return position_ids.unsqueeze(0).expand(input_shape) + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaSelfAttention with Roberta->XLMRoberta +class XLMRobertaSelfAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, 'embedding_size'): + raise ValueError( + f'The hidden size ({config.hidden_size}) is not a multiple of the number of attention ' + f'heads ({config.num_attention_heads})') + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size + / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = position_embedding_type or getattr( + config, 'position_embedding_type', 'absolute') + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, + self.attention_head_size) + + self.is_decoder = config.is_decoder + + def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor: + new_x_shape = x.size()[:-1] + (self.num_attention_heads, + self.attention_head_size) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores( + self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores( + self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + use_cache = past_key_value is not None + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, + key_layer.transpose(-1, -2)) + + if self.position_embedding_type == 'relative_key' or self.position_embedding_type == 'relative_key_query': + query_length, key_length = query_layer.shape[2], key_layer.shape[2] + if use_cache: + position_ids_l = torch.tensor( + key_length - 1, + dtype=torch.long, + device=hidden_states.device).view(-1, 1) + else: + position_ids_l = torch.arange( + query_length, + dtype=torch.long, + device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange( + key_length, dtype=torch.long, + device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == 'relative_key': + relative_position_scores = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == 'relative_key_query': + relative_position_scores_query = torch.einsum( + 'bhld,lrd->bhlr', query_layer, positional_embedding) + relative_position_scores_key = torch.einsum( + 'bhrd,lrd->bhlr', key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt( + self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in XLMRobertaModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + ( + self.all_head_size, ) + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, + attention_probs) if output_attentions else (context_layer, ) + + if self.is_decoder: + outputs = outputs + (past_key_value, ) + return outputs + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaSelfOutput +class XLMRobertaSelfOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states: torch.Tensor, + input_tensor: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaAttention with Roberta->XLMRoberta +class XLMRobertaAttention(nn.Module): + + def __init__(self, config, position_embedding_type=None): + super().__init__() + self.self = XLMRobertaSelfAttention( + config, position_embedding_type=position_embedding_type) + self.output = XLMRobertaSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, + self.self.attention_head_size, self.pruned_heads) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len( + heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output, + ) + self_outputs[1:] # add attentions if we output them + return outputs + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaIntermediate +class XLMRobertaIntermediate(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaOutput +class XLMRobertaOutput(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states: torch.Tensor, + input_tensor: torch.Tensor) -> torch.Tensor: + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaLayer with Roberta->XLMRoberta +class XLMRobertaLayer(nn.Module): + + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = XLMRobertaAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError( + f'{self} should be used as a decoder model if cross attention is added' + ) + self.crossattention = XLMRobertaAttention( + config, position_embedding_type='absolute') + self.intermediate = XLMRobertaIntermediate(config) + self.output = XLMRobertaOutput(config) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[: + 2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + # if decoder, the last output is tuple of self-attn cache + if self.is_decoder: + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + else: + outputs = self_attention_outputs[ + 1:] # add self attentions if we output attention weights + + cross_attn_present_key_value = None + if self.is_decoder and encoder_hidden_states is not None: + if not hasattr(self, 'crossattention'): + raise ValueError( + f'If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers' + ' by setting `config.add_cross_attention=True`') + + # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple + cross_attn_past_key_value = past_key_value[ + -2:] if past_key_value is not None else None + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + cross_attn_past_key_value, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[ + 1:-1] # add cross attentions if we output attention weights + + # add cross-attn cache to positions 3,4 of present_key_value tuple + cross_attn_present_key_value = cross_attention_outputs[-1] + present_key_value = present_key_value + cross_attn_present_key_value + + layer_output = apply_chunking_to_forward(self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output) + outputs = (layer_output, ) + outputs + + # if decoder, return the attn key/values as the last output + if self.is_decoder: + outputs = outputs + (present_key_value, ) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaEncoder with Roberta->XLMRoberta +class XLMRobertaEncoder(nn.Module): + + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [XLMRobertaLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = ( + ) if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[ + i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' + ) + use_cache = False + + def create_custom_forward(module): + + def custom_forward(*inputs): + return module(*inputs, past_key_value, + output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1], ) + if output_attentions: + all_self_attentions = all_self_attentions + ( + layer_outputs[1], ) + if self.config.add_cross_attention: + all_cross_attentions = all_cross_attentions + ( + layer_outputs[2], ) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states, ) + + if not return_dict: + return tuple(v for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] if v is not None) + return AttentionBackboneModelOutput( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaPooler +class XLMRobertaPooler(nn.Module): + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +# Copied from transformers.models.roberta.modeling_roberta.RobertaPreTrainedModel with Roberta->XLMRoberta +class XLMRobertaPreTrainedModel(TorchModel, PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = XLMRobertaConfig + base_model_prefix = 'roberta' + supports_gradient_checkpointing = True + _no_split_modules = [] + + # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, XLMRobertaEncoder): + module.gradient_checkpointing = value + + def update_keys_to_ignore(self, config, del_keys_to_ignore): + """Remove some keys from ignore list""" + if not config.tie_word_embeddings: + # must make a new list, or the class variable gets modified! + self._keys_to_ignore_on_save = [ + k for k in self._keys_to_ignore_on_save + if k not in del_keys_to_ignore + ] + self._keys_to_ignore_on_load_missing = [ + k for k in self._keys_to_ignore_on_load_missing + if k not in del_keys_to_ignore + ] + + @classmethod + def _instantiate(cls, **kwargs): + """Instantiate the model. + + Args: + kwargs: Input args. + model_dir: The model dir used to load the checkpoint and the label information. + num_labels: An optional arg to tell the model how many classes to initialize. + Method will call utils.parse_label_mapping if num_labels not supplied. + If num_labels is not found, the model will use the default setting (2 classes). + + Returns: + The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained + """ + + model_dir = kwargs.pop('model_dir', None) + cfg = kwargs.pop('cfg', None) + model_args = parse_labels_in_order(model_dir, cfg, **kwargs) + if model_dir is None: + config = XLMRobertaConfig(**model_args) + model = cls(config) + else: + model = super(Model, cls).from_pretrained( + pretrained_model_name_or_path=model_dir, **model_args) + model.model_dir = model_dir + return model + + +@MODELS.register_module( + group_key=Tasks.backbone, module_name=Models.xlm_roberta) +# Copied from transformers.models.roberta.modeling_roberta.RobertaModel with Roberta->XLMRoberta, ROBERTA->XLM_ROBERTA +class XLMRobertaModel(XLMRobertaPreTrainedModel): + """ + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in *Attention is + all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz + Kaiser and Illia Polosukhin. + + To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set + to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and + `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. + + .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 + + """ + + _keys_to_ignore_on_load_missing = [r'position_ids'] + + # Copied from transformers.models.bert.modeling_bert.BertModel.__init__ with Bert->XLMRoberta + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = XLMRobertaEmbeddings(config) + self.encoder = XLMRobertaEncoder(config) + + self.pooler = XLMRobertaPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + # Copied from transformers.models.bert.modeling_bert.BertModel.forward + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`BertTokenizer`]. See + [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] + for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `((batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask + values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the + inputs. Indices are selected in `[0, 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position + embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, + num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask + values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length, hidden_size)`, + *optional*): + Optionally, instead of passing `input_ids` you can choose to + directly pass an embedded representation. This is useful if you want + more control over how to convert `input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention + layers. See `attentions` under returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See + `hidden_states` under returned tensors for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a + plain tuple. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors + of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else + self.config.output_hidden_states) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + 'You cannot specify both input_ids and inputs_embeds at the same time' + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + 'You have to specify either input_ids or inputs_embeds') + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[ + 2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), + device=device) + + if token_type_ids is None: + if hasattr(self.embeddings, 'token_type_ids'): + buffered_token_type_ids = self.embeddings.token_type_ids[:, : + seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + batch_size, seq_length) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size( + ) + encoder_hidden_shape = (encoder_batch_size, + encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, + self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler( + sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return AttentionBackboneModelOutput( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) diff --git a/runtime/Lib/site_packages/models/nlp/xlm_roberta/configuration.py b/runtime/Lib/site_packages/models/nlp/xlm_roberta/configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..3a245945a76bb47a8895ae4448c4fb17fb607e09 --- /dev/null +++ b/runtime/Lib/site_packages/models/nlp/xlm_roberta/configuration.py @@ -0,0 +1,152 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLM-RoBERTa configuration""" +from collections import OrderedDict +from typing import Mapping + +from transformers.configuration_utils import PretrainedConfig +from transformers.onnx import OnnxConfig + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class XLMRobertaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`XLMRobertaModel`] or a [`TFXLMRobertaModel`]. It + is used to instantiate a XLM-RoBERTa model according to the specified arguments, defining the model architecture. + Instantiating a configuration with the defaults will yield a similar configuration to that of the XLMRoBERTa + [xlm-roberta-base](https://huggingface.co/xlm-roberta-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 30522): + Vocabulary size of the XLM-RoBERTa model. Defines the number of different tokens that can be represented by + the `inputs_ids` passed when calling [`XLMRobertaModel`] or [`TFXLMRobertaModel`]. + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (`int`, *optional*, defaults to 512): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling [`XLMRobertaModel`] or + [`TFXLMRobertaModel`]. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + position_embedding_type (`str`, *optional*, defaults to `"absolute"`): + Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For + positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to + [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). + For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models + with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). + is_decoder (`bool`, *optional*, defaults to `False`): + Whether the model is used as a decoder or not. If `False`, the model is used as an encoder. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + classifier_dropout (`float`, *optional*): + The dropout ratio for the classification head. + + Examples: + + ```python + >>> from transformers import XLMRobertaConfig, XLMRobertaModel + + >>> # Initializing a XLM-RoBERTa xlm-roberta-base style configuration + >>> configuration = XLMRobertaConfig() + + >>> # Initializing a model (with random weights) from the xlm-roberta-base style configuration + >>> model = XLMRobertaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = 'xlm-roberta' + + def __init__(self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + position_embedding_type='absolute', + use_cache=True, + classifier_dropout=None, + **kwargs): + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.position_embedding_type = position_embedding_type + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + + +# Copied from transformers.models.roberta.configuration_roberta.RobertaOnnxConfig with Roberta->XLMRoberta +class XLMRobertaOnnxConfig(OnnxConfig): + + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + if self.task == 'multiple-choice': + dynamic_axis = {0: 'batch', 1: 'choice', 2: 'sequence'} + else: + dynamic_axis = {0: 'batch', 1: 'sequence'} + return OrderedDict([ + ('input_ids', dynamic_axis), + ('attention_mask', dynamic_axis), + ]) diff --git a/runtime/Lib/site_packages/models/science/__init__.py b/runtime/Lib/site_packages/models/science/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..50ab55d73c281bf5231d1f98133c7a628a307526 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import TYPE_CHECKING + +from modelscope.utils.import_utils import LazyImportModule + +if TYPE_CHECKING: + + from .unifold import UnifoldForProteinStructrue + +else: + _import_structure = {'unifold': ['UnifoldForProteinStructrue']} + + import sys + + sys.modules[__name__] = LazyImportModule( + __name__, + globals()['__file__'], + _import_structure, + module_spec=__spec__, + extra_objects={}, + ) diff --git a/runtime/Lib/site_packages/models/science/unifold/__init__.py b/runtime/Lib/site_packages/models/science/unifold/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..75435fed338c68de176526ea0177a17462ced6f3 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/__init__.py @@ -0,0 +1 @@ +from .model import UnifoldForProteinStructrue diff --git a/runtime/Lib/site_packages/models/science/unifold/config.py b/runtime/Lib/site_packages/models/science/unifold/config.py new file mode 100644 index 0000000000000000000000000000000000000000..e760fbf9e4903b7c0835536f17145d0c6c9d469c --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/config.py @@ -0,0 +1,636 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import copy +from typing import Any + +import ml_collections as mlc + +N_RES = 'number of residues' +N_MSA = 'number of MSA sequences' +N_EXTRA_MSA = 'number of extra MSA sequences' +N_TPL = 'number of templates' + +d_pair = mlc.FieldReference(128, field_type=int) +d_msa = mlc.FieldReference(256, field_type=int) +d_template = mlc.FieldReference(64, field_type=int) +d_extra_msa = mlc.FieldReference(64, field_type=int) +d_single = mlc.FieldReference(384, field_type=int) +max_recycling_iters = mlc.FieldReference(3, field_type=int) +chunk_size = mlc.FieldReference(4, field_type=int) +aux_distogram_bins = mlc.FieldReference(64, field_type=int) +eps = mlc.FieldReference(1e-8, field_type=float) +inf = mlc.FieldReference(3e4, field_type=float) +use_templates = mlc.FieldReference(True, field_type=bool) +is_multimer = mlc.FieldReference(False, field_type=bool) + + +def base_config(): + return mlc.ConfigDict({ + 'data': { + 'common': { + 'features': { + 'aatype': [N_RES], + 'all_atom_mask': [N_RES, None], + 'all_atom_positions': [N_RES, None, None], + 'alt_chi_angles': [N_RES, None], + 'atom14_alt_gt_exists': [N_RES, None], + 'atom14_alt_gt_positions': [N_RES, None, None], + 'atom14_atom_exists': [N_RES, None], + 'atom14_atom_is_ambiguous': [N_RES, None], + 'atom14_gt_exists': [N_RES, None], + 'atom14_gt_positions': [N_RES, None, None], + 'atom37_atom_exists': [N_RES, None], + 'frame_mask': [N_RES], + 'true_frame_tensor': [N_RES, None, None], + 'bert_mask': [N_MSA, N_RES], + 'chi_angles_sin_cos': [N_RES, None, None], + 'chi_mask': [N_RES, None], + 'extra_msa_deletion_value': [N_EXTRA_MSA, N_RES], + 'extra_msa_has_deletion': [N_EXTRA_MSA, N_RES], + 'extra_msa': [N_EXTRA_MSA, N_RES], + 'extra_msa_mask': [N_EXTRA_MSA, N_RES], + 'extra_msa_row_mask': [N_EXTRA_MSA], + 'is_distillation': [], + 'msa_feat': [N_MSA, N_RES, None], + 'msa_mask': [N_MSA, N_RES], + 'msa_chains': [N_MSA, None], + 'msa_row_mask': [N_MSA], + 'num_recycling_iters': [], + 'pseudo_beta': [N_RES, None], + 'pseudo_beta_mask': [N_RES], + 'residue_index': [N_RES], + 'residx_atom14_to_atom37': [N_RES, None], + 'residx_atom37_to_atom14': [N_RES, None], + 'resolution': [], + 'rigidgroups_alt_gt_frames': [N_RES, None, None, None], + 'rigidgroups_group_exists': [N_RES, None], + 'rigidgroups_group_is_ambiguous': [N_RES, None], + 'rigidgroups_gt_exists': [N_RES, None], + 'rigidgroups_gt_frames': [N_RES, None, None, None], + 'seq_length': [], + 'seq_mask': [N_RES], + 'target_feat': [N_RES, None], + 'template_aatype': [N_TPL, N_RES], + 'template_all_atom_mask': [N_TPL, N_RES, None], + 'template_all_atom_positions': [N_TPL, N_RES, None, None], + 'template_alt_torsion_angles_sin_cos': [ + N_TPL, + N_RES, + None, + None, + ], + 'template_frame_mask': [N_TPL, N_RES], + 'template_frame_tensor': [N_TPL, N_RES, None, None], + 'template_mask': [N_TPL], + 'template_pseudo_beta': [N_TPL, N_RES, None], + 'template_pseudo_beta_mask': [N_TPL, N_RES], + 'template_sum_probs': [N_TPL, None], + 'template_torsion_angles_mask': [N_TPL, N_RES, None], + 'template_torsion_angles_sin_cos': + [N_TPL, N_RES, None, None], + 'true_msa': [N_MSA, N_RES], + 'use_clamped_fape': [], + 'assembly_num_chains': [1], + 'asym_id': [N_RES], + 'sym_id': [N_RES], + 'entity_id': [N_RES], + 'num_sym': [N_RES], + 'asym_len': [None], + 'cluster_bias_mask': [N_MSA], + }, + 'masked_msa': { + 'profile_prob': 0.1, + 'same_prob': 0.1, + 'uniform_prob': 0.1, + }, + 'block_delete_msa': { + 'msa_fraction_per_block': 0.3, + 'randomize_num_blocks': False, + 'num_blocks': 5, + 'min_num_msa': 16, + }, + 'random_delete_msa': { + 'max_msa_entry': 1 << 25, # := 33554432 + }, + 'v2_feature': + False, + 'gumbel_sample': + False, + 'max_extra_msa': + 1024, + 'msa_cluster_features': + True, + 'reduce_msa_clusters_by_max_templates': + True, + 'resample_msa_in_recycling': + True, + 'template_features': [ + 'template_all_atom_positions', + 'template_sum_probs', + 'template_aatype', + 'template_all_atom_mask', + ], + 'unsupervised_features': [ + 'aatype', + 'residue_index', + 'msa', + 'msa_chains', + 'num_alignments', + 'seq_length', + 'between_segment_residues', + 'deletion_matrix', + 'num_recycling_iters', + 'crop_and_fix_size_seed', + ], + 'recycling_features': [ + 'msa_chains', + 'msa_mask', + 'msa_row_mask', + 'bert_mask', + 'true_msa', + 'msa_feat', + 'extra_msa_deletion_value', + 'extra_msa_has_deletion', + 'extra_msa', + 'extra_msa_mask', + 'extra_msa_row_mask', + 'is_distillation', + ], + 'multimer_features': [ + 'assembly_num_chains', + 'asym_id', + 'sym_id', + 'num_sym', + 'entity_id', + 'asym_len', + 'cluster_bias_mask', + ], + 'use_templates': + use_templates, + 'is_multimer': + is_multimer, + 'use_template_torsion_angles': + use_templates, + 'max_recycling_iters': + max_recycling_iters, + }, + 'supervised': { + 'use_clamped_fape_prob': + 1.0, + 'supervised_features': [ + 'all_atom_mask', + 'all_atom_positions', + 'resolution', + 'use_clamped_fape', + 'is_distillation', + ], + }, + 'predict': { + 'fixed_size': True, + 'subsample_templates': False, + 'block_delete_msa': False, + 'random_delete_msa': True, + 'masked_msa_replace_fraction': 0.15, + 'max_msa_clusters': 128, + 'max_templates': 4, + 'num_ensembles': 2, + 'crop': False, + 'crop_size': None, + 'supervised': False, + 'biased_msa_by_chain': False, + 'share_mask': False, + }, + 'eval': { + 'fixed_size': True, + 'subsample_templates': False, + 'block_delete_msa': False, + 'random_delete_msa': True, + 'masked_msa_replace_fraction': 0.15, + 'max_msa_clusters': 128, + 'max_templates': 4, + 'num_ensembles': 1, + 'crop': False, + 'crop_size': None, + 'spatial_crop_prob': 0.5, + 'ca_ca_threshold': 10.0, + 'supervised': True, + 'biased_msa_by_chain': False, + 'share_mask': False, + }, + 'train': { + 'fixed_size': True, + 'subsample_templates': True, + 'block_delete_msa': True, + 'random_delete_msa': True, + 'masked_msa_replace_fraction': 0.15, + 'max_msa_clusters': 128, + 'max_templates': 4, + 'num_ensembles': 1, + 'crop': True, + 'crop_size': 256, + 'spatial_crop_prob': 0.5, + 'ca_ca_threshold': 10.0, + 'supervised': True, + 'use_clamped_fape_prob': 1.0, + 'max_distillation_msa_clusters': 1000, + 'biased_msa_by_chain': True, + 'share_mask': True, + }, + }, + 'globals': { + 'chunk_size': chunk_size, + 'block_size': None, + 'd_pair': d_pair, + 'd_msa': d_msa, + 'd_template': d_template, + 'd_extra_msa': d_extra_msa, + 'd_single': d_single, + 'eps': eps, + 'inf': inf, + 'max_recycling_iters': max_recycling_iters, + 'alphafold_original_mode': False, + }, + 'model': { + 'is_multimer': is_multimer, + 'input_embedder': { + 'tf_dim': 22, + 'msa_dim': 49, + 'd_pair': d_pair, + 'd_msa': d_msa, + 'relpos_k': 32, + 'max_relative_chain': 2, + }, + 'recycling_embedder': { + 'd_pair': d_pair, + 'd_msa': d_msa, + 'min_bin': 3.25, + 'max_bin': 20.75, + 'num_bins': 15, + 'inf': 1e8, + }, + 'template': { + 'distogram': { + 'min_bin': 3.25, + 'max_bin': 50.75, + 'num_bins': 39, + }, + 'template_angle_embedder': { + 'd_in': 57, + 'd_out': d_msa, + }, + 'template_pair_embedder': { + 'd_in': 88, + 'v2_d_in': [39, 1, 22, 22, 1, 1, 1, 1], + 'd_pair': d_pair, + 'd_out': d_template, + 'v2_feature': False, + }, + 'template_pair_stack': { + 'd_template': d_template, + 'd_hid_tri_att': 16, + 'd_hid_tri_mul': 64, + 'num_blocks': 2, + 'num_heads': 4, + 'pair_transition_n': 2, + 'dropout_rate': 0.25, + 'inf': 1e9, + 'tri_attn_first': True, + }, + 'template_pointwise_attention': { + 'enabled': True, + 'd_template': d_template, + 'd_pair': d_pair, + 'd_hid': 16, + 'num_heads': 4, + 'inf': 1e5, + }, + 'inf': 1e5, + 'eps': 1e-6, + 'enabled': use_templates, + 'embed_angles': use_templates, + }, + 'extra_msa': { + 'extra_msa_embedder': { + 'd_in': 25, + 'd_out': d_extra_msa, + }, + 'extra_msa_stack': { + 'd_msa': d_extra_msa, + 'd_pair': d_pair, + 'd_hid_msa_att': 8, + 'd_hid_opm': 32, + 'd_hid_mul': 128, + 'd_hid_pair_att': 32, + 'num_heads_msa': 8, + 'num_heads_pair': 4, + 'num_blocks': 4, + 'transition_n': 4, + 'msa_dropout': 0.15, + 'pair_dropout': 0.25, + 'inf': 1e9, + 'eps': 1e-10, + 'outer_product_mean_first': False, + }, + 'enabled': True, + }, + 'evoformer_stack': { + 'd_msa': d_msa, + 'd_pair': d_pair, + 'd_hid_msa_att': 32, + 'd_hid_opm': 32, + 'd_hid_mul': 128, + 'd_hid_pair_att': 32, + 'd_single': d_single, + 'num_heads_msa': 8, + 'num_heads_pair': 4, + 'num_blocks': 48, + 'transition_n': 4, + 'msa_dropout': 0.15, + 'pair_dropout': 0.25, + 'inf': 1e9, + 'eps': 1e-10, + 'outer_product_mean_first': False, + }, + 'structure_module': { + 'd_single': d_single, + 'd_pair': d_pair, + 'd_ipa': 16, + 'd_angle': 128, + 'num_heads_ipa': 12, + 'num_qk_points': 4, + 'num_v_points': 8, + 'dropout_rate': 0.1, + 'num_blocks': 8, + 'no_transition_layers': 1, + 'num_resnet_blocks': 2, + 'num_angles': 7, + 'trans_scale_factor': 10, + 'epsilon': 1e-12, + 'inf': 1e5, + 'separate_kv': False, + 'ipa_bias': True, + }, + 'heads': { + 'plddt': { + 'num_bins': 50, + 'd_in': d_single, + 'd_hid': 128, + }, + 'distogram': { + 'd_pair': d_pair, + 'num_bins': aux_distogram_bins, + 'disable_enhance_head': False, + }, + 'pae': { + 'd_pair': d_pair, + 'num_bins': aux_distogram_bins, + 'enabled': False, + 'iptm_weight': 0.8, + 'disable_enhance_head': False, + }, + 'masked_msa': { + 'd_msa': d_msa, + 'd_out': 23, + 'disable_enhance_head': False, + }, + 'experimentally_resolved': { + 'd_single': d_single, + 'd_out': 37, + 'enabled': False, + 'disable_enhance_head': False, + }, + }, + }, + 'loss': { + 'distogram': { + 'min_bin': 2.3125, + 'max_bin': 21.6875, + 'num_bins': 64, + 'eps': 1e-6, + 'weight': 0.3, + }, + 'experimentally_resolved': { + 'eps': 1e-8, + 'min_resolution': 0.1, + 'max_resolution': 3.0, + 'weight': 0.0, + }, + 'fape': { + 'backbone': { + 'clamp_distance': 10.0, + 'clamp_distance_between_chains': 30.0, + 'loss_unit_distance': 10.0, + 'loss_unit_distance_between_chains': 20.0, + 'weight': 0.5, + 'eps': 1e-4, + }, + 'sidechain': { + 'clamp_distance': 10.0, + 'length_scale': 10.0, + 'weight': 0.5, + 'eps': 1e-4, + }, + 'weight': 1.0, + }, + 'plddt': { + 'min_resolution': 0.1, + 'max_resolution': 3.0, + 'cutoff': 15.0, + 'num_bins': 50, + 'eps': 1e-10, + 'weight': 0.01, + }, + 'masked_msa': { + 'eps': 1e-8, + 'weight': 2.0, + }, + 'supervised_chi': { + 'chi_weight': 0.5, + 'angle_norm_weight': 0.01, + 'eps': 1e-6, + 'weight': 1.0, + }, + 'violation': { + 'violation_tolerance_factor': 12.0, + 'clash_overlap_tolerance': 1.5, + 'bond_angle_loss_weight': 0.3, + 'eps': 1e-6, + 'weight': 0.0, + }, + 'pae': { + 'max_bin': 31, + 'num_bins': 64, + 'min_resolution': 0.1, + 'max_resolution': 3.0, + 'eps': 1e-8, + 'weight': 0.0, + }, + 'repr_norm': { + 'weight': 0.01, + 'tolerance': 1.0, + }, + 'chain_centre_mass': { + 'weight': 0.0, + 'eps': 1e-8, + }, + }, + }) + + +def recursive_set(c: mlc.ConfigDict, key: str, value: Any, ignore: str = None): + with c.unlocked(): + for k, v in c.items(): + if ignore is not None and k == ignore: + continue + if isinstance(v, mlc.ConfigDict): + recursive_set(v, key, value) + elif k == key: + c[k] = value + + +def model_config(name, train=False): + c = copy.deepcopy(base_config()) + + def model_2_v2(c): + recursive_set(c, 'v2_feature', True) + recursive_set(c, 'gumbel_sample', True) + c.model.heads.masked_msa.d_out = 22 + c.model.structure_module.separate_kv = True + c.model.structure_module.ipa_bias = False + c.model.template.template_angle_embedder.d_in = 34 + return c + + def multimer(c): + recursive_set(c, 'is_multimer', True) + recursive_set(c, 'max_extra_msa', 1152) + recursive_set(c, 'max_msa_clusters', 128) + recursive_set(c, 'v2_feature', True) + recursive_set(c, 'gumbel_sample', True) + c.model.template.template_angle_embedder.d_in = 34 + c.model.template.template_pair_stack.tri_attn_first = False + c.model.template.template_pointwise_attention.enabled = False + c.model.heads.pae.enabled = True + # we forget to enable it in our training, so disable it here + c.model.heads.pae.disable_enhance_head = True + c.model.heads.masked_msa.d_out = 22 + c.model.structure_module.separate_kv = True + c.model.structure_module.ipa_bias = False + c.model.structure_module.trans_scale_factor = 20 + c.loss.pae.weight = 0.1 + c.model.input_embedder.tf_dim = 21 + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + c.loss.chain_centre_mass.weight = 1.0 + return c + + if name == 'model_1': + pass + elif name == 'model_1_ft': + recursive_set(c, 'max_extra_msa', 5120) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + elif name == 'model_1_af2': + recursive_set(c, 'max_extra_msa', 5120) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + c.loss.repr_norm.weight = 0 + c.model.heads.experimentally_resolved.enabled = True + c.loss.experimentally_resolved.weight = 0.01 + c.globals.alphafold_original_mode = True + elif name == 'model_2': + pass + elif name == 'model_init': + pass + elif name == 'model_init_af2': + c.globals.alphafold_original_mode = True + pass + elif name == 'model_2_ft': + recursive_set(c, 'max_extra_msa', 1024) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + elif name == 'model_2_af2': + recursive_set(c, 'max_extra_msa', 1024) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + c.loss.repr_norm.weight = 0 + c.model.heads.experimentally_resolved.enabled = True + c.loss.experimentally_resolved.weight = 0.01 + c.globals.alphafold_original_mode = True + elif name == 'model_2_v2': + c = model_2_v2(c) + elif name == 'model_2_v2_ft': + c = model_2_v2(c) + recursive_set(c, 'max_extra_msa', 1024) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + elif name == 'model_3_af2' or name == 'model_4_af2': + recursive_set(c, 'max_extra_msa', 5120) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + c.loss.repr_norm.weight = 0 + c.model.heads.experimentally_resolved.enabled = True + c.loss.experimentally_resolved.weight = 0.01 + c.globals.alphafold_original_mode = True + c.model.template.enabled = False + c.model.template.embed_angles = False + recursive_set(c, 'use_templates', False) + recursive_set(c, 'use_template_torsion_angles', False) + elif name == 'model_5_af2': + recursive_set(c, 'max_extra_msa', 1024) + recursive_set(c, 'max_msa_clusters', 512) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.02 + c.loss.repr_norm.weight = 0 + c.model.heads.experimentally_resolved.enabled = True + c.loss.experimentally_resolved.weight = 0.01 + c.globals.alphafold_original_mode = True + c.model.template.enabled = False + c.model.template.embed_angles = False + recursive_set(c, 'use_templates', False) + recursive_set(c, 'use_template_torsion_angles', False) + elif name == 'multimer': + c = multimer(c) + elif name == 'multimer_ft': + c = multimer(c) + recursive_set(c, 'max_extra_msa', 1152) + recursive_set(c, 'max_msa_clusters', 256) + c.data.train.crop_size = 384 + c.loss.violation.weight = 0.5 + elif name == 'multimer_af2': + recursive_set(c, 'max_extra_msa', 1152) + recursive_set(c, 'max_msa_clusters', 256) + recursive_set(c, 'is_multimer', True) + recursive_set(c, 'v2_feature', True) + recursive_set(c, 'gumbel_sample', True) + c.model.template.template_angle_embedder.d_in = 34 + c.model.template.template_pair_stack.tri_attn_first = False + c.model.template.template_pointwise_attention.enabled = False + c.model.heads.pae.enabled = True + c.model.heads.experimentally_resolved.enabled = True + c.model.heads.masked_msa.d_out = 22 + c.model.structure_module.separate_kv = True + c.model.structure_module.ipa_bias = False + c.model.structure_module.trans_scale_factor = 20 + c.loss.pae.weight = 0.1 + c.loss.violation.weight = 0.5 + c.loss.experimentally_resolved.weight = 0.01 + c.model.input_embedder.tf_dim = 21 + c.globals.alphafold_original_mode = True + c.data.train.crop_size = 384 + c.loss.repr_norm.weight = 0 + c.loss.chain_centre_mass.weight = 1.0 + recursive_set(c, 'outer_product_mean_first', True) + else: + raise ValueError(f'invalid --model-name: {name}.') + if train: + c.globals.chunk_size = None + recursive_set(c, 'inf', 3e4) + recursive_set(c, 'eps', 1e-5, 'loss') + return c diff --git a/runtime/Lib/site_packages/models/science/unifold/data/__init__.py b/runtime/Lib/site_packages/models/science/unifold/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9821d212c3c7781e601ea8d2137493942d0937d4 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Data pipeline for model features.""" diff --git a/runtime/Lib/site_packages/models/science/unifold/data/data_ops.py b/runtime/Lib/site_packages/models/science/unifold/data/data_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..c6acbfe2348cf995f73633d909d20a83a91de5c2 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/data_ops.py @@ -0,0 +1,1397 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import itertools +from functools import reduce, wraps +from operator import add +from typing import List, MutableMapping, Optional + +import numpy as np +import torch +from unicore.data import data_utils +from unicore.utils import batched_gather, one_hot, tensor_tree_map, tree_map + +from modelscope.models.science.unifold.config import (N_EXTRA_MSA, N_MSA, + N_RES, N_TPL) +from modelscope.models.science.unifold.data import residue_constants as rc +from modelscope.models.science.unifold.modules.frame import Frame, Rotation + +NumpyDict = MutableMapping[str, np.ndarray] +TorchDict = MutableMapping[str, np.ndarray] + +protein: TorchDict + +MSA_FEATURE_NAMES = [ + 'msa', + 'deletion_matrix', + 'msa_mask', + 'msa_row_mask', + 'bert_mask', + 'true_msa', + 'msa_chains', +] + + +def cast_to_64bit_ints(protein): + # We keep all ints as int64 + for k, v in protein.items(): + if k.endswith('_mask'): + protein[k] = v.type(torch.float32) + elif v.dtype in (torch.int32, torch.uint8, torch.int8): + protein[k] = v.type(torch.int64) + + return protein + + +def make_seq_mask(protein): + protein['seq_mask'] = torch.ones( + protein['aatype'].shape, dtype=torch.float32) + return protein + + +def make_template_mask(protein): + protein['template_mask'] = torch.ones( + protein['template_aatype'].shape[0], dtype=torch.float32) + return protein + + +def curry1(f): + """Supply all arguments but the first.""" + + @wraps(f) + def fc(*args, **kwargs): + return lambda x: f(x, *args, **kwargs) + + return fc + + +def correct_msa_restypes(protein): + """Correct MSA restype to have the same order as rc.""" + protein['msa'] = protein['msa'].long() + new_order_list = rc.MAP_HHBLITS_AATYPE_TO_OUR_AATYPE + new_order = ( + torch.tensor(new_order_list, dtype=torch.int8).unsqueeze(-1).expand( + -1, protein['msa'].shape[1])) + protein['msa'] = torch.gather(new_order, 0, protein['msa']).long() + + return protein + + +def squeeze_features(protein): + """Remove singleton and repeated dimensions in protein features.""" + if len(protein['aatype'].shape) == 2: + protein['aatype'] = torch.argmax(protein['aatype'], dim=-1) + if 'resolution' in protein and len(protein['resolution'].shape) == 1: + # use tensor for resolution + protein['resolution'] = protein['resolution'][0] + for k in [ + 'domain_name', + 'msa', + 'num_alignments', + 'seq_length', + 'sequence', + 'superfamily', + 'deletion_matrix', + 'between_segment_residues', + 'residue_index', + 'template_all_atom_mask', + ]: + if k in protein and len(protein[k].shape): + final_dim = protein[k].shape[-1] + if isinstance(final_dim, int) and final_dim == 1: + if torch.is_tensor(protein[k]): + protein[k] = torch.squeeze(protein[k], dim=-1) + else: + protein[k] = np.squeeze(protein[k], axis=-1) + + for k in ['seq_length', 'num_alignments']: + if k in protein and len(protein[k].shape): + protein[k] = protein[k][0] + + return protein + + +@curry1 +def randomly_replace_msa_with_unknown(protein, replace_proportion): + """Replace a portion of the MSA with 'X'.""" + if replace_proportion > 0.0: + msa_mask = np.random.rand(protein['msa'].shape) < replace_proportion + x_idx = 20 + gap_idx = 21 + msa_mask = torch.logical_and(msa_mask, protein['msa'] != gap_idx) + protein['msa'] = torch.where(msa_mask, + torch.ones_like(protein['msa']) * x_idx, + protein['msa']) + aatype_mask = np.random.rand( + protein['aatype'].shape) < replace_proportion + + protein['aatype'] = torch.where( + aatype_mask, + torch.ones_like(protein['aatype']) * x_idx, + protein['aatype'], + ) + return protein + + +def gumbel_noise(shape): + """Generate Gumbel Noise of given Shape. + This generates samples from Gumbel(0, 1). + Args: + shape: Shape of noise to return. + Returns: + Gumbel noise of given shape. + """ + epsilon = 1e-6 + uniform_noise = torch.from_numpy(np.random.uniform(0, 1, shape)) + gumbel = -torch.log(-torch.log(uniform_noise + epsilon) + epsilon) + return gumbel + + +def gumbel_max_sample(logits): + """Samples from a probability distribution given by 'logits'. + This uses Gumbel-max trick to implement the sampling in an efficient manner. + Args: + logits: Logarithm of probabilities to sample from, probabilities can be + unnormalized. + Returns: + Sample from logprobs in one-hot form. + """ + z = gumbel_noise(logits.shape) + return torch.argmax(logits + z, dim=-1) + + +def gumbel_argsort_sample_idx(logits): + """Samples with replacement from a distribution given by 'logits'. + This uses Gumbel trick to implement the sampling an efficient manner. For a + distribution over k items this samples k times without replacement, so this + is effectively sampling a random permutation with probabilities over the + permutations derived from the logprobs. + Args: + logits: Logarithm of probabilities to sample from, probabilities can be + unnormalized. + Returns: + Sample from logprobs in index + """ + z = gumbel_noise(logits.shape) + return torch.argsort(logits + z, dim=-1, descending=True) + + +def uniform_permutation(num_seq): + shuffled = torch.from_numpy(np.random.permutation(num_seq - 1) + 1) + return torch.cat((torch.tensor([0]), shuffled), dim=0) + + +def gumbel_permutation(msa_mask, msa_chains=None): + has_msa = torch.sum(msa_mask.long(), dim=-1) > 0 + # default logits is zero + logits = torch.zeros_like(has_msa, dtype=torch.float32) + logits[~has_msa] = -1e6 + # one sample only + assert len(logits.shape) == 1 + # skip first row + logits = logits[1:] + has_msa = has_msa[1:] + if logits.shape[0] == 0: + return torch.tensor([0]) + if msa_chains is not None: + # skip first row + msa_chains = msa_chains[1:].reshape(-1) + msa_chains[~has_msa] = 0 + keys, counts = np.unique(msa_chains, return_counts=True) + num_has_msa = has_msa.sum() + num_pair = (msa_chains == 1).sum() + num_unpair = num_has_msa - num_pair + num_chains = (keys > 1).sum() + logits[has_msa] = 1.0 / (num_has_msa + 1e-6) + logits[~has_msa] = 0 + for k in keys: + if k > 1: + cur_mask = msa_chains == k + cur_cnt = cur_mask.sum() + if cur_cnt > 0: + logits[cur_mask] *= num_unpair / (num_chains * cur_cnt) + logits = torch.log(logits + 1e-6) + shuffled = gumbel_argsort_sample_idx(logits) + 1 + return torch.cat((torch.tensor([0]), shuffled), dim=0) + + +@curry1 +def sample_msa(protein, + max_seq, + keep_extra, + gumbel_sample=False, + biased_msa_by_chain=False): + """Sample MSA randomly, remaining sequences are stored are stored as `extra_*`.""" + num_seq = protein['msa'].shape[0] + num_sel = min(max_seq, num_seq) + if not gumbel_sample: + index_order = uniform_permutation(num_seq) + else: + msa_chains = ( + protein['msa_chains'] if + (biased_msa_by_chain and 'msa_chains' in protein) else None) + index_order = gumbel_permutation(protein['msa_mask'], msa_chains) + num_sel = min(max_seq, num_seq) + sel_seq, not_sel_seq = torch.split(index_order, + [num_sel, num_seq - num_sel]) + + for k in MSA_FEATURE_NAMES: + if k in protein: + if keep_extra: + protein['extra_' + k] = torch.index_select( + protein[k], 0, not_sel_seq) + protein[k] = torch.index_select(protein[k], 0, sel_seq) + + return protein + + +@curry1 +def sample_msa_distillation(protein, max_seq): + if 'is_distillation' in protein and protein['is_distillation'] == 1: + protein = sample_msa(max_seq, keep_extra=False)(protein) + return protein + + +@curry1 +def random_delete_msa(protein, config): + # to reduce the cost of msa features + num_seq = protein['msa'].shape[0] + seq_len = protein['msa'].shape[1] + max_seq = config.max_msa_entry // seq_len + if num_seq > max_seq: + keep_index = ( + torch.from_numpy( + np.random.choice(num_seq - 1, max_seq - 1, + replace=False)).long() + 1) + keep_index = torch.sort(keep_index)[0] + keep_index = torch.cat((torch.tensor([0]), keep_index), dim=0) + for k in MSA_FEATURE_NAMES: + if k in protein: + protein[k] = torch.index_select(protein[k], 0, keep_index) + return protein + + +@curry1 +def crop_extra_msa(protein, max_extra_msa): + num_seq = protein['extra_msa'].shape[0] + num_sel = min(max_extra_msa, num_seq) + select_indices = torch.from_numpy(np.random.permutation(num_seq)[:num_sel]) + for k in MSA_FEATURE_NAMES: + if 'extra_' + k in protein: + protein['extra_' + k] = torch.index_select(protein['extra_' + k], + 0, select_indices) + + return protein + + +def delete_extra_msa(protein): + for k in MSA_FEATURE_NAMES: + if 'extra_' + k in protein: + del protein['extra_' + k] + return protein + + +@curry1 +def block_delete_msa(protein, config): + if 'is_distillation' in protein and protein['is_distillation'] == 1: + return protein + num_seq = protein['msa'].shape[0] + if num_seq <= config.min_num_msa: + return protein + block_num_seq = torch.floor( + torch.tensor(num_seq, dtype=torch.float32) + * config.msa_fraction_per_block).to(torch.int32) + + if config.randomize_num_blocks: + nb = np.random.randint(0, config.num_blocks + 1) + else: + nb = config.num_blocks + + del_block_starts = torch.from_numpy(np.random.randint(0, num_seq, [nb])) + del_blocks = del_block_starts[:, None] + torch.arange(0, block_num_seq) + del_blocks = torch.clip(del_blocks, 0, num_seq - 1) + del_indices = torch.unique(del_blocks.view(-1)) + # add zeros to ensure cnt_zero > 1 + combined = torch.hstack((torch.arange(0, num_seq)[None], del_indices[None], + torch.zeros(2)[None])).long() + uniques, counts = combined.unique(return_counts=True) + difference = uniques[counts == 1] + # intersection = uniques[counts > 1] + keep_indices = difference.view(-1) + keep_indices = torch.hstack( + [torch.zeros(1).long()[None], keep_indices[None]]).view(-1) + assert int(keep_indices[0]) == 0 + for k in MSA_FEATURE_NAMES: + if k in protein: + protein[k] = torch.index_select(protein[k], 0, index=keep_indices) + return protein + + +@curry1 +def nearest_neighbor_clusters(protein, gap_agreement_weight=0.0): + weights = torch.cat( + [torch.ones(21), gap_agreement_weight * torch.ones(1), + torch.zeros(1)], + 0, + ) + + msa_one_hot = one_hot(protein['msa'], 23) + sample_one_hot = protein['msa_mask'][:, :, None] * msa_one_hot + extra_msa_one_hot = one_hot(protein['extra_msa'], 23) + extra_one_hot = protein['extra_msa_mask'][:, :, None] * extra_msa_one_hot + + num_seq, num_res, _ = sample_one_hot.shape + extra_num_seq, _, _ = extra_one_hot.shape + + # Compute tf.einsum('mrc,nrc,c->mn', sample_one_hot, extra_one_hot, weights) + # in an optimized fashion to avoid possible memory or computation blowup. + a = extra_one_hot.view(extra_num_seq, num_res * 23) + b = (sample_one_hot * weights).view(num_seq, num_res * 23).transpose(0, 1) + agreement = a @ b + # Assign each sequence in the extra sequences to the closest MSA sample + protein['extra_cluster_assignment'] = torch.argmax(agreement, dim=1).long() + + return protein + + +def unsorted_segment_sum(data, segment_ids, num_segments): + assert len( + segment_ids.shape) == 1 and segment_ids.shape[0] == data.shape[0] + segment_ids = segment_ids.view(segment_ids.shape[0], + *((1, ) * len(data.shape[1:]))) + segment_ids = segment_ids.expand(data.shape) + shape = [num_segments] + list(data.shape[1:]) + tensor = torch.zeros(*shape).scatter_add_(0, segment_ids, data.float()) + tensor = tensor.type(data.dtype) + return tensor + + +def summarize_clusters(protein): + """Produce profile and deletion_matrix_mean within each cluster.""" + num_seq = protein['msa'].shape[0] + + def csum(x): + return unsorted_segment_sum(x, protein['extra_cluster_assignment'], + num_seq) + + mask = protein['extra_msa_mask'] + mask_counts = 1e-6 + protein['msa_mask'] + csum(mask) # Include center + + # TODO: this line is very slow + msa_sum = csum(mask[:, :, None] * one_hot(protein['extra_msa'], 23)) + msa_sum += one_hot(protein['msa'], 23) # Original sequence + protein['cluster_profile'] = msa_sum / mask_counts[:, :, None] + del msa_sum + + del_sum = csum(mask * protein['extra_deletion_matrix']) + del_sum += protein['deletion_matrix'] # Original sequence + protein['cluster_deletion_mean'] = del_sum / mask_counts + del del_sum + + return protein + + +@curry1 +def nearest_neighbor_clusters_v2(batch, gap_agreement_weight=0.0): + """Assign each extra MSA sequence to its nearest neighbor in sampled MSA.""" + + # Determine how much weight we assign to each agreement. In theory, we could + # use a full blosum matrix here, but right now let's just down-weight gap + # agreement because it could be spurious. + # Never put weight on agreeing on BERT mask. + + weights = torch.tensor( + [1.0] * 21 + [gap_agreement_weight] + [0.0], dtype=torch.float32) + + msa_mask = batch['msa_mask'] + extra_mask = batch['extra_msa_mask'] + msa_one_hot = one_hot(batch['msa'], 23) + extra_one_hot = one_hot(batch['extra_msa'], 23) + + msa_one_hot_masked = msa_mask[:, :, None] * msa_one_hot + extra_one_hot_masked = extra_mask[:, :, None] * extra_one_hot + + t1 = weights * msa_one_hot_masked + t1 = t1.view(t1.shape[0], t1.shape[1] * t1.shape[2]) + t2 = extra_one_hot_masked.view( + extra_one_hot.shape[0], + extra_one_hot.shape[1] * extra_one_hot.shape[2]) + agreement = t1 @ t2.T + + cluster_assignment = torch.nn.functional.softmax(1e3 * agreement, dim=0) + cluster_assignment *= torch.einsum('mr, nr->mn', msa_mask, extra_mask) + + cluster_count = torch.sum(cluster_assignment, dim=-1) + cluster_count += 1.0 # We always include the sequence itself. + + msa_sum = torch.einsum('nm, mrc->nrc', cluster_assignment, + extra_one_hot_masked) + msa_sum += msa_one_hot_masked + + cluster_profile = msa_sum / cluster_count[:, None, None] + + deletion_matrix = batch['deletion_matrix'] + extra_deletion_matrix = batch['extra_deletion_matrix'] + + del_sum = torch.einsum('nm, mc->nc', cluster_assignment, + extra_mask * extra_deletion_matrix) + del_sum += deletion_matrix # Original sequence. + cluster_deletion_mean = del_sum / cluster_count[:, None] + batch['cluster_profile'] = cluster_profile + batch['cluster_deletion_mean'] = cluster_deletion_mean + + return batch + + +def make_msa_mask(protein): + """Mask features are all ones, but will later be zero-padded.""" + if 'msa_mask' not in protein: + protein['msa_mask'] = torch.ones( + protein['msa'].shape, dtype=torch.float32) + protein['msa_row_mask'] = torch.ones((protein['msa'].shape[0]), + dtype=torch.float32) + return protein + + +def pseudo_beta_fn(aatype, all_atom_positions, all_atom_mask): + """Create pseudo beta features.""" + if aatype.shape[0] > 0: + is_gly = torch.eq(aatype, rc.restype_order['G']) + ca_idx = rc.atom_order['CA'] + cb_idx = rc.atom_order['CB'] + pseudo_beta = torch.where( + torch.tile(is_gly[..., None], [1] * len(is_gly.shape) + [3]), + all_atom_positions[..., ca_idx, :], + all_atom_positions[..., cb_idx, :], + ) + else: + pseudo_beta = all_atom_positions.new_zeros(*aatype.shape, 3) + if all_atom_mask is not None: + if aatype.shape[0] > 0: + pseudo_beta_mask = torch.where(is_gly, all_atom_mask[..., ca_idx], + all_atom_mask[..., cb_idx]) + else: + pseudo_beta_mask = torch.zeros_like(aatype).float() + return pseudo_beta, pseudo_beta_mask + else: + return pseudo_beta + + +@curry1 +def make_pseudo_beta(protein, prefix=''): + """Create pseudo-beta (alpha for glycine) position and mask.""" + assert prefix in ['', 'template_'] + ( + protein[prefix + 'pseudo_beta'], + protein[prefix + 'pseudo_beta_mask'], + ) = pseudo_beta_fn( + protein['template_aatype' if prefix else 'aatype'], + protein[prefix + 'all_atom_positions'], + protein['template_all_atom_mask' if prefix else 'all_atom_mask'], + ) + return protein + + +@curry1 +def add_constant_field(protein, key, value): + protein[key] = torch.tensor(value) + return protein + + +def shaped_categorical(probs, epsilon=1e-10): + ds = probs.shape + num_classes = ds[-1] + probs = torch.reshape(probs + epsilon, [-1, num_classes]) + gen = torch.Generator() + gen.manual_seed(np.random.randint(65535)) + counts = torch.multinomial(probs, 1, generator=gen) + return torch.reshape(counts, ds[:-1]) + + +def make_hhblits_profile(protein): + """Compute the HHblits MSA profile if not already present.""" + if 'hhblits_profile' in protein: + return protein + + # Compute the profile for every residue (over all MSA sequences). + msa_one_hot = one_hot(protein['msa'], 22) + + protein['hhblits_profile'] = torch.mean(msa_one_hot, dim=0) + return protein + + +def make_msa_profile(batch): + """Compute the MSA profile.""" + # Compute the profile for every residue (over all MSA sequences). + oh = one_hot(batch['msa'], 22) + mask = batch['msa_mask'][:, :, None] + oh *= mask + return oh.sum(dim=0) / (mask.sum(dim=0) + 1e-10) + + +def make_hhblits_profile_v2(protein): + """Compute the HHblits MSA profile if not already present.""" + if 'hhblits_profile' in protein: + return protein + protein['hhblits_profile'] = make_msa_profile(protein) + return protein + + +def share_mask_by_entity(mask_position, protein): # new in unifold + if 'num_sym' not in protein: + return mask_position + entity_id = protein['entity_id'] + sym_id = protein['sym_id'] + num_sym = protein['num_sym'] + unique_entity_ids = entity_id.unique() + first_sym_mask = sym_id == 1 + for cur_entity_id in unique_entity_ids: + cur_entity_mask = entity_id == cur_entity_id + cur_num_sym = int(num_sym[cur_entity_mask][0]) + if cur_num_sym > 1: + cur_sym_mask = first_sym_mask & cur_entity_mask + cur_sym_bert_mask = mask_position[:, cur_sym_mask] + mask_position[:, cur_entity_mask] = cur_sym_bert_mask.repeat( + 1, cur_num_sym) + return mask_position + + +@curry1 +def make_masked_msa(protein, + config, + replace_fraction, + gumbel_sample=False, + share_mask=False): + """Create data for BERT on raw MSA.""" + # Add a random amino acid uniformly. + random_aa = torch.tensor([0.05] * 20 + [0.0, 0.0], dtype=torch.float32) + + categorical_probs = ( + config.uniform_prob * random_aa + + config.profile_prob * protein['hhblits_profile'] + + config.same_prob * one_hot(protein['msa'], 22)) + + # Put all remaining probability on [MASK] which is a new column + pad_shapes = list( + reduce(add, [(0, 0) for _ in range(len(categorical_probs.shape))])) + pad_shapes[1] = 1 + mask_prob = 1.0 - config.profile_prob - config.same_prob - config.uniform_prob + assert mask_prob >= 0.0 + categorical_probs = torch.nn.functional.pad( + categorical_probs, pad_shapes, value=mask_prob) + sh = protein['msa'].shape + mask_position = torch.from_numpy(np.random.rand(*sh) < replace_fraction) + mask_position &= protein['msa_mask'].bool() + + if 'bert_mask' in protein: + mask_position &= protein['bert_mask'].bool() + + if share_mask: + mask_position = share_mask_by_entity(mask_position, protein) + if gumbel_sample: + logits = torch.log(categorical_probs + 1e-6) + bert_msa = gumbel_max_sample(logits) + else: + bert_msa = shaped_categorical(categorical_probs) + bert_msa = torch.where(mask_position, bert_msa, protein['msa']) + bert_msa *= protein['msa_mask'].long() + + # Mix real and masked MSA + protein['bert_mask'] = mask_position.to(torch.float32) + protein['true_msa'] = protein['msa'] + protein['msa'] = bert_msa + + return protein + + +@curry1 +def make_fixed_size( + protein, + shape_schema, + msa_cluster_size, + extra_msa_size, + num_res=0, + num_templates=0, +): + """Guess at the MSA and sequence dimension to make fixed size.""" + + def get_pad_size(cur_size, multiplier=4): + return max(multiplier, + ((cur_size + multiplier - 1) // multiplier) * multiplier) + + if num_res is not None: + input_num_res = ( + protein['aatype'].shape[0] + if 'aatype' in protein else protein['msa_mask'].shape[1]) + if input_num_res != num_res: + num_res = get_pad_size(input_num_res, 4) + if 'extra_msa_mask' in protein: + input_extra_msa_size = protein['extra_msa_mask'].shape[0] + if input_extra_msa_size != extra_msa_size: + extra_msa_size = get_pad_size(input_extra_msa_size, 8) + pad_size_map = { + N_RES: num_res, + N_MSA: msa_cluster_size, + N_EXTRA_MSA: extra_msa_size, + N_TPL: num_templates, + } + + for k, v in protein.items(): + # Don't transfer this to the accelerator. + if k == 'extra_cluster_assignment': + continue + shape = list(v.shape) + schema = shape_schema[k] + msg = 'Rank mismatch between shape and shape schema for' + assert len(shape) == len(schema), f'{msg} {k}: {shape} vs {schema}' + pad_size = [ + pad_size_map.get(s2, None) or s1 + for (s1, s2) in zip(shape, schema) + ] + + padding = [(0, p - v.shape[i]) for i, p in enumerate(pad_size)] + padding.reverse() + padding = list(itertools.chain(*padding)) + if padding: + protein[k] = torch.nn.functional.pad(v, padding) + protein[k] = torch.reshape(protein[k], pad_size) + + return protein + + +def make_target_feat(protein): + """Create and concatenate MSA features.""" + protein['aatype'] = protein['aatype'].long() + + if 'between_segment_residues' in protein: + has_break = torch.clip( + protein['between_segment_residues'].to(torch.float32), 0, 1) + else: + has_break = torch.zeros_like(protein['aatype'], dtype=torch.float32) + if 'asym_len' in protein: + asym_len = protein['asym_len'] + entity_ends = torch.cumsum(asym_len, dim=-1)[:-1] + has_break[entity_ends] = 1.0 + has_break = has_break.float() + aatype_1hot = one_hot(protein['aatype'], 21) + target_feat = [ + torch.unsqueeze(has_break, dim=-1), + aatype_1hot, # Everyone gets the original sequence. + ] + protein['target_feat'] = torch.cat(target_feat, dim=-1) + return protein + + +def make_msa_feat(protein): + """Create and concatenate MSA features.""" + msa_1hot = one_hot(protein['msa'], 23) + has_deletion = torch.clip(protein['deletion_matrix'], 0.0, 1.0) + deletion_value = torch.atan( + protein['deletion_matrix'] / 3.0) * (2.0 / np.pi) + msa_feat = [ + msa_1hot, + torch.unsqueeze(has_deletion, dim=-1), + torch.unsqueeze(deletion_value, dim=-1), + ] + if 'cluster_profile' in protein: + deletion_mean_value = torch.atan( + protein['cluster_deletion_mean'] / 3.0) * (2.0 / np.pi) + msa_feat.extend([ + protein['cluster_profile'], + torch.unsqueeze(deletion_mean_value, dim=-1), + ]) + + if 'extra_deletion_matrix' in protein: + protein['extra_msa_has_deletion'] = torch.clip( + protein['extra_deletion_matrix'], 0.0, 1.0) + protein['extra_msa_deletion_value'] = torch.atan( + protein['extra_deletion_matrix'] / 3.0) * (2.0 / np.pi) + + protein['msa_feat'] = torch.cat(msa_feat, dim=-1) + return protein + + +def make_msa_feat_v2(batch): + """Create and concatenate MSA features.""" + msa_1hot = one_hot(batch['msa'], 23) + deletion_matrix = batch['deletion_matrix'] + has_deletion = torch.clip(deletion_matrix, 0.0, 1.0)[..., None] + deletion_value = (torch.atan(deletion_matrix / 3.0) * (2.0 / np.pi))[..., + None] + + deletion_mean_value = ( + torch.arctan(batch['cluster_deletion_mean'] / 3.0) * # noqa W504 + (2.0 / np.pi))[..., None] + + msa_feat = [ + msa_1hot, + has_deletion, + deletion_value, + batch['cluster_profile'], + deletion_mean_value, + ] + batch['msa_feat'] = torch.cat(msa_feat, dim=-1) + return batch + + +@curry1 +def make_extra_msa_feat(batch, num_extra_msa): + # 23 = 20 amino acids + 'X' for unknown + gap + bert mask + extra_msa = batch['extra_msa'][:num_extra_msa] + deletion_matrix = batch['extra_deletion_matrix'][:num_extra_msa] + has_deletion = torch.clip(deletion_matrix, 0.0, 1.0) + deletion_value = torch.atan(deletion_matrix / 3.0) * (2.0 / np.pi) + extra_msa_mask = batch['extra_msa_mask'][:num_extra_msa] + batch['extra_msa'] = extra_msa + batch['extra_msa_mask'] = extra_msa_mask + batch['extra_msa_has_deletion'] = has_deletion + batch['extra_msa_deletion_value'] = deletion_value + return batch + + +@curry1 +def select_feat(protein, feature_list): + return {k: v for k, v in protein.items() if k in feature_list} + + +def make_atom14_masks(protein): + """Construct denser atom positions (14 dimensions instead of 37).""" + + if 'atom14_atom_exists' in protein: # lazy move + return protein + + restype_atom14_to_atom37 = torch.tensor( + rc.restype_atom14_to_atom37, + dtype=torch.int64, + device=protein['aatype'].device, + ) + restype_atom37_to_atom14 = torch.tensor( + rc.restype_atom37_to_atom14, + dtype=torch.int64, + device=protein['aatype'].device, + ) + restype_atom14_mask = torch.tensor( + rc.restype_atom14_mask, + dtype=torch.float32, + device=protein['aatype'].device, + ) + restype_atom37_mask = torch.tensor( + rc.restype_atom37_mask, + dtype=torch.float32, + device=protein['aatype'].device) + + protein_aatype = protein['aatype'].long() + protein['residx_atom14_to_atom37'] = restype_atom14_to_atom37[ + protein_aatype].long() + protein['residx_atom37_to_atom14'] = restype_atom37_to_atom14[ + protein_aatype].long() + protein['atom14_atom_exists'] = restype_atom14_mask[protein_aatype] + protein['atom37_atom_exists'] = restype_atom37_mask[protein_aatype] + + return protein + + +def make_atom14_masks_np(batch): + batch = tree_map(lambda n: torch.tensor(n), batch, np.ndarray) + out = make_atom14_masks(batch) + out = tensor_tree_map(lambda t: np.array(t), out) + return out + + +def make_atom14_positions(protein): + """Constructs denser atom positions (14 dimensions instead of 37).""" + protein['aatype'] = protein['aatype'].long() + protein['all_atom_mask'] = protein['all_atom_mask'].float() + protein['all_atom_positions'] = protein['all_atom_positions'].float() + residx_atom14_mask = protein['atom14_atom_exists'] + residx_atom14_to_atom37 = protein['residx_atom14_to_atom37'] + + # Create a mask for known ground truth positions. + residx_atom14_gt_mask = residx_atom14_mask * batched_gather( + protein['all_atom_mask'], + residx_atom14_to_atom37, + dim=-1, + num_batch_dims=len(protein['all_atom_mask'].shape[:-1]), + ) + + # Gather the ground truth positions. + residx_atom14_gt_positions = residx_atom14_gt_mask[..., None] * ( + batched_gather( + protein['all_atom_positions'], + residx_atom14_to_atom37, + dim=-2, + num_batch_dims=len(protein['all_atom_positions'].shape[:-2]), + )) + + protein['atom14_atom_exists'] = residx_atom14_mask + protein['atom14_gt_exists'] = residx_atom14_gt_mask + protein['atom14_gt_positions'] = residx_atom14_gt_positions + + renaming_matrices = torch.tensor( + rc.renaming_matrices, + dtype=protein['all_atom_mask'].dtype, + device=protein['all_atom_mask'].device, + ) + + # Pick the transformation matrices for the given residue sequence + # shape (num_res, 14, 14). + renaming_transform = renaming_matrices[protein['aatype']] + + # Apply it to the ground truth positions. shape (num_res, 14, 3). + alternative_gt_positions = torch.einsum('...rac,...rab->...rbc', + residx_atom14_gt_positions, + renaming_transform) + protein['atom14_alt_gt_positions'] = alternative_gt_positions + + # Create the mask for the alternative ground truth (differs from the + # ground truth mask, if only one of the atoms in an ambiguous pair has a + # ground truth position). + alternative_gt_mask = torch.einsum('...ra,...rab->...rb', + residx_atom14_gt_mask, + renaming_transform) + protein['atom14_alt_gt_exists'] = alternative_gt_mask + + restype_atom14_is_ambiguous = torch.tensor( + rc.restype_atom14_is_ambiguous, + dtype=protein['all_atom_mask'].dtype, + device=protein['all_atom_mask'].device, + ) + # From this create an ambiguous_mask for the given sequence. + protein['atom14_atom_is_ambiguous'] = restype_atom14_is_ambiguous[ + protein['aatype']] + + return protein + + +def atom37_to_frames(protein, eps=1e-8): + # TODO: extract common part and put them into residue constants. + aatype = protein['aatype'] + all_atom_positions = protein['all_atom_positions'] + all_atom_mask = protein['all_atom_mask'] + + batch_dims = len(aatype.shape[:-1]) + + restype_rigidgroup_base_atom_names = np.full([21, 8, 3], '', dtype=object) + restype_rigidgroup_base_atom_names[:, 0, :] = ['C', 'CA', 'N'] + restype_rigidgroup_base_atom_names[:, 3, :] = ['CA', 'C', 'O'] + + for restype, restype_letter in enumerate(rc.restypes): + resname = rc.restype_1to3[restype_letter] + for chi_idx in range(4): + if rc.chi_angles_mask[restype][chi_idx]: + names = rc.chi_angles_atoms[resname][chi_idx] + restype_rigidgroup_base_atom_names[restype, + chi_idx + 4, :] = names[1:] + + restype_rigidgroup_mask = all_atom_mask.new_zeros( + (*aatype.shape[:-1], 21, 8), ) + restype_rigidgroup_mask[..., 0] = 1 + restype_rigidgroup_mask[..., 3] = 1 + restype_rigidgroup_mask[..., :20, + 4:] = all_atom_mask.new_tensor(rc.chi_angles_mask) + + lookuptable = rc.atom_order.copy() + lookuptable[''] = 0 + lookup = np.vectorize(lambda x: lookuptable[x]) + restype_rigidgroup_base_atom37_idx = lookup( + restype_rigidgroup_base_atom_names, ) + restype_rigidgroup_base_atom37_idx = aatype.new_tensor( + restype_rigidgroup_base_atom37_idx, ) + restype_rigidgroup_base_atom37_idx = restype_rigidgroup_base_atom37_idx.view( + *((1, ) * batch_dims), *restype_rigidgroup_base_atom37_idx.shape) + + residx_rigidgroup_base_atom37_idx = batched_gather( + restype_rigidgroup_base_atom37_idx, + aatype, + dim=-3, + num_batch_dims=batch_dims, + ) + + base_atom_pos = batched_gather( + all_atom_positions, + residx_rigidgroup_base_atom37_idx, + dim=-2, + num_batch_dims=len(all_atom_positions.shape[:-2]), + ) + + gt_frames = Frame.from_3_points( + p_neg_x_axis=base_atom_pos[..., 0, :], + origin=base_atom_pos[..., 1, :], + p_xy_plane=base_atom_pos[..., 2, :], + eps=eps, + ) + + group_exists = batched_gather( + restype_rigidgroup_mask, + aatype, + dim=-2, + num_batch_dims=batch_dims, + ) + + gt_atoms_exist = batched_gather( + all_atom_mask, + residx_rigidgroup_base_atom37_idx, + dim=-1, + num_batch_dims=len(all_atom_mask.shape[:-1]), + ) + gt_exists = torch.min(gt_atoms_exist, dim=-1)[0] * group_exists + + rots = torch.eye(3, dtype=all_atom_mask.dtype, device=aatype.device) + rots = torch.tile(rots, (*((1, ) * batch_dims), 8, 1, 1)) + rots[..., 0, 0, 0] = -1 + rots[..., 0, 2, 2] = -1 + rots = Rotation(mat=rots) + + gt_frames = gt_frames.compose(Frame(rots, None)) + + restype_rigidgroup_is_ambiguous = all_atom_mask.new_zeros( + *((1, ) * batch_dims), 21, 8) + restype_rigidgroup_rots = torch.eye( + 3, dtype=all_atom_mask.dtype, device=aatype.device) + restype_rigidgroup_rots = torch.tile( + restype_rigidgroup_rots, + (*((1, ) * batch_dims), 21, 8, 1, 1), + ) + + for resname, _ in rc.residue_atom_renaming_swaps.items(): + restype = rc.restype_order[rc.restype_3to1[resname]] + chi_idx = int(sum(rc.chi_angles_mask[restype]) - 1) + restype_rigidgroup_is_ambiguous[..., restype, chi_idx + 4] = 1 + restype_rigidgroup_rots[..., restype, chi_idx + 4, 1, 1] = -1 + restype_rigidgroup_rots[..., restype, chi_idx + 4, 2, 2] = -1 + + residx_rigidgroup_is_ambiguous = batched_gather( + restype_rigidgroup_is_ambiguous, + aatype, + dim=-2, + num_batch_dims=batch_dims, + ) + + residx_rigidgroup_ambiguity_rot = batched_gather( + restype_rigidgroup_rots, + aatype, + dim=-4, + num_batch_dims=batch_dims, + ) + + residx_rigidgroup_ambiguity_rot = Rotation( + mat=residx_rigidgroup_ambiguity_rot) + alt_gt_frames = gt_frames.compose( + Frame(residx_rigidgroup_ambiguity_rot, None)) + + gt_frames_tensor = gt_frames.to_tensor_4x4() + alt_gt_frames_tensor = alt_gt_frames.to_tensor_4x4() + + protein['rigidgroups_gt_frames'] = gt_frames_tensor + protein['rigidgroups_gt_exists'] = gt_exists + protein['rigidgroups_group_exists'] = group_exists + protein['rigidgroups_group_is_ambiguous'] = residx_rigidgroup_is_ambiguous + protein['rigidgroups_alt_gt_frames'] = alt_gt_frames_tensor + + return protein + + +@curry1 +def atom37_to_torsion_angles( + protein, + prefix='', +): + aatype = protein[prefix + 'aatype'] + all_atom_positions = protein[prefix + 'all_atom_positions'] + all_atom_mask = protein[prefix + 'all_atom_mask'] + if aatype.shape[-1] == 0: + base_shape = aatype.shape + protein[prefix + + 'torsion_angles_sin_cos'] = all_atom_positions.new_zeros( + *base_shape, 7, 2) + protein[prefix + + 'alt_torsion_angles_sin_cos'] = all_atom_positions.new_zeros( + *base_shape, 7, 2) + protein[prefix + 'torsion_angles_mask'] = all_atom_positions.new_zeros( + *base_shape, 7) + return protein + + aatype = torch.clamp(aatype, max=20) + + pad = all_atom_positions.new_zeros( + [*all_atom_positions.shape[:-3], 1, 37, 3]) + prev_all_atom_positions = torch.cat( + [pad, all_atom_positions[..., :-1, :, :]], dim=-3) + + pad = all_atom_mask.new_zeros([*all_atom_mask.shape[:-2], 1, 37]) + prev_all_atom_mask = torch.cat([pad, all_atom_mask[..., :-1, :]], dim=-2) + + pre_omega_atom_pos = torch.cat( + [prev_all_atom_positions[..., 1:3, :], all_atom_positions[..., :2, :]], + dim=-2, + ) + phi_atom_pos = torch.cat( + [prev_all_atom_positions[..., 2:3, :], all_atom_positions[..., :3, :]], + dim=-2, + ) + psi_atom_pos = torch.cat( + [all_atom_positions[..., :3, :], all_atom_positions[..., 4:5, :]], + dim=-2, + ) + + pre_omega_mask = torch.prod( + prev_all_atom_mask[..., 1:3], dim=-1) * torch.prod( + all_atom_mask[..., :2], dim=-1) + phi_mask = prev_all_atom_mask[..., 2] * torch.prod( + all_atom_mask[..., :3], dim=-1, dtype=all_atom_mask.dtype) + psi_mask = ( + torch.prod(all_atom_mask[..., :3], dim=-1, dtype=all_atom_mask.dtype) + * all_atom_mask[..., 4]) + + chi_atom_indices = torch.as_tensor( + rc.chi_atom_indices, device=aatype.device) + + atom_indices = chi_atom_indices[..., aatype, :, :] + chis_atom_pos = batched_gather(all_atom_positions, atom_indices, -2, + len(atom_indices.shape[:-2])) + + chi_angles_mask = list(rc.chi_angles_mask) + chi_angles_mask.append([0.0, 0.0, 0.0, 0.0]) + chi_angles_mask = all_atom_mask.new_tensor(chi_angles_mask) + + chis_mask = chi_angles_mask[aatype, :] + + chi_angle_atoms_mask = batched_gather( + all_atom_mask, + atom_indices, + dim=-1, + num_batch_dims=len(atom_indices.shape[:-2]), + ) + chi_angle_atoms_mask = torch.prod( + chi_angle_atoms_mask, dim=-1, dtype=chi_angle_atoms_mask.dtype) + chis_mask = chis_mask * chi_angle_atoms_mask + + torsions_atom_pos = torch.cat( + [ + pre_omega_atom_pos[..., None, :, :], + phi_atom_pos[..., None, :, :], + psi_atom_pos[..., None, :, :], + chis_atom_pos, + ], + dim=-3, + ) + + torsion_angles_mask = torch.cat( + [ + pre_omega_mask[..., None], + phi_mask[..., None], + psi_mask[..., None], + chis_mask, + ], + dim=-1, + ) + + torsion_frames = Frame.from_3_points( + torsions_atom_pos[..., 1, :], + torsions_atom_pos[..., 2, :], + torsions_atom_pos[..., 0, :], + eps=1e-8, + ) + + fourth_atom_rel_pos = torsion_frames.invert().apply( + torsions_atom_pos[..., 3, :]) + + torsion_angles_sin_cos = torch.stack( + [fourth_atom_rel_pos[..., 2], fourth_atom_rel_pos[..., 1]], dim=-1) + + denom = torch.sqrt( + torch.sum( + torch.square(torsion_angles_sin_cos), + dim=-1, + dtype=torsion_angles_sin_cos.dtype, + keepdims=True, + ) + 1e-8) + torsion_angles_sin_cos = torsion_angles_sin_cos / denom + + torsion_angles_sin_cos = ( + torsion_angles_sin_cos + * all_atom_mask.new_tensor([1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0], )[ + ((None, ) * len(torsion_angles_sin_cos.shape[:-2])) + + (slice(None), None)]) + + chi_is_ambiguous = torsion_angles_sin_cos.new_tensor( + rc.chi_pi_periodic, )[aatype, ...] + + mirror_torsion_angles = torch.cat( + [ + all_atom_mask.new_ones(*aatype.shape, 3), + 1.0 - 2.0 * chi_is_ambiguous, + ], + dim=-1, + ) + + alt_torsion_angles_sin_cos = ( + torsion_angles_sin_cos * mirror_torsion_angles[..., None]) + + if prefix == '': + # consistent to uni-fold. use [1, 0] placeholder + placeholder_torsions = torch.stack( + [ + torch.ones(torsion_angles_sin_cos.shape[:-1]), + torch.zeros(torsion_angles_sin_cos.shape[:-1]), + ], + dim=-1, + ) + torsion_angles_sin_cos = torsion_angles_sin_cos * torsion_angles_mask[ + ..., + None] + placeholder_torsions * (1 - torsion_angles_mask[..., None]) + alt_torsion_angles_sin_cos = alt_torsion_angles_sin_cos * torsion_angles_mask[ + ..., + None] + placeholder_torsions * (1 - torsion_angles_mask[..., None]) + + protein[prefix + 'torsion_angles_sin_cos'] = torsion_angles_sin_cos + protein[prefix + 'alt_torsion_angles_sin_cos'] = alt_torsion_angles_sin_cos + protein[prefix + 'torsion_angles_mask'] = torsion_angles_mask + + return protein + + +def get_backbone_frames(protein): + protein['true_frame_tensor'] = protein['rigidgroups_gt_frames'][..., + 0, :, :] + protein['frame_mask'] = protein['rigidgroups_gt_exists'][..., 0] + + return protein + + +def get_chi_angles(protein): + dtype = protein['all_atom_mask'].dtype + protein['chi_angles_sin_cos'] = ( + protein['torsion_angles_sin_cos'][..., 3:, :]).to(dtype) + protein['chi_mask'] = protein['torsion_angles_mask'][..., 3:].to(dtype) + + return protein + + +@curry1 +def crop_templates( + protein, + max_templates, + subsample_templates=False, +): + if 'template_mask' in protein: + num_templates = protein['template_mask'].shape[-1] + else: + num_templates = 0 + + # don't sample when there are no templates + if num_templates > 0: + if subsample_templates: + # af2's sampling, min(4, uniform[0, n]) + max_templates = min(max_templates, + np.random.randint(0, num_templates + 1)) + template_idx = torch.tensor( + np.random.choice(num_templates, max_templates, replace=False), + dtype=torch.int64, + ) + else: + # use top templates + template_idx = torch.arange( + min(num_templates, max_templates), dtype=torch.int64) + for k, v in protein.items(): + if k.startswith('template'): + try: + v = v[template_idx] + except Exception as ex: + print(ex.__class__, ex) + print('num_templates', num_templates) + print(k, v.shape) + print('protein:', protein) + print( + 'protein_shape:', + { + k: v.shape + for k, v in protein.items() if 'shape' in dir(v) + }, + ) + protein[k] = v + + return protein + + +@curry1 +def crop_to_size_single(protein, crop_size, shape_schema, seed): + """crop to size.""" + num_res = ( + protein['aatype'].shape[0] + if 'aatype' in protein else protein['msa_mask'].shape[1]) + crop_idx = get_single_crop_idx(num_res, crop_size, seed) + protein = apply_crop_idx(protein, shape_schema, crop_idx) + return protein + + +@curry1 +def crop_to_size_multimer(protein, crop_size, shape_schema, seed, + spatial_crop_prob, ca_ca_threshold): + """crop to size.""" + with data_utils.numpy_seed(seed, key='multimer_crop'): + use_spatial_crop = np.random.rand() < spatial_crop_prob + is_distillation = 'is_distillation' in protein and protein[ + 'is_distillation'] == 1 + if is_distillation: + return crop_to_size_single( + crop_size=crop_size, shape_schema=shape_schema, seed=seed)( + protein) + elif use_spatial_crop: + crop_idx = get_spatial_crop_idx(protein, crop_size, seed, + ca_ca_threshold) + else: + crop_idx = get_contiguous_crop_idx(protein, crop_size, seed) + return apply_crop_idx(protein, shape_schema, crop_idx) + + +def get_single_crop_idx(num_res: NumpyDict, crop_size: int, + random_seed: Optional[int]) -> torch.Tensor: + + if num_res < crop_size: + return torch.arange(num_res) + with data_utils.numpy_seed(random_seed): + crop_start = int(np.random.randint(0, num_res - crop_size + 1)) + return torch.arange(crop_start, crop_start + crop_size) + + +def get_crop_sizes_each_chain( + asym_len: torch.Tensor, + crop_size: int, + random_seed: Optional[int] = None, + use_multinomial: bool = False, +) -> torch.Tensor: + """get crop sizes for contiguous crop""" + if not use_multinomial: + with data_utils.numpy_seed( + random_seed, key='multimer_contiguous_perm'): + shuffle_idx = np.random.permutation(len(asym_len)) + num_left = asym_len.sum() + num_budget = torch.tensor(crop_size) + crop_sizes = [0 for _ in asym_len] + for j, idx in enumerate(shuffle_idx): + this_len = asym_len[idx] + num_left -= this_len + # num res at most we can keep in this ent + max_size = min(num_budget, this_len) + # num res at least we shall keep in this ent + min_size = min(this_len, max(0, num_budget - num_left)) + with data_utils.numpy_seed( + random_seed, j, key='multimer_contiguous_crop_size'): + this_crop_size = int( + np.random.randint( + low=int(min_size), high=int(max_size) + 1)) + num_budget -= this_crop_size + crop_sizes[idx] = this_crop_size + crop_sizes = torch.tensor(crop_sizes) + else: # use multinomial + # TODO: better multimer + entity_probs = asym_len / torch.sum(asym_len) + crop_sizes = torch.from_numpy( + np.random.multinomial(crop_size, pvals=entity_probs)) + crop_sizes = torch.min(crop_sizes, asym_len) + return crop_sizes + + +def get_contiguous_crop_idx( + protein: NumpyDict, + crop_size: int, + random_seed: Optional[int] = None, + use_multinomial: bool = False, +) -> torch.Tensor: + + num_res = protein['aatype'].shape[0] + if num_res <= crop_size: + return torch.arange(num_res) + + assert 'asym_len' in protein + asym_len = protein['asym_len'] + + crop_sizes = get_crop_sizes_each_chain(asym_len, crop_size, random_seed, + use_multinomial) + crop_idxs = [] + asym_offset = torch.tensor(0, dtype=torch.int64) + with data_utils.numpy_seed( + random_seed, key='multimer_contiguous_crop_start_idx'): + for ll, csz in zip(asym_len, crop_sizes): + this_start = np.random.randint(0, int(ll - csz) + 1) + crop_idxs.append( + torch.arange(asym_offset + this_start, + asym_offset + this_start + csz)) + asym_offset += ll + + return torch.cat(crop_idxs) + + +def get_spatial_crop_idx( + protein: NumpyDict, + crop_size: int, + random_seed: int, + ca_ca_threshold: float, + inf: float = 3e4, +) -> List[int]: + + ca_idx = rc.atom_order['CA'] + ca_coords = protein['all_atom_positions'][..., ca_idx, :] + ca_mask = protein['all_atom_mask'][..., ca_idx].bool() + # if there are not enough atoms to construct interface, use contiguous crop + if (ca_mask.sum(dim=-1) <= 1).all(): + return get_contiguous_crop_idx(protein, crop_size, random_seed) + + pair_mask = ca_mask[..., None] * ca_mask[..., None, :] + ca_distances = get_pairwise_distances(ca_coords) + + interface_candidates = get_interface_candidates(ca_distances, + protein['asym_id'], + pair_mask, ca_ca_threshold) + + if torch.any(interface_candidates): + with data_utils.numpy_seed(random_seed, key='multimer_spatial_crop'): + target_res = int(np.random.choice(interface_candidates)) + else: + return get_contiguous_crop_idx(protein, crop_size, random_seed) + + to_target_distances = ca_distances[target_res] + # set inf to non-position residues + to_target_distances[~ca_mask] = inf + break_tie = ( + torch.arange( + 0, + to_target_distances.shape[-1], + device=to_target_distances.device).float() * 1e-3) + to_target_distances += break_tie + ret = torch.argsort(to_target_distances)[:crop_size] + return ret.sort().values + + +def get_pairwise_distances(coords: torch.Tensor) -> torch.Tensor: + coord_diff = coords.unsqueeze(-2) - coords.unsqueeze(-3) + return torch.sqrt(torch.sum(coord_diff**2, dim=-1)) + + +def get_interface_candidates( + ca_distances: torch.Tensor, + asym_id: torch.Tensor, + pair_mask: torch.Tensor, + ca_ca_threshold, +) -> torch.Tensor: + + in_same_asym = asym_id[..., None] == asym_id[..., None, :] + # set distance in the same entity to zero + ca_distances = ca_distances * (1.0 - in_same_asym.float()) * pair_mask + cnt_interfaces = torch.sum( + (ca_distances > 0) & (ca_distances < ca_ca_threshold), dim=-1) + interface_candidates = cnt_interfaces.nonzero(as_tuple=True)[0] + return interface_candidates + + +def apply_crop_idx(protein, shape_schema, crop_idx): + cropped_protein = {} + for k, v in protein.items(): + if k not in shape_schema: # skip items with unknown shape schema + continue + for i, dim_size in enumerate(shape_schema[k]): + if dim_size == N_RES: + v = torch.index_select(v, i, crop_idx) + cropped_protein[k] = v + return cropped_protein diff --git a/runtime/Lib/site_packages/models/science/unifold/data/msa_pairing.py b/runtime/Lib/site_packages/models/science/unifold/data/msa_pairing.py new file mode 100644 index 0000000000000000000000000000000000000000..77c4e9a6c6e4c1a83517ecfeb8f7d78e20bee64a --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/msa_pairing.py @@ -0,0 +1,526 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pairing logic for multimer data """ + +import collections +from typing import Dict, Iterable, List, Sequence + +import numpy as np +import pandas as pd +import scipy.linalg + +from .data_ops import NumpyDict +from .residue_constants import restypes_with_x_and_gap + +MSA_GAP_IDX = restypes_with_x_and_gap.index('-') +SEQUENCE_GAP_CUTOFF = 0.5 +SEQUENCE_SIMILARITY_CUTOFF = 0.9 + +MSA_PAD_VALUES = { + 'msa_all_seq': MSA_GAP_IDX, + 'msa_mask_all_seq': 1, + 'deletion_matrix_all_seq': 0, + 'deletion_matrix_int_all_seq': 0, + 'msa': MSA_GAP_IDX, + 'msa_mask': 1, + 'deletion_matrix': 0, + 'deletion_matrix_int': 0, +} + +MSA_FEATURES = ('msa', 'msa_mask', 'deletion_matrix', 'deletion_matrix_int') +SEQ_FEATURES = ( + 'residue_index', + 'aatype', + 'all_atom_positions', + 'all_atom_mask', + 'seq_mask', + 'between_segment_residues', + 'has_alt_locations', + 'has_hetatoms', + 'asym_id', + 'entity_id', + 'sym_id', + 'entity_mask', + 'deletion_mean', + 'prediction_atom_mask', + 'literature_positions', + 'atom_indices_to_group_indices', + 'rigid_group_default_frame', + # zy + 'num_sym', +) +TEMPLATE_FEATURES = ( + 'template_aatype', + 'template_all_atom_positions', + 'template_all_atom_mask', +) +CHAIN_FEATURES = ('num_alignments', 'seq_length') + + +def create_paired_features(chains: Iterable[NumpyDict], ) -> List[NumpyDict]: + """Returns the original chains with paired NUM_SEQ features. + + Args: + chains: A list of feature dictionaries for each chain. + + Returns: + A list of feature dictionaries with sequence features including only + rows to be paired. + """ + chains = list(chains) + chain_keys = chains[0].keys() + + if len(chains) < 2: + return chains + else: + updated_chains = [] + paired_chains_to_paired_row_indices = pair_sequences(chains) + paired_rows = reorder_paired_rows(paired_chains_to_paired_row_indices) + + for chain_num, chain in enumerate(chains): + new_chain = {k: v for k, v in chain.items() if '_all_seq' not in k} + for feature_name in chain_keys: + if feature_name.endswith('_all_seq'): + feats_padded = pad_features(chain[feature_name], + feature_name) + new_chain[feature_name] = feats_padded[ + paired_rows[:, chain_num]] + new_chain['num_alignments_all_seq'] = np.asarray( + len(paired_rows[:, chain_num])) + updated_chains.append(new_chain) + return updated_chains + + +def pad_features(feature: np.ndarray, feature_name: str) -> np.ndarray: + """Add a 'padding' row at the end of the features list. + + The padding row will be selected as a 'paired' row in the case of partial + alignment - for the chain that doesn't have paired alignment. + + Args: + feature: The feature to be padded. + feature_name: The name of the feature to be padded. + + Returns: + The feature with an additional padding row. + """ + assert feature.dtype != np.dtype(np.str_) + if feature_name in ( + 'msa_all_seq', + 'msa_mask_all_seq', + 'deletion_matrix_all_seq', + 'deletion_matrix_int_all_seq', + ): + num_res = feature.shape[1] + padding = MSA_PAD_VALUES[feature_name] * np.ones([1, num_res], + feature.dtype) + elif feature_name == 'msa_species_identifiers_all_seq': + padding = [b''] + else: + return feature + feats_padded = np.concatenate([feature, padding], axis=0) + return feats_padded + + +def _make_msa_df(chain_features: NumpyDict) -> pd.DataFrame: + """Makes dataframe with msa features needed for msa pairing.""" + chain_msa = chain_features['msa_all_seq'] + query_seq = chain_msa[0] + per_seq_similarity = np.sum( + query_seq[None] == chain_msa, axis=-1) / float(len(query_seq)) + per_seq_gap = np.sum(chain_msa == 21, axis=-1) / float(len(query_seq)) + msa_df = pd.DataFrame({ + 'msa_species_identifiers': + chain_features['msa_species_identifiers_all_seq'], + 'msa_row': + np.arange(len(chain_features['msa_species_identifiers_all_seq'])), + 'msa_similarity': + per_seq_similarity, + 'gap': + per_seq_gap, + }) + return msa_df + + +def _create_species_dict(msa_df: pd.DataFrame) -> Dict[bytes, pd.DataFrame]: + """Creates mapping from species to msa dataframe of that species.""" + species_lookup = {} + for species, species_df in msa_df.groupby('msa_species_identifiers'): + species_lookup[species] = species_df + return species_lookup + + +def _match_rows_by_sequence_similarity( + this_species_msa_dfs: List[pd.DataFrame], ) -> List[List[int]]: # noqa + """Finds MSA sequence pairings across chains based on sequence similarity. + + Each chain's MSA sequences are first sorted by their sequence similarity to + their respective target sequence. The sequences are then paired, starting + from the sequences most similar to their target sequence. + + Args: + this_species_msa_dfs: a list of dataframes containing MSA features for + sequences for a specific species. + + Returns: + A list of lists, each containing M indices corresponding to paired MSA rows, + where M is the number of chains. + """ + all_paired_msa_rows = [] + + num_seqs = [ + len(species_df) for species_df in this_species_msa_dfs + if species_df is not None + ] + take_num_seqs = np.min(num_seqs) + + # sort_by_similarity = lambda x: x.sort_values( + # 'msa_similarity', axis=0, ascending=False) + + def sort_by_similarity(x): + return x.sort_values('msa_similarity', axis=0, ascending=False) + + for species_df in this_species_msa_dfs: + if species_df is not None: + species_df_sorted = sort_by_similarity(species_df) + msa_rows = species_df_sorted.msa_row.iloc[:take_num_seqs].values + else: + msa_rows = [-1] * take_num_seqs # take the last 'padding' row + all_paired_msa_rows.append(msa_rows) + all_paired_msa_rows = list(np.array(all_paired_msa_rows).transpose()) + return all_paired_msa_rows + + +def pair_sequences(examples: List[NumpyDict]) -> Dict[int, np.ndarray]: + """Returns indices for paired MSA sequences across chains.""" + + num_examples = len(examples) + + all_chain_species_dict = [] + common_species = set() + for chain_features in examples: + msa_df = _make_msa_df(chain_features) + species_dict = _create_species_dict(msa_df) + all_chain_species_dict.append(species_dict) + common_species.update(set(species_dict)) + + common_species = sorted(common_species) + common_species.remove(b'') # Remove target sequence species. + + all_paired_msa_rows = [np.zeros(len(examples), int)] + all_paired_msa_rows_dict = {k: [] for k in range(num_examples)} + all_paired_msa_rows_dict[num_examples] = [np.zeros(len(examples), int)] + + for species in common_species: + if not species: + continue + this_species_msa_dfs = [] + species_dfs_present = 0 + for species_dict in all_chain_species_dict: + if species in species_dict: + this_species_msa_dfs.append(species_dict[species]) + species_dfs_present += 1 + else: + this_species_msa_dfs.append(None) + + # Skip species that are present in only one chain. + if species_dfs_present <= 1: + continue + + if np.any( + np.array([ + len(species_df) for species_df in this_species_msa_dfs + if isinstance(species_df, pd.DataFrame) + ]) > 600): + continue + + paired_msa_rows = _match_rows_by_sequence_similarity( + this_species_msa_dfs) + all_paired_msa_rows.extend(paired_msa_rows) + all_paired_msa_rows_dict[species_dfs_present].extend(paired_msa_rows) + all_paired_msa_rows_dict = { + num_examples: np.array(paired_msa_rows) + for num_examples, paired_msa_rows in all_paired_msa_rows_dict.items() + } + return all_paired_msa_rows_dict + + +def reorder_paired_rows( + all_paired_msa_rows_dict: Dict[int, np.ndarray]) -> np.ndarray: + """Creates a list of indices of paired MSA rows across chains. + + Args: + all_paired_msa_rows_dict: a mapping from the number of paired chains to the + paired indices. + + Returns: + a list of lists, each containing indices of paired MSA rows across chains. + The paired-index lists are ordered by: + 1) the number of chains in the paired alignment, i.e, all-chain pairings + will come first. + 2) e-values + """ + all_paired_msa_rows = [] + + for num_pairings in sorted(all_paired_msa_rows_dict, reverse=True): + paired_rows = all_paired_msa_rows_dict[num_pairings] + paired_rows_product = np.abs( + np.array( + [np.prod(rows.astype(np.float64)) for rows in paired_rows])) + paired_rows_sort_index = np.argsort(paired_rows_product) + all_paired_msa_rows.extend(paired_rows[paired_rows_sort_index]) + + return np.array(all_paired_msa_rows) + + +def block_diag(*arrs: np.ndarray, pad_value: float = 0.0) -> np.ndarray: + """Like scipy.linalg.block_diag but with an optional padding value.""" + ones_arrs = [np.ones_like(x) for x in arrs] + off_diag_mask = 1 - scipy.linalg.block_diag(*ones_arrs) + diag = scipy.linalg.block_diag(*arrs) + diag += (off_diag_mask * pad_value).astype(diag.dtype) + return diag + + +def _correct_post_merged_feats(np_example: NumpyDict, + np_chains_list: Sequence[NumpyDict], + pair_msa_sequences: bool) -> NumpyDict: + """Adds features that need to be computed/recomputed post merging.""" + + np_example['seq_length'] = np.asarray( + np_example['aatype'].shape[0], dtype=np.int32) + np_example['num_alignments'] = np.asarray( + np_example['msa'].shape[0], dtype=np.int32) + + if not pair_msa_sequences: + # Generate a bias that is 1 for the first row of every block in the + # block diagonal MSA - i.e. make sure the cluster stack always includes + # the query sequences for each chain (since the first row is the query + # sequence). + cluster_bias_masks = [] + for chain in np_chains_list: + mask = np.zeros(chain['msa'].shape[0]) + mask[0] = 1 + cluster_bias_masks.append(mask) + np_example['cluster_bias_mask'] = np.concatenate(cluster_bias_masks) + + # Initialize Bert mask with masked out off diagonals. + msa_masks = [ + np.ones(x['msa'].shape, dtype=np.int8) for x in np_chains_list + ] + + np_example['bert_mask'] = block_diag(*msa_masks, pad_value=0) + else: + np_example['cluster_bias_mask'] = np.zeros(np_example['msa'].shape[0]) + np_example['cluster_bias_mask'][0] = 1 + + # Initialize Bert mask with masked out off diagonals. + msa_masks = [ + np.ones(x['msa'].shape, dtype=np.int8) for x in np_chains_list + ] + msa_masks_all_seq = [ + np.ones(x['msa_all_seq'].shape, dtype=np.int8) + for x in np_chains_list + ] + + msa_mask_block_diag = block_diag(*msa_masks, pad_value=0) + msa_mask_all_seq = np.concatenate(msa_masks_all_seq, axis=1) + np_example['bert_mask'] = np.concatenate( + [msa_mask_all_seq, msa_mask_block_diag], axis=0) + return np_example + + +def _pad_templates(chains: Sequence[NumpyDict], + max_templates: int) -> Sequence[NumpyDict]: + """For each chain pad the number of templates to a fixed size. + + Args: + chains: A list of protein chains. + max_templates: Each chain will be padded to have this many templates. + + Returns: + The list of chains, updated to have template features padded to + max_templates. + """ + for chain in chains: + for k, v in chain.items(): + if k in TEMPLATE_FEATURES: + padding = np.zeros_like(v.shape) + padding[0] = max_templates - v.shape[0] + padding = [(0, p) for p in padding] + chain[k] = np.pad(v, padding, mode='constant') + return chains + + +def _merge_features_from_multiple_chains( + chains: Sequence[NumpyDict], pair_msa_sequences: bool) -> NumpyDict: + """Merge features from multiple chains. + + Args: + chains: A list of feature dictionaries that we want to merge. + pair_msa_sequences: Whether to concatenate MSA features along the + num_res dimension (if True), or to block diagonalize them (if False). + + Returns: + A feature dictionary for the merged example. + """ + merged_example = {} + for feature_name in chains[0]: + feats = [x[feature_name] for x in chains] + feature_name_split = feature_name.split('_all_seq')[0] + if feature_name_split in MSA_FEATURES: + if pair_msa_sequences or '_all_seq' in feature_name: + merged_example[feature_name] = np.concatenate(feats, axis=1) + if feature_name_split == 'msa': + merged_example['msa_chains_all_seq'] = np.ones( + merged_example[feature_name].shape[0]).reshape(-1, 1) + else: + merged_example[feature_name] = block_diag( + *feats, pad_value=MSA_PAD_VALUES[feature_name]) + if feature_name_split == 'msa': + msa_chains = [] + for i, feat in enumerate(feats): + cur_shape = feat.shape[0] + vals = np.ones(cur_shape) * (i + 2) + msa_chains.append(vals) + merged_example['msa_chains'] = np.concatenate( + msa_chains).reshape(-1, 1) + elif feature_name_split in SEQ_FEATURES: + merged_example[feature_name] = np.concatenate(feats, axis=0) + elif feature_name_split in TEMPLATE_FEATURES: + merged_example[feature_name] = np.concatenate(feats, axis=1) + elif feature_name_split in CHAIN_FEATURES: + merged_example[feature_name] = np.sum(feats).astype(np.int32) + else: + merged_example[feature_name] = feats[0] + return merged_example + + +def _merge_homomers_dense_msa( + chains: Iterable[NumpyDict]) -> Sequence[NumpyDict]: + """Merge all identical chains, making the resulting MSA dense. + + Args: + chains: An iterable of features for each chain. + + Returns: + A list of feature dictionaries. All features with the same entity_id + will be merged - MSA features will be concatenated along the num_res + dimension - making them dense. + """ + entity_chains = collections.defaultdict(list) + for chain in chains: + entity_id = chain['entity_id'][0] + entity_chains[entity_id].append(chain) + + grouped_chains = [] + for entity_id in sorted(entity_chains): + chains = entity_chains[entity_id] + grouped_chains.append(chains) + chains = [ + _merge_features_from_multiple_chains(chains, pair_msa_sequences=True) + for chains in grouped_chains + ] + return chains + + +def _concatenate_paired_and_unpaired_features(example: NumpyDict) -> NumpyDict: + """Merges paired and block-diagonalised features.""" + features = MSA_FEATURES + ('msa_chains', ) + for feature_name in features: + if feature_name in example: + feat = example[feature_name] + feat_all_seq = example[feature_name + '_all_seq'] + try: + merged_feat = np.concatenate([feat_all_seq, feat], axis=0) + except Exception as ex: + raise Exception( + 'concat failed.', + feature_name, + feat_all_seq.shape, + feat.shape, + ex.__class__, + ex, + ) + example[feature_name] = merged_feat + example['num_alignments'] = np.array( + example['msa'].shape[0], dtype=np.int32) + return example + + +def merge_chain_features(np_chains_list: List[NumpyDict], + pair_msa_sequences: bool, + max_templates: int) -> NumpyDict: + """Merges features for multiple chains to single FeatureDict. + + Args: + np_chains_list: List of FeatureDicts for each chain. + pair_msa_sequences: Whether to merge paired MSAs. + max_templates: The maximum number of templates to include. + + Returns: + Single FeatureDict for entire complex. + """ + np_chains_list = _pad_templates( + np_chains_list, max_templates=max_templates) + np_chains_list = _merge_homomers_dense_msa(np_chains_list) + # Unpaired MSA features will be always block-diagonalised; paired MSA + # features will be concatenated. + np_example = _merge_features_from_multiple_chains( + np_chains_list, pair_msa_sequences=False) + if pair_msa_sequences: + np_example = _concatenate_paired_and_unpaired_features(np_example) + np_example = _correct_post_merged_feats( + np_example=np_example, + np_chains_list=np_chains_list, + pair_msa_sequences=pair_msa_sequences, + ) + + return np_example + + +def deduplicate_unpaired_sequences( + np_chains: List[NumpyDict]) -> List[NumpyDict]: + """Removes unpaired sequences which duplicate a paired sequence.""" + + feature_names = np_chains[0].keys() + msa_features = MSA_FEATURES + cache_msa_features = {} + for chain in np_chains: + entity_id = int(chain['entity_id'][0]) + if entity_id not in cache_msa_features: + sequence_set = set(s.tobytes() for s in chain['msa_all_seq']) + keep_rows = [] + # Go through unpaired MSA seqs and remove any rows that correspond to the + # sequences that are already present in the paired MSA. + for row_num, seq in enumerate(chain['msa']): + if seq.tobytes() not in sequence_set: + keep_rows.append(row_num) + new_msa_features = {} + for feature_name in feature_names: + if feature_name in msa_features: + if keep_rows: + new_msa_features[feature_name] = chain[feature_name][ + keep_rows] + else: + new_shape = list(chain[feature_name].shape) + new_shape[0] = 0 + new_msa_features[feature_name] = np.zeros( + new_shape, dtype=chain[feature_name].dtype) + cache_msa_features[entity_id] = new_msa_features + for feature_name in cache_msa_features[entity_id]: + chain[feature_name] = cache_msa_features[entity_id][feature_name] + chain['num_alignments'] = np.array( + chain['msa'].shape[0], dtype=np.int32) + return np_chains diff --git a/runtime/Lib/site_packages/models/science/unifold/data/process.py b/runtime/Lib/site_packages/models/science/unifold/data/process.py new file mode 100644 index 0000000000000000000000000000000000000000..3987cb1c8bc2f6f1f70d971b1942cfd806825217 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/process.py @@ -0,0 +1,264 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from typing import Optional + +import numpy as np +import torch + +from modelscope.models.science.unifold.data import data_ops + + +def nonensembled_fns(common_cfg, mode_cfg): + """Input pipeline data transformers that are not ensembled.""" + v2_feature = common_cfg.v2_feature + operators = [] + if mode_cfg.random_delete_msa: + operators.append( + data_ops.random_delete_msa(common_cfg.random_delete_msa)) + operators.extend([ + data_ops.cast_to_64bit_ints, + data_ops.correct_msa_restypes, + data_ops.squeeze_features, + data_ops.randomly_replace_msa_with_unknown(0.0), + data_ops.make_seq_mask, + data_ops.make_msa_mask, + ]) + operators.append(data_ops.make_hhblits_profile_v2 + if v2_feature else data_ops.make_hhblits_profile) + if common_cfg.use_templates: + operators.extend([ + data_ops.make_template_mask, + data_ops.make_pseudo_beta('template_'), + ]) + operators.append( + data_ops.crop_templates( + max_templates=mode_cfg.max_templates, + subsample_templates=mode_cfg.subsample_templates, + )) + + if common_cfg.use_template_torsion_angles: + operators.extend([ + data_ops.atom37_to_torsion_angles('template_'), + ]) + + operators.append(data_ops.make_atom14_masks) + operators.append(data_ops.make_target_feat) + + return operators + + +def crop_and_fix_size_fns(common_cfg, mode_cfg, crop_and_fix_size_seed): + operators = [] + if common_cfg.reduce_msa_clusters_by_max_templates: + pad_msa_clusters = mode_cfg.max_msa_clusters - mode_cfg.max_templates + else: + pad_msa_clusters = mode_cfg.max_msa_clusters + crop_feats = dict(common_cfg.features) + if mode_cfg.fixed_size: + if mode_cfg.crop: + if common_cfg.is_multimer: + crop_fn = data_ops.crop_to_size_multimer( + crop_size=mode_cfg.crop_size, + shape_schema=crop_feats, + seed=crop_and_fix_size_seed, + spatial_crop_prob=mode_cfg.spatial_crop_prob, + ca_ca_threshold=mode_cfg.ca_ca_threshold, + ) + else: + crop_fn = data_ops.crop_to_size_single( + crop_size=mode_cfg.crop_size, + shape_schema=crop_feats, + seed=crop_and_fix_size_seed, + ) + operators.append(crop_fn) + + operators.append(data_ops.select_feat(crop_feats)) + + operators.append( + data_ops.make_fixed_size( + crop_feats, + pad_msa_clusters, + common_cfg.max_extra_msa, + mode_cfg.crop_size, + mode_cfg.max_templates, + )) + return operators + + +def ensembled_fns(common_cfg, mode_cfg): + """Input pipeline data transformers that can be ensembled and averaged.""" + operators = [] + multimer_mode = common_cfg.is_multimer + v2_feature = common_cfg.v2_feature + # multimer don't use block delete msa + if mode_cfg.block_delete_msa and not multimer_mode: + operators.append( + data_ops.block_delete_msa(common_cfg.block_delete_msa)) + if 'max_distillation_msa_clusters' in mode_cfg: + operators.append( + data_ops.sample_msa_distillation( + mode_cfg.max_distillation_msa_clusters)) + + if common_cfg.reduce_msa_clusters_by_max_templates: + pad_msa_clusters = mode_cfg.max_msa_clusters - mode_cfg.max_templates + else: + pad_msa_clusters = mode_cfg.max_msa_clusters + + max_msa_clusters = pad_msa_clusters + max_extra_msa = common_cfg.max_extra_msa + + assert common_cfg.resample_msa_in_recycling + gumbel_sample = common_cfg.gumbel_sample + operators.append( + data_ops.sample_msa( + max_msa_clusters, + keep_extra=True, + gumbel_sample=gumbel_sample, + biased_msa_by_chain=mode_cfg.biased_msa_by_chain, + )) + + if 'masked_msa' in common_cfg: + # Masked MSA should come *before* MSA clustering so that + # the clustering and full MSA profile do not leak information about + # the masked locations and secret corrupted locations. + operators.append( + data_ops.make_masked_msa( + common_cfg.masked_msa, + mode_cfg.masked_msa_replace_fraction, + gumbel_sample=gumbel_sample, + share_mask=mode_cfg.share_mask, + )) + + if common_cfg.msa_cluster_features: + if v2_feature: + operators.append(data_ops.nearest_neighbor_clusters_v2()) + else: + operators.append(data_ops.nearest_neighbor_clusters()) + operators.append(data_ops.summarize_clusters) + + if v2_feature: + operators.append(data_ops.make_msa_feat_v2) + else: + operators.append(data_ops.make_msa_feat) + # Crop after creating the cluster profiles. + if max_extra_msa: + if v2_feature: + operators.append(data_ops.make_extra_msa_feat(max_extra_msa)) + else: + operators.append(data_ops.crop_extra_msa(max_extra_msa)) + else: + operators.append(data_ops.delete_extra_msa) + # operators.append(data_operators.select_feat(common_cfg.recycling_features)) + return operators + + +def process_features(tensors, common_cfg, mode_cfg): + """Based on the config, apply filters and transformations to the data.""" + is_distillation = bool(tensors.get('is_distillation', 0)) + multimer_mode = common_cfg.is_multimer + crop_and_fix_size_seed = int(tensors['crop_and_fix_size_seed']) + crop_fn = crop_and_fix_size_fns( + common_cfg, + mode_cfg, + crop_and_fix_size_seed, + ) + + def wrap_ensemble_fn(data, i): + """Function to be mapped over the ensemble dimension.""" + d = data.copy() + fns = ensembled_fns( + common_cfg, + mode_cfg, + ) + new_d = compose(fns)(d) + if not multimer_mode or is_distillation: + new_d = data_ops.select_feat(common_cfg.recycling_features)(new_d) + return compose(crop_fn)(new_d) + else: # select after crop for spatial cropping + d = compose(crop_fn)(d) + d = data_ops.select_feat(common_cfg.recycling_features)(d) + return d + + nonensembled = nonensembled_fns(common_cfg, mode_cfg) + + if mode_cfg.supervised and (not multimer_mode or is_distillation): + nonensembled.extend(label_transform_fn()) + + tensors = compose(nonensembled)(tensors) + + num_recycling = int(tensors['num_recycling_iters']) + 1 + num_ensembles = mode_cfg.num_ensembles + + ensemble_tensors = map_fn( + lambda x: wrap_ensemble_fn(tensors, x), + torch.arange(num_recycling * num_ensembles), + ) + tensors = compose(crop_fn)(tensors) + # add a dummy dim to align with recycling features + tensors = {k: torch.stack([tensors[k]], dim=0) for k in tensors} + tensors.update(ensemble_tensors) + return tensors + + +@data_ops.curry1 +def compose(x, fs): + for f in fs: + x = f(x) + return x + + +def pad_then_stack(values, ): + if len(values[0].shape) >= 1: + size = max(v.shape[0] for v in values) + new_values = [] + for v in values: + if v.shape[0] < size: + res = values[0].new_zeros(size, *v.shape[1:]) + res[:v.shape[0], ...] = v + else: + res = v + new_values.append(res) + else: + new_values = values + return torch.stack(new_values, dim=0) + + +def map_fn(fun, x): + ensembles = [fun(elem) for elem in x] + features = ensembles[0].keys() + ensembled_dict = {} + for feat in features: + ensembled_dict[feat] = pad_then_stack( + [dict_i[feat] for dict_i in ensembles]) + return ensembled_dict + + +def process_single_label(label: dict, + num_ensemble: Optional[int] = None) -> dict: + assert 'aatype' in label + assert 'all_atom_positions' in label + assert 'all_atom_mask' in label + label = compose(label_transform_fn())(label) + if num_ensemble is not None: + label = { + k: torch.stack([v for _ in range(num_ensemble)]) + for k, v in label.items() + } + return label + + +def process_labels(labels_list, num_ensemble: Optional[int] = None): + return [process_single_label(ll, num_ensemble) for ll in labels_list] + + +def label_transform_fn(): + return [ + data_ops.make_atom14_masks, + data_ops.make_atom14_positions, + data_ops.atom37_to_frames, + data_ops.atom37_to_torsion_angles(''), + data_ops.make_pseudo_beta(''), + data_ops.get_backbone_frames, + data_ops.get_chi_angles, + ] diff --git a/runtime/Lib/site_packages/models/science/unifold/data/process_multimer.py b/runtime/Lib/site_packages/models/science/unifold/data/process_multimer.py new file mode 100644 index 0000000000000000000000000000000000000000..04572d2de6933fd3f370cc979b1f2331475e2998 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/process_multimer.py @@ -0,0 +1,417 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Feature processing logic for multimer data """ + +import collections +from typing import Iterable, List, MutableMapping + +import numpy as np + +from modelscope.models.science.unifold.data import (msa_pairing, + residue_constants) +from .utils import correct_template_restypes + +FeatureDict = MutableMapping[str, np.ndarray] + +REQUIRED_FEATURES = frozenset({ + 'aatype', + 'all_atom_mask', + 'all_atom_positions', + 'all_chains_entity_ids', + 'all_crops_all_chains_mask', + 'all_crops_all_chains_positions', + 'all_crops_all_chains_residue_ids', + 'assembly_num_chains', + 'asym_id', + 'bert_mask', + 'cluster_bias_mask', + 'deletion_matrix', + 'deletion_mean', + 'entity_id', + 'entity_mask', + 'mem_peak', + 'msa', + 'msa_mask', + 'num_alignments', + 'num_templates', + 'queue_size', + 'residue_index', + 'resolution', + 'seq_length', + 'seq_mask', + 'sym_id', + 'template_aatype', + 'template_all_atom_mask', + 'template_all_atom_positions', + # zy added: + 'asym_len', + 'template_sum_probs', + 'num_sym', + 'msa_chains', +}) + +MAX_TEMPLATES = 4 +MSA_CROP_SIZE = 2048 + + +def _is_homomer_or_monomer(chains: Iterable[FeatureDict]) -> bool: + """Checks if a list of chains represents a homomer/monomer example.""" + # Note that an entity_id of 0 indicates padding. + num_unique_chains = len( + np.unique( + np.concatenate([ + np.unique(chain['entity_id'][chain['entity_id'] > 0]) + for chain in chains + ]))) + return num_unique_chains == 1 + + +def pair_and_merge( + all_chain_features: MutableMapping[str, FeatureDict]) -> FeatureDict: + """Runs processing on features to augment, pair and merge. + + Args: + all_chain_features: A MutableMap of dictionaries of features for each chain. + + Returns: + A dictionary of features. + """ + + process_unmerged_features(all_chain_features) + + np_chains_list = all_chain_features + + pair_msa_sequences = not _is_homomer_or_monomer(np_chains_list) + + if pair_msa_sequences: + np_chains_list = msa_pairing.create_paired_features( + chains=np_chains_list) + np_chains_list = msa_pairing.deduplicate_unpaired_sequences( + np_chains_list) + np_chains_list = crop_chains( + np_chains_list, + msa_crop_size=MSA_CROP_SIZE, + pair_msa_sequences=pair_msa_sequences, + max_templates=MAX_TEMPLATES, + ) + np_example = msa_pairing.merge_chain_features( + np_chains_list=np_chains_list, + pair_msa_sequences=pair_msa_sequences, + max_templates=MAX_TEMPLATES, + ) + np_example = process_final(np_example) + return np_example + + +def crop_chains( + chains_list: List[FeatureDict], + msa_crop_size: int, + pair_msa_sequences: bool, + max_templates: int, +) -> List[FeatureDict]: + """Crops the MSAs for a set of chains. + + Args: + chains_list: A list of chains to be cropped. + msa_crop_size: The total number of sequences to crop from the MSA. + pair_msa_sequences: Whether we are operating in sequence-pairing mode. + max_templates: The maximum templates to use per chain. + + Returns: + The chains cropped. + """ + + # Apply the cropping. + cropped_chains = [] + for chain in chains_list: + cropped_chain = _crop_single_chain( + chain, + msa_crop_size=msa_crop_size, + pair_msa_sequences=pair_msa_sequences, + max_templates=max_templates, + ) + cropped_chains.append(cropped_chain) + + return cropped_chains + + +def _crop_single_chain(chain: FeatureDict, msa_crop_size: int, + pair_msa_sequences: bool, + max_templates: int) -> FeatureDict: + """Crops msa sequences to `msa_crop_size`.""" + msa_size = chain['num_alignments'] + + if pair_msa_sequences: + msa_size_all_seq = chain['num_alignments_all_seq'] + msa_crop_size_all_seq = np.minimum(msa_size_all_seq, + msa_crop_size // 2) + + # We reduce the number of un-paired sequences, by the number of times a + # sequence from this chain's MSA is included in the paired MSA. This keeps + # the MSA size for each chain roughly constant. + msa_all_seq = chain['msa_all_seq'][:msa_crop_size_all_seq, :] + num_non_gapped_pairs = np.sum( + np.any(msa_all_seq != msa_pairing.MSA_GAP_IDX, axis=1)) + num_non_gapped_pairs = np.minimum(num_non_gapped_pairs, + msa_crop_size_all_seq) + + # Restrict the unpaired crop size so that paired+unpaired sequences do not + # exceed msa_seqs_per_chain for each chain. + max_msa_crop_size = np.maximum(msa_crop_size - num_non_gapped_pairs, 0) + msa_crop_size = np.minimum(msa_size, max_msa_crop_size) + else: + msa_crop_size = np.minimum(msa_size, msa_crop_size) + + include_templates = 'template_aatype' in chain and max_templates + if include_templates: + num_templates = chain['template_aatype'].shape[0] + templates_crop_size = np.minimum(num_templates, max_templates) + + for k in chain: + k_split = k.split('_all_seq')[0] + if k_split in msa_pairing.TEMPLATE_FEATURES: + chain[k] = chain[k][:templates_crop_size, :] + elif k_split in msa_pairing.MSA_FEATURES: + if '_all_seq' in k and pair_msa_sequences: + chain[k] = chain[k][:msa_crop_size_all_seq, :] + else: + chain[k] = chain[k][:msa_crop_size, :] + + chain['num_alignments'] = np.asarray(msa_crop_size, dtype=np.int32) + if include_templates: + chain['num_templates'] = np.asarray( + templates_crop_size, dtype=np.int32) + if pair_msa_sequences: + chain['num_alignments_all_seq'] = np.asarray( + msa_crop_size_all_seq, dtype=np.int32) + return chain + + +def process_final(np_example: FeatureDict) -> FeatureDict: + """Final processing steps in data pipeline, after merging and pairing.""" + np_example = _make_seq_mask(np_example) + np_example = _make_msa_mask(np_example) + np_example = _filter_features(np_example) + return np_example + + +def _make_seq_mask(np_example): + np_example['seq_mask'] = (np_example['entity_id'] > 0).astype(np.float32) + return np_example + + +def _make_msa_mask(np_example): + """Mask features are all ones, but will later be zero-padded.""" + + np_example['msa_mask'] = np.ones_like(np_example['msa'], dtype=np.int8) + + seq_mask = (np_example['entity_id'] > 0).astype(np.int8) + np_example['msa_mask'] *= seq_mask[None] + + return np_example + + +def _filter_features(np_example: FeatureDict) -> FeatureDict: + """Filters features of example to only those requested.""" + return {k: v for (k, v) in np_example.items() if k in REQUIRED_FEATURES} + + +def process_unmerged_features(all_chain_features: MutableMapping[str, + FeatureDict]): + """Postprocessing stage for per-chain features before merging.""" + num_chains = len(all_chain_features) + for chain_features in all_chain_features: + # Convert deletion matrices to float. + if 'deletion_matrix_int' in chain_features: + chain_features['deletion_matrix'] = np.asarray( + chain_features.pop('deletion_matrix_int'), dtype=np.float32) + if 'deletion_matrix_int_all_seq' in chain_features: + chain_features['deletion_matrix_all_seq'] = np.asarray( + chain_features.pop('deletion_matrix_int_all_seq'), + dtype=np.float32) + + chain_features['deletion_mean'] = np.mean( + chain_features['deletion_matrix'], axis=0) + + if 'all_atom_positions' not in chain_features: + # Add all_atom_mask and dummy all_atom_positions based on aatype. + all_atom_mask = residue_constants.STANDARD_ATOM_MASK[ + chain_features['aatype']] + chain_features['all_atom_mask'] = all_atom_mask + chain_features['all_atom_positions'] = np.zeros( + list(all_atom_mask.shape) + [3]) + + # Add assembly_num_chains. + chain_features['assembly_num_chains'] = np.asarray(num_chains) + + # Add entity_mask. + for chain_features in all_chain_features: + chain_features['entity_mask'] = ( + chain_features['entity_id'] != # noqa W504 + 0).astype(np.int32) + + +def empty_template_feats(n_res): + return { + 'template_aatype': + np.zeros((0, n_res)).astype(np.int64), + 'template_all_atom_positions': + np.zeros((0, n_res, 37, 3)).astype(np.float32), + 'template_sum_probs': + np.zeros((0, 1)).astype(np.float32), + 'template_all_atom_mask': + np.zeros((0, n_res, 37)).astype(np.float32), + } + + +def convert_monomer_features(monomer_features: FeatureDict) -> FeatureDict: + """Reshapes and modifies monomer features for multimer models.""" + if monomer_features['template_aatype'].shape[0] == 0: + monomer_features.update( + empty_template_feats(monomer_features['aatype'].shape[0])) + converted = {} + unnecessary_leading_dim_feats = { + 'sequence', + 'domain_name', + 'num_alignments', + 'seq_length', + } + for feature_name, feature in monomer_features.items(): + if feature_name in unnecessary_leading_dim_feats: + # asarray ensures it's a np.ndarray. + feature = np.asarray(feature[0], dtype=feature.dtype) + elif feature_name == 'aatype': + # The multimer model performs the one-hot operation itself. + feature = np.argmax(feature, axis=-1).astype(np.int32) + elif feature_name == 'template_aatype': + if feature.shape[0] > 0: + feature = correct_template_restypes(feature) + elif feature_name == 'template_all_atom_masks': + feature_name = 'template_all_atom_mask' + elif feature_name == 'msa': + feature = feature.astype(np.uint8) + + if feature_name.endswith('_mask'): + feature = feature.astype(np.float32) + + converted[feature_name] = feature + + if 'deletion_matrix_int' in monomer_features: + monomer_features['deletion_matrix'] = monomer_features.pop( + 'deletion_matrix_int').astype(np.float32) + + converted.pop( + 'template_sum_probs' + ) # zy: this input is checked to be dirty in shape. TODO: figure out why and make it right. + return converted + + +def int_id_to_str_id(num: int) -> str: + """Encodes a number as a string, using reverse spreadsheet style naming. + + Args: + num: A positive integer. + + Returns: + A string that encodes the positive integer using reverse spreadsheet style, + naming e.g. 1 = A, 2 = B, ..., 27 = AA, 28 = BA, 29 = CA, ... This is the + usual way to encode chain IDs in mmCIF files. + """ + if num <= 0: + raise ValueError(f'Only positive integers allowed, got {num}.') + + num = num - 1 # 1-based indexing. + output = [] + while num >= 0: + output.append(chr(num % 26 + ord('A'))) + num = num // 26 - 1 + return ''.join(output) + + +def add_assembly_features(all_chain_features, ): + """Add features to distinguish between chains. + + Args: + all_chain_features: A dictionary which maps chain_id to a dictionary of + features for each chain. + + Returns: + all_chain_features: A dictionary which maps strings of the form + `_` to the corresponding chain features. E.g. two + chains from a homodimer would have keys A_1 and A_2. Two chains from a + heterodimer would have keys A_1 and B_1. + """ + # Group the chains by sequence + seq_to_entity_id = {} + grouped_chains = collections.defaultdict(list) + for chain_features in all_chain_features: + assert 'sequence' in chain_features + seq = str(chain_features['sequence']) + if seq not in seq_to_entity_id: + seq_to_entity_id[seq] = len(seq_to_entity_id) + 1 + grouped_chains[seq_to_entity_id[seq]].append(chain_features) + + new_all_chain_features = [] + chain_id = 1 + for entity_id, group_chain_features in grouped_chains.items(): + num_sym = len(group_chain_features) # zy + for sym_id, chain_features in enumerate(group_chain_features, start=1): + seq_length = chain_features['seq_length'] + chain_features['asym_id'] = chain_id * np.ones(seq_length) + chain_features['sym_id'] = sym_id * np.ones(seq_length) + chain_features['entity_id'] = entity_id * np.ones(seq_length) + chain_features['num_sym'] = num_sym * np.ones(seq_length) + chain_id += 1 + new_all_chain_features.append(chain_features) + + return new_all_chain_features + + +def pad_msa(np_example, min_num_seq): + np_example = dict(np_example) + num_seq = np_example['msa'].shape[0] + if num_seq < min_num_seq: + for feat in ('msa', 'deletion_matrix', 'bert_mask', 'msa_mask', + 'msa_chains'): + np_example[feat] = np.pad(np_example[feat], + ((0, min_num_seq - num_seq), (0, 0))) + np_example['cluster_bias_mask'] = np.pad( + np_example['cluster_bias_mask'], ((0, min_num_seq - num_seq), )) + return np_example + + +def post_process(np_example): + np_example = pad_msa(np_example, 512) + no_dim_keys = [ + 'num_alignments', + 'assembly_num_chains', + 'num_templates', + 'seq_length', + 'resolution', + ] + for k in no_dim_keys: + if k in np_example: + np_example[k] = np_example[k].reshape(-1) + return np_example + + +def merge_msas(msa, del_mat, new_msa, new_del_mat): + cur_msa_set = set([tuple(m) for m in msa]) + new_rows = [] + for i, s in enumerate(new_msa): + if tuple(s) not in cur_msa_set: + new_rows.append(i) + ret_msa = np.concatenate([msa, new_msa[new_rows]], axis=0) + ret_del_mat = np.concatenate([del_mat, new_del_mat[new_rows]], axis=0) + return ret_msa, ret_del_mat diff --git a/runtime/Lib/site_packages/models/science/unifold/data/protein.py b/runtime/Lib/site_packages/models/science/unifold/data/protein.py new file mode 100644 index 0000000000000000000000000000000000000000..42308d0434a15213823c4c337e28dc1449af0a55 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/protein.py @@ -0,0 +1,322 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Protein data type.""" +import dataclasses +import io +from typing import Any, Mapping, Optional + +import numpy as np +from Bio.PDB import PDBParser + +from modelscope.models.science.unifold.data import residue_constants + +FeatureDict = Mapping[str, np.ndarray] +ModelOutput = Mapping[str, Any] # Is a nested dict. + +# Complete sequence of chain IDs supported by the PDB format. +PDB_CHAIN_IDS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' +PDB_MAX_CHAINS = len(PDB_CHAIN_IDS) # := 62. + + +@dataclasses.dataclass(frozen=True) +class Protein: + """Protein structure representation.""" + + # Cartesian coordinates of atoms in angstroms. The atom types correspond to + # residue_constants.atom_types, i.e. the first three are N, CA, CB. + atom_positions: np.ndarray # [num_res, num_atom_type, 3] + + # Amino-acid type for each residue represented as an integer between 0 and + # 20, where 20 is 'X'. + aatype: np.ndarray # [num_res] + + # Binary float mask to indicate presence of a particular atom. 1.0 if an atom + # is present and 0.0 if not. This should be used for loss masking. + atom_mask: np.ndarray # [num_res, num_atom_type] + + # Residue index as used in PDB. It is not necessarily continuous or 0-indexed. + residue_index: np.ndarray # [num_res] + + # 0-indexed number corresponding to the chain in the protein that this residue + # belongs to. + chain_index: np.ndarray # [num_res] + + # B-factors, or temperature factors, of each residue (in sq. angstroms units), + # representing the displacement of the residue from its ground truth mean + # value. + b_factors: np.ndarray # [num_res, num_atom_type] + + def __post_init__(self): + if len(np.unique(self.chain_index)) > PDB_MAX_CHAINS: + raise ValueError( + f'Cannot build an instance with more than {PDB_MAX_CHAINS} chains ' + 'because these cannot be written to PDB format.') + + +def from_pdb_string(pdb_str: str, chain_id: Optional[str] = None) -> Protein: + """Takes a PDB string and constructs a Protein object. + + WARNING: All non-standard residue types will be converted into UNK. All + non-standard atoms will be ignored. + + Args: + pdb_str: The contents of the pdb file + chain_id: If chain_id is specified (e.g. A), then only that chain + is parsed. Otherwise all chains are parsed. + + Returns: + A new `Protein` parsed from the pdb contents. + """ + pdb_fh = io.StringIO(pdb_str) + parser = PDBParser(QUIET=True) + structure = parser.get_structure('none', pdb_fh) + models = list(structure.get_models()) + if len(models) != 1: + raise ValueError( + f'Only single model PDBs are supported. Found {len(models)} models.' + ) + model = models[0] + + atom_positions = [] + aatype = [] + atom_mask = [] + residue_index = [] + chain_ids = [] + b_factors = [] + + for chain in model: + if chain_id is not None and chain.id != chain_id: + continue + for res in chain: + if res.id[2] != ' ': + raise ValueError( + f'PDB contains an insertion code at chain {chain.id} and residue ' + f'index {res.id[1]}. These are not supported.') + res_shortname = residue_constants.restype_3to1.get( + res.resname, 'X') + restype_idx = residue_constants.restype_order.get( + res_shortname, residue_constants.restype_num) + pos = np.zeros((residue_constants.atom_type_num, 3)) + mask = np.zeros((residue_constants.atom_type_num, )) + res_b_factors = np.zeros((residue_constants.atom_type_num, )) + for atom in res: + if atom.name not in residue_constants.atom_types: + continue + pos[residue_constants.atom_order[atom.name]] = atom.coord + mask[residue_constants.atom_order[atom.name]] = 1.0 + res_b_factors[residue_constants.atom_order[ + atom.name]] = atom.bfactor + if np.sum(mask) < 0.5: + # If no known atom positions are reported for the residue then skip it. + continue + aatype.append(restype_idx) + atom_positions.append(pos) + atom_mask.append(mask) + residue_index.append(res.id[1]) + chain_ids.append(chain.id) + b_factors.append(res_b_factors) + + # Chain IDs are usually characters so map these to ints. + unique_chain_ids = np.unique(chain_ids) + chain_id_mapping = {cid: n for n, cid in enumerate(unique_chain_ids)} + chain_index = np.array([chain_id_mapping[cid] for cid in chain_ids]) + + return Protein( + atom_positions=np.array(atom_positions), + atom_mask=np.array(atom_mask), + aatype=np.array(aatype), + residue_index=np.array(residue_index), + chain_index=chain_index, + b_factors=np.array(b_factors), + ) + + +def _chain_end(atom_index, end_resname, chain_name, residue_index) -> str: + chain_end = 'TER' + return (f'{chain_end:<6}{atom_index:>5} {end_resname:>3} ' + f'{chain_name:>1}{residue_index:>4}') + + +def to_pdb(prot: Protein) -> str: + """Converts a `Protein` instance to a PDB string. + + Args: + prot: The protein to convert to PDB. + + Returns: + PDB string. + """ + restypes = residue_constants.restypes + ['X'] + + # res_1to3 = lambda r: residue_constants.restype_1to3.get(restypes[r], 'UNK') + def res_1to3(r): + return residue_constants.restype_1to3.get(restypes[r], 'UNK') + + atom_types = residue_constants.atom_types + + pdb_lines = [] + + atom_mask = prot.atom_mask + aatype = prot.aatype + atom_positions = prot.atom_positions + residue_index = prot.residue_index.astype(np.int32) + chain_index = prot.chain_index.astype(np.int32) + b_factors = prot.b_factors + + if np.any(aatype > residue_constants.restype_num): + raise ValueError('Invalid aatypes.') + + # Construct a mapping from chain integer indices to chain ID strings. + chain_ids = {} + for i in np.unique(chain_index): # np.unique gives sorted output. + if i >= PDB_MAX_CHAINS: + raise ValueError( + f'The PDB format supports at most {PDB_MAX_CHAINS} chains.') + chain_ids[i] = PDB_CHAIN_IDS[i] + + pdb_lines.append('MODEL 1') + atom_index = 1 + last_chain_index = chain_index[0] + # Add all atom sites. + for i in range(aatype.shape[0]): + # Close the previous chain if in a multichain PDB. + if last_chain_index != chain_index[i]: + pdb_lines.append( + _chain_end( + atom_index, + res_1to3(aatype[i - 1]), + chain_ids[chain_index[i - 1]], + residue_index[i - 1], + )) + last_chain_index = chain_index[i] + atom_index += 1 # Atom index increases at the TER symbol. + + res_name_3 = res_1to3(aatype[i]) + for atom_name, pos, mask, b_factor in zip(atom_types, + atom_positions[i], + atom_mask[i], b_factors[i]): + if mask < 0.5: + continue + + record_type = 'ATOM' + name = atom_name if len(atom_name) == 4 else f' {atom_name}' + alt_loc = '' + insertion_code = '' + occupancy = 1.00 + element = atom_name[ + 0] # Protein supports only C, N, O, S, this works. + charge = '' + # PDB is a columnar format, every space matters here! + atom_line = ( + f'{record_type:<6}{atom_index:>5} {name:<4}{alt_loc:>1}' + f'{res_name_3:>3} {chain_ids[chain_index[i]]:>1}' + f'{residue_index[i]:>4}{insertion_code:>1} ' + f'{pos[0]:>8.3f}{pos[1]:>8.3f}{pos[2]:>8.3f}' + f'{occupancy:>6.2f}{b_factor:>6.2f} ' + f'{element:>2}{charge:>2}') + pdb_lines.append(atom_line) + atom_index += 1 + + # Close the final chain. + pdb_lines.append( + _chain_end( + atom_index, + res_1to3(aatype[-1]), + chain_ids[chain_index[-1]], + residue_index[-1], + )) + pdb_lines.append('ENDMDL') + pdb_lines.append('END') + + # Pad all lines to 80 characters. + pdb_lines = [line.ljust(80) for line in pdb_lines] + return '\n'.join(pdb_lines) + '\n' # Add terminating newline. + + +def ideal_atom_mask(prot: Protein) -> np.ndarray: + """Computes an ideal atom mask. + + `Protein.atom_mask` typically is defined according to the atoms that are + reported in the PDB. This function computes a mask according to heavy atoms + that should be present in the given sequence of amino acids. + + Args: + prot: `Protein` whose fields are `numpy.ndarray` objects. + + Returns: + An ideal atom mask. + """ + return residue_constants.STANDARD_ATOM_MASK[prot.aatype] + + +def from_prediction(features: FeatureDict, + result: ModelOutput, + b_factors: Optional[np.ndarray] = None) -> Protein: + """Assembles a protein from a prediction. + + Args: + features: Dictionary holding model inputs. + fold_output: Dictionary holding model outputs. + b_factors: (Optional) B-factors to use for the protein. + + Returns: + A protein instance. + """ + + if 'asym_id' in features: + chain_index = features['asym_id'] - 1 + else: + chain_index = np.zeros_like((features['aatype'])) + + if b_factors is None: + b_factors = np.zeros_like(result['final_atom_mask']) + + return Protein( + aatype=features['aatype'], + atom_positions=result['final_atom_positions'], + atom_mask=result['final_atom_mask'], + residue_index=features['residue_index'] + 1, + chain_index=chain_index, + b_factors=b_factors, + ) + + +def from_feature(features: FeatureDict, + b_factors: Optional[np.ndarray] = None) -> Protein: + """Assembles a standard pdb from input atom positions & mask. + + Args: + features: Dictionary holding model inputs. + b_factors: (Optional) B-factors to use for the protein. + + Returns: + A protein instance. + """ + + if 'asym_id' in features: + chain_index = features['asym_id'] - 1 + else: + chain_index = np.zeros_like((features['aatype'])) + + if b_factors is None: + b_factors = np.zeros_like(features['all_atom_mask']) + + return Protein( + aatype=features['aatype'], + atom_positions=features['all_atom_positions'], + atom_mask=features['all_atom_mask'], + residue_index=features['residue_index'] + 1, + chain_index=chain_index, + b_factors=b_factors, + ) diff --git a/runtime/Lib/site_packages/models/science/unifold/data/residue_constants.py b/runtime/Lib/site_packages/models/science/unifold/data/residue_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..2701ee38e5078d4fcd876c370225ed1cf273b432 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/residue_constants.py @@ -0,0 +1,1212 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Constants used in AlphaFold.""" + +import collections +import functools +import os +from typing import List, Mapping, Tuple + +import numpy as np +from unicore.utils import tree_map + +# Distance from one CA to next CA [trans configuration: omega = 180]. +ca_ca = 3.80209737096 + +# Format: The list for each AA type contains chi1, chi2, chi3, chi4 in +# this order (or a relevant subset from chi1 onwards). ALA and GLY don't have +# chi angles so their chi angle lists are empty. +chi_angles_atoms = { + 'ALA': [], + # Chi5 in arginine is always 0 +- 5 degrees, so ignore it. + 'ARG': [ + ['N', 'CA', 'CB', 'CG'], + ['CA', 'CB', 'CG', 'CD'], + ['CB', 'CG', 'CD', 'NE'], + ['CG', 'CD', 'NE', 'CZ'], + ], + 'ASN': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'OD1']], + 'ASP': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'OD1']], + 'CYS': [['N', 'CA', 'CB', 'SG']], + 'GLN': [ + ['N', 'CA', 'CB', 'CG'], + ['CA', 'CB', 'CG', 'CD'], + ['CB', 'CG', 'CD', 'OE1'], + ], + 'GLU': [ + ['N', 'CA', 'CB', 'CG'], + ['CA', 'CB', 'CG', 'CD'], + ['CB', 'CG', 'CD', 'OE1'], + ], + 'GLY': [], + 'HIS': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'ND1']], + 'ILE': [['N', 'CA', 'CB', 'CG1'], ['CA', 'CB', 'CG1', 'CD1']], + 'LEU': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'CD1']], + 'LYS': [ + ['N', 'CA', 'CB', 'CG'], + ['CA', 'CB', 'CG', 'CD'], + ['CB', 'CG', 'CD', 'CE'], + ['CG', 'CD', 'CE', 'NZ'], + ], + 'MET': [ + ['N', 'CA', 'CB', 'CG'], + ['CA', 'CB', 'CG', 'SD'], + ['CB', 'CG', 'SD', 'CE'], + ], + 'PHE': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'CD1']], + 'PRO': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'CD']], + 'SER': [['N', 'CA', 'CB', 'OG']], + 'THR': [['N', 'CA', 'CB', 'OG1']], + 'TRP': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'CD1']], + 'TYR': [['N', 'CA', 'CB', 'CG'], ['CA', 'CB', 'CG', 'CD1']], + 'VAL': [['N', 'CA', 'CB', 'CG1']], +} + +# If chi angles given in fixed-length array, this matrix determines how to mask +# them for each AA type. The order is as per restype_order (see below). +chi_angles_mask = [ + [0.0, 0.0, 0.0, 0.0], # ALA + [1.0, 1.0, 1.0, 1.0], # ARG + [1.0, 1.0, 0.0, 0.0], # ASN + [1.0, 1.0, 0.0, 0.0], # ASP + [1.0, 0.0, 0.0, 0.0], # CYS + [1.0, 1.0, 1.0, 0.0], # GLN + [1.0, 1.0, 1.0, 0.0], # GLU + [0.0, 0.0, 0.0, 0.0], # GLY + [1.0, 1.0, 0.0, 0.0], # HIS + [1.0, 1.0, 0.0, 0.0], # ILE + [1.0, 1.0, 0.0, 0.0], # LEU + [1.0, 1.0, 1.0, 1.0], # LYS + [1.0, 1.0, 1.0, 0.0], # MET + [1.0, 1.0, 0.0, 0.0], # PHE + [1.0, 1.0, 0.0, 0.0], # PRO + [1.0, 0.0, 0.0, 0.0], # SER + [1.0, 0.0, 0.0, 0.0], # THR + [1.0, 1.0, 0.0, 0.0], # TRP + [1.0, 1.0, 0.0, 0.0], # TYR + [1.0, 0.0, 0.0, 0.0], # VAL +] + +# The following chi angles are pi periodic: they can be rotated by a multiple +# of pi without affecting the structure. +chi_pi_periodic = [ + [0.0, 0.0, 0.0, 0.0], # ALA + [0.0, 0.0, 0.0, 0.0], # ARG + [0.0, 0.0, 0.0, 0.0], # ASN + [0.0, 1.0, 0.0, 0.0], # ASP + [0.0, 0.0, 0.0, 0.0], # CYS + [0.0, 0.0, 0.0, 0.0], # GLN + [0.0, 0.0, 1.0, 0.0], # GLU + [0.0, 0.0, 0.0, 0.0], # GLY + [0.0, 0.0, 0.0, 0.0], # HIS + [0.0, 0.0, 0.0, 0.0], # ILE + [0.0, 0.0, 0.0, 0.0], # LEU + [0.0, 0.0, 0.0, 0.0], # LYS + [0.0, 0.0, 0.0, 0.0], # MET + [0.0, 1.0, 0.0, 0.0], # PHE + [0.0, 0.0, 0.0, 0.0], # PRO + [0.0, 0.0, 0.0, 0.0], # SER + [0.0, 0.0, 0.0, 0.0], # THR + [0.0, 0.0, 0.0, 0.0], # TRP + [0.0, 1.0, 0.0, 0.0], # TYR + [0.0, 0.0, 0.0, 0.0], # VAL + [0.0, 0.0, 0.0, 0.0], # UNK +] + +# Atoms positions relative to the 8 rigid groups, defined by the pre-omega, phi, +# psi and chi angles: +# 0: 'backbone group', +# 1: 'pre-omega-group', (empty) +# 2: 'phi-group', (currently empty, because it defines only hydrogens) +# 3: 'psi-group', +# 4,5,6,7: 'chi1,2,3,4-group' +# The atom positions are relative to the axis-end-atom of the corresponding +# rotation axis. The x-axis is in direction of the rotation axis, and the y-axis +# is defined such that the dihedral-angle-definiting atom (the last entry in +# chi_angles_atoms above) is in the xy-plane (with a positive y-coordinate). +# format: [atomname, group_idx, rel_position] +rigid_group_atom_positions = { + 'ALA': [ + ['N', 0, (-0.525, 1.363, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, -0.000, -0.000)], + ['CB', 0, (-0.529, -0.774, -1.205)], + ['O', 3, (0.627, 1.062, 0.000)], + ], + 'ARG': [ + ['N', 0, (-0.524, 1.362, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, -0.000, -0.000)], + ['CB', 0, (-0.524, -0.778, -1.209)], + ['O', 3, (0.626, 1.062, 0.000)], + ['CG', 4, (0.616, 1.390, -0.000)], + ['CD', 5, (0.564, 1.414, 0.000)], + ['NE', 6, (0.539, 1.357, -0.000)], + ['NH1', 7, (0.206, 2.301, 0.000)], + ['NH2', 7, (2.078, 0.978, -0.000)], + ['CZ', 7, (0.758, 1.093, -0.000)], + ], + 'ASN': [ + ['N', 0, (-0.536, 1.357, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, -0.000, -0.000)], + ['CB', 0, (-0.531, -0.787, -1.200)], + ['O', 3, (0.625, 1.062, 0.000)], + ['CG', 4, (0.584, 1.399, 0.000)], + ['ND2', 5, (0.593, -1.188, 0.001)], + ['OD1', 5, (0.633, 1.059, 0.000)], + ], + 'ASP': [ + ['N', 0, (-0.525, 1.362, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.527, 0.000, -0.000)], + ['CB', 0, (-0.526, -0.778, -1.208)], + ['O', 3, (0.626, 1.062, -0.000)], + ['CG', 4, (0.593, 1.398, -0.000)], + ['OD1', 5, (0.610, 1.091, 0.000)], + ['OD2', 5, (0.592, -1.101, -0.003)], + ], + 'CYS': [ + ['N', 0, (-0.522, 1.362, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.524, 0.000, 0.000)], + ['CB', 0, (-0.519, -0.773, -1.212)], + ['O', 3, (0.625, 1.062, -0.000)], + ['SG', 4, (0.728, 1.653, 0.000)], + ], + 'GLN': [ + ['N', 0, (-0.526, 1.361, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, 0.000, 0.000)], + ['CB', 0, (-0.525, -0.779, -1.207)], + ['O', 3, (0.626, 1.062, -0.000)], + ['CG', 4, (0.615, 1.393, 0.000)], + ['CD', 5, (0.587, 1.399, -0.000)], + ['NE2', 6, (0.593, -1.189, -0.001)], + ['OE1', 6, (0.634, 1.060, 0.000)], + ], + 'GLU': [ + ['N', 0, (-0.528, 1.361, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, -0.000, -0.000)], + ['CB', 0, (-0.526, -0.781, -1.207)], + ['O', 3, (0.626, 1.062, 0.000)], + ['CG', 4, (0.615, 1.392, 0.000)], + ['CD', 5, (0.600, 1.397, 0.000)], + ['OE1', 6, (0.607, 1.095, -0.000)], + ['OE2', 6, (0.589, -1.104, -0.001)], + ], + 'GLY': [ + ['N', 0, (-0.572, 1.337, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.517, -0.000, -0.000)], + ['O', 3, (0.626, 1.062, -0.000)], + ], + 'HIS': [ + ['N', 0, (-0.527, 1.360, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, 0.000, 0.000)], + ['CB', 0, (-0.525, -0.778, -1.208)], + ['O', 3, (0.625, 1.063, 0.000)], + ['CG', 4, (0.600, 1.370, -0.000)], + ['CD2', 5, (0.889, -1.021, 0.003)], + ['ND1', 5, (0.744, 1.160, -0.000)], + ['CE1', 5, (2.030, 0.851, 0.002)], + ['NE2', 5, (2.145, -0.466, 0.004)], + ], + 'ILE': [ + ['N', 0, (-0.493, 1.373, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.527, -0.000, -0.000)], + ['CB', 0, (-0.536, -0.793, -1.213)], + ['O', 3, (0.627, 1.062, -0.000)], + ['CG1', 4, (0.534, 1.437, -0.000)], + ['CG2', 4, (0.540, -0.785, -1.199)], + ['CD1', 5, (0.619, 1.391, 0.000)], + ], + 'LEU': [ + ['N', 0, (-0.520, 1.363, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, -0.000, -0.000)], + ['CB', 0, (-0.522, -0.773, -1.214)], + ['O', 3, (0.625, 1.063, -0.000)], + ['CG', 4, (0.678, 1.371, 0.000)], + ['CD1', 5, (0.530, 1.430, -0.000)], + ['CD2', 5, (0.535, -0.774, 1.200)], + ], + 'LYS': [ + ['N', 0, (-0.526, 1.362, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, 0.000, 0.000)], + ['CB', 0, (-0.524, -0.778, -1.208)], + ['O', 3, (0.626, 1.062, -0.000)], + ['CG', 4, (0.619, 1.390, 0.000)], + ['CD', 5, (0.559, 1.417, 0.000)], + ['CE', 6, (0.560, 1.416, 0.000)], + ['NZ', 7, (0.554, 1.387, 0.000)], + ], + 'MET': [ + ['N', 0, (-0.521, 1.364, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, 0.000, 0.000)], + ['CB', 0, (-0.523, -0.776, -1.210)], + ['O', 3, (0.625, 1.062, -0.000)], + ['CG', 4, (0.613, 1.391, -0.000)], + ['SD', 5, (0.703, 1.695, 0.000)], + ['CE', 6, (0.320, 1.786, -0.000)], + ], + 'PHE': [ + ['N', 0, (-0.518, 1.363, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.524, 0.000, -0.000)], + ['CB', 0, (-0.525, -0.776, -1.212)], + ['O', 3, (0.626, 1.062, -0.000)], + ['CG', 4, (0.607, 1.377, 0.000)], + ['CD1', 5, (0.709, 1.195, -0.000)], + ['CD2', 5, (0.706, -1.196, 0.000)], + ['CE1', 5, (2.102, 1.198, -0.000)], + ['CE2', 5, (2.098, -1.201, -0.000)], + ['CZ', 5, (2.794, -0.003, -0.001)], + ], + 'PRO': [ + ['N', 0, (-0.566, 1.351, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.527, -0.000, 0.000)], + ['CB', 0, (-0.546, -0.611, -1.293)], + ['O', 3, (0.621, 1.066, 0.000)], + ['CG', 4, (0.382, 1.445, 0.0)], + # ['CD', 5, (0.427, 1.440, 0.0)], + ['CD', 5, (0.477, 1.424, 0.0)], # manually made angle 2 degrees larger + ], + 'SER': [ + ['N', 0, (-0.529, 1.360, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, -0.000, -0.000)], + ['CB', 0, (-0.518, -0.777, -1.211)], + ['O', 3, (0.626, 1.062, -0.000)], + ['OG', 4, (0.503, 1.325, 0.000)], + ], + 'THR': [ + ['N', 0, (-0.517, 1.364, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.526, 0.000, -0.000)], + ['CB', 0, (-0.516, -0.793, -1.215)], + ['O', 3, (0.626, 1.062, 0.000)], + ['CG2', 4, (0.550, -0.718, -1.228)], + ['OG1', 4, (0.472, 1.353, 0.000)], + ], + 'TRP': [ + ['N', 0, (-0.521, 1.363, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.525, -0.000, 0.000)], + ['CB', 0, (-0.523, -0.776, -1.212)], + ['O', 3, (0.627, 1.062, 0.000)], + ['CG', 4, (0.609, 1.370, -0.000)], + ['CD1', 5, (0.824, 1.091, 0.000)], + ['CD2', 5, (0.854, -1.148, -0.005)], + ['CE2', 5, (2.186, -0.678, -0.007)], + ['CE3', 5, (0.622, -2.530, -0.007)], + ['NE1', 5, (2.140, 0.690, -0.004)], + ['CH2', 5, (3.028, -2.890, -0.013)], + ['CZ2', 5, (3.283, -1.543, -0.011)], + ['CZ3', 5, (1.715, -3.389, -0.011)], + ], + 'TYR': [ + ['N', 0, (-0.522, 1.362, 0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.524, -0.000, -0.000)], + ['CB', 0, (-0.522, -0.776, -1.213)], + ['O', 3, (0.627, 1.062, -0.000)], + ['CG', 4, (0.607, 1.382, -0.000)], + ['CD1', 5, (0.716, 1.195, -0.000)], + ['CD2', 5, (0.713, -1.194, -0.001)], + ['CE1', 5, (2.107, 1.200, -0.002)], + ['CE2', 5, (2.104, -1.201, -0.003)], + ['OH', 5, (4.168, -0.002, -0.005)], + ['CZ', 5, (2.791, -0.001, -0.003)], + ], + 'VAL': [ + ['N', 0, (-0.494, 1.373, -0.000)], + ['CA', 0, (0.000, 0.000, 0.000)], + ['C', 0, (1.527, -0.000, -0.000)], + ['CB', 0, (-0.533, -0.795, -1.213)], + ['O', 3, (0.627, 1.062, -0.000)], + ['CG1', 4, (0.540, 1.429, -0.000)], + ['CG2', 4, (0.533, -0.776, 1.203)], + ], +} + +# A list of atoms (excluding hydrogen) for each AA type. PDB naming convention. +residue_atoms = { + 'ALA': ['C', 'CA', 'CB', 'N', 'O'], + 'ARG': ['C', 'CA', 'CB', 'CG', 'CD', 'CZ', 'N', 'NE', 'O', 'NH1', 'NH2'], + 'ASP': ['C', 'CA', 'CB', 'CG', 'N', 'O', 'OD1', 'OD2'], + 'ASN': ['C', 'CA', 'CB', 'CG', 'N', 'ND2', 'O', 'OD1'], + 'CYS': ['C', 'CA', 'CB', 'N', 'O', 'SG'], + 'GLU': ['C', 'CA', 'CB', 'CG', 'CD', 'N', 'O', 'OE1', 'OE2'], + 'GLN': ['C', 'CA', 'CB', 'CG', 'CD', 'N', 'NE2', 'O', 'OE1'], + 'GLY': ['C', 'CA', 'N', 'O'], + 'HIS': ['C', 'CA', 'CB', 'CG', 'CD2', 'CE1', 'N', 'ND1', 'NE2', 'O'], + 'ILE': ['C', 'CA', 'CB', 'CG1', 'CG2', 'CD1', 'N', 'O'], + 'LEU': ['C', 'CA', 'CB', 'CG', 'CD1', 'CD2', 'N', 'O'], + 'LYS': ['C', 'CA', 'CB', 'CG', 'CD', 'CE', 'N', 'NZ', 'O'], + 'MET': ['C', 'CA', 'CB', 'CG', 'CE', 'N', 'O', 'SD'], + 'PHE': ['C', 'CA', 'CB', 'CG', 'CD1', 'CD2', 'CE1', 'CE2', 'CZ', 'N', 'O'], + 'PRO': ['C', 'CA', 'CB', 'CG', 'CD', 'N', 'O'], + 'SER': ['C', 'CA', 'CB', 'N', 'O', 'OG'], + 'THR': ['C', 'CA', 'CB', 'CG2', 'N', 'O', 'OG1'], + 'TRP': [ + 'C', + 'CA', + 'CB', + 'CG', + 'CD1', + 'CD2', + 'CE2', + 'CE3', + 'CZ2', + 'CZ3', + 'CH2', + 'N', + 'NE1', + 'O', + ], + 'TYR': + ['C', 'CA', 'CB', 'CG', 'CD1', 'CD2', 'CE1', 'CE2', 'CZ', 'N', 'O', 'OH'], + 'VAL': ['C', 'CA', 'CB', 'CG1', 'CG2', 'N', 'O'], +} + +# Naming swaps for ambiguous atom names. +# Due to symmetries in the amino acids the naming of atoms is ambiguous in +# 4 of the 20 amino acids. +# (The LDDT paper lists 7 amino acids as ambiguous, but the naming ambiguities +# in LEU, VAL and ARG can be resolved by using the 3d constellations of +# the 'ambiguous' atoms and their neighbours) +residue_atom_renaming_swaps = { + 'ASP': { + 'OD1': 'OD2' + }, + 'GLU': { + 'OE1': 'OE2' + }, + 'PHE': { + 'CD1': 'CD2', + 'CE1': 'CE2' + }, + 'TYR': { + 'CD1': 'CD2', + 'CE1': 'CE2' + }, +} + +# Van der Waals radii [Angstroem] of the atoms (from Wikipedia) +van_der_waals_radius = { + 'C': 1.7, + 'N': 1.55, + 'O': 1.52, + 'S': 1.8, +} + +Bond = collections.namedtuple('Bond', + ['atom1_name', 'atom2_name', 'length', 'stddev']) +BondAngle = collections.namedtuple( + 'BondAngle', + ['atom1_name', 'atom2_name', 'atom3name', 'angle_rad', 'stddev']) + + +@functools.lru_cache(maxsize=None) +# def load_stereo_chemical_props() -> Tuple[Mapping[str, List[Bond]], Mapping[ #noqa +# str, List[Bond]], Mapping[str, List[BondAngle]]]: +def load_stereo_chemical_props(): + """Load stereo_chemical_props.txt into a nice structure. + + Load literature values for bond lengths and bond angles and translate + bond angles into the length of the opposite edge of the triangle + ("residue_virtual_bonds"). + + Returns: + residue_bonds: Dict that maps resname -> list of Bond tuples. + residue_virtual_bonds: Dict that maps resname -> list of Bond tuples. + residue_bond_angles: Dict that maps resname -> list of BondAngle tuples. + """ + stereo_chemical_props_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'stereo_chemical_props.txt') + with open(stereo_chemical_props_path, 'rt', encoding='utf-8') as f: + stereo_chemical_props = f.read() + lines_iter = iter(stereo_chemical_props.splitlines()) + # Load bond lengths. + residue_bonds = {} + next(lines_iter) # Skip header line. + for line in lines_iter: + if line.strip() == '-': + break + bond, resname, length, stddev = line.split() + atom1, atom2 = bond.split('-') + if resname not in residue_bonds: + residue_bonds[resname] = [] + residue_bonds[resname].append( + Bond(atom1, atom2, float(length), float(stddev))) + residue_bonds['UNK'] = [] + + # Load bond angles. + residue_bond_angles = {} + next(lines_iter) # Skip empty line. + next(lines_iter) # Skip header line. + for line in lines_iter: + if line.strip() == '-': + break + bond, resname, angle_degree, stddev_degree = line.split() + atom1, atom2, atom3 = bond.split('-') + if resname not in residue_bond_angles: + residue_bond_angles[resname] = [] + residue_bond_angles[resname].append( + BondAngle( + atom1, + atom2, + atom3, + float(angle_degree) / 180.0 * np.pi, + float(stddev_degree) / 180.0 * np.pi, + )) + residue_bond_angles['UNK'] = [] + + def make_bond_key(atom1_name, atom2_name): + """Unique key to lookup bonds.""" + return '-'.join(sorted([atom1_name, atom2_name])) + + # Translate bond angles into distances ("virtual bonds"). + residue_virtual_bonds = {} + for resname, bond_angles in residue_bond_angles.items(): + # Create a fast lookup dict for bond lengths. + bond_cache = {} + for b in residue_bonds[resname]: + bond_cache[make_bond_key(b.atom1_name, b.atom2_name)] = b + residue_virtual_bonds[resname] = [] + for ba in bond_angles: + bond1 = bond_cache[make_bond_key(ba.atom1_name, ba.atom2_name)] + bond2 = bond_cache[make_bond_key(ba.atom2_name, ba.atom3name)] + + # Compute distance between atom1 and atom3 using the law of cosines + # c^2 = a^2 + b^2 - 2ab*cos(gamma). + gamma = ba.angle_rad + length = np.sqrt(bond1.length**2 + bond2.length**2 + - 2 * bond1.length * bond2.length * np.cos(gamma)) + + # Propagation of uncertainty assuming uncorrelated errors. + dl_outer = 0.5 / length + dl_dgamma = (2 * bond1.length * bond2.length + * np.sin(gamma)) * dl_outer + dl_db1 = (2 * bond1.length + - 2 * bond2.length * np.cos(gamma)) * dl_outer + dl_db2 = (2 * bond2.length + - 2 * bond1.length * np.cos(gamma)) * dl_outer + stddev = np.sqrt((dl_dgamma * ba.stddev)**2 + + (dl_db1 * bond1.stddev)**2 + + (dl_db2 * bond2.stddev)**2) + residue_virtual_bonds[resname].append( + Bond(ba.atom1_name, ba.atom3name, length, stddev)) + + return (residue_bonds, residue_virtual_bonds, residue_bond_angles) + + +# Between-residue bond lengths for general bonds (first element) and for Proline +# (second element). +between_res_bond_length_c_n = [1.329, 1.341] +between_res_bond_length_stddev_c_n = [0.014, 0.016] + +# Between-residue cos_angles. +between_res_cos_angles_c_n_ca = [-0.5203, 0.0353] # degrees: 121.352 +- 2.315 +between_res_cos_angles_ca_c_n = [-0.4473, 0.0311] # degrees: 116.568 +- 1.995 + +# This mapping is used when we need to store atom data in a format that requires +# fixed atom data size for every residue (e.g. a numpy array). +atom_types = [ + 'N', + 'CA', + 'C', + 'CB', + 'O', + 'CG', + 'CG1', + 'CG2', + 'OG', + 'OG1', + 'SG', + 'CD', + 'CD1', + 'CD2', + 'ND1', + 'ND2', + 'OD1', + 'OD2', + 'SD', + 'CE', + 'CE1', + 'CE2', + 'CE3', + 'NE', + 'NE1', + 'NE2', + 'OE1', + 'OE2', + 'CH2', + 'NH1', + 'NH2', + 'OH', + 'CZ', + 'CZ2', + 'CZ3', + 'NZ', + 'OXT', +] +atom_order = {atom_type: i for i, atom_type in enumerate(atom_types)} +atom_type_num = len(atom_types) # := 37. + +# A compact atom encoding with 14 columns +# pylint: disable=line-too-long +# pylint: disable=bad-whitespace +restype_name_to_atom14_names = { + 'ALA': ['N', 'CA', 'C', 'O', 'CB', '', '', '', '', '', '', '', '', ''], + 'ARG': [ + 'N', + 'CA', + 'C', + 'O', + 'CB', + 'CG', + 'CD', + 'NE', + 'CZ', + 'NH1', + 'NH2', + '', + '', + '', + ], + 'ASN': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'OD1', 'ND2', '', '', '', '', '', ''], + 'ASP': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'OD1', 'OD2', '', '', '', '', '', ''], + 'CYS': ['N', 'CA', 'C', 'O', 'CB', 'SG', '', '', '', '', '', '', '', ''], + 'GLN': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'CD', 'OE1', 'NE2', '', '', '', '', ''], + 'GLU': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'CD', 'OE1', 'OE2', '', '', '', '', ''], + 'GLY': ['N', 'CA', 'C', 'O', '', '', '', '', '', '', '', '', '', ''], + 'HIS': [ + 'N', + 'CA', + 'C', + 'O', + 'CB', + 'CG', + 'ND1', + 'CD2', + 'CE1', + 'NE2', + '', + '', + '', + '', + ], + 'ILE': + ['N', 'CA', 'C', 'O', 'CB', 'CG1', 'CG2', 'CD1', '', '', '', '', '', ''], + 'LEU': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'CD1', 'CD2', '', '', '', '', '', ''], + 'LYS': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'CD', 'CE', 'NZ', '', '', '', '', ''], + 'MET': + ['N', 'CA', 'C', 'O', 'CB', 'CG', 'SD', 'CE', '', '', '', '', '', ''], + 'PHE': [ + 'N', + 'CA', + 'C', + 'O', + 'CB', + 'CG', + 'CD1', + 'CD2', + 'CE1', + 'CE2', + 'CZ', + '', + '', + '', + ], + 'PRO': ['N', 'CA', 'C', 'O', 'CB', 'CG', 'CD', '', '', '', '', '', '', ''], + 'SER': ['N', 'CA', 'C', 'O', 'CB', 'OG', '', '', '', '', '', '', '', ''], + 'THR': + ['N', 'CA', 'C', 'O', 'CB', 'OG1', 'CG2', '', '', '', '', '', '', ''], + 'TRP': [ + 'N', + 'CA', + 'C', + 'O', + 'CB', + 'CG', + 'CD1', + 'CD2', + 'NE1', + 'CE2', + 'CE3', + 'CZ2', + 'CZ3', + 'CH2', + ], + 'TYR': [ + 'N', + 'CA', + 'C', + 'O', + 'CB', + 'CG', + 'CD1', + 'CD2', + 'CE1', + 'CE2', + 'CZ', + 'OH', + '', + '', + ], + 'VAL': + ['N', 'CA', 'C', 'O', 'CB', 'CG1', 'CG2', '', '', '', '', '', '', ''], + 'UNK': ['', '', '', '', '', '', '', '', '', '', '', '', '', ''], +} +# pylint: enable=line-too-long +# pylint: enable=bad-whitespace + +# This is the standard residue order when coding AA type as a number. +# Reproduce it by taking 3-letter AA codes and sorting them alphabetically. +restypes = [ + 'A', + 'R', + 'N', + 'D', + 'C', + 'Q', + 'E', + 'G', + 'H', + 'I', + 'L', + 'K', + 'M', + 'F', + 'P', + 'S', + 'T', + 'W', + 'Y', + 'V', +] +restype_order = {restype: i for i, restype in enumerate(restypes)} +restype_num = len(restypes) # := 20. +unk_restype_index = restype_num # Catch-all index for unknown restypes. + +restypes_with_x = restypes + ['X'] +restype_order_with_x = { + restype: i + for i, restype in enumerate(restypes_with_x) +} + + +def sequence_to_onehot(sequence: str, + mapping: Mapping[str, int], + map_unknown_to_x: bool = False) -> np.ndarray: + """Maps the given sequence into a one-hot encoded matrix. + + Args: + sequence: An amino acid sequence. + mapping: A dictionary mapping amino acids to integers. + map_unknown_to_x: If True, any amino acid that is not in the mapping will be + mapped to the unknown amino acid 'X'. If the mapping doesn't contain + amino acid 'X', an error will be thrown. If False, any amino acid not in + the mapping will throw an error. + + Returns: + A numpy array of shape (seq_len, num_unique_aas) with one-hot encoding of + the sequence. + + Raises: + ValueError: If the mapping doesn't contain values from 0 to + num_unique_aas - 1 without any gaps. + """ + num_entries = max(mapping.values()) + 1 + + if sorted(set(mapping.values())) != list(range(num_entries)): + raise ValueError( + 'The mapping must have values from 0 to num_unique_aas-1 ' + 'without any gaps. Got: %s' % sorted(mapping.values())) + + one_hot_arr = np.zeros((len(sequence), num_entries), dtype=np.int32) + + for aa_index, aa_type in enumerate(sequence): + if map_unknown_to_x: + if aa_type.isalpha() and aa_type.isupper(): + aa_id = mapping.get(aa_type, mapping['X']) + else: + raise ValueError( + f'Invalid character in the sequence: {aa_type}') + else: + aa_id = mapping[aa_type] + one_hot_arr[aa_index, aa_id] = 1 + + return one_hot_arr + + +restype_1to3 = { + 'A': 'ALA', + 'R': 'ARG', + 'N': 'ASN', + 'D': 'ASP', + 'C': 'CYS', + 'Q': 'GLN', + 'E': 'GLU', + 'G': 'GLY', + 'H': 'HIS', + 'I': 'ILE', + 'L': 'LEU', + 'K': 'LYS', + 'M': 'MET', + 'F': 'PHE', + 'P': 'PRO', + 'S': 'SER', + 'T': 'THR', + 'W': 'TRP', + 'Y': 'TYR', + 'V': 'VAL', +} + +# NB: restype_3to1 differs from Bio.PDB.protein_letters_3to1 by being a simple +# 1-to-1 mapping of 3 letter names to one letter names. The latter contains +# many more, and less common, three letter names as keys and maps many of these +# to the same one letter name (including 'X' and 'U' which we don't use here). +restype_3to1 = {v: k for k, v in restype_1to3.items()} + +# Define a restype name for all unknown residues. +unk_restype = 'UNK' + +resnames = [restype_1to3[r] for r in restypes] + [unk_restype] +resname_to_idx = {resname: i for i, resname in enumerate(resnames)} + +# The mapping here uses hhblits convention, so that B is mapped to D, J and O +# are mapped to X, U is mapped to C, and Z is mapped to E. Other than that the +# remaining 20 amino acids are kept in alphabetical order. +# There are 2 non-amino acid codes, X (representing any amino acid) and +# "-" representing a missing amino acid in an alignment. The id for these +# codes is put at the end (20 and 21) so that they can easily be ignored if +# desired. +HHBLITS_AA_TO_ID = { + 'A': 0, + 'B': 2, + 'C': 1, + 'D': 2, + 'E': 3, + 'F': 4, + 'G': 5, + 'H': 6, + 'I': 7, + 'J': 20, + 'K': 8, + 'L': 9, + 'M': 10, + 'N': 11, + 'O': 20, + 'P': 12, + 'Q': 13, + 'R': 14, + 'S': 15, + 'T': 16, + 'U': 1, + 'V': 17, + 'W': 18, + 'X': 20, + 'Y': 19, + 'Z': 3, + '-': 21, +} + +# Partial inversion of HHBLITS_AA_TO_ID. +ID_TO_HHBLITS_AA = { + 0: 'A', + 1: 'C', # Also U. + 2: 'D', # Also B. + 3: 'E', # Also Z. + 4: 'F', + 5: 'G', + 6: 'H', + 7: 'I', + 8: 'K', + 9: 'L', + 10: 'M', + 11: 'N', + 12: 'P', + 13: 'Q', + 14: 'R', + 15: 'S', + 16: 'T', + 17: 'V', + 18: 'W', + 19: 'Y', + 20: 'X', # Includes J and O. + 21: '-', +} + +restypes_with_x_and_gap = restypes + ['X', '-'] +MAP_HHBLITS_AATYPE_TO_OUR_AATYPE = tuple( + restypes_with_x_and_gap.index(ID_TO_HHBLITS_AA[i]) + for i in range(len(restypes_with_x_and_gap))) + + +def _make_standard_atom_mask() -> np.ndarray: + """Returns [num_res_types, num_atom_types] mask array.""" + # +1 to account for unknown (all 0s). + mask = np.zeros([restype_num + 1, atom_type_num], dtype=np.int32) + for restype, restype_letter in enumerate(restypes): + restype_name = restype_1to3[restype_letter] + atom_names = residue_atoms[restype_name] + for atom_name in atom_names: + atom_type = atom_order[atom_name] + mask[restype, atom_type] = 1 + return mask + + +STANDARD_ATOM_MASK = _make_standard_atom_mask() + + +# A one hot representation for the first and second atoms defining the axis +# of rotation for each chi-angle in each residue. +def chi_angle_atom(atom_index: int) -> np.ndarray: + """Define chi-angle rigid groups via one-hot representations.""" + chi_angles_index = {} + one_hots = [] + + for k, v in chi_angles_atoms.items(): + indices = [atom_types.index(s[atom_index]) for s in v] + indices.extend([-1] * (4 - len(indices))) + chi_angles_index[k] = indices + + for r in restypes: + res3 = restype_1to3[r] + one_hot = np.eye(atom_type_num)[chi_angles_index[res3]] + one_hots.append(one_hot) + + one_hots.append(np.zeros([4, atom_type_num])) # Add zeros for residue `X`. + one_hot = np.stack(one_hots, axis=0) + one_hot = np.transpose(one_hot, [0, 2, 1]) + + return one_hot + + +chi_atom_1_one_hot = chi_angle_atom(1) +chi_atom_2_one_hot = chi_angle_atom(2) + +# An array like chi_angles_atoms but using indices rather than names. +chi_angles_atom_indices = [chi_angles_atoms[restype_1to3[r]] for r in restypes] +chi_angles_atom_indices = tree_map( + lambda n: atom_order[n], chi_angles_atom_indices, leaf_type=str) +chi_angles_atom_indices = np.array([ + chi_atoms + ([[0, 0, 0, 0]] * (4 - len(chi_atoms))) + for chi_atoms in chi_angles_atom_indices +]) + +# Mapping from (res_name, atom_name) pairs to the atom's chi group index +# and atom index within that group. +chi_groups_for_atom = collections.defaultdict(list) +for res_name, chi_angle_atoms_for_res in chi_angles_atoms.items(): + for chi_group_i, chi_group in enumerate(chi_angle_atoms_for_res): + for atom_i, atom in enumerate(chi_group): + chi_groups_for_atom[(res_name, atom)].append((chi_group_i, atom_i)) +chi_groups_for_atom = dict(chi_groups_for_atom) + + +def _make_rigid_transformation_4x4(ex, ey, translation): + """Create a rigid 4x4 transformation matrix from two axes and transl.""" + # Normalize ex. + ex_normalized = ex / np.linalg.norm(ex) + + # make ey perpendicular to ex + ey_normalized = ey - np.dot(ey, ex_normalized) * ex_normalized + ey_normalized /= np.linalg.norm(ey_normalized) + + # compute ez as cross product + eznorm = np.cross(ex_normalized, ey_normalized) + m = np.stack([ex_normalized, ey_normalized, eznorm, + translation]).transpose() + m = np.concatenate([m, [[0.0, 0.0, 0.0, 1.0]]], axis=0) + return m + + +# create an array with (restype, atomtype) --> rigid_group_idx +# and an array with (restype, atomtype, coord) for the atom positions +# and compute affine transformation matrices (4,4) from one rigid group to the +# previous group +restype_atom37_to_rigid_group = np.zeros([21, 37], dtype=np.int_) +restype_atom37_mask = np.zeros([21, 37], dtype=np.float32) +restype_atom37_rigid_group_positions = np.zeros([21, 37, 3], dtype=np.float32) +restype_atom14_to_rigid_group = np.zeros([21, 14], dtype=np.int_) +restype_atom14_mask = np.zeros([21, 14], dtype=np.float32) +restype_atom14_rigid_group_positions = np.zeros([21, 14, 3], dtype=np.float32) +restype_rigid_group_default_frame = np.zeros([21, 8, 4, 4], dtype=np.float32) + + +def _make_rigid_group_constants(): + """Fill the arrays above.""" + for restype, restype_letter in enumerate(restypes): + resname = restype_1to3[restype_letter] + for atomname, group_idx, atom_position in rigid_group_atom_positions[ + resname]: + atomtype = atom_order[atomname] + restype_atom37_to_rigid_group[restype, atomtype] = group_idx + restype_atom37_mask[restype, atomtype] = 1 + restype_atom37_rigid_group_positions[restype, + atomtype, :] = atom_position + + atom14idx = restype_name_to_atom14_names[resname].index(atomname) + restype_atom14_to_rigid_group[restype, atom14idx] = group_idx + restype_atom14_mask[restype, atom14idx] = 1 + restype_atom14_rigid_group_positions[restype, + atom14idx, :] = atom_position + + for restype, restype_letter in enumerate(restypes): + resname = restype_1to3[restype_letter] + atom_positions = { + name: np.array(pos) + for name, _, pos in rigid_group_atom_positions[resname] + } + + # backbone to backbone is the identity transform + restype_rigid_group_default_frame[restype, 0, :, :] = np.eye(4) + + # pre-omega-frame to backbone (currently dummy identity matrix) + restype_rigid_group_default_frame[restype, 1, :, :] = np.eye(4) + + # phi-frame to backbone + mat = _make_rigid_transformation_4x4( + ex=atom_positions['N'] - atom_positions['CA'], + ey=np.array([1.0, 0.0, 0.0]), + translation=atom_positions['N'], + ) + restype_rigid_group_default_frame[restype, 2, :, :] = mat + + # psi-frame to backbone + mat = _make_rigid_transformation_4x4( + ex=atom_positions['C'] - atom_positions['CA'], + ey=atom_positions['CA'] - atom_positions['N'], + translation=atom_positions['C'], + ) + restype_rigid_group_default_frame[restype, 3, :, :] = mat + + # chi1-frame to backbone + if chi_angles_mask[restype][0]: + base_atom_names = chi_angles_atoms[resname][0] + base_atom_positions = [ + atom_positions[name] for name in base_atom_names + ] + mat = _make_rigid_transformation_4x4( + ex=base_atom_positions[2] - base_atom_positions[1], + ey=base_atom_positions[0] - base_atom_positions[1], + translation=base_atom_positions[2], + ) + restype_rigid_group_default_frame[restype, 4, :, :] = mat + + # chi2-frame to chi1-frame + # chi3-frame to chi2-frame + # chi4-frame to chi3-frame + # luckily all rotation axes for the next frame start at (0,0,0) of the + # previous frame + for chi_idx in range(1, 4): + if chi_angles_mask[restype][chi_idx]: + axis_end_atom_name = chi_angles_atoms[resname][chi_idx][2] + axis_end_atom_position = atom_positions[axis_end_atom_name] + mat = _make_rigid_transformation_4x4( + ex=axis_end_atom_position, + ey=np.array([-1.0, 0.0, 0.0]), + translation=axis_end_atom_position, + ) + restype_rigid_group_default_frame[restype, + 4 + chi_idx, :, :] = mat + + +_make_rigid_group_constants() + + +def make_atom14_dists_bounds(overlap_tolerance=1.5, + bond_length_tolerance_factor=15): + """compute upper and lower bounds for bonds to assess violations.""" + restype_atom14_bond_lower_bound = np.zeros([21, 14, 14], np.float32) + restype_atom14_bond_upper_bound = np.zeros([21, 14, 14], np.float32) + restype_atom14_bond_stddev = np.zeros([21, 14, 14], np.float32) + residue_bonds, residue_virtual_bonds, _ = load_stereo_chemical_props() + for restype, restype_letter in enumerate(restypes): + resname = restype_1to3[restype_letter] + atom_list = restype_name_to_atom14_names[resname] + + # create lower and upper bounds for clashes + for atom1_idx, atom1_name in enumerate(atom_list): + if not atom1_name: + continue + atom1_radius = van_der_waals_radius[atom1_name[0]] + for atom2_idx, atom2_name in enumerate(atom_list): + if (not atom2_name) or atom1_idx == atom2_idx: + continue + atom2_radius = van_der_waals_radius[atom2_name[0]] + lower = atom1_radius + atom2_radius - overlap_tolerance + upper = 1e10 + restype_atom14_bond_lower_bound[restype, atom1_idx, + atom2_idx] = lower + restype_atom14_bond_lower_bound[restype, atom2_idx, + atom1_idx] = lower + restype_atom14_bond_upper_bound[restype, atom1_idx, + atom2_idx] = upper + restype_atom14_bond_upper_bound[restype, atom2_idx, + atom1_idx] = upper + + # overwrite lower and upper bounds for bonds and angles + for b in residue_bonds[resname] + residue_virtual_bonds[resname]: + atom1_idx = atom_list.index(b.atom1_name) + atom2_idx = atom_list.index(b.atom2_name) + lower = b.length - bond_length_tolerance_factor * b.stddev + upper = b.length + bond_length_tolerance_factor * b.stddev + restype_atom14_bond_lower_bound[restype, atom1_idx, + atom2_idx] = lower + restype_atom14_bond_lower_bound[restype, atom2_idx, + atom1_idx] = lower + restype_atom14_bond_upper_bound[restype, atom1_idx, + atom2_idx] = upper + restype_atom14_bond_upper_bound[restype, atom2_idx, + atom1_idx] = upper + restype_atom14_bond_stddev[restype, atom1_idx, + atom2_idx] = b.stddev + restype_atom14_bond_stddev[restype, atom2_idx, + atom1_idx] = b.stddev + return { + 'lower_bound': restype_atom14_bond_lower_bound, # shape (21,14,14) + 'upper_bound': restype_atom14_bond_upper_bound, # shape (21,14,14) + 'stddev': restype_atom14_bond_stddev, # shape (21,14,14) + } + + +def _make_atom14_and_atom37_constants(): + restype_atom14_to_atom37 = [] + restype_atom37_to_atom14 = [] + restype_atom14_mask = [] + + for rt in restypes: + atom_names = restype_name_to_atom14_names[restype_1to3[rt]] + restype_atom14_to_atom37.append([(atom_order[name] if name else 0) + for name in atom_names]) + atom_name_to_idx14 = {name: i for i, name in enumerate(atom_names)} + restype_atom37_to_atom14.append([ + (atom_name_to_idx14[name] if name in atom_name_to_idx14 else 0) + for name in atom_types + ]) + + restype_atom14_mask.append([(1.0 if name else 0.0) + for name in atom_names]) + + # Add dummy mapping for restype 'UNK' + restype_atom14_to_atom37.append([0] * 14) + restype_atom37_to_atom14.append([0] * 37) + restype_atom14_mask.append([0.0] * 14) + + restype_atom14_to_atom37 = np.array( + restype_atom14_to_atom37, dtype=np.int32) + restype_atom37_to_atom14 = np.array( + restype_atom37_to_atom14, dtype=np.int32) + restype_atom14_mask = np.array(restype_atom14_mask, dtype=np.float32) + + return restype_atom14_to_atom37, restype_atom37_to_atom14, restype_atom14_mask + + +( + restype_atom14_to_atom37, + restype_atom37_to_atom14, + restype_atom14_mask, +) = _make_atom14_and_atom37_constants() + + +def _make_renaming_matrices(): + # As the atom naming is ambiguous for 7 of the 20 amino acids, provide + # alternative ground truth coordinates where the naming is swapped + restype_3 = [restype_1to3[res] for res in restypes] + restype_3 += ['UNK'] + + # Matrices for renaming ambiguous atoms. + all_matrices = {res: np.eye(14) for res in restype_3} + for resname, swap in residue_atom_renaming_swaps.items(): + correspondences = np.arange(14) + for source_atom_swap, target_atom_swap in swap.items(): + source_index = restype_name_to_atom14_names[resname].index( + source_atom_swap) + target_index = restype_name_to_atom14_names[resname].index( + target_atom_swap) + correspondences[source_index] = target_index + correspondences[target_index] = source_index + renaming_matrix = np.zeros((14, 14)) + for index, correspondence in enumerate(correspondences): + renaming_matrix[index, correspondence] = 1.0 + all_matrices[resname] = renaming_matrix + renaming_matrices = np.stack( + [all_matrices[restype] for restype in restype_3]) + return renaming_matrices + + +renaming_matrices = _make_renaming_matrices() + + +def _make_atom14_is_ambiguous(): + # Create an ambiguous atoms mask. shape: (21, 14). + restype_atom14_is_ambiguous = np.zeros((21, 14)) + for resname, swap in residue_atom_renaming_swaps.items(): + for atom_name1, atom_name2 in swap.items(): + restype = restype_order[restype_3to1[resname]] + atom_idx1 = restype_name_to_atom14_names[resname].index(atom_name1) + atom_idx2 = restype_name_to_atom14_names[resname].index(atom_name2) + restype_atom14_is_ambiguous[restype, atom_idx1] = 1 + restype_atom14_is_ambiguous[restype, atom_idx2] = 1 + return restype_atom14_is_ambiguous + + +restype_atom14_is_ambiguous = _make_atom14_is_ambiguous() + + +def get_chi_atom_indices(): + """Returns atom indices needed to compute chi angles for all residue types. + + Returns: + A tensor of shape [residue_types=21, chis=4, atoms=4]. The residue types are + in the order specified in restypes + unknown residue type + at the end. For chi angles which are not defined on the residue, the + positions indices are by default set to 0. + """ + chi_atom_indices = [] + for residue_name in restypes: + residue_name = restype_1to3[residue_name] + residue_chi_angles = chi_angles_atoms[residue_name] + atom_indices = [] + for chi_angle in residue_chi_angles: + atom_indices.append([atom_order[atom] for atom in chi_angle]) + for _ in range(4 - len(atom_indices)): + atom_indices.append([0, 0, 0, + 0]) # For chi angles not defined on the AA. + chi_atom_indices.append(atom_indices) + + chi_atom_indices.append([[0, 0, 0, 0]] * 4) # For UNKNOWN residue. + + return chi_atom_indices + + +chi_atom_indices = get_chi_atom_indices() diff --git a/runtime/Lib/site_packages/models/science/unifold/data/utils.py b/runtime/Lib/site_packages/models/science/unifold/data/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2be91ef0d5cbe83fbba85ac8fe6928ee8925bfb0 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/data/utils.py @@ -0,0 +1,161 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import copy as copy_lib +import functools +import gzip +import pickle +from typing import Any, Dict + +import json +import numpy as np +from scipy import sparse as sp + +from . import residue_constants as rc +from .data_ops import NumpyDict + +# from typing import * + + +def lru_cache(maxsize=16, typed=False, copy=False, deepcopy=False): + if deepcopy: + + def decorator(f): + cached_func = functools.lru_cache(maxsize, typed)(f) + + @functools.wraps(f) + def wrapper(*args, **kwargs): + return copy_lib.deepcopy(cached_func(*args, **kwargs)) + + return wrapper + + elif copy: + + def decorator(f): + cached_func = functools.lru_cache(maxsize, typed)(f) + + @functools.wraps(f) + def wrapper(*args, **kwargs): + return copy_lib.copy(cached_func(*args, **kwargs)) + + return wrapper + + else: + decorator = functools.lru_cache(maxsize, typed) + return decorator + + +@lru_cache(maxsize=8, deepcopy=True) +def load_pickle_safe(path: str) -> Dict[str, Any]: + + def load(path): + assert path.endswith('.pkl') or path.endswith( + '.pkl.gz'), f'bad suffix in {path} as pickle file.' + open_fn = gzip.open if path.endswith('.gz') else open + with open_fn(path, 'rb') as f: + return pickle.load(f) + + ret = load(path) + ret = uncompress_features(ret) + return ret + + +@lru_cache(maxsize=8, copy=True) +def load_pickle(path: str) -> Dict[str, Any]: + + def load(path): + assert path.endswith('.pkl') or path.endswith( + '.pkl.gz'), f'bad suffix in {path} as pickle file.' + open_fn = gzip.open if path.endswith('.gz') else open + with open_fn(path, 'rb') as f: + return pickle.load(f) + + ret = load(path) + ret = uncompress_features(ret) + return ret + + +def correct_template_restypes(feature): + """Correct template restype to have the same order as residue_constants.""" + feature = np.argmax(feature, axis=-1).astype(np.int32) + new_order_list = rc.MAP_HHBLITS_AATYPE_TO_OUR_AATYPE + feature = np.take(new_order_list, feature.astype(np.int32), axis=0) + return feature + + +def convert_all_seq_feature(feature: NumpyDict) -> NumpyDict: + feature['msa'] = feature['msa'].astype(np.uint8) + if 'num_alignments' in feature: + feature.pop('num_alignments') + # make_all_seq_key = lambda k: f'{k}_all_seq' if not k.endswith('_all_seq') else k + + def make_all_seq_key(k): + if not k.endswith('_all_seq'): + return f'{k}_all_seq' + return k + + return {make_all_seq_key(k): v for k, v in feature.items()} + + +def to_dense_matrix(spmat_dict: NumpyDict): + spmat = sp.coo_matrix( + (spmat_dict['data'], (spmat_dict['row'], spmat_dict['col'])), + shape=spmat_dict['shape'], + dtype=np.float32, + ) + return spmat.toarray() + + +FEATS_DTYPE = {'msa': np.int32} + + +def uncompress_features(feats: NumpyDict) -> NumpyDict: + if 'sparse_deletion_matrix_int' in feats: + v = feats.pop('sparse_deletion_matrix_int') + v = to_dense_matrix(v) + feats['deletion_matrix'] = v + return feats + + +def filter(feature: NumpyDict, **kwargs) -> NumpyDict: + assert len(kwargs) == 1, f'wrong usage of filter with kwargs: {kwargs}' + if 'desired_keys' in kwargs: + feature = { + k: v + for k, v in feature.items() if k in kwargs['desired_keys'] + } + elif 'required_keys' in kwargs: + for k in kwargs['required_keys']: + assert k in feature, f'cannot find required key {k}.' + elif 'ignored_keys' in kwargs: + feature = { + k: v + for k, v in feature.items() if k not in kwargs['ignored_keys'] + } + else: + raise AssertionError(f'wrong usage of filter with kwargs: {kwargs}') + return feature + + +def compress_features(features: NumpyDict): + change_dtype = { + 'msa': np.uint8, + } + sparse_keys = ['deletion_matrix_int'] + + compressed_features = {} + for k, v in features.items(): + if k in change_dtype: + v = v.astype(change_dtype[k]) + if k in sparse_keys: + v = sp.coo_matrix(v, dtype=v.dtype) + sp_v = { + 'shape': v.shape, + 'row': v.row, + 'col': v.col, + 'data': v.data + } + k = f'sparse_{k}' + v = sp_v + compressed_features[k] = v + return compressed_features diff --git a/runtime/Lib/site_packages/models/science/unifold/dataset.py b/runtime/Lib/site_packages/models/science/unifold/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f14c2ef795d9ee5cfb6fb463401a19c865127e3f --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/dataset.py @@ -0,0 +1,518 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import copy +import logging +import os +# from typing import * +from typing import Dict, Iterable, List, Optional, Tuple, Union + +import json +import ml_collections as mlc +import numpy as np +import torch +from unicore.data import UnicoreDataset, data_utils +from unicore.distributed import utils as distributed_utils + +from .data import utils +from .data.data_ops import NumpyDict, TorchDict +from .data.process import process_features, process_labels +from .data.process_multimer import (add_assembly_features, + convert_monomer_features, merge_msas, + pair_and_merge, post_process) + +Rotation = Iterable[Iterable] +Translation = Iterable +Operation = Union[str, Tuple[Rotation, Translation]] +NumpyExample = Tuple[NumpyDict, Optional[List[NumpyDict]]] +TorchExample = Tuple[TorchDict, Optional[List[TorchDict]]] + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + +def make_data_config( + config: mlc.ConfigDict, + mode: str, + num_res: int, +) -> Tuple[mlc.ConfigDict, List[str]]: + cfg = copy.deepcopy(config) + mode_cfg = cfg[mode] + with cfg.unlocked(): + if mode_cfg.crop_size is None: + mode_cfg.crop_size = num_res + feature_names = cfg.common.unsupervised_features + cfg.common.recycling_features + if cfg.common.use_templates: + feature_names += cfg.common.template_features + if cfg.common.is_multimer: + feature_names += cfg.common.multimer_features + if cfg[mode].supervised: + feature_names += cfg.supervised.supervised_features + + return cfg, feature_names + + +def process_label(all_atom_positions: np.ndarray, + operation: Operation) -> np.ndarray: + if operation == 'I': + return all_atom_positions + rot, trans = operation + rot = np.array(rot).reshape(3, 3) + trans = np.array(trans).reshape(3) + return all_atom_positions @ rot.T + trans + + +@utils.lru_cache(maxsize=8, copy=True) +def load_single_feature( + sequence_id: str, + monomer_feature_dir: str, + uniprot_msa_dir: Optional[str] = None, + is_monomer: bool = False, +) -> NumpyDict: + + monomer_feature = utils.load_pickle( + os.path.join(monomer_feature_dir, f'{sequence_id}.feature.pkl.gz')) + monomer_feature = convert_monomer_features(monomer_feature) + chain_feature = {**monomer_feature} + + if uniprot_msa_dir is not None: + all_seq_feature = utils.load_pickle( + os.path.join(uniprot_msa_dir, f'{sequence_id}.uniprot.pkl.gz')) + if is_monomer: + chain_feature['msa'], chain_feature[ + 'deletion_matrix'] = merge_msas( + chain_feature['msa'], + chain_feature['deletion_matrix'], + all_seq_feature['msa'], + all_seq_feature['deletion_matrix'], + ) # noqa + else: + all_seq_feature = utils.convert_all_seq_feature(all_seq_feature) + for key in [ + 'msa_all_seq', + 'msa_species_identifiers_all_seq', + 'deletion_matrix_all_seq', + ]: + chain_feature[key] = all_seq_feature[key] + + return chain_feature + + +def load_single_label( + label_id: str, + label_dir: str, + symmetry_operation: Optional[Operation] = None, +) -> NumpyDict: + label = utils.load_pickle( + os.path.join(label_dir, f'{label_id}.label.pkl.gz')) + if symmetry_operation is not None: + label['all_atom_positions'] = process_label( + label['all_atom_positions'], symmetry_operation) + label = { + k: v + for k, v in label.items() if k in + ['aatype', 'all_atom_positions', 'all_atom_mask', 'resolution'] + } + return label + + +def load( + sequence_ids: List[str], + monomer_feature_dir: str, + uniprot_msa_dir: Optional[str] = None, + label_ids: Optional[List[str]] = None, + label_dir: Optional[str] = None, + symmetry_operations: Optional[List[Operation]] = None, + is_monomer: bool = False, +) -> NumpyExample: + + all_chain_features = [ + load_single_feature(s, monomer_feature_dir, uniprot_msa_dir, + is_monomer) for s in sequence_ids + ] + + if label_ids is not None: + # load labels + assert len(label_ids) == len(sequence_ids) + assert label_dir is not None + if symmetry_operations is None: + symmetry_operations = ['I' for _ in label_ids] + all_chain_labels = [ + load_single_label(ll, label_dir, o) + for ll, o in zip(label_ids, symmetry_operations) + ] + # update labels into features to calculate spatial cropping etc. + [f.update(ll) for f, ll in zip(all_chain_features, all_chain_labels)] + + all_chain_features = add_assembly_features(all_chain_features) + + # get labels back from features, as add_assembly_features may alter the order of inputs. + if label_ids is not None: + all_chain_labels = [{ + k: f[k] + for k in + ['aatype', 'all_atom_positions', 'all_atom_mask', 'resolution'] + } for f in all_chain_features] + else: + all_chain_labels = None + + asym_len = np.array([c['seq_length'] for c in all_chain_features], + dtype=np.int64) + if is_monomer: + all_chain_features = all_chain_features[0] + else: + all_chain_features = pair_and_merge(all_chain_features) + all_chain_features = post_process(all_chain_features) + all_chain_features['asym_len'] = asym_len + + return all_chain_features, all_chain_labels + + +def process( + config: mlc.ConfigDict, + mode: str, + features: NumpyDict, + labels: Optional[List[NumpyDict]] = None, + seed: int = 0, + batch_idx: Optional[int] = None, + data_idx: Optional[int] = None, + is_distillation: bool = False, +) -> TorchExample: + + if mode == 'train': + assert batch_idx is not None + with data_utils.numpy_seed(seed, batch_idx, key='recycling'): + num_iters = np.random.randint( + 0, config.common.max_recycling_iters + 1) + use_clamped_fape = np.random.rand( + ) < config[mode].use_clamped_fape_prob + else: + num_iters = config.common.max_recycling_iters + use_clamped_fape = 1 + + features['num_recycling_iters'] = int(num_iters) + features['use_clamped_fape'] = int(use_clamped_fape) + features['is_distillation'] = int(is_distillation) + if is_distillation and 'msa_chains' in features: + features.pop('msa_chains') + + num_res = int(features['seq_length']) + cfg, feature_names = make_data_config(config, mode=mode, num_res=num_res) + + if labels is not None: + features['resolution'] = labels[0]['resolution'].reshape(-1) + + with data_utils.numpy_seed(seed, data_idx, key='protein_feature'): + features['crop_and_fix_size_seed'] = np.random.randint(0, 63355) + features = utils.filter(features, desired_keys=feature_names) + features = {k: torch.tensor(v) for k, v in features.items()} + with torch.no_grad(): + features = process_features(features, cfg.common, cfg[mode]) + + if labels is not None: + labels = [{k: torch.tensor(v) for k, v in ll.items()} for ll in labels] + with torch.no_grad(): + labels = process_labels(labels) + + return features, labels + + +def load_and_process( + config: mlc.ConfigDict, + mode: str, + seed: int = 0, + batch_idx: Optional[int] = None, + data_idx: Optional[int] = None, + is_distillation: bool = False, + **load_kwargs, +): + is_monomer = ( + is_distillation + if 'is_monomer' not in load_kwargs else load_kwargs.pop('is_monomer')) + features, labels = load(**load_kwargs, is_monomer=is_monomer) + features, labels = process(config, mode, features, labels, seed, batch_idx, + data_idx, is_distillation) + return features, labels + + +class UnifoldDataset(UnicoreDataset): + + def __init__( + self, + args, + seed, + config, + data_path, + mode='train', + max_step=None, + disable_sd=False, + json_prefix='', + ): + self.path = data_path + + def load_json(filename): + return json.load(open(filename, 'r', encoding='utf-8')) + + sample_weight = load_json( + os.path.join(self.path, + json_prefix + mode + '_sample_weight.json')) + self.multi_label = load_json( + os.path.join(self.path, json_prefix + mode + '_multi_label.json')) + self.inverse_multi_label = self._inverse_map(self.multi_label) + self.sample_weight = {} + for chain in self.inverse_multi_label: + entity = self.inverse_multi_label[chain] + self.sample_weight[chain] = sample_weight[entity] + self.seq_sample_weight = sample_weight + logger.info('load {} chains (unique {} sequences)'.format( + len(self.sample_weight), len(self.seq_sample_weight))) + self.feature_path = os.path.join(self.path, 'pdb_features') + self.label_path = os.path.join(self.path, 'pdb_labels') + sd_sample_weight_path = os.path.join( + self.path, json_prefix + 'sd_train_sample_weight.json') + if mode == 'train' and os.path.isfile( + sd_sample_weight_path) and not disable_sd: + self.sd_sample_weight = load_json(sd_sample_weight_path) + logger.info('load {} self-distillation samples.'.format( + len(self.sd_sample_weight))) + self.sd_feature_path = os.path.join(self.path, 'sd_features') + self.sd_label_path = os.path.join(self.path, 'sd_labels') + else: + self.sd_sample_weight = None + self.batch_size = ( + args.batch_size * distributed_utils.get_data_parallel_world_size() + * args.update_freq[0]) + self.data_len = ( + max_step * self.batch_size + if max_step is not None else len(self.sample_weight)) + self.mode = mode + self.num_seq, self.seq_keys, self.seq_sample_prob = self.cal_sample_weight( + self.seq_sample_weight) + self.num_chain, self.chain_keys, self.sample_prob = self.cal_sample_weight( + self.sample_weight) + if self.sd_sample_weight is not None: + ( + self.sd_num_chain, + self.sd_chain_keys, + self.sd_sample_prob, + ) = self.cal_sample_weight(self.sd_sample_weight) + self.config = config.data + self.seed = seed + self.sd_prob = args.sd_prob + + def cal_sample_weight(self, sample_weight): + prot_keys = list(sample_weight.keys()) + sum_weight = sum(sample_weight.values()) + sample_prob = [sample_weight[k] / sum_weight for k in prot_keys] + num_prot = len(prot_keys) + return num_prot, prot_keys, sample_prob + + def sample_chain(self, idx, sample_by_seq=False): + is_distillation = False + if self.mode == 'train': + with data_utils.numpy_seed(self.seed, idx, key='data_sample'): + is_distillation = ((np.random.rand(1)[0] < self.sd_prob) + if self.sd_sample_weight is not None else + False) + if is_distillation: + prot_idx = np.random.choice( + self.sd_num_chain, p=self.sd_sample_prob) + label_name = self.sd_chain_keys[prot_idx] + seq_name = label_name + else: + if not sample_by_seq: + prot_idx = np.random.choice( + self.num_chain, p=self.sample_prob) + label_name = self.chain_keys[prot_idx] + seq_name = self.inverse_multi_label[label_name] + else: + seq_idx = np.random.choice( + self.num_seq, p=self.seq_sample_prob) + seq_name = self.seq_keys[seq_idx] + label_name = np.random.choice( + self.multi_label[seq_name]) + else: + label_name = self.chain_keys[idx] + seq_name = self.inverse_multi_label[label_name] + return seq_name, label_name, is_distillation + + def __getitem__(self, idx): + sequence_id, label_id, is_distillation = self.sample_chain( + idx, sample_by_seq=True) + feature_dir, label_dir = ((self.feature_path, + self.label_path) if not is_distillation else + (self.sd_feature_path, self.sd_label_path)) + features, _ = load_and_process( + self.config, + self.mode, + self.seed, + batch_idx=(idx // self.batch_size), + data_idx=idx, + is_distillation=is_distillation, + sequence_ids=[sequence_id], + monomer_feature_dir=feature_dir, + uniprot_msa_dir=None, + label_ids=[label_id], + label_dir=label_dir, + symmetry_operations=None, + is_monomer=True, + ) + return features + + def __len__(self): + return self.data_len + + @staticmethod + def collater(samples): + # first dim is recyling. bsz is at the 2nd dim + return data_utils.collate_dict(samples, dim=1) + + @staticmethod + def _inverse_map(mapping: Dict[str, List[str]]): + inverse_mapping = {} + for ent, refs in mapping.items(): + for ref in refs: + if ref in inverse_mapping: # duplicated ent for this ref. + ent_2 = inverse_mapping[ref] + assert ( + ent == ent_2 + ), f'multiple entities ({ent_2}, {ent}) exist for reference {ref}.' + inverse_mapping[ref] = ent + return inverse_mapping + + +class UnifoldMultimerDataset(UnifoldDataset): + + def __init__( + self, + args: mlc.ConfigDict, + seed: int, + config: mlc.ConfigDict, + data_path: str, + mode: str = 'train', + max_step: Optional[int] = None, + disable_sd: bool = False, + json_prefix: str = '', + **kwargs, + ): + super().__init__(args, seed, config, data_path, mode, max_step, + disable_sd, json_prefix) + self.data_path = data_path + self.pdb_assembly = json.load( + open( + os.path.join(self.data_path, + json_prefix + 'pdb_assembly.json'), + encoding='utf-8')) + self.pdb_chains = self.get_chains(self.inverse_multi_label) + self.monomer_feature_path = os.path.join(self.data_path, + 'pdb_features') + self.uniprot_msa_path = os.path.join(self.data_path, 'pdb_uniprots') + self.label_path = os.path.join(self.data_path, 'pdb_labels') + self.max_chains = args.max_chains + if self.mode == 'train': + self.pdb_chains, self.sample_weight = self.filter_pdb_by_max_chains( + self.pdb_chains, self.pdb_assembly, self.sample_weight, + self.max_chains) + self.num_chain, self.chain_keys, self.sample_prob = self.cal_sample_weight( + self.sample_weight) + + def __getitem__(self, idx): + seq_id, label_id, is_distillation = self.sample_chain(idx) + if is_distillation: + label_ids = [label_id] + sequence_ids = [seq_id] + monomer_feature_path, uniprot_msa_path, label_path = ( + self.sd_feature_path, + None, + self.sd_label_path, + ) + symmetry_operations = None + else: + pdb_id = self.get_pdb_name(label_id) + if pdb_id in self.pdb_assembly and self.mode == 'train': + label_ids = [ + pdb_id + '_' + id + for id in self.pdb_assembly[pdb_id]['chains'] + ] + symmetry_operations = [ + t for t in self.pdb_assembly[pdb_id]['opers'] + ] + else: + label_ids = self.pdb_chains[pdb_id] + symmetry_operations = None + sequence_ids = [ + self.inverse_multi_label[chain_id] for chain_id in label_ids + ] + monomer_feature_path, uniprot_msa_path, label_path = ( + self.monomer_feature_path, + self.uniprot_msa_path, + self.label_path, + ) + + return load_and_process( + self.config, + self.mode, + self.seed, + batch_idx=(idx // self.batch_size), + data_idx=idx, + is_distillation=is_distillation, + sequence_ids=sequence_ids, + monomer_feature_dir=monomer_feature_path, + uniprot_msa_dir=uniprot_msa_path, + label_ids=label_ids, + label_dir=label_path, + symmetry_operations=symmetry_operations, + is_monomer=False, + ) + + @staticmethod + def collater(samples): + # first dim is recyling. bsz is at the 2nd dim + if len(samples) <= 0: # tackle empty batch + return None + feats = [s[0] for s in samples] + labs = [s[1] for s in samples if s[1] is not None] + try: + feats = data_utils.collate_dict(feats, dim=1) + except BaseException: + raise ValueError('cannot collate features', feats) + if not labs: + labs = None + return feats, labs + + @staticmethod + def get_pdb_name(chain): + return chain.split('_')[0] + + @staticmethod + def get_chains(canon_chain_map): + pdb_chains = {} + for chain in canon_chain_map: + pdb = UnifoldMultimerDataset.get_pdb_name(chain) + if pdb not in pdb_chains: + pdb_chains[pdb] = [] + pdb_chains[pdb].append(chain) + return pdb_chains + + @staticmethod + def filter_pdb_by_max_chains(pdb_chains, pdb_assembly, sample_weight, + max_chains): + new_pdb_chains = {} + for chain in pdb_chains: + if chain in pdb_assembly: + size = len(pdb_assembly[chain]['chains']) + if size <= max_chains: + new_pdb_chains[chain] = pdb_chains[chain] + else: + size = len(pdb_chains[chain]) + if size == 1: + new_pdb_chains[chain] = pdb_chains[chain] + new_sample_weight = { + k: sample_weight[k] + for k in sample_weight + if UnifoldMultimerDataset.get_pdb_name(k) in new_pdb_chains + } + logger.info( + f'filtered out {len(pdb_chains) - len(new_pdb_chains)} / {len(pdb_chains)} PDBs ' + f'({len(sample_weight) - len(new_sample_weight)} / {len(sample_weight)} chains) ' + f'by max_chains {max_chains}') + return new_pdb_chains, new_sample_weight diff --git a/runtime/Lib/site_packages/models/science/unifold/model.py b/runtime/Lib/site_packages/models/science/unifold/model.py new file mode 100644 index 0000000000000000000000000000000000000000..7f28f18dd8b28d7992a0381bdc5810adf37e8559 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/model.py @@ -0,0 +1,78 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import argparse +import os +from typing import Any + +import torch + +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.utils.constant import ModelFile, Tasks +from .config import model_config +from .modules.alphafold import AlphaFold + +__all__ = ['UnifoldForProteinStructrue'] + + +@MODELS.register_module(Tasks.protein_structure, module_name=Models.unifold) +class UnifoldForProteinStructrue(TorchModel): + + @staticmethod + def add_args(parser): + """Add model-specific arguments to the parser.""" + parser.add_argument( + '--model-name', + help='choose the model config', + ) + + def __init__(self, **kwargs): + super().__init__() + parser = argparse.ArgumentParser() + parse_comm = [] + for key in kwargs: + parser.add_argument(f'--{key}') + parse_comm.append(f'--{key}') + parse_comm.append(kwargs[key]) + args = parser.parse_args(parse_comm) + base_architecture(args) + self.args = args + config = model_config( + self.args.model_name, + train=True, + ) + self.model = AlphaFold(config) + self.config = config + + # load model state dict + param_path = os.path.join(kwargs['model_dir'], + ModelFile.TORCH_MODEL_BIN_FILE) + state_dict = torch.load(param_path)['ema']['params'] + state_dict = { + '.'.join(k.split('.')[1:]): v + for k, v in state_dict.items() + } + self.model.load_state_dict(state_dict) + + def half(self): + self.model = self.model.half() + return self + + def bfloat16(self): + self.model = self.model.bfloat16() + return self + + @classmethod + def build_model(cls, args, task): + """Build a new model instance.""" + return cls(args) + + def forward(self, batch, **kwargs): + outputs = self.model.forward(batch) + return outputs, self.config.loss + + +def base_architecture(args): + args.model_name = getattr(args, 'model_name', 'model_2') diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/__init__.py b/runtime/Lib/site_packages/models/science/unifold/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..63aa84ed5a8f9cbf91be500b67ae4c3af542d7c7 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/__init__.py @@ -0,0 +1,3 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. +"""Unifold Modules.""" diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/alphafold.py b/runtime/Lib/site_packages/models/science/unifold/modules/alphafold.py new file mode 100644 index 0000000000000000000000000000000000000000..71a1b3103e393c60e0964430fd5632839ca90393 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/alphafold.py @@ -0,0 +1,450 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import torch +import torch.nn as nn +from unicore.utils import tensor_tree_map + +from ..data import residue_constants +from .attentions import gen_msa_attn_mask, gen_tri_attn_mask +from .auxillary_heads import AuxiliaryHeads +from .common import residual +from .embedders import (ExtraMSAEmbedder, InputEmbedder, RecyclingEmbedder, + TemplateAngleEmbedder, TemplatePairEmbedder) +from .evoformer import EvoformerStack, ExtraMSAStack +from .featurization import (atom14_to_atom37, build_extra_msa_feat, + build_template_angle_feat, + build_template_pair_feat, + build_template_pair_feat_v2, pseudo_beta_fn) +from .structure_module import StructureModule +from .template import (TemplatePairStack, TemplatePointwiseAttention, + TemplateProjection) + + +class AlphaFold(nn.Module): + + def __init__(self, config): + super(AlphaFold, self).__init__() + + self.globals = config.globals + config = config.model + template_config = config.template + extra_msa_config = config.extra_msa + + self.input_embedder = InputEmbedder( + **config['input_embedder'], + use_chain_relative=config.is_multimer, + ) + self.recycling_embedder = RecyclingEmbedder( + **config['recycling_embedder'], ) + if config.template.enabled: + self.template_angle_embedder = TemplateAngleEmbedder( + **template_config['template_angle_embedder'], ) + self.template_pair_embedder = TemplatePairEmbedder( + **template_config['template_pair_embedder'], ) + self.template_pair_stack = TemplatePairStack( + **template_config['template_pair_stack'], ) + else: + self.template_pair_stack = None + self.enable_template_pointwise_attention = template_config[ + 'template_pointwise_attention'].enabled + if self.enable_template_pointwise_attention: + self.template_pointwise_att = TemplatePointwiseAttention( + **template_config['template_pointwise_attention'], ) + else: + self.template_proj = TemplateProjection( + **template_config['template_pointwise_attention'], ) + self.extra_msa_embedder = ExtraMSAEmbedder( + **extra_msa_config['extra_msa_embedder'], ) + self.extra_msa_stack = ExtraMSAStack( + **extra_msa_config['extra_msa_stack'], ) + self.evoformer = EvoformerStack(**config['evoformer_stack'], ) + self.structure_module = StructureModule(**config['structure_module'], ) + + self.aux_heads = AuxiliaryHeads(config['heads'], ) + + self.config = config + self.dtype = torch.float + self.inf = self.globals.inf + if self.globals.alphafold_original_mode: + self.alphafold_original_mode() + + def __make_input_float__(self): + self.input_embedder = self.input_embedder.float() + self.recycling_embedder = self.recycling_embedder.float() + + def half(self): + super().half() + if (not getattr(self, 'inference', False)): + self.__make_input_float__() + self.dtype = torch.half + return self + + def bfloat16(self): + super().bfloat16() + if (not getattr(self, 'inference', False)): + self.__make_input_float__() + self.dtype = torch.bfloat16 + return self + + def alphafold_original_mode(self): + + def set_alphafold_original_mode(module): + if hasattr(module, 'apply_alphafold_original_mode'): + module.apply_alphafold_original_mode() + if hasattr(module, 'act'): + module.act = nn.ReLU() + + self.apply(set_alphafold_original_mode) + + def inference_mode(self): + + def set_inference_mode(module): + setattr(module, 'inference', True) + + self.apply(set_inference_mode) + + def __convert_input_dtype__(self, batch): + for key in batch: + # only convert features with mask + if batch[key].dtype != self.dtype and 'mask' in key: + batch[key] = batch[key].type(self.dtype) + return batch + + def embed_templates_pair_core(self, batch, z, pair_mask, + tri_start_attn_mask, tri_end_attn_mask, + templ_dim, multichain_mask_2d): + if self.config.template.template_pair_embedder.v2_feature: + t = build_template_pair_feat_v2( + batch, + inf=self.config.template.inf, + eps=self.config.template.eps, + multichain_mask_2d=multichain_mask_2d, + **self.config.template.distogram, + ) + num_template = t[0].shape[-4] + single_templates = [ + self.template_pair_embedder([x[..., ti, :, :, :] + for x in t], z) + for ti in range(num_template) + ] + else: + t = build_template_pair_feat( + batch, + inf=self.config.template.inf, + eps=self.config.template.eps, + **self.config.template.distogram, + ) + single_templates = [ + self.template_pair_embedder(x, z) + for x in torch.unbind(t, dim=templ_dim) + ] + + t = self.template_pair_stack( + single_templates, + pair_mask, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + templ_dim=templ_dim, + chunk_size=self.globals.chunk_size, + block_size=self.globals.block_size, + return_mean=not self.enable_template_pointwise_attention, + ) + return t + + def embed_templates_pair(self, batch, z, pair_mask, tri_start_attn_mask, + tri_end_attn_mask, templ_dim): + if self.config.template.template_pair_embedder.v2_feature and 'asym_id' in batch: + multichain_mask_2d = ( + batch['asym_id'][..., :, None] == batch['asym_id'][..., + None, :]) + multichain_mask_2d = multichain_mask_2d.unsqueeze(0) + else: + multichain_mask_2d = None + + if self.training or self.enable_template_pointwise_attention: + t = self.embed_templates_pair_core(batch, z, pair_mask, + tri_start_attn_mask, + tri_end_attn_mask, templ_dim, + multichain_mask_2d) + if self.enable_template_pointwise_attention: + t = self.template_pointwise_att( + t, + z, + template_mask=batch['template_mask'], + chunk_size=self.globals.chunk_size, + ) + t_mask = torch.sum( + batch['template_mask'], dim=-1, keepdims=True) > 0 + t_mask = t_mask[..., None, None].type(t.dtype) + t *= t_mask + else: + t = self.template_proj(t, z) + else: + template_aatype_shape = batch['template_aatype'].shape + # template_aatype is either [n_template, n_res] or [1, n_template_, n_res] + batch_templ_dim = 1 if len(template_aatype_shape) == 3 else 0 + n_templ = batch['template_aatype'].shape[batch_templ_dim] + + if n_templ <= 0: + t = None + else: + template_batch = { + k: v + for k, v in batch.items() if k.startswith('template_') + } + + def embed_one_template(i): + + def slice_template_tensor(t): + s = [slice(None) for _ in t.shape] + s[batch_templ_dim] = slice(i, i + 1) + return t[s] + + template_feats = tensor_tree_map( + slice_template_tensor, + template_batch, + ) + t = self.embed_templates_pair_core( + template_feats, z, pair_mask, tri_start_attn_mask, + tri_end_attn_mask, templ_dim, multichain_mask_2d) + return t + + t = embed_one_template(0) + # iterate templates one by one + for i in range(1, n_templ): + t += embed_one_template(i) + t /= n_templ + t = self.template_proj(t, z) + return t + + def embed_templates_angle(self, batch): + template_angle_feat, template_angle_mask = build_template_angle_feat( + batch, + v2_feature=self.config.template.template_pair_embedder.v2_feature) + t = self.template_angle_embedder(template_angle_feat) + return t, template_angle_mask + + def iteration_evoformer(self, feats, m_1_prev, z_prev, x_prev): + batch_dims = feats['target_feat'].shape[:-2] + n = feats['target_feat'].shape[-2] + seq_mask = feats['seq_mask'] + pair_mask = seq_mask[..., None] * seq_mask[..., None, :] + msa_mask = feats['msa_mask'] + + m, z = self.input_embedder( + feats['target_feat'], + feats['msa_feat'], + ) + + if m_1_prev is None: + m_1_prev = m.new_zeros( + (*batch_dims, n, self.config.input_embedder.d_msa), + requires_grad=False, + ) + if z_prev is None: + z_prev = z.new_zeros( + (*batch_dims, n, n, self.config.input_embedder.d_pair), + requires_grad=False, + ) + if x_prev is None: + x_prev = z.new_zeros( + (*batch_dims, n, residue_constants.atom_type_num, 3), + requires_grad=False, + ) + x_prev = pseudo_beta_fn(feats['aatype'], x_prev, None) + + z += self.recycling_embedder.recyle_pos(x_prev) + + m_1_prev_emb, z_prev_emb = self.recycling_embedder( + m_1_prev, + z_prev, + ) + + m[..., 0, :, :] += m_1_prev_emb + + z += z_prev_emb + + z += self.input_embedder.relpos_emb( + feats['residue_index'].long(), + feats.get('sym_id', None), + feats.get('asym_id', None), + feats.get('entity_id', None), + feats.get('num_sym', None), + ) + + m = m.type(self.dtype) + z = z.type(self.dtype) + tri_start_attn_mask, tri_end_attn_mask = gen_tri_attn_mask( + pair_mask, self.inf) + + if self.config.template.enabled: + template_mask = feats['template_mask'] + if torch.any(template_mask): + z = residual( + z, + self.embed_templates_pair( + feats, + z, + pair_mask, + tri_start_attn_mask, + tri_end_attn_mask, + templ_dim=-4, + ), + self.training, + ) + + if self.config.extra_msa.enabled: + a = self.extra_msa_embedder(build_extra_msa_feat(feats)) + extra_msa_row_mask = gen_msa_attn_mask( + feats['extra_msa_mask'], + inf=self.inf, + gen_col_mask=False, + ) + z = self.extra_msa_stack( + a, + z, + msa_mask=feats['extra_msa_mask'], + chunk_size=self.globals.chunk_size, + block_size=self.globals.block_size, + pair_mask=pair_mask, + msa_row_attn_mask=extra_msa_row_mask, + msa_col_attn_mask=None, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + ) + + if self.config.template.embed_angles: + template_1d_feat, template_1d_mask = self.embed_templates_angle( + feats) + m = torch.cat([m, template_1d_feat], dim=-3) + msa_mask = torch.cat([feats['msa_mask'], template_1d_mask], dim=-2) + + msa_row_mask, msa_col_mask = gen_msa_attn_mask( + msa_mask, + inf=self.inf, + ) + + m, z, s = self.evoformer( + m, + z, + msa_mask=msa_mask, + pair_mask=pair_mask, + msa_row_attn_mask=msa_row_mask, + msa_col_attn_mask=msa_col_mask, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + chunk_size=self.globals.chunk_size, + block_size=self.globals.block_size, + ) + return m, z, s, msa_mask, m_1_prev_emb, z_prev_emb + + def iteration_evoformer_structure_module(self, + batch, + m_1_prev, + z_prev, + x_prev, + cycle_no, + num_recycling, + num_ensembles=1): + z, s = 0, 0 + n_seq = batch['msa_feat'].shape[-3] + assert num_ensembles >= 1 + for ensemble_no in range(num_ensembles): + idx = cycle_no * num_ensembles + ensemble_no + + # fetch_cur_batch = lambda t: t[min(t.shape[0] - 1, idx), ...] + def fetch_cur_batch(t): + return t[min(t.shape[0] - 1, idx), ...] + + feats = tensor_tree_map(fetch_cur_batch, batch) + m, z0, s0, msa_mask, m_1_prev_emb, z_prev_emb = self.iteration_evoformer( + feats, m_1_prev, z_prev, x_prev) + z += z0 + s += s0 + del z0, s0 + if num_ensembles > 1: + z /= float(num_ensembles) + s /= float(num_ensembles) + + outputs = {} + + outputs['msa'] = m[..., :n_seq, :, :] + outputs['pair'] = z + outputs['single'] = s + + # norm loss + if (not getattr(self, 'inference', + False)) and num_recycling == (cycle_no + 1): + delta_msa = m + delta_msa[..., + 0, :, :] = delta_msa[..., + 0, :, :] - m_1_prev_emb.detach() + delta_pair = z - z_prev_emb.detach() + outputs['delta_msa'] = delta_msa + outputs['delta_pair'] = delta_pair + outputs['msa_norm_mask'] = msa_mask + + outputs['sm'] = self.structure_module( + s, + z, + feats['aatype'], + mask=feats['seq_mask'], + ) + outputs['final_atom_positions'] = atom14_to_atom37( + outputs['sm']['positions'], feats) + outputs['final_atom_mask'] = feats['atom37_atom_exists'] + outputs['pred_frame_tensor'] = outputs['sm']['frames'][-1] + + # use float32 for numerical stability + if (not getattr(self, 'inference', False)): + m_1_prev = m[..., 0, :, :].float() + z_prev = z.float() + x_prev = outputs['final_atom_positions'].float() + else: + m_1_prev = m[..., 0, :, :] + z_prev = z + x_prev = outputs['final_atom_positions'] + + return outputs, m_1_prev, z_prev, x_prev + + def forward(self, batch): + + m_1_prev = batch.get('m_1_prev', None) + z_prev = batch.get('z_prev', None) + x_prev = batch.get('x_prev', None) + + is_grad_enabled = torch.is_grad_enabled() + + num_iters = int(batch['num_recycling_iters']) + 1 + num_ensembles = int(batch['msa_mask'].shape[0]) // num_iters + if self.training: + # don't use ensemble during training + assert num_ensembles == 1 + + # convert dtypes in batch + batch = self.__convert_input_dtype__(batch) + for cycle_no in range(num_iters): + is_final_iter = cycle_no == (num_iters - 1) + with torch.set_grad_enabled(is_grad_enabled and is_final_iter): + ( + outputs, + m_1_prev, + z_prev, + x_prev, + ) = self.iteration_evoformer_structure_module( + batch, + m_1_prev, + z_prev, + x_prev, + cycle_no=cycle_no, + num_recycling=num_iters, + num_ensembles=num_ensembles, + ) + if not is_final_iter: + del outputs + + if 'asym_id' in batch: + outputs['asym_id'] = batch['asym_id'][0, ...] + outputs.update(self.aux_heads(outputs)) + return outputs diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/attentions.py b/runtime/Lib/site_packages/models/science/unifold/modules/attentions.py new file mode 100644 index 0000000000000000000000000000000000000000..21d92ffd1b178418f2793bbcdbd0edc3f2e556f7 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/attentions.py @@ -0,0 +1,430 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from functools import partialmethod +from typing import List, Optional + +import torch +import torch.nn as nn +from unicore.modules import LayerNorm, softmax_dropout +from unicore.utils import permute_final_dims + +from .common import Linear, chunk_layer + + +def gen_attn_mask(mask, neg_inf): + assert neg_inf < -1e4 + attn_mask = torch.zeros_like(mask) + attn_mask[mask == 0] = neg_inf + return attn_mask + + +class Attention(nn.Module): + + def __init__( + self, + q_dim: int, + k_dim: int, + v_dim: int, + head_dim: int, + num_heads: int, + gating: bool = True, + ): + super(Attention, self).__init__() + + self.num_heads = num_heads + total_dim = head_dim * self.num_heads + self.gating = gating + self.linear_q = Linear(q_dim, total_dim, bias=False, init='glorot') + self.linear_k = Linear(k_dim, total_dim, bias=False, init='glorot') + self.linear_v = Linear(v_dim, total_dim, bias=False, init='glorot') + self.linear_o = Linear(total_dim, q_dim, init='final') + self.linear_g = None + if self.gating: + self.linear_g = Linear(q_dim, total_dim, init='gating') + # precompute the 1/sqrt(head_dim) + self.norm = head_dim**-0.5 + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + mask: torch.Tensor = None, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + g = None + if self.linear_g is not None: + # gating, use raw query input + g = self.linear_g(q) + + q = self.linear_q(q) + q *= self.norm + k = self.linear_k(k) + v = self.linear_v(v) + + q = q.view(q.shape[:-1] + (self.num_heads, -1)).transpose( + -2, -3).contiguous() + k = k.view(k.shape[:-1] + (self.num_heads, -1)).transpose( + -2, -3).contiguous() + v = v.view(v.shape[:-1] + (self.num_heads, -1)).transpose(-2, -3) + + attn = torch.matmul(q, k.transpose(-1, -2)) + del q, k + + attn = softmax_dropout(attn, 0, self.training, mask=mask, bias=bias) + o = torch.matmul(attn, v) + del attn, v + + o = o.transpose(-2, -3).contiguous() + o = o.view(*o.shape[:-2], -1) + + if g is not None: + o = torch.sigmoid(g) * o + + # merge heads + o = nn.functional.linear(o, self.linear_o.weight) + return o + + def get_output_bias(self): + return self.linear_o.bias + + +class GlobalAttention(nn.Module): + + def __init__(self, input_dim, head_dim, num_heads, inf, eps): + super(GlobalAttention, self).__init__() + + self.num_heads = num_heads + self.inf = inf + self.eps = eps + self.linear_q = Linear( + input_dim, head_dim * num_heads, bias=False, init='glorot') + self.linear_k = Linear(input_dim, head_dim, bias=False, init='glorot') + self.linear_v = Linear(input_dim, head_dim, bias=False, init='glorot') + self.linear_g = Linear(input_dim, head_dim * num_heads, init='gating') + self.linear_o = Linear(head_dim * num_heads, input_dim, init='final') + self.sigmoid = nn.Sigmoid() + # precompute the 1/sqrt(head_dim) + self.norm = head_dim**-0.5 + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + + # gating + g = self.sigmoid(self.linear_g(x)) + + k = self.linear_k(x) + v = self.linear_v(x) + + q = torch.sum( + x * mask.unsqueeze(-1), dim=-2) / ( + torch.sum(mask, dim=-1, keepdims=True) + self.eps) + q = self.linear_q(q) + q *= self.norm + q = q.view(q.shape[:-1] + (self.num_heads, -1)) + + attn = torch.matmul(q, k.transpose(-1, -2)) + del q, k + + attn_mask = gen_attn_mask(mask, -self.inf)[..., :, None, :] + attn = softmax_dropout(attn, 0, self.training, mask=attn_mask) + + o = torch.matmul( + attn, + v, + ) + del attn, v + + g = g.view(g.shape[:-1] + (self.num_heads, -1)) + o = o.unsqueeze(-3) * g + del g + + # merge heads + o = o.reshape(o.shape[:-2] + (-1, )) + return self.linear_o(o) + + +def gen_msa_attn_mask(mask, inf, gen_col_mask=True): + row_mask = gen_attn_mask(mask, -inf)[..., :, None, None, :] + if gen_col_mask: + col_mask = gen_attn_mask(mask.transpose(-1, -2), -inf)[..., :, None, + None, :] + return row_mask, col_mask + else: + return row_mask + + +class MSAAttention(nn.Module): + + def __init__( + self, + d_in, + d_hid, + num_heads, + pair_bias=False, + d_pair=None, + ): + super(MSAAttention, self).__init__() + + self.pair_bias = pair_bias + self.layer_norm_m = LayerNorm(d_in) + self.layer_norm_z = None + self.linear_z = None + if self.pair_bias: + self.layer_norm_z = LayerNorm(d_pair) + self.linear_z = Linear( + d_pair, num_heads, bias=False, init='normal') + + self.mha = Attention(d_in, d_in, d_in, d_hid, num_heads) + + @torch.jit.ignore + def _chunk( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + chunk_size: int = None, + ) -> torch.Tensor: + + return chunk_layer( + self._attn_forward, + { + 'm': m, + 'mask': mask, + 'bias': bias + }, + chunk_size=chunk_size, + num_batch_dims=len(m.shape[:-2]), + ) + + @torch.jit.ignore + def _attn_chunk_forward( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = 2560, + ) -> torch.Tensor: + m = self.layer_norm_m(m) + num_chunk = (m.shape[-3] + chunk_size - 1) // chunk_size + outputs = [] + for i in range(num_chunk): + chunk_start = i * chunk_size + chunk_end = min(m.shape[-3], chunk_start + chunk_size) + cur_m = m[..., chunk_start:chunk_end, :, :] + cur_mask = ( + mask[..., chunk_start:chunk_end, :, :, :] + if mask is not None else None) + outputs.append( + self.mha(q=cur_m, k=cur_m, v=cur_m, mask=cur_mask, bias=bias)) + return torch.cat(outputs, dim=-3) + + def _attn_forward(self, m, mask, bias: Optional[torch.Tensor] = None): + m = self.layer_norm_m(m) + return self.mha(q=m, k=m, v=m, mask=mask, bias=bias) + + def forward( + self, + m: torch.Tensor, + z: Optional[torch.Tensor] = None, + attn_mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + + bias = None + if self.pair_bias: + z = self.layer_norm_z(z) + bias = ( + permute_final_dims(self.linear_z(z), + (2, 0, 1)).unsqueeze(-4).contiguous()) + + if chunk_size is not None: + m = self._chunk(m, attn_mask, bias, chunk_size) + else: + attn_chunk_size = 2560 + if m.shape[-3] <= attn_chunk_size: + m = self._attn_forward(m, attn_mask, bias) + else: + # reduce the peak memory cost in extra_msa_stack + return self._attn_chunk_forward( + m, attn_mask, bias, chunk_size=attn_chunk_size) + + return m + + def get_output_bias(self): + return self.mha.get_output_bias() + + +class MSARowAttentionWithPairBias(MSAAttention): + + def __init__(self, d_msa, d_pair, d_hid, num_heads): + super(MSARowAttentionWithPairBias, self).__init__( + d_msa, + d_hid, + num_heads, + pair_bias=True, + d_pair=d_pair, + ) + + +class MSAColumnAttention(MSAAttention): + + def __init__(self, d_msa, d_hid, num_heads): + super(MSAColumnAttention, self).__init__( + d_in=d_msa, + d_hid=d_hid, + num_heads=num_heads, + pair_bias=False, + d_pair=None, + ) + + def forward( + self, + m: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + m = m.transpose(-2, -3) + m = super().forward(m, attn_mask=attn_mask, chunk_size=chunk_size) + m = m.transpose(-2, -3) + + return m + + +class MSAColumnGlobalAttention(nn.Module): + + def __init__( + self, + d_in, + d_hid, + num_heads, + inf=1e9, + eps=1e-10, + ): + super(MSAColumnGlobalAttention, self).__init__() + + self.layer_norm_m = LayerNorm(d_in) + self.global_attention = GlobalAttention( + d_in, + d_hid, + num_heads, + inf=inf, + eps=eps, + ) + + @torch.jit.ignore + def _chunk( + self, + m: torch.Tensor, + mask: torch.Tensor, + chunk_size: int, + ) -> torch.Tensor: + return chunk_layer( + self._attn_forward, + { + 'm': m, + 'mask': mask + }, + chunk_size=chunk_size, + num_batch_dims=len(m.shape[:-2]), + ) + + def _attn_forward(self, m, mask): + m = self.layer_norm_m(m) + return self.global_attention(m, mask=mask) + + def forward( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + + m = m.transpose(-2, -3) + mask = mask.transpose(-1, -2) + + if chunk_size is not None: + m = self._chunk(m, mask, chunk_size) + else: + m = self._attn_forward(m, mask=mask) + + m = m.transpose(-2, -3) + return m + + +def gen_tri_attn_mask(mask, inf): + start_mask = gen_attn_mask(mask, -inf)[..., :, None, None, :] + end_mask = gen_attn_mask(mask.transpose(-1, -2), -inf)[..., :, None, + None, :] + return start_mask, end_mask + + +class TriangleAttention(nn.Module): + + def __init__( + self, + d_in, + d_hid, + num_heads, + starting, + ): + super(TriangleAttention, self).__init__() + self.starting = starting + self.layer_norm = LayerNorm(d_in) + self.linear = Linear(d_in, num_heads, bias=False, init='normal') + self.mha = Attention(d_in, d_in, d_in, d_hid, num_heads) + + @torch.jit.ignore + def _chunk( + self, + x: torch.Tensor, + mask: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + chunk_size: int = None, + ) -> torch.Tensor: + return chunk_layer( + self.mha, + { + 'q': x, + 'k': x, + 'v': x, + 'mask': mask, + 'bias': bias + }, + chunk_size=chunk_size, + num_batch_dims=len(x.shape[:-2]), + ) + + def forward( + self, + x: torch.Tensor, + attn_mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + if not self.starting: + x = x.transpose(-2, -3) + + x = self.layer_norm(x) + triangle_bias = ( + permute_final_dims(self.linear(x), + (2, 0, 1)).unsqueeze(-4).contiguous()) + + if chunk_size is not None: + x = self._chunk(x, attn_mask, triangle_bias, chunk_size) + else: + x = self.mha(q=x, k=x, v=x, mask=attn_mask, bias=triangle_bias) + + if not self.starting: + x = x.transpose(-2, -3) + return x + + def get_output_bias(self): + return self.mha.get_output_bias() + + +class TriangleAttentionStarting(TriangleAttention): + __init__ = partialmethod(TriangleAttention.__init__, starting=True) + + +class TriangleAttentionEnding(TriangleAttention): + __init__ = partialmethod(TriangleAttention.__init__, starting=False) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/auxillary_heads.py b/runtime/Lib/site_packages/models/science/unifold/modules/auxillary_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..2daf5d55b6adfda384bd7c65d3ed388156bba9a2 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/auxillary_heads.py @@ -0,0 +1,171 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from typing import Dict + +import torch.nn as nn +from unicore.modules import LayerNorm + +from .common import Linear +from .confidence import (predicted_aligned_error, predicted_lddt, + predicted_tm_score) + + +class AuxiliaryHeads(nn.Module): + + def __init__(self, config): + super(AuxiliaryHeads, self).__init__() + + self.plddt = PredictedLDDTHead(**config['plddt'], ) + + self.distogram = DistogramHead(**config['distogram'], ) + + self.masked_msa = MaskedMSAHead(**config['masked_msa'], ) + + if config.experimentally_resolved.enabled: + self.experimentally_resolved = ExperimentallyResolvedHead( + **config['experimentally_resolved'], ) + + if config.pae.enabled: + self.pae = PredictedAlignedErrorHead(**config.pae, ) + + self.config = config + + def forward(self, outputs): + aux_out = {} + plddt_logits = self.plddt(outputs['sm']['single']) + aux_out['plddt_logits'] = plddt_logits + + aux_out['plddt'] = predicted_lddt(plddt_logits.detach()) + + distogram_logits = self.distogram(outputs['pair']) + aux_out['distogram_logits'] = distogram_logits + + masked_msa_logits = self.masked_msa(outputs['msa']) + aux_out['masked_msa_logits'] = masked_msa_logits + + if self.config.experimentally_resolved.enabled: + exp_res_logits = self.experimentally_resolved(outputs['single']) + aux_out['experimentally_resolved_logits'] = exp_res_logits + + if self.config.pae.enabled: + pae_logits = self.pae(outputs['pair']) + aux_out['pae_logits'] = pae_logits + pae_logits = pae_logits.detach() + aux_out.update( + predicted_aligned_error( + pae_logits, + **self.config.pae, + )) + aux_out['ptm'] = predicted_tm_score( + pae_logits, interface=False, **self.config.pae) + + iptm_weight = self.config.pae.get('iptm_weight', 0.0) + if iptm_weight > 0.0: + aux_out['iptm'] = predicted_tm_score( + pae_logits, + interface=True, + asym_id=outputs['asym_id'], + **self.config.pae, + ) + aux_out['iptm+ptm'] = ( + iptm_weight * aux_out['iptm'] + # noqa W504 + (1.0 - iptm_weight) * aux_out['ptm']) + + return aux_out + + +class PredictedLDDTHead(nn.Module): + + def __init__(self, num_bins, d_in, d_hid): + super(PredictedLDDTHead, self).__init__() + + self.num_bins = num_bins + self.d_in = d_in + self.d_hid = d_hid + + self.layer_norm = LayerNorm(self.d_in) + + self.linear_1 = Linear(self.d_in, self.d_hid, init='relu') + self.linear_2 = Linear(self.d_hid, self.d_hid, init='relu') + self.act = nn.GELU() + self.linear_3 = Linear(self.d_hid, self.num_bins, init='final') + + def forward(self, s): + s = self.layer_norm(s) + s = self.linear_1(s) + s = self.act(s) + s = self.linear_2(s) + s = self.act(s) + s = self.linear_3(s) + return s + + +class EnhancedHeadBase(nn.Module): + + def __init__(self, d_in, d_out, disable_enhance_head): + super(EnhancedHeadBase, self).__init__() + if disable_enhance_head: + self.layer_norm = None + self.linear_in = None + else: + self.layer_norm = LayerNorm(d_in) + self.linear_in = Linear(d_in, d_in, init='relu') + self.act = nn.GELU() + self.linear = Linear(d_in, d_out, init='final') + + def apply_alphafold_original_mode(self): + self.layer_norm = None + self.linear_in = None + + def forward(self, x): + if self.layer_norm is not None: + x = self.layer_norm(x) + x = self.act(self.linear_in(x)) + logits = self.linear(x) + return logits + + +class DistogramHead(EnhancedHeadBase): + + def __init__(self, d_pair, num_bins, disable_enhance_head, **kwargs): + super(DistogramHead, self).__init__( + d_in=d_pair, + d_out=num_bins, + disable_enhance_head=disable_enhance_head, + ) + + def forward(self, x): + logits = super().forward(x) + logits = logits + logits.transpose(-2, -3) + return logits + + +class PredictedAlignedErrorHead(EnhancedHeadBase): + + def __init__(self, d_pair, num_bins, disable_enhance_head, **kwargs): + super(PredictedAlignedErrorHead, self).__init__( + d_in=d_pair, + d_out=num_bins, + disable_enhance_head=disable_enhance_head, + ) + + +class MaskedMSAHead(EnhancedHeadBase): + + def __init__(self, d_msa, d_out, disable_enhance_head, **kwargs): + super(MaskedMSAHead, self).__init__( + d_in=d_msa, + d_out=d_out, + disable_enhance_head=disable_enhance_head, + ) + + +class ExperimentallyResolvedHead(EnhancedHeadBase): + + def __init__(self, d_single, d_out, disable_enhance_head, **kwargs): + super(ExperimentallyResolvedHead, self).__init__( + d_in=d_single, + d_out=d_out, + disable_enhance_head=disable_enhance_head, + ) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/common.py b/runtime/Lib/site_packages/models/science/unifold/modules/common.py new file mode 100644 index 0000000000000000000000000000000000000000..186f25671ac8d8f9797b25e9fcb4eaaa84b32d3e --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/common.py @@ -0,0 +1,387 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from functools import partial +from typing import Any, Callable, Dict, Iterable, List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from unicore.modules import LayerNorm +from unicore.utils import tensor_tree_map + + +class Linear(nn.Linear): + + def __init__( + self, + d_in: int, + d_out: int, + bias: bool = True, + init: str = 'default', + ): + super(Linear, self).__init__(d_in, d_out, bias=bias) + + self.use_bias = bias + + if self.use_bias: + with torch.no_grad(): + self.bias.fill_(0) + + if init == 'default': + self._trunc_normal_init(1.0) + elif init == 'relu': + self._trunc_normal_init(2.0) + elif init == 'glorot': + self._glorot_uniform_init() + elif init == 'gating': + self._zero_init(self.use_bias) + elif init == 'normal': + self._normal_init() + elif init == 'final': + self._zero_init(False) + else: + raise ValueError('Invalid init method.') + + def _trunc_normal_init(self, scale=1.0): + # Constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) + TRUNCATED_NORMAL_STDDEV_FACTOR = 0.87962566103423978 + _, fan_in = self.weight.shape + scale = scale / max(1, fan_in) + std = (scale**0.5) / TRUNCATED_NORMAL_STDDEV_FACTOR + nn.init.trunc_normal_(self.weight, mean=0.0, std=std) + + def _glorot_uniform_init(self): + nn.init.xavier_uniform_(self.weight, gain=1) + + def _zero_init(self, use_bias=True): + with torch.no_grad(): + self.weight.fill_(0.0) + if use_bias: + with torch.no_grad(): + self.bias.fill_(1.0) + + def _normal_init(self): + torch.nn.init.kaiming_normal_(self.weight, nonlinearity='linear') + + +class Transition(nn.Module): + + def __init__(self, d_in, n): + + super(Transition, self).__init__() + + self.d_in = d_in + self.n = n + + self.layer_norm = LayerNorm(self.d_in) + self.linear_1 = Linear(self.d_in, self.n * self.d_in, init='relu') + self.act = nn.GELU() + self.linear_2 = Linear(self.n * self.d_in, d_in, init='final') + + def _transition(self, x): + x = self.layer_norm(x) + x = self.linear_1(x) + x = self.act(x) + x = self.linear_2(x) + return x + + @torch.jit.ignore + def _chunk( + self, + x: torch.Tensor, + chunk_size: int, + ) -> torch.Tensor: + return chunk_layer( + self._transition, + {'x': x}, + chunk_size=chunk_size, + num_batch_dims=len(x.shape[:-2]), + ) + + def forward( + self, + x: torch.Tensor, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + + if chunk_size is not None: + x = self._chunk(x, chunk_size) + else: + x = self._transition(x=x) + + return x + + +class OuterProductMean(nn.Module): + + def __init__(self, d_msa, d_pair, d_hid, eps=1e-3): + super(OuterProductMean, self).__init__() + + self.d_msa = d_msa + self.d_pair = d_pair + self.d_hid = d_hid + self.eps = eps + + self.layer_norm = LayerNorm(d_msa) + self.linear_1 = Linear(d_msa, d_hid) + self.linear_2 = Linear(d_msa, d_hid) + self.linear_out = Linear(d_hid**2, d_pair, init='relu') + self.act = nn.GELU() + self.linear_z = Linear(self.d_pair, self.d_pair, init='final') + self.layer_norm_out = LayerNorm(self.d_pair) + + def _opm(self, a, b): + outer = torch.einsum('...bac,...dae->...bdce', a, b) + outer = outer.reshape(outer.shape[:-2] + (-1, )) + outer = self.linear_out(outer) + return outer + + @torch.jit.ignore + def _chunk(self, a: torch.Tensor, b: torch.Tensor, + chunk_size: int) -> torch.Tensor: + a = a.reshape((-1, ) + a.shape[-3:]) + b = b.reshape((-1, ) + b.shape[-3:]) + out = [] + # TODO: optimize this + for a_prime, b_prime in zip(a, b): + outer = chunk_layer( + partial(self._opm, b=b_prime), + {'a': a_prime}, + chunk_size=chunk_size, + num_batch_dims=1, + ) + out.append(outer) + if len(out) == 1: + outer = out[0].unsqueeze(0) + else: + outer = torch.stack(out, dim=0) + outer = outer.reshape(a.shape[:-3] + outer.shape[1:]) + + return outer + + def apply_alphafold_original_mode(self): + self.linear_z = None + self.layer_norm_out = None + + def forward( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + + m = self.layer_norm(m) + mask = mask.unsqueeze(-1) + if self.layer_norm_out is not None: + # for numerical stability + mask = mask * (mask.size(-2)**-0.5) + a = self.linear_1(m) + b = self.linear_2(m) + if self.training: + a = a * mask + b = b * mask + else: + a *= mask + b *= mask + + a = a.transpose(-2, -3) + b = b.transpose(-2, -3) + + if chunk_size is not None: + z = self._chunk(a, b, chunk_size) + else: + z = self._opm(a, b) + + norm = torch.einsum('...abc,...adc->...bdc', mask, mask) + z /= self.eps + norm + if self.layer_norm_out is not None: + z = self.act(z) + z = self.layer_norm_out(z) + z = self.linear_z(z) + return z + + +def residual(residual, x, training): + if training: + return x + residual + else: + residual += x + return residual + + +@torch.jit.script +def fused_bias_dropout_add( + x: torch.Tensor, + bias: torch.Tensor, + residual: torch.Tensor, + dropmask: torch.Tensor, + prob: float, +) -> torch.Tensor: + return (x + bias) * F.dropout(dropmask, p=prob, training=True) + residual + + +@torch.jit.script +def fused_bias_dropout_add_inference( + x: torch.Tensor, + bias: torch.Tensor, + residual: torch.Tensor, +) -> torch.Tensor: + residual += bias + x + return residual + + +def bias_dropout_residual(module, residual, x, dropout_shared_dim, prob, + training): + bias = module.get_output_bias() + if training: + shape = list(x.shape) + shape[dropout_shared_dim] = 1 + with torch.no_grad(): + mask = x.new_ones(shape) + return fused_bias_dropout_add(x, bias, residual, mask, prob) + else: + return fused_bias_dropout_add_inference(x, bias, residual) + + +@torch.jit.script +def fused_bias_gated_dropout_add( + x: torch.Tensor, + bias: torch.Tensor, + g: torch.Tensor, + g_bias: torch.Tensor, + residual: torch.Tensor, + dropout_mask: torch.Tensor, + prob: float, +) -> torch.Tensor: + return (torch.sigmoid(g + g_bias) * (x + bias)) * F.dropout( + dropout_mask, + p=prob, + training=True, + ) + residual + + +def tri_mul_residual( + module, + residual, + outputs, + dropout_shared_dim, + prob, + training, + block_size, +): + if training: + x, g = outputs + bias, g_bias = module.get_output_bias() + shape = list(x.shape) + shape[dropout_shared_dim] = 1 + with torch.no_grad(): + mask = x.new_ones(shape) + return fused_bias_gated_dropout_add( + x, + bias, + g, + g_bias, + residual, + mask, + prob, + ) + elif block_size is None: + x, g = outputs + bias, g_bias = module.get_output_bias() + residual += (torch.sigmoid(g + g_bias) * (x + bias)) + return residual + else: + # gated is not used here + residual += outputs + return residual + + +class SimpleModuleList(nn.ModuleList): + + def __repr__(self): + return str(len(self)) + ' X ...\n' + self[0].__repr__() + + +def chunk_layer( + layer: Callable, + inputs: Dict[str, Any], + chunk_size: int, + num_batch_dims: int, +) -> Any: + # TODO: support inplace add to output + if not (len(inputs) > 0): + raise ValueError('Must provide at least one input') + + def _dict_get_shapes(input): + shapes = [] + if type(input) is torch.Tensor: + shapes.append(input.shape) + elif type(input) is dict: + for v in input.values(): + shapes.extend(_dict_get_shapes(v)) + elif isinstance(input, Iterable): + for v in input: + shapes.extend(_dict_get_shapes(v)) + else: + raise ValueError('Not supported') + + return shapes + + inputs = {k: v for k, v in inputs.items() if v is not None} + initial_dims = [ + shape[:num_batch_dims] for shape in _dict_get_shapes(inputs) + ] + orig_batch_dims = tuple([max(s) for s in zip(*initial_dims)]) + + flat_batch_dim = 1 + for d in orig_batch_dims: + flat_batch_dim *= d + num_chunks = (flat_batch_dim + chunk_size - 1) // chunk_size + + def _flat_inputs(t): + t = t.view(-1, *t.shape[num_batch_dims:]) + assert ( + t.shape[0] == flat_batch_dim or t.shape[0] == 1 + ), 'batch dimension must be 1 or equal to the flat batch dimension' + return t + + flat_inputs = tensor_tree_map(_flat_inputs, inputs) + + out = None + for i in range(num_chunks): + chunk_start = i * chunk_size + chunk_end = min((i + 1) * chunk_size, flat_batch_dim) + + def select_chunk(t): + if t.shape[0] == 1: + return t[0:1] + else: + return t[chunk_start:chunk_end] + + chunkes = tensor_tree_map(select_chunk, flat_inputs) + + output_chunk = layer(**chunkes) + + if out is None: + out = tensor_tree_map( + lambda t: t.new_zeros((flat_batch_dim, ) + t.shape[1:]), + output_chunk) + + out_type = type(output_chunk) + if out_type is tuple: + for x, y in zip(out, output_chunk): + x[chunk_start:chunk_end] = y + elif out_type is torch.Tensor: + out[chunk_start:chunk_end] = output_chunk + else: + raise ValueError('Not supported') + + # reshape = lambda t: t.view(orig_batch_dims + t.shape[1:]) + def reshape(t): + return t.view(orig_batch_dims + t.shape[1:]) + + out = tensor_tree_map(reshape, out) + + return out diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/confidence.py b/runtime/Lib/site_packages/models/science/unifold/modules/confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..7574689c3331ff571b90b06946cf9a2f9ad9aaaf --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/confidence.py @@ -0,0 +1,159 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from typing import Dict, Optional, Tuple + +import torch + + +def predicted_lddt(plddt_logits: torch.Tensor) -> torch.Tensor: + """Computes per-residue pLDDT from logits. + Args: + logits: [num_res, num_bins] output from the PredictedLDDTHead. + Returns: + plddt: [num_res] per-residue pLDDT. + """ + num_bins = plddt_logits.shape[-1] + bin_probs = torch.nn.functional.softmax(plddt_logits.float(), dim=-1) + bin_width = 1.0 / num_bins + bounds = torch.arange( + start=0.5 * bin_width, + end=1.0, + step=bin_width, + device=plddt_logits.device) + plddt = torch.sum( + bin_probs + * bounds.view(*((1, ) * len(bin_probs.shape[:-1])), *bounds.shape), + dim=-1, + ) + return plddt + + +def compute_bin_values(breaks: torch.Tensor): + """Gets the bin centers from the bin edges. + Args: + breaks: [num_bins - 1] the error bin edges. + Returns: + bin_centers: [num_bins] the error bin centers. + """ + step = breaks[1] - breaks[0] + bin_values = breaks + step / 2 + bin_values = torch.cat([bin_values, (bin_values[-1] + step).unsqueeze(-1)], + dim=0) + return bin_values + + +def compute_predicted_aligned_error( + bin_edges: torch.Tensor, + bin_probs: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Calculates expected aligned distance errors for every pair of residues. + Args: + alignment_confidence_breaks: [num_bins - 1] the error bin edges. + aligned_distance_error_probs: [num_res, num_res, num_bins] the predicted + probs for each error bin, for each pair of residues. + Returns: + predicted_aligned_error: [num_res, num_res] the expected aligned distance + error for each pair of residues. + max_predicted_aligned_error: The maximum predicted error possible. + """ + bin_values = compute_bin_values(bin_edges) + return torch.sum(bin_probs * bin_values, dim=-1) + + +def predicted_aligned_error( + pae_logits: torch.Tensor, + max_bin: int = 31, + num_bins: int = 64, + **kwargs, +) -> Dict[str, torch.Tensor]: + """Computes aligned confidence metrics from logits. + Args: + logits: [num_res, num_res, num_bins] the logits output from + PredictedAlignedErrorHead. + breaks: [num_bins - 1] the error bin edges. + Returns: + aligned_confidence_probs: [num_res, num_res, num_bins] the predicted + aligned error probabilities over bins for each residue pair. + predicted_aligned_error: [num_res, num_res] the expected aligned distance + error for each pair of residues. + max_predicted_aligned_error: The maximum predicted error possible. + """ + bin_probs = torch.nn.functional.softmax(pae_logits.float(), dim=-1) + bin_edges = torch.linspace( + 0, max_bin, steps=(num_bins - 1), device=pae_logits.device) + + predicted_aligned_error = compute_predicted_aligned_error( + bin_edges=bin_edges, + bin_probs=bin_probs, + ) + + return { + 'aligned_error_probs_per_bin': bin_probs, + 'predicted_aligned_error': predicted_aligned_error, + } + + +def predicted_tm_score( + pae_logits: torch.Tensor, + residue_weights: Optional[torch.Tensor] = None, + max_bin: int = 31, + num_bins: int = 64, + eps: float = 1e-8, + asym_id: Optional[torch.Tensor] = None, + interface: bool = False, + **kwargs, +) -> torch.Tensor: + """Computes predicted TM alignment or predicted interface TM alignment score. + Args: + logits: [num_res, num_res, num_bins] the logits output from + PredictedAlignedErrorHead. + breaks: [num_bins] the error bins. + residue_weights: [num_res] the per residue weights to use for the + expectation. + asym_id: [num_res] the asymmetric unit ID - the chain ID. Only needed for + ipTM calculation, i.e. when interface=True. + interface: If True, interface predicted TM score is computed. + Returns: + ptm_score: The predicted TM alignment or the predicted iTM score. + """ + pae_logits = pae_logits.float() + if residue_weights is None: + residue_weights = pae_logits.new_ones(pae_logits.shape[:-2]) + + breaks = torch.linspace( + 0, max_bin, steps=(num_bins - 1), device=pae_logits.device) + + def tm_kernal(nres): + clipped_n = max(nres, 19) + d0 = 1.24 * (clipped_n - 15)**(1.0 / 3.0) - 1.8 + return lambda x: 1.0 / (1.0 + (x / d0)**2) + + def rmsd_kernal(eps): # leave for compute pRMS + return lambda x: 1. / (x + eps) + + bin_centers = compute_bin_values(breaks) + probs = torch.nn.functional.softmax(pae_logits, dim=-1) + tm_per_bin = tm_kernal(nres=pae_logits.shape[-2])(bin_centers) + # tm_per_bin = 1.0 / (1 + (bin_centers ** 2) / (d0 ** 2)) + # rmsd_per_bin = rmsd_kernal()(bin_centers) + predicted_tm_term = torch.sum(probs * tm_per_bin, dim=-1) + + pair_mask = predicted_tm_term.new_ones(predicted_tm_term.shape) + if interface: + assert asym_id is not None, 'must provide asym_id for iptm calculation.' + pair_mask *= asym_id[..., :, None] != asym_id[..., None, :] + + predicted_tm_term *= pair_mask + + pair_residue_weights = pair_mask * ( + residue_weights[None, :] * residue_weights[:, None]) + normed_residue_mask = pair_residue_weights / ( + eps + pair_residue_weights.sum(dim=-1, keepdim=True)) + + per_alignment = torch.sum(predicted_tm_term * normed_residue_mask, dim=-1) + weighted = per_alignment * residue_weights + ret = per_alignment.gather( + dim=-1, index=weighted.max(dim=-1, + keepdim=True).indices).squeeze(dim=-1) + return ret diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/embedders.py b/runtime/Lib/site_packages/models/science/unifold/modules/embedders.py new file mode 100644 index 0000000000000000000000000000000000000000..84e87e2dc5f3fc04d1cbdf16274265af927c46d8 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/embedders.py @@ -0,0 +1,290 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from typing import Optional, Tuple + +import torch +import torch.nn as nn +from unicore.modules import LayerNorm +from unicore.utils import one_hot + +from .common import Linear, SimpleModuleList, residual + + +class InputEmbedder(nn.Module): + + def __init__( + self, + tf_dim: int, + msa_dim: int, + d_pair: int, + d_msa: int, + relpos_k: int, + use_chain_relative: bool = False, + max_relative_chain: Optional[int] = None, + **kwargs, + ): + super(InputEmbedder, self).__init__() + + self.tf_dim = tf_dim + self.msa_dim = msa_dim + + self.d_pair = d_pair + self.d_msa = d_msa + + self.linear_tf_z_i = Linear(tf_dim, d_pair) + self.linear_tf_z_j = Linear(tf_dim, d_pair) + self.linear_tf_m = Linear(tf_dim, d_msa) + self.linear_msa_m = Linear(msa_dim, d_msa) + + # RPE stuff + self.relpos_k = relpos_k + self.use_chain_relative = use_chain_relative + self.max_relative_chain = max_relative_chain + if not self.use_chain_relative: + self.num_bins = 2 * self.relpos_k + 1 + else: + self.num_bins = 2 * self.relpos_k + 2 + self.num_bins += 1 # entity id + self.num_bins += 2 * max_relative_chain + 2 + + self.linear_relpos = Linear(self.num_bins, d_pair) + + def _relpos_indices( + self, + res_id: torch.Tensor, + sym_id: Optional[torch.Tensor] = None, + asym_id: Optional[torch.Tensor] = None, + entity_id: Optional[torch.Tensor] = None, + ): + + max_rel_res = self.relpos_k + rp = res_id[..., None] - res_id[..., None, :] + rp = rp.clip(-max_rel_res, max_rel_res) + max_rel_res + if not self.use_chain_relative: + return rp + else: + asym_id_same = asym_id[..., :, None] == asym_id[..., None, :] + rp[~asym_id_same] = 2 * max_rel_res + 1 + entity_id_same = entity_id[..., :, None] == entity_id[..., None, :] + rp_entity_id = entity_id_same.type(rp.dtype)[..., None] + + rel_sym_id = sym_id[..., :, None] - sym_id[..., None, :] + + max_rel_chain = self.max_relative_chain + + clipped_rel_chain = torch.clamp( + rel_sym_id + max_rel_chain, min=0, max=2 * max_rel_chain) + + clipped_rel_chain[~entity_id_same] = 2 * max_rel_chain + 1 + return rp, rp_entity_id, clipped_rel_chain + + def relpos_emb( + self, + res_id: torch.Tensor, + sym_id: Optional[torch.Tensor] = None, + asym_id: Optional[torch.Tensor] = None, + entity_id: Optional[torch.Tensor] = None, + num_sym: Optional[torch.Tensor] = None, + ): + + dtype = self.linear_relpos.weight.dtype + if not self.use_chain_relative: + rp = self._relpos_indices(res_id=res_id) + return self.linear_relpos( + one_hot(rp, num_classes=self.num_bins, dtype=dtype)) + else: + rp, rp_entity_id, rp_rel_chain = self._relpos_indices( + res_id=res_id, + sym_id=sym_id, + asym_id=asym_id, + entity_id=entity_id) + rp = one_hot(rp, num_classes=(2 * self.relpos_k + 2), dtype=dtype) + rp_entity_id = rp_entity_id.type(dtype) + rp_rel_chain = one_hot( + rp_rel_chain, + num_classes=(2 * self.max_relative_chain + 2), + dtype=dtype) + return self.linear_relpos( + torch.cat([rp, rp_entity_id, rp_rel_chain], dim=-1)) + + def forward( + self, + tf: torch.Tensor, + msa: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # [*, N_res, d_pair] + if self.tf_dim == 21: + # multimer use 21 target dim + tf = tf[..., 1:] + # convert type if necessary + tf = tf.type(self.linear_tf_z_i.weight.dtype) + msa = msa.type(self.linear_tf_z_i.weight.dtype) + n_clust = msa.shape[-3] + + msa_emb = self.linear_msa_m(msa) + # target_feat (aatype) into msa representation + tf_m = ( + self.linear_tf_m(tf).unsqueeze(-3).expand( + ((-1, ) * len(tf.shape[:-2]) + # noqa W504 + (n_clust, -1, -1)))) + msa_emb += tf_m + + tf_emb_i = self.linear_tf_z_i(tf) + tf_emb_j = self.linear_tf_z_j(tf) + pair_emb = tf_emb_i[..., None, :] + tf_emb_j[..., None, :, :] + + return msa_emb, pair_emb + + +class RecyclingEmbedder(nn.Module): + + def __init__( + self, + d_msa: int, + d_pair: int, + min_bin: float, + max_bin: float, + num_bins: int, + inf: float = 1e8, + **kwargs, + ): + super(RecyclingEmbedder, self).__init__() + + self.d_msa = d_msa + self.d_pair = d_pair + self.min_bin = min_bin + self.max_bin = max_bin + self.num_bins = num_bins + self.inf = inf + + self.squared_bins = None + + self.linear = Linear(self.num_bins, self.d_pair) + self.layer_norm_m = LayerNorm(self.d_msa) + self.layer_norm_z = LayerNorm(self.d_pair) + + def forward( + self, + m: torch.Tensor, + z: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + m_update = self.layer_norm_m(m) + z_update = self.layer_norm_z(z) + + return m_update, z_update + + def recyle_pos( + self, + x: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + if self.squared_bins is None: + bins = torch.linspace( + self.min_bin, + self.max_bin, + self.num_bins, + dtype=torch.float if self.training else x.dtype, + device=x.device, + requires_grad=False, + ) + self.squared_bins = bins**2 + upper = torch.cat( + [self.squared_bins[1:], + self.squared_bins.new_tensor([self.inf])], + dim=-1) + if self.training: + x = x.float() + d = torch.sum( + (x[..., None, :] - x[..., None, :, :])**2, dim=-1, keepdims=True) + d = ((d > self.squared_bins) * # noqa W504 + (d < upper)).type(self.linear.weight.dtype) + d = self.linear(d) + return d + + +class TemplateAngleEmbedder(nn.Module): + + def __init__( + self, + d_in: int, + d_out: int, + **kwargs, + ): + super(TemplateAngleEmbedder, self).__init__() + + self.d_out = d_out + self.d_in = d_in + + self.linear_1 = Linear(self.d_in, self.d_out, init='relu') + self.act = nn.GELU() + self.linear_2 = Linear(self.d_out, self.d_out, init='relu') + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.linear_1(x.type(self.linear_1.weight.dtype)) + x = self.act(x) + x = self.linear_2(x) + return x + + +class TemplatePairEmbedder(nn.Module): + + def __init__( + self, + d_in: int, + v2_d_in: list, + d_out: int, + d_pair: int, + v2_feature: bool = False, + **kwargs, + ): + super(TemplatePairEmbedder, self).__init__() + + self.d_out = d_out + self.v2_feature = v2_feature + if self.v2_feature: + self.d_in = v2_d_in + self.linear = SimpleModuleList() + for d_in in self.d_in: + self.linear.append(Linear(d_in, self.d_out, init='relu')) + self.z_layer_norm = LayerNorm(d_pair) + self.z_linear = Linear(d_pair, self.d_out, init='relu') + else: + self.d_in = d_in + self.linear = Linear(self.d_in, self.d_out, init='relu') + + def forward( + self, + x, + z, + ) -> torch.Tensor: + if not self.v2_feature: + x = self.linear(x.type(self.linear.weight.dtype)) + return x + else: + dtype = self.z_linear.weight.dtype + t = self.linear[0](x[0].type(dtype)) + for i, s in enumerate(x[1:]): + t = residual(t, self.linear[i + 1](s.type(dtype)), + self.training) + t = residual(t, self.z_linear(self.z_layer_norm(z)), self.training) + return t + + +class ExtraMSAEmbedder(nn.Module): + + def __init__( + self, + d_in: int, + d_out: int, + **kwargs, + ): + super(ExtraMSAEmbedder, self).__init__() + + self.d_in = d_in + self.d_out = d_out + self.linear = Linear(self.d_in, self.d_out) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x.type(self.linear.weight.dtype)) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/evoformer.py b/runtime/Lib/site_packages/models/science/unifold/modules/evoformer.py new file mode 100644 index 0000000000000000000000000000000000000000..b083498691cad3836a6623e3a8b687c1031e2fde --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/evoformer.py @@ -0,0 +1,362 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from functools import partial +from typing import Optional, Tuple + +import torch +import torch.nn as nn +from unicore.utils import checkpoint_sequential + +from .attentions import (MSAColumnAttention, MSAColumnGlobalAttention, + MSARowAttentionWithPairBias, TriangleAttentionEnding, + TriangleAttentionStarting) +from .common import (Linear, OuterProductMean, SimpleModuleList, Transition, + bias_dropout_residual, residual, tri_mul_residual) +from .triangle_multiplication import (TriangleMultiplicationIncoming, + TriangleMultiplicationOutgoing) + + +class EvoformerIteration(nn.Module): + + def __init__( + self, + d_msa: int, + d_pair: int, + d_hid_msa_att: int, + d_hid_opm: int, + d_hid_mul: int, + d_hid_pair_att: int, + num_heads_msa: int, + num_heads_pair: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + outer_product_mean_first: bool, + inf: float, + eps: float, + _is_extra_msa_stack: bool = False, + ): + super(EvoformerIteration, self).__init__() + + self._is_extra_msa_stack = _is_extra_msa_stack + self.outer_product_mean_first = outer_product_mean_first + + self.msa_att_row = MSARowAttentionWithPairBias( + d_msa=d_msa, + d_pair=d_pair, + d_hid=d_hid_msa_att, + num_heads=num_heads_msa, + ) + + if _is_extra_msa_stack: + self.msa_att_col = MSAColumnGlobalAttention( + d_in=d_msa, + d_hid=d_hid_msa_att, + num_heads=num_heads_msa, + inf=inf, + eps=eps, + ) + else: + self.msa_att_col = MSAColumnAttention( + d_msa, + d_hid_msa_att, + num_heads_msa, + ) + + self.msa_transition = Transition( + d_in=d_msa, + n=transition_n, + ) + + self.outer_product_mean = OuterProductMean( + d_msa, + d_pair, + d_hid_opm, + ) + + self.tri_mul_out = TriangleMultiplicationOutgoing( + d_pair, + d_hid_mul, + ) + self.tri_mul_in = TriangleMultiplicationIncoming( + d_pair, + d_hid_mul, + ) + + self.tri_att_start = TriangleAttentionStarting( + d_pair, + d_hid_pair_att, + num_heads_pair, + ) + self.tri_att_end = TriangleAttentionEnding( + d_pair, + d_hid_pair_att, + num_heads_pair, + ) + + self.pair_transition = Transition( + d_in=d_pair, + n=transition_n, + ) + + self.row_dropout_share_dim = -3 + self.col_dropout_share_dim = -2 + self.msa_dropout = msa_dropout + self.pair_dropout = pair_dropout + + def forward( + self, + m: torch.Tensor, + z: torch.Tensor, + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + msa_row_attn_mask: torch.Tensor, + msa_col_attn_mask: Optional[torch.Tensor], + tri_start_attn_mask: torch.Tensor, + tri_end_attn_mask: torch.Tensor, + chunk_size: Optional[int] = None, + block_size: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + if self.outer_product_mean_first: + z = residual( + z, + self.outer_product_mean( + m, mask=msa_mask, chunk_size=chunk_size), self.training) + + m = bias_dropout_residual( + self.msa_att_row, + m, + self.msa_att_row( + m, z=z, attn_mask=msa_row_attn_mask, chunk_size=chunk_size), + self.row_dropout_share_dim, + self.msa_dropout, + self.training, + ) + if self._is_extra_msa_stack: + m = residual( + m, self.msa_att_col(m, mask=msa_mask, chunk_size=chunk_size), + self.training) + else: + m = bias_dropout_residual( + self.msa_att_col, + m, + self.msa_att_col( + m, attn_mask=msa_col_attn_mask, chunk_size=chunk_size), + self.col_dropout_share_dim, + self.msa_dropout, + self.training, + ) + m = residual(m, self.msa_transition(m, chunk_size=chunk_size), + self.training) + if not self.outer_product_mean_first: + z = residual( + z, + self.outer_product_mean( + m, mask=msa_mask, chunk_size=chunk_size), self.training) + + z = tri_mul_residual( + self.tri_mul_out, + z, + self.tri_mul_out(z, mask=pair_mask, block_size=block_size), + self.row_dropout_share_dim, + self.pair_dropout, + self.training, + block_size=block_size, + ) + + z = tri_mul_residual( + self.tri_mul_in, + z, + self.tri_mul_in(z, mask=pair_mask, block_size=block_size), + self.row_dropout_share_dim, + self.pair_dropout, + self.training, + block_size=block_size, + ) + + z = bias_dropout_residual( + self.tri_att_start, + z, + self.tri_att_start( + z, attn_mask=tri_start_attn_mask, chunk_size=chunk_size), + self.row_dropout_share_dim, + self.pair_dropout, + self.training, + ) + + z = bias_dropout_residual( + self.tri_att_end, + z, + self.tri_att_end( + z, attn_mask=tri_end_attn_mask, chunk_size=chunk_size), + self.col_dropout_share_dim, + self.pair_dropout, + self.training, + ) + z = residual(z, self.pair_transition(z, chunk_size=chunk_size), + self.training) + return m, z + + +class EvoformerStack(nn.Module): + + def __init__( + self, + d_msa: int, + d_pair: int, + d_hid_msa_att: int, + d_hid_opm: int, + d_hid_mul: int, + d_hid_pair_att: int, + d_single: int, + num_heads_msa: int, + num_heads_pair: int, + num_blocks: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + outer_product_mean_first: bool, + inf: float, + eps: float, + _is_extra_msa_stack: bool = False, + **kwargs, + ): + super(EvoformerStack, self).__init__() + + self._is_extra_msa_stack = _is_extra_msa_stack + + self.blocks = SimpleModuleList() + + for _ in range(num_blocks): + self.blocks.append( + EvoformerIteration( + d_msa=d_msa, + d_pair=d_pair, + d_hid_msa_att=d_hid_msa_att, + d_hid_opm=d_hid_opm, + d_hid_mul=d_hid_mul, + d_hid_pair_att=d_hid_pair_att, + num_heads_msa=num_heads_msa, + num_heads_pair=num_heads_pair, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + outer_product_mean_first=outer_product_mean_first, + inf=inf, + eps=eps, + _is_extra_msa_stack=_is_extra_msa_stack, + )) + if not self._is_extra_msa_stack: + self.linear = Linear(d_msa, d_single) + else: + self.linear = None + + def forward( + self, + m: torch.Tensor, + z: torch.Tensor, + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + msa_row_attn_mask: torch.Tensor, + msa_col_attn_mask: torch.Tensor, + tri_start_attn_mask: torch.Tensor, + tri_end_attn_mask: torch.Tensor, + chunk_size: int, + block_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + blocks = [ + partial( + b, + msa_mask=msa_mask, + pair_mask=pair_mask, + msa_row_attn_mask=msa_row_attn_mask, + msa_col_attn_mask=msa_col_attn_mask, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + chunk_size=chunk_size, + block_size=block_size) for b in self.blocks + ] + + m, z = checkpoint_sequential( + blocks, + input=(m, z), + ) + + s = None + if not self._is_extra_msa_stack: + seq_dim = -3 + index = torch.tensor([0], device=m.device) + s = self.linear(torch.index_select(m, dim=seq_dim, index=index)) + s = s.squeeze(seq_dim) + + return m, z, s + + +class ExtraMSAStack(EvoformerStack): + + def __init__( + self, + d_msa: int, + d_pair: int, + d_hid_msa_att: int, + d_hid_opm: int, + d_hid_mul: int, + d_hid_pair_att: int, + num_heads_msa: int, + num_heads_pair: int, + num_blocks: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + outer_product_mean_first: bool, + inf: float, + eps: float, + **kwargs, + ): + super(ExtraMSAStack, self).__init__( + d_msa=d_msa, + d_pair=d_pair, + d_hid_msa_att=d_hid_msa_att, + d_hid_opm=d_hid_opm, + d_hid_mul=d_hid_mul, + d_hid_pair_att=d_hid_pair_att, + d_single=None, + num_heads_msa=num_heads_msa, + num_heads_pair=num_heads_pair, + num_blocks=num_blocks, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + outer_product_mean_first=outer_product_mean_first, + inf=inf, + eps=eps, + _is_extra_msa_stack=True, + ) + + def forward( + self, + m: torch.Tensor, + z: torch.Tensor, + msa_mask: Optional[torch.Tensor] = None, + pair_mask: Optional[torch.Tensor] = None, + msa_row_attn_mask: torch.Tensor = None, + msa_col_attn_mask: torch.Tensor = None, + tri_start_attn_mask: torch.Tensor = None, + tri_end_attn_mask: torch.Tensor = None, + chunk_size: int = None, + block_size: int = None, + ) -> torch.Tensor: + _, z, _ = super().forward( + m, + z, + msa_mask=msa_mask, + pair_mask=pair_mask, + msa_row_attn_mask=msa_row_attn_mask, + msa_col_attn_mask=msa_col_attn_mask, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + chunk_size=chunk_size, + block_size=block_size) + return z diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/featurization.py b/runtime/Lib/site_packages/models/science/unifold/modules/featurization.py new file mode 100644 index 0000000000000000000000000000000000000000..b62adc9d705df534b470db12eb7e12b1cf535447 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/featurization.py @@ -0,0 +1,195 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from typing import Dict + +import torch +import torch.nn as nn +from unicore.utils import batched_gather, one_hot + +from modelscope.models.science.unifold.data import residue_constants as rc +from .frame import Frame + + +def pseudo_beta_fn(aatype, all_atom_positions, all_atom_masks): + is_gly = aatype == rc.restype_order['G'] + ca_idx = rc.atom_order['CA'] + cb_idx = rc.atom_order['CB'] + pseudo_beta = torch.where( + is_gly[..., None].expand(*((-1, ) * len(is_gly.shape)), 3), + all_atom_positions[..., ca_idx, :], + all_atom_positions[..., cb_idx, :], + ) + + if all_atom_masks is not None: + pseudo_beta_mask = torch.where( + is_gly, + all_atom_masks[..., ca_idx], + all_atom_masks[..., cb_idx], + ) + return pseudo_beta, pseudo_beta_mask + else: + return pseudo_beta + + +def atom14_to_atom37(atom14, batch): + atom37_data = batched_gather( + atom14, + batch['residx_atom37_to_atom14'], + dim=-2, + num_batch_dims=len(atom14.shape[:-2]), + ) + + atom37_data = atom37_data * batch['atom37_atom_exists'][..., None] + + return atom37_data + + +def build_template_angle_feat(template_feats, v2_feature=False): + template_aatype = template_feats['template_aatype'] + torsion_angles_sin_cos = template_feats['template_torsion_angles_sin_cos'] + torsion_angles_mask = template_feats['template_torsion_angles_mask'] + if not v2_feature: + alt_torsion_angles_sin_cos = template_feats[ + 'template_alt_torsion_angles_sin_cos'] + template_angle_feat = torch.cat( + [ + one_hot(template_aatype, 22), + torsion_angles_sin_cos.reshape( + *torsion_angles_sin_cos.shape[:-2], 14), + alt_torsion_angles_sin_cos.reshape( + *alt_torsion_angles_sin_cos.shape[:-2], 14), + torsion_angles_mask, + ], + dim=-1, + ) + template_angle_mask = torsion_angles_mask[..., 2] + else: + chi_mask = torsion_angles_mask[..., 3:] + chi_angles_sin = torsion_angles_sin_cos[..., 3:, 0] * chi_mask + chi_angles_cos = torsion_angles_sin_cos[..., 3:, 1] * chi_mask + template_angle_feat = torch.cat( + [ + one_hot(template_aatype, 22), + chi_angles_sin, + chi_angles_cos, + chi_mask, + ], + dim=-1, + ) + template_angle_mask = chi_mask[..., 0] + return template_angle_feat, template_angle_mask + + +def build_template_pair_feat( + batch, + min_bin, + max_bin, + num_bins, + eps=1e-20, + inf=1e8, +): + template_mask = batch['template_pseudo_beta_mask'] + template_mask_2d = template_mask[..., None] * template_mask[..., None, :] + + tpb = batch['template_pseudo_beta'] + dgram = torch.sum( + (tpb[..., None, :] - tpb[..., None, :, :])**2, dim=-1, keepdim=True) + lower = torch.linspace(min_bin, max_bin, num_bins, device=tpb.device)**2 + upper = torch.cat([lower[1:], lower.new_tensor([inf])], dim=-1) + dgram = ((dgram > lower) * (dgram < upper)).type(dgram.dtype) + + to_concat = [dgram, template_mask_2d[..., None]] + + aatype_one_hot = nn.functional.one_hot( + batch['template_aatype'], + rc.restype_num + 2, + ) + + n_res = batch['template_aatype'].shape[-1] + to_concat.append(aatype_one_hot[..., None, :, :].expand( + *aatype_one_hot.shape[:-2], n_res, -1, -1)) + to_concat.append(aatype_one_hot[..., + None, :].expand(*aatype_one_hot.shape[:-2], + -1, n_res, -1)) + + to_concat.append(template_mask_2d.new_zeros(*template_mask_2d.shape, 3)) + to_concat.append(template_mask_2d[..., None]) + + act = torch.cat(to_concat, dim=-1) + act = act * template_mask_2d[..., None] + + return act + + +def build_template_pair_feat_v2( + batch, + min_bin, + max_bin, + num_bins, + multichain_mask_2d=None, + eps=1e-20, + inf=1e8, +): + template_mask = batch['template_pseudo_beta_mask'] + template_mask_2d = template_mask[..., None] * template_mask[..., None, :] + if multichain_mask_2d is not None: + template_mask_2d *= multichain_mask_2d + + tpb = batch['template_pseudo_beta'] + dgram = torch.sum( + (tpb[..., None, :] - tpb[..., None, :, :])**2, dim=-1, keepdim=True) + lower = torch.linspace(min_bin, max_bin, num_bins, device=tpb.device)**2 + upper = torch.cat([lower[1:], lower.new_tensor([inf])], dim=-1) + dgram = ((dgram > lower) * (dgram < upper)).type(dgram.dtype) + dgram *= template_mask_2d[..., None] + to_concat = [dgram, template_mask_2d[..., None]] + + aatype_one_hot = one_hot( + batch['template_aatype'], + rc.restype_num + 2, + ) + + n_res = batch['template_aatype'].shape[-1] + to_concat.append(aatype_one_hot[..., None, :, :].expand( + *aatype_one_hot.shape[:-2], n_res, -1, -1)) + to_concat.append(aatype_one_hot[..., + None, :].expand(*aatype_one_hot.shape[:-2], + -1, n_res, -1)) + + n, ca, c = [rc.atom_order[a] for a in ['N', 'CA', 'C']] + rigids = Frame.make_transform_from_reference( + n_xyz=batch['template_all_atom_positions'][..., n, :], + ca_xyz=batch['template_all_atom_positions'][..., ca, :], + c_xyz=batch['template_all_atom_positions'][..., c, :], + eps=eps, + ) + points = rigids.get_trans()[..., None, :, :] + rigid_vec = rigids[..., None].invert_apply(points) + + inv_distance_scalar = torch.rsqrt(eps + torch.sum(rigid_vec**2, dim=-1)) + + t_aa_masks = batch['template_all_atom_mask'] + backbone_mask = t_aa_masks[..., n] * t_aa_masks[..., ca] * t_aa_masks[..., + c] + backbone_mask_2d = backbone_mask[..., :, None] * backbone_mask[..., + None, :] + if multichain_mask_2d is not None: + backbone_mask_2d *= multichain_mask_2d + + inv_distance_scalar = inv_distance_scalar * backbone_mask_2d + unit_vector_data = rigid_vec * inv_distance_scalar[..., None] + to_concat.extend(torch.unbind(unit_vector_data[..., None, :], dim=-1)) + to_concat.append(backbone_mask_2d[..., None]) + + return to_concat + + +def build_extra_msa_feat(batch): + msa_1hot = one_hot(batch['extra_msa'], 23) + msa_feat = [ + msa_1hot, + batch['extra_msa_has_deletion'].unsqueeze(-1), + batch['extra_msa_deletion_value'].unsqueeze(-1), + ] + return torch.cat(msa_feat, dim=-1) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/frame.py b/runtime/Lib/site_packages/models/science/unifold/modules/frame.py new file mode 100644 index 0000000000000000000000000000000000000000..5a0e4d6a7bfc5d02325857d0b27ec13635f95827 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/frame.py @@ -0,0 +1,562 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from __future__ import annotations # noqa +from typing import Any, Callable, Iterable, Optional, Sequence, Tuple + +import numpy as np +import torch + + +def zero_translation( + batch_dims: Tuple[int], + dtype: Optional[torch.dtype] = torch.float, + device: Optional[torch.device] = torch.device('cpu'), + requires_grad: bool = False, +) -> torch.Tensor: + trans = torch.zeros((*batch_dims, 3), + dtype=dtype, + device=device, + requires_grad=requires_grad) + return trans + + +# pylint: disable=bad-whitespace +_QUAT_TO_ROT = np.zeros((4, 4, 3, 3), dtype=np.float32) + +_QUAT_TO_ROT[0, 0] = [[1, 0, 0], [0, 1, 0], [0, 0, 1]] # rr +_QUAT_TO_ROT[1, 1] = [[1, 0, 0], [0, -1, 0], [0, 0, -1]] # ii +_QUAT_TO_ROT[2, 2] = [[-1, 0, 0], [0, 1, 0], [0, 0, -1]] # jj +_QUAT_TO_ROT[3, 3] = [[-1, 0, 0], [0, -1, 0], [0, 0, 1]] # kk + +_QUAT_TO_ROT[1, 2] = [[0, 2, 0], [2, 0, 0], [0, 0, 0]] # ij +_QUAT_TO_ROT[1, 3] = [[0, 0, 2], [0, 0, 0], [2, 0, 0]] # ik +_QUAT_TO_ROT[2, 3] = [[0, 0, 0], [0, 0, 2], [0, 2, 0]] # jk + +_QUAT_TO_ROT[0, 1] = [[0, 0, 0], [0, 0, -2], [0, 2, 0]] # ir +_QUAT_TO_ROT[0, 2] = [[0, 0, 2], [0, 0, 0], [-2, 0, 0]] # jr +_QUAT_TO_ROT[0, 3] = [[0, -2, 0], [2, 0, 0], [0, 0, 0]] # kr + +_QUAT_TO_ROT = _QUAT_TO_ROT.reshape(4, 4, 9) +_QUAT_TO_ROT_tensor = torch.from_numpy(_QUAT_TO_ROT) + +_QUAT_MULTIPLY = np.zeros((4, 4, 4)) +_QUAT_MULTIPLY[:, :, 0] = [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], + [0, 0, 0, -1]] + +_QUAT_MULTIPLY[:, :, 1] = [[0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1], + [0, 0, -1, 0]] + +_QUAT_MULTIPLY[:, :, 2] = [[0, 0, 1, 0], [0, 0, 0, -1], [1, 0, 0, 0], + [0, 1, 0, 0]] + +_QUAT_MULTIPLY[:, :, 3] = [[0, 0, 0, 1], [0, 0, 1, 0], [0, -1, 0, 0], + [1, 0, 0, 0]] + +_QUAT_MULTIPLY_BY_VEC = _QUAT_MULTIPLY[:, 1:, :] +_QUAT_MULTIPLY_BY_VEC_tensor = torch.from_numpy(_QUAT_MULTIPLY_BY_VEC) + + +class Rotation: + + def __init__( + self, + mat: torch.Tensor, + ): + if mat.shape[-2:] != (3, 3): + raise ValueError(f'incorrect rotation shape: {mat.shape}') + self._mat = mat + + @staticmethod + def identity( + shape, + dtype: Optional[torch.dtype] = torch.float, + device: Optional[torch.device] = torch.device('cpu'), + requires_grad: bool = False, + ) -> Rotation: + mat = torch.eye( + 3, dtype=dtype, device=device, requires_grad=requires_grad) + mat = mat.view(*((1, ) * len(shape)), 3, 3) + mat = mat.expand(*shape, -1, -1) + return Rotation(mat) + + @staticmethod + def mat_mul_mat(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + return (a.float() @ b.float()).type(a.dtype) + + @staticmethod + def mat_mul_vec(r: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + return (r.float() @ t.float().unsqueeze(-1)).squeeze(-1).type(t.dtype) + + def __getitem__(self, index: Any) -> Rotation: + if not isinstance(index, tuple): + index = (index, ) + return Rotation(mat=self._mat[index + (slice(None), slice(None))]) + + def __mul__(self, right: Any) -> Rotation: + if isinstance(right, (int, float)): + return Rotation(mat=self._mat * right) + elif isinstance(right, torch.Tensor): + return Rotation(mat=self._mat * right[..., None, None]) + else: + raise TypeError( + f'multiplicand must be a tensor or a number, got {type(right)}.' + ) + + def __rmul__(self, left: Any) -> Rotation: + return self.__mul__(left) + + def __matmul__(self, other: Rotation) -> Rotation: + new_mat = Rotation.mat_mul_mat(self.rot_mat, other.rot_mat) + return Rotation(mat=new_mat) + + @property + def _inv_mat(self): + return self._mat.transpose(-1, -2) + + @property + def rot_mat(self) -> torch.Tensor: + return self._mat + + def invert(self) -> Rotation: + return Rotation(mat=self._inv_mat) + + def apply(self, pts: torch.Tensor) -> torch.Tensor: + return Rotation.mat_mul_vec(self._mat, pts) + + def invert_apply(self, pts: torch.Tensor) -> torch.Tensor: + return Rotation.mat_mul_vec(self._inv_mat, pts) + + # inherit tensor behaviors + @property + def shape(self) -> torch.Size: + s = self._mat.shape[:-2] + return s + + @property + def dtype(self) -> torch.dtype: + return self._mat.dtype + + @property + def device(self) -> torch.device: + return self._mat.device + + @property + def requires_grad(self) -> bool: + return self._mat.requires_grad + + def unsqueeze(self, dim: int) -> Rotation: + if dim >= len(self.shape): + raise ValueError('Invalid dimension') + + rot_mats = self._mat.unsqueeze(dim if dim >= 0 else dim - 2) + return Rotation(mat=rot_mats) + + def map_tensor_fn(self, fn: Callable[[torch.Tensor], + torch.Tensor]) -> Rotation: + mat = self._mat.view(self._mat.shape[:-2] + (9, )) + mat = torch.stack(list(map(fn, torch.unbind(mat, dim=-1))), dim=-1) + mat = mat.view(mat.shape[:-1] + (3, 3)) + return Rotation(mat=mat) + + @staticmethod + def cat(rs: Sequence[Rotation], dim: int) -> Rotation: + rot_mats = [r.rot_mat for r in rs] + rot_mats = torch.cat(rot_mats, dim=dim if dim >= 0 else dim - 2) + + return Rotation(mat=rot_mats) + + def cuda(self) -> Rotation: + return Rotation(mat=self._mat.cuda()) + + def to(self, device: Optional[torch.device], + dtype: Optional[torch.dtype]) -> Rotation: + return Rotation(mat=self._mat.to(device=device, dtype=dtype)) + + def type(self, dtype: Optional[torch.dtype]) -> Rotation: + return Rotation(mat=self._mat.type(dtype)) + + def detach(self) -> Rotation: + return Rotation(mat=self._mat.detach()) + + +class Frame: + + def __init__( + self, + rotation: Optional[Rotation], + translation: Optional[torch.Tensor], + ): + if rotation is None and translation is None: + rotation = Rotation.identity((0, )) + translation = zero_translation((0, )) + elif translation is None: + translation = zero_translation(rotation.shape, rotation.dtype, + rotation.device, + rotation.requires_grad) + + elif rotation is None: + rotation = Rotation.identity( + translation.shape[:-1], + translation.dtype, + translation.device, + translation.requires_grad, + ) + + if (rotation.shape != translation.shape[:-1]) or (rotation.device + != # noqa W504 + translation.device): + raise ValueError('RotationMatrix and translation incompatible') + + self._r = rotation + self._t = translation + + @staticmethod + def identity( + shape: Iterable[int], + dtype: Optional[torch.dtype] = torch.float, + device: Optional[torch.device] = torch.device('cpu'), + requires_grad: bool = False, + ) -> Frame: + return Frame( + Rotation.identity(shape, dtype, device, requires_grad), + zero_translation(shape, dtype, device, requires_grad), + ) + + def __getitem__( + self, + index: Any, + ) -> Frame: + if type(index) != tuple: + index = (index, ) + + return Frame( + self._r[index], + self._t[index + (slice(None), )], + ) + + def __mul__( + self, + right: torch.Tensor, + ) -> Frame: + if not (isinstance(right, torch.Tensor)): + raise TypeError('The other multiplicand must be a Tensor') + + new_rots = self._r * right + new_trans = self._t * right[..., None] + + return Frame(new_rots, new_trans) + + def __rmul__( + self, + left: torch.Tensor, + ) -> Frame: + return self.__mul__(left) + + @property + def shape(self) -> torch.Size: + s = self._t.shape[:-1] + return s + + @property + def device(self) -> torch.device: + return self._t.device + + def get_rots(self) -> Rotation: + return self._r + + def get_trans(self) -> torch.Tensor: + return self._t + + def compose( + self, + other: Frame, + ) -> Frame: + new_rot = self._r @ other._r + new_trans = self._r.apply(other._t) + self._t + return Frame(new_rot, new_trans) + + def apply( + self, + pts: torch.Tensor, + ) -> torch.Tensor: + rotated = self._r.apply(pts) + return rotated + self._t + + def invert_apply(self, pts: torch.Tensor) -> torch.Tensor: + pts = pts - self._t + return self._r.invert_apply(pts) + + def invert(self) -> Frame: + rot_inv = self._r.invert() + trn_inv = rot_inv.apply(self._t) + + return Frame(rot_inv, -1 * trn_inv) + + def map_tensor_fn(self, fn: Callable[[torch.Tensor], + torch.Tensor]) -> Frame: + new_rots = self._r.map_tensor_fn(fn) + new_trans = torch.stack( + list(map(fn, torch.unbind(self._t, dim=-1))), dim=-1) + + return Frame(new_rots, new_trans) + + def to_tensor_4x4(self) -> torch.Tensor: + tensor = self._t.new_zeros((*self.shape, 4, 4)) + tensor[..., :3, :3] = self._r.rot_mat + tensor[..., :3, 3] = self._t + tensor[..., 3, 3] = 1 + return tensor + + @staticmethod + def from_tensor_4x4(t: torch.Tensor) -> Frame: + if t.shape[-2:] != (4, 4): + raise ValueError('Incorrectly shaped input tensor') + + rots = Rotation(mat=t[..., :3, :3]) + trans = t[..., :3, 3] + + return Frame(rots, trans) + + @staticmethod + def from_3_points( + p_neg_x_axis: torch.Tensor, + origin: torch.Tensor, + p_xy_plane: torch.Tensor, + eps: float = 1e-8, + ) -> Frame: + p_neg_x_axis = torch.unbind(p_neg_x_axis, dim=-1) + origin = torch.unbind(origin, dim=-1) + p_xy_plane = torch.unbind(p_xy_plane, dim=-1) + + e0 = [c1 - c2 for c1, c2 in zip(origin, p_neg_x_axis)] + e1 = [c1 - c2 for c1, c2 in zip(p_xy_plane, origin)] + + denom = torch.sqrt(sum((c * c for c in e0)) + eps) + e0 = [c / denom for c in e0] + dot = sum((c1 * c2 for c1, c2 in zip(e0, e1))) + e1 = [c2 - c1 * dot for c1, c2 in zip(e0, e1)] + denom = torch.sqrt(sum((c * c for c in e1)) + eps) + e1 = [c / denom for c in e1] + e2 = [ + e0[1] * e1[2] - e0[2] * e1[1], + e0[2] * e1[0] - e0[0] * e1[2], + e0[0] * e1[1] - e0[1] * e1[0], + ] + + rots = torch.stack([c for tup in zip(e0, e1, e2) for c in tup], dim=-1) + rots = rots.reshape(rots.shape[:-1] + (3, 3)) + + rot_obj = Rotation(mat=rots) + + return Frame(rot_obj, torch.stack(origin, dim=-1)) + + def unsqueeze( + self, + dim: int, + ) -> Frame: + if dim >= len(self.shape): + raise ValueError('Invalid dimension') + rots = self._r.unsqueeze(dim) + trans = self._t.unsqueeze(dim if dim >= 0 else dim - 1) + + return Frame(rots, trans) + + @staticmethod + def cat( + Ts: Sequence[Frame], + dim: int, + ) -> Frame: + rots = Rotation.cat([T._r for T in Ts], dim) + trans = torch.cat([T._t for T in Ts], dim=dim if dim >= 0 else dim - 1) + + return Frame(rots, trans) + + def apply_rot_fn(self, fn: Callable[[Rotation], Rotation]) -> Frame: + return Frame(fn(self._r), self._t) + + def apply_trans_fn(self, fn: Callable[[torch.Tensor], + torch.Tensor]) -> Frame: + return Frame(self._r, fn(self._t)) + + def scale_translation(self, trans_scale_factor: float) -> Frame: + # fn = lambda t: t * trans_scale_factor + def fn(t): + return t * trans_scale_factor + + return self.apply_trans_fn(fn) + + def stop_rot_gradient(self) -> Frame: + # fn = lambda r: r.detach() + def fn(r): + return r.detach() + + return self.apply_rot_fn(fn) + + @staticmethod + def make_transform_from_reference(n_xyz, ca_xyz, c_xyz, eps=1e-20): + input_dtype = ca_xyz.dtype + n_xyz = n_xyz.float() + ca_xyz = ca_xyz.float() + c_xyz = c_xyz.float() + n_xyz = n_xyz - ca_xyz + c_xyz = c_xyz - ca_xyz + + c_x, c_y, d_pair = [c_xyz[..., i] for i in range(3)] + norm = torch.sqrt(eps + c_x**2 + c_y**2) + sin_c1 = -c_y / norm + cos_c1 = c_x / norm + + c1_rots = sin_c1.new_zeros((*sin_c1.shape, 3, 3)) + c1_rots[..., 0, 0] = cos_c1 + c1_rots[..., 0, 1] = -1 * sin_c1 + c1_rots[..., 1, 0] = sin_c1 + c1_rots[..., 1, 1] = cos_c1 + c1_rots[..., 2, 2] = 1 + + norm = torch.sqrt(eps + c_x**2 + c_y**2 + d_pair**2) + sin_c2 = d_pair / norm + cos_c2 = torch.sqrt(c_x**2 + c_y**2) / norm + + c2_rots = sin_c2.new_zeros((*sin_c2.shape, 3, 3)) + c2_rots[..., 0, 0] = cos_c2 + c2_rots[..., 0, 2] = sin_c2 + c2_rots[..., 1, 1] = 1 + c2_rots[..., 2, 0] = -1 * sin_c2 + c2_rots[..., 2, 2] = cos_c2 + + c_rots = Rotation.mat_mul_mat(c2_rots, c1_rots) + n_xyz = Rotation.mat_mul_vec(c_rots, n_xyz) + + _, n_y, n_z = [n_xyz[..., i] for i in range(3)] + norm = torch.sqrt(eps + n_y**2 + n_z**2) + sin_n = -n_z / norm + cos_n = n_y / norm + + n_rots = sin_c2.new_zeros((*sin_c2.shape, 3, 3)) + n_rots[..., 0, 0] = 1 + n_rots[..., 1, 1] = cos_n + n_rots[..., 1, 2] = -1 * sin_n + n_rots[..., 2, 1] = sin_n + n_rots[..., 2, 2] = cos_n + + rots = Rotation.mat_mul_mat(n_rots, c_rots) + + rots = rots.transpose(-1, -2) + rot_obj = Rotation(mat=rots.type(input_dtype)) + + return Frame(rot_obj, ca_xyz.type(input_dtype)) + + def cuda(self) -> Frame: + return Frame(self._r.cuda(), self._t.cuda()) + + @property + def dtype(self) -> torch.dtype: + assert self._r.dtype == self._t.dtype + return self._r.dtype + + def type(self, dtype) -> Frame: + return Frame(self._r.type(dtype), self._t.type(dtype)) + + +class Quaternion: + + def __init__(self, quaternion: torch.Tensor, translation: torch.Tensor): + if quaternion.shape[-1] != 4: + raise ValueError(f'incorrect quaternion shape: {quaternion.shape}') + self._q = quaternion + self._t = translation + + @staticmethod + def identity( + shape: Iterable[int], + dtype: Optional[torch.dtype] = torch.float, + device: Optional[torch.device] = torch.device('cpu'), + requires_grad: bool = False, + ) -> Quaternion: + trans = zero_translation(shape, dtype, device, requires_grad) + quats = torch.zeros((*shape, 4), + dtype=dtype, + device=device, + requires_grad=requires_grad) + with torch.no_grad(): + quats[..., 0] = 1 + return Quaternion(quats, trans) + + def get_quats(self): + return self._q + + def get_trans(self): + return self._t + + def get_rot_mats(self): + quats = self.get_quats() + rot_mats = Quaternion.quat_to_rot(quats) + return rot_mats + + @staticmethod + def quat_to_rot(normalized_quat): + global _QUAT_TO_ROT_tensor + dtype = normalized_quat.dtype + normalized_quat = normalized_quat.float() + if _QUAT_TO_ROT_tensor.device != normalized_quat.device: + _QUAT_TO_ROT_tensor = _QUAT_TO_ROT_tensor.to( + normalized_quat.device) + rot_tensor = torch.sum( + _QUAT_TO_ROT_tensor * normalized_quat[..., :, None, None] + * normalized_quat[..., None, :, None], + dim=(-3, -2), + ) + rot_tensor = rot_tensor.type(dtype) + rot_tensor = rot_tensor.view(*rot_tensor.shape[:-1], 3, 3) + return rot_tensor + + @staticmethod + def normalize_quat(quats): + dtype = quats.dtype + quats = quats.float() + quats = quats / torch.linalg.norm(quats, dim=-1, keepdim=True) + quats = quats.type(dtype) + return quats + + @staticmethod + def quat_multiply_by_vec(quat, vec): + dtype = quat.dtype + quat = quat.float() + vec = vec.float() + global _QUAT_MULTIPLY_BY_VEC_tensor + if _QUAT_MULTIPLY_BY_VEC_tensor.device != quat.device: + _QUAT_MULTIPLY_BY_VEC_tensor = _QUAT_MULTIPLY_BY_VEC_tensor.to( + quat.device) + mat = _QUAT_MULTIPLY_BY_VEC_tensor + reshaped_mat = mat.view((1, ) * len(quat.shape[:-1]) + mat.shape) + return torch.sum( + reshaped_mat * quat[..., :, None, None] * vec[..., None, :, None], + dim=(-3, -2), + ).type(dtype) + + def compose_q_update_vec(self, + q_update_vec: torch.Tensor, + normalize_quats: bool = True) -> torch.Tensor: + quats = self.get_quats() + new_quats = quats + Quaternion.quat_multiply_by_vec( + quats, q_update_vec) + if normalize_quats: + new_quats = Quaternion.normalize_quat(new_quats) + return new_quats + + def compose_update_vec( + self, + update_vec: torch.Tensor, + pre_rot_mat: Rotation, + ) -> Quaternion: + q_vec, t_vec = update_vec[..., :3], update_vec[..., 3:] + new_quats = self.compose_q_update_vec(q_vec) + + trans_update = pre_rot_mat.apply(t_vec) + new_trans = self._t + trans_update + + return Quaternion(new_quats, new_trans) + + def stop_rot_gradient(self) -> Quaternion: + return Quaternion(self._q.detach(), self._t) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/structure_module.py b/runtime/Lib/site_packages/models/science/unifold/modules/structure_module.py new file mode 100644 index 0000000000000000000000000000000000000000..4872d5c6448540f04475002b72aae690ee6de406 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/structure_module.py @@ -0,0 +1,592 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import math +from typing import Tuple + +import torch +import torch.nn as nn +from unicore.modules import LayerNorm, softmax_dropout +from unicore.utils import dict_multimap, one_hot, permute_final_dims + +from modelscope.models.science.unifold.data.residue_constants import ( + restype_atom14_mask, restype_atom14_rigid_group_positions, + restype_atom14_to_rigid_group, restype_rigid_group_default_frame) +from .attentions import gen_attn_mask +from .common import Linear, SimpleModuleList, residual +from .frame import Frame, Quaternion, Rotation + + +def ipa_point_weights_init_(weights): + with torch.no_grad(): + softplus_inverse_1 = 0.541324854612918 + weights.fill_(softplus_inverse_1) + + +def torsion_angles_to_frames( + frame: Frame, + alpha: torch.Tensor, + aatype: torch.Tensor, + default_frames: torch.Tensor, +): + default_frame = Frame.from_tensor_4x4(default_frames[aatype, ...]) + + bb_rot = alpha.new_zeros((*((1, ) * len(alpha.shape[:-1])), 2)) + bb_rot[..., 1] = 1 + + alpha = torch.cat([bb_rot.expand(*alpha.shape[:-2], -1, -1), alpha], + dim=-2) + + all_rots = alpha.new_zeros(default_frame.get_rots().rot_mat.shape) + all_rots[..., 0, 0] = 1 + all_rots[..., 1, 1] = alpha[..., 1] + all_rots[..., 1, 2] = -alpha[..., 0] + all_rots[..., 2, 1:] = alpha + + all_rots = Frame(Rotation(mat=all_rots), None) + + all_frames = default_frame.compose(all_rots) + + chi2_frame_to_frame = all_frames[..., 5] + chi3_frame_to_frame = all_frames[..., 6] + chi4_frame_to_frame = all_frames[..., 7] + + chi1_frame_to_bb = all_frames[..., 4] + chi2_frame_to_bb = chi1_frame_to_bb.compose(chi2_frame_to_frame) + chi3_frame_to_bb = chi2_frame_to_bb.compose(chi3_frame_to_frame) + chi4_frame_to_bb = chi3_frame_to_bb.compose(chi4_frame_to_frame) + + all_frames_to_bb = Frame.cat( + [ + all_frames[..., :5], + chi2_frame_to_bb.unsqueeze(-1), + chi3_frame_to_bb.unsqueeze(-1), + chi4_frame_to_bb.unsqueeze(-1), + ], + dim=-1, + ) + + all_frames_to_global = frame[..., None].compose(all_frames_to_bb) + + return all_frames_to_global + + +def frames_and_literature_positions_to_atom14_pos( + frame: Frame, + aatype: torch.Tensor, + default_frames, + group_idx, + atom_mask, + lit_positions, +): + group_mask = group_idx[aatype, ...] + group_mask = one_hot( + group_mask, + num_classes=default_frames.shape[-3], + ) + + t_atoms_to_global = frame[..., None, :] * group_mask + t_atoms_to_global = t_atoms_to_global.map_tensor_fn( + lambda x: torch.sum(x, dim=-1)) + + atom_mask = atom_mask[aatype, ...].unsqueeze(-1) + + lit_positions = lit_positions[aatype, ...] + pred_positions = t_atoms_to_global.apply(lit_positions) + pred_positions = pred_positions * atom_mask + + return pred_positions + + +class SideChainAngleResnetIteration(nn.Module): + + def __init__(self, d_hid): + super(SideChainAngleResnetIteration, self).__init__() + + self.d_hid = d_hid + + self.linear_1 = Linear(self.d_hid, self.d_hid, init='relu') + self.act = nn.GELU() + self.linear_2 = Linear(self.d_hid, self.d_hid, init='final') + + def forward(self, s: torch.Tensor) -> torch.Tensor: + + x = self.act(s) + x = self.linear_1(x) + x = self.act(x) + x = self.linear_2(x) + + return residual(s, x, self.training) + + +class SidechainAngleResnet(nn.Module): + + def __init__(self, d_in, d_hid, num_blocks, num_angles): + super(SidechainAngleResnet, self).__init__() + + self.linear_in = Linear(d_in, d_hid) + self.act = nn.GELU() + self.linear_initial = Linear(d_in, d_hid) + + self.layers = SimpleModuleList() + for _ in range(num_blocks): + self.layers.append(SideChainAngleResnetIteration(d_hid=d_hid)) + + self.linear_out = Linear(d_hid, num_angles * 2) + + def forward(self, s: torch.Tensor, + initial_s: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + + initial_s = self.linear_initial(self.act(initial_s)) + s = self.linear_in(self.act(s)) + + s = s + initial_s + + for layer in self.layers: + s = layer(s) + + s = self.linear_out(self.act(s)) + + s = s.view(s.shape[:-1] + (-1, 2)) + + unnormalized_s = s + norm_denom = torch.sqrt( + torch.clamp( + torch.sum(s.float()**2, dim=-1, keepdim=True), + min=1e-12, + )) + s = s.float() / norm_denom + + return unnormalized_s, s.type(unnormalized_s.dtype) + + +class InvariantPointAttention(nn.Module): + + def __init__( + self, + d_single: int, + d_pair: int, + d_hid: int, + num_heads: int, + num_qk_points: int, + num_v_points: int, + separate_kv: bool = False, + bias: bool = True, + eps: float = 1e-8, + ): + super(InvariantPointAttention, self).__init__() + + self.d_hid = d_hid + self.num_heads = num_heads + self.num_qk_points = num_qk_points + self.num_v_points = num_v_points + self.eps = eps + + hc = self.d_hid * self.num_heads + self.linear_q = Linear(d_single, hc, bias=bias) + self.separate_kv = separate_kv + if self.separate_kv: + self.linear_k = Linear(d_single, hc, bias=bias) + self.linear_v = Linear(d_single, hc, bias=bias) + else: + self.linear_kv = Linear(d_single, 2 * hc, bias=bias) + + hpq = self.num_heads * self.num_qk_points * 3 + self.linear_q_points = Linear(d_single, hpq) + hpk = self.num_heads * self.num_qk_points * 3 + hpv = self.num_heads * self.num_v_points * 3 + if self.separate_kv: + self.linear_k_points = Linear(d_single, hpk) + self.linear_v_points = Linear(d_single, hpv) + else: + hpkv = hpk + hpv + self.linear_kv_points = Linear(d_single, hpkv) + + self.linear_b = Linear(d_pair, self.num_heads) + + self.head_weights = nn.Parameter(torch.zeros((num_heads))) + ipa_point_weights_init_(self.head_weights) + + concat_out_dim = self.num_heads * ( + d_pair + self.d_hid + self.num_v_points * 4) + self.linear_out = Linear(concat_out_dim, d_single, init='final') + + self.softplus = nn.Softplus() + + def forward( + self, + s: torch.Tensor, + z: torch.Tensor, + f: Frame, + square_mask: torch.Tensor, + ) -> torch.Tensor: + q = self.linear_q(s) + + q = q.view(q.shape[:-1] + (self.num_heads, -1)) + + if self.separate_kv: + k = self.linear_k(s) + v = self.linear_v(s) + k = k.view(k.shape[:-1] + (self.num_heads, -1)) + v = v.view(v.shape[:-1] + (self.num_heads, -1)) + else: + kv = self.linear_kv(s) + kv = kv.view(kv.shape[:-1] + (self.num_heads, -1)) + k, v = torch.split(kv, self.d_hid, dim=-1) + + q_pts = self.linear_q_points(s) + + def process_points(pts, no_points): + shape = pts.shape[:-1] + (pts.shape[-1] // 3, 3) + if self.separate_kv: + # alphafold-multimer uses different layout + pts = pts.view(pts.shape[:-1] + + (self.num_heads, no_points * 3)) + pts = torch.split(pts, pts.shape[-1] // 3, dim=-1) + pts = torch.stack(pts, dim=-1).view(*shape) + pts = f[..., None].apply(pts) + + pts = pts.view(pts.shape[:-2] + (self.num_heads, no_points, 3)) + return pts + + q_pts = process_points(q_pts, self.num_qk_points) + + if self.separate_kv: + k_pts = self.linear_k_points(s) + v_pts = self.linear_v_points(s) + k_pts = process_points(k_pts, self.num_qk_points) + v_pts = process_points(v_pts, self.num_v_points) + else: + kv_pts = self.linear_kv_points(s) + + kv_pts = torch.split(kv_pts, kv_pts.shape[-1] // 3, dim=-1) + kv_pts = torch.stack(kv_pts, dim=-1) + kv_pts = f[..., None].apply(kv_pts) + + kv_pts = kv_pts.view(kv_pts.shape[:-2] + (self.num_heads, -1, 3)) + + k_pts, v_pts = torch.split( + kv_pts, [self.num_qk_points, self.num_v_points], dim=-2) + + bias = self.linear_b(z) + + attn = torch.matmul( + permute_final_dims(q, (1, 0, 2)), + permute_final_dims(k, (1, 2, 0)), + ) + + if self.training: + attn = attn * math.sqrt(1.0 / (3 * self.d_hid)) + attn = attn + ( + math.sqrt(1.0 / 3) * permute_final_dims(bias, (2, 0, 1))) + pt_att = q_pts.unsqueeze(-4) - k_pts.unsqueeze(-5) + pt_att = pt_att.float()**2 + else: + attn *= math.sqrt(1.0 / (3 * self.d_hid)) + attn += (math.sqrt(1.0 / 3) * permute_final_dims(bias, (2, 0, 1))) + pt_att = q_pts.unsqueeze(-4) - k_pts.unsqueeze(-5) + pt_att *= pt_att + + pt_att = pt_att.sum(dim=-1) + head_weights = self.softplus(self.head_weights).view( # noqa + *((1, ) * len(pt_att.shape[:-2]) + (-1, 1))) # noqa + head_weights = head_weights * math.sqrt( + 1.0 / (3 * (self.num_qk_points * 9.0 / 2))) + pt_att *= head_weights * (-0.5) + + pt_att = torch.sum(pt_att, dim=-1) + + pt_att = permute_final_dims(pt_att, (2, 0, 1)) + attn += square_mask + attn = softmax_dropout( + attn, 0, self.training, bias=pt_att.type(attn.dtype)) + del pt_att, q_pts, k_pts, bias + o = torch.matmul(attn, v.transpose(-2, -3)).transpose(-2, -3) + o = o.contiguous().view(*o.shape[:-2], -1) + del q, k, v + o_pts = torch.sum( + (attn[..., None, :, :, None] + * permute_final_dims(v_pts, (1, 3, 0, 2))[..., None, :, :]), + dim=-2, + ) + + o_pts = permute_final_dims(o_pts, (2, 0, 3, 1)) + o_pts = f[..., None, None].invert_apply(o_pts) + if self.training: + o_pts_norm = torch.sqrt( + torch.sum(o_pts.float()**2, dim=-1) + self.eps).type( + o_pts.dtype) + else: + o_pts_norm = torch.sqrt(torch.sum(o_pts**2, dim=-1) + + self.eps).type(o_pts.dtype) + + o_pts_norm = o_pts_norm.view(*o_pts_norm.shape[:-2], -1) + + o_pts = o_pts.view(*o_pts.shape[:-3], -1, 3) + + o_pair = torch.matmul(attn.transpose(-2, -3), z) + + o_pair = o_pair.view(*o_pair.shape[:-2], -1) + + s = self.linear_out( + torch.cat((o, *torch.unbind(o_pts, dim=-1), o_pts_norm, o_pair), + dim=-1)) + + return s + + +class BackboneUpdate(nn.Module): + + def __init__(self, d_single): + super(BackboneUpdate, self).__init__() + self.linear = Linear(d_single, 6, init='final') + + def forward(self, s: torch.Tensor): + return self.linear(s) + + +class StructureModuleTransitionLayer(nn.Module): + + def __init__(self, c): + super(StructureModuleTransitionLayer, self).__init__() + + self.linear_1 = Linear(c, c, init='relu') + self.linear_2 = Linear(c, c, init='relu') + self.act = nn.GELU() + self.linear_3 = Linear(c, c, init='final') + + def forward(self, s): + s_old = s + s = self.linear_1(s) + s = self.act(s) + s = self.linear_2(s) + s = self.act(s) + s = self.linear_3(s) + + s = residual(s_old, s, self.training) + + return s + + +class StructureModuleTransition(nn.Module): + + def __init__(self, c, num_layers, dropout_rate): + super(StructureModuleTransition, self).__init__() + + self.num_layers = num_layers + self.dropout_rate = dropout_rate + + self.layers = SimpleModuleList() + for _ in range(self.num_layers): + self.layers.append(StructureModuleTransitionLayer(c)) + + self.dropout = nn.Dropout(self.dropout_rate) + self.layer_norm = LayerNorm(c) + + def forward(self, s): + for layer in self.layers: + s = layer(s) + + s = self.dropout(s) + s = self.layer_norm(s) + + return s + + +class StructureModule(nn.Module): + + def __init__( + self, + d_single, + d_pair, + d_ipa, + d_angle, + num_heads_ipa, + num_qk_points, + num_v_points, + dropout_rate, + num_blocks, + no_transition_layers, + num_resnet_blocks, + num_angles, + trans_scale_factor, + separate_kv, + ipa_bias, + epsilon, + inf, + **kwargs, + ): + super(StructureModule, self).__init__() + + self.num_blocks = num_blocks + self.trans_scale_factor = trans_scale_factor + self.default_frames = None + self.group_idx = None + self.atom_mask = None + self.lit_positions = None + self.inf = inf + + self.layer_norm_s = LayerNorm(d_single) + self.layer_norm_z = LayerNorm(d_pair) + + self.linear_in = Linear(d_single, d_single) + + self.ipa = InvariantPointAttention( + d_single, + d_pair, + d_ipa, + num_heads_ipa, + num_qk_points, + num_v_points, + separate_kv=separate_kv, + bias=ipa_bias, + eps=epsilon, + ) + + self.ipa_dropout = nn.Dropout(dropout_rate) + self.layer_norm_ipa = LayerNorm(d_single) + + self.transition = StructureModuleTransition( + d_single, + no_transition_layers, + dropout_rate, + ) + + self.bb_update = BackboneUpdate(d_single) + + self.angle_resnet = SidechainAngleResnet( + d_single, + d_angle, + num_resnet_blocks, + num_angles, + ) + + def forward( + self, + s, + z, + aatype, + mask=None, + ): + if mask is None: + mask = s.new_ones(s.shape[:-1]) + + # generate square mask + square_mask = mask.unsqueeze(-1) * mask.unsqueeze(-2) + square_mask = gen_attn_mask(square_mask, -self.inf).unsqueeze(-3) + s = self.layer_norm_s(s) + z = self.layer_norm_z(z) + initial_s = s + s = self.linear_in(s) + + quat_encoder = Quaternion.identity( + s.shape[:-1], + s.dtype, + s.device, + requires_grad=False, + ) + backb_to_global = Frame( + Rotation(mat=quat_encoder.get_rot_mats(), ), + quat_encoder.get_trans(), + ) + outputs = [] + for i in range(self.num_blocks): + s = residual(s, self.ipa(s, z, backb_to_global, square_mask), + self.training) + s = self.ipa_dropout(s) + s = self.layer_norm_ipa(s) + s = self.transition(s) + + # update quaternion encoder + # use backb_to_global to avoid quat-to-rot conversion + quat_encoder = quat_encoder.compose_update_vec( + self.bb_update(s), pre_rot_mat=backb_to_global.get_rots()) + + # initial_s is always used to update the backbone + unnormalized_angles, angles = self.angle_resnet(s, initial_s) + + # convert quaternion to rotation matrix + backb_to_global = Frame( + Rotation(mat=quat_encoder.get_rot_mats(), ), + quat_encoder.get_trans(), + ) + if i == self.num_blocks - 1: + all_frames_to_global = self.torsion_angles_to_frames( + backb_to_global.scale_translation(self.trans_scale_factor), + angles, + aatype, + ) + + pred_positions = self.frames_and_literature_positions_to_atom14_pos( + all_frames_to_global, + aatype, + ) + + preds = { + 'frames': + backb_to_global.scale_translation( + self.trans_scale_factor).to_tensor_4x4(), + 'unnormalized_angles': + unnormalized_angles, + 'angles': + angles, + } + + outputs.append(preds) + if i < (self.num_blocks - 1): + # stop gradient in iteration + quat_encoder = quat_encoder.stop_rot_gradient() + backb_to_global = backb_to_global.stop_rot_gradient() + + outputs = dict_multimap(torch.stack, outputs) + outputs['sidechain_frames'] = all_frames_to_global.to_tensor_4x4() + outputs['positions'] = pred_positions + outputs['single'] = s + + return outputs + + def _init_residue_constants(self, float_dtype, device): + if self.default_frames is None: + self.default_frames = torch.tensor( + restype_rigid_group_default_frame, + dtype=float_dtype, + device=device, + requires_grad=False, + ) + if self.group_idx is None: + self.group_idx = torch.tensor( + restype_atom14_to_rigid_group, + device=device, + requires_grad=False, + ) + if self.atom_mask is None: + self.atom_mask = torch.tensor( + restype_atom14_mask, + dtype=float_dtype, + device=device, + requires_grad=False, + ) + if self.lit_positions is None: + self.lit_positions = torch.tensor( + restype_atom14_rigid_group_positions, + dtype=float_dtype, + device=device, + requires_grad=False, + ) + + def torsion_angles_to_frames(self, frame, alpha, aatype): + self._init_residue_constants(alpha.dtype, alpha.device) + return torsion_angles_to_frames(frame, alpha, aatype, + self.default_frames) + + def frames_and_literature_positions_to_atom14_pos(self, frame, aatype): + self._init_residue_constants(frame.get_rots().dtype, + frame.get_rots().device) + return frames_and_literature_positions_to_atom14_pos( + frame, + aatype, + self.default_frames, + self.group_idx, + self.atom_mask, + self.lit_positions, + ) diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/template.py b/runtime/Lib/site_packages/models/science/unifold/modules/template.py new file mode 100644 index 0000000000000000000000000000000000000000..49e5bec0045e4956cc24732bbb7313ec38848d13 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/template.py @@ -0,0 +1,330 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import math +from functools import partial +from typing import List, Optional, Tuple + +import torch +import torch.nn as nn +from unicore.modules import LayerNorm +from unicore.utils import (checkpoint_sequential, permute_final_dims, + tensor_tree_map) + +from .attentions import (Attention, TriangleAttentionEnding, + TriangleAttentionStarting, gen_attn_mask) +from .common import (Linear, SimpleModuleList, Transition, + bias_dropout_residual, chunk_layer, residual, + tri_mul_residual) +from .featurization import build_template_pair_feat_v2 +from .triangle_multiplication import (TriangleMultiplicationIncoming, + TriangleMultiplicationOutgoing) + + +class TemplatePointwiseAttention(nn.Module): + + def __init__(self, d_template, d_pair, d_hid, num_heads, inf, **kwargs): + super(TemplatePointwiseAttention, self).__init__() + + self.inf = inf + + self.mha = Attention( + d_pair, + d_template, + d_template, + d_hid, + num_heads, + gating=False, + ) + + def _chunk( + self, + z: torch.Tensor, + t: torch.Tensor, + mask: torch.Tensor, + chunk_size: int, + ) -> torch.Tensor: + mha_inputs = { + 'q': z, + 'k': t, + 'v': t, + 'mask': mask, + } + return chunk_layer( + self.mha, + mha_inputs, + chunk_size=chunk_size, + num_batch_dims=len(z.shape[:-2]), + ) + + def forward( + self, + t: torch.Tensor, + z: torch.Tensor, + template_mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + if template_mask is None: + template_mask = t.new_ones(t.shape[:-3]) + + mask = gen_attn_mask(template_mask, -self.inf)[..., None, None, None, + None, :] + z = z.unsqueeze(-2) + + t = permute_final_dims(t, (1, 2, 0, 3)) + + if chunk_size is not None: + z = self._chunk(z, t, mask, chunk_size) + else: + z = self.mha(z, t, t, mask=mask) + + z = z.squeeze(-2) + + return z + + +class TemplateProjection(nn.Module): + + def __init__(self, d_template, d_pair, **kwargs): + super(TemplateProjection, self).__init__() + + self.d_pair = d_pair + self.act = nn.ReLU() + self.output_linear = Linear(d_template, d_pair, init='relu') + + def forward(self, t, z) -> torch.Tensor: + if t is None: + # handle for non-template case + shape = z.shape + shape[-1] = self.d_pair + t = torch.zeros(shape, dtype=z.dtype, device=z.device) + t = self.act(t) + z_t = self.output_linear(t) + return z_t + + +class TemplatePairStackBlock(nn.Module): + + def __init__( + self, + d_template: int, + d_hid_tri_att: int, + d_hid_tri_mul: int, + num_heads: int, + pair_transition_n: int, + dropout_rate: float, + tri_attn_first: bool, + inf: float, + **kwargs, + ): + super(TemplatePairStackBlock, self).__init__() + + self.tri_att_start = TriangleAttentionStarting( + d_template, + d_hid_tri_att, + num_heads, + ) + self.tri_att_end = TriangleAttentionEnding( + d_template, + d_hid_tri_att, + num_heads, + ) + + self.tri_mul_out = TriangleMultiplicationOutgoing( + d_template, + d_hid_tri_mul, + ) + self.tri_mul_in = TriangleMultiplicationIncoming( + d_template, + d_hid_tri_mul, + ) + + self.pair_transition = Transition( + d_template, + pair_transition_n, + ) + self.tri_attn_first = tri_attn_first + self.dropout = dropout_rate + self.row_dropout_share_dim = -3 + self.col_dropout_share_dim = -2 + + def forward( + self, + s: torch.Tensor, + mask: torch.Tensor, + tri_start_attn_mask: torch.Tensor, + tri_end_attn_mask: torch.Tensor, + chunk_size: Optional[int] = None, + block_size: Optional[int] = None, + ): + if self.tri_attn_first: + s = bias_dropout_residual( + self.tri_att_start, + s, + self.tri_att_start( + s, attn_mask=tri_start_attn_mask, chunk_size=chunk_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + ) + + s = bias_dropout_residual( + self.tri_att_end, + s, + self.tri_att_end( + s, attn_mask=tri_end_attn_mask, chunk_size=chunk_size), + self.col_dropout_share_dim, + self.dropout, + self.training, + ) + s = tri_mul_residual( + self.tri_mul_out, + s, + self.tri_mul_out(s, mask=mask, block_size=block_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + block_size=block_size, + ) + + s = tri_mul_residual( + self.tri_mul_in, + s, + self.tri_mul_in(s, mask=mask, block_size=block_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + block_size=block_size, + ) + else: + s = tri_mul_residual( + self.tri_mul_out, + s, + self.tri_mul_out(s, mask=mask, block_size=block_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + block_size=block_size, + ) + + s = tri_mul_residual( + self.tri_mul_in, + s, + self.tri_mul_in(s, mask=mask, block_size=block_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + block_size=block_size, + ) + + s = bias_dropout_residual( + self.tri_att_start, + s, + self.tri_att_start( + s, attn_mask=tri_start_attn_mask, chunk_size=chunk_size), + self.row_dropout_share_dim, + self.dropout, + self.training, + ) + + s = bias_dropout_residual( + self.tri_att_end, + s, + self.tri_att_end( + s, attn_mask=tri_end_attn_mask, chunk_size=chunk_size), + self.col_dropout_share_dim, + self.dropout, + self.training, + ) + s = residual(s, self.pair_transition( + s, + chunk_size=chunk_size, + ), self.training) + return s + + +class TemplatePairStack(nn.Module): + + def __init__( + self, + d_template, + d_hid_tri_att, + d_hid_tri_mul, + num_blocks, + num_heads, + pair_transition_n, + dropout_rate, + tri_attn_first, + inf=1e9, + **kwargs, + ): + super(TemplatePairStack, self).__init__() + + self.blocks = SimpleModuleList() + for _ in range(num_blocks): + self.blocks.append( + TemplatePairStackBlock( + d_template=d_template, + d_hid_tri_att=d_hid_tri_att, + d_hid_tri_mul=d_hid_tri_mul, + num_heads=num_heads, + pair_transition_n=pair_transition_n, + dropout_rate=dropout_rate, + inf=inf, + tri_attn_first=tri_attn_first, + )) + + self.layer_norm = LayerNorm(d_template) + + def forward( + self, + single_templates: Tuple[torch.Tensor], + mask: torch.tensor, + tri_start_attn_mask: torch.Tensor, + tri_end_attn_mask: torch.Tensor, + templ_dim: int, + chunk_size: int, + block_size: int, + return_mean: bool, + ): + + def one_template(i): + (s, ) = checkpoint_sequential( + functions=[ + partial( + b, + mask=mask, + tri_start_attn_mask=tri_start_attn_mask, + tri_end_attn_mask=tri_end_attn_mask, + chunk_size=chunk_size, + block_size=block_size, + ) for b in self.blocks + ], + input=(single_templates[i], ), + ) + return s + + n_templ = len(single_templates) + if n_templ > 0: + new_single_templates = [one_template(0)] + if return_mean: + t = self.layer_norm(new_single_templates[0]) + for i in range(1, n_templ): + s = one_template(i) + if return_mean: + t = residual(t, self.layer_norm(s), self.training) + else: + new_single_templates.append(s) + + if return_mean: + if n_templ > 0: + t /= n_templ + else: + t = None + else: + t = torch.cat( + [s.unsqueeze(templ_dim) for s in new_single_templates], + dim=templ_dim) + t = self.layer_norm(t) + + return t diff --git a/runtime/Lib/site_packages/models/science/unifold/modules/triangle_multiplication.py b/runtime/Lib/site_packages/models/science/unifold/modules/triangle_multiplication.py new file mode 100644 index 0000000000000000000000000000000000000000..c4094cd21cc6de146fe37943de9193659402e2e0 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/modules/triangle_multiplication.py @@ -0,0 +1,158 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +from functools import partialmethod +from typing import List, Optional + +import torch +import torch.nn as nn +from unicore.modules import LayerNorm +from unicore.utils import permute_final_dims + +from .common import Linear + + +class TriangleMultiplication(nn.Module): + + def __init__(self, d_pair, d_hid, outgoing=True): + super(TriangleMultiplication, self).__init__() + self.outgoing = outgoing + + self.linear_ab_p = Linear(d_pair, d_hid * 2) + self.linear_ab_g = Linear(d_pair, d_hid * 2, init='gating') + + self.linear_g = Linear(d_pair, d_pair, init='gating') + self.linear_z = Linear(d_hid, d_pair, init='final') + + self.layer_norm_in = LayerNorm(d_pair) + self.layer_norm_out = LayerNorm(d_hid) + + self._alphafold_original_mode = False + + def _chunk_2d( + self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + block_size: int = None, + ) -> torch.Tensor: + + # avoid too small chunk size + # block_size = max(block_size, 256) + new_z = z.new_zeros(z.shape) + dim1 = z.shape[-3] + + def _slice_linear(z, linear: Linear, a=True): + d_hid = linear.bias.shape[0] // 2 + index = 0 if a else d_hid + p = ( + nn.functional.linear(z, linear.weight[index:index + d_hid]) + + linear.bias[index:index + d_hid]) + return p + + def _chunk_projection(z, mask, a=True): + p = _slice_linear(z, self.linear_ab_p, a) * mask + p *= torch.sigmoid(_slice_linear(z, self.linear_ab_g, a)) + return p + + num_chunk = (dim1 + block_size - 1) // block_size + for i in range(num_chunk): + chunk_start = i * block_size + chunk_end = min(chunk_start + block_size, dim1) + if self.outgoing: + a_chunk = _chunk_projection( + z[..., chunk_start:chunk_end, :, :], + mask[..., chunk_start:chunk_end, :, :], + a=True, + ) + a_chunk = permute_final_dims(a_chunk, (2, 0, 1)) + else: + a_chunk = _chunk_projection( + z[..., :, chunk_start:chunk_end, :], + mask[..., :, chunk_start:chunk_end, :], + a=True, + ) + a_chunk = a_chunk.transpose(-1, -3) + + for j in range(num_chunk): + j_chunk_start = j * block_size + j_chunk_end = min(j_chunk_start + block_size, dim1) + if self.outgoing: + b_chunk = _chunk_projection( + z[..., j_chunk_start:j_chunk_end, :, :], + mask[..., j_chunk_start:j_chunk_end, :, :], + a=False, + ) + b_chunk = b_chunk.transpose(-1, -3) + else: + b_chunk = _chunk_projection( + z[..., :, j_chunk_start:j_chunk_end, :], + mask[..., :, j_chunk_start:j_chunk_end, :], + a=False, + ) + b_chunk = permute_final_dims(b_chunk, (2, 0, 1)) + x_chunk = torch.matmul(a_chunk, b_chunk) + del b_chunk + x_chunk = permute_final_dims(x_chunk, (1, 2, 0)) + x_chunk = self.layer_norm_out(x_chunk) + x_chunk = self.linear_z(x_chunk) + x_chunk *= torch.sigmoid( + self.linear_g(z[..., chunk_start:chunk_end, + j_chunk_start:j_chunk_end, :])) + new_z[..., chunk_start:chunk_end, + j_chunk_start:j_chunk_end, :] = x_chunk + del x_chunk + del a_chunk + return new_z + + def forward( + self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + block_size=None, + ) -> torch.Tensor: + + mask = mask.unsqueeze(-1) + if not self._alphafold_original_mode: + # divided by 1/sqrt(dim) for numerical stability + mask = mask * (mask.shape[-2]**-0.5) + + z = self.layer_norm_in(z) + if not self.training and block_size is not None: + return self._chunk_2d(z, mask, block_size=block_size) + + g = nn.functional.linear(z, self.linear_g.weight) + if self.training: + ab = self.linear_ab_p(z) * mask * torch.sigmoid( + self.linear_ab_g(z)) + else: + ab = self.linear_ab_p(z) + ab *= mask + ab *= torch.sigmoid(self.linear_ab_g(z)) + a, b = torch.chunk(ab, 2, dim=-1) + del z, ab + + if self.outgoing: + a = permute_final_dims(a, (2, 0, 1)) + b = b.transpose(-1, -3) + else: + b = permute_final_dims(b, (2, 0, 1)) + a = a.transpose(-1, -3) + x = torch.matmul(a, b) + del a, b + + x = permute_final_dims(x, (1, 2, 0)) + + x = self.layer_norm_out(x) + x = nn.functional.linear(x, self.linear_z.weight) + return x, g + + def get_output_bias(self): + return self.linear_z.bias, self.linear_g.bias + + +class TriangleMultiplicationOutgoing(TriangleMultiplication): + __init__ = partialmethod(TriangleMultiplication.__init__, outgoing=True) + + +class TriangleMultiplicationIncoming(TriangleMultiplication): + __init__ = partialmethod(TriangleMultiplication.__init__, outgoing=False) diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/__init__.py b/runtime/Lib/site_packages/models/science/unifold/msa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2121062c41a4c7ff43b4f2f2148c70e5f2601bd8 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/__init__.py @@ -0,0 +1 @@ +""" Scripts for MSA & template searching. """ diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/mmcif.py b/runtime/Lib/site_packages/models/science/unifold/msa/mmcif.py new file mode 100644 index 0000000000000000000000000000000000000000..cf67239f078be34de3697d400ff1730de16a20f8 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/mmcif.py @@ -0,0 +1,483 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Parses the mmCIF file format.""" +import collections +import dataclasses +import functools +import io +from typing import Any, Mapping, Optional, Sequence, Tuple + +from absl import logging +from Bio import PDB +from Bio.Data import SCOPData +from Bio.PDB.MMCIFParser import MMCIFParser + +# Type aliases: +ChainId = str +PdbHeader = Mapping[str, Any] +PdbStructure = PDB.Structure.Structure +SeqRes = str +MmCIFDict = Mapping[str, Sequence[str]] + + +@dataclasses.dataclass(frozen=True) +class Monomer: + id: str + num: int + + +# Note - mmCIF format provides no guarantees on the type of author-assigned +# sequence numbers. They need not be integers. +@dataclasses.dataclass(frozen=True) +class AtomSite: + residue_name: str + author_chain_id: str + mmcif_chain_id: str + author_seq_num: str + mmcif_seq_num: int + insertion_code: str + hetatm_atom: str + model_num: int + + +# Used to map SEQRES index to a residue in the structure. +@dataclasses.dataclass(frozen=True) +class ResiduePosition: + chain_id: str + residue_number: int + insertion_code: str + + +@dataclasses.dataclass(frozen=True) +class ResidueAtPosition: + position: Optional[ResiduePosition] + name: str + is_missing: bool + hetflag: str + + +@dataclasses.dataclass(frozen=True) +class MmcifObject: + """Representation of a parsed mmCIF file. + + Contains: + file_id: A meaningful name, e.g. a pdb_id. Should be unique amongst all + files being processed. + header: Biopython header. + structure: Biopython structure. + chain_to_seqres: Dict mapping chain_id to 1 letter amino acid sequence. E.g. + {'A': 'ABCDEFG'} + seqres_to_structure: Dict; for each chain_id contains a mapping between + SEQRES index and a ResidueAtPosition. e.g. {'A': {0: ResidueAtPosition, 1: ResidueAtPosition, ...}} + raw_string: The raw string used to construct the MmcifObject. + """ + + file_id: str + header: PdbHeader + structure: PdbStructure + chain_to_seqres: Mapping[ChainId, SeqRes] + seqres_to_structure: Mapping[ChainId, Mapping[int, ResidueAtPosition]] + raw_string: Any + mmcif_to_author_chain_id: Mapping[ChainId, ChainId] + valid_chains: Mapping[ChainId, str] + + +@dataclasses.dataclass(frozen=True) +class ParsingResult: + """Returned by the parse function. + + Contains: + mmcif_object: A MmcifObject, may be None if no chain could be successfully + parsed. + errors: A dict mapping (file_id, chain_id) to any exception generated. + """ + + mmcif_object: Optional[MmcifObject] + errors: Mapping[Tuple[str, str], Any] + + +class ParseError(Exception): + """An error indicating that an mmCIF file could not be parsed.""" + + +def mmcif_loop_to_list(prefix: str, + parsed_info: MmCIFDict) -> Sequence[Mapping[str, str]]: + """Extracts loop associated with a prefix from mmCIF data as a list. + + Reference for loop_ in mmCIF: + http://mmcif.wwpdb.org/docs/tutorials/mechanics/pdbx-mmcif-syntax.html + + Args: + prefix: Prefix shared by each of the data items in the loop. + e.g. '_entity_poly_seq.', where the data items are _entity_poly_seq.num, + _entity_poly_seq.mon_id. Should include the trailing period. + parsed_info: A dict of parsed mmCIF data, e.g. _mmcif_dict from a Biopython + parser. + + Returns: + Returns a list of dicts; each dict represents 1 entry from an mmCIF loop. + """ + cols = [] + data = [] + for key, value in parsed_info.items(): + if key.startswith(prefix): + cols.append(key) + data.append(value) + + assert all([ + len(xs) == len(data[0]) for xs in data + ]), ('mmCIF error: Not all loops are the same length: %s' % cols) + + return [dict(zip(cols, xs)) for xs in zip(*data)] + + +def mmcif_loop_to_dict( + prefix: str, + index: str, + parsed_info: MmCIFDict, +) -> Mapping[str, Mapping[str, str]]: + """Extracts loop associated with a prefix from mmCIF data as a dictionary. + + Args: + prefix: Prefix shared by each of the data items in the loop. + e.g. '_entity_poly_seq.', where the data items are _entity_poly_seq.num, + _entity_poly_seq.mon_id. Should include the trailing period. + index: Which item of loop data should serve as the key. + parsed_info: A dict of parsed mmCIF data, e.g. _mmcif_dict from a Biopython + parser. + + Returns: + Returns a dict of dicts; each dict represents 1 entry from an mmCIF loop, + indexed by the index column. + """ + entries = mmcif_loop_to_list(prefix, parsed_info) + return {entry[index]: entry for entry in entries} + + +@functools.lru_cache(16, typed=False) +def fast_parse(*, + file_id: str, + mmcif_string: str, + catch_all_errors: bool = True) -> ParsingResult: + """Entry point, parses an mmcif_string. + + Args: + file_id: A string identifier for this file. Should be unique within the + collection of files being processed. + mmcif_string: Contents of an mmCIF file. + catch_all_errors: If True, all exceptions are caught and error messages are + returned as part of the ParsingResult. If False exceptions will be allowed + to propagate. + + Returns: + A ParsingResult. + """ + errors = {} + try: + parser = MMCIFParser(QUIET=True) + # handle = io.StringIO(mmcif_string) + # full_structure = parser.get_structure('', handle) + parsed_info = parser._mmcif_dict # pylint:disable=protected-access + + # Ensure all values are lists, even if singletons. + for key, value in parsed_info.items(): + if not isinstance(value, list): + parsed_info[key] = [value] + + header = _get_header(parsed_info) + + # Determine the protein chains, and their start numbers according to the + # internal mmCIF numbering scheme (likely but not guaranteed to be 1). + valid_chains = _get_protein_chains(parsed_info=parsed_info) + if not valid_chains: + return ParsingResult( + None, {(file_id, ''): 'No protein chains found in this file.'}) + + mmcif_to_author_chain_id = {} + # seq_to_structure_mappings = {} + for atom in _get_atom_site_list(parsed_info): + if atom.model_num != '1': + # We only process the first model at the moment. + continue + mmcif_to_author_chain_id[ + atom.mmcif_chain_id] = atom.author_chain_id + + mmcif_object = MmcifObject( + file_id=file_id, + header=header, + structure=None, + chain_to_seqres=None, + seqres_to_structure=None, + raw_string=parsed_info, + mmcif_to_author_chain_id=mmcif_to_author_chain_id, + valid_chains=valid_chains, + ) + + return ParsingResult(mmcif_object=mmcif_object, errors=errors) + except Exception as e: # pylint:disable=broad-except + errors[(file_id, '')] = e + if not catch_all_errors: + raise + return ParsingResult(mmcif_object=None, errors=errors) + + +@functools.lru_cache(16, typed=False) +def parse(*, + file_id: str, + mmcif_string: str, + catch_all_errors: bool = True) -> ParsingResult: + """Entry point, parses an mmcif_string. + + Args: + file_id: A string identifier for this file. Should be unique within the + collection of files being processed. + mmcif_string: Contents of an mmCIF file. + catch_all_errors: If True, all exceptions are caught and error messages are + returned as part of the ParsingResult. If False exceptions will be allowed + to propagate. + + Returns: + A ParsingResult. + """ + errors = {} + try: + parser = PDB.MMCIFParser(QUIET=True) + handle = io.StringIO(mmcif_string) + full_structure = parser.get_structure('', handle) + first_model_structure = _get_first_model(full_structure) + # Extract the _mmcif_dict from the parser, which contains useful fields not + # reflected in the Biopython structure. + parsed_info = parser._mmcif_dict # pylint:disable=protected-access + + # Ensure all values are lists, even if singletons. + for key, value in parsed_info.items(): + if not isinstance(value, list): + parsed_info[key] = [value] + + header = _get_header(parsed_info) + + # Determine the protein chains, and their start numbers according to the + # internal mmCIF numbering scheme (likely but not guaranteed to be 1). + valid_chains = _get_protein_chains(parsed_info=parsed_info) + if not valid_chains: + return ParsingResult( + None, {(file_id, ''): 'No protein chains found in this file.'}) + seq_start_num = { + chain_id: min([monomer.num for monomer in seq]) + for chain_id, seq in valid_chains.items() + } + + # Loop over the atoms for which we have coordinates. Populate two mappings: + # -mmcif_to_author_chain_id (maps internal mmCIF chain ids to chain ids used + # the authors / Biopython). + # -seq_to_structure_mappings (maps idx into sequence to ResidueAtPosition). + mmcif_to_author_chain_id = {} + seq_to_structure_mappings = {} + for atom in _get_atom_site_list(parsed_info): + if atom.model_num != '1': + # We only process the first model at the moment. + continue + + mmcif_to_author_chain_id[ + atom.mmcif_chain_id] = atom.author_chain_id + + if atom.mmcif_chain_id in valid_chains: + hetflag = ' ' + if atom.hetatm_atom == 'HETATM': + # Water atoms are assigned a special hetflag of W in Biopython. We + # need to do the same, so that this hetflag can be used to fetch + # a residue from the Biopython structure by id. + if atom.residue_name in ('HOH', 'WAT'): + hetflag = 'W' + else: + hetflag = 'H_' + atom.residue_name + insertion_code = atom.insertion_code + if not _is_set(atom.insertion_code): + insertion_code = ' ' + position = ResiduePosition( + chain_id=atom.author_chain_id, + residue_number=int(atom.author_seq_num), + insertion_code=insertion_code, + ) + seq_idx = int( + atom.mmcif_seq_num) - seq_start_num[atom.mmcif_chain_id] + current = seq_to_structure_mappings.get( + atom.author_chain_id, {}) + current[seq_idx] = ResidueAtPosition( + position=position, + name=atom.residue_name, + is_missing=False, + hetflag=hetflag, + ) + seq_to_structure_mappings[atom.author_chain_id] = current + + # Add missing residue information to seq_to_structure_mappings. + for chain_id, seq_info in valid_chains.items(): + author_chain = mmcif_to_author_chain_id[chain_id] + current_mapping = seq_to_structure_mappings[author_chain] + for idx, monomer in enumerate(seq_info): + if idx not in current_mapping: + current_mapping[idx] = ResidueAtPosition( + position=None, + name=monomer.id, + is_missing=True, + hetflag=' ') + + author_chain_to_sequence = {} + for chain_id, seq_info in valid_chains.items(): + author_chain = mmcif_to_author_chain_id[chain_id] + seq = [] + for monomer in seq_info: + code = SCOPData.protein_letters_3to1.get(monomer.id, 'X') + seq.append(code if len(code) == 1 else 'X') + seq = ''.join(seq) + author_chain_to_sequence[author_chain] = seq + + mmcif_object = MmcifObject( + file_id=file_id, + header=header, + structure=first_model_structure, + chain_to_seqres=author_chain_to_sequence, + seqres_to_structure=seq_to_structure_mappings, + raw_string=parsed_info, + mmcif_to_author_chain_id=mmcif_to_author_chain_id, + valid_chains=valid_chains, + ) + + return ParsingResult(mmcif_object=mmcif_object, errors=errors) + except Exception as e: # pylint:disable=broad-except + errors[(file_id, '')] = e + if not catch_all_errors: + raise + return ParsingResult(mmcif_object=None, errors=errors) + + +def _get_first_model(structure: PdbStructure) -> PdbStructure: + """Returns the first model in a Biopython structure.""" + return next(structure.get_models()) + + +_MIN_LENGTH_OF_CHAIN_TO_BE_COUNTED_AS_PEPTIDE = 21 + + +def get_release_date(parsed_info: MmCIFDict) -> str: + """Returns the oldest revision date.""" + revision_dates = parsed_info['_pdbx_audit_revision_history.revision_date'] + return min(revision_dates) + + +def _get_header(parsed_info: MmCIFDict) -> PdbHeader: + """Returns a basic header containing method, release date and resolution.""" + header = {} + + experiments = mmcif_loop_to_list('_exptl.', parsed_info) + header['structure_method'] = ','.join( + [experiment['_exptl.method'].lower() for experiment in experiments]) + + # Note: The release_date here corresponds to the oldest revision. We prefer to + # use this for dataset filtering over the deposition_date. + if '_pdbx_audit_revision_history.revision_date' in parsed_info: + header['release_date'] = get_release_date(parsed_info) + else: + logging.warning('Could not determine release_date: %s', + parsed_info['_entry.id']) + + header['resolution'] = 0.00 + for res_key in ( + '_refine.ls_d_res_high', + '_em_3d_reconstruction.resolution', + '_reflns.d_resolution_high', + ): + if res_key in parsed_info: + try: + raw_resolution = parsed_info[res_key][0] + header['resolution'] = float(raw_resolution) + except ValueError: + logging.debug('Invalid resolution format: %s', + parsed_info[res_key]) + + return header + + +def _get_atom_site_list(parsed_info: MmCIFDict) -> Sequence[AtomSite]: + """Returns list of atom sites; contains data not present in the structure.""" + return [ + AtomSite(*site) for site in zip( # pylint:disable=g-complex-comprehension + parsed_info['_atom_site.label_comp_id'], + parsed_info['_atom_site.auth_asym_id'], + parsed_info['_atom_site.label_asym_id'], + parsed_info['_atom_site.auth_seq_id'], + parsed_info['_atom_site.label_seq_id'], + parsed_info['_atom_site.pdbx_PDB_ins_code'], + parsed_info['_atom_site.group_PDB'], + parsed_info['_atom_site.pdbx_PDB_model_num'], + ) + ] + + +def _get_protein_chains( + *, parsed_info: Mapping[str, + Any]) -> Mapping[ChainId, Sequence[Monomer]]: + """Extracts polymer information for protein chains only. + + Args: + parsed_info: _mmcif_dict produced by the Biopython parser. + + Returns: + A dict mapping mmcif chain id to a list of Monomers. + """ + # Get polymer information for each entity in the structure. + entity_poly_seqs = mmcif_loop_to_list('_entity_poly_seq.', parsed_info) + + polymers = collections.defaultdict(list) + for entity_poly_seq in entity_poly_seqs: + polymers[entity_poly_seq['_entity_poly_seq.entity_id']].append( + Monomer( + id=entity_poly_seq['_entity_poly_seq.mon_id'], + num=int(entity_poly_seq['_entity_poly_seq.num']), + )) + + # Get chemical compositions. Will allow us to identify which of these polymers + # are proteins. + chem_comps = mmcif_loop_to_dict('_chem_comp.', '_chem_comp.id', + parsed_info) + + # Get chains information for each entity. Necessary so that we can return a + # dict keyed on chain id rather than entity. + struct_asyms = mmcif_loop_to_list('_struct_asym.', parsed_info) + + entity_to_mmcif_chains = collections.defaultdict(list) + for struct_asym in struct_asyms: + chain_id = struct_asym['_struct_asym.id'] + entity_id = struct_asym['_struct_asym.entity_id'] + entity_to_mmcif_chains[entity_id].append(chain_id) + + # Identify and return the valid protein chains. + valid_chains = {} + for entity_id, seq_info in polymers.items(): + chain_ids = entity_to_mmcif_chains[entity_id] + + # Reject polymers without any peptide-like components, such as DNA/RNA. + if any([ + 'peptide' in chem_comps[monomer.id]['_chem_comp.type'] + for monomer in seq_info + ]): + for chain_id in chain_ids: + valid_chains[chain_id] = seq_info + return valid_chains + + +def _is_set(data: str) -> bool: + """Returns False if data is a special mmCIF character indicating 'unset'.""" + return data not in ('.', '?') diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/msa_identifiers.py b/runtime/Lib/site_packages/models/science/unifold/msa/msa_identifiers.py new file mode 100644 index 0000000000000000000000000000000000000000..366239db3bfb47b7aaf364a5182afab60fb142ff --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/msa_identifiers.py @@ -0,0 +1,88 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for extracting identifiers from MSA sequence descriptions.""" + +import dataclasses +import re +from typing import Optional + +# Sequences coming from UniProtKB database come in the +# `db|UniqueIdentifier|EntryName` format, e.g. `tr|A0A146SKV9|A0A146SKV9_FUNHE` +# or `sp|P0C2L1|A3X1_LOXLA` (for TREMBL/Swiss-Prot respectively). +_UNIPROT_PATTERN = re.compile( + r""" + ^ + # UniProtKB/TrEMBL or UniProtKB/Swiss-Prot + (?:tr|sp) + \| + # A primary accession number of the UniProtKB entry. + (?P[A-Za-z0-9]{6,10}) + # Occasionally there is a _0 or _1 isoform suffix, which we ignore. + (?:_\d)? + \| + # TREMBL repeats the accession ID here. Swiss-Prot has a mnemonic + # protein ID code. + (?:[A-Za-z0-9]+) + _ + # A mnemonic species identification code. + (?P([A-Za-z0-9]){1,5}) + # Small BFD uses a final value after an underscore, which we ignore. + (?:_\d+)? + $ + """, + re.VERBOSE, +) + + +@dataclasses.dataclass(frozen=True) +class Identifiers: + species_id: str = '' + + +def _parse_sequence_identifier(msa_sequence_identifier: str) -> Identifiers: + """Gets accession id and species from an msa sequence identifier. + + The sequence identifier has the format specified by + _UNIPROT_TREMBL_ENTRY_NAME_PATTERN or _UNIPROT_SWISSPROT_ENTRY_NAME_PATTERN. + An example of a sequence identifier: `tr|A0A146SKV9|A0A146SKV9_FUNHE` + + Args: + msa_sequence_identifier: a sequence identifier. + + Returns: + An `Identifiers` instance with a species_id. These + can be empty in the case where no identifier was found. + """ + matches = re.search(_UNIPROT_PATTERN, msa_sequence_identifier.strip()) + if matches: + return Identifiers(species_id=matches.group('SpeciesIdentifier')) + return Identifiers() + + +def _extract_sequence_identifier(description: str) -> Optional[str]: + """Extracts sequence identifier from description. Returns None if no match.""" + split_description = description.split() + if split_description: + return split_description[0].partition('/')[0] + else: + return None + + +def get_identifiers(description: str) -> Identifiers: + """Computes extra MSA features from the description.""" + sequence_identifier = _extract_sequence_identifier(description) + if sequence_identifier is None: + return Identifiers() + else: + return _parse_sequence_identifier(sequence_identifier) diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/parsers.py b/runtime/Lib/site_packages/models/science/unifold/msa/parsers.py new file mode 100644 index 0000000000000000000000000000000000000000..bf36c8168bd3b0de534d640ccd71423676cdd19a --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/parsers.py @@ -0,0 +1,627 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions for parsing various file formats.""" +import collections +import dataclasses +import itertools +import re +import string +from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple + +DeletionMatrix = Sequence[Sequence[int]] + + +@dataclasses.dataclass(frozen=True) +class Msa: + """Class representing a parsed MSA file.""" + + sequences: Sequence[str] + deletion_matrix: DeletionMatrix + descriptions: Sequence[str] + + def __post_init__(self): + if not (len(self.sequences) == len(self.deletion_matrix) == len( + self.descriptions)): + raise ValueError( + 'All fields for an MSA must have the same length. ' + f'Got {len(self.sequences)} sequences, ' + f'{len(self.deletion_matrix)} rows in the deletion matrix and ' + f'{len(self.descriptions)} descriptions.') + + def __len__(self): + return len(self.sequences) + + def truncate(self, max_seqs: int): + return Msa( + sequences=self.sequences[:max_seqs], + deletion_matrix=self.deletion_matrix[:max_seqs], + descriptions=self.descriptions[:max_seqs], + ) + + +@dataclasses.dataclass(frozen=True) +class TemplateHit: + """Class representing a template hit.""" + + index: int + name: str + aligned_cols: int + sum_probs: Optional[float] + query: str + hit_sequence: str + indices_query: List[int] + indices_hit: List[int] + + +def parse_fasta(fasta_string: str) -> Tuple[Sequence[str], Sequence[str]]: + """Parses FASTA string and returns list of strings with amino-acid sequences. + + Arguments: + fasta_string: The string contents of a FASTA file. + + Returns: + A tuple of two lists: + * A list of sequences. + * A list of sequence descriptions taken from the comment lines. In the + same order as the sequences. + """ + sequences = [] + descriptions = [] + index = -1 + for line in fasta_string.splitlines(): + line = line.strip() + if line.startswith('>'): + index += 1 + descriptions.append(line[1:]) # Remove the '>' at the beginning. + sequences.append('') + continue + elif not line: + continue # Skip blank lines. + sequences[index] += line + + return sequences, descriptions + + +def parse_stockholm(stockholm_string: str) -> Msa: + """Parses sequences and deletion matrix from stockholm format alignment. + + Args: + stockholm_string: The string contents of a stockholm file. The first + sequence in the file should be the query sequence. + + Returns: + A tuple of: + * A list of sequences that have been aligned to the query. These + might contain duplicates. + * The deletion matrix for the alignment as a list of lists. The element + at `deletion_matrix[i][j]` is the number of residues deleted from + the aligned sequence i at residue position j. + * The names of the targets matched, including the jackhmmer subsequence + suffix. + """ + name_to_sequence = collections.OrderedDict() + for line in stockholm_string.splitlines(): + line = line.strip() + if not line or line.startswith(('#', '//')): + continue + name, sequence = line.split() + if name not in name_to_sequence: + name_to_sequence[name] = '' + name_to_sequence[name] += sequence + + msa = [] + deletion_matrix = [] + + query = '' + keep_columns = [] + for seq_index, sequence in enumerate(name_to_sequence.values()): + if seq_index == 0: + # Gather the columns with gaps from the query + query = sequence + keep_columns = [i for i, res in enumerate(query) if res != '-'] + + # Remove the columns with gaps in the query from all sequences. + aligned_sequence = ''.join([sequence[c] for c in keep_columns]) + + msa.append(aligned_sequence) + + # Count the number of deletions w.r.t. query. + deletion_vec = [] + deletion_count = 0 + for seq_res, query_res in zip(sequence, query): + if seq_res != '-' or query_res != '-': + if query_res == '-': + deletion_count += 1 + else: + deletion_vec.append(deletion_count) + deletion_count = 0 + deletion_matrix.append(deletion_vec) + + return Msa( + sequences=msa, + deletion_matrix=deletion_matrix, + descriptions=list(name_to_sequence.keys()), + ) + + +def parse_a3m(a3m_string: str) -> Msa: + """Parses sequences and deletion matrix from a3m format alignment. + + Args: + a3m_string: The string contents of a a3m file. The first sequence in the + file should be the query sequence. + + Returns: + A tuple of: + * A list of sequences that have been aligned to the query. These + might contain duplicates. + * The deletion matrix for the alignment as a list of lists. The element + at `deletion_matrix[i][j]` is the number of residues deleted from + the aligned sequence i at residue position j. + * A list of descriptions, one per sequence, from the a3m file. + """ + sequences, descriptions = parse_fasta(a3m_string) + deletion_matrix = [] + for msa_sequence in sequences: + deletion_vec = [] + deletion_count = 0 + for j in msa_sequence: + if j.islower(): + deletion_count += 1 + else: + deletion_vec.append(deletion_count) + deletion_count = 0 + deletion_matrix.append(deletion_vec) + + # Make the MSA matrix out of aligned (deletion-free) sequences. + deletion_table = str.maketrans('', '', string.ascii_lowercase) + aligned_sequences = [s.translate(deletion_table) for s in sequences] + return Msa( + sequences=aligned_sequences, + deletion_matrix=deletion_matrix, + descriptions=descriptions, + ) + + +def _convert_sto_seq_to_a3m(query_non_gaps: Sequence[bool], + sto_seq: str) -> Iterable[str]: + for is_query_res_non_gap, sequence_res in zip(query_non_gaps, sto_seq): + if is_query_res_non_gap: + yield sequence_res + elif sequence_res != '-': + yield sequence_res.lower() + + +def convert_stockholm_to_a3m( + stockholm_format: str, + max_sequences: Optional[int] = None, + remove_first_row_gaps: bool = True, +) -> str: + """Converts MSA in Stockholm format to the A3M format.""" + descriptions = {} + sequences = {} + reached_max_sequences = False + + for line in stockholm_format.splitlines(): + reached_max_sequences = max_sequences and len( + sequences) >= max_sequences + if line.strip() and not line.startswith(('#', '//')): + # Ignore blank lines, markup and end symbols - remainder are alignment + # sequence parts. + seqname, aligned_seq = line.split(maxsplit=1) + if seqname not in sequences: + if reached_max_sequences: + continue + sequences[seqname] = '' + sequences[seqname] += aligned_seq + + for line in stockholm_format.splitlines(): + if line[:4] == '#=GS': + # Description row - example format is: + # #=GS UniRef90_Q9H5Z4/4-78 DE [subseq from] cDNA: FLJ22755 ... + columns = line.split(maxsplit=3) + seqname, feature = columns[1:3] + value = columns[3] if len(columns) == 4 else '' + if feature != 'DE': + continue + if reached_max_sequences and seqname not in sequences: + continue + descriptions[seqname] = value + if len(descriptions) == len(sequences): + break + + # Convert sto format to a3m line by line + a3m_sequences = {} + if remove_first_row_gaps: + # query_sequence is assumed to be the first sequence + query_sequence = next(iter(sequences.values())) + query_non_gaps = [res != '-' for res in query_sequence] + for seqname, sto_sequence in sequences.items(): + # Dots are optional in a3m format and are commonly removed. + out_sequence = sto_sequence.replace('.', '') + if remove_first_row_gaps: + out_sequence = ''.join( + _convert_sto_seq_to_a3m(query_non_gaps, out_sequence)) + a3m_sequences[seqname] = out_sequence + + fasta_chunks = (f">{k} {descriptions.get(k, '')}\n{a3m_sequences[k]}" + for k in a3m_sequences) + return '\n'.join(fasta_chunks) + '\n' # Include terminating newline. + + +def _keep_line(line: str, seqnames: Set[str]) -> bool: + """Function to decide which lines to keep.""" + if not line.strip(): + return True + if line.strip() == '//': # End tag + return True + if line.startswith('# STOCKHOLM'): # Start tag + return True + if line.startswith('#=GC RF'): # Reference Annotation Line + return True + if line[:4] == '#=GS': # Description lines - keep if sequence in list. + _, seqname, _ = line.split(maxsplit=2) + return seqname in seqnames + elif line.startswith('#'): # Other markup - filter out + return False + else: # Alignment data - keep if sequence in list. + seqname = line.partition(' ')[0] + return seqname in seqnames + + +def truncate_stockholm_msa(stockholm_msa: str, max_sequences: int) -> str: + """Truncates a stockholm file to a maximum number of sequences.""" + seqnames = set() + filtered_lines = [] + for line in stockholm_msa.splitlines(): + if line.strip() and not line.startswith(('#', '//')): + # Ignore blank lines, markup and end symbols - remainder are alignment + # sequence parts. + seqname = line.partition(' ')[0] + seqnames.add(seqname) + if len(seqnames) >= max_sequences: + break + + for line in stockholm_msa.splitlines(): + if _keep_line(line, seqnames): + filtered_lines.append(line) + + return '\n'.join(filtered_lines) + '\n' + + +def remove_empty_columns_from_stockholm_msa(stockholm_msa: str) -> str: + """Removes empty columns (dashes-only) from a Stockholm MSA.""" + processed_lines = {} + unprocessed_lines = {} + for i, line in enumerate(stockholm_msa.splitlines()): + if line.startswith('#=GC RF'): + reference_annotation_i = i + reference_annotation_line = line + # Reached the end of this chunk of the alignment. Process chunk. + _, _, first_alignment = line.rpartition(' ') + mask = [] + for j in range(len(first_alignment)): + for _, unprocessed_line in unprocessed_lines.items(): + prefix, _, alignment = unprocessed_line.rpartition(' ') + if alignment[j] != '-': + mask.append(True) + break + else: # Every row contained a hyphen - empty column. + mask.append(False) + # Add reference annotation for processing with mask. + unprocessed_lines[ + reference_annotation_i] = reference_annotation_line + + if not any( + mask + ): # All columns were empty. Output empty lines for chunk. + for line_index in unprocessed_lines: + processed_lines[line_index] = '' + else: + for line_index, unprocessed_line in unprocessed_lines.items(): + prefix, _, alignment = unprocessed_line.rpartition(' ') + masked_alignment = ''.join( + itertools.compress(alignment, mask)) + processed_lines[ + line_index] = f'{prefix} {masked_alignment}' + + # Clear raw_alignments. + unprocessed_lines = {} + elif line.strip() and not line.startswith(('#', '//')): + unprocessed_lines[i] = line + else: + processed_lines[i] = line + return '\n'.join((processed_lines[i] for i in range(len(processed_lines)))) + + +def deduplicate_stockholm_msa(stockholm_msa: str) -> str: + """Remove duplicate sequences (ignoring insertions wrt query).""" + sequence_dict = collections.defaultdict(str) + + # First we must extract all sequences from the MSA. + for line in stockholm_msa.splitlines(): + # Only consider the alignments - ignore reference annotation, empty lines, + # descriptions or markup. + if line.strip() and not line.startswith(('#', '//')): + line = line.strip() + seqname, alignment = line.split() + sequence_dict[seqname] += alignment + + seen_sequences = set() + seqnames = set() + # First alignment is the query. + query_align = next(iter(sequence_dict.values())) + mask = [c != '-' for c in query_align] # Mask is False for insertions. + for seqname, alignment in sequence_dict.items(): + # Apply mask to remove all insertions from the string. + masked_alignment = ''.join(itertools.compress(alignment, mask)) + if masked_alignment in seen_sequences: + continue + else: + seen_sequences.add(masked_alignment) + seqnames.add(seqname) + + filtered_lines = [] + for line in stockholm_msa.splitlines(): + if _keep_line(line, seqnames): + filtered_lines.append(line) + + return '\n'.join(filtered_lines) + '\n' + + +def _get_hhr_line_regex_groups(regex_pattern: str, + line: str) -> Sequence[Optional[str]]: + match = re.match(regex_pattern, line) + if match is None: + raise RuntimeError(f'Could not parse query line {line}') + return match.groups() + + +def _update_hhr_residue_indices_list(sequence: str, start_index: int, + indices_list: List[int]): + """Computes the relative indices for each residue with respect to the original sequence.""" + counter = start_index + for symbol in sequence: + if symbol == '-': + indices_list.append(-1) + else: + indices_list.append(counter) + counter += 1 + + +def _parse_hhr_hit(detailed_lines: Sequence[str]) -> TemplateHit: + """Parses the detailed HMM HMM comparison section for a single Hit. + + This works on .hhr files generated from both HHBlits and HHSearch. + + Args: + detailed_lines: A list of lines from a single comparison section between 2 + sequences (which each have their own HMM's) + + Returns: + A dictionary with the information from that detailed comparison section + + Raises: + RuntimeError: If a certain line cannot be processed + """ + # Parse first 2 lines. + number_of_hit = int(detailed_lines[0].split()[-1]) + name_hit = detailed_lines[1][1:] + + # Parse the summary line. + pattern = ( + 'Probab=(.*)[\t ]*E-value=(.*)[\t ]*Score=(.*)[\t ]*Aligned_cols=(.*)[\t' + ' ]*Identities=(.*)%[\t ]*Similarity=(.*)[\t ]*Sum_probs=(.*)[\t ' + ']*Template_Neff=(.*)') + match = re.match(pattern, detailed_lines[2]) + if match is None: + raise RuntimeError( + 'Could not parse section: %s. Expected this: \n%s to contain summary.' + % (detailed_lines, detailed_lines[2])) + (_, _, _, aligned_cols, _, _, sum_probs, + _) = [float(x) for x in match.groups()] + + # The next section reads the detailed comparisons. These are in a 'human + # readable' format which has a fixed length. The strategy employed is to + # assume that each block starts with the query sequence line, and to parse + # that with a regexp in order to deduce the fixed length used for that block. + query = '' + hit_sequence = '' + indices_query = [] + indices_hit = [] + length_block = None + + for line in detailed_lines[3:]: + # Parse the query sequence line + if (line.startswith('Q ') and not line.startswith('Q ss_dssp') + and not line.startswith('Q ss_pred') + and not line.startswith('Q Consensus')): + # Thus the first 17 characters must be 'Q ', and we can parse + # everything after that. + # start sequence end total_sequence_length + patt = r'[\t ]*([0-9]*) ([A-Z-]*)[\t ]*([0-9]*) \([0-9]*\)' + groups = _get_hhr_line_regex_groups(patt, line[17:]) + + # Get the length of the parsed block using the start and finish indices, + # and ensure it is the same as the actual block length. + start = int(groups[0]) - 1 # Make index zero based. + delta_query = groups[1] + end = int(groups[2]) + num_insertions = len([x for x in delta_query if x == '-']) + length_block = end - start + num_insertions + assert length_block == len(delta_query) + + # Update the query sequence and indices list. + query += delta_query + _update_hhr_residue_indices_list(delta_query, start, indices_query) + + elif line.startswith('T '): + # Parse the hit sequence. + if (not line.startswith('T ss_dssp') + and not line.startswith('T ss_pred') + and not line.startswith('T Consensus')): + # Thus the first 17 characters must be 'T ', and we can + # parse everything after that. + # start sequence end total_sequence_length + patt = r'[\t ]*([0-9]*) ([A-Z-]*)[\t ]*[0-9]* \([0-9]*\)' + groups = _get_hhr_line_regex_groups(patt, line[17:]) + start = int(groups[0]) - 1 # Make index zero based. + delta_hit_sequence = groups[1] + assert length_block == len(delta_hit_sequence) + + # Update the hit sequence and indices list. + hit_sequence += delta_hit_sequence + _update_hhr_residue_indices_list(delta_hit_sequence, start, + indices_hit) + + return TemplateHit( + index=number_of_hit, + name=name_hit, + aligned_cols=int(aligned_cols), + sum_probs=sum_probs, + query=query, + hit_sequence=hit_sequence, + indices_query=indices_query, + indices_hit=indices_hit, + ) + + +def parse_hhr(hhr_string: str) -> Sequence[TemplateHit]: + """Parses the content of an entire HHR file.""" + lines = hhr_string.splitlines() + + # Each .hhr file starts with a results table, then has a sequence of hit + # "paragraphs", each paragraph starting with a line 'No '. We + # iterate through each paragraph to parse each hit. + + block_starts = [ + i for i, line in enumerate(lines) if line.startswith('No ') + ] + + hits = [] + if block_starts: + block_starts.append(len(lines)) # Add the end of the final block. + for i in range(len(block_starts) - 1): + hits.append( + _parse_hhr_hit(lines[block_starts[i]:block_starts[i + 1]])) + return hits + + +def parse_e_values_from_tblout(tblout: str) -> Dict[str, float]: + """Parse target to e-value mapping parsed from Jackhmmer tblout string.""" + e_values = {'query': 0} + lines = [line for line in tblout.splitlines() if line[0] != '#'] + # As per http://eddylab.org/software/hmmer/Userguide.pdf fields are + # space-delimited. Relevant fields are (1) target name: and + # (5) E-value (full sequence) (numbering from 1). + for line in lines: + fields = line.split() + e_value = fields[4] + target_name = fields[0] + e_values[target_name] = float(e_value) + return e_values + + +def _get_indices(sequence: str, start: int) -> List[int]: + """Returns indices for non-gap/insert residues starting at the given index.""" + indices = [] + counter = start + for symbol in sequence: + # Skip gaps but add a placeholder so that the alignment is preserved. + if symbol == '-': + indices.append(-1) + # Skip deleted residues, but increase the counter. + elif symbol.islower(): + counter += 1 + # Normal aligned residue. Increase the counter and append to indices. + else: + indices.append(counter) + counter += 1 + return indices + + +@dataclasses.dataclass(frozen=True) +class HitMetadata: + pdb_id: str + chain: str + start: int + end: int + length: int + text: str + + +def _parse_hmmsearch_description(description: str) -> HitMetadata: + """Parses the hmmsearch A3M sequence description line.""" + # Example 1: >4pqx_A/2-217 [subseq from] mol:protein length:217 Free text + # Example 2: >5g3r_A/1-55 [subseq from] mol:protein length:352 + match = re.match( + r'^>?([a-z0-9]+)_(\w+)/([0-9]+)-([0-9]+).*protein length:([0-9]+) *(.*)$', + description.strip(), + ) + + if not match: + raise ValueError(f'Could not parse description: "{description}".') + + return HitMetadata( + pdb_id=match[1], + chain=match[2], + start=int(match[3]), + end=int(match[4]), + length=int(match[5]), + text=match[6], + ) + + +def parse_hmmsearch_a3m(query_sequence: str, + a3m_string: str, + skip_first: bool = True) -> Sequence[TemplateHit]: + """Parses an a3m string produced by hmmsearch. + + Args: + query_sequence: The query sequence. + a3m_string: The a3m string produced by hmmsearch. + skip_first: Whether to skip the first sequence in the a3m string. + + Returns: + A sequence of `TemplateHit` results. + """ + # Zip the descriptions and MSAs together, skip the first query sequence. + parsed_a3m = list(zip(*parse_fasta(a3m_string))) + if skip_first: + parsed_a3m = parsed_a3m[1:] + + indices_query = _get_indices(query_sequence, start=0) + + hits = [] + for i, (hit_sequence, hit_description) in enumerate(parsed_a3m, start=1): + if 'mol:protein' not in hit_description: + continue # Skip non-protein chains. + metadata = _parse_hmmsearch_description(hit_description) + # Aligned columns are only the match states. + aligned_cols = sum([r.isupper() and r != '-' for r in hit_sequence]) + indices_hit = _get_indices(hit_sequence, start=metadata.start - 1) + + hit = TemplateHit( + index=i, + name=f'{metadata.pdb_id}_{metadata.chain}', + aligned_cols=aligned_cols, + sum_probs=None, + query=query_sequence, + hit_sequence=hit_sequence.upper(), + indices_query=indices_query, + indices_hit=indices_hit, + ) + hits.append(hit) + + return hits diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/pipeline.py b/runtime/Lib/site_packages/models/science/unifold/msa/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..8037e50e6726546adb03857b3900a957fdc0e0ab --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/pipeline.py @@ -0,0 +1,282 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions for building the input features for the unifold model.""" + +import os +from typing import Any, Mapping, MutableMapping, Optional, Sequence, Union + +import numpy as np +from absl import logging + +from modelscope.models.science.unifold.data import residue_constants +from modelscope.models.science.unifold.msa import (msa_identifiers, parsers, + templates) +from modelscope.models.science.unifold.msa.tools import (hhblits, hhsearch, + hmmsearch, jackhmmer) + +FeatureDict = MutableMapping[str, np.ndarray] +TemplateSearcher = Union[hhsearch.HHSearch, hmmsearch.Hmmsearch] + + +def make_sequence_features(sequence: str, description: str, + num_res: int) -> FeatureDict: + """Constructs a feature dict of sequence features.""" + features = {} + features['aatype'] = residue_constants.sequence_to_onehot( + sequence=sequence, + mapping=residue_constants.restype_order_with_x, + map_unknown_to_x=True, + ) + features['between_segment_residues'] = np.zeros((num_res, ), + dtype=np.int32) + features['domain_name'] = np.array([description.encode('utf-8')], + dtype=np.object_) + features['residue_index'] = np.array(range(num_res), dtype=np.int32) + features['seq_length'] = np.array([num_res] * num_res, dtype=np.int32) + features['sequence'] = np.array([sequence.encode('utf-8')], + dtype=np.object_) + return features + + +def make_msa_features(msas: Sequence[parsers.Msa]) -> FeatureDict: + """Constructs a feature dict of MSA features.""" + if not msas: + raise ValueError('At least one MSA must be provided.') + + int_msa = [] + deletion_matrix = [] + species_ids = [] + seen_sequences = set() + for msa_index, msa in enumerate(msas): + if not msa: + raise ValueError( + f'MSA {msa_index} must contain at least one sequence.') + for sequence_index, sequence in enumerate(msa.sequences): + if sequence in seen_sequences: + continue + seen_sequences.add(sequence) + int_msa.append( + [residue_constants.HHBLITS_AA_TO_ID[res] for res in sequence]) + deletion_matrix.append(msa.deletion_matrix[sequence_index]) + identifiers = msa_identifiers.get_identifiers( + msa.descriptions[sequence_index]) + species_ids.append(identifiers.species_id.encode('utf-8')) + + num_res = len(msas[0].sequences[0]) + num_alignments = len(int_msa) + features = {} + features['deletion_matrix_int'] = np.array(deletion_matrix, dtype=np.int32) + features['msa'] = np.array(int_msa, dtype=np.int32) + features['num_alignments'] = np.array( + [num_alignments] * num_res, dtype=np.int32) + features['msa_species_identifiers'] = np.array( + species_ids, dtype=np.object_) + return features + + +def run_msa_tool( + msa_runner, + input_fasta_path: str, + msa_out_path: str, + msa_format: str, + use_precomputed_msas: bool, +) -> Mapping[str, Any]: + """Runs an MSA tool, checking if output already exists first.""" + if not use_precomputed_msas or not os.path.exists(msa_out_path): + result = msa_runner.query(input_fasta_path)[0] + with open(msa_out_path, 'w') as f: + f.write(result[msa_format]) + else: + logging.warning('Reading MSA from file %s', msa_out_path) + with open(msa_out_path, 'r', encoding='utf-8') as f: + result = {msa_format: f.read()} + return result + + +class DataPipeline: + """Runs the alignment tools and assembles the input features.""" + + def __init__( + self, + jackhmmer_binary_path: str, + hhblits_binary_path: str, + uniref90_database_path: str, + mgnify_database_path: str, + bfd_database_path: Optional[str], + uniclust30_database_path: Optional[str], + small_bfd_database_path: Optional[str], + uniprot_database_path: Optional[str], + template_searcher: TemplateSearcher, + template_featurizer: templates.TemplateHitFeaturizer, + use_small_bfd: bool, + mgnify_max_hits: int = 501, + uniref_max_hits: int = 10000, + use_precomputed_msas: bool = False, + ): + """Initializes the data pipeline.""" + self._use_small_bfd = use_small_bfd + self.jackhmmer_uniref90_runner = jackhmmer.Jackhmmer( + binary_path=jackhmmer_binary_path, + database_path=uniref90_database_path) + if use_small_bfd: + self.jackhmmer_small_bfd_runner = jackhmmer.Jackhmmer( + binary_path=jackhmmer_binary_path, + database_path=small_bfd_database_path) + else: + self.hhblits_bfd_uniclust_runner = hhblits.HHBlits( + binary_path=hhblits_binary_path, + databases=[bfd_database_path, uniclust30_database_path], + ) + self.jackhmmer_mgnify_runner = jackhmmer.Jackhmmer( + binary_path=jackhmmer_binary_path, + database_path=mgnify_database_path) + self.jackhmmer_uniprot_runner = jackhmmer.Jackhmmer( + binary_path=jackhmmer_binary_path, + database_path=uniprot_database_path) + self.template_searcher = template_searcher + self.template_featurizer = template_featurizer + self.mgnify_max_hits = mgnify_max_hits + self.uniref_max_hits = uniref_max_hits + self.use_precomputed_msas = use_precomputed_msas + + def process(self, input_fasta_path: str, + msa_output_dir: str) -> FeatureDict: + """Runs alignment tools on the input sequence and creates features.""" + with open(input_fasta_path, encoding='utf-8') as f: + input_fasta_str = f.read() + input_seqs, input_descs = parsers.parse_fasta(input_fasta_str) + if len(input_seqs) != 1: + raise ValueError( + f'More than one input sequence found in {input_fasta_path}.') + input_sequence = input_seqs[0] + input_description = input_descs[0] + num_res = len(input_sequence) + + uniref90_out_path = os.path.join(msa_output_dir, 'uniref90_hits.sto') + jackhmmer_uniref90_result = run_msa_tool( + self.jackhmmer_uniref90_runner, + input_fasta_path, + uniref90_out_path, + 'sto', + self.use_precomputed_msas, + ) + mgnify_out_path = os.path.join(msa_output_dir, 'mgnify_hits.sto') + jackhmmer_mgnify_result = run_msa_tool( + self.jackhmmer_mgnify_runner, + input_fasta_path, + mgnify_out_path, + 'sto', + self.use_precomputed_msas, + ) + + msa_for_templates = jackhmmer_uniref90_result['sto'] + msa_for_templates = parsers.truncate_stockholm_msa( + msa_for_templates, max_sequences=self.uniref_max_hits) + msa_for_templates = parsers.deduplicate_stockholm_msa( + msa_for_templates) + msa_for_templates = parsers.remove_empty_columns_from_stockholm_msa( + msa_for_templates) + + if self.template_searcher.input_format == 'sto': + pdb_templates_result = self.template_searcher.query( + msa_for_templates) + elif self.template_searcher.input_format == 'a3m': + uniref90_msa_as_a3m = parsers.convert_stockholm_to_a3m( + msa_for_templates) + pdb_templates_result = self.template_searcher.query( + uniref90_msa_as_a3m) + else: + raise ValueError('Unrecognized template input format: ' + f'{self.template_searcher.input_format}') + + pdb_hits_out_path = os.path.join( + msa_output_dir, f'pdb_hits.{self.template_searcher.output_format}') + with open(pdb_hits_out_path, 'w') as f: + f.write(pdb_templates_result) + + uniref90_msa = parsers.parse_stockholm( + jackhmmer_uniref90_result['sto']) + uniref90_msa = uniref90_msa.truncate(max_seqs=self.uniref_max_hits) + mgnify_msa = parsers.parse_stockholm(jackhmmer_mgnify_result['sto']) + mgnify_msa = mgnify_msa.truncate(max_seqs=self.mgnify_max_hits) + + pdb_template_hits = self.template_searcher.get_template_hits( + output_string=pdb_templates_result, input_sequence=input_sequence) + + if self._use_small_bfd: + bfd_out_path = os.path.join(msa_output_dir, 'small_bfd_hits.sto') + jackhmmer_small_bfd_result = run_msa_tool( + self.jackhmmer_small_bfd_runner, + input_fasta_path, + bfd_out_path, + 'sto', + self.use_precomputed_msas, + ) + bfd_msa = parsers.parse_stockholm( + jackhmmer_small_bfd_result['sto']) + else: + bfd_out_path = os.path.join(msa_output_dir, + 'bfd_uniclust_hits.a3m') + hhblits_bfd_uniclust_result = run_msa_tool( + self.hhblits_bfd_uniclust_runner, + input_fasta_path, + bfd_out_path, + 'a3m', + self.use_precomputed_msas, + ) + bfd_msa = parsers.parse_a3m(hhblits_bfd_uniclust_result['a3m']) + + templates_result = self.template_featurizer.get_templates( + query_sequence=input_sequence, hits=pdb_template_hits) + + sequence_features = make_sequence_features( + sequence=input_sequence, + description=input_description, + num_res=num_res) + + msa_features = make_msa_features((uniref90_msa, bfd_msa, mgnify_msa)) + + logging.info('Uniref90 MSA size: %d sequences.', len(uniref90_msa)) + logging.info('BFD MSA size: %d sequences.', len(bfd_msa)) + logging.info('MGnify MSA size: %d sequences.', len(mgnify_msa)) + logging.info( + 'Final (deduplicated) MSA size: %d sequences.', + msa_features['num_alignments'][0], + ) + logging.info( + 'Total number of templates (NB: this can include bad ' + 'templates and is later filtered to top 4): %d.', + templates_result.features['template_domain_names'].shape[0], + ) + + return { + **sequence_features, + **msa_features, + **templates_result.features + } + + def process_uniprot(self, input_fasta_path: str, + msa_output_dir: str) -> FeatureDict: + uniprot_path = os.path.join(msa_output_dir, 'uniprot_hits.sto') + uniprot_result = run_msa_tool( + self.jackhmmer_uniprot_runner, + input_fasta_path, + uniprot_path, + 'sto', + self.use_precomputed_msas, + ) + msa = parsers.parse_stockholm(uniprot_result['sto']) + msa = msa.truncate(max_seqs=50000) + all_seq_dict = make_msa_features([msa]) + return all_seq_dict diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/templates.py b/runtime/Lib/site_packages/models/science/unifold/msa/templates.py new file mode 100644 index 0000000000000000000000000000000000000000..f2d3d79c977e409ae28015d065bc06d1aa8fe0b7 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/templates.py @@ -0,0 +1,1110 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions for getting templates and calculating template features.""" +import abc +import dataclasses +import datetime +import functools +import glob +import os +import re +from typing import Any, Dict, Mapping, Optional, Sequence, Tuple + +import numpy as np +from absl import logging + +from modelscope.models.science.unifold.data import residue_constants +from modelscope.models.science.unifold.msa import mmcif, parsers +from modelscope.models.science.unifold.msa.tools import kalign + + +class Error(Exception): + """Base class for exceptions.""" + + +class NoChainsError(Error): + """An error indicating that template mmCIF didn't have any chains.""" + + +class SequenceNotInTemplateError(Error): + """An error indicating that template mmCIF didn't contain the sequence.""" + + +class NoAtomDataInTemplateError(Error): + """An error indicating that template mmCIF didn't contain atom positions.""" + + +class TemplateAtomMaskAllZerosError(Error): + """An error indicating that template mmCIF had all atom positions masked.""" + + +class QueryToTemplateAlignError(Error): + """An error indicating that the query can't be aligned to the template.""" + + +class CaDistanceError(Error): + """An error indicating that a CA atom distance exceeds a threshold.""" + + +class MultipleChainsError(Error): + """An error indicating that multiple chains were found for a given ID.""" + + +# Prefilter exceptions. +class PrefilterError(Exception): + """A base class for template prefilter exceptions.""" + + +class DateError(PrefilterError): + """An error indicating that the hit date was after the max allowed date.""" + + +class AlignRatioError(PrefilterError): + """An error indicating that the hit align ratio to the query was too small.""" + + +class DuplicateError(PrefilterError): + """An error indicating that the hit was an exact subsequence of the query.""" + + +class LengthError(PrefilterError): + """An error indicating that the hit was too short.""" + + +TEMPLATE_FEATURES = { + 'template_aatype': np.float32, + 'template_all_atom_mask': np.float32, + 'template_all_atom_positions': np.float32, + 'template_domain_names': np.object_, + 'template_sequence': np.object_, + 'template_sum_probs': np.float32, +} + + +def _get_pdb_id_and_chain(hit: parsers.TemplateHit) -> Tuple[str, str]: + """Returns PDB id and chain id for an HHSearch Hit.""" + # PDB ID: 4 letters. Chain ID: 1+ alphanumeric letters or "." if unknown. + id_match = re.match(r'[a-zA-Z\d]{4}_[a-zA-Z0-9.]+', hit.name) + if not id_match: + raise ValueError( + f'hit.name did not start with PDBID_chain: {hit.name}') + pdb_id, chain_id = id_match.group(0).split('_') + return pdb_id.lower(), chain_id + + +def _is_after_cutoff( + pdb_id: str, + release_dates: Mapping[str, datetime.datetime], + release_date_cutoff: Optional[datetime.datetime], +) -> bool: + """Checks if the template date is after the release date cutoff. + + Args: + pdb_id: 4 letter pdb code. + release_dates: Dictionary mapping PDB ids to their structure release dates. + release_date_cutoff: Max release date that is valid for this query. + + Returns: + True if the template release date is after the cutoff, False otherwise. + """ + if release_date_cutoff is None: + raise ValueError('The release_date_cutoff must not be None.') + if pdb_id in release_dates: + return release_dates[pdb_id] > release_date_cutoff + else: + # Since this is just a quick prefilter to reduce the number of mmCIF files + # we need to parse, we don't have to worry about returning True here. + return False + + +def _parse_obsolete(obsolete_file_path: str) -> Mapping[str, Optional[str]]: + """Parses the data file from PDB that lists which pdb_ids are obsolete.""" + with open(obsolete_file_path) as f: + result = {} + for line in f: + line = line.strip() + # Format: Date From To + # 'OBSLTE 06-NOV-19 6G9Y' - Removed, rare + # 'OBSLTE 31-JUL-94 116L 216L' - Replaced, common + # 'OBSLTE 26-SEP-06 2H33 2JM5 2OWI' - Replaced by multiple, rare + if line.startswith('OBSLTE'): + if len(line) > 30: + # Replaced by at least one structure. + from_id = line[20:24].lower() + to_id = line[29:33].lower() + result[from_id] = to_id + elif len(line) == 24: + # Removed. + from_id = line[20:24].lower() + result[from_id] = None + return result + + +def _parse_release_dates(path: str) -> Mapping[str, datetime.datetime]: + """Parses release dates file, returns a mapping from PDBs to release dates.""" + if path.endswith('txt'): + release_dates = {} + with open(path, 'r', encoding='utf-8') as f: + for line in f: + pdb_id, date = line.split(':') + date = date.strip() + # Python 3.6 doesn't have datetime.date.fromisoformat() which is about + # 90x faster than strptime. However, splitting the string manually is + # about 10x faster than strptime. + release_dates[pdb_id.strip()] = datetime.datetime( + year=int(date[:4]), + month=int(date[5:7]), + day=int(date[8:10])) + return release_dates + else: + raise ValueError('Invalid format of the release date file %s.' % path) + + +def _assess_hhsearch_hit( + hit: parsers.TemplateHit, + hit_pdb_code: str, + query_sequence: str, + release_dates: Mapping[str, datetime.datetime], + release_date_cutoff: datetime.datetime, + max_subsequence_ratio: float = 0.95, + min_align_ratio: float = 0.1, +) -> bool: + """Determines if template is valid (without parsing the template mmcif file). + + Args: + hit: HhrHit for the template. + hit_pdb_code: The 4 letter pdb code of the template hit. This might be + different from the value in the actual hit since the original pdb might + have become obsolete. + query_sequence: Amino acid sequence of the query. + release_dates: Dictionary mapping pdb codes to their structure release + dates. + release_date_cutoff: Max release date that is valid for this query. + max_subsequence_ratio: Exclude any exact matches with this much overlap. + min_align_ratio: Minimum overlap between the template and query. + + Returns: + True if the hit passed the prefilter. Raises an exception otherwise. + + Raises: + DateError: If the hit date was after the max allowed date. + AlignRatioError: If the hit align ratio to the query was too small. + DuplicateError: If the hit was an exact subsequence of the query. + LengthError: If the hit was too short. + """ + aligned_cols = hit.aligned_cols + align_ratio = aligned_cols / len(query_sequence) + + template_sequence = hit.hit_sequence.replace('-', '') + length_ratio = float(len(template_sequence)) / len(query_sequence) + + # Check whether the template is a large subsequence or duplicate of original + # query. This can happen due to duplicate entries in the PDB database. + duplicate = ( + template_sequence in query_sequence + and length_ratio > max_subsequence_ratio) + + if _is_after_cutoff(hit_pdb_code, release_dates, release_date_cutoff): + raise DateError( + f'Date ({release_dates[hit_pdb_code]}) > max template date ' + f'({release_date_cutoff}).') + + if align_ratio <= min_align_ratio: + raise AlignRatioError( + 'Proportion of residues aligned to query too small. ' + f'Align ratio: {align_ratio}.') + + if duplicate: + raise DuplicateError( + 'Template is an exact subsequence of query with large ' + f'coverage. Length ratio: {length_ratio}.') + + if len(template_sequence) < 10: + raise LengthError( + f'Template too short. Length: {len(template_sequence)}.') + + return True + + +def _find_template_in_pdb( + template_chain_id: str, template_sequence: str, + mmcif_object: mmcif.MmcifObject) -> Tuple[str, str, int]: + """Tries to find the template chain in the given pdb file. + + This method tries the three following things in order: + 1. Tries if there is an exact match in both the chain ID and the sequence. + If yes, the chain sequence is returned. Otherwise: + 2. Tries if there is an exact match only in the sequence. + If yes, the chain sequence is returned. Otherwise: + 3. Tries if there is a fuzzy match (X = wildcard) in the sequence. + If yes, the chain sequence is returned. + If none of these succeed, a SequenceNotInTemplateError is thrown. + + Args: + template_chain_id: The template chain ID. + template_sequence: The template chain sequence. + mmcif_object: The PDB object to search for the template in. + + Returns: + A tuple with: + * The chain sequence that was found to match the template in the PDB object. + * The ID of the chain that is being returned. + * The offset where the template sequence starts in the chain sequence. + + Raises: + SequenceNotInTemplateError: If no match is found after the steps described + above. + """ + # Try if there is an exact match in both the chain ID and the (sub)sequence. + pdb_id = mmcif_object.file_id + chain_sequence = mmcif_object.chain_to_seqres.get(template_chain_id) + if chain_sequence and (template_sequence in chain_sequence): + logging.info('Found an exact template match %s_%s.', pdb_id, + template_chain_id) + mapping_offset = chain_sequence.find(template_sequence) + return chain_sequence, template_chain_id, mapping_offset + + # Try if there is an exact match in the (sub)sequence only. + for chain_id, chain_sequence in mmcif_object.chain_to_seqres.items(): + if chain_sequence and (template_sequence in chain_sequence): + logging.info('Found a sequence-only match %s_%s.', pdb_id, + chain_id) + mapping_offset = chain_sequence.find(template_sequence) + return chain_sequence, chain_id, mapping_offset + + # Return a chain sequence that fuzzy matches (X = wildcard) the template. + # Make parentheses unnamed groups (?:_) to avoid the 100 named groups limit. + regex = ['.' if aa == 'X' else '(?:%s|X)' % aa for aa in template_sequence] + regex = re.compile(''.join(regex)) + for chain_id, chain_sequence in mmcif_object.chain_to_seqres.items(): + match = re.search(regex, chain_sequence) + if match: + logging.info('Found a fuzzy sequence-only match %s_%s.', pdb_id, + chain_id) + mapping_offset = match.start() + return chain_sequence, chain_id, mapping_offset + + # No hits, raise an error. + raise SequenceNotInTemplateError( + 'Could not find the template sequence in %s_%s. Template sequence: %s, ' + 'chain_to_seqres: %s' % (pdb_id, template_chain_id, template_sequence, + mmcif_object.chain_to_seqres)) + + +def _realign_pdb_template_to_query( + old_template_sequence: str, + template_chain_id: str, + mmcif_object: mmcif.MmcifObject, + old_mapping: Mapping[int, int], + kalign_binary_path: str, +) -> Tuple[str, Mapping[int, int]]: + """Aligns template from the mmcif_object to the query. + + In case PDB70 contains a different version of the template sequence, we need + to perform a realignment to the actual sequence that is in the mmCIF file. + This method performs such realignment, but returns the new sequence and + mapping only if the sequence in the mmCIF file is 90% identical to the old + sequence. + + Note that the old_template_sequence comes from the hit, and contains only that + part of the chain that matches with the query while the new_template_sequence + is the full chain. + + Args: + old_template_sequence: The template sequence that was returned by the PDB + template search (typically done using HHSearch). + template_chain_id: The template chain id was returned by the PDB template + search (typically done using HHSearch). This is used to find the right + chain in the mmcif_object chain_to_seqres mapping. + mmcif_object: A mmcif_object which holds the actual template data. + old_mapping: A mapping from the query sequence to the template sequence. + This mapping will be used to compute the new mapping from the query + sequence to the actual mmcif_object template sequence by aligning the + old_template_sequence and the actual template sequence. + kalign_binary_path: The path to a kalign executable. + + Returns: + A tuple (new_template_sequence, new_query_to_template_mapping) where: + * new_template_sequence is the actual template sequence that was found in + the mmcif_object. + * new_query_to_template_mapping is the new mapping from the query to the + actual template found in the mmcif_object. + + Raises: + QueryToTemplateAlignError: + * If there was an error thrown by the alignment tool. + * Or if the actual template sequence differs by more than 10% from the + old_template_sequence. + """ + aligner = kalign.Kalign(binary_path=kalign_binary_path) + new_template_sequence = mmcif_object.chain_to_seqres.get( + template_chain_id, '') + + # Sometimes the template chain id is unknown. But if there is only a single + # sequence within the mmcif_object, it is safe to assume it is that one. + if not new_template_sequence: + if len(mmcif_object.chain_to_seqres) == 1: + logging.info( + 'Could not find %s in %s, but there is only 1 sequence, so ' + 'using that one.', + template_chain_id, + mmcif_object.file_id, + ) + new_template_sequence = list( + mmcif_object.chain_to_seqres.values())[0] + else: + raise QueryToTemplateAlignError( + f'Could not find chain {template_chain_id} in {mmcif_object.file_id}. ' + 'If there are no mmCIF parsing errors, it is possible it was not a ' + 'protein chain.') + + try: + parsed_a3m = parsers.parse_a3m( + aligner.align([old_template_sequence, new_template_sequence])) + old_aligned_template, new_aligned_template = parsed_a3m.sequences + except Exception as e: + raise QueryToTemplateAlignError( + 'Could not align old template %s to template %s (%s_%s). Error: %s' + % ( + old_template_sequence, + new_template_sequence, + mmcif_object.file_id, + template_chain_id, + str(e), + )) + + logging.info( + 'Old aligned template: %s\nNew aligned template: %s', + old_aligned_template, + new_aligned_template, + ) + + old_to_new_template_mapping = {} + old_template_index = -1 + new_template_index = -1 + num_same = 0 + for old_template_aa, new_template_aa in zip(old_aligned_template, + new_aligned_template): + if old_template_aa != '-': + old_template_index += 1 + if new_template_aa != '-': + new_template_index += 1 + if old_template_aa != '-' and new_template_aa != '-': + old_to_new_template_mapping[ + old_template_index] = new_template_index + if old_template_aa == new_template_aa: + num_same += 1 + + # Require at least 90 % sequence identity wrt to the shorter of the sequences. + if (float(num_same) + / min(len(old_template_sequence), len(new_template_sequence)) + < # noqa W504 + 0.9): + raise QueryToTemplateAlignError( + 'Insufficient similarity of the sequence in the database: %s to the ' + 'actual sequence in the mmCIF file %s_%s: %s. We require at least ' + '90 %% similarity wrt to the shorter of the sequences. This is not a ' + 'problem unless you think this is a template that should be included.' + % ( + old_template_sequence, + mmcif_object.file_id, + template_chain_id, + new_template_sequence, + )) + + new_query_to_template_mapping = {} + for query_index, old_template_index in old_mapping.items(): + new_query_to_template_mapping[ + query_index] = old_to_new_template_mapping.get( + old_template_index, -1) + + new_template_sequence = new_template_sequence.replace('-', '') + + return new_template_sequence, new_query_to_template_mapping + + +def _check_residue_distances(all_positions: np.ndarray, + all_positions_mask: np.ndarray, + max_ca_ca_distance: float): + """Checks if the distance between unmasked neighbor residues is ok.""" + ca_position = residue_constants.atom_order['CA'] + prev_is_unmasked = False + prev_calpha = None + for i, (coords, mask) in enumerate(zip(all_positions, all_positions_mask)): + this_is_unmasked = bool(mask[ca_position]) + if this_is_unmasked: + this_calpha = coords[ca_position] + if prev_is_unmasked: + distance = np.linalg.norm(this_calpha - prev_calpha) + if distance > max_ca_ca_distance: + raise CaDistanceError( + 'The distance between residues %d and %d is %f > limit %f.' + % (i, i + 1, distance, max_ca_ca_distance)) + prev_calpha = this_calpha + prev_is_unmasked = this_is_unmasked + + +def _get_atom_positions( + mmcif_object: mmcif.MmcifObject, auth_chain_id: str, + max_ca_ca_distance: float) -> Tuple[np.ndarray, np.ndarray]: + """Gets atom positions and mask from a list of Biopython Residues.""" + num_res = len(mmcif_object.chain_to_seqres[auth_chain_id]) + + relevant_chains = [ + c for c in mmcif_object.structure.get_chains() if c.id == auth_chain_id + ] + if len(relevant_chains) != 1: + raise MultipleChainsError( + f'Expected exactly one chain in structure with id {auth_chain_id}.' + ) + chain = relevant_chains[0] + + all_positions = np.zeros([num_res, residue_constants.atom_type_num, 3]) + all_positions_mask = np.zeros([num_res, residue_constants.atom_type_num], + dtype=np.int64) + for res_index in range(num_res): + pos = np.zeros([residue_constants.atom_type_num, 3], dtype=np.float32) + mask = np.zeros([residue_constants.atom_type_num], dtype=np.float32) + res_at_position = mmcif_object.seqres_to_structure[auth_chain_id][ + res_index] + if not res_at_position.is_missing: + res = chain[( + res_at_position.hetflag, + res_at_position.position.residue_number, + res_at_position.position.insertion_code, + )] + for atom in res.get_atoms(): + atom_name = atom.get_name() + x, y, z = atom.get_coord() + if atom_name in residue_constants.atom_order.keys(): + pos[residue_constants.atom_order[atom_name]] = [x, y, z] + mask[residue_constants.atom_order[atom_name]] = 1.0 + elif atom_name.upper() == 'SE' and res.get_resname() == 'MSE': + # Put the coordinates of the selenium atom in the sulphur column. + pos[residue_constants.atom_order['SD']] = [x, y, z] + mask[residue_constants.atom_order['SD']] = 1.0 + + # Fix naming errors in arginine residues where NH2 is incorrectly + # assigned to be closer to CD than NH1. + cd = residue_constants.atom_order['CD'] + nh1 = residue_constants.atom_order['NH1'] + nh2 = residue_constants.atom_order['NH2'] + if (res.get_resname() == 'ARG' + and all(mask[atom_index] for atom_index in (cd, nh1, nh2)) + and (np.linalg.norm(pos[nh1] - pos[cd]) > # noqa W504 + np.linalg.norm(pos[nh2] - pos[cd]))): + pos[nh1], pos[nh2] = pos[nh2].copy(), pos[nh1].copy() + mask[nh1], mask[nh2] = mask[nh2].copy(), mask[nh1].copy() + + all_positions[res_index] = pos + all_positions_mask[res_index] = mask + _check_residue_distances(all_positions, all_positions_mask, + max_ca_ca_distance) + return all_positions, all_positions_mask + + +def _extract_template_features( + mmcif_object: mmcif.MmcifObject, + pdb_id: str, + mapping: Mapping[int, int], + template_sequence: str, + query_sequence: str, + template_chain_id: str, + kalign_binary_path: str, +) -> Tuple[Dict[str, Any], Optional[str]]: + """Parses atom positions in the target structure and aligns with the query. + + Atoms for each residue in the template structure are indexed to coincide + with their corresponding residue in the query sequence, according to the + alignment mapping provided. + + Args: + mmcif_object: mmcif_parsing.MmcifObject representing the template. + pdb_id: PDB code for the template. + mapping: Dictionary mapping indices in the query sequence to indices in + the template sequence. + template_sequence: String describing the amino acid sequence for the + template protein. + query_sequence: String describing the amino acid sequence for the query + protein. + template_chain_id: String ID describing which chain in the structure proto + should be used. + kalign_binary_path: The path to a kalign executable used for template + realignment. + + Returns: + A tuple with: + * A dictionary containing the extra features derived from the template + protein structure. + * A warning message if the hit was realigned to the actual mmCIF sequence. + Otherwise None. + + Raises: + NoChainsError: If the mmcif object doesn't contain any chains. + SequenceNotInTemplateError: If the given chain id / sequence can't + be found in the mmcif object. + QueryToTemplateAlignError: If the actual template in the mmCIF file + can't be aligned to the query. + NoAtomDataInTemplateError: If the mmcif object doesn't contain + atom positions. + TemplateAtomMaskAllZerosError: If the mmcif object doesn't have any + unmasked residues. + """ + if mmcif_object is None or not mmcif_object.chain_to_seqres: + raise NoChainsError('No chains in PDB: %s_%s' % + (pdb_id, template_chain_id)) + + warning = None + try: + seqres, chain_id, mapping_offset = _find_template_in_pdb( + template_chain_id=template_chain_id, + template_sequence=template_sequence, + mmcif_object=mmcif_object, + ) + except SequenceNotInTemplateError: + # If PDB70 contains a different version of the template, we use the sequence + # from the mmcif_object. + chain_id = template_chain_id + warning = ( + f'The exact sequence {template_sequence} was not found in ' + f'{pdb_id}_{chain_id}. Realigning the template to the actual sequence.' + ) + logging.warning(warning) + # This throws an exception if it fails to realign the hit. + seqres, mapping = _realign_pdb_template_to_query( + old_template_sequence=template_sequence, + template_chain_id=template_chain_id, + mmcif_object=mmcif_object, + old_mapping=mapping, + kalign_binary_path=kalign_binary_path, + ) + logging.info( + 'Sequence in %s_%s: %s successfully realigned to %s', + pdb_id, + chain_id, + template_sequence, + seqres, + ) + # The template sequence changed. + template_sequence = seqres + # No mapping offset, the query is aligned to the actual sequence. + mapping_offset = 0 + + try: + # Essentially set to infinity - we don't want to reject templates unless + # they're really really bad. + all_atom_positions, all_atom_mask = _get_atom_positions( + mmcif_object, chain_id, max_ca_ca_distance=150.0) + except (CaDistanceError, KeyError) as ex: + raise NoAtomDataInTemplateError('Could not get atom data (%s_%s): %s' % + (pdb_id, chain_id, str(ex))) from ex + + all_atom_positions = np.split(all_atom_positions, + all_atom_positions.shape[0]) + all_atom_masks = np.split(all_atom_mask, all_atom_mask.shape[0]) + + output_templates_sequence = [] + templates_all_atom_positions = [] + templates_all_atom_masks = [] + + for _ in query_sequence: + # Residues in the query_sequence that are not in the template_sequence: + templates_all_atom_positions.append( + np.zeros((residue_constants.atom_type_num, 3))) + templates_all_atom_masks.append( + np.zeros(residue_constants.atom_type_num)) + output_templates_sequence.append('-') + + for k, v in mapping.items(): + template_index = v + mapping_offset + templates_all_atom_positions[k] = all_atom_positions[template_index][0] + templates_all_atom_masks[k] = all_atom_masks[template_index][0] + output_templates_sequence[k] = template_sequence[v] + + # Alanine (AA with the lowest number of atoms) has 5 atoms (C, CA, CB, N, O). + if np.sum(templates_all_atom_masks) < 5: + raise TemplateAtomMaskAllZerosError( + 'Template all atom mask was all zeros: %s_%s. Residue range: %d-%d' + % ( + pdb_id, + chain_id, + min(mapping.values()) + mapping_offset, + max(mapping.values()) + mapping_offset, + )) + + output_templates_sequence = ''.join(output_templates_sequence) + + templates_aatype = residue_constants.sequence_to_onehot( + output_templates_sequence, residue_constants.HHBLITS_AA_TO_ID) + + return ( + { + 'template_all_atom_positions': + np.array(templates_all_atom_positions), + 'template_all_atom_mask': np.array(templates_all_atom_masks), + 'template_sequence': output_templates_sequence.encode(), + 'template_aatype': np.array(templates_aatype), + 'template_domain_names': f'{pdb_id.lower()}_{chain_id}'.encode(), + }, + warning, + ) + + +def _build_query_to_hit_index_mapping( + hit_query_sequence: str, + hit_sequence: str, + indices_hit: Sequence[int], + indices_query: Sequence[int], + original_query_sequence: str, +) -> Mapping[int, int]: + """Gets mapping from indices in original query sequence to indices in the hit. + + hit_query_sequence and hit_sequence are two aligned sequences containing gap + characters. hit_query_sequence contains only the part of the original query + sequence that matched the hit. When interpreting the indices from the .hhr, we + need to correct for this to recover a mapping from original query sequence to + the hit sequence. + + Args: + hit_query_sequence: The portion of the query sequence that is in the .hhr + hit + hit_sequence: The portion of the hit sequence that is in the .hhr + indices_hit: The indices for each aminoacid relative to the hit sequence + indices_query: The indices for each aminoacid relative to the original query + sequence + original_query_sequence: String describing the original query sequence. + + Returns: + Dictionary with indices in the original query sequence as keys and indices + in the hit sequence as values. + """ + # If the hit is empty (no aligned residues), return empty mapping + if not hit_query_sequence: + return {} + + # Remove gaps and find the offset of hit.query relative to original query. + hhsearch_query_sequence = hit_query_sequence.replace('-', '') + hit_sequence = hit_sequence.replace('-', '') + hhsearch_query_offset = original_query_sequence.find( + hhsearch_query_sequence) + + # Index of -1 used for gap characters. Subtract the min index ignoring gaps. + min_idx = min(x for x in indices_hit if x > -1) + fixed_indices_hit = [x - min_idx if x > -1 else -1 for x in indices_hit] + + min_idx = min(x for x in indices_query if x > -1) + fixed_indices_query = [ + x - min_idx if x > -1 else -1 for x in indices_query + ] + + # Zip the corrected indices, ignore case where both seqs have gap characters. + mapping = {} + for q_i, q_t in zip(fixed_indices_query, fixed_indices_hit): + if q_t != -1 and q_i != -1: + if q_t >= len(hit_sequence) or q_i + hhsearch_query_offset >= len( + original_query_sequence): + continue + mapping[q_i + hhsearch_query_offset] = q_t + + return mapping + + +@dataclasses.dataclass(frozen=True) +class SingleHitResult: + features: Optional[Mapping[str, Any]] + error: Optional[str] + warning: Optional[str] + + +@functools.lru_cache(16, typed=False) +def _read_file(path): + with open(path, 'r') as f: + file_data = f.read() + return file_data + + +def _process_single_hit( + query_sequence: str, + hit: parsers.TemplateHit, + mmcif_dir: str, + max_template_date: datetime.datetime, + release_dates: Mapping[str, datetime.datetime], + obsolete_pdbs: Mapping[str, Optional[str]], + kalign_binary_path: str, + strict_error_check: bool = False, +) -> SingleHitResult: + """Tries to extract template features from a single HHSearch hit.""" + # Fail hard if we can't get the PDB ID and chain name from the hit. + hit_pdb_code, hit_chain_id = _get_pdb_id_and_chain(hit) + + # This hit has been removed (obsoleted) from PDB, skip it. + if hit_pdb_code in obsolete_pdbs and obsolete_pdbs[hit_pdb_code] is None: + return SingleHitResult( + features=None, + error=None, + warning=f'Hit {hit_pdb_code} is obsolete.') + + if hit_pdb_code not in release_dates: + if hit_pdb_code in obsolete_pdbs: + hit_pdb_code = obsolete_pdbs[hit_pdb_code] + + # Pass hit_pdb_code since it might have changed due to the pdb being obsolete. + try: + _assess_hhsearch_hit( + hit=hit, + hit_pdb_code=hit_pdb_code, + query_sequence=query_sequence, + release_dates=release_dates, + release_date_cutoff=max_template_date, + ) + except PrefilterError as e: + msg = f'hit {hit_pdb_code}_{hit_chain_id} did not pass prefilter: {str(e)}' + logging.info(msg) + if strict_error_check and isinstance(e, (DateError, DuplicateError)): + # In strict mode we treat some prefilter cases as errors. + return SingleHitResult(features=None, error=msg, warning=None) + + return SingleHitResult(features=None, error=None, warning=None) + + mapping = _build_query_to_hit_index_mapping(hit.query, hit.hit_sequence, + hit.indices_hit, + hit.indices_query, + query_sequence) + + # The mapping is from the query to the actual hit sequence, so we need to + # remove gaps (which regardless have a missing confidence score). + template_sequence = hit.hit_sequence.replace('-', '') + + cif_path = os.path.join(mmcif_dir, hit_pdb_code + '.cif') + logging.debug( + 'Reading PDB entry from %s. Query: %s, template: %s', + cif_path, + query_sequence, + template_sequence, + ) + # Fail if we can't find the mmCIF file. + cif_string = _read_file(cif_path) + + parsing_result = mmcif.parse(file_id=hit_pdb_code, mmcif_string=cif_string) + + if parsing_result.mmcif_object is not None: + hit_release_date = datetime.datetime.strptime( + parsing_result.mmcif_object.header['release_date'], '%Y-%m-%d') + if hit_release_date > max_template_date: + error = 'Template %s date (%s) > max template date (%s).' % ( + hit_pdb_code, + hit_release_date, + max_template_date, + ) + if strict_error_check: + return SingleHitResult( + features=None, error=error, warning=None) + else: + logging.debug(error) + return SingleHitResult(features=None, error=None, warning=None) + + try: + features, realign_warning = _extract_template_features( + mmcif_object=parsing_result.mmcif_object, + pdb_id=hit_pdb_code, + mapping=mapping, + template_sequence=template_sequence, + query_sequence=query_sequence, + template_chain_id=hit_chain_id, + kalign_binary_path=kalign_binary_path, + ) + if hit.sum_probs is None: + features['template_sum_probs'] = [0] + else: + features['template_sum_probs'] = [hit.sum_probs] + + # It is possible there were some errors when parsing the other chains in the + # mmCIF file, but the template features for the chain we want were still + # computed. In such case the mmCIF parsing errors are not relevant. + return SingleHitResult( + features=features, error=None, warning=realign_warning) + except ( + NoChainsError, + NoAtomDataInTemplateError, + TemplateAtomMaskAllZerosError, + ) as e: + # These 3 errors indicate missing mmCIF experimental data rather than a + # problem with the template search, so turn them into warnings. + warning = ( + '%s_%s (sum_probs: %s, rank: %s): feature extracting errors: ' + '%s, mmCIF parsing errors: %s' % ( + hit_pdb_code, + hit_chain_id, + hit.sum_probs, + hit.index, + str(e), + parsing_result.errors, + )) + if strict_error_check: + return SingleHitResult(features=None, error=warning, warning=None) + else: + return SingleHitResult(features=None, error=None, warning=warning) + except Error as e: + error = ( + '%s_%s (sum_probs: %.2f, rank: %d): feature extracting errors: ' + '%s, mmCIF parsing errors: %s' % ( + hit_pdb_code, + hit_chain_id, + hit.sum_probs, + hit.index, + str(e), + parsing_result.errors, + )) + return SingleHitResult(features=None, error=error, warning=None) + + +@dataclasses.dataclass(frozen=True) +class TemplateSearchResult: + features: Mapping[str, Any] + errors: Sequence[str] + warnings: Sequence[str] + + +class TemplateHitFeaturizer(abc.ABC): + """An abstract base class for turning template hits to template features.""" + + def __init__( + self, + mmcif_dir: str, + max_template_date: str, + max_hits: int, + kalign_binary_path: str, + release_dates_path: Optional[str], + obsolete_pdbs_path: Optional[str], + strict_error_check: bool = False, + ): + """Initializes the Template Search. + + Args: + mmcif_dir: Path to a directory with mmCIF structures. Once a template ID + is found by HHSearch, this directory is used to retrieve the template + data. + max_template_date: The maximum date permitted for template structures. No + template with date higher than this date will be returned. In ISO8601 + date format, YYYY-MM-DD. + max_hits: The maximum number of templates that will be returned. + kalign_binary_path: The path to a kalign executable used for template + realignment. + release_dates_path: An optional path to a file with a mapping from PDB IDs + to their release dates. Thanks to this we don't have to redundantly + parse mmCIF files to get that information. + obsolete_pdbs_path: An optional path to a file containing a mapping from + obsolete PDB IDs to the PDB IDs of their replacements. + strict_error_check: If True, then the following will be treated as errors: + * If any template date is after the max_template_date. + * If any template has identical PDB ID to the query. + * If any template is a duplicate of the query. + * Any feature computation errors. + """ + self._mmcif_dir = mmcif_dir + if not glob.glob(os.path.join(self._mmcif_dir, '*.cif')): + logging.error('Could not find CIFs in %s', self._mmcif_dir) + raise ValueError(f'Could not find CIFs in {self._mmcif_dir}') + + try: + self._max_template_date = datetime.datetime.strptime( + max_template_date, '%Y-%m-%d') + except ValueError: + raise ValueError( + 'max_template_date must be set and have format YYYY-MM-DD.') + self._max_hits = max_hits + self._kalign_binary_path = kalign_binary_path + self._strict_error_check = strict_error_check + + if release_dates_path: + logging.info('Using precomputed release dates %s.', + release_dates_path) + self._release_dates = _parse_release_dates(release_dates_path) + else: + self._release_dates = {} + + if obsolete_pdbs_path: + logging.info('Using precomputed obsolete pdbs %s.', + obsolete_pdbs_path) + self._obsolete_pdbs = _parse_obsolete(obsolete_pdbs_path) + else: + self._obsolete_pdbs = {} + + @abc.abstractmethod + def get_templates( + self, query_sequence: str, + hits: Sequence[parsers.TemplateHit]) -> TemplateSearchResult: + """Computes the templates for given query sequence.""" + + +class HhsearchHitFeaturizer(TemplateHitFeaturizer): + """A class for turning a3m hits from hhsearch to template features.""" + + def get_templates( + self, query_sequence: str, + hits: Sequence[parsers.TemplateHit]) -> TemplateSearchResult: + """Computes the templates for given query sequence (more details above).""" + logging.info('Searching for template for: %s', query_sequence) + + template_features = {} + for template_feature_name in TEMPLATE_FEATURES: + template_features[template_feature_name] = [] + + num_hits = 0 + errors = [] + warnings = [] + + for hit in sorted(hits, key=lambda x: x.sum_probs, reverse=True): + # We got all the templates we wanted, stop processing hits. + if num_hits >= self._max_hits: + break + + result = _process_single_hit( + query_sequence=query_sequence, + hit=hit, + mmcif_dir=self._mmcif_dir, + max_template_date=self._max_template_date, + release_dates=self._release_dates, + obsolete_pdbs=self._obsolete_pdbs, + strict_error_check=self._strict_error_check, + kalign_binary_path=self._kalign_binary_path, + ) + + if result.error: + errors.append(result.error) + + # There could be an error even if there are some results, e.g. thrown by + # other unparsable chains in the same mmCIF file. + if result.warning: + warnings.append(result.warning) + + if result.features is None: + logging.info( + 'Skipped invalid hit %s, error: %s, warning: %s', + hit.name, + result.error, + result.warning, + ) + else: + # Increment the hit counter, since we got features out of this hit. + num_hits += 1 + for k in template_features: + template_features[k].append(result.features[k]) + + for name in template_features: + if num_hits > 0: + template_features[name] = np.stack( + template_features[name], + axis=0).astype(TEMPLATE_FEATURES[name]) + else: + # Make sure the feature has correct dtype even if empty. + template_features[name] = np.array( + [], dtype=TEMPLATE_FEATURES[name]) + + return TemplateSearchResult( + features=template_features, errors=errors, warnings=warnings) + + +class HmmsearchHitFeaturizer(TemplateHitFeaturizer): + """A class for turning a3m hits from hmmsearch to template features.""" + + def get_templates( + self, query_sequence: str, + hits: Sequence[parsers.TemplateHit]) -> TemplateSearchResult: + """Computes the templates for given query sequence (more details above).""" + logging.info('Searching for template for: %s', query_sequence) + + template_features = {} + for template_feature_name in TEMPLATE_FEATURES: + template_features[template_feature_name] = [] + + already_seen = set() + errors = [] + warnings = [] + + if not hits or hits[0].sum_probs is None: + sorted_hits = hits + else: + sorted_hits = sorted(hits, key=lambda x: x.sum_probs, reverse=True) + + for hit in sorted_hits: + # We got all the templates we wanted, stop processing hits. + if len(already_seen) >= self._max_hits: + break + + result = _process_single_hit( + query_sequence=query_sequence, + hit=hit, + mmcif_dir=self._mmcif_dir, + max_template_date=self._max_template_date, + release_dates=self._release_dates, + obsolete_pdbs=self._obsolete_pdbs, + strict_error_check=self._strict_error_check, + kalign_binary_path=self._kalign_binary_path, + ) + + if result.error: + errors.append(result.error) + + # There could be an error even if there are some results, e.g. thrown by + # other unparsable chains in the same mmCIF file. + if result.warning: + warnings.append(result.warning) + + if result.features is None: + logging.debug( + 'Skipped invalid hit %s, error: %s, warning: %s', + hit.name, + result.error, + result.warning, + ) + else: + already_seen_key = result.features['template_sequence'] + if already_seen_key in already_seen: + continue + # Increment the hit counter, since we got features out of this hit. + already_seen.add(already_seen_key) + for k in template_features: + template_features[k].append(result.features[k]) + + if already_seen: + for name in template_features: + template_features[name] = np.stack( + template_features[name], + axis=0).astype(TEMPLATE_FEATURES[name]) + else: + num_res = len(query_sequence) + # Construct a default template with all zeros. + template_features = { + 'template_aatype': + np.zeros( + (1, num_res, len( + residue_constants.restypes_with_x_and_gap)), + np.float32, + ), + 'template_all_atom_mask': + np.zeros((1, num_res, residue_constants.atom_type_num), + np.float32), + 'template_all_atom_positions': + np.zeros((1, num_res, residue_constants.atom_type_num, 3), + np.float32), + 'template_domain_names': + np.array([''.encode()], dtype=np.object_), + 'template_sequence': + np.array([''.encode()], dtype=np.object_), + 'template_sum_probs': + np.array([0], dtype=np.float32), + } + return TemplateSearchResult( + features=template_features, errors=errors, warnings=warnings) diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/__init__.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..903d09793c39d08491dd9a6fecefd463b058a251 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python wrappers for third party tools.""" diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhblits.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhblits.py new file mode 100644 index 0000000000000000000000000000000000000000..ee442e39e3167ec0a3e163108942df73731eddef --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhblits.py @@ -0,0 +1,170 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Library to run HHblits from Python.""" + +import glob +import os +import subprocess +from typing import Any, List, Mapping, Optional, Sequence + +from absl import logging + +from . import utils + +_HHBLITS_DEFAULT_P = 20 +_HHBLITS_DEFAULT_Z = 500 + + +class HHBlits: + """Python wrapper of the HHblits binary.""" + + def __init__( + self, + *, + binary_path: str, + databases: Sequence[str], + n_cpu: int = 4, + n_iter: int = 3, + e_value: float = 0.001, + maxseq: int = 1_000_000, + realign_max: int = 100_000, + maxfilt: int = 100_000, + min_prefilter_hits: int = 1000, + all_seqs: bool = False, + alt: Optional[int] = None, + p: int = _HHBLITS_DEFAULT_P, + z: int = _HHBLITS_DEFAULT_Z, + ): + """Initializes the Python HHblits wrapper. + + Args: + binary_path: The path to the HHblits executable. + databases: A sequence of HHblits database paths. This should be the + common prefix for the database files (i.e. up to but not including + _hhm.ffindex etc.) + n_cpu: The number of CPUs to give HHblits. + n_iter: The number of HHblits iterations. + e_value: The E-value, see HHblits docs for more details. + maxseq: The maximum number of rows in an input alignment. Note that this + parameter is only supported in HHBlits version 3.1 and higher. + realign_max: Max number of HMM-HMM hits to realign. HHblits default: 500. + maxfilt: Max number of hits allowed to pass the 2nd prefilter. + HHblits default: 20000. + min_prefilter_hits: Min number of hits to pass prefilter. + HHblits default: 100. + all_seqs: Return all sequences in the MSA / Do not filter the result MSA. + HHblits default: False. + alt: Show up to this many alternative alignments. + p: Minimum Prob for a hit to be included in the output hhr file. + HHblits default: 20. + z: Hard cap on number of hits reported in the hhr file. + HHblits default: 500. NB: The relevant HHblits flag is -Z not -z. + + Raises: + RuntimeError: If HHblits binary not found within the path. + """ + self.binary_path = binary_path + self.databases = databases + + for database_path in self.databases: + if not glob.glob(database_path + '_*'): + logging.error('Could not find HHBlits database %s', + database_path) + raise ValueError( + f'Could not find HHBlits database {database_path}') + + self.n_cpu = n_cpu + self.n_iter = n_iter + self.e_value = e_value + self.maxseq = maxseq + self.realign_max = realign_max + self.maxfilt = maxfilt + self.min_prefilter_hits = min_prefilter_hits + self.all_seqs = all_seqs + self.alt = alt + self.p = p + self.z = z + + def query(self, input_fasta_path: str) -> List[Mapping[str, Any]]: + """Queries the database using HHblits.""" + with utils.tmpdir_manager() as query_tmp_dir: + a3m_path = os.path.join(query_tmp_dir, 'output.a3m') + + db_cmd = [] + for db_path in self.databases: + db_cmd.append('-d') + db_cmd.append(db_path) + cmd = [ + self.binary_path, + '-i', + input_fasta_path, + '-cpu', + str(self.n_cpu), + '-oa3m', + a3m_path, + '-o', + '/dev/null', + '-n', + str(self.n_iter), + '-e', + str(self.e_value), + '-maxseq', + str(self.maxseq), + '-realign_max', + str(self.realign_max), + '-maxfilt', + str(self.maxfilt), + '-min_prefilter_hits', + str(self.min_prefilter_hits), + ] + if self.all_seqs: + cmd += ['-all'] + if self.alt: + cmd += ['-alt', str(self.alt)] + if self.p != _HHBLITS_DEFAULT_P: + cmd += ['-p', str(self.p)] + if self.z != _HHBLITS_DEFAULT_Z: + cmd += ['-Z', str(self.z)] + cmd += db_cmd + + logging.info('Launching subprocess "%s"', ' '.join(cmd)) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + with utils.timing('HHblits query'): + stdout, stderr = process.communicate() + retcode = process.wait() + + if retcode: + # Logs have a 15k character limit, so log HHblits error line by line. + logging.error('HHblits failed. HHblits stderr begin:') + for error_line in stderr.decode('utf-8').splitlines(): + if error_line.strip(): + logging.error(error_line.strip()) + logging.error('HHblits stderr end') + raise RuntimeError( + 'HHblits failed\nstdout:\n%s\n\nstderr:\n%s\n' % + (stdout.decode('utf-8'), stderr[:500_000].decode('utf-8'))) + + with open(a3m_path) as f: + a3m = f.read() + + raw_output = dict( + a3m=a3m, + output=stdout, + stderr=stderr, + n_iter=self.n_iter, + e_value=self.e_value, + ) + return [raw_output] diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhsearch.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhsearch.py new file mode 100644 index 0000000000000000000000000000000000000000..ac7f3b55bfaf7369f2e33d725f085511880d71ae --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hhsearch.py @@ -0,0 +1,111 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Library to run HHsearch from Python.""" + +import glob +import os +import subprocess +from typing import Sequence + +from absl import logging + +from modelscope.models.science.unifold.msa import parsers +from . import utils + + +class HHSearch: + """Python wrapper of the HHsearch binary.""" + + def __init__(self, + *, + binary_path: str, + databases: Sequence[str], + maxseq: int = 1_000_000): + """Initializes the Python HHsearch wrapper. + + Args: + binary_path: The path to the HHsearch executable. + databases: A sequence of HHsearch database paths. This should be the + common prefix for the database files (i.e. up to but not including + _hhm.ffindex etc.) + maxseq: The maximum number of rows in an input alignment. Note that this + parameter is only supported in HHBlits version 3.1 and higher. + + Raises: + RuntimeError: If HHsearch binary not found within the path. + """ + self.binary_path = binary_path + self.databases = databases + self.maxseq = maxseq + + for database_path in self.databases: + if not glob.glob(database_path + '_*'): + logging.error('Could not find HHsearch database %s', + database_path) + raise ValueError( + f'Could not find HHsearch database {database_path}') + + @property + def output_format(self) -> str: + return 'hhr' + + @property + def input_format(self) -> str: + return 'a3m' + + def query(self, a3m: str) -> str: + """Queries the database using HHsearch using a given a3m.""" + with utils.tmpdir_manager() as query_tmp_dir: + input_path = os.path.join(query_tmp_dir, 'query.a3m') + hhr_path = os.path.join(query_tmp_dir, 'output.hhr') + with open(input_path, 'w') as f: + f.write(a3m) + + db_cmd = [] + for db_path in self.databases: + db_cmd.append('-d') + db_cmd.append(db_path) + cmd = [ + self.binary_path, + '-i', + input_path, + '-o', + hhr_path, + '-maxseq', + str(self.maxseq), + ] + db_cmd + + logging.info('Launching subprocess "%s"', ' '.join(cmd)) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with utils.timing('HHsearch query'): + stdout, stderr = process.communicate() + retcode = process.wait() + + if retcode: + # Stderr is truncated to prevent proto size errors in Beam. + raise RuntimeError( + 'HHSearch failed:\nstdout:\n%s\n\nstderr:\n%s\n' % + (stdout.decode('utf-8'), stderr[:100_000].decode('utf-8'))) + + with open(hhr_path) as f: + hhr = f.read() + return hhr + + def get_template_hits( + self, output_string: str, + input_sequence: str) -> Sequence[parsers.TemplateHit]: + """Gets parsed template hits from the raw string output by the tool.""" + del input_sequence # Used by hmmseach but not needed for hhsearch. + return parsers.parse_hhr(output_string) diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmbuild.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmbuild.py new file mode 100644 index 0000000000000000000000000000000000000000..84f205d6ed458eba00e535c3b79f21cb1f4cbe25 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmbuild.py @@ -0,0 +1,143 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A Python wrapper for hmmbuild - construct HMM profiles from MSA.""" + +import os +import re +import subprocess + +from absl import logging + +from . import utils + + +class Hmmbuild(object): + """Python wrapper of the hmmbuild binary.""" + + def __init__(self, *, binary_path: str, singlemx: bool = False): + """Initializes the Python hmmbuild wrapper. + + Args: + binary_path: The path to the hmmbuild executable. + singlemx: Whether to use --singlemx flag. If True, it forces HMMBuild to + just use a common substitution score matrix. + + Raises: + RuntimeError: If hmmbuild binary not found within the path. + """ + self.binary_path = binary_path + self.singlemx = singlemx + + def build_profile_from_sto(self, + sto: str, + model_construction='fast') -> str: + """Builds a HHM for the aligned sequences given as an A3M string. + + Args: + sto: A string with the aligned sequences in the Stockholm format. + model_construction: Whether to use reference annotation in the msa to + determine consensus columns ('hand') or default ('fast'). + + Returns: + A string with the profile in the HMM format. + + Raises: + RuntimeError: If hmmbuild fails. + """ + return self._build_profile(sto, model_construction=model_construction) + + def build_profile_from_a3m(self, a3m: str) -> str: + """Builds a HHM for the aligned sequences given as an A3M string. + + Args: + a3m: A string with the aligned sequences in the A3M format. + + Returns: + A string with the profile in the HMM format. + + Raises: + RuntimeError: If hmmbuild fails. + """ + lines = [] + for line in a3m.splitlines(): + if not line.startswith('>'): + line = re.sub('[a-z]+', '', line) # Remove inserted residues. + lines.append(line + '\n') + msa = ''.join(lines) + return self._build_profile(msa, model_construction='fast') + + def _build_profile(self, + msa: str, + model_construction: str = 'fast') -> str: + """Builds a HMM for the aligned sequences given as an MSA string. + + Args: + msa: A string with the aligned sequences, in A3M or STO format. + model_construction: Whether to use reference annotation in the msa to + determine consensus columns ('hand') or default ('fast'). + + Returns: + A string with the profile in the HMM format. + + Raises: + RuntimeError: If hmmbuild fails. + ValueError: If unspecified arguments are provided. + """ + if model_construction not in {'hand', 'fast'}: + raise ValueError( + f'Invalid model_construction {model_construction} - only' + 'hand and fast supported.') + + with utils.tmpdir_manager() as query_tmp_dir: + input_query = os.path.join(query_tmp_dir, 'query.msa') + output_hmm_path = os.path.join(query_tmp_dir, 'output.hmm') + + with open(input_query, 'w') as f: + f.write(msa) + + cmd = [self.binary_path] + # If adding flags, we have to do so before the output and input: + + if model_construction == 'hand': + cmd.append(f'--{model_construction}') + if self.singlemx: + cmd.append('--singlemx') + cmd.extend([ + '--amino', + output_hmm_path, + input_query, + ]) + + logging.info('Launching subprocess %s', cmd) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + with utils.timing('hmmbuild query'): + stdout, stderr = process.communicate() + retcode = process.wait() + logging.info( + 'hmmbuild stdout:\n%s\n\nstderr:\n%s\n', + stdout.decode('utf-8'), + stderr.decode('utf-8'), + ) + + if retcode: + raise RuntimeError( + 'hmmbuild failed\nstdout:\n%s\n\nstderr:\n%s\n' % + (stdout.decode('utf-8'), stderr.decode('utf-8'))) + + with open(output_hmm_path, encoding='utf-8') as f: + hmm = f.read() + + return hmm diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmsearch.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmsearch.py new file mode 100644 index 0000000000000000000000000000000000000000..445970cab483115df8ae69246d4961ac6ff589ba --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/hmmsearch.py @@ -0,0 +1,146 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A Python wrapper for hmmsearch - search profile against a sequence db.""" + +import os +import subprocess +from typing import Optional, Sequence + +from absl import logging + +from modelscope.models.science.unifold.msa import parsers +from . import hmmbuild, utils + + +class Hmmsearch(object): + """Python wrapper of the hmmsearch binary.""" + + def __init__( + self, + *, + binary_path: str, + hmmbuild_binary_path: str, + database_path: str, + flags: Optional[Sequence[str]] = None, + ): + """Initializes the Python hmmsearch wrapper. + + Args: + binary_path: The path to the hmmsearch executable. + hmmbuild_binary_path: The path to the hmmbuild executable. Used to build + an hmm from an input a3m. + database_path: The path to the hmmsearch database (FASTA format). + flags: List of flags to be used by hmmsearch. + + Raises: + RuntimeError: If hmmsearch binary not found within the path. + """ + self.binary_path = binary_path + self.hmmbuild_runner = hmmbuild.Hmmbuild( + binary_path=hmmbuild_binary_path) + self.database_path = database_path + if flags is None: + # Default hmmsearch run settings. + flags = [ + '--F1', + '0.1', + '--F2', + '0.1', + '--F3', + '0.1', + '--incE', + '100', + '-E', + '100', + '--domE', + '100', + '--incdomE', + '100', + ] + self.flags = flags + + if not os.path.exists(self.database_path): + logging.error('Could not find hmmsearch database %s', + database_path) + raise ValueError( + f'Could not find hmmsearch database {database_path}') + + @property + def output_format(self) -> str: + return 'sto' + + @property + def input_format(self) -> str: + return 'sto' + + def query(self, msa_sto: str) -> str: + """Queries the database using hmmsearch using a given stockholm msa.""" + hmm = self.hmmbuild_runner.build_profile_from_sto( + msa_sto, model_construction='hand') + return self.query_with_hmm(hmm) + + def query_with_hmm(self, hmm: str) -> str: + """Queries the database using hmmsearch using a given hmm.""" + with utils.tmpdir_manager() as query_tmp_dir: + hmm_input_path = os.path.join(query_tmp_dir, 'query.hmm') + out_path = os.path.join(query_tmp_dir, 'output.sto') + with open(hmm_input_path, 'w') as f: + f.write(hmm) + + cmd = [ + self.binary_path, + '--noali', # Don't include the alignment in stdout. + '--cpu', + '8', + ] + # If adding flags, we have to do so before the output and input: + if self.flags: + cmd.extend(self.flags) + cmd.extend([ + '-A', + out_path, + hmm_input_path, + self.database_path, + ]) + + logging.info('Launching sub-process %s', cmd) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with utils.timing( + f'hmmsearch ({os.path.basename(self.database_path)}) query' + ): + stdout, stderr = process.communicate() + retcode = process.wait() + + if retcode: + raise RuntimeError( + 'hmmsearch failed:\nstdout:\n%s\n\nstderr:\n%s\n' % + (stdout.decode('utf-8'), stderr.decode('utf-8'))) + + with open(out_path) as f: + out_msa = f.read() + + return out_msa + + def get_template_hits( + self, output_string: str, + input_sequence: str) -> Sequence[parsers.TemplateHit]: + """Gets parsed template hits from the raw string output by the tool.""" + a3m_string = parsers.convert_stockholm_to_a3m( + output_string, remove_first_row_gaps=False) + template_hits = parsers.parse_hmmsearch_a3m( + query_sequence=input_sequence, + a3m_string=a3m_string, + skip_first=False) + return template_hits diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/jackhmmer.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/jackhmmer.py new file mode 100644 index 0000000000000000000000000000000000000000..3e29eec9197ec8dd6c3db3db5b55d456f909e0c8 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/jackhmmer.py @@ -0,0 +1,224 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Library to run Jackhmmer from Python.""" + +import glob +import os +import subprocess +from concurrent import futures +from typing import Any, Callable, Mapping, Optional, Sequence +from urllib import request + +from absl import logging + +from . import utils + + +class Jackhmmer: + """Python wrapper of the Jackhmmer binary.""" + + def __init__( + self, + *, + binary_path: str, + database_path: str, + n_cpu: int = 8, + n_iter: int = 1, + e_value: float = 0.0001, + z_value: Optional[int] = None, + get_tblout: bool = False, + filter_f1: float = 0.0005, + filter_f2: float = 0.00005, + filter_f3: float = 0.0000005, + incdom_e: Optional[float] = None, + dom_e: Optional[float] = None, + num_streamed_chunks: Optional[int] = None, + streaming_callback: Optional[Callable[[int], None]] = None, + ): + """Initializes the Python Jackhmmer wrapper. + + Args: + binary_path: The path to the jackhmmer executable. + database_path: The path to the jackhmmer database (FASTA format). + n_cpu: The number of CPUs to give Jackhmmer. + n_iter: The number of Jackhmmer iterations. + e_value: The E-value, see Jackhmmer docs for more details. + z_value: The Z-value, see Jackhmmer docs for more details. + get_tblout: Whether to save tblout string. + filter_f1: MSV and biased composition pre-filter, set to >1.0 to turn off. + filter_f2: Viterbi pre-filter, set to >1.0 to turn off. + filter_f3: Forward pre-filter, set to >1.0 to turn off. + incdom_e: Domain e-value criteria for inclusion of domains in MSA/next + round. + dom_e: Domain e-value criteria for inclusion in tblout. + num_streamed_chunks: Number of database chunks to stream over. + streaming_callback: Callback function run after each chunk iteration with + the iteration number as argument. + """ + self.binary_path = binary_path + self.database_path = database_path + self.num_streamed_chunks = num_streamed_chunks + + if not os.path.exists( + self.database_path) and num_streamed_chunks is None: + logging.error('Could not find Jackhmmer database %s', + database_path) + raise ValueError( + f'Could not find Jackhmmer database {database_path}') + + self.n_cpu = n_cpu + self.n_iter = n_iter + self.e_value = e_value + self.z_value = z_value + self.filter_f1 = filter_f1 + self.filter_f2 = filter_f2 + self.filter_f3 = filter_f3 + self.incdom_e = incdom_e + self.dom_e = dom_e + self.get_tblout = get_tblout + self.streaming_callback = streaming_callback + + def _query_chunk(self, input_fasta_path: str, + database_path: str) -> Mapping[str, Any]: + """Queries the database chunk using Jackhmmer.""" + with utils.tmpdir_manager() as query_tmp_dir: + sto_path = os.path.join(query_tmp_dir, 'output.sto') + + # The F1/F2/F3 are the expected proportion to pass each of the filtering + # stages (which get progressively more expensive), reducing these + # speeds up the pipeline at the expensive of sensitivity. They are + # currently set very low to make querying Mgnify run in a reasonable + # amount of time. + cmd_flags = [ + # Don't pollute stdout with Jackhmmer output. + '-o', + '/dev/null', + '-A', + sto_path, + '--noali', + '--F1', + str(self.filter_f1), + '--F2', + str(self.filter_f2), + '--F3', + str(self.filter_f3), + '--incE', + str(self.e_value), + # Report only sequences with E-values <= x in per-sequence output. + '-E', + str(self.e_value), + '--cpu', + str(self.n_cpu), + '-N', + str(self.n_iter), + ] + if self.get_tblout: + tblout_path = os.path.join(query_tmp_dir, 'tblout.txt') + cmd_flags.extend(['--tblout', tblout_path]) + + if self.z_value: + cmd_flags.extend(['-Z', str(self.z_value)]) + + if self.dom_e is not None: + cmd_flags.extend(['--domE', str(self.dom_e)]) + + if self.incdom_e is not None: + cmd_flags.extend(['--incdomE', str(self.incdom_e)]) + + cmd = [self.binary_path + ] + cmd_flags + [input_fasta_path, database_path] + + logging.info('Launching subprocess "%s"', ' '.join(cmd)) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with utils.timing( + f'Jackhmmer ({os.path.basename(database_path)}) query'): + _, stderr = process.communicate() + retcode = process.wait() + + if retcode: + raise RuntimeError('Jackhmmer failed\nstderr:\n%s\n' + % stderr.decode('utf-8')) + + # Get e-values for each target name + tbl = '' + if self.get_tblout: + with open(tblout_path) as f: + tbl = f.read() + + with open(sto_path) as f: + sto = f.read() + + raw_output = dict( + sto=sto, + tbl=tbl, + stderr=stderr, + n_iter=self.n_iter, + e_value=self.e_value) + + return raw_output + + def query(self, input_fasta_path: str) -> Sequence[Mapping[str, Any]]: + """Queries the database using Jackhmmer.""" + if self.num_streamed_chunks is None: + return [self._query_chunk(input_fasta_path, self.database_path)] + + db_basename = os.path.basename(self.database_path) + + def db_remote_chunk(db_idx): + return f'{self.database_path}.{db_idx}' + + def db_local_chunk(db_idx): + return f'/tmp/ramdisk/{db_basename}.{db_idx}' + + # db_remote_chunk = lambda db_idx: f'{self.database_path}.{db_idx}' + # db_local_chunk = lambda db_idx: f'/tmp/ramdisk/{db_basename}.{db_idx}' + + # Remove existing files to prevent OOM + for f in glob.glob(db_local_chunk('[0-9]*')): + try: + os.remove(f) + except OSError: + print(f'OSError while deleting {f}') + + # Download the (i+1)-th chunk while Jackhmmer is running on the i-th chunk + with futures.ThreadPoolExecutor(max_workers=2) as executor: + chunked_output = [] + for i in range(1, self.num_streamed_chunks + 1): + # Copy the chunk locally + if i == 1: + future = executor.submit(request.urlretrieve, + db_remote_chunk(i), + db_local_chunk(i)) + if i < self.num_streamed_chunks: + next_future = executor.submit( + request.urlretrieve, + db_remote_chunk(i + 1), + db_local_chunk(i + 1), + ) + + # Run Jackhmmer with the chunk + future.result() + chunked_output.append( + self._query_chunk(input_fasta_path, db_local_chunk(i))) + + # Remove the local copy of the chunk + os.remove(db_local_chunk(i)) + # Do not set next_future for the last chunk so that this works even for + # databases with only 1 chunk. + if i < self.num_streamed_chunks: + future = next_future + if self.streaming_callback: + self.streaming_callback(i) + return chunked_output diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/kalign.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/kalign.py new file mode 100644 index 0000000000000000000000000000000000000000..1ea997facfef0a2690b631f2bcc0f9ad3a76939c --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/kalign.py @@ -0,0 +1,110 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A Python wrapper for Kalign.""" +import os +import subprocess +from typing import Sequence + +from absl import logging + +from . import utils + + +def _to_a3m(sequences: Sequence[str]) -> str: + """Converts sequences to an a3m file.""" + names = ['sequence %d' % i for i in range(1, len(sequences) + 1)] + a3m = [] + for sequence, name in zip(sequences, names): + a3m.append('>' + name + '\n') + a3m.append(sequence + '\n') + return ''.join(a3m) + + +class Kalign: + """Python wrapper of the Kalign binary.""" + + def __init__(self, *, binary_path: str): + """Initializes the Python Kalign wrapper. + + Args: + binary_path: The path to the Kalign binary. + + Raises: + RuntimeError: If Kalign binary not found within the path. + """ + self.binary_path = binary_path + + def align(self, sequences: Sequence[str]) -> str: + """Aligns the sequences and returns the alignment in A3M string. + + Args: + sequences: A list of query sequence strings. The sequences have to be at + least 6 residues long (Kalign requires this). Note that the order in + which you give the sequences might alter the output slightly as + different alignment tree might get constructed. + + Returns: + A string with the alignment in a3m format. + + Raises: + RuntimeError: If Kalign fails. + ValueError: If any of the sequences is less than 6 residues long. + """ + logging.info('Aligning %d sequences', len(sequences)) + + for s in sequences: + if len(s) < 6: + raise ValueError( + 'Kalign requires all sequences to be at least 6 ' + 'residues long. Got %s (%d residues).' % (s, len(s))) + + with utils.tmpdir_manager() as query_tmp_dir: + input_fasta_path = os.path.join(query_tmp_dir, 'input.fasta') + output_a3m_path = os.path.join(query_tmp_dir, 'output.a3m') + + with open(input_fasta_path, 'w') as f: + f.write(_to_a3m(sequences)) + + cmd = [ + self.binary_path, + '-i', + input_fasta_path, + '-o', + output_a3m_path, + '-format', + 'fasta', + ] + + logging.info('Launching subprocess "%s"', ' '.join(cmd)) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + with utils.timing('Kalign query'): + stdout, stderr = process.communicate() + retcode = process.wait() + logging.info( + 'Kalign stdout:\n%s\n\nstderr:\n%s\n', + stdout.decode('utf-8'), + stderr.decode('utf-8'), + ) + + if retcode: + raise RuntimeError( + 'Kalign failed\nstdout:\n%s\n\nstderr:\n%s\n' % + (stdout.decode('utf-8'), stderr.decode('utf-8'))) + + with open(output_a3m_path) as f: + a3m = f.read() + + return a3m diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/tools/utils.py b/runtime/Lib/site_packages/models/science/unifold/msa/tools/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1c2af9368fafbc0b03495fa1963eceacda925029 --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/tools/utils.py @@ -0,0 +1,40 @@ +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common utilities for data pipeline tools.""" +import contextlib +import shutil +import tempfile +import time +from typing import Optional + +from absl import logging + + +@contextlib.contextmanager +def tmpdir_manager(base_dir: Optional[str] = None): + """Context manager that deletes a temporary directory on exit.""" + tmpdir = tempfile.mkdtemp(dir=base_dir) + try: + yield tmpdir + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +@contextlib.contextmanager +def timing(msg: str): + logging.info('Started %s', msg) + tic = time.time() + yield + toc = time.time() + logging.info('Finished %s in %.3f seconds', msg, toc - tic) diff --git a/runtime/Lib/site_packages/models/science/unifold/msa/utils.py b/runtime/Lib/site_packages/models/science/unifold/msa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..50e380d44c3a61bd04062948238ee64ddd1618cb --- /dev/null +++ b/runtime/Lib/site_packages/models/science/unifold/msa/utils.py @@ -0,0 +1,89 @@ +# The Uni-fold implementation is also open-sourced by the authors under Apache-2.0 license, +# and is publicly available at https://github.com/dptech-corp/Uni-Fold. + +import os +from typing import Mapping, Sequence + +import json +from absl import logging + +from modelscope.models.science.unifold.data import protein + + +def get_chain_id_map( + sequences: Sequence[str], + descriptions: Sequence[str], +): + """ + Makes a mapping from PDB-format chain ID to sequence and description, + and parses the order of multi-chains + """ + unique_seqs = [] + for seq in sequences: + if seq not in unique_seqs: + unique_seqs.append(seq) + + chain_id_map = { + chain_id: { + 'descriptions': [], + 'sequence': seq + } + for chain_id, seq in zip(protein.PDB_CHAIN_IDS, unique_seqs) + } + chain_order = [] + + for seq, des in zip(sequences, descriptions): + chain_id = protein.PDB_CHAIN_IDS[unique_seqs.index(seq)] + chain_id_map[chain_id]['descriptions'].append(des) + chain_order.append(chain_id) + + return chain_id_map, chain_order + + +def divide_multi_chains( + fasta_name: str, + output_dir_base: str, + sequences: Sequence[str], + descriptions: Sequence[str], +): + """ + Divides the multi-chains fasta into several single fasta files and + records multi-chains mapping information. + """ + if len(sequences) != len(descriptions): + raise ValueError('sequences and descriptions must have equal length. ' + f'Got {len(sequences)} != {len(descriptions)}.') + if len(sequences) > protein.PDB_MAX_CHAINS: + raise ValueError( + 'Cannot process more chains than the PDB format supports. ' + f'Got {len(sequences)} chains.') + + chain_id_map, chain_order = get_chain_id_map(sequences, descriptions) + + output_dir = os.path.join(output_dir_base, fasta_name) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + chain_id_map_path = os.path.join(output_dir, 'chain_id_map.json') + with open(chain_id_map_path, 'w') as f: + json.dump(chain_id_map, f, indent=4, sort_keys=True) + + chain_order_path = os.path.join(output_dir, 'chains.txt') + with open(chain_order_path, 'w') as f: + f.write(' '.join(chain_order)) + + logging.info('Mapping multi-chains fasta with chain order: %s', + ' '.join(chain_order)) + + temp_names = [] + temp_paths = [] + for chain_id in chain_id_map.keys(): + temp_name = fasta_name + '_{}'.format(chain_id) + temp_path = os.path.join(output_dir, temp_name + '.fasta') + des = 'chain_{}'.format(chain_id) + seq = chain_id_map[chain_id]['sequence'] + with open(temp_path, 'w') as f: + f.write('>' + des + '\n' + seq) + temp_names.append(temp_name) + temp_paths.append(temp_path) + return temp_names, temp_paths